├── .jshintrc ├── test ├── mocha.opts ├── .jshintrc ├── fixtures │ ├── simple-package │ │ └── package.json │ └── native-dependencies-package │ │ └── package.json ├── helpers │ ├── exists.js │ ├── tmp.js │ └── zip.js ├── support │ └── common.js └── acceptance │ └── basic-test.js ├── .gitignore ├── assets ├── diagram.png └── diagram.graffle.zip ├── lib ├── util │ ├── prompt.js │ └── ui.js ├── commands │ ├── package.js │ └── deploy.js ├── errors │ └── lambda-error.js ├── aws │ ├── s3.js │ ├── lambda.js │ └── cloud-formation.js ├── zip-file.js ├── config.js ├── packager.js ├── http-dependency-builder.js └── server │ └── build-dependencies.js ├── bin ├── build-install-zip └── lambda-packager ├── server.js ├── package.json └── README.md /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "node": true 3 | } 4 | -------------------------------------------------------------------------------- /test/mocha.opts: -------------------------------------------------------------------------------- 1 | --require test/support/common 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | blueprints/builder/node_modules 3 | -------------------------------------------------------------------------------- /test/.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "node": true, 3 | "expr": true 4 | } 5 | -------------------------------------------------------------------------------- /assets/diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomdale/lambda-packager/HEAD/assets/diagram.png -------------------------------------------------------------------------------- /assets/diagram.graffle.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomdale/lambda-packager/HEAD/assets/diagram.graffle.zip -------------------------------------------------------------------------------- /test/fixtures/simple-package/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "simple-package", 3 | "version": "0.1.0", 4 | "dependencies": { 5 | "chalk": "1.1.1" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /test/fixtures/native-dependencies-package/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "simple-package", 3 | "version": "0.1.0", 4 | "dependencies": { 5 | "chalk": "1.1.1", 6 | "contextify": "0.1.14" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /test/helpers/exists.js: -------------------------------------------------------------------------------- 1 | var fsp = require('fs-promise'); 2 | 3 | module.exports = function(path) { 4 | return fsp.stat(path) 5 | .then(function() { 6 | return true; 7 | }, function() { 8 | return false; 9 | }); 10 | }; 11 | -------------------------------------------------------------------------------- /test/helpers/tmp.js: -------------------------------------------------------------------------------- 1 | var temp = require('temp').track(); 2 | 3 | module.exports = { 4 | create: function() { 5 | if (this.tmpDir) { 6 | return this.tmpDir; 7 | } 8 | 9 | return temp.mkdirSync('lambda-packager'); 10 | }, 11 | 12 | tmpDir: null 13 | }; 14 | -------------------------------------------------------------------------------- /lib/util/prompt.js: -------------------------------------------------------------------------------- 1 | var inquirer = require('inquirer'); 2 | var RSVP = require('rsvp'); 3 | 4 | function prompt(questions) { 5 | return new RSVP.Promise(function(resolve) { 6 | inquirer.prompt(questions, function(answers) { 7 | resolve(answers); 8 | }); 9 | }); 10 | } 11 | module.exports = prompt; 12 | -------------------------------------------------------------------------------- /test/support/common.js: -------------------------------------------------------------------------------- 1 | var chai = require('chai'); 2 | var chaiAsPromised = require('chai-as-promised'); 3 | 4 | chai.use(chaiAsPromised); 5 | 6 | chaiAsPromised.transferPromiseness = function (assertion, promise) { 7 | assertion.then = promise.then.bind(promise); // this is all you get by default 8 | assertion.finally = promise.finally.bind(promise); 9 | assertion.catch = promise.catch.bind(promise); 10 | }; 11 | -------------------------------------------------------------------------------- /lib/util/ui.js: -------------------------------------------------------------------------------- 1 | var chalk = require('chalk'); 2 | var util = require('util'); 3 | 4 | function log(message) { 5 | if (!ui.silent) { 6 | console.log(chalk.green(util.format.apply(this, arguments))); 7 | } 8 | } 9 | 10 | function error() { 11 | if (!ui.silent) { 12 | console.log(chalk.red(util.format.apply(this, arguments))); 13 | } 14 | } 15 | 16 | var ui = { 17 | silent: true, 18 | log: log, 19 | error: error 20 | }; 21 | 22 | module.exports = ui; 23 | -------------------------------------------------------------------------------- /lib/commands/package.js: -------------------------------------------------------------------------------- 1 | var packager = require('../packager'); 2 | var green = require('chalk').green; 3 | var ui = require('../util/ui'); 4 | var util = require('util'); 5 | 6 | var PackageCommand = { 7 | run: function(packageDirectory, outputZip) { 8 | return packager.build({ 9 | from: packageDirectory, 10 | to: outputZip 11 | }) 12 | .then(function() { 13 | ui.log("Packaged " + packageDirectory + " into " + outputZip + "."); 14 | }) 15 | .catch(function(err) { 16 | ui.error(err.message || util.inspect(err)); 17 | ui.error(err.stack); 18 | }); 19 | } 20 | }; 21 | 22 | module.exports = PackageCommand; 23 | -------------------------------------------------------------------------------- /test/helpers/zip.js: -------------------------------------------------------------------------------- 1 | var AdmZip = require('adm-zip'); 2 | 3 | module.exports = function(zipPath) { 4 | var zip = new AdmZip(zipPath); 5 | return new Zip(zip); 6 | }; 7 | 8 | function Zip(zip) { 9 | this.zip = zip; 10 | } 11 | 12 | Zip.prototype.shouldInclude = function(included) { 13 | var entries = this.zip.getEntries().map(function(e) { 14 | return e.entryName; 15 | }); 16 | 17 | var found = []; 18 | included.forEach(function(i) { 19 | if (entries.indexOf(i) > -1) { 20 | found.push(i); 21 | } 22 | }); 23 | 24 | if (found.length < included.length) { 25 | var error = new Error("Files not found in zip file"); 26 | error.expected = included; 27 | error.actual = found; 28 | throw error; 29 | } 30 | }; 31 | -------------------------------------------------------------------------------- /lib/errors/lambda-error.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | 3 | function LambdaError(data) { 4 | if (data.FunctionError) { 5 | this.handled = data.FunctionError === 'Handled'; 6 | } 7 | 8 | this.statusCode = data.StatusCode; 9 | this.payload = data.Payload; 10 | } 11 | 12 | LambdaError.prototype = Object.create(Error.prototype); 13 | LambdaError.prototype.constructor = LambdaError; 14 | 15 | LambdaError.detect = function(data) { 16 | return data.StatusCode === 200; 17 | }; 18 | 19 | Object.defineProperty(LambdaError.prototype, 'message', { 20 | get: function() { 21 | var payload = JSON.parse(this.payload); 22 | 23 | return "Error while invoking Lambda function.\n" + 24 | "Status code: " + this.statusCode + "\n" + 25 | "Error message: " + payload.errorMessage; 26 | } 27 | }); 28 | 29 | module.exports = LambdaError; 30 | -------------------------------------------------------------------------------- /bin/build-install-zip: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var AWS = require('aws-sdk'); 4 | var RSVP = require('rsvp'); 5 | var temp = require('temp').track(); 6 | var mkdir = RSVP.denodeify(temp.mkdir); 7 | var exec = RSVP.denodeify(require('child_process').exec); 8 | var path = require('path'); 9 | var fs = require('fs'); 10 | 11 | // This script is for maintainers to build the Lambda function that does the actual 12 | // compiling on Lambda. This is the file pointed to in the CloudFormation template. 13 | var zipPath; 14 | 15 | mkdir('lambda-packager') 16 | .then(function(tmpPath) { 17 | zipPath = path.join(tmpPath, 'lambda-packager.zip'); 18 | }) 19 | .then(function() { 20 | return exec("zip -qr " + zipPath + " *", { cwd: 'blueprints/builder' }); 21 | }) 22 | .then(function() { 23 | return exec("cp " + zipPath + " ./lambda-builder.zip"); 24 | }) 25 | .catch(function(err) { 26 | console.log(err); 27 | console.log(err.stack); 28 | }); 29 | -------------------------------------------------------------------------------- /lib/aws/s3.js: -------------------------------------------------------------------------------- 1 | var AWS = require('aws-sdk'); 2 | var RSVP = require('rsvp'); 3 | var fsp = require('fs-promise'); 4 | 5 | function S3(config) { 6 | this.bucket = config.bucket; 7 | this.s3 = new AWS.S3({ 8 | apiVersion: '2006-03-01', 9 | region: config.region 10 | }); 11 | } 12 | 13 | S3.prototype.download = function(options) { 14 | var bucket = this.bucket; 15 | var s3 = this.s3; 16 | var key = options.key; 17 | var destination = options.destination; 18 | 19 | return new RSVP.Promise(function(resolve, reject) { 20 | var params = { 21 | Bucket: bucket, 22 | Key: key 23 | }; 24 | 25 | s3.getObject(params, function(err, data) { 26 | if (err) { reject(err); } 27 | 28 | resolve(data); 29 | }); 30 | }).then(function(data) { 31 | // Extract the raw bytes from the response and 32 | // write them to disk. 33 | var body = data.Body; 34 | return fsp.writeFile(destination, body); 35 | }); 36 | }; 37 | 38 | module.exports = S3; 39 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var express = require('express'); 4 | var bodyParser = require('body-parser'); 5 | var uuid = require('uuid'); 6 | var buildDependencies = require('./lib/server/build-dependencies'); 7 | 8 | var app = express(); 9 | app.use(bodyParser.json()); 10 | 11 | app.post('/build', function(req, res) { 12 | var packages = req.body.packages; 13 | var bucket = req.body.bucket; 14 | 15 | buildDependencies(packages, bucket, uuid.v4()) 16 | .then(function(data) { 17 | res.send(data); 18 | }) 19 | .catch(function(err) { 20 | console.log(err); 21 | response.status(500).send(); 22 | }); 23 | }); 24 | 25 | module.exports = app; 26 | 27 | app.set('port', app.get('env') === 'production' ? 80 : 8387); 28 | 29 | var server = app.listen(app.get('port'), function() { 30 | var host = server.address().address; 31 | var port = server.address().port; 32 | 33 | console.log('Packager server running at http://%s:%s', host, port); 34 | }); 35 | 36 | server.timeout = 5 * 60 * 1000; 37 | -------------------------------------------------------------------------------- /lib/zip-file.js: -------------------------------------------------------------------------------- 1 | var archiver = require('archiver'); 2 | var Promise = require('rsvp').Promise; 3 | var fs = require('fs'); 4 | var debug = require('debug')('lambda-packager:zip-file'); 5 | 6 | var ZipFile = function(options) { 7 | this.sourceDirectory = options.sourceDirectory; 8 | this.destinationZip = options.destinationZip; 9 | }; 10 | 11 | ZipFile.prototype.zip = function() { 12 | var destinationZip = this.destinationZip; 13 | var sourceDirectory = this.sourceDirectory; 14 | 15 | debug('zipping; from=', sourceDirectory, '; to=', destinationZip); 16 | 17 | return new Promise(function(resolve, reject) { 18 | var archive = archiver.create('zip', {}); 19 | var output = fs.createWriteStream(destinationZip); 20 | 21 | output.on('close', resolve); 22 | archive.on('error', reject); 23 | 24 | archive.pipe(output); 25 | 26 | archive.bulk([{ 27 | expand: true, 28 | cwd: sourceDirectory, 29 | src: ['**'] 30 | }]); 31 | 32 | archive.finalize(); 33 | }); 34 | }; 35 | 36 | module.exports = ZipFile; 37 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lambda-packager", 3 | "version": "0.2.1", 4 | "description": "Builds dependencies for Node.js AWS Lambda functions", 5 | "main": "lib/packager.js", 6 | "scripts": { 7 | "test": "mocha test/**/*-test.js" 8 | }, 9 | "bin": { 10 | "lambda-packager": "bin/lambda-packager" 11 | }, 12 | "repository": "tomdale/lambda-packager", 13 | "keywords": [ 14 | "aws", 15 | "lambda" 16 | ], 17 | "author": "Tom Dale ", 18 | "license": "MIT", 19 | "bugs": { 20 | "url": "https://github.com/tomdale/lambda-packager/issues" 21 | }, 22 | "homepage": "https://github.com/tomdale/lambda-packager#readme", 23 | "devDependencies": { 24 | "adm-zip": "^0.4.7", 25 | "chai": "^3.4.0", 26 | "chai-as-promised": "^5.1.0", 27 | "mocha": "^2.3.3" 28 | }, 29 | "dependencies": { 30 | "archiver": "^0.16.0", 31 | "aws-sdk": "^2.2.13", 32 | "body-parser": "^1.14.1", 33 | "chalk": "^1.1.1", 34 | "commander": "^2.9.0", 35 | "debug": "^2.2.0", 36 | "express": "^4.13.3", 37 | "fs-extra": "^0.24.0", 38 | "fs-promise": "^0.3.1", 39 | "inquirer": "^0.11.0", 40 | "lodash.merge": "^3.3.2", 41 | "request-promise": "^1.0.2", 42 | "rsvp": "^3.1.0", 43 | "temp": "^0.8.3", 44 | "user-home": "^2.0.0", 45 | "uuid": "^2.0.1" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /bin/lambda-packager: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var program = require('commander'); 4 | var pkg = require('../package.json'); 5 | var chalk = require('chalk'); 6 | var red = chalk.red; 7 | var ui = require('../lib/util/ui'); 8 | 9 | // Enable logging 10 | ui.silent = false; 11 | 12 | // Commands 13 | var PackageCommand = require('../lib/commands/package'); 14 | var DeployCommand = require('../lib/commands/deploy'); 15 | 16 | program 17 | .version(pkg.version) 18 | .description("Tool for building Node packages to be deployed to AWS Lambda"); 19 | 20 | program.command('package ') 21 | .description('Builds dependencies for the specified package and saves the package plus dependencies in the specific zip file') 22 | .action(function(packageDirectory, outputZip) { 23 | PackageCommand.run(packageDirectory, outputZip); 24 | }); 25 | 26 | program.command('deploy') 27 | .description('Provisions an AWS CloudFormation stack for building dependencies') 28 | .action(function() { 29 | DeployCommand.run(); 30 | }); 31 | 32 | program.command('*', null, { noHelp: true } ) 33 | .action(function (cmd){ 34 | console.log(red(" " + cmd + " is not a valid command.")); 35 | program.outputHelp(); 36 | }); 37 | 38 | // If no command is provided, print help and exit 39 | if (process.argv.length == 2) { 40 | program.parse(process.argv); 41 | program.outputHelp(); 42 | process.exit(); 43 | } 44 | 45 | program.parse(process.argv); 46 | -------------------------------------------------------------------------------- /lib/aws/lambda.js: -------------------------------------------------------------------------------- 1 | var AWS = require('aws-sdk'); 2 | var RSVP = require('rsvp'); 3 | var debug = require('debug')('lambda-packager:lambda'); 4 | var ui = require('../util/ui'); 5 | var LambdaError = require('../errors/lambda-error'); 6 | 7 | function Lambda(config) { 8 | this.name = config.lambdaFunction; 9 | this.lambda = new AWS.Lambda({ 10 | apiVersion: '2015-03-31', 11 | region: config.region 12 | }); 13 | } 14 | 15 | Lambda.prototype.invoke = function(payload) { 16 | var lambda = this.lambda; 17 | var name = this.name; 18 | 19 | debug('invoking lambda function; name=', name, '; payload=', payload); 20 | 21 | return new RSVP.Promise(function(resolve, reject) { 22 | var params = { 23 | FunctionName: name, 24 | Payload: JSON.stringify(payload), 25 | LogType: "Tail" 26 | }; 27 | 28 | lambda.invoke(params, function(err, data) { 29 | if (err) { 30 | debug('lambda error; err=', err); 31 | return reject(err); 32 | } 33 | 34 | if (LambdaError.detect(data)) { 35 | debug('lambda function error'); 36 | return reject(new LambdaError(data)); 37 | } 38 | 39 | var payload = JSON.parse(data.Payload); 40 | 41 | printLambdaLog(data.LogResult); 42 | 43 | debug('lambda succeeded; payload=', payload); 44 | resolve(payload); 45 | }); 46 | }); 47 | }; 48 | 49 | function printLambdaLog(log) { 50 | // Convert Base64 to string 51 | log = new Buffer(log, "base64").toString(); 52 | 53 | ui.log(log); 54 | } 55 | 56 | module.exports = Lambda; 57 | -------------------------------------------------------------------------------- /lib/config.js: -------------------------------------------------------------------------------- 1 | var fsp = require('fs-promise'); 2 | var merge = require('lodash.merge'); 3 | var path = require('path'); 4 | var userHome = require('user-home'); 5 | var ui = require('./util/ui'); 6 | 7 | var CONFIG_PATH = path.join(userHome, ".lambda-packager/config.json"); 8 | 9 | // The Config class reads settings from disk on instantiation and uses a set of 10 | // default configuration options if they cannot be found. 11 | function Config() { 12 | } 13 | 14 | Config.prototype.read = function() { 15 | var config; 16 | 17 | try { 18 | config = fsp.readJsonSync(CONFIG_PATH); 19 | } catch(e) { 20 | ui.error("Unable to read " + CONFIG_PATH); 21 | ui.error("Run lambda-packager deploy to deploy and configure Lambda Packager."); 22 | process.exit(); 23 | } 24 | 25 | this.region = config.region; 26 | this.bucket = config.bucket; 27 | this.lambdaFunction = config.lambdaFunction; 28 | this.serverURL = config.serverURL; 29 | 30 | return this; 31 | }; 32 | 33 | Config.prototype.save = function() { 34 | var region = this.region; 35 | var bucket = this.bucket; 36 | var lambdaFunction = this.lambdaFunction; 37 | var serverURL = this.serverURL; 38 | 39 | fsp.ensureDir(path.dirname(CONFIG_PATH)) 40 | .then(function() { 41 | return fsp.writeJson(CONFIG_PATH, { 42 | region: region, 43 | bucket: bucket, 44 | lambdaFunction: lambdaFunction, 45 | serverURL: serverURL 46 | }); 47 | }) 48 | .catch(function(err) { 49 | console.log(err.stack); 50 | }); 51 | }; 52 | 53 | module.exports = Config; 54 | -------------------------------------------------------------------------------- /test/acceptance/basic-test.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | var expect = require('chai').expect; 3 | var fsp = require('fs-promise'); 4 | 5 | var packager = require('../../lib/packager'); 6 | var tmpDir = require('../helpers/tmp').create(); 7 | var exists = require('../helpers/exists'); 8 | var zip = require('../helpers/zip'); 9 | 10 | describe("Package", function() { 11 | this.timeout(500000); 12 | 13 | describe("with pure JS dependencies", function() { 14 | var zipPath = path.join(tmpDir, 'simple-package.zip'); 15 | 16 | before(function() { 17 | return packager.build({ 18 | from: path.resolve(__dirname, '../fixtures/simple-package'), 19 | to: zipPath 20 | }); 21 | }); 22 | 23 | it("should create a zip file", function() { 24 | return expect(exists(zipPath)).to.eventually.be.true; 25 | }); 26 | 27 | it("should have a node_modules directory", function() { 28 | zip(zipPath).shouldInclude([ 29 | 'node_modules/', 30 | 'node_modules/chalk/', 31 | 'node_modules/chalk/package.json', 32 | 'package.json' 33 | ]); 34 | }); 35 | 36 | }); 37 | 38 | describe("with native dependencies", function() { 39 | var zipPath = path.join(tmpDir, 'native-dependencies-package.zip'); 40 | 41 | before(function() { 42 | return packager.build({ 43 | from: path.resolve(__dirname, '../fixtures/native-dependencies-package'), 44 | to: zipPath 45 | }); 46 | }); 47 | 48 | it("should create a zip file", function() { 49 | return expect(exists(zipPath)).to.eventually.be.true; 50 | }); 51 | 52 | it("should have a node_modules directory", function() { 53 | zip(zipPath).shouldInclude([ 54 | 'node_modules/', 55 | 'node_modules/chalk/', 56 | 'node_modules/chalk/package.json', 57 | 'node_modules/contextify/', 58 | 'node_modules/contextify/package.json', 59 | 'package.json' 60 | ]); 61 | }); 62 | 63 | }); 64 | 65 | }); 66 | 67 | -------------------------------------------------------------------------------- /lib/commands/deploy.js: -------------------------------------------------------------------------------- 1 | var CloudFormation = require('../aws/cloud-formation'); 2 | var Config = require('../config'); 3 | var prompt = require('../util/prompt'); 4 | var green = require('chalk').green; 5 | var red = require('chalk').red; 6 | var cyan = require('chalk').cyan; 7 | var path = require('path'); 8 | var util = require('util'); 9 | var ui = require('../util/ui'); 10 | var root = __dirname; 11 | 12 | var DeployCommand = { 13 | run: function() { 14 | var config = new Config(); 15 | 16 | var cloudFormation; 17 | 18 | return askQuestions() 19 | .then(function(answers) { 20 | ui.log("Creating stack " + answers.stackName + "..."); 21 | ui.log("This may take a few minutes."); 22 | 23 | config.region = answers.region; 24 | 25 | cloudFormation = new CloudFormation({ 26 | apiVersion: '2010-05-15', 27 | region: config.region 28 | }); 29 | 30 | var templatePath = path.join(root, '../../blueprints/cloudformation-template.json'); 31 | return cloudFormation.createStack(answers.stackName, templatePath); 32 | }) 33 | .then(function(stackID) { 34 | return cloudFormation.poll(stackID); 35 | }) 36 | .then(function(outputs) { 37 | config.bucket = outputs.Bucket; 38 | config.lambdaFunction = outputs.Function; 39 | config.save(); 40 | 41 | ui.log("Saved AWS configuration:"); 42 | ui.log("Bucket: " + cyan(config.bucket)); 43 | ui.log("Lambda Function: " + cyan(config.lambdaFunction)); 44 | }) 45 | .catch(function(err) { 46 | ui.error(err.message || util.inspect(err)); 47 | }); 48 | } 49 | }; 50 | 51 | function askQuestions() { 52 | var questions = [{ 53 | type: 'input', 54 | name: 'stackName', 55 | default: 'lambda-packager', 56 | message: 'Stack name' 57 | }, { 58 | type: 'region', 59 | name: 'region', 60 | default: 'us-east-1', 61 | message: 'Region' 62 | }]; 63 | 64 | return prompt(questions); 65 | } 66 | 67 | module.exports = DeployCommand; 68 | -------------------------------------------------------------------------------- /lib/aws/cloud-formation.js: -------------------------------------------------------------------------------- 1 | var AWS = require('aws-sdk'); 2 | var RSVP = require('rsvp'); 3 | var debug = require('debug')('lambda-packager:cloud-formation'); 4 | var fsp = require('fs-promise'); 5 | var ui = require('../util/ui'); 6 | var green = require('chalk').green; 7 | 8 | function CloudFormation(config) { 9 | this.cloudFormation = new AWS.CloudFormation({ 10 | apiVersion: '2015-03-31', 11 | region: config.region 12 | }); 13 | } 14 | 15 | CloudFormation.prototype.createStack = function(name, templatePath) { 16 | var cloudFormation = this.cloudFormation; 17 | 18 | return fsp.readFile(templatePath) 19 | .then(function(template) { 20 | template = template.toString(); 21 | 22 | debug('creating stack; name=', name, '; template=', template); 23 | 24 | return new RSVP.Promise(function(resolve, reject) { 25 | var params = { 26 | StackName: name, 27 | Capabilities: [ 28 | 'CAPABILITY_IAM' 29 | ], 30 | OnFailure: 'DELETE', 31 | TemplateBody: template 32 | }; 33 | 34 | cloudFormation.createStack(params, function(err, data) { 35 | if (err) { 36 | debug('cloudformation error; err=', err); 37 | return reject(err); 38 | } 39 | 40 | debug('createStack succeeded; data=', data); 41 | resolve(data.StackId); 42 | }); 43 | }); 44 | }); 45 | 46 | }; 47 | 48 | CloudFormation.prototype.poll = function(stackID) { 49 | var cloudFormation = this.cloudFormation; 50 | 51 | return new RSVP.Promise(function(resolve, reject) { 52 | var timer = setInterval(function() { 53 | var params = { 54 | StackName: stackID 55 | }; 56 | 57 | cloudFormation.describeStacks(params, function(err, data) { 58 | if (err) { 59 | clearTimer(); 60 | reject(err); 61 | return; 62 | } 63 | 64 | var status = data.Stacks[0].StackStatus; 65 | switch(status) { 66 | case 'CREATE_IN_PROGRESS': 67 | process.stdout.write(green('.')); 68 | break; 69 | case 'CREATE_COMPLETE': 70 | clearTimer(); 71 | ui.log("\nStack created"); 72 | resolve(mapOutputs(data.Stacks[0].Outputs)); 73 | break; 74 | default: 75 | clearTimer(); 76 | reject(data); 77 | } 78 | }); 79 | }, 2000); 80 | 81 | function clearTimer() { 82 | clearInterval(timer); 83 | } 84 | }); 85 | }; 86 | 87 | function mapOutputs(outputs) { 88 | var mapped = {}; 89 | 90 | outputs.forEach(function(item) { 91 | mapped[item.OutputKey] = item.OutputValue; 92 | }); 93 | 94 | return mapped; 95 | } 96 | 97 | module.exports = CloudFormation; 98 | -------------------------------------------------------------------------------- /lib/packager.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | var fsp = require('fs-promise'); 4 | var RSVP = require('rsvp'); 5 | var Promise = require('rsvp').Promise; 6 | var path = require('path'); 7 | var util = require('util'); 8 | var exec = RSVP.denodeify(require('child_process').exec); 9 | var ZipFile = require('./zip-file'); 10 | var HTTPDependencyBuilder = require('./http-dependency-builder'); 11 | var temp = require('temp').track(); 12 | var mktempdir = RSVP.denodeify(temp.mkdir); 13 | var Config = require('./config'); 14 | var red = require('chalk').red; 15 | var ui = require('./util/ui'); 16 | var debug = require('debug')('lambda-packager:packager'); 17 | 18 | var Packager = function() { 19 | }; 20 | 21 | Packager.prototype.build = function(options) { 22 | var packagePath = options.from; 23 | var zipPath = options.to; 24 | var tmpDir; 25 | var outputPath; 26 | 27 | var config = new Config().read(); 28 | 29 | return createTempDirectory() 30 | .then(verifyInputs) 31 | .then(copySourceDirectory) 32 | .then(buildLambdaDependencies) 33 | .then(zipPackage); 34 | 35 | function createTempDirectory() { 36 | debug('creating temp directory'); 37 | return mktempdir('lambda-packager') 38 | .then(function(dirPath) { 39 | tmpDir = dirPath; 40 | outputPath = path.join(tmpDir, path.basename(packagePath)); 41 | }); 42 | } 43 | 44 | function verifyInputs() { 45 | return fsp.stat(packagePath) 46 | .then(function(stats) { 47 | if (!stats.isDirectory()) { 48 | throw new Error(packagePath + " is not a directory"); 49 | } 50 | }); 51 | } 52 | 53 | function copySourceDirectory() { 54 | debug('copying source directory'); 55 | // Arguments to cp: 56 | // 57 | // -R Recursively copies the directory. 58 | // -L Resolves all symlinks. Symlinks out of this directory ("../../") 59 | // won't work once copied to a tmp directory 60 | return exec("cp -LR " + packagePath + " " + outputPath); 61 | } 62 | 63 | function buildLambdaDependencies() { 64 | var packageJSONPath = path.join(outputPath, 'package.json'); 65 | 66 | var builder = new HTTPDependencyBuilder({ 67 | config: config, 68 | packageJSONPath: packageJSONPath, 69 | destination: outputPath 70 | }); 71 | 72 | return builder.build(); 73 | } 74 | 75 | function zipPackage() { 76 | ui.log("Saving package to " + zipPath); 77 | 78 | var zipFile = new ZipFile({ 79 | sourceDirectory: outputPath, 80 | destinationZip: zipPath 81 | }); 82 | 83 | return zipFile.zip(); 84 | } 85 | }; 86 | 87 | module.exports = new Packager(); 88 | -------------------------------------------------------------------------------- /lib/http-dependency-builder.js: -------------------------------------------------------------------------------- 1 | var AWS = require('aws-sdk'); 2 | var RSVP = require('rsvp'); 3 | var fsp = require('fs-promise'); 4 | var path = require('path'); 5 | var S3 = require('./aws/s3'); 6 | var ui = require('./util/ui'); 7 | var green = require('chalk').green; 8 | var exec = RSVP.denodeify(require('child_process').exec); 9 | var request = require('request-promise'); 10 | 11 | /** 12 | * Given the path to a package.json file, the DependencyBuilder 13 | * will transmit those packages to a Lambda Packager-compatible 14 | * server for compilation and save the resulting node_modules 15 | * directory in the `destination` folder. 16 | */ 17 | var DependencyBuilder = function(options) { 18 | this.config = options.config; 19 | this.packageJSONPath = options.packageJSONPath; 20 | this.destination = options.destination; 21 | }; 22 | 23 | DependencyBuilder.prototype.build = function() { 24 | var config = this.config; 25 | var dest = this.destination; 26 | var s3 = new S3(config); 27 | var nodeModulesZip = path.join(dest, 'node_modules.zip'); 28 | 29 | var timeout = printDots(); 30 | 31 | // Transmit our package.json to the Lambda function to 32 | // be built. 33 | function buildDependencies(packageJSON) { 34 | var payload = { 35 | packages: JSON.parse(packageJSON), 36 | bucket: config.bucket 37 | }; 38 | 39 | return postJSON(payload) 40 | .finally(stopDots(timeout)); 41 | } 42 | 43 | function postJSON(payload) { 44 | var serverURL = config.serverURL; 45 | 46 | ui.log('Sending package.json to ' + serverURL); 47 | ui.log('Compiling dependencies remotely'); 48 | 49 | var options = { 50 | method: 'POST', 51 | uri: config.serverURL, 52 | body: payload, 53 | timeout: 5 * 60 * 1000, 54 | json: true 55 | }; 56 | 57 | return request(options); 58 | } 59 | 60 | // Once the Lambda function returns the request ID, 61 | // we can fetch the zip of node_modules from S3. 62 | function downloadZippedDependencies(result) { 63 | var requestID = result.requestID; 64 | 65 | ui.log('Downloading zipped dependencies'); 66 | 67 | return s3.download({ 68 | key: requestID + "/node_modules.zip", 69 | destination: nodeModulesZip 70 | }); 71 | } 72 | 73 | // Unzip the zipped node_modules directory that we fetched from S3, 74 | // then delete the archive once decompressed. 75 | function unzipDependencies() { 76 | ui.log('Unzipping dependencies'); 77 | 78 | return exec('unzip -q ' + nodeModulesZip + " -d " + dest) 79 | .then(function() { 80 | return fsp.remove(nodeModulesZip); 81 | }); 82 | } 83 | 84 | return fsp.readFile(this.packageJSONPath) 85 | .then(buildDependencies) 86 | .then(downloadZippedDependencies) 87 | .then(unzipDependencies); 88 | }; 89 | 90 | function printDots() { 91 | return setInterval(function() { 92 | process.stdout.write(green('.')); 93 | }, 1000); 94 | } 95 | 96 | function stopDots(timeout) { 97 | return function(requestID) { 98 | ui.log(''); 99 | clearInterval(timeout); 100 | return requestID; 101 | }; 102 | } 103 | 104 | module.exports = DependencyBuilder; 105 | -------------------------------------------------------------------------------- /lib/server/build-dependencies.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'); 2 | var childProcess = require('child_process'); 3 | var path = require('path'); 4 | var AWS = require('aws-sdk'); 5 | var RSVP = require('rsvp'); 6 | var archiver = require('archiver'); 7 | var chalk = require('chalk'); 8 | var util = require('util'); 9 | 10 | var mkdir = RSVP.denodeify(fs.mkdir); 11 | var writeFile = RSVP.denodeify(fs.writeFile); 12 | var exec = RSVP.denodeify(childProcess.exec); 13 | 14 | var root = __dirname; 15 | 16 | // This is the main entry point that Lambda invokes to handle the 17 | // request to build the provided packages. 18 | // 19 | // The workflow is: 20 | // 21 | // 1. Create /tmp// 22 | // 2. Copy the provided package.json to /tmp//package.json 23 | // 3. Run npm install in that directory 24 | // 4. Zip the resulting node_modules directory 25 | // 5. Upload the zip file to //node_modules.zip 26 | 27 | module.exports = function buildDependencies(packages, bucket, requestID) { 28 | var tmpDir = path.join("/tmp/", requestID); 29 | var zipPath = path.join(tmpDir, "node_modules.zip"); 30 | var output = []; 31 | var error; 32 | 33 | return createTempDirectory() 34 | .then(writePackageJSON(packages)) 35 | .then(installPackages) 36 | .then(zipPackages) 37 | .then(uploadPackagesToS3) 38 | .then(function() { 39 | return { 40 | requestID: requestID 41 | }; 42 | }) 43 | .finally(function() { 44 | cleanupTempDirectory(); 45 | }); 46 | 47 | function createTempDirectory() { 48 | log('creating tmpdir; path=' + tmpDir); 49 | return mkdir(tmpDir); 50 | } 51 | 52 | function cleanupTempDirectory() { 53 | log('cleaning up tmpdir; path=' + tmpDir ); 54 | return exec("rm -rf " + tmpDir); 55 | } 56 | 57 | function writePackageJSON(packages) { 58 | return function() { 59 | var packageJSONPath = path.join(tmpDir, "package.json"); 60 | log("writing package.json; path=" + packageJSONPath); 61 | return writeFile(packageJSONPath, JSON.stringify(packages)); 62 | }; 63 | } 64 | 65 | function installPackages() { 66 | //var command = "node " + root + "/node_modules/npm/bin/npm-cli.js install"; 67 | var command = "npm install"; 68 | 69 | log("execing; cwd=" + tmpDir + "; cmd=" + command); 70 | 71 | return exec(command, { 72 | cwd: tmpDir 73 | }); 74 | } 75 | 76 | function zipPackages() { 77 | var command = "zip -qr " + zipPath + " node_modules"; 78 | 79 | log("zipping; from=" + tmpDir + "; to=" + zipPath); 80 | log("execing; cwd=" + tmpDir + "; cmd=" + command); 81 | 82 | return exec(command, { 83 | cwd: tmpDir 84 | }); 85 | } 86 | 87 | function uploadPackagesToS3() { 88 | var s3 = new AWS.S3(); 89 | var upload = RSVP.denodeify(s3.upload.bind(s3)); 90 | var stream = fs.createReadStream(zipPath); 91 | var key = requestID + '/node_modules.zip'; 92 | 93 | log("uploading to S3; bucket=" + bucket + "; key=" + key); 94 | 95 | return upload({ 96 | Bucket: bucket, 97 | Key: key, 98 | Body: stream 99 | }); 100 | } 101 | 102 | function log() { 103 | var message = util.format.apply(this, arguments); 104 | _log(chalk.green(message)); 105 | } 106 | 107 | function logError() { 108 | var message = util.format.apply(this, arguments); 109 | _log(chalk.red(message)); 110 | } 111 | 112 | function _log(message) { 113 | var date = (new Date()).toUTCString(); 114 | console.log("[" + date + "] [" + requestID + "] " + message); 115 | } 116 | }; 117 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Lambda Packager 2 | 3 | Lambda Packager builds your project's npm packages for use on AWS 4 | Lambda using AWS Lambda. 5 | 6 | ### Motivation 7 | 8 | AWS Lambda runs Node.js apps, but you have to provide the `node_modules` 9 | directory yourself. Building it on your local machine won't work if your 10 | dependencies contain native code that needs to be compiled. Amazon 11 | suggests running an EC2 instance, compiling the dependencies there 12 | manually, then copying them back to your machine over SSH. 13 | 14 | Lambda Packager makes deploying Node.js code to AWS Lambda easy, by 15 | using Lambda itself to compile your dependencies. 16 | 17 | Just provide Lambda Packager with a directory containing your Lambda 18 | function (it must contains a `package.json` file with a list of 19 | dependencies), and it will build a zip file of that directory with a 20 | Lambda-compatible `node_modules` directory that's ready to deploy. 21 | 22 | ### Usage 23 | 24 | **Note**: You must deploy the Lambda builder before these commands will 25 | work. See [Deployment](#deployment). 26 | 27 | #### Command Line 28 | 29 | ```sh 30 | lambda-packager package my-package output.zip 31 | ``` 32 | 33 | #### Programmatic 34 | 35 | ```js 36 | var lambdaPackager = require('lambda-packager'); 37 | 38 | lambdaPackager.build({ 39 | from: 'my-package', 40 | to: 'output.zip' 41 | }); 42 | ``` 43 | 44 | Assuming `my-package` is a path to a directory with a `package.json` 45 | file, its dependencies will be compiled via Lambda, then the package 46 | plus the dependencies will be placed into `my-package-function.zip`. 47 | 48 | ## Deployment 49 | 50 | To build dependencies, Lambda Packager uploads your `package.json` to a 51 | Lambda function that builds your dependencies in the AWS environment. 52 | 53 | To deploy this builder function to AWS, run the `lambda-packager deploy` 54 | command, which will prompt you for the name to use for the 55 | CloudFormation stack, as well as what region to create it in. It will 56 | automatically use the same credentials as the AWS CLI. 57 | 58 | This command builds a CloudFormation stack that provisions 59 | everything needed to build dependencies for Lambda Packager: 60 | 61 | * IAM Role 62 | * Lambda Function 63 | * S3 Bucket 64 | 65 | Make sure that the AWS account you have authorized via the AWS CLI has 66 | permission to create each of these resources. 67 | 68 | ### Example 69 | 70 | Imagine I have a Node.js Lambda function that I want to deploy. It's 71 | file structure looks like this: 72 | 73 | ``` 74 | simple-package 75 | ├── index.js 76 | └── package.json 77 | ``` 78 | 79 | Your `package.json` contains a list of dependencies, like this: 80 | 81 | ```js 82 | { 83 | "dependencies": { 84 | "contextify": "0.1.14" 85 | } 86 | } 87 | ``` 88 | 89 | `contextify` contains native code, so if I run `npm install` on my 90 | computer and upload my project to Lambda, it won't work. Instead, I'll 91 | use `lambda-packager` to build it: 92 | 93 | ```sh 94 | lambda-packager package simple-package simple-package-lambda.zip 95 | ``` 96 | 97 | After a few seconds to a few minutes (depending on the number of 98 | dependencies), the `package` command will produce the 99 | `simple-package-lambda.zip` file in my current directory. That zip file, 100 | if expanded, would look like this: 101 | 102 | ``` 103 | simple-package 104 | ├── index.js 105 | └── package.json 106 | └── node_modules 107 |     └── contextify 108 | ``` 109 | 110 | Because the zip file is just your package with a Lambda-compatible 111 | `node_modules` directory, it's ready to upload to your Lambda function, 112 | either via the AWS CLI, the AWS console, or via another tool. 113 | 114 | ## How It Works 115 | 116 | ![diagram of lambda packager architecture](assets/diagram.png) 117 | 118 | npm packages written in pure JavaScript run fine on Lambda, but many 119 | packages contain native code (written in C or C++) that must be 120 | compiled. If you build those dependencies on your local machine, they're 121 | unlikely to work on the custom version of Amazon Linux that powers AWS 122 | Lambda. 123 | 124 | Lambda Packager works by invoking a Lambda function running on AWS and 125 | uploading your project's `package.json` to it. It copies that 126 | `package.json` to a temporary directory, then runs `npm install` to 127 | compile the dependencies in the Lambda environments 128 | 129 | Once compilation is complete, it uploads the Lambda-compatible 130 | dependencies to S3. Those dependencies are then downloaded back to your 131 | local machine. 132 | 133 | To facilitate deployment, Lambda Packager will create a copy of your 134 | Node package, copy in the Lambda-built `node_modules` directory, and 135 | create a zip file that is ready to deploy via the AWS console or CLI 136 | utility. 137 | 138 | ## Thanks 139 | 140 | Lambda Packager was inspired by the [Thaumaturgy][thaumaturgy] project. 141 | I wanted to make something more automated that used my projects' 142 | `package.json`, rather than specifying dependencies manually. I also 143 | wanted something that bundled everything into a ready-to-deploy zip. 144 | 145 | Work on this project is generously sponsored by [Bustle Labs][bustle-labs]. 146 | 147 | [thaumaturgy]: https://github.com/node-hocus-pocus/thaumaturgy 148 | [bustle-labs]: http://www.bustle.com/labs 149 | --------------------------------------------------------------------------------