├── .eslintrc ├── .gitignore ├── CHANGELOG.md ├── Gruntfile.js ├── LICENSE ├── README.md ├── docs └── images │ ├── lambda-complex-overview.odp │ └── lambda-complex-overview.png ├── examples ├── exampleApplicationConfig.js └── simple │ ├── README.md │ ├── applicationConfig.js │ ├── messageTransformer │ ├── index.js │ └── package.json │ └── processor │ ├── index.js │ └── package.json ├── index.js ├── lib ├── build │ ├── cloudFormationTemplateUtilities.js │ ├── common.js │ ├── configValidator.js │ ├── installUtilities.js │ ├── packageUtilities.js │ └── template │ │ └── index.js.hbs ├── deploy │ ├── cloudFormationUtilities.js │ └── s3Utilities.js ├── grunt │ └── common.js ├── lambdaFunctions │ └── coordinator │ │ ├── README.md │ │ ├── common.js │ │ ├── coordinator.js │ │ ├── index.js │ │ ├── invoker.js │ │ └── package.json └── shared │ ├── constants.js │ └── utilities.js ├── package-lock.json ├── package.json ├── tasks ├── build.js └── deploy.js └── test ├── .eslintrc ├── index.spec.js ├── lib ├── build │ ├── cloudFormationTemplateUtilities.spec.js │ ├── common.spec.js │ ├── configValidator.spec.js │ └── template │ │ └── index.spec.js ├── deploy │ ├── cloudFormationUtilities.spec.js │ └── s3Utilities.spec.js ├── grunt │ └── common.spec.js ├── lambdaFunctions │ └── coordinator │ │ ├── common.spec.js │ │ ├── coordinator.spec.js │ │ ├── index.spec.js │ │ └── invoker.spec.js └── shared │ └── utilities.spec.js ├── mochaInit.js ├── resources ├── index.js ├── mockApplication │ ├── applicationConfig.js │ ├── cloudFormation.json │ └── mockLambdaFunction │ │ ├── index.js │ │ └── package.json └── uncaughtExceptionTest.js └── tasks ├── build.spec.js └── deploy.spec.js /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "node": true 4 | }, 5 | "rules": { 6 | "brace-style": [2, "stroustrup", { "allowSingleLine": true }], 7 | "consistent-return": 0, 8 | "curly": 2, 9 | "eqeqeq": 2, 10 | "indent": [2, 2, { "SwitchCase": 1 }], 11 | "key-spacing": [2, { 12 | "beforeColon": false, 13 | "afterColon": true 14 | }], 15 | "keyword-spacing": [2, { "before": true, "after": true }], 16 | "no-multiple-empty-lines": 2, 17 | "no-throw-literal": 2, 18 | "no-underscore-dangle": 0, 19 | "no-unused-vars": [2, { 20 | "vars": "all", 21 | "args": "none" 22 | }], 23 | "no-use-before-define": [2, "nofunc"], 24 | "object-curly-spacing": [2, "always"], 25 | "quote-props": [2, "as-needed"], 26 | "quotes": [2, "single"], 27 | "radix": 2, 28 | "space-before-blocks": [2, "always"], 29 | "space-before-function-paren": [2, "always"], 30 | "space-in-parens": [2, "never"], 31 | "space-unary-ops": [2, { 32 | "words": true, 33 | "nonwords": false 34 | }], 35 | "strict": [2, "never"], 36 | "wrap-iife": [2, "inside"] 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Sublime Text 2 | *sublime-* 3 | 4 | # Lock files for LibreOffice. 5 | **/.~lock* 6 | 7 | # Node, NPM 8 | **/node_modules 9 | npm* 10 | 11 | # TODO lists. 12 | TODO* 13 | 14 | # Built packages. 15 | build/ 16 | 17 | # Testing. 18 | test/scratch 19 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 0.7.0 4 | 5 | * Deal with the changes in NPM since the last update, particularly local 6 | packages now installing as symlinks rather than being copied into place. 7 | 8 | ## 0.6.0 9 | 10 | * Rename and export Grunt tasks. 11 | * Update documentation, add an overview diagram. 12 | 13 | ## 0.5.1 14 | 15 | * Minor configuration validation improvement. 16 | 17 | ## 0.5.0 18 | 19 | * Improve configuration validation. 20 | * Expand unit test converage. 21 | 22 | ## 0.4.0 23 | 24 | * Add option to delete old CloudWatch log groups on deployment. 25 | * Make deployment options for stack deletion more granular. 26 | 27 | ## 0.3.2 28 | 29 | * `VisibilityTimeout` cannot be 0 in the `decrementConcurrencyCount` function, as this can cause message deletion to fail silently. 30 | 31 | ## 0.3.1 32 | 33 | * Alter increment/decrement error behavior in invoker to match coordinator. 34 | * Fix concurrency decrement for the case in which no message is found. 35 | 36 | ## 0.3.0 37 | 38 | * Upload `config.js` alongside the other deployment files. Helpful to keep a record. 39 | * The first coordinator instances following deployment upload `confirm.txt` on success. 40 | * Deployment waits for `confirm.txt` to exist. 41 | 42 | ## 0.2.0 43 | 44 | * Add a maximum concurrency limit for `eventFromMessage` components. 45 | * Allow multiple concurrent coordinator instances. 46 | * Improve documentation. 47 | 48 | ## 0.1.0 49 | 50 | * Initial release. 51 | -------------------------------------------------------------------------------- /Gruntfile.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Definition file for grunt tasks. 3 | */ 4 | 5 | // Core. 6 | var path = require('path'); 7 | 8 | module.exports = function (grunt) { 9 | 10 | // Always output stack traces. 11 | grunt.option('stack', true); 12 | 13 | grunt.initConfig({ 14 | clean: [ 15 | 'build', 16 | 'test/scratch' 17 | ], 18 | 19 | eslint: { 20 | target: [ 21 | '**/*.js', 22 | '!**/node_modules/**', 23 | '!build/**', 24 | '!test/scratch/**' 25 | ] 26 | }, 27 | 28 | mochaTest: { 29 | test: { 30 | options: { 31 | reporter: 'spec', 32 | quiet: false, 33 | clearRequireCache: false, 34 | require: [ 35 | path.join(__dirname, 'test/mochaInit.js') 36 | ] 37 | }, 38 | src: [ 39 | 'test/**/*.spec.js' 40 | ] 41 | } 42 | } 43 | }); 44 | 45 | // Loads local tasks for this module. 46 | grunt.loadTasks('tasks'); 47 | 48 | // Loading NPM module grunt tasks. 49 | grunt.loadNpmTasks('grunt-contrib-clean'); 50 | grunt.loadNpmTasks('grunt-eslint'); 51 | grunt.loadNpmTasks('grunt-mocha-test'); 52 | 53 | grunt.registerTask('test', [ 54 | 'clean', 55 | 'eslint', 56 | 'mochaTest' 57 | ]); 58 | }; 59 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2015 Reason [reason -A- exratione.com] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /docs/images/lambda-complex-overview.odp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exratione/lambda-complex/1c9018e9a8b4164f86d61704291475bc0a18aa21/docs/images/lambda-complex-overview.odp -------------------------------------------------------------------------------- /docs/images/lambda-complex-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exratione/lambda-complex/1c9018e9a8b4164f86d61704291475bc0a18aa21/docs/images/lambda-complex-overview.png -------------------------------------------------------------------------------- /examples/exampleApplicationConfig.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview An example configuration file, with documentation. 3 | * 4 | * It is very important to note that the configuration object will, after it is 5 | * loaded by the deployment code, be: 6 | * 7 | * - Regenerated into code and added as a file in Lambda function modules. 8 | * - Loaded in Lambda by all Lambda functions in a Lambda Complex application. 9 | * 10 | * This means that almost everything that results in a valid configuration 11 | * object will just work, but there are things you can do that will result in 12 | * failure to run in Lambda. 13 | * 14 | * In particular note the restrictions on routing functions, described below in 15 | * the component inline comments. 16 | */ 17 | 18 | module.exports = { 19 | // The application name. It must be unique for a given account. 20 | // 21 | // When deploying an application the queue names and Lambda function names 22 | // are prefixed with the application name. 23 | name: 'example', 24 | 25 | // The application version. This is used in tags and some names. 26 | version: '1.0.0', 27 | 28 | // A deployment ID that is unique to this deployment of this application. For 29 | // example the BUILD_NUMBER value generated by Jenkins tasks, or a unix 30 | // timestamp, or similar. It can be alphanumeric. 31 | // 32 | // This is included in most tags, paths, and names. 33 | deployId: 15, 34 | 35 | // AWS deployment settings. 36 | // 37 | // The application is deployed as a CloudFormation stack, and after successful 38 | // completion any prior deployed stacks for this application are destroyed. On 39 | // a failure to deploy the new stack is destroyed. 40 | deployment: { 41 | 42 | // ---------------------------------------------------------------------- 43 | // Required. 44 | // ---------------------------------------------------------------------- 45 | 46 | // AWS region. 47 | // 48 | // This must match whatever region is configured in the deployment 49 | // environment. 50 | // 51 | // TODO: having this and the environment configuration is clumsy. Look at 52 | // cutting that down to one declaration one way or another. 53 | region: 'us-east-1', 54 | 55 | // The S3 bucket into which packaged Lambda functions will be uploaded. 56 | s3Bucket: 'lambda-complex', 57 | 58 | // All packaged Lambda function keys will be prefixed with this string. 59 | s3KeyPrefix: 'applications/', 60 | 61 | // ---------------------------------------------------------------------- 62 | // Optional. 63 | // ---------------------------------------------------------------------- 64 | 65 | // Any additional tags to add to the CloudFormation stack. 66 | tags: { 67 | tagName: 'tagValue' 68 | }, 69 | 70 | // An asynchronous function invoked after the new stack is deployed but 71 | // before the old stack is destroyed. This should perform any functions 72 | // needed to switch over to use the new stack and then call back. 73 | // 74 | // Generally this will mean switching to use new queues to introduce data 75 | // into the application and then waiting for the old queues to clear, but 76 | // the precise details will depend on the application. 77 | // 78 | // This function will not be invoked in Lambda functions when deployed, so 79 | // it can require() and invoke any resources available to Lambda Complex 80 | // during deployment. 81 | // 82 | // @param {Object} stackDescription Stack description, including outputs. 83 | // @param {Object} config This configuration object. 84 | // @param {Function} callback Of the form function (error). 85 | switchoverFunction: function (stackDescription, config, callback) { 86 | callback(); 87 | }, 88 | 89 | // If set true then CloudFormation stacks created in prior deployments of 90 | // this application are NOT destroyed on a successful deployment. 91 | // 92 | // This can be helpful during development. 93 | skipPriorCloudFormationStackDeletion: false, 94 | 95 | // If set true, then CloudWatch log groups created by the Lambda function 96 | // instances from prior deployments of this application are NOT destroyed 97 | // on a successful deployment. 98 | // 99 | // CloudWatch log groups are clutter for some people, others want to keep 100 | // them. 101 | skipPriorCloudWatchLogGroupsDeletion: false, 102 | 103 | // If set true, then the CloudFormation stack for this deployment is NOT 104 | // destroyed on deployment failure. 105 | // 106 | // This can be helpful during development. 107 | skipCloudFormationStackDeletionOnFailure: false 108 | }, 109 | 110 | // The coordinator is a lambda function package that is expected to consume 111 | // this configuration and manage the invocation of invoker lambda functions in 112 | // the required concurrency. 113 | coordinator: { 114 | // The maximum number of concurrent coordinators to run. 115 | // 116 | // Each coordinator will carry out its fraction of the overall work to be 117 | // done. E.g. for 2 concurrent coordinators, each will undertake half of the 118 | // necessary invocations. 119 | // 120 | // More than one coordinator provides a little redundancy against the random 121 | // slings and arrows of AWS API failure. The application relies on 122 | // coordinators invoking themselves, and while the coordinator code tries to 123 | // be bulletproof, there are still a number of points at which failure can 124 | // occur. 125 | // 126 | // So long as one coordinator is running, it will invoke as many other 127 | // instances as are needed to maintain this specified concurrency. 128 | coordinatorConcurrency: 2, 129 | 130 | // How many AWS API requests can the coordinator make at any one time? 131 | // Too many and you'll tend to see errors. 132 | maxApiConcurrency: 10, 133 | 134 | // How many Lambda functions, excluding the coordinator itself, can a 135 | // single coordinator instance launch? Too many and it can get timed out. 136 | // Too few and concurrency won't be maxed out. When making invocation 137 | // requests, the setting is obeyed, so it will limit 138 | // how many requests are made at one time. 139 | maxInvocationCount: 50, 140 | 141 | // Minimum interval in seconds between invocations. Has to be equal to 142 | // or less than the execution time limit for a lambda function, see: 143 | // http://docs.aws.amazon.com/lambda/latest/dg/limits.html 144 | // 145 | // A coordinator may well run longer than this interval before invoking the 146 | // next coordinator if it has a lot to do. 147 | // 148 | // The shorter the interval the more frequently that new component Lambda 149 | // function instances will be launched in response to messages in their 150 | // queues. This goes a long way towards determining the pace of the 151 | // application for small applications and small amounts of data. 152 | minInterval: 10 153 | }, 154 | 155 | // Every Lambda function, and thus every component, is associated with a 156 | // single IAM role. This role provides it with all of the necessary 157 | // permissions to access AWS resources. E.g. S3 buckets, etc. 158 | // 159 | // Component definitions each specify the use of one of these roles. 160 | // 161 | roles: [ 162 | // In the rare case that a Lambda function needs no permissions, it is 163 | // possible to assign an empty role. Lambda Complex will add the necessary 164 | // internal permissions to it, such as queue access and logging. 165 | { 166 | name: 'default', 167 | statements: [] 168 | }, 169 | // More commonly a role will contain one or more statements. These follow 170 | // the normal form for policy statements inside a CloudFormation role 171 | // definition (but with lowercase property names, for consistency with the 172 | // rest of this config file). 173 | // 174 | // In addition to the statements provided here, Lambda Complex will add 175 | // necessary internal permissions, such as queue access and logging. 176 | { 177 | name: 's3Read', 178 | statements: [ 179 | { 180 | effect: 'Allow', 181 | action: [ 182 | 's3:get*', 183 | 's3:list*' 184 | ], 185 | resource: [ 186 | 'arn:aws:s3:::exampleBucket1/*', 187 | 'arn:aws:s3:::exampleBucket2/*' 188 | ] 189 | } 190 | ] 191 | } 192 | ], 193 | 194 | // A component consists of a lambda function package that will be triggered in 195 | // response to circumstances, such as a message in a queue or the completion 196 | // of another Lambda function. 197 | // 198 | // There are several different types of component. 199 | // 200 | // Any number of components can be defined. 201 | components: [ 202 | 203 | // Example of a lambda function that consumes a message from a queue when 204 | // invoked, and the message data becomes the event passed to the lambda 205 | // function handler. 206 | // 207 | // This type of component can be run at a given concurrency, controlled by 208 | // the coordinator. 209 | { 210 | // The name of the lambda function definition to be created in 211 | // AWS. 212 | name: 'eventFromMessageExample', 213 | 214 | // Type of the component. 215 | type: 'eventFromMessage', 216 | 217 | // The maximum number of Lambda function instances for this component that 218 | // will run at any one time. This, coupled with coordinator.minInterval, 219 | // goes towards determining the rate at which this component's message 220 | // queue will drain. 221 | maxConcurrency: 10, 222 | 223 | // Seconds to wait for a message to show up in the queue before giving up. 224 | // This should not be high as it cuts into short time limit for a Lambda. 225 | // function. Setting a few seconds may smooth out the operation of an 226 | // application in future versions of Lambda Complex, but at the present 227 | // time should make no difference either way. 228 | queueWaitTime: 5, 229 | 230 | // Define where the results from this Lambda function are sent. The data 231 | // passed to context.succeed(data) or context.done(null, data) will be 232 | // send on to another component as the event passed to the handle of its 233 | // Lambda function. 234 | // 235 | // This is an optional property. If not defined, then data is not passed 236 | // on to any other component. 237 | // 238 | // If a string, then results on success will always be sent to an 239 | // invocation of the specified component's Lambda function. On failure 240 | // there is no invocation and no data sent. 241 | routing: 'eventFromInvocationExample', 242 | // 243 | // If an array of strings, then a copy of the results on success is sent 244 | // to an invocation of the Lambda function for each of the specified 245 | // components. 246 | //routing: ['componentA', 'componentB'], 247 | // 248 | // If a function, then the destination components and the data sent can 249 | // be specified as desired. 250 | // 251 | // Note that this function will be called inside a Lambda function context 252 | // and so cannot rely on any NPM modules not in that context. Further, 253 | // the require() statements for resources must be inside the routing 254 | // function context, as this config file will be loaded in other contexts 255 | // that do not have those modules available. 256 | // 257 | //routing: function (error, data) { 258 | // // This is safe provided that the Lambda function for this component 259 | // // includes the underscore module. 260 | // var _ = require('underscore'); 261 | // 262 | // // Don't send on any data if an error resulted. 263 | // if (error) { 264 | // return []; 265 | // } 266 | // 267 | // // Otherwise split up or manipulate data and send it to other 268 | // // components as desired. 269 | // return [ 270 | // { name: 'componentA', data: _.keys(data) }, 271 | // { name: 'componentA', data: _.values(data) } 272 | // ]; 273 | //}, 274 | 275 | // Detailing the lambda function to be used by this component. 276 | lambda: { 277 | // If an NPM package, this is the package name or location for use when 278 | // installing: npm install . 279 | // 280 | // Note that you can specify a local absolute path instead of a package 281 | // name. This is the usual methology when constructing an application 282 | // from multiple components: all of the lambda function definitions are 283 | // kept locally. 284 | npmPackage: 'package-name', 285 | // npmPackage: '/absolute/path/to/local/package/dir', 286 | 287 | // The name of the lambda function handle exported in the package code. 288 | // This doesn't have to be the same as the name of the lambda function 289 | // definition. 290 | handler: 'index.handler', 291 | 292 | // Memory size in MB, with a minimum of 128. 293 | memorySize: 128, 294 | 295 | // Timeout for the function in seconds. 296 | timeout: 60, 297 | 298 | // The role used by this function. 299 | role: 's3Read' 300 | } 301 | }, 302 | 303 | // Example of a lambda function that is directly invoked with data from a 304 | // preceding invocation. 305 | // 306 | // Note that there is no concept of concurrency here; this is invoked 307 | // directly by a prior lambda function execution, so operates at the same 308 | // concurrency. 309 | { 310 | // The name of the lambda function definition to be created in 311 | // AWS. 312 | name: 'eventFromInvocationExample', 313 | 314 | // Type of the component. 315 | type: 'eventFromInvocation', 316 | 317 | // Define where the results from this Lambda function are sent. The data 318 | // passed to context.succeed(data) or context.done(null, data) will be 319 | // send on to another component as the event passed to the handle of its 320 | // Lambda function. 321 | // 322 | // In this case, no destination is defined so no other function will be 323 | // invoked as a result of this function completing. 324 | routing: undefined, 325 | 326 | // Detailing the lambda function to be used by this component. 327 | lambda: { 328 | // If an NPM package, this is the package name or location for use when 329 | // installing: npm install . 330 | // 331 | // Note that you can specify a local absolute path instead of a package 332 | // name. This is the usual methology when constructing an application 333 | // from multiple components: all of the lambda function definitions are 334 | // kept locally. 335 | npmPackage: 'package-name', 336 | // npmPackage: '/absolute/path/to/local/package/dir', 337 | 338 | // The lambda function file and function property exported in the 339 | // package code. For this example that is index.js and the exported 340 | // property name 'handle'. 341 | handler: 'index.handler', 342 | 343 | // Memory size in MB, with a minimum of 128. 344 | memorySize: 128, 345 | 346 | // Timeout for the function in seconds. 347 | timeout: 60, 348 | 349 | // The role used by this function. 350 | role: 'default' 351 | } 352 | } 353 | ] 354 | 355 | }; 356 | -------------------------------------------------------------------------------- /examples/simple/README.md: -------------------------------------------------------------------------------- 1 | # Simple Lambda Complex Example 2 | 3 | This is a very simple example Lambda Complex application that reads input data 4 | from a queue, transforms it, and takes action on it at a set concurrency. The 5 | components created on deployment are: 6 | 7 | * `simple--MessageTransformerQueue` SQS queue. 8 | * `simple--MessageProcessorQueue` SQS queue. 9 | * `simple--MessageTransformerConcurrencyQueue` SQS queue. 10 | * `simple--MessageProcessorConcurrencyQueue` SQS queue. 11 | * `simple--LambdaComplexCoordinatorConcurrencyQueue` SQS queue. 12 | * `simple--LambdaComplexInvokerConcurrencyQueue` SQS queue. 13 | * `simple--MessageTransformer-` Lambda function. 14 | * `simple--MessageProcessor-` Lambda function. 15 | * `simple--InvokedProcessor-` Lambda function. 16 | * `simple--LambdaComplexCoordinator-` Lambda function. 17 | * `simple--LambdaComplexInvoker-` Lambda function. 18 | 19 | ## How the Application Functions 20 | 21 | The `simple--LambdaComplexCoordinator-` Lambda function 22 | constantly invokes itself to monitors queues and invoke other Lambda functions 23 | in response to the presence of queue messages. The entry point for data into the 24 | application is the `simple--MessageTransformerQueue` queue. Messages 25 | added there will cause the coordinator to invoke the 26 | `simple--MessageTransformer-` Lambda function once per 27 | message. 28 | 29 | The `simple--MessageTransformer-` Lambda function consumes 30 | a message from the `simple--MessageTransformerQueue` queue, alters the 31 | message contents, and then passes that data on to either the 32 | `simple--MessageProcessor-` function or the 33 | `simple--InvokedProcessor-` Lambda function depending on 34 | the length of the contents. 35 | 36 | If sending the data to the `simple--MessageProcessor-` 37 | function it adds a message containing the data to the 38 | `simple--MessageProcessorQueue` queue. 39 | 40 | If sending the data to the `simple--InvokedProcessor-` 41 | function, then it invokes that function directly. 42 | 43 | Both of the processor Lambda functions use the same underlying NPM package and 44 | function. The only difference is how they are invoked, either directly or in 45 | response to a queue message. 46 | 47 | The processor function simply logs the message it receives and then exits. 48 | 49 | ## Illustrating the Basics 50 | 51 | Obviously this is trivial and contrived, but serves to show off the basics: 52 | 53 | * Lambda Complex application configuration and deployment. 54 | * Lambda functions reading from queues. 55 | * Lambda function `A` passing data to Lambda function `B` via a queue. 56 | * Lambda function `A` directly invoking Lambda function `B` to pass data. 57 | 58 | A real Lambda Complex application will use these same building blocks, but to a 59 | more constructive end. 60 | 61 | ## Configuring the Application 62 | 63 | Update `examples/simple/applicationConfig.js` to set the following properties to 64 | match the AWS account to deploy into: 65 | 66 | * `region` 67 | * `s3Bucket` 68 | * `s3KeyPrefix` 69 | 70 | ## Launching the Application 71 | 72 | After configuring `examples/simple/applicationConfig.js`, run the following: 73 | 74 | ``` 75 | grunt lambda-complex-deploy --config-path=examples/simple/applicationConfig.js 76 | ``` 77 | 78 | ## Setting the Application in Motion 79 | 80 | Once deployment is complete, add messages containing any valid JSON string to 81 | the `simple--MessageTransformer-` queue to see the Lambda 82 | functions triggered in response. 83 | -------------------------------------------------------------------------------- /examples/simple/applicationConfig.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Configuration for a simple Lambda Complex example application. 3 | * 4 | * See examples/exampleApplicationConfig.js for comments explaining the various 5 | * options and parameters. 6 | */ 7 | 8 | var path = require('path'); 9 | 10 | module.exports = { 11 | name: 'simple', 12 | version: '0.1.0', 13 | // Use a unix timestamp for the deploy ID for the sake of simplicity. 14 | deployId: Math.round((new Date()).getTime() / 1000), 15 | 16 | deployment: { 17 | // Set the desired region. 18 | region: 'us-east-1', 19 | s3Bucket: 'lambda-complex', 20 | s3KeyPrefix: 'applications/', 21 | // No additional tags. 22 | tags: {}, 23 | // No additional duties for the switchover between versions of the 24 | // deployed application. 25 | switchoverFunction: function (stackDescription, config, callback) { 26 | callback(); 27 | }, 28 | skipPriorCloudFormationStackDeletion: false, 29 | skipPriorCloudWatchLogGroupsDeletion: false, 30 | skipCloudFormationStackDeletionOnFailure: false 31 | }, 32 | 33 | coordinator: { 34 | coordinatorConcurrency: 2, 35 | maxApiConcurrency: 10, 36 | maxInvocationCount: 20, 37 | minInterval: 10 38 | }, 39 | 40 | roles: [ 41 | { 42 | name: 'default', 43 | // No extra statements beyond those added by default to access queues. 44 | statements: [] 45 | } 46 | ], 47 | 48 | components: [ 49 | { 50 | name: 'invokedProcessor', 51 | type: 'eventFromInvocation', 52 | // Since this defines no routing, this is a dead end: events are delivered 53 | // here and no further processing results. 54 | // routing: undefined, 55 | lambda: { 56 | npmPackage: path.join(__dirname, 'processor'), 57 | handler: 'index.handler', 58 | memorySize: 128, 59 | timeout: 60, 60 | role: 'default' 61 | } 62 | }, 63 | 64 | { 65 | name: 'messageProcessor', 66 | type: 'eventFromMessage', 67 | queueWaitTime: 5, 68 | maxConcurrency: 10, 69 | // Since this defines no routing, this is a dead end: events are delivered 70 | // here and no further processing results. 71 | // routing: undefined, 72 | lambda: { 73 | npmPackage: path.join(__dirname, 'processor'), 74 | handler: 'index.handler', 75 | memorySize: 128, 76 | timeout: 60, 77 | role: 'default' 78 | } 79 | }, 80 | 81 | { 82 | name: 'messageTransformer', 83 | type: 'eventFromMessage', 84 | maxConcurrency: 10, 85 | queueWaitTime: 5, 86 | /** 87 | * A routing function to send data resulting from this component's Lambda 88 | * function invocation to other components based on its contents. 89 | * 90 | * Remember that (a) the routing function cannot include any reference to 91 | * resources that don't exist in this component Lambda function, and 92 | * (b) this config will be included and loaded in other places that don't 93 | * have the same set of NPM modules available as this Lambda function. 94 | * 95 | * That doesn't matter in this case, but it would if we used require() to 96 | * load modules. 97 | * 98 | * @param {Error} error An Error instance. 99 | * @param {Mixed} data Results of the Lambda function invocation. 100 | */ 101 | routing: function (error, data) { 102 | // Don't send on any data to other components if there was an error in 103 | // processing. 104 | if (error) { 105 | return []; 106 | } 107 | 108 | var json = JSON.stringify(data); 109 | var destination; 110 | 111 | if (json.length % 2 === 1) { 112 | destination = 'messageProcessor'; 113 | } 114 | else { 115 | destination = 'invokedProcessor'; 116 | } 117 | 118 | return [ 119 | { 120 | name: destination, 121 | data: data 122 | } 123 | ]; 124 | }, 125 | lambda: { 126 | npmPackage: path.join(__dirname, 'messageTransformer'), 127 | handler: 'index.handler', 128 | memorySize: 128, 129 | timeout: 60, 130 | role: 'default' 131 | } 132 | } 133 | ] 134 | }; 135 | -------------------------------------------------------------------------------- /examples/simple/messageTransformer/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview The messageTransformer Lambda function. 3 | */ 4 | 5 | /** 6 | * Send on the event data with a little decoration. 7 | * 8 | * @param {Object} event 9 | * @param {Object} context 10 | */ 11 | exports.handler = function (event, context) { 12 | event = event || {}; 13 | event.param = 'transformed'; 14 | 15 | context.succeed(event); 16 | }; 17 | -------------------------------------------------------------------------------- /examples/simple/messageTransformer/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "message-transformer", 3 | "description": "Part of the lambda-complex simple example.", 4 | "private": true, 5 | "version": "0.1.0", 6 | "author": "Reason ", 7 | "engines": { 8 | "node": ">= 0.10" 9 | }, 10 | "dependencies": {} 11 | } 12 | -------------------------------------------------------------------------------- /examples/simple/processor/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview A processor Lambda function definition. 3 | */ 4 | 5 | /** 6 | * Process the provided event. 7 | * 8 | * @param {Object} event 9 | * @param {Object} context 10 | */ 11 | exports.handler = function (event, context) { 12 | // Processing is this case is nothing more exciting than logging the event 13 | // JSON. 14 | console.info(JSON.stringify(event, null, ' ')); 15 | context.succeed(); 16 | }; 17 | -------------------------------------------------------------------------------- /examples/simple/processor/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "processor", 3 | "description": "Part of the lambda-complex simple example.", 4 | "private": true, 5 | "version": "0.1.0", 6 | "author": "Reason ", 7 | "engines": { 8 | "node": ">= 0.10" 9 | }, 10 | "dependencies": {} 11 | } 12 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Expose a programmatic interface for using Lambda Complex. 3 | */ 4 | 5 | // Core. 6 | var util = require('util'); 7 | 8 | // NPM. 9 | var async = require('async'); 10 | var Janitor = require('cloudwatch-logs-janitor'); 11 | var fs = require('fs-extra'); 12 | 13 | // Local. 14 | var common = require('./lib/build/common'); 15 | var configValidator = require('./lib/build/configValidator'); 16 | var installUtilities = require('./lib/build/installUtilities'); 17 | var packageUtilities = require('./lib/build/packageUtilities'); 18 | var cloudFormationTemplateUtilities = require('./lib/build/cloudFormationTemplateUtilities'); 19 | var cloudFormationUtilities = require('./lib/deploy/cloudFormationUtilities'); 20 | var s3Utilities = require('./lib/deploy/s3Utilities'); 21 | 22 | /** 23 | * Build a Lambda Complex application. 24 | * 25 | * @param {Object} config The application configuration. 26 | * @param {Function} callback Of the form function (error). 27 | */ 28 | exports.build = function (config, callback) { 29 | // Validate the configuration first of all. 30 | var results = configValidator.validate(config); 31 | if (results.length) { 32 | return callback(new Error(util.format( 33 | 'Invalid configuration: %s', 34 | JSON.stringify(results, null, ' ') 35 | ))); 36 | } 37 | 38 | async.series({ 39 | // Clear out the build directory for this application. 40 | clean: function (asyncCallback) { 41 | fs.remove(common.getApplicationBuildDirectory(config), asyncCallback); 42 | }, 43 | 44 | // Download or copy the NPM packages containing Lambda function handlers. 45 | install: function (asyncCallback) { 46 | installUtilities.installLambdaFunctions(config, asyncCallback); 47 | }, 48 | 49 | // Zip up the NPM packages. 50 | package: function (asyncCallback) { 51 | packageUtilities.packageLambdaFunctions(config, asyncCallback); 52 | }, 53 | 54 | // Create the CloudFormation template for this application deployment. 55 | generateCloudFormationTemplate: function (asyncCallback) { 56 | cloudFormationTemplateUtilities.generateTemplate(config, asyncCallback); 57 | } 58 | }, callback); 59 | }; 60 | 61 | /** 62 | * Build and deploy a Lambda Complex application. 63 | * 64 | * @param {Object} config The application configuration. 65 | * @param {Function} callback Of the form function (error, results). 66 | */ 67 | exports.deploy = function (config, callback) { 68 | var startTime = new Date(); 69 | var results; 70 | 71 | async.series({ 72 | // Run the build task. 73 | build: function (asyncCallback) { 74 | exports.build(config, asyncCallback); 75 | }, 76 | 77 | // Take the packaged Lambda functions and upload them to S3. 78 | uploadLambdaFunctions: function (asyncCallback) { 79 | s3Utilities.uploadLambdaFunctions(config, asyncCallback); 80 | }, 81 | 82 | // Upload the configuration to S3. 83 | // 84 | // The application doesn't use this config file instance, rather loading 85 | // config files from the packages, but keeping a record of it in S3 86 | // alongside the other files for each deployment seems like a smart idea. 87 | uploadConfig: function (asyncCallback) { 88 | s3Utilities.uploadConfig(config, asyncCallback); 89 | }, 90 | 91 | // Now on to the actual CloudFormation deployment, including the activities 92 | // needed to switch over resources to use the new application, and deletion 93 | // of old stacks. 94 | deployCloudFormationStack: function (asyncCallback) { 95 | cloudFormationUtilities.deployStack(config, function (error, _results) { 96 | results = _results; 97 | asyncCallback(error); 98 | }); 99 | }, 100 | 101 | // Every Lambda function generates a CloudWatch log group, and this can 102 | // build up clutter pretty quickly during development. Some people may 103 | // prefer to keep the old groups around for production deployments. 104 | deletePriorCloudwatchLogGroups: function (asyncCallback) { 105 | if (config.deployment.skipPriorCloudWatchLogGroupsDeletion) { 106 | return asyncCallback(); 107 | } 108 | 109 | // This will use AWS config from the environment, like the rest of Lambda 110 | // Complex. 111 | var janitor = new Janitor(); 112 | 113 | janitor.deleteMatchingLogGroups({ 114 | // Using this should be sufficient to avoid deleting log groups for 115 | // the just-deployed Lambda functions. 116 | createdBefore: startTime, 117 | // Lambda function log groups for this application will have names that 118 | // begin with this prefix. 119 | prefix: '/aws/lambda/' + config.name + '-' 120 | }, function (error) { 121 | // Cleaning up old log groups may require a bunch of API requests, and 122 | // these have in the past proven to be somewhat flaky and/or subject to 123 | // very low throttling levels. Failure here isn't critical, as anything 124 | // that is missed will be addressed by the next deployment. So just add 125 | // the error to the results. 126 | if (error) { 127 | results.cloudWatchLogDeletionError = error; 128 | } 129 | 130 | asyncCallback(); 131 | }); 132 | } 133 | }, function (error) { 134 | callback(error, results); 135 | }); 136 | }; 137 | -------------------------------------------------------------------------------- /lib/build/cloudFormationTemplateUtilities.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview CloudFormation template construction utilities. 3 | */ 4 | 5 | // Core. 6 | var util = require('util'); 7 | 8 | // NPM. 9 | var fs = require('fs-extra'); 10 | var _ = require('lodash'); 11 | 12 | // Local. 13 | var common = require('../build/common'); 14 | var constants = require('../shared/constants'); 15 | var utilities = require('../shared/utilities'); 16 | 17 | // --------------------------------------------------------------------------- 18 | // Functions exported for unit test purposes. 19 | // --------------------------------------------------------------------------- 20 | 21 | /** 22 | * Return an array of all role definitions, including internal ones. 23 | * 24 | * @param {Object} config Configuration object. 25 | * @return {Object[]} Component definitions. 26 | */ 27 | exports.getAllRoles = function (config) { 28 | return [ 29 | // The internal role, not specified in config. 30 | { 31 | name: constants.coordinator.ROLE, 32 | // No custom statements. The standard set of default statements added 33 | // in CloudFormation template construction are all that is needed. 34 | statements: [] 35 | } 36 | ].concat(config.roles); 37 | }; 38 | 39 | // --------------------------------------------------------------------------- 40 | // Functions. 41 | // --------------------------------------------------------------------------- 42 | 43 | /** 44 | * Every role is given an additional policy allowing read/write access to the 45 | * queues that are a part of the application. 46 | * 47 | * @param {Object} config The application configuration. 48 | * @return {Object} The statement. 49 | */ 50 | function getQueuesStatement (config) { 51 | return { 52 | Effect: 'Allow', 53 | Action: [ 54 | 'sqs:DeleteMessage', 55 | 'sqs:GetQueueAttributes', 56 | 'sqs:ReceiveMessage', 57 | 'sqs:SendMessage' 58 | ], 59 | Resource: _.map( 60 | common.getEventFromMessageComponents(config), 61 | function (component) { 62 | return { 63 | 'Fn::GetAtt': [ 64 | utilities.getQueueName(component.name), 65 | 'Arn' 66 | ] 67 | }; 68 | } 69 | ) 70 | }; 71 | } 72 | 73 | /** 74 | * Every role is given an additional policy allowing read/write access to the 75 | * concurrency queues that are a part of the application. 76 | * 77 | * @param {Object} config The application configuration. 78 | * @return {Object} The statement. 79 | */ 80 | function getConcurrencyQueuesStatement (config) { 81 | return { 82 | Effect: 'Allow', 83 | Action: [ 84 | 'sqs:DeleteMessage', 85 | 'sqs:GetQueueAttributes', 86 | 'sqs:ReceiveMessage', 87 | 'sqs:SendMessage' 88 | ], 89 | Resource: _.map( 90 | common.getAllComponents(config), 91 | function (component) { 92 | return { 93 | 'Fn::GetAtt': [ 94 | utilities.getConcurrencyQueueName(component.name), 95 | 'Arn' 96 | ] 97 | }; 98 | } 99 | ) 100 | }; 101 | } 102 | 103 | /** 104 | * Every role is given an additional policy allowing permissions relating to the 105 | * Lambda functions that are a part of the application. 106 | * 107 | * @param {Object} config The application configuration. 108 | * @return {Object} The statement. 109 | */ 110 | function getLambdaFunctionsStatement (config) { 111 | return { 112 | Effect: 'Allow', 113 | Action: [ 114 | 'lambda:InvokeFunction' 115 | ], 116 | // We can't use the actual Lambda function ARNs via Fn::GetAtt as that 117 | // would create circular references. So make use of the fact that all of the 118 | // ARNs will be prefixed by the application name, and use wildcards. 119 | Resource: [ 120 | util.format( 121 | // The first wildcard is the account ID, which we also don't know at this 122 | // point. 123 | 'arn:aws:lambda:%s:*:function:%s-*', 124 | config.deployment.region, 125 | config.name 126 | ) 127 | ] 128 | }; 129 | } 130 | 131 | /** 132 | * Every role is given an additional policy allowing Lambda functions to write 133 | * to CloudWatch Logs. 134 | * 135 | * This is necessary for console#log statements to show up in CloudWatch Logs. 136 | */ 137 | function getCloudWatchLogsStatement (config) { 138 | return { 139 | Effect: 'Allow', 140 | Action: [ 141 | 'logs:CreateLogGroup', 142 | 'logs:CreateLogStream', 143 | 'logs:PutLogEvents' 144 | ], 145 | Resource: [ 146 | util.format( 147 | // TODO: look at whether this can be more specific. 148 | 'arn:aws:logs:%s:*:*', 149 | config.deployment.region 150 | ) 151 | ] 152 | }; 153 | } 154 | 155 | /** 156 | * Every role is given additional policies allowing permissions relating to S3 157 | * keys that are a part of the application. 158 | * 159 | * @param {Object} config The application configuration. 160 | * @return {Object} The statement. 161 | */ 162 | function getS3ArnMapStatement (config) { 163 | return { 164 | Effect: 'Allow', 165 | Action: [ 166 | 's3:GetObject' 167 | ], 168 | Resource: [ 169 | util.format( 170 | 'arn:aws:s3:::%s/%s', 171 | config.deployment.s3Bucket, 172 | utilities.getArnMapS3Key(config) 173 | ) 174 | ] 175 | }; 176 | } 177 | 178 | /** 179 | * Every role is given additional policies allowing permissions relating to S3 180 | * keys that are a part of the application. 181 | * 182 | * @param {Object} config The application configuration. 183 | * @return {Object} The statement. 184 | */ 185 | function getS3ApplicationConfirmationStatement (config) { 186 | return { 187 | Effect: 'Allow', 188 | Action: [ 189 | 's3:GetObject', 190 | 's3:PutObject', 191 | 's3:PutObjectAcl' 192 | ], 193 | Resource: [ 194 | util.format( 195 | 'arn:aws:s3:::%s/%s', 196 | config.deployment.s3Bucket, 197 | utilities.getApplicationConfirmationS3Key(config) 198 | ) 199 | ] 200 | }; 201 | } 202 | 203 | /** 204 | * Add to the outputs section of the template. 205 | * 206 | * @param {Object} template The CloudFormation template under construction. 207 | * @param {String} name The logical ID of the output. 208 | * @param {String} description A description. 209 | * @param {Object|String} value Usually a function call to obtain an ID. 210 | */ 211 | function setOutput (template, name, description, value) { 212 | template.Outputs[name] = { 213 | Description: description, 214 | Value: value 215 | }; 216 | } 217 | 218 | /** 219 | * Set the description property for the template. 220 | * 221 | * @param {Object} template The CloudFormation template under construction. 222 | * @param {Object} config The application configuration. 223 | */ 224 | function setDescription (template, config) { 225 | template.Description = util.format( 226 | '%s: %s', 227 | config.name, 228 | config.version 229 | ); 230 | } 231 | 232 | /** 233 | * Add the role resources specified in the configuration to the template. 234 | * 235 | * Each role is associated with one or more Lambda functions and provide 236 | * permissions allowing the Lambda functions to: 237 | * 238 | * - fetch the ARN map from S3. 239 | * - invoke the Lambda functions associated with this application. 240 | * - write to CloudWatch Logs. 241 | * - interact with the SQS queues associated with this application. 242 | * - perform actions associated with custom policy statements provided in the 243 | * application configuration. 244 | * 245 | * @param {Object} template The CloudFormation template under construction. 246 | * @param {Object} config The application configuration. 247 | */ 248 | function setRoles (template, config) { 249 | var roleConfigurations = exports.getAllRoles(config); 250 | 251 | _.each(roleConfigurations, function (roleConfig) { 252 | // First create the skeleton of the role and its policy. 253 | var roleName = utilities.getRoleName(roleConfig.name); 254 | var role = { 255 | Type: 'AWS::IAM::Role', 256 | Properties: { 257 | AssumeRolePolicyDocument: { 258 | Version: '2012-10-17', 259 | Statement: [ 260 | { 261 | Sid: 'AssumeRole', 262 | Effect: 'Allow', 263 | Principal: { 264 | Service: 'lambda.amazonaws.com' 265 | }, 266 | Action: 'sts:AssumeRole' 267 | } 268 | ] 269 | }, 270 | Path: '/', 271 | Policies: [ 272 | { 273 | PolicyName: roleName, 274 | PolicyDocument: { 275 | Version: '2012-10-17', 276 | Statement: [] 277 | } 278 | } 279 | ] 280 | } 281 | }; 282 | 283 | // Add the custom statements provided in the configuration. 284 | var statements = role.Properties.Policies[0].PolicyDocument.Statement = _.map( 285 | roleConfig.statements, 286 | function (statement) { 287 | return { 288 | Effect: statement.effect, 289 | Action: _.clone(statement.action), 290 | Resource: _.clone(statement.resource) 291 | }; 292 | } 293 | ); 294 | 295 | // Add statements for various permissions needed by all of the application 296 | // Lambda functions. 297 | statements.push(getQueuesStatement(config)); 298 | statements.push(getConcurrencyQueuesStatement(config)); 299 | statements.push(getLambdaFunctionsStatement(config)); 300 | statements.push(getS3ArnMapStatement(config)); 301 | statements.push(getS3ApplicationConfirmationStatement(config)); 302 | statements.push(getCloudWatchLogsStatement(config)); 303 | 304 | // Add this role to the template. 305 | template.Resources[roleName] = role; 306 | }); 307 | } 308 | 309 | /** 310 | * Add the Lambda function resources to the template. 311 | * 312 | * @param {Object} template The CloudFormation template under construction. 313 | * @param {Object} config The application configuration. 314 | */ 315 | function setLambdaFunctions (template, config) { 316 | var components = common.getAllComponents(config); 317 | 318 | _.each(components, function (component) { 319 | var lambdaFunctionName = utilities.getLambdaFunctionName(component.name); 320 | var lambda = { 321 | Type: 'AWS::Lambda::Function', 322 | Properties: { 323 | Code: { 324 | S3Bucket: config.deployment.s3Bucket, 325 | S3Key: common.getComponentS3Key(component, config) 326 | // Not used here. 327 | //S3ObjectVersion: '' 328 | }, 329 | Description: lambdaFunctionName, 330 | Handler: component.lambda.handler, 331 | MemorySize: component.lambda.memorySize || constants.lambda.MIN_MEMORY_SIZE, 332 | // Reference the role in this template for this Lambda function. 333 | Role: { 334 | 'Fn::GetAtt': [ 335 | utilities.getRoleName(component.lambda.role), 336 | 'Arn' 337 | ] 338 | }, 339 | Runtime: 'nodejs', 340 | Timeout: component.lambda.timeout || constants.lambda.MIN_TIMEOUT 341 | } 342 | }; 343 | 344 | // Add Lambda function to template. 345 | template.Resources[lambdaFunctionName] = lambda; 346 | 347 | // Add a related output for the Lambda function ARN, as we'll need it to 348 | // set up the ARN map after deployment, but before starting up the 349 | // application. 350 | setOutput( 351 | template, 352 | utilities.getLambdaFunctionArnOutputName(component.name, config), 353 | lambdaFunctionName + ' ARN.', 354 | { 355 | 'Fn::GetAtt': [ 356 | lambdaFunctionName, 357 | 'Arn' 358 | ] 359 | } 360 | ); 361 | }); 362 | } 363 | 364 | /** 365 | * Add the SQS queue resources for message components to the template. 366 | * 367 | * These are the queues used to deliver data to message component Lambda 368 | * functions. 369 | * 370 | * @param {Object} template The CloudFormation template under construction. 371 | * @param {Object} config The application configuration. 372 | */ 373 | function setMessageComponentQueues (template, config) { 374 | var components = common.getEventFromMessageComponents(config); 375 | 376 | _.each(components, function (component) { 377 | var queueName = utilities.getQueueName(component.name); 378 | var queue = { 379 | Type: 'AWS::SQS::Queue', 380 | Properties: { 381 | QueueName: utilities.getFullQueueName(component.name, config), 382 | // This will be set to the same value as the timeout for the associated 383 | // component Lambda function. 384 | VisibilityTimeout: component.lambda.timeout || constants.lambda.MAX_TIMEOUT 385 | // Default values, not set. 386 | //DelaySeconds: 0, 387 | //MaximumMessageSize: 262144, 388 | //MessageRetentionPeriod: 345600, 389 | //ReceiveMessageWaitTimeSeconds: 0, 390 | // Not used. 391 | //RedrivePolicy: { 392 | // deadLetterTargetArn: '', 393 | // maxReceiveCount: 1 394 | //} 395 | } 396 | }; 397 | 398 | // Add queue to the template. 399 | template.Resources[queueName] = queue; 400 | 401 | // Add a related output to obtain the queue ARN, as we'll need it to set up 402 | // the ARN map after deployment, but before starting up the application. 403 | setOutput( 404 | template, 405 | utilities.getQueueArnOutputName(component.name), 406 | queueName + ' ARN.', 407 | { 408 | 'Fn::GetAtt': [ 409 | queueName, 410 | 'Arn' 411 | ] 412 | } 413 | ); 414 | }); 415 | } 416 | 417 | /** 418 | * Add the SQS queue resources for tracking concurrency to the template. 419 | * 420 | * Every component has an associated concurrency queue. 421 | * 422 | * @param {Object} template The CloudFormation template under construction. 423 | * @param {Object} config The application configuration. 424 | */ 425 | function setConcurrencyQueues (template, config) { 426 | var components = common.getAllComponents(config); 427 | 428 | _.each(components, function (component) { 429 | // Message retention period has a minimum of 60 seconds, unfortunately. 430 | // 431 | // Otherwise this should be set to the same value as the timeout for the 432 | // associated component Lambda function. If a component fails to remove a 433 | // concurrency-tracking message, that message should evaporate at the 434 | // timeout. 435 | var messageRetentionPeriod = Math.max( 436 | 60, 437 | component.lambda.timeout || constants.lambda.MAX_TIMEOUT 438 | ); 439 | 440 | var queueName = utilities.getConcurrencyQueueName(component.name); 441 | var queue = { 442 | Type: 'AWS::SQS::Queue', 443 | Properties: { 444 | QueueName: utilities.getFullConcurrencyQueueName(component.name, config), 445 | // This will be set to the same value as the timeout for the associated 446 | // component Lambda function. 447 | VisibilityTimeout: component.lambda.timeout || constants.lambda.MAX_TIMEOUT, 448 | MessageRetentionPeriod: messageRetentionPeriod 449 | // Default values, not set. 450 | //DelaySeconds: 0, 451 | //MaximumMessageSize: 262144, 452 | //ReceiveMessageWaitTimeSeconds: 0, 453 | // Not used. 454 | //RedrivePolicy: { 455 | // deadLetterTargetArn: '', 456 | // maxReceiveCount: 1 457 | //} 458 | } 459 | }; 460 | 461 | // Add queue to the template. 462 | template.Resources[queueName] = queue; 463 | 464 | // Add a related output to obtain the queue ARN, as we'll need it to set up 465 | // the ARN map after deployment, but before starting up the application. 466 | setOutput( 467 | template, 468 | utilities.getConcurrencyQueueArnOutputName(component.name), 469 | queueName + ' ARN.', 470 | { 471 | 'Fn::GetAtt': [ 472 | queueName, 473 | 'Arn' 474 | ] 475 | } 476 | ); 477 | }); 478 | } 479 | 480 | // --------------------------------------------------------------------------- 481 | // Exported functions. 482 | // --------------------------------------------------------------------------- 483 | 484 | /** 485 | * Generate the CloudFormation template for the application. 486 | * 487 | * @param {Object} config Application config. 488 | * @param {Function} callback Of the form function (error). 489 | */ 490 | exports.generateTemplate = function (config, callback) { 491 | var template = { 492 | AWSTemplateFormatVersion: '2010-09-09', 493 | Description: '', 494 | // The generated template won't have any parameters, while resources and 495 | // outputs are filled in by below. 496 | Parameters: {}, 497 | Resources: {}, 498 | Outputs: {} 499 | }; 500 | 501 | setDescription(template, config); 502 | setRoles(template, config); 503 | setLambdaFunctions(template, config); 504 | setMessageComponentQueues(template, config); 505 | setConcurrencyQueues(template, config); 506 | 507 | fs.writeJSON( 508 | common.getCloudFormationTemplatePath(config), 509 | template, 510 | { 511 | spaces: 2 512 | }, 513 | callback 514 | ); 515 | }; 516 | -------------------------------------------------------------------------------- /lib/build/common.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Common utils for the deploy code. 3 | */ 4 | 5 | // Core. 6 | var fs = require('fs'); 7 | var path = require('path'); 8 | var util = require('util'); 9 | 10 | // NPM. 11 | var async = require('async'); 12 | var _ = require('lodash'); 13 | 14 | // Local 15 | var constants = require('../shared/constants'); 16 | var utilities = require('../shared/utilities'); 17 | 18 | // -------------------------------------------------------------------------- 19 | // Directory and listing tools. 20 | // -------------------------------------------------------------------------- 21 | 22 | /** 23 | * Obtain the absolute path to the build subdirectory for an application. 24 | * 25 | * @param {Object} config Configuration object. 26 | * @return {String} The absolute path. 27 | */ 28 | exports.getApplicationBuildDirectory = function (config) { 29 | return path.resolve( 30 | __dirname, 31 | '../../build', 32 | config.name, 33 | // Might be a number, and path.resolve only accepts strings. 34 | '' + config.deployId 35 | ); 36 | }; 37 | 38 | /** 39 | * Obtain the absolute path to the build node_modules directory for an 40 | * application. 41 | * 42 | * @param {Object} config Configuration object. 43 | * @return {String} The absolute path. 44 | */ 45 | exports.getApplicationBuildNodeModulesDirectory = function (config) { 46 | return path.join(exports.getApplicationBuildDirectory(config), 'node_modules'); 47 | }; 48 | 49 | /** 50 | * Obtain an array of absolute paths to the installed Lambda function NPM 51 | * packages for an application, in the node_modules directory, prior to their 52 | * being moved. 53 | * 54 | * @param {Object} config Configuration object. 55 | * @param {Function} callback Of the form function (error, string[]). 56 | */ 57 | exports.getApplicationPackageDirectories = function (config, callback) { 58 | var dir = exports.getApplicationBuildNodeModulesDirectory(config); 59 | 60 | fs.readdir(dir, function (error, subdirs) { 61 | if (error) { 62 | return callback(error); 63 | } 64 | 65 | async.map(subdirs, function (subdir, asyncCallback) { 66 | var absoluteDir = path.join(dir, subdir); 67 | fs.stat(absoluteDir, function (statError, stat) { 68 | if (error) { 69 | return asyncCallback(error); 70 | } 71 | 72 | if (stat.isDirectory()) { 73 | asyncCallback(null, absoluteDir); 74 | } 75 | else { 76 | asyncCallback(); 77 | } 78 | }); 79 | }, function (mapError, absoluteDirs) { 80 | callback( 81 | mapError, 82 | _.chain(absoluteDirs || []).compact().difference([ 83 | // NPM may create a node_modules/.bin directory. Ignore that. 84 | path.join(dir, '.bin') 85 | ]).value() 86 | ); 87 | }); 88 | }); 89 | }; 90 | 91 | // -------------------------------------------------------------------------- 92 | // Path tools. 93 | // -------------------------------------------------------------------------- 94 | 95 | /** 96 | * Obtain the S3 key for a zipped Lambda function NPM module. 97 | * 98 | * @param {Object} component Component definition object. 99 | * @param {Object} config Application configuration object. 100 | * @return {String} The key. 101 | */ 102 | exports.getComponentS3Key = function (component, config) { 103 | return path.join( 104 | utilities.getFullS3KeyPrefix(config), 105 | component.name + '.zip' 106 | ); 107 | }; 108 | 109 | /** 110 | * Obtain the absolute path to the zip file for a Lambda function NPM module. 111 | * 112 | * @param {Object} component Component definition object. 113 | * @param {Object} config Configuration object. 114 | * @return {String} The absolute path. 115 | */ 116 | exports.getComponentZipFilePath = function (component, config) { 117 | return path.join( 118 | exports.getApplicationBuildDirectory(config), 119 | component.name + '.zip' 120 | ); 121 | }; 122 | 123 | /** 124 | * Obtain the absolute path to the CloudFormation template for a given Lambda 125 | * Complex application. 126 | * 127 | * @param {Object} config Configuration object. 128 | * @return {String} The absolute path. 129 | */ 130 | exports.getCloudFormationTemplatePath = function (config) { 131 | return path.join( 132 | exports.getApplicationBuildDirectory(config), 133 | 'cloudFormation.json' 134 | ); 135 | }; 136 | 137 | // -------------------------------------------------------------------------- 138 | // Configuration tools. 139 | // -------------------------------------------------------------------------- 140 | 141 | /** 142 | * It is sometimes helpful to have a component definition for the coordinator. 143 | * 144 | * @return {Object} A component definition. 145 | */ 146 | exports.getCoordinatorComponentDefinition = function () { 147 | var component = _.cloneDeep(constants.coordinator.COMPONENT); 148 | 149 | component.lambda.npmPackage = path.resolve( 150 | __dirname, 151 | '../lambdaFunctions/coordinator' 152 | ); 153 | return component; 154 | }; 155 | 156 | /** 157 | * It is sometimes helpful to have a component definition for the invoker. 158 | * 159 | * @return {Object} A component definition. 160 | */ 161 | exports.getInvokerComponentDefinition = function () { 162 | var component = _.cloneDeep(constants.invoker.COMPONENT); 163 | 164 | component.lambda.npmPackage = path.resolve( 165 | __dirname, 166 | '../lambdaFunctions/coordinator' 167 | ); 168 | return component; 169 | }; 170 | 171 | /** 172 | * Return an array of all components, including internal ones. 173 | * 174 | * @param {Object} config Configuration object. 175 | * @return {Object[]} Component definitions. 176 | */ 177 | exports.getAllComponents = function (config) { 178 | return [ 179 | exports.getCoordinatorComponentDefinition(), 180 | exports.getInvokerComponentDefinition() 181 | ].concat(config.components); 182 | }; 183 | 184 | /** 185 | * Return an array containing the event from message type components only. 186 | * 187 | * @param {Object} config The application configuration. 188 | * @return {Object[]} Only event from message components. 189 | */ 190 | exports.getEventFromMessageComponents = function (config) { 191 | return _.filter(exports.getAllComponents(config), function (component) { 192 | return component.type === constants.componentType.EVENT_FROM_MESSAGE; 193 | }); 194 | } 195 | 196 | /** 197 | * Given a config object generate the contents of the config.js file to be 198 | * included in Lambda function NPM modules. 199 | * 200 | * Since we have to include function definitions this isn't as simple as just 201 | * generating JSON. 202 | * 203 | * @param {Object} config The application configuration. 204 | * @return {String} Contents to be written to a file. 205 | */ 206 | exports.generateConfigContents = function (config) { 207 | var token = '__ROUTING_FN_%s__'; 208 | var quotedToken = '"' + token + '"'; 209 | var routingFns = []; 210 | var switchoverFn; 211 | var switchoverToken = '__SWITCHOVER_FN__'; 212 | var quotedSwitchoverToken = '"' + switchoverToken + '"'; 213 | 214 | // Take a copy to manipulate. 215 | config = _.cloneDeep(config); 216 | 217 | // Replace the switchover function with a token. 218 | if (config.deployment.switchoverFunction) { 219 | switchoverFn = config.deployment.switchoverFunction; 220 | config.deployment.switchoverFunction = switchoverToken; 221 | } 222 | 223 | // Replace all of the routing functions with string tokens. 224 | _.each(config.components, function (component) { 225 | if (typeof component.routing !== 'function') { 226 | return; 227 | } 228 | 229 | var index = routingFns.length; 230 | routingFns[index] = component.routing; 231 | component.routing = util.format(token, index); 232 | }); 233 | 234 | // Generate the content, dropping any remaining functions such as the 235 | // switchoverFunction. 236 | var contents = util.format( 237 | 'module.exports = %s;', 238 | JSON.stringify(config, null, ' ') 239 | ); 240 | 241 | // Now replace the tokens with string representations of the replaced 242 | // routing functions. 243 | if (switchoverFn) { 244 | contents = contents.replace(quotedSwitchoverToken, switchoverFn.toString()); 245 | } 246 | 247 | _.each(routingFns, function (fn, index) { 248 | contents = contents.replace( 249 | util.format(quotedToken, index), 250 | fn.toString() 251 | ); 252 | }); 253 | 254 | return contents; 255 | }; 256 | -------------------------------------------------------------------------------- /lib/build/configValidator.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview A configuration validator. 3 | */ 4 | 5 | // Core. 6 | var util = require('util'); 7 | 8 | // NPM. 9 | var jsonschema = require('jsonschema'); 10 | var jsonschemaHelpers = require('jsonschema/lib/helpers'); 11 | var _ = require('lodash'); 12 | 13 | // Local. 14 | var constants = require('../shared/constants'); 15 | 16 | // -------------------------------------------------------------------------- 17 | // Schema definitions. 18 | // -------------------------------------------------------------------------- 19 | 20 | var configSchema = { 21 | id: '/Config', 22 | type: 'object', 23 | additionalProperties: false, 24 | properties: { 25 | components: { 26 | type: 'array', 27 | items: { 28 | anyOf: [ 29 | { 30 | $ref: '/EventFromMessageComponent' 31 | }, 32 | { 33 | $ref: '/EventFromInvocationComponent' 34 | } 35 | ] 36 | }, 37 | minItems: 1, 38 | noDuplicatePropertyValuesFor: ['name'], 39 | required: true 40 | }, 41 | coordinator: { 42 | $ref: '/Coordinator', 43 | required: true 44 | }, 45 | deployId: { 46 | anyOf: [ 47 | { 48 | type: 'string', 49 | pattern: /[a-z0-9]+/i 50 | }, 51 | { 52 | type: 'number', 53 | minimum: 0 54 | } 55 | ], 56 | required: true 57 | }, 58 | deployment: { 59 | $ref: '/Deployment', 60 | required: true 61 | }, 62 | name: { 63 | type: 'string', 64 | pattern: /[a-z0-9]+/i, 65 | required: true 66 | }, 67 | roles: { 68 | type: 'array', 69 | items: { 70 | $ref: '/Role' 71 | }, 72 | minItems: 1, 73 | noDuplicatePropertyValuesFor: ['name'], 74 | required: true 75 | }, 76 | version: { 77 | type: 'string', 78 | minLength: 1, 79 | required: true 80 | } 81 | }, 82 | required: true 83 | }; 84 | 85 | var coordinatorSchema = { 86 | id: '/Coordinator', 87 | type: 'object', 88 | additionalProperties: false, 89 | properties: { 90 | coordinatorConcurrency: { 91 | type: 'number', 92 | minimum: 1, 93 | required: true 94 | }, 95 | maxApiConcurrency: { 96 | type: 'number', 97 | minimum: 1, 98 | required: true 99 | }, 100 | maxInvocationCount: { 101 | type: 'number', 102 | minimum: 1, 103 | required: true 104 | }, 105 | minInterval: { 106 | type: 'number', 107 | minimum: 0, 108 | maximum: constants.lambda.MAX_TIMEOUT, 109 | required: true 110 | } 111 | } 112 | }; 113 | 114 | var deploymentSchema = { 115 | id: '/Deployment', 116 | type: 'object', 117 | additionalProperties: false, 118 | properties: { 119 | region: { 120 | type: 'string', 121 | pattern: /(ap\-northeast|ap\-southeast|eu\-central|eu\-west|sa\-east|us\-east|us\-west)\-\d/, 122 | required: true 123 | }, 124 | s3Bucket: { 125 | type: 'string', 126 | minLength: 1, 127 | required: true 128 | }, 129 | s3KeyPrefix: { 130 | type: 'string', 131 | minLength: 1, 132 | required: true 133 | }, 134 | skipPriorCloudFormationStackDeletion: { 135 | type: 'boolean', 136 | required: false 137 | }, 138 | skipPriorCloudWatchLogGroupsDeletion: { 139 | type: 'boolean', 140 | required: false 141 | }, 142 | skipCloudFormationStackDeletionOnFailure: { 143 | type: 'boolean', 144 | required: false 145 | }, 146 | switchoverFunction: { 147 | isFunction: true, 148 | required: false 149 | }, 150 | tags: { 151 | type: 'object', 152 | patternProperties: { 153 | // TODO: a better regexp here would match tag name restrictions in the 154 | // AWS API. 155 | '.*': { 156 | type: 'string', 157 | required: false 158 | } 159 | }, 160 | required: false 161 | } 162 | } 163 | }; 164 | 165 | var roleSchema = { 166 | id: '/Role', 167 | type: 'object', 168 | additionalProperties: false, 169 | properties: { 170 | name: { 171 | type: 'string', 172 | pattern: /[a-z0-9]+/i, 173 | required: true 174 | }, 175 | statements: { 176 | type: 'array', 177 | items: { 178 | $ref: '/Statement' 179 | }, 180 | required: true 181 | } 182 | } 183 | }; 184 | 185 | var statementSchema = { 186 | id: '/Statement', 187 | type: 'object', 188 | additionalProperties: false, 189 | properties: { 190 | effect: { 191 | type: 'string', 192 | enum: [ 193 | 'Allow', 194 | 'Deny' 195 | ], 196 | required: true 197 | }, 198 | // TODO: could probably write a matcher that catches at least some action 199 | // name errors. 200 | action: { 201 | anyOf: [ 202 | { 203 | type: 'string', 204 | minLength: 1 205 | }, 206 | { 207 | type: 'array', 208 | items: { 209 | type: 'string', 210 | minLength: 1 211 | }, 212 | minItems: 1 213 | } 214 | ], 215 | required: true 216 | }, 217 | // TODO: could probably write a matcher that catches at least some resource 218 | // ARN errors. 219 | resource: { 220 | anyOf: [ 221 | { 222 | type: 'string', 223 | minLength: 1 224 | }, 225 | { 226 | type: 'array', 227 | items: { 228 | type: 'string', 229 | minLength: 1 230 | }, 231 | minItems: 1 232 | } 233 | ], 234 | required: true 235 | } 236 | } 237 | }; 238 | 239 | var eventFromMessageComponentSchema = { 240 | id: '/EventFromMessageComponent', 241 | type: 'object', 242 | additionalProperties: false, 243 | properties: { 244 | lambda: { 245 | $ref: '/Lambda', 246 | required: true 247 | }, 248 | name: { 249 | type: 'string', 250 | pattern: /[a-z0-9]+/i, 251 | invalidValues: [ 252 | constants.coordinator.NAME, 253 | constants.invoker.NAME 254 | ], 255 | required: true 256 | }, 257 | maxConcurrency: { 258 | type: 'number', 259 | minimum: 1, 260 | required: true 261 | }, 262 | queueWaitTime: { 263 | type: 'number', 264 | minimum: 0, 265 | maximum: constants.lambda.MAX_TIMEOUT, 266 | required: true 267 | }, 268 | routing: { 269 | anyOf: [ 270 | { 271 | type: 'string', 272 | pattern: /[a-z0-9]+/i 273 | }, 274 | { 275 | type: 'array', 276 | items: { 277 | type: 'string', 278 | pattern: /[a-z0-9]+/i 279 | } 280 | }, 281 | { 282 | isFunction: true 283 | } 284 | ], 285 | required: false 286 | }, 287 | type: { 288 | type: 'string', 289 | enum: [ 290 | constants.componentType.EVENT_FROM_MESSAGE 291 | ], 292 | required: true 293 | } 294 | } 295 | }; 296 | 297 | var eventFromInvocationComponentSchema = { 298 | id: '/EventFromInvocationComponent', 299 | type: 'object', 300 | additionalProperties: false, 301 | properties: { 302 | lambda: { 303 | $ref: '/Lambda', 304 | required: true 305 | }, 306 | name: { 307 | type: 'string', 308 | pattern: /[a-z0-9]+/i, 309 | invalidValues: [ 310 | constants.coordinator.NAME, 311 | constants.invoker.NAME 312 | ], 313 | required: true 314 | }, 315 | routing: { 316 | anyOf: [ 317 | { 318 | type: 'string', 319 | pattern: /[a-z0-9]+/i 320 | }, 321 | { 322 | type: 'array', 323 | items: { 324 | type: 'string', 325 | pattern: /[a-z0-9]+/i 326 | } 327 | }, 328 | { 329 | isFunction: true 330 | } 331 | ], 332 | required: false 333 | }, 334 | type: { 335 | type: 'string', 336 | enum: [ 337 | constants.componentType.EVENT_FROM_INVOCATION 338 | ], 339 | required: true 340 | } 341 | } 342 | }; 343 | 344 | var lambdaSchema = { 345 | id: '/Lambda', 346 | type: 'object', 347 | additionalProperties: false, 348 | properties: { 349 | handler: { 350 | type: 'string', 351 | // This is 'index.handler' or similar. Technically I suppose the file name 352 | // could be something like index.file.name.segments.js, so that 353 | // possibility has to be supported as well. 354 | pattern: /^.+\.[^\.]+$/, 355 | // Not allowed to use 'lc' as the handler name, as this will mess up the 356 | // wrapper code. 357 | invalidPattern: /\.lc$/, 358 | required: true 359 | }, 360 | npmPackage: { 361 | type: 'string', 362 | minLength: 1, 363 | required: true 364 | }, 365 | memorySize: { 366 | type: 'number', 367 | minimum: constants.lambda.MIN_MEMORY_SIZE, 368 | maximum: constants.lambda.MAX_MEMORY_SIZE, 369 | required: true 370 | }, 371 | role: { 372 | type: 'string', 373 | pattern: /[a-z0-9]+/i, 374 | required: true 375 | }, 376 | timeout: { 377 | type: 'number', 378 | minimum: constants.lambda.MIN_TIMEOUT, 379 | maximum: constants.lambda.MAX_TIMEOUT, 380 | required: true 381 | } 382 | } 383 | }; 384 | 385 | // -------------------------------------------------------------------------- 386 | // Set up the validator. 387 | // -------------------------------------------------------------------------- 388 | 389 | var validator = new jsonschema.Validator(); 390 | 391 | validator.addSchema( 392 | coordinatorSchema, 393 | '/Coordinator' 394 | ); 395 | validator.addSchema( 396 | deploymentSchema, 397 | '/Deployment' 398 | ); 399 | validator.addSchema( 400 | roleSchema, 401 | '/Role' 402 | ); 403 | validator.addSchema( 404 | statementSchema, 405 | '/Statement' 406 | ); 407 | validator.addSchema( 408 | eventFromMessageComponentSchema, 409 | '/EventFromMessageComponent' 410 | ); 411 | validator.addSchema( 412 | eventFromInvocationComponentSchema, 413 | '/EventFromInvocationComponent' 414 | ); 415 | validator.addSchema( 416 | lambdaSchema, 417 | '/Lambda' 418 | ); 419 | 420 | /** 421 | * Since jsonschema doesn't seem to test function types properly at this point 422 | * in time, hack in an additional test. 423 | */ 424 | validator.attributes.isFunction = function (instance, schema, options, ctx) { 425 | var result = new jsonschema.ValidatorResult(instance, schema, options, ctx); 426 | 427 | if (!_.isBoolean(schema.isFunction)) { 428 | return result; 429 | } 430 | 431 | if (schema.isFunction) { 432 | if ((instance !== undefined) && (typeof instance !== 'function')) { 433 | result.addError('Required to be a function.'); 434 | } 435 | } 436 | else { 437 | if (typeof instance === 'function') { 438 | result.addError('Required to not be a function.'); 439 | } 440 | } 441 | 442 | return result; 443 | }; 444 | 445 | /** 446 | * Validate against a blacklist of invalid values. 447 | */ 448 | validator.attributes.invalidValues = function (instance, schema, options, ctx) { 449 | var result = new jsonschema.ValidatorResult(instance, schema, options, ctx); 450 | 451 | if (!_.isArray(schema.invalidValues) || !schema.invalidValues.length) { 452 | return result; 453 | } 454 | 455 | var isInvalid = _.some(schema.invalidValues, function (invalidValue) { 456 | return jsonschemaHelpers.deepCompareStrict(invalidValue, instance); 457 | }); 458 | 459 | if (isInvalid) { 460 | result.addError(util.format( 461 | 'Value appears in list of invalid values: %s', 462 | JSON.stringify(instance, null, ' ') 463 | )); 464 | } 465 | 466 | return result; 467 | }; 468 | 469 | /** 470 | * Validate against a blacklist regexp. 471 | */ 472 | validator.attributes.invalidPattern = function (instance, schema, options, ctx) { 473 | var result = new jsonschema.ValidatorResult(instance, schema, options, ctx); 474 | 475 | if (!_.isString(instance) || !_.isRegExp(schema.invalidPattern)) { 476 | return result; 477 | } 478 | 479 | if (instance.match(schema.invalidPattern)) { 480 | result.addError(util.format( 481 | 'Value matches invalid pattern: %s', 482 | instance 483 | )); 484 | } 485 | 486 | return result; 487 | }; 488 | 489 | /** 490 | * Check an array of objects for duplicate values for specified property names. 491 | */ 492 | validator.attributes.noDuplicatePropertyValuesFor = function (instance, schema, options, ctx) { 493 | var result = new jsonschema.ValidatorResult(instance, schema, options, ctx); 494 | 495 | if (_.isString(schema.noDuplicatePropertyValuesFor)) { 496 | schema.noDuplicatePropertyValuesFor = [schema.noDuplicatePropertyValuesFor]; 497 | } 498 | 499 | if (!_.isArray(schema.noDuplicatePropertyValuesFor) || !_.isArray(instance)) { 500 | return result; 501 | } 502 | 503 | _.each(schema.noDuplicatePropertyValuesFor, function (prop) { 504 | var duplicates = _.chain(instance).countBy(function (component) { 505 | return component[prop]; 506 | }).map(function (count, value) { 507 | if (count > 1) { 508 | return value; 509 | } 510 | }).compact().value(); 511 | 512 | if (duplicates.length) { 513 | result.addError(util.format( 514 | 'Duplicate values for property %s: %s', 515 | prop, 516 | duplicates.join(', ') 517 | )); 518 | } 519 | }); 520 | 521 | return result; 522 | }; 523 | 524 | // -------------------------------------------------------------------------- 525 | // Validation checks that do not use jsonschema. 526 | // -------------------------------------------------------------------------- 527 | 528 | // Some of the checks have to cross-reference between different parts of the 529 | // configuration, which jsonschema isn't good at. 530 | 531 | /** 532 | * Check to see that the role names speficied in components are correct. 533 | * 534 | * Append errors to the provided array. 535 | * 536 | * @param {Object} config A configuration object. 537 | * @param {Error[]} An array of errors. 538 | */ 539 | function validateComponentRoleNames (config, errors) { 540 | var roleNames = _.map(config.roles, function (role) { 541 | return role.name; 542 | }); 543 | 544 | var invalidComponentRoles = _.chain( 545 | config.components 546 | ).map(function (component) { 547 | return component.lambda.role; 548 | }).filter(function (name) { 549 | return !_.contains(roleNames, name); 550 | }).value(); 551 | 552 | if (invalidComponentRoles.length) { 553 | errors.push(new Error(util.format( 554 | 'One or more invalid role names specified in components: %s', 555 | invalidComponentRoles.join(', ') 556 | ))); 557 | } 558 | } 559 | 560 | /** 561 | * Check to see that string routing destinations are valid component names. 562 | * 563 | * Append errors to the provided array. 564 | * 565 | * @param {Object} config A configuration object. 566 | * @param {Error[]} An array of errors. 567 | */ 568 | function validateRoutingComponentNames (config, errors) { 569 | var componentNames = _.map(config.components, function (component) { 570 | return component.name; 571 | }); 572 | 573 | var invalidComponentNames = []; 574 | 575 | var routings = _.chain( 576 | config.components 577 | ).map(function (component) { 578 | return component.routing; 579 | }).filter(function (routing) { 580 | return _.isString(routing) || _.isArray(routing); 581 | }).value(); 582 | 583 | _.each(routings, function (routing) { 584 | if (!_.isArray(routing)) { 585 | routing = [routing]; 586 | } 587 | 588 | invalidComponentNames = invalidComponentNames.concat(_.difference( 589 | routing, 590 | componentNames 591 | )); 592 | }); 593 | 594 | if (invalidComponentNames.length) { 595 | errors.push(new Error(util.format( 596 | 'One or more invalid component names specified in routing: %s', 597 | invalidComponentNames.join(', ') 598 | ))); 599 | } 600 | } 601 | 602 | // -------------------------------------------------------------------------- 603 | // Exported functions. 604 | // -------------------------------------------------------------------------- 605 | 606 | /** 607 | * Validate the provided configuration. 608 | * 609 | * @param {Object} config A configuration object. 610 | * @return {Error[]} An array of errors. 611 | */ 612 | exports.validate = function (config) { 613 | var result = validator.validate(config, configSchema) || {}; 614 | var errors = result.errors || []; 615 | 616 | // Non-jsonschema checks. Not worth getting into these if there are already 617 | // errors, as they presuppose that the structure of the JSON is correct. 618 | if (!errors.length) { 619 | validateComponentRoleNames(config, errors); 620 | validateRoutingComponentNames(config, errors); 621 | } 622 | 623 | return errors; 624 | }; 625 | -------------------------------------------------------------------------------- /lib/build/installUtilities.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview For the initial install of Lambda function NPM packages. 3 | * 4 | * This handles: 5 | * 6 | * - NPM installation. 7 | * - Copying modules to sensible places. 8 | * - Wrapping the handler function in Lambda Complex code. 9 | */ 10 | 11 | // Core. 12 | var childProcess = require('child_process'); 13 | var path = require('path'); 14 | var util = require('util'); 15 | 16 | // NPM. 17 | var async = require('async'); 18 | var fs = require('fs-extra'); 19 | var handlebars = require('handlebars'); 20 | var _ = require('lodash'); 21 | 22 | // Local. 23 | var common = require('./common'); 24 | var constants = require('../shared/constants'); 25 | var utilities = require('../shared/utilities'); 26 | 27 | // --------------------------------------------------------------------------- 28 | // Functions exported only for testability. 29 | // --------------------------------------------------------------------------- 30 | 31 | /** 32 | * For the Lambda function package in the provided directory, identify the file 33 | * exporting the handle function, then: 34 | * 35 | * - move filename.js to _filename.js 36 | * - create a new filename.js with a wrapper handler function. 37 | * 38 | * @param {Object} component Component definition. 39 | * @param {Object} config The application configuration. 40 | * @param {Function} wrapperTemplate Template function for the wrapper script. 41 | * @param {Function} callback Of the form function (error). 42 | */ 43 | exports.wrapHandlerForInstalledLambdaFunction = function ( 44 | component, 45 | config, 46 | wrapperTemplate, 47 | callback 48 | ) { 49 | // Figure out the file to wrap from the handle. 50 | var fileBaseName = utilities.getFileBaseNameFromHandle( 51 | component.lambda.handler 52 | ); 53 | var packageDir = path.join( 54 | common.getApplicationBuildDirectory(config), 55 | component.name 56 | ); 57 | var sourcePath = path.join(packageDir, fileBaseName + '.js'); 58 | var destinationPath = path.join(packageDir, '_' + fileBaseName + '.js'); 59 | 60 | async.series({ 61 | move: function (asyncCallback) { 62 | fs.move(sourcePath, destinationPath, asyncCallback); 63 | }, 64 | write: function (asyncCallback) { 65 | var contents = wrapperTemplate({ 66 | componentName: component.name 67 | }); 68 | 69 | fs.outputFile(sourcePath, contents, { 70 | encoding: 'utf-8' 71 | }, asyncCallback); 72 | } 73 | }, callback); 74 | }; 75 | 76 | /** 77 | * Obtain the handlebars template function for the wrapper to replace the 78 | * handle file. 79 | * 80 | * @param {Function} callback Of the form function (error, templateFn). 81 | */ 82 | exports.getWrapperTemplate = function (callback) { 83 | var templateFilePath = path.join(__dirname, 'template/index.js.hbs'); 84 | fs.readFile(templateFilePath, { 85 | encoding: 'utf8' 86 | }, function (error, contents) { 87 | if (error) { 88 | return callback(error); 89 | } 90 | 91 | callback(null, handlebars.compile(contents)); 92 | }); 93 | }; 94 | 95 | /** 96 | * Run npm install for a Lambda function package to install it to the 97 | * application build directory. 98 | * 99 | * @param {Object} config The application configuration. 100 | * @param {String} npmPackage A package name or a path to a package. 101 | * @param {Function} callback Of the form function (error). 102 | */ 103 | exports.npmInstallLambdaFunction = function (config, npmPackage, callback) { 104 | var buildDir = common.getApplicationBuildDirectory(config); 105 | var command = util.format( 106 | // We don't want a package-lock.json or package.json to result from this, 107 | // so --no-package-lock --no-save. 108 | // 109 | // --prefix is needed to get later NPM versions to ignore the package.json 110 | // in the project root and just focus on this build directory. 111 | 'npm --no-package-lock --no-save --prefix "%s" install "%s"', 112 | buildDir, 113 | npmPackage 114 | ); 115 | 116 | childProcess.exec( 117 | command, 118 | { 119 | // Always a good idea to pass over the whole environment. 120 | env: process.env, 121 | cwd: buildDir, 122 | encoding: 'utf-8' 123 | }, 124 | function (error) { 125 | // NPM randomly creates an etc folder when using --prefix. Get rid of it. 126 | // See: https://github.com/npm/npm/pull/7249 127 | fs.remove(path.join(buildDir, 'etc'), callback); 128 | } 129 | ); 130 | }; 131 | 132 | /** 133 | * Run the necessary steps to install the Lambda function for a component. 134 | * 135 | * @param {Object} component Component definition from configuration. 136 | * @param {Object} config The application configuration. 137 | * @param {Function} wrapperTemplate Handlebars template. 138 | * @param {Function} callback Of the form function (error). 139 | */ 140 | exports.installLambdaFunction = function ( 141 | component, 142 | config, 143 | wrapperTemplate, 144 | callback 145 | ) { 146 | // Configuration file contents. 147 | var configContents = common.generateConfigContents(config); 148 | // We figure out where the NPM installation landed by listing directories 149 | // before and after. It is otherwise hard to say what a string that may be a 150 | // package name or directory will produce via npm install. 151 | var installDirsBefore; 152 | var installDirsAfter; 153 | // Where it is initially installed. 154 | var installDir; 155 | // Where it will be moved to after installation. 156 | var destinationDir; 157 | 158 | async.series({ 159 | listDirsBefore: function (asyncCallback) { 160 | common.getApplicationPackageDirectories(config, function (error, dirs) { 161 | installDirsBefore = dirs; 162 | asyncCallback(error); 163 | }); 164 | }, 165 | npmInstall: function (asyncCallback) { 166 | exports.npmInstallLambdaFunction( 167 | config, 168 | component.lambda.npmPackage, 169 | asyncCallback 170 | ); 171 | }, 172 | listDirsAfter: function (asyncCallback) { 173 | common.getApplicationPackageDirectories(config, function (error, dirs) { 174 | if (error) { 175 | return asyncCallback(error); 176 | } 177 | 178 | installDirsAfter = dirs; 179 | installDir = _.difference(installDirsAfter, installDirsBefore)[0]; 180 | asyncCallback(); 181 | }); 182 | }, 183 | // Since two or more components can use the same NPM package, but will need 184 | // different wrapper templates applied to them, we move the installed 185 | // package to a different directory. 186 | moveInstall: function (asyncCallback) { 187 | destinationDir = path.join( 188 | common.getApplicationBuildDirectory(config), 189 | component.name 190 | ); 191 | 192 | // Did we just install a local package, and so we have a symlink rather 193 | // than a directory? NPM 3 creates symlinks to local packages, unlike 194 | // earlier versions. 195 | fs.lstat(installDir, function (error, stats) { 196 | if (error) { 197 | return asyncCallback(error); 198 | } 199 | 200 | // If it is a symlink, then copy the destination and remove the link. 201 | if (stats.isSymbolicLink()) { 202 | var targetDir; 203 | 204 | async.series({ 205 | readlink: function (innerAsyncCallback) { 206 | fs.readlink(installDir, function (error, target) { 207 | // The target will probably be relative to the installation 208 | // location, as that is how NPM 3 likes to do things. If so, 209 | // make it absolute. 210 | if (path.isAbsolute(target)) { 211 | targetDir = target; 212 | } 213 | else { 214 | targetDir = path.resolve(path.dirname(installDir), target); 215 | } 216 | 217 | innerAsyncCallback(error); 218 | }); 219 | }, 220 | // TODO: if the contents of the target directory include further 221 | // symlinks, they will probably break. 222 | copy: function (innerAsyncCallback) { 223 | fs.copy(targetDir, destinationDir, innerAsyncCallback); 224 | }, 225 | unlink: function (innerAsyncCallback) { 226 | fs.unlink(installDir, innerAsyncCallback); 227 | } 228 | }, asyncCallback); 229 | 230 | } 231 | // Otherwise, just move the directory. It would be nice if move() just 232 | // followed symlinks, but sadly not - it moves them, breaking them. 233 | else { 234 | fs.move(installDir, destinationDir, asyncCallback); 235 | } 236 | }); 237 | }, 238 | // Write a copy of the configuration to _config.js in the installed package. 239 | copyConfig: function (asyncCallback) { 240 | fs.outputFile(path.join(destinationDir, '_config.js'), configContents, { 241 | encoding: 'utf-8' 242 | }, asyncCallback); 243 | }, 244 | // Write a copy of the constants file to _constants.js in the installed 245 | // package. 246 | copySharedConstants: function (asyncCallback) { 247 | fs.copy( 248 | path.join(__dirname, '../shared/constants.js'), 249 | path.join(destinationDir, '_constants.js'), 250 | asyncCallback 251 | ); 252 | }, 253 | // Write a copy of the utilities file to _utilities.js in the installed 254 | // package. 255 | copySharedUtilities: function (asyncCallback) { 256 | fs.copy( 257 | path.join(__dirname, '../shared/utilities.js'), 258 | path.join(destinationDir, '_utilities.js'), 259 | asyncCallback 260 | ); 261 | }, 262 | // Move the Lambda function handle Javascript file and replace it with a 263 | // wrapper. 264 | wrapHandleFile: function (asyncCallback) { 265 | // We don't do the wrapping for internal components such as the 266 | // coordinator. 267 | if (component.type === constants.componentType.INTERNAL) { 268 | return asyncCallback(); 269 | } 270 | 271 | exports.wrapHandlerForInstalledLambdaFunction( 272 | component, 273 | config, 274 | wrapperTemplate, 275 | asyncCallback 276 | ); 277 | } 278 | }, callback); 279 | }; 280 | 281 | // --------------------------------------------------------------------------- 282 | // Exported functions (public interface). 283 | // --------------------------------------------------------------------------- 284 | 285 | /** 286 | * Run npm install for all of the specified lambda function packages. 287 | * 288 | * TODO: this won't work for native packages unless this is run on the right 289 | * version of Amazon Linux as the packages have to be built against the right 290 | * Amazon binaries to work in Lambda. See: 291 | * https://aws.amazon.com/blogs/compute/nodejs-packages-in-lambda/ 292 | * 293 | * @param {Object} config The configuration for this deployment. 294 | * @param {Function} callback Of the form function (error). 295 | */ 296 | exports.installLambdaFunctions = function (config, callback) { 297 | var nodeModulesDir = common.getApplicationBuildNodeModulesDirectory(config); 298 | var components = common.getAllComponents(config); 299 | var wrapperTemplate; 300 | 301 | 302 | async.series({ 303 | ensureDirectory: function (asyncCallback) { 304 | fs.mkdirs(nodeModulesDir, asyncCallback); 305 | }, 306 | loadWrapperTemplate: function (asyncCallback) { 307 | exports.getWrapperTemplate(function (error, template) { 308 | wrapperTemplate = template; 309 | asyncCallback(error); 310 | }); 311 | }, 312 | installLambdaFunctions: function (asyncCallback) { 313 | // This has to run in series because we figure out what was installed and 314 | // where by checking directories before and after. This is the easiest 315 | // way to understand what npm install actually did with the string you 316 | // gave it. 317 | // 318 | // Also NPM can be cranky about running in parallel. 319 | async.eachSeries(components, function (component, innerAsyncCallback) { 320 | exports.installLambdaFunction( 321 | component, 322 | config, 323 | wrapperTemplate, 324 | innerAsyncCallback 325 | ); 326 | }, asyncCallback); 327 | }, 328 | // Get rid of the node_modules directory left over after installations. It 329 | // should be empty. 330 | removeNodeModules: function (asyncCallback) { 331 | fs.rmdir( 332 | common.getApplicationBuildNodeModulesDirectory(config), 333 | asyncCallback 334 | ); 335 | } 336 | }, callback); 337 | }; 338 | -------------------------------------------------------------------------------- /lib/build/packageUtilities.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Packaging Lambda function NPM modules. 3 | * 4 | * This handles archiving Lambda function package directories in preparation for 5 | * upload to S3. 6 | */ 7 | 8 | // Core. 9 | var os = require('os'); 10 | var path = require('path'); 11 | 12 | // NPM. 13 | var archiver = require('archiver'); 14 | var async = require('async'); 15 | var fs = require('fs-extra'); 16 | var _ = require('lodash'); 17 | 18 | // Local. 19 | var common = require('./common'); 20 | 21 | // --------------------------------------------------------------------------- 22 | // Variables. 23 | // --------------------------------------------------------------------------- 24 | 25 | // Assuming the setting of credentials via environment variable, credentials 26 | // file, role, etc. 27 | var cpuCount = os.cpus().length; 28 | 29 | // --------------------------------------------------------------------------- 30 | // Functions exported only for testability. 31 | // --------------------------------------------------------------------------- 32 | 33 | /** 34 | * Bundle an installed Lambda function NPM module into a zip file. 35 | * 36 | * @param {Object} component Component definition. 37 | * @param {Object} config The application config. 38 | * @param {Function} callback Of the form function (error). 39 | */ 40 | exports.packageLambdaFunction = function (component, config, callback) { 41 | var zipper = archiver('zip', {}); 42 | var distDir = common.getApplicationBuildDirectory(config); 43 | var output = fs.createWriteStream(common.getComponentZipFilePath( 44 | component, 45 | config 46 | )); 47 | 48 | callback = _.once(callback); 49 | zipper.pipe(output); 50 | 51 | zipper.on('error', callback); 52 | output.on('close', callback); 53 | 54 | zipper.bulk([ 55 | { 56 | cwd: path.join(distDir, component.name), 57 | // Make the glob matcher see dotfiles. 58 | dot: true, 59 | expand: true, 60 | // Archive everything in the installed NPM module. 61 | src: ['**'] 62 | } 63 | ]).finalize(); 64 | }; 65 | 66 | // --------------------------------------------------------------------------- 67 | // Exported functions. 68 | // --------------------------------------------------------------------------- 69 | 70 | /** 71 | * Zip up the installed Lambda function NPM modules. 72 | * 73 | * @param {Object} config The application config. 74 | * @param {Function} callback Of the form function (error). 75 | */ 76 | exports.packageLambdaFunctions = function (config, callback) { 77 | var components = common.getAllComponents(config); 78 | 79 | // Concurrently package modules. 80 | var queue = async.queue(function (component, asyncCallback) { 81 | exports.packageLambdaFunction( 82 | component, 83 | config, 84 | asyncCallback 85 | ); 86 | }, cpuCount); 87 | 88 | queue.drain = _.once(callback); 89 | 90 | function onTaskCompletion (error) { 91 | if (error) { 92 | queue.kill(); 93 | callback(error); 94 | } 95 | } 96 | 97 | _.each(components, function (component) { 98 | queue.push(component, onTaskCompletion); 99 | }); 100 | }; 101 | -------------------------------------------------------------------------------- /lib/deploy/cloudFormationUtilities.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview CloudFormation related utilities. 3 | */ 4 | 5 | // NPM. 6 | var async = require('async'); 7 | var cloudFormationDeploy = require('cloudformation-deploy'); 8 | var fs = require('fs-extra'); 9 | var _ = require('lodash'); 10 | 11 | // Local. 12 | var common = require('../build/common'); 13 | var constants = require('../shared/constants'); 14 | var s3Utilities = require('./s3Utilities'); 15 | var utilities = require('../shared/utilities'); 16 | 17 | // --------------------------------------------------------------------------- 18 | // Exported functions. 19 | // --------------------------------------------------------------------------- 20 | 21 | /** 22 | * Obtain a map of components to relevant ARNs (queues and Lambda functions). 23 | * 24 | * This is in essence a more compact representation of the stack description 25 | * outputs property. 26 | * 27 | * @param {Object[]} outputs The outputs from the stack description. 28 | * @param {Function} callback Of the form function (error, arnMap). 29 | */ 30 | exports.arnMapFromOutputs = function (outputs, callback) { 31 | var arnMap = {}; 32 | 33 | if (!_.isArray(outputs) || !outputs.length) { 34 | return callback(new Error( 35 | 'The stack description Outputs is empty, which should not be the case.' 36 | )); 37 | } 38 | 39 | _.each(outputs, function (output) { 40 | arnMap[output.OutputKey] = output.OutputValue; 41 | }); 42 | 43 | callback(null, arnMap); 44 | }; 45 | 46 | /** 47 | * Start the deployed application by invoking coordinator Lambda function 48 | * instances. 49 | * 50 | * @param {Object} arnMap The ARN map. 51 | * @param {Object} config Lambda Complex configuration. 52 | * @param {Function} callback Of the form function (error). 53 | */ 54 | exports.startApplication = function (arnMap, config, callback) { 55 | var coordinatorArn = utilities.getLambdaFunctionArn( 56 | constants.coordinator.NAME, 57 | arnMap 58 | ); 59 | // The coordinator doesn't need any specific event data. 60 | var event = {}; 61 | 62 | // Fire up the number of coordinators specified in the configuration, 63 | // but space them out across the span of coordinator.minInterval. 64 | var timeout = 0; 65 | if (config.coordinator.coordinatorConcurrency > 1) { 66 | timeout = Math.floor( 67 | config.coordinator.minInterval * 1000 / config.coordinator.coordinatorConcurrency 68 | ); 69 | } 70 | 71 | async.timesSeries( 72 | config.coordinator.coordinatorConcurrency, 73 | function (index, asyncCallback) { 74 | utilities.invoke(coordinatorArn, event, function (error) { 75 | setTimeout(function () { 76 | asyncCallback(error); 77 | }, timeout) 78 | }); 79 | }, 80 | callback 81 | ); 82 | }; 83 | 84 | /** 85 | * Wait for the initial coordinator instances to write a success file to S3. 86 | * 87 | * @param {Object} config Lambda Complex configuration. 88 | * @param {Function} callback Of the form function (error, arnMap). 89 | */ 90 | exports.awaitApplicationConfirmation = function (config, callback) { 91 | var confirmationFileExists = false; 92 | var timedOut = false; 93 | var timeout = (config.coordinator.minInterval + 1) * 2 * 1000; 94 | 95 | // Time this out after twice the standard minimum interval for a coordinator 96 | // that has nothing to do. That should be good enough and with margin for 97 | // error. 98 | var timeoutId = setTimeout(function () { 99 | timedOut = true; 100 | }, timeout); 101 | 102 | async.until( 103 | // The test. When it returns true, stop and trigger the callback function. 104 | function () { 105 | return confirmationFileExists || timedOut; 106 | }, 107 | 108 | // Action function. Run this until the test function returns true. 109 | function (asyncCallback) { 110 | // Insert a short pause between requests. 111 | setTimeout(function () { 112 | utilities.applicationConfirmationExists(config, function (error, exists) { 113 | confirmationFileExists = exists; 114 | asyncCallback(error); 115 | }); 116 | }, 2000); 117 | }, 118 | 119 | // Callback function. 120 | function (error) { 121 | clearTimeout(timeoutId); 122 | 123 | if (error) { 124 | callback(error); 125 | } 126 | else if (timedOut) { 127 | callback(new Error( 128 | 'Timed out waiting on application confirmation file to be created.' 129 | )); 130 | } 131 | else { 132 | callback(); 133 | } 134 | } 135 | ); 136 | }; 137 | 138 | /** 139 | * Create a switchover function that performs the extra tasks we need it to 140 | * carry out, such as: 141 | * 142 | * - Upload the ARN map file. 143 | * - Start the new Lambda Complex application by invoking the coordinator. 144 | * - Wait for the signal that the first coordinators worked. 145 | * 146 | * @param {Object} config Lambda Complex configuration. 147 | * @return {Function} The hybrid switchover function. 148 | */ 149 | exports.getSwitchoverFunction = function (config) { 150 | return function (stackDescription, callback) { 151 | var arnMap; 152 | 153 | async.series({ 154 | createArnMap: function (asyncCallback) { 155 | exports.arnMapFromOutputs(stackDescription.Outputs, function (error, map) { 156 | arnMap = map; 157 | asyncCallback(error); 158 | }); 159 | }, 160 | 161 | uploadArnMap: function (asyncCallback) { 162 | s3Utilities.uploadArnMap(arnMap, config, asyncCallback); 163 | }, 164 | 165 | // Start the new application stack running by invoking the coordinator 166 | // Lambda function. 167 | startApplication: function (asyncCallback) { 168 | exports.startApplication(arnMap, config, asyncCallback); 169 | }, 170 | 171 | // The initial coordinators should write a file to S3 if they worked as 172 | // expected and were able to invoke themselves. 173 | awaitApplicationConfirmation: function (asyncCallback) { 174 | exports.awaitApplicationConfirmation(config, asyncCallback); 175 | }, 176 | 177 | // Lastly, if everything else worked then invoke the switchover function 178 | // provided in the configuration and await its completion. 179 | invokeProvidedSwitchoverFunction: function (asyncCallback) { 180 | if (typeof config.deployment.switchoverFunction === 'function') { 181 | config.deployment.switchoverFunction( 182 | stackDescription, 183 | config, 184 | asyncCallback 185 | ); 186 | } 187 | else { 188 | asyncCallback(); 189 | } 190 | } 191 | }, callback); 192 | }; 193 | }; 194 | 195 | /** 196 | * Create the configuration used by the cloudformation-deploy package. 197 | * 198 | * @param {Object} config Lambda Complex configuration. 199 | * @return {Object} CloudFormation Deploy configuration. 200 | */ 201 | exports.generateCloudFormationDeployConfig = function (config) { 202 | var cfdConfig = { 203 | baseName: config.name, 204 | version: config.version, 205 | deployId: config.deployId, 206 | 207 | // Timeout in minutes for the process of stack creation. 208 | createStackTimeoutInMinutes: 10, 209 | 210 | // Specify additional tags to apply to the stack. Might be missing. 211 | tags: config.deployment.tags, 212 | 213 | // Seconds to wait between each check on the progress of stack creation or 214 | // deletion. 215 | progressCheckIntervalInSeconds: 10, 216 | 217 | // A function invoked whenever a CloudFormation event is created during 218 | // stack creation or deletion. We don't use this. 219 | //onEventFn: function (event) {}, 220 | 221 | // An optional function invoked after the CloudFormation stack is 222 | // successfully created but before any prior stack is deleted. This allows 223 | // for a clean switchover of resources to use the new stack. 224 | postCreationFn: exports.getSwitchoverFunction(config), 225 | 226 | // Delete past stack instances for this application on successful 227 | // deployment. 228 | priorInstance: cloudFormationDeploy.priorInstance.DELETE, 229 | // Delete this stack on failure to deploy. 230 | onDeployFailure: cloudFormationDeploy.onDeployFailure.DELETE 231 | }; 232 | 233 | // Are we in fact cleaning up the wreckage of failure and deleting prior 234 | // application instances, as is the default behavior? 235 | if (config.deployment.skipPriorCloudFormationStackDeletion) { 236 | cfdConfig.priorInstance = cloudFormationDeploy.priorInstance.DO_NOTHING; 237 | } 238 | if (config.deployment.skipCloudFormationStackDeletionOnFailure) { 239 | cfdConfig.onDeployFailure = cloudFormationDeploy.onDeployFailure.DO_NOTHING; 240 | } 241 | 242 | return cfdConfig; 243 | }; 244 | 245 | /** 246 | * Deploy the new stack and on success transition away from and delete any 247 | * prior stacks for this application. 248 | * 249 | * @param {Object} config Application configuration. 250 | * @param {Function} callback Of the form function (error, results). 251 | */ 252 | exports.deployStack = function (config, callback) { 253 | var cfdConfig = exports.generateCloudFormationDeployConfig(config); 254 | var template; 255 | var results; 256 | 257 | async.series({ 258 | loadTemplate: function (asyncCallback) { 259 | fs.readFile( 260 | common.getCloudFormationTemplatePath(config), 261 | { 262 | encoding: 'utf8' 263 | }, 264 | function (error, contents) { 265 | template = contents; 266 | asyncCallback(error); 267 | } 268 | ); 269 | }, 270 | deploy: function (asyncCallback) { 271 | cloudFormationDeploy.deploy(cfdConfig, template, function (error, _results) { 272 | results = _results; 273 | asyncCallback(error); 274 | }); 275 | } 276 | }, function (error) { 277 | callback(error, results); 278 | }); 279 | }; 280 | -------------------------------------------------------------------------------- /lib/deploy/s3Utilities.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview S3 related utilities. 3 | */ 4 | 5 | // Core. 6 | var os = require('os'); 7 | 8 | // NPM. 9 | var AWS = require('aws-sdk'); 10 | var async = require('async'); 11 | var fs = require('fs-extra'); 12 | var _ = require('lodash'); 13 | 14 | // Local. 15 | var common = require('../build/common'); 16 | var utilities = require('../shared/utilities'); 17 | 18 | // --------------------------------------------------------------------------- 19 | // Variables. 20 | // --------------------------------------------------------------------------- 21 | 22 | // Assuming the setting of credentials via environment variable, credentials 23 | // file, role, etc. 24 | // 25 | // This is exported for test purposes. 26 | exports.s3Client = new AWS.S3(); 27 | 28 | var cpuCount = os.cpus().length; 29 | 30 | // --------------------------------------------------------------------------- 31 | // Exported functions. 32 | // --------------------------------------------------------------------------- 33 | 34 | /** 35 | * Upload the map of ARNs for the application as JSON. 36 | * 37 | * @param {Object} map The map of ARNs. 38 | * @param {Object} config The application config. 39 | * @param {Function} callback Of the form function (error). 40 | */ 41 | exports.uploadArnMap = function (map, config, callback) { 42 | // TODO: ACL options; what will be needed here for additional customization? 43 | var params = { 44 | Body: JSON.stringify(map), 45 | Bucket: config.deployment.s3Bucket, 46 | // Not strictly necessary, but helpful for human inspection. 47 | ContentType: 'application/json', 48 | Key: utilities.getArnMapS3Key(config) 49 | }; 50 | 51 | // S3 uploads are flaky enough to always need a retry. 52 | async.retry(3, function (asyncCallback) { 53 | exports.s3Client.putObject(params, asyncCallback); 54 | }, callback); 55 | }; 56 | 57 | /** 58 | * Upload the configuration file to S3, alongside the other items relating to 59 | * this deployment. 60 | * 61 | * This will be useful for later tooling and manual reference, but is not used 62 | * by the running Lambda Complex application. 63 | * 64 | * @param {Object} config The application config. 65 | * @param {Function} callback Of the form function (error). 66 | */ 67 | exports.uploadConfig = function (config, callback) { 68 | // TODO: ACL options; what will be needed here for additional customization? 69 | var params = { 70 | Body: common.generateConfigContents(config), 71 | Bucket: config.deployment.s3Bucket, 72 | // Not strictly necessary, but helpful for human inspection. 73 | ContentType: 'application/javascript', 74 | Key: utilities.getConfigS3Key(config) 75 | }; 76 | 77 | // S3 uploads are flaky enough to always need a retry. 78 | async.retry(3, function (asyncCallback) { 79 | exports.s3Client.putObject(params, asyncCallback); 80 | }, callback); 81 | }; 82 | 83 | /** 84 | * Upload a zipped Lambda function NPM module to S3. 85 | * 86 | * The uploaded zip file will later be referenced in a CloudFormation template. 87 | * 88 | * @param {Object} component Component definition. 89 | * @param {Object} config The application config. 90 | * @param {Function} callback Of the form function (error). 91 | */ 92 | exports.uploadLambdaFunction = function (component, config, callback) { 93 | var params; 94 | 95 | // S3 uploads are flaky enough to always need a retry. 96 | async.retry(3, function (asyncCallback) { 97 | // Since we're using a stream, recreate the params each time we retry. 98 | // 99 | // TODO: ACL options; what will be needed here for additional customization? 100 | params = { 101 | Body: fs.createReadStream(common.getComponentZipFilePath( 102 | component, 103 | config 104 | )), 105 | Bucket: config.deployment.s3Bucket, 106 | Key: common.getComponentS3Key(component, config) 107 | }; 108 | 109 | exports.s3Client.putObject(params, asyncCallback); 110 | }, callback); 111 | } 112 | 113 | /** 114 | * Upload the Lambda function zip files to S3. 115 | * 116 | * @param {Object} config The application config. 117 | * @param {Function} callback Of the form function (error). 118 | */ 119 | exports.uploadLambdaFunctions = function (config, callback) { 120 | var components = common.getAllComponents(config); 121 | 122 | // Concurrently package modules. 123 | var queue = async.queue(function (component, asyncCallback) { 124 | exports.uploadLambdaFunction( 125 | component, 126 | config, 127 | asyncCallback 128 | ); 129 | }, cpuCount); 130 | 131 | queue.drain = _.once(callback); 132 | 133 | function onTaskCompletion (error) { 134 | if (error) { 135 | queue.kill(); 136 | callback(error); 137 | } 138 | } 139 | 140 | _.each(components, function (component) { 141 | queue.push(component, onTaskCompletion); 142 | }); 143 | }; 144 | -------------------------------------------------------------------------------- /lib/grunt/common.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Common code for grunt tasks. 3 | */ 4 | 5 | // Core. 6 | var path = require('path'); 7 | var util = require('util'); 8 | 9 | /** 10 | * Return the configuration object based on a path passed in via --config-path, 11 | * or fail the grunt task. 12 | * 13 | * @param {Object} grunt A grunt instance. 14 | * @return {Object} A configuration object. 15 | */ 16 | exports.getConfigurationFromOptionOrFail = function (grunt) { 17 | var configPath = grunt.option('config-path'); 18 | var config; 19 | 20 | // Is this configuration option missing? 21 | if (!configPath) { 22 | return grunt.fail.fatal(new Error( 23 | 'The --config-path option is required, e.g.: --config-path=path/to/applicationConfig.js' 24 | )); 25 | } 26 | 27 | // Convert relative to absolute path. 28 | if (!path.isAbsolute(configPath)) { 29 | configPath = path.resolve(process.cwd(), configPath); 30 | } 31 | 32 | // Load the file. 33 | try { 34 | config = require(configPath); 35 | } 36 | catch (error) { 37 | return grunt.fail.fatal(new Error(util.format( 38 | 'No configuration Javascript file at: %s', 39 | configPath 40 | ), error)); 41 | } 42 | 43 | return config; 44 | }; 45 | -------------------------------------------------------------------------------- /lib/lambdaFunctions/coordinator/README.md: -------------------------------------------------------------------------------- 1 | # Coordinator Lambda Function 2 | 3 | This package provides a default implementation of the Lambda Complex 4 | coordinator Lambda function. It includes both `coordinator` and `invoker` 5 | handles. 6 | 7 | ## Coordinator 8 | 9 | The coordinator carries out the following functions: 10 | 11 | - Assess current queue sizes per component. 12 | - Invoke invokers or other component Lambda functions so as to clear queues at 13 | the desired concurrency. 14 | - Invoke other coordinator instances once the current instance is done to keep 15 | the coordinator concurrency at the desired level. 16 | 17 | ## Invoker 18 | 19 | The invoker exists to amplify the number of invocations a single Lambda function 20 | instance can create in a given time. API requests take time and start to fail if 21 | launched with too great a concurrency in any one Node.js process. 22 | -------------------------------------------------------------------------------- /lib/lambdaFunctions/coordinator/common.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Common functionality. 3 | */ 4 | 5 | // Other NPM. 6 | var async = require('async'); 7 | var _ = require('lodash'); 8 | 9 | // Local. 10 | // 11 | // The coordinator expects that the build process will have placed _config.js in 12 | // the same directory as this file. 13 | var config = require('./_config'); 14 | // Expect lambda-complex application constants to be placed in _constants.js. 15 | var constants = require('./_constants'); 16 | // Expect lambda-complex application utilities to be placed in _utilities.js. 17 | var utilities = require('./_utilities'); 18 | 19 | // --------------------------------------------------------------------------- 20 | // Utility functions. 21 | // --------------------------------------------------------------------------- 22 | 23 | /** 24 | * Wait for as much time is needed to make up the remainder of the interval, 25 | * given how much time has elapsed already since this lambda function was 26 | * invoked. 27 | * 28 | * @param {Number} startTime Timestamp for the invocation start. 29 | * @param {Number} interval Interval to wait in milliseconds. 30 | * @param {Object} context Lambda context instance. 31 | * @param {Function} callback 32 | */ 33 | exports.ensureInterval = function (startTime, interval, context, callback) { 34 | var elapsed = new Date().getTime() - startTime; 35 | var remainingWaitTime = Math.max(0, interval - elapsed); 36 | 37 | // Make sure we don't overshoot the hard time limits. So check how long is 38 | // left and subtract an extra five seconds to allow enough time to wrap up and 39 | // sort out the remaining tasks, like invoking another instance of this lambda 40 | // function. 41 | var remainingLimitTime = Math.max(0, context.getRemainingTimeInMillis() - 5000); 42 | if (remainingWaitTime > remainingLimitTime) { 43 | remainingWaitTime = remainingLimitTime; 44 | } 45 | 46 | setTimeout(callback, remainingWaitTime); 47 | }; 48 | 49 | /** 50 | * Provide an array of all components, including internal ones. 51 | * 52 | * Note that this somewhat replicates functionality in lib/build/common.js, but 53 | * there are differences between the two. 54 | */ 55 | exports.getAllComponents = function () { 56 | return [ 57 | _.cloneDeep(constants.coordinator.COMPONENT), 58 | _.cloneDeep(constants.invoker.COMPONENT) 59 | ].concat(config.components); 60 | }; 61 | 62 | // --------------------------------------------------------------------------- 63 | // Flow control. 64 | // --------------------------------------------------------------------------- 65 | 66 | /** 67 | * Execute the provided async functions concurrently, but no more than the 68 | * provided concurrency limit at any one time. 69 | * 70 | * Failures are logged but will not disrupt ongoing execution. 71 | * 72 | * @param {Function[]} fns Functions to execute, of the form fn(callback). 73 | * @param {Function} callback Of the form callback (error). 74 | */ 75 | exports.executeConcurrently = function (fns, concurrency, callback) { 76 | if (!fns.length) { 77 | return callback(); 78 | } 79 | 80 | var queue = async.queue(function (fn, asyncCallback) { 81 | fn(asyncCallback); 82 | }, concurrency); 83 | 84 | queue.drain = _.once(callback); 85 | 86 | function onTaskCompletion (error) { 87 | if (error) { 88 | console.error(error); 89 | } 90 | } 91 | 92 | _.each(fns, function (fn) { 93 | queue.push(fn, onTaskCompletion); 94 | }); 95 | }; 96 | 97 | // --------------------------------------------------------------------------- 98 | // Invocation count functions. 99 | // --------------------------------------------------------------------------- 100 | 101 | /** 102 | * Produce invocation counts from the application status. This determines 103 | * which event from message type Lambda functions should be invoked and how many 104 | * times each. 105 | * 106 | * [ 107 | * { name: 'componentName', count: 10 }, 108 | * ... 109 | * ] 110 | * 111 | * The invocation counts are limited by the maxConcurrency specified in the 112 | * component definition. 113 | * 114 | * Further when there are multiple coordinators each only does its share of the 115 | * work. For two coordinators, each does half, for example. 116 | * 117 | * @param {Object} status Application status. 118 | * @return {Object[]} The invocation counts. 119 | */ 120 | exports.getInvocationCounts = function (status) { 121 | return _.chain(status.components).filter(function (component) { 122 | return ( 123 | // We only run this for event from message components. 124 | (component.type === constants.componentType.EVENT_FROM_MESSAGE) && 125 | // This is only a number when the check on queue size worked. If it 126 | // didn't work, we choose to do nothing for that component this time. 127 | // The error will have been logged when it occurred. 128 | (typeof component.queuedMessageCount === 'number') && 129 | // This is only a number when the concurrency count check worked. If it 130 | // didn't work, we choose to do nothing for that component this time. 131 | // The error will have been logged when it occurred. 132 | (typeof component.concurrency === 'number') 133 | ); 134 | }).map(function (component) { 135 | var count = Math.min( 136 | // How many messages to act on. 137 | component.queuedMessageCount, 138 | // How much space we have left for concurrent invocations. 139 | Math.max(0, component.maxConcurrency - component.concurrency) 140 | ); 141 | 142 | // Next cut down the count by the coordinator concurrency; each coordinator 143 | // only does its share of the work. For two coordinators, each does half. 144 | // 145 | // If there are fractional counts, then we round up and just do more. This 146 | // is most pronounced at low levels of activity, where every coordinator 147 | // will chase the same message if they are in sequence. 148 | // 149 | // The alternative is to randomize odds, which may lead to a message 150 | // lingering and adds to complexity. 151 | count = Math.ceil(count / config.coordinator.coordinatorConcurrency); 152 | 153 | return { 154 | name: component.name, 155 | count: count 156 | }; 157 | }).value(); 158 | }; 159 | 160 | /** 161 | * Sum the counts in the invocation count objects provided. 162 | * 163 | * [ 164 | * { name: 'componentName', count: 10 }, 165 | * ... 166 | * ] 167 | * 168 | * @param {[type]} invocationCounts The invocation counts. 169 | * @return {Number} The sum. 170 | */ 171 | exports.sumOfInvocationCounts = function (invocationCounts) { 172 | return _.reduce(invocationCounts, function (sum, invocationCount) { 173 | return sum + invocationCount.count; 174 | }, 0); 175 | }; 176 | 177 | /** 178 | * Split up invocation counts into sets. Returns the following format: 179 | * 180 | * { 181 | * // What to execute locally. 182 | * localInvoker: [ 183 | * { name: 'componentName', count: 10 }, 184 | * ... 185 | * ], 186 | * // What to pass on to other invokers, split into groups of a size of 187 | * // maxInvocationCount. 188 | * otherInvoker: [ 189 | * [ 190 | * { name: 'componentName', count: 10 }, 191 | * ... 192 | * ] 193 | * ... 194 | * 195 | * ] 196 | * } 197 | * 198 | * @param {Object[]} invocationCounts 199 | * @return {Object} The desired arrangement of invocations. 200 | */ 201 | exports.splitInvocationCounts = function (invocationCounts) { 202 | var split = { 203 | localInvoker: [], 204 | otherInvoker: [] 205 | }; 206 | var remainingTotalCount = exports.sumOfInvocationCounts(invocationCounts); 207 | var maxInvocationCount = config.coordinator.maxInvocationCount; 208 | 209 | // Cloning to ensure we don't mess anything up while manipulating this. 210 | invocationCounts = _.cloneDeep(invocationCounts); 211 | 212 | // If there are few enough invocations to run in this instance, then things 213 | // are simple - just allot them to the local bucket and return. 214 | if (remainingTotalCount <= maxInvocationCount) { 215 | split.localInvoker = invocationCounts; 216 | return split; 217 | } 218 | 219 | // Otherwise pull out lumps of maxInvocationCount a maximum number of times 220 | // equal to (maxInvocationCount - 1) for sending to another invoker. Do this 221 | // until left with a remainder that is either small enough to run or large 222 | // enough to be sent on. 223 | do { 224 | var pulledCounts = []; 225 | var invocationCount; 226 | var spaceLeftInInvoker = maxInvocationCount; 227 | 228 | while (spaceLeftInInvoker > 0 && invocationCounts.length) { 229 | invocationCount = _.last(invocationCounts); 230 | 231 | // If there's a larger count than the target invoker can handle, 232 | // considering what we've already assigned, then split out 233 | // maxInvocationCount from it, and reduce its count by that much. 234 | if (invocationCount.count > spaceLeftInInvoker) { 235 | invocationCount.count -= spaceLeftInInvoker; 236 | 237 | // Remove from the components array if this empties it. 238 | if (invocationCount.count === 0) { 239 | invocationCounts.pop(); 240 | } 241 | 242 | var copy = _.clone(invocationCount); 243 | copy.count = spaceLeftInInvoker; 244 | pulledCounts.push(copy); 245 | } 246 | // Otherwise pull the whole thing and remove it from the array. 247 | else { 248 | invocationCounts.pop(); 249 | pulledCounts.push(invocationCount); 250 | } 251 | 252 | // Update the remaining space count. 253 | spaceLeftInInvoker = maxInvocationCount - exports.sumOfInvocationCounts( 254 | pulledCounts 255 | ); 256 | } 257 | 258 | // Add the pulled counts to the bucket of those intended to be send on to 259 | // other invokers. 260 | split.otherInvoker.push(pulledCounts); 261 | 262 | // Update the count for the loop check. 263 | remainingTotalCount = exports.sumOfInvocationCounts(invocationCounts); 264 | } while ( 265 | remainingTotalCount > maxInvocationCount && 266 | split.otherInvoker.length < maxInvocationCount - 1 267 | ) 268 | 269 | // Now we're left with whatever is left. Is it small enough to run here? 270 | if (remainingTotalCount <= maxInvocationCount - split.otherInvoker.length) { 271 | split.localInvoker = invocationCounts; 272 | } 273 | // Otherwise send it on to another invoker. 274 | else { 275 | split.otherInvoker.push(invocationCounts); 276 | } 277 | 278 | return split; 279 | }; 280 | 281 | // --------------------------------------------------------------------------- 282 | // Clarifying invocation wrappers. 283 | // --------------------------------------------------------------------------- 284 | 285 | /** 286 | * Launch a Lambda function instance for an "event from message" component. 287 | * 288 | * This needs no data passed to it, as it will pull its data from its queue. 289 | * 290 | * @param {Object} name The component name. 291 | * @param {Object} arnMap The ARN map. 292 | * @param {Function} callback Of the form function (error). 293 | */ 294 | exports.invokeEventFromMessageFunction = function (name, arnMap, callback) { 295 | // Empty payload; no event needs passing to this invocation. 296 | var payload = {}; 297 | utilities.invoke( 298 | utilities.getLambdaFunctionArn(name, arnMap), 299 | payload, 300 | callback 301 | ); 302 | }; 303 | 304 | /** 305 | * Launch an instance of an invoker, and pass a set of invocation counts for 306 | * event from message Lambda functions. The invoker is then responsible for 307 | * invoking them. 308 | * 309 | * The form of the invocation counts is: 310 | * 311 | * [ 312 | * { name: 'component1', count: 10 }, 313 | * ... 314 | * ] 315 | * 316 | * @param {Object[]} invocationCounts The invocation count data. 317 | * @param {Object} arnMap The ARN map. 318 | * @param {Function} callback Of the form function (error). 319 | */ 320 | exports.invokeInvoker = function (invocationCounts, arnMap, callback) { 321 | // Empty payload; no event needs passing to this invocation. 322 | utilities.invoke( 323 | utilities.getLambdaFunctionArn(constants.invoker.NAME, arnMap), 324 | { 325 | components: invocationCounts 326 | }, 327 | callback 328 | ); 329 | }; 330 | 331 | /** 332 | * Invoke invoker and other Lambda functions as needed to process queue 333 | * contents. 334 | * 335 | * The form of the invocation counts is: 336 | * 337 | * [ 338 | * { name: 'component1', count: 10 }, 339 | * ... 340 | * ] 341 | * 342 | * @param {Object[]} invocationCounts Data on which functions are to be invoked. 343 | * @param {Object} arnMap The ARN map. 344 | * @param {Function} callback Of the form function (error). 345 | */ 346 | exports.invokeApplicationLambdaFunctions = function (invocationCounts, arnMap, callback) { 347 | var split = exports.splitInvocationCounts(invocationCounts); 348 | var fns = []; 349 | 350 | // Local direct invocations of application Lambda functions, many times over. 351 | _.each(split.localInvoker, function (invocationCount) { 352 | _.times(invocationCount.count, function () { 353 | fns.push(function (asyncCallback) { 354 | exports.invokeEventFromMessageFunction( 355 | invocationCount.name, 356 | arnMap, 357 | asyncCallback 358 | ); 359 | }); 360 | }); 361 | }); 362 | 363 | // Launch other invokers to invoke specific functions many times. 364 | fns = fns.concat( 365 | _.map(split.otherInvoker, function (invocationCountSet) { 366 | return function (asyncCallback) { 367 | exports.invokeInvoker(invocationCountSet, arnMap, asyncCallback); 368 | }; 369 | }) 370 | ); 371 | 372 | exports.executeConcurrently( 373 | fns, 374 | config.coordinator.maxApiConcurrency, 375 | callback 376 | ); 377 | }; 378 | -------------------------------------------------------------------------------- /lib/lambdaFunctions/coordinator/coordinator.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Coordinator handler implementation. 3 | */ 4 | 5 | // Core. 6 | var util = require('util'); 7 | 8 | // Other NPM packages. 9 | var async = require('async'); 10 | var _ = require('lodash'); 11 | 12 | // Local. 13 | var common = require('./common'); 14 | // The coordinator expects that the build process will have placed _config.js in 15 | // the same directory as this file. 16 | var config = require('./_config'); 17 | // Expect lambda-complex application constants to be placed in _constants.js. 18 | var constants = require('./_constants'); 19 | // Expect lambda-complex application utilities to be placed in _utilities.js. 20 | var utilities = require('./_utilities'); 21 | 22 | // --------------------------------------------------------------------------- 23 | // Properties. 24 | // --------------------------------------------------------------------------- 25 | 26 | // Will be set by the handler before any other action. 27 | exports.arnMap = undefined; 28 | 29 | // --------------------------------------------------------------------------- 30 | // Functions. 31 | // --------------------------------------------------------------------------- 32 | 33 | /** 34 | * Obtain data on the application status. 35 | * 36 | * The response is of the form: 37 | * 38 | * { 39 | * components: [ 40 | * { 41 | * name: 'functionX', 42 | * type: 'eventFromMessage', 43 | * // A roughly accurate count of how many Lambda functions are presently 44 | * // running for this component. 45 | * concurrency: 1, 46 | * // For eventFromMessage type only. 47 | * maxConcurrency: 10, 48 | * queuedMessageCount: 0 49 | * }, 50 | * ... 51 | * ] 52 | * } 53 | * 54 | * @param {Function} callback Of the form function (error, status). 55 | */ 56 | exports.determineApplicationStatus = function (callback) { 57 | var allComponents = common.getAllComponents(); 58 | var dataByName = {}; 59 | var status = { 60 | components: [] 61 | }; 62 | 63 | _.each(allComponents, function (component) { 64 | var data = { 65 | name: component.name, 66 | type: component.type, 67 | concurrency: null 68 | }; 69 | 70 | if (component.type === constants.componentType.EVENT_FROM_MESSAGE) { 71 | data.maxConcurrency = component.maxConcurrency; 72 | data.queuedMessageCount = null; 73 | } 74 | 75 | dataByName[component.name] = data; 76 | status.components.push(data); 77 | }); 78 | 79 | // For all components, look at the concurrency queue message count, which 80 | // indicates how many Lambda functions are running. 81 | var fns = _.map(allComponents, function (component) { 82 | return function (mapCallback) { 83 | utilities.getQueueMessageCount( 84 | utilities.getConcurrencyQueueUrl(component.name, exports.arnMap), 85 | function (error, count) { 86 | // Just log the error for an individual failed request. 87 | if (error) { 88 | console.error(error); 89 | } 90 | else { 91 | dataByName[component.name].concurrency = count; 92 | } 93 | 94 | mapCallback(); 95 | } 96 | ); 97 | }; 98 | }); 99 | 100 | // For event from message type components, check the status of the queue that 101 | // feeds the component. 102 | fns = fns.concat( 103 | _.chain(allComponents).filter(function (component) { 104 | return component.type === constants.componentType.EVENT_FROM_MESSAGE; 105 | }).map(function (component) { 106 | return function (mapCallback) { 107 | utilities.getQueueMessageCount( 108 | utilities.getQueueUrl(component.name, exports.arnMap), 109 | function (error, count) { 110 | // Just log the error for an individual failed request. 111 | if (error) { 112 | console.error(error); 113 | } 114 | else { 115 | dataByName[component.name].queuedMessageCount = count; 116 | } 117 | 118 | mapCallback(); 119 | } 120 | ); 121 | }; 122 | }).value() 123 | ); 124 | 125 | common.executeConcurrently( 126 | fns, 127 | config.coordinator.maxApiConcurrency, 128 | function (error) { 129 | callback(error, status); 130 | } 131 | ); 132 | }; 133 | 134 | /** 135 | * Given the application status, launch additional coordinator instances if 136 | * needed. This helps to repair an application with multiple coordinators and 137 | * where one more have failed to invoke their successors. 138 | * 139 | * @param {Object} applicationStatus The application status. 140 | * @param {Object} event The event passed in to this instance handler. 141 | * @param {Function} callback Of the form function (error, status). 142 | */ 143 | exports.ensureCoordinatorConcurrency = function (applicationStatus, event, callback) { 144 | // Don't run this on the first generation coordinators, as the application 145 | // is still starting up and the coordinator launch is staggered. 146 | if (event.generation < 2) { 147 | return callback(); 148 | } 149 | 150 | var coordinator = _.find(applicationStatus.components, function (component) { 151 | return ( 152 | component.name === constants.coordinator.NAME && 153 | component.type === constants.componentType.INTERNAL 154 | ); 155 | }); 156 | 157 | // If we don't have the necessary data, then skip this. 158 | if (typeof coordinator.concurrency !== 'number') { 159 | return callback(); 160 | } 161 | 162 | // If we have enough coordinators, then skip this. 163 | if (coordinator.concurrency >= config.coordinator.coordinatorConcurrency) { 164 | return callback(); 165 | } 166 | 167 | var arn = utilities.getLambdaFunctionArn( 168 | constants.coordinator.NAME, 169 | exports.arnMap 170 | ); 171 | 172 | async.times( 173 | config.coordinator.coordinatorConcurrency - coordinator.concurrency, 174 | function (index, timesCallback) { 175 | utilities.invoke(arn, event, timesCallback); 176 | }, 177 | callback 178 | ); 179 | }; 180 | 181 | // --------------------------------------------------------------------------- 182 | // Lambda handler function. 183 | // --------------------------------------------------------------------------- 184 | 185 | /** 186 | * Acts as a coordinator to: 187 | * 188 | * - View queue message counts in the application status. 189 | * - Invoke invokers and other Lambda functions for queues with messages. 190 | * 191 | * @param {Object} event Event instance. 192 | * @param {Object} context Lambda context instance. 193 | */ 194 | exports.handler = function (event, context) { 195 | var startTime = new Date().getTime(); 196 | var applicationStatus; 197 | var invocationCounts; 198 | 199 | // Ensure that we're tracking generation: the count in the chain of 200 | // coordinator Lambda function instances invoking themselves. 201 | event.generation = event.generation || 0; 202 | event.generation++; 203 | 204 | async.series([ 205 | // Load the ARN map first of all. 206 | function (asyncCallback) { 207 | utilities.loadArnMap(config, function (error, arnMap) { 208 | exports.arnMap = arnMap; 209 | 210 | // If this errors out, we can't do anything, not even invoke this 211 | // Lambda function again. So log and exit immediately. 212 | if (error) { 213 | console.error( 214 | 'Critical: failed to load ARN map, aborting immediately.', 215 | error 216 | ); 217 | return context.done(error); 218 | } 219 | 220 | asyncCallback(); 221 | }); 222 | }, 223 | 224 | // Increment the concurrency count. 225 | function (asyncCallback) { 226 | utilities.incrementConcurrencyCount( 227 | constants.coordinator.COMPONENT, 228 | exports.arnMap, 229 | function (error) { 230 | asyncCallback(error); 231 | } 232 | ); 233 | }, 234 | 235 | // Obtain the application status and derived data. 236 | function (asyncCallback) { 237 | exports.determineApplicationStatus(function (error, status) { 238 | if (error) { 239 | return asyncCallback(error); 240 | } 241 | 242 | applicationStatus = status; 243 | invocationCounts = common.getInvocationCounts(applicationStatus); 244 | 245 | console.info(util.format( 246 | 'Generation: %s\nApplication status: %s\nInvocation counts: %s', 247 | event.generation, 248 | JSON.stringify(applicationStatus), 249 | JSON.stringify(invocationCounts) 250 | )); 251 | 252 | asyncCallback(); 253 | }); 254 | }, 255 | 256 | // Ensure that we have enough coordinators running concurrently. This helps 257 | // rescue the application from any unexpected issues that might prevent a 258 | // coordinator from invoking its successor. 259 | function (asyncCallback) { 260 | exports.ensureCoordinatorConcurrency( 261 | applicationStatus, 262 | event, 263 | asyncCallback 264 | ); 265 | }, 266 | 267 | // Next take that data and make API requests to launch other functions as 268 | // needed. 269 | function (asyncCallback) { 270 | common.invokeApplicationLambdaFunctions( 271 | invocationCounts, 272 | exports.arnMap, 273 | asyncCallback 274 | ); 275 | }, 276 | 277 | // If there is time left to wait before the next invocation of the 278 | // coordinator, then wait. 279 | function (asyncCallback) { 280 | common.ensureInterval( 281 | startTime, 282 | // Seconds to milliseconds. 283 | config.coordinator.minInterval * 1000, 284 | context, 285 | asyncCallback 286 | ); 287 | }, 288 | 289 | // Decrement the concurrency count. 290 | // 291 | // It isn't a disaster if this doesn't happen due to earlier errors - it 292 | // just means the count is one high until the message expires, which should 293 | // happen fairly rapidly. 294 | function (asyncCallback) { 295 | utilities.decrementConcurrencyCount( 296 | constants.coordinator.COMPONENT, 297 | exports.arnMap, 298 | asyncCallback 299 | ); 300 | } 301 | ], function (error) { 302 | // Here on in we have to get to the decrement and the invocation of the next 303 | // coordinator instance regardless of issues, so errors are logged only. 304 | if (error) { 305 | console.error(error); 306 | } 307 | 308 | console.info('Invoking the next coordinator.'); 309 | 310 | utilities.invoke( 311 | utilities.getLambdaFunctionArn( 312 | constants.coordinator.NAME, 313 | exports.arnMap 314 | ), 315 | // The event passes on with event.generation, which will be incremented by 316 | // the next coordinator instance. 317 | event, 318 | function (invokeError) { 319 | if (invokeError) { 320 | console.error( 321 | 'Critical: failed to invoke next coordinator.', 322 | invokeError 323 | ); 324 | } 325 | 326 | // On the first generation upload the application confirmation file 327 | // if there is no error. 328 | // 329 | // Lack of an uploaded confirmation file will be considered a failure to 330 | // deploy, which is as it should be. 331 | if (error || invokeError || event.generation > 1) { 332 | context.done( 333 | error || invokeError, 334 | applicationStatus 335 | ); 336 | } 337 | else { 338 | utilities.uploadApplicationConfirmation(config, function (uploadError) { 339 | if (uploadError) { 340 | console.error( 341 | 'Critical: failed to upload application confirmation file.', 342 | uploadError 343 | ); 344 | } 345 | 346 | context.done( 347 | uploadError, 348 | applicationStatus 349 | ); 350 | }); 351 | } 352 | } 353 | ); 354 | }); 355 | }; 356 | -------------------------------------------------------------------------------- /lib/lambdaFunctions/coordinator/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Main coordinator/invoker lambda function. 3 | * 4 | * When the coordinator handle is invoked this is responsible for: 5 | * 6 | * - Respawning itself. 7 | * - Monitoring queues. 8 | * - Invoking invoker lambda function instances. 9 | * 10 | * When the invoker handle is invoked this is responsible for: 11 | * 12 | * - Invoking component lambda function instances. 13 | */ 14 | 15 | var coordinator = require('./coordinator'); 16 | var invoker = require('./invoker'); 17 | 18 | exports.coordinator = coordinator.handler; 19 | exports.invoker = invoker.handler; 20 | -------------------------------------------------------------------------------- /lib/lambdaFunctions/coordinator/invoker.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Invoker handler implementation. 3 | */ 4 | 5 | // Core. 6 | var util = require('util'); 7 | 8 | // NPM. 9 | var async = require('async'); 10 | 11 | // Local. 12 | var common = require('./common'); 13 | // The coordinator expects that the build process will have placed _config.js in 14 | // the same directory as this file. 15 | var config = require('./_config'); 16 | // Expect lambda-complex application constants to be placed in _constants.js. 17 | var constants = require('./_constants'); 18 | // Expect lambda-complex application utilities to be placed in _utilities.js. 19 | var utilities = require('./_utilities'); 20 | 21 | // --------------------------------------------------------------------------- 22 | // Lambda handler function. 23 | // --------------------------------------------------------------------------- 24 | 25 | /** 26 | * An invoker instance is responsible for invoking multiple instances of one or 27 | * more component Lambda functions. In effect this is just an amplifier to allow 28 | * large numbers of invocations to be carried out given that each instance is 29 | * limited in how many invocations it can make. 30 | * 31 | * If asked to invoke too many Lambda function instances, this instance will 32 | * invoke other invoker instances. 33 | * 34 | * This is only used to invoke components that are checking a queue in order to 35 | * obtain concurrency of message processing. 36 | * 37 | * This expects an event of the form: 38 | * 39 | * { 40 | * components: [ 41 | * { 42 | * name: 'componentName', 43 | * count: 10 44 | * }, 45 | * ... 46 | * ] 47 | * } 48 | * 49 | * @param {Object} event Event instance. 50 | * @param {Object} context Lambda context instance. 51 | */ 52 | exports.handler = function (event, context) { 53 | event = event || {}; 54 | var invocationCounts = event.components || []; 55 | var arnMap; 56 | 57 | console.info(util.format( 58 | 'Invocation counts: %s', 59 | JSON.stringify(invocationCounts) 60 | )); 61 | 62 | async.series([ 63 | // Load the ARN map first of all. 64 | function (asyncCallback) { 65 | utilities.loadArnMap(config, function (error, loadedArnMap) { 66 | arnMap = loadedArnMap; 67 | asyncCallback(error); 68 | }); 69 | }, 70 | 71 | // Increment the concurrency count. 72 | function (asyncCallback) { 73 | utilities.incrementConcurrencyCount( 74 | constants.invoker.COMPONENT, 75 | arnMap, 76 | asyncCallback 77 | ); 78 | }, 79 | 80 | // Get on with the invoking. 81 | function (asyncCallback) { 82 | common.invokeApplicationLambdaFunctions( 83 | invocationCounts, 84 | arnMap, 85 | asyncCallback 86 | ); 87 | }, 88 | 89 | // Decrement the concurrency count. 90 | // 91 | // It isn't a disaster if this doesn't happen due to earlier errors - it 92 | // just means the count is one high until the message expires, which should 93 | // happen fairly rapidly. 94 | function (asyncCallback) { 95 | utilities.decrementConcurrencyCount( 96 | constants.invoker.COMPONENT, 97 | arnMap, 98 | asyncCallback 99 | ); 100 | } 101 | ], function (error) { 102 | if (error) { 103 | console.error(error); 104 | } 105 | 106 | context.done(error, invocationCounts); 107 | }); 108 | }; 109 | -------------------------------------------------------------------------------- /lib/lambdaFunctions/coordinator/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lambda-complex-coordinator", 3 | "private": true, 4 | "description": "The default lambda-complex coordinator package.", 5 | "keywords": [ 6 | "lambda", 7 | "complex", 8 | "coordinator" 9 | ], 10 | "version": "0.1.0", 11 | "homepage": "https://github.com/exratione/lambda-complex", 12 | "author": "Reason ", 13 | "engines": { 14 | "node": ">= 0.10" 15 | }, 16 | "dependencies": { 17 | "async": "1.4.2", 18 | "lodash": "3.10.0" 19 | }, 20 | "repository": { 21 | "type": "git", 22 | "url": "https://github.com/exratione/lambda-complex" 23 | }, 24 | "licenses": [{ 25 | "type": "MIT", 26 | "url": "https://github.com/exratione/lambda-complex/raw/master/LICENSE" 27 | }] 28 | } 29 | -------------------------------------------------------------------------------- /lib/shared/constants.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Various constant values. 3 | * 4 | * Shared between deployment and deployed code. 5 | */ 6 | 7 | // Relating to Lambda limits. 8 | exports.lambda = { 9 | // In MBs. 10 | MIN_MEMORY_SIZE: 128, 11 | MAX_MEMORY_SIZE: 1536, 12 | // In seconds. 13 | MIN_TIMEOUT: 3, 14 | MAX_TIMEOUT: 300 15 | }; 16 | 17 | exports.componentType = { 18 | INTERNAL: 'internal', 19 | EVENT_FROM_MESSAGE: 'eventFromMessage', 20 | EVENT_FROM_INVOCATION: 'eventFromInvocation' 21 | }; 22 | 23 | exports.coordinator = { 24 | NAME: 'lambdaComplexCoordinator', 25 | HANDLER: 'index.coordinator', 26 | MEMORY_SIZE: exports.lambda.MIN_MEMORY_SIZE, 27 | TIMEOUT: exports.lambda.MAX_TIMEOUT, 28 | ROLE: 'internalLambdaComplex', 29 | }; 30 | // Useful to have a base component definition. Note that anything using this 31 | // on the deployment side of the house will have to do something useful with 32 | // the npmPackage property - point it in the right direction. 33 | // 34 | // On the deployed side of the house nothing cares about npmPackage. 35 | exports.coordinator.COMPONENT = { 36 | name: exports.coordinator.NAME, 37 | type: exports.componentType.INTERNAL, 38 | lambda: { 39 | npmPackage: undefined, 40 | handler: exports.coordinator.HANDLER, 41 | memorySize: exports.coordinator.MEMORY_SIZE, 42 | timeout: exports.coordinator.TIMEOUT, 43 | role: exports.coordinator.ROLE 44 | } 45 | } 46 | 47 | exports.invoker = { 48 | NAME: 'lambdaComplexInvoker', 49 | HANDLER: 'index.invoker', 50 | MEMORY_SIZE: exports.lambda.MIN_MEMORY_SIZE, 51 | TIMEOUT: exports.lambda.MAX_TIMEOUT, 52 | ROLE: 'internalLambdaComplex', 53 | }; 54 | // Useful to have a base component definition. Note that anything using this 55 | // on the deployment side of the house will have to do something useful with 56 | // the npmPackage property - point it in the right direction. 57 | // 58 | // On the deployed side of the house nothing cares about npmPackage. 59 | exports.invoker.COMPONENT = { 60 | name: exports.invoker.NAME, 61 | type: exports.componentType.INTERNAL, 62 | lambda: { 63 | npmPackage: undefined, 64 | handler: exports.invoker.HANDLER, 65 | memorySize: exports.invoker.MEMORY_SIZE, 66 | timeout: exports.invoker.TIMEOUT, 67 | role: exports.invoker.ROLE 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lambda-complex", 3 | "description": "Managing more complex applications in AWS Lambda.", 4 | "keywords": [ 5 | "aws", 6 | "lambda", 7 | "complex" 8 | ], 9 | "version": "0.7.0", 10 | "homepage": "https://github.com/exratione/lambda-complex", 11 | "author": "Reason ", 12 | "engines": { 13 | "node": ">= 8.0.0" 14 | }, 15 | "dependencies": { 16 | "archiver": "0.14.4", 17 | "async": "1.4.2", 18 | "aws-sdk": "2.2.3", 19 | "chai": "3.1.0", 20 | "cloudformation-deploy": "0.5.0", 21 | "cloudwatch-logs-janitor": "0.2.0", 22 | "dir-compare": "0.0.2", 23 | "extract-zip": "1.6.6", 24 | "fs-extra": "5.0.0", 25 | "grunt": "0.4.5", 26 | "grunt-contrib-clean": "1.1.0", 27 | "grunt-eslint": "20.1.0", 28 | "grunt-mocha-test": "0.13.3", 29 | "handlebars": "4.0.11", 30 | "jsonschema": "1.0.2", 31 | "lodash": "3.10.0", 32 | "mocha": "5.0.0", 33 | "sinon": "1.15.4" 34 | }, 35 | "repository": { 36 | "type": "git", 37 | "url": "https://github.com/exratione/lambda-complex" 38 | }, 39 | "license": "MIT" 40 | } 41 | -------------------------------------------------------------------------------- /tasks/build.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Grunt task lambda-complex-build. 3 | * 4 | * Build a Lambda Complex application. 5 | */ 6 | 7 | // Local. 8 | var common = require('../lib/grunt/common'); 9 | var index = require('../index'); 10 | 11 | module.exports = function (grunt) { 12 | grunt.registerTask( 13 | 'lambda-complex-build', 14 | 'Build a Lambda Complex application.', 15 | function () { 16 | var done = this.async(); 17 | var config = common.getConfigurationFromOptionOrFail(grunt); 18 | 19 | index.build(config, done); 20 | } 21 | ); 22 | }; 23 | -------------------------------------------------------------------------------- /tasks/deploy.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Grunt task lambda-complex-deploy. 3 | * 4 | * Build and deploy a Lambda Complex application as a CloudFormation stack. 5 | */ 6 | 7 | // Local. 8 | var common = require('../lib/grunt/common'); 9 | var index = require('../index'); 10 | 11 | module.exports = function (grunt) { 12 | grunt.registerTask( 13 | 'lambda-complex-deploy', 14 | 'Build and deploy a Lambda Complex application as a CloudFormation stack.', 15 | function () { 16 | var done = this.async(); 17 | var config = common.getConfigurationFromOptionOrFail(grunt); 18 | 19 | index.deploy(config, done); 20 | } 21 | ); 22 | }; 23 | -------------------------------------------------------------------------------- /test/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "mocha": true 4 | }, 5 | "globals": { 6 | "assert": true, 7 | "expect": true, 8 | "should": true, 9 | "sinon": true 10 | } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /test/index.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Tests for the top level index.js interface. 3 | */ 4 | 5 | // NPM. 6 | var fs = require('fs-extra'); 7 | var Janitor = require('cloudwatch-logs-janitor'); 8 | 9 | // Local. 10 | var applicationConfigValidator = require('../lib/build/configValidator'); 11 | var common = require('../lib/build/common'); 12 | var cloudFormationTemplateUtilities = require('../lib/build/cloudFormationTemplateUtilities'); 13 | var cloudFormationUtilities = require('../lib/deploy/cloudFormationUtilities'); 14 | var installUtilities = require('../lib/build/installUtilities'); 15 | var packageUtilities = require('../lib/build/packageUtilities'); 16 | var s3Utilities = require('../lib/deploy/s3Utilities'); 17 | 18 | var index = require('../index'); 19 | 20 | describe('index', function () { 21 | var applicationConfig; 22 | var sandbox; 23 | 24 | beforeEach(function () { 25 | applicationConfig = require('./resources/mockApplication/applicationConfig'); 26 | 27 | sandbox = sinon.sandbox.create(); 28 | sandbox.stub(applicationConfigValidator, 'validate').returns([]); 29 | sandbox.stub(Janitor.prototype, 'deleteMatchingLogGroups').yields(); 30 | }); 31 | 32 | afterEach(function () { 33 | sandbox.restore(); 34 | delete require.cache[require.resolve('./resources/mockApplication/applicationConfig')]; 35 | }); 36 | 37 | describe('build', function () { 38 | 39 | beforeEach(function () { 40 | sandbox.stub(fs, 'remove').yields(); 41 | sandbox.stub(installUtilities, 'installLambdaFunctions').yields(); 42 | sandbox.stub(packageUtilities, 'packageLambdaFunctions').yields(); 43 | sandbox.stub(cloudFormationTemplateUtilities, 'generateTemplate').yields(); 44 | }); 45 | 46 | it('calls underlying functions', function (done) { 47 | index.build(applicationConfig, function (error) { 48 | sinon.assert.callOrder( 49 | applicationConfigValidator.validate, 50 | fs.remove, 51 | installUtilities.installLambdaFunctions, 52 | packageUtilities.packageLambdaFunctions, 53 | cloudFormationTemplateUtilities.generateTemplate 54 | ); 55 | 56 | sinon.assert.calledWith( 57 | applicationConfigValidator.validate, 58 | applicationConfig 59 | ); 60 | sinon.assert.calledWith( 61 | fs.remove, 62 | common.getApplicationBuildDirectory(applicationConfig), 63 | sinon.match.func 64 | ); 65 | sinon.assert.calledWith( 66 | installUtilities.installLambdaFunctions, 67 | applicationConfig, 68 | sinon.match.func 69 | ); 70 | sinon.assert.calledWith( 71 | packageUtilities.packageLambdaFunctions, 72 | applicationConfig, 73 | sinon.match.func 74 | ); 75 | sinon.assert.calledWith( 76 | cloudFormationTemplateUtilities.generateTemplate, 77 | applicationConfig, 78 | sinon.match.func 79 | ); 80 | 81 | done(error); 82 | }); 83 | }); 84 | }); 85 | 86 | describe('deploy', function () { 87 | var results; 88 | 89 | beforeEach(function () { 90 | results = {}; 91 | 92 | sandbox.stub(index, 'build').yields(); 93 | sandbox.stub(s3Utilities, 'uploadLambdaFunctions').yields(); 94 | sandbox.stub(s3Utilities, 'uploadConfig').yields(); 95 | sandbox.stub(cloudFormationUtilities, 'deployStack').yields(null, results); 96 | }); 97 | 98 | it('calls underlying functions', function (done) { 99 | index.deploy(applicationConfig, function (error, obtainedResults) { 100 | expect(obtainedResults).to.equal(results); 101 | 102 | sinon.assert.callOrder( 103 | index.build, 104 | s3Utilities.uploadLambdaFunctions, 105 | cloudFormationUtilities.deployStack 106 | ); 107 | 108 | sinon.assert.calledWith( 109 | index.build, 110 | applicationConfig, 111 | sinon.match.func 112 | ); 113 | sinon.assert.calledWith( 114 | s3Utilities.uploadLambdaFunctions, 115 | applicationConfig, 116 | sinon.match.func 117 | ); 118 | sinon.assert.calledWith( 119 | s3Utilities.uploadConfig, 120 | applicationConfig, 121 | sinon.match.func 122 | ); 123 | sinon.assert.calledWith( 124 | cloudFormationUtilities.deployStack, 125 | applicationConfig, 126 | sinon.match.func 127 | ); 128 | sinon.assert.calledWith( 129 | Janitor.prototype.deleteMatchingLogGroups, 130 | sinon.match(function (options) { 131 | return ( 132 | (options.createdBefore instanceof Date) && 133 | (options.prefix === '/aws/lambda/' + applicationConfig.name + '-') 134 | ); 135 | }, 'Options object does not match.'), 136 | sinon.match.func 137 | ); 138 | 139 | done(error); 140 | }); 141 | }); 142 | 143 | it('skips CloudWatch log deletion if so configured', function (done) { 144 | applicationConfig.deployment.skipPriorCloudWatchLogGroupsDeletion = true; 145 | 146 | index.deploy(applicationConfig, function (error, obtainedResults) { 147 | expect(obtainedResults).to.equal(results); 148 | sinon.assert.notCalled(Janitor.prototype.deleteMatchingLogGroups); 149 | done(error); 150 | }); 151 | }); 152 | }); 153 | }); 154 | -------------------------------------------------------------------------------- /test/lib/build/cloudFormationTemplateUtilities.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Tests for lib/build/cloudFormationTemplateUtilities. 3 | */ 4 | 5 | // Core. 6 | var path = require('path'); 7 | 8 | // Local. 9 | var cloudFormationTemplateUtilities = require('../../../lib/build/cloudFormationTemplateUtilities'); 10 | var constants = require('../../../lib/shared/constants'); 11 | var resources = require('../../resources'); 12 | var applicationConfig = require('../../resources/mockApplication/applicationConfig'); 13 | 14 | describe('lib/build/cloudFormationTemplateUtilities', function () { 15 | var sandbox; 16 | 17 | before(function (done) { 18 | // Needs time to set up the mock application as there are npm install 19 | // commands in there. 20 | this.timeout(30000); 21 | // Set up the mock application. 22 | resources.setUpMockApplication(applicationConfig, done); 23 | }); 24 | 25 | beforeEach(function () { 26 | sandbox = sinon.sandbox.create(); 27 | }); 28 | 29 | afterEach(function () { 30 | sandbox.restore(); 31 | }); 32 | 33 | describe('getAllRoles', function () { 34 | it('functions correctly', function () { 35 | expect(cloudFormationTemplateUtilities.getAllRoles(applicationConfig)).to.eql([ 36 | { 37 | name: constants.coordinator.ROLE, 38 | statements: [] 39 | } 40 | ].concat(applicationConfig.roles)); 41 | }) 42 | }); 43 | 44 | describe('generateTemplate', function () { 45 | it('produces the correct CloudFormation template', function () { 46 | // The template for the mock app will already have been generated via 47 | // this function, so load it and compare with the expected one. 48 | var actual = require(path.resolve( 49 | __dirname, 50 | '../..', 51 | resources.getScratchDirectory(), 52 | applicationConfig.name, 53 | 'cloudFormation' 54 | )); 55 | var expected = resources.getExpectedCloudFormationTemplate(); 56 | 57 | expect(actual).to.eql(expected); 58 | }) 59 | }); 60 | 61 | }); 62 | -------------------------------------------------------------------------------- /test/lib/build/common.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Tests for lib/build/common. 3 | */ 4 | 5 | // Core. 6 | var path = require('path'); 7 | 8 | // NPM. 9 | var fs = require('fs-extra'); 10 | var _ = require('lodash'); 11 | 12 | // Local. 13 | var buildCommon = require('../../../lib/build/common'); 14 | var constants = require('../../../lib/shared/constants'); 15 | var applicationConfig = require('../../resources/mockApplication/applicationConfig'); 16 | var resources = require('../../resources'); 17 | 18 | var scratchDir = resources.getScratchDirectory(); 19 | 20 | describe('lib/build/common', function () { 21 | var sandbox; 22 | 23 | beforeEach(function () { 24 | sandbox = sinon.sandbox.create(); 25 | }); 26 | 27 | afterEach(function () { 28 | sandbox.restore(); 29 | }); 30 | 31 | describe('getApplicationBuildDirectory', function () { 32 | it('functions correctly', function () { 33 | expect( 34 | buildCommon.getApplicationBuildDirectory(applicationConfig) 35 | ).to.equal( 36 | path.resolve( 37 | __dirname, 38 | '../../../build', 39 | applicationConfig.name, 40 | // Might be a number, and path.resolve only wants strings. 41 | '' + applicationConfig.deployId 42 | ) 43 | ); 44 | }); 45 | }); 46 | 47 | describe('getApplicationBuildNodeModulesDirectory', function () { 48 | it('functions correctly', function () { 49 | expect( 50 | buildCommon.getApplicationBuildNodeModulesDirectory(applicationConfig) 51 | ).to.equal( 52 | path.resolve( 53 | __dirname, 54 | '../../../build', 55 | applicationConfig.name, 56 | // Might be a number, and path.resolve only wants strings. 57 | '' + applicationConfig.deployId, 58 | 'node_modules' 59 | ) 60 | ); 61 | }); 62 | }); 63 | 64 | describe('getApplicationPackageDirectories', function () { 65 | var fakePackageDirs; 66 | var fakeApplicationDir; 67 | 68 | beforeEach(function () { 69 | fakeApplicationDir = path.join(scratchDir, 'fake'); 70 | 71 | fakePackageDirs = [ 72 | path.join(fakeApplicationDir, 'node_modules/.bin'), 73 | path.join(fakeApplicationDir, 'node_modules/x'), 74 | path.join(fakeApplicationDir, 'node_modules/y'), 75 | path.join(fakeApplicationDir, 'node_modules/z') 76 | ]; 77 | 78 | _.each(fakePackageDirs, function (dir) { 79 | fs.mkdirsSync(dir); 80 | }); 81 | 82 | // Changing the application build directory to the test scratch directory. 83 | sandbox.stub( 84 | buildCommon, 85 | 'getApplicationBuildDirectory' 86 | ).returns(fakeApplicationDir); 87 | }); 88 | 89 | it('functions correctly, ignores .bin directory', function (done) { 90 | buildCommon.getApplicationPackageDirectories( 91 | applicationConfig, 92 | function (error, dirs) { 93 | expect(error).to.not.be.instanceof(Error); 94 | expect(dirs).to.eql(fakePackageDirs.slice(1)); 95 | done(); 96 | } 97 | ); 98 | }); 99 | }); 100 | 101 | describe('getComponentS3Key', function () { 102 | it('functions correctly', function () { 103 | expect( 104 | buildCommon.getComponentS3Key( 105 | applicationConfig.components[0], 106 | applicationConfig 107 | ) 108 | ).to.equal( 109 | path.join( 110 | applicationConfig.deployment.s3KeyPrefix, 111 | applicationConfig.name, 112 | // Could be a number or a string, but path#join wants strings only. 113 | '' + applicationConfig.deployId, 114 | applicationConfig.components[0].name + '.zip' 115 | ) 116 | ); 117 | }); 118 | }); 119 | 120 | describe('getComponentZipFilePath', function () { 121 | it('functions correctly', function () { 122 | expect( 123 | buildCommon.getComponentZipFilePath( 124 | applicationConfig.components[0], 125 | applicationConfig 126 | ) 127 | ).to.equal( 128 | path.join( 129 | buildCommon.getApplicationBuildDirectory(applicationConfig), 130 | applicationConfig.components[0].name + '.zip' 131 | ) 132 | ); 133 | }); 134 | }); 135 | 136 | describe('getCoordinatorComponentDefinition', function () { 137 | it('functions correctly', function () { 138 | expect(buildCommon.getCoordinatorComponentDefinition()).to.eql({ 139 | name: constants.coordinator.NAME, 140 | type: constants.componentType.INTERNAL, 141 | lambda: { 142 | npmPackage: path.resolve( 143 | __dirname, 144 | '../../../lib/lambdaFunctions/coordinator' 145 | ), 146 | handler: constants.coordinator.HANDLER, 147 | memorySize: constants.coordinator.MEMORY_SIZE, 148 | timeout: constants.coordinator.TIMEOUT, 149 | role: constants.coordinator.ROLE 150 | } 151 | }); 152 | }); 153 | }); 154 | 155 | describe('getInvokerComponentDefinition', function () { 156 | it('functions correctly', function () { 157 | expect(buildCommon.getInvokerComponentDefinition()).to.eql({ 158 | name: constants.invoker.NAME, 159 | type: constants.componentType.INTERNAL, 160 | lambda: { 161 | // It uses a different handle in the coordinator package. 162 | npmPackage: path.resolve( 163 | __dirname, 164 | '../../../lib/lambdaFunctions/coordinator' 165 | ), 166 | handler: constants.invoker.HANDLER, 167 | memorySize: constants.invoker.MEMORY_SIZE, 168 | timeout: constants.invoker.TIMEOUT, 169 | role: constants.invoker.ROLE 170 | } 171 | }); 172 | }); 173 | }); 174 | 175 | describe('getAllComponents', function () { 176 | it('functions correctly', function () { 177 | expect(buildCommon.getAllComponents(applicationConfig)).to.eql([ 178 | buildCommon.getCoordinatorComponentDefinition(), 179 | buildCommon.getInvokerComponentDefinition() 180 | ].concat(applicationConfig.components)) 181 | }); 182 | }); 183 | 184 | describe('getEventFromMessageComponents', function () { 185 | it('functions correctly', function () { 186 | expect(buildCommon.getEventFromMessageComponents(applicationConfig)).to.eql([ 187 | applicationConfig.components[0] 188 | ]); 189 | }); 190 | }); 191 | 192 | describe('generateConfigContents', function () { 193 | it('produces suitable duplicate config Javascript', function () { 194 | var jsPath = path.join(scratchDir, 'configContent.js'); 195 | 196 | fs.writeFileSync( 197 | jsPath, 198 | buildCommon.generateConfigContents(applicationConfig) 199 | ); 200 | 201 | var config = require(jsPath); 202 | // Hijack the Sinon matcher to make the comparison, since that is what it 203 | // is for. 204 | var matcher = resources.getConfigMatcher(applicationConfig); 205 | 206 | expect(matcher.test(config)).to.equal(true); 207 | }); 208 | }); 209 | 210 | }); 211 | -------------------------------------------------------------------------------- /test/lib/deploy/cloudFormationUtilities.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Tests for lib/deploy/cloudFormationUtilities. 3 | */ 4 | 5 | // NPM. 6 | var cloudFormationDeploy = require('cloudformation-deploy'); 7 | var fs = require('fs-extra'); 8 | 9 | // Local. 10 | var common = require('../../../lib/build/common'); 11 | var constants = require('../../../lib/shared/constants'); 12 | var s3Utilities = require('../../../lib/deploy/s3Utilities'); 13 | var utilities = require('../../../lib/shared/utilities'); 14 | 15 | var resources = require('../../resources'); 16 | 17 | describe('lib/deploy/cloudFormationUtilities', function () { 18 | 19 | var applicationConfig; 20 | var clock; 21 | var cloudFormationUtilities; 22 | var sandbox; 23 | 24 | beforeEach(function () { 25 | applicationConfig = require('../../resources/mockApplication/applicationConfig'); 26 | 27 | sandbox = sinon.sandbox.create(); 28 | clock = sandbox.useFakeTimers(); 29 | // Load after creating the clock so that we get the fake timers. 30 | cloudFormationUtilities = require('../../../lib/deploy/cloudFormationUtilities'); 31 | 32 | sandbox.stub(utilities, 'applicationConfirmationExists').yields(null, true); 33 | }); 34 | 35 | afterEach(function () { 36 | sandbox.restore(); 37 | delete require.cache[require.resolve('../../../lib/deploy/cloudFormationUtilities')]; 38 | delete require.cache[require.resolve('../../resources/mockApplication/applicationConfig')]; 39 | }); 40 | 41 | describe('arnMapFromOutputs', function () { 42 | it('functions correctly', function (done) { 43 | var outputs = [ 44 | { 45 | OutputKey: 'key1', 46 | OutputValue: 'value1' 47 | }, 48 | { 49 | OutputKey: 'key2', 50 | OutputValue: 'value2' 51 | }, 52 | ]; 53 | 54 | cloudFormationUtilities.arnMapFromOutputs(outputs, function (error, arnMap) { 55 | expect(arnMap).to.eql({ 56 | key1: 'value1', 57 | key2: 'value2' 58 | }); 59 | done(error); 60 | }); 61 | }); 62 | 63 | it('errors on empty outputs', function (done) { 64 | var outputs = []; 65 | 66 | cloudFormationUtilities.arnMapFromOutputs(outputs, function (error, arnMap) { 67 | expect(error).to.be.instanceOf(Error); 68 | done(); 69 | }); 70 | }); 71 | 72 | it('errors on undefined outputs', function (done) { 73 | var outputs = undefined; 74 | 75 | cloudFormationUtilities.arnMapFromOutputs(outputs, function (error, arnMap) { 76 | expect(error).to.be.instanceOf(Error); 77 | done(); 78 | }); 79 | }); 80 | }); 81 | 82 | describe('startApplication', function () { 83 | var arnMap; 84 | var event; 85 | 86 | beforeEach(function () { 87 | arnMap = resources.getMockArnMap(applicationConfig); 88 | event = {}; 89 | 90 | sandbox.stub(utilities, 'invoke').yields(); 91 | }); 92 | 93 | it('functions as expected', function (done) { 94 | cloudFormationUtilities.startApplication(arnMap, applicationConfig, function (error) { 95 | sinon.assert.callCount( 96 | utilities.invoke, 97 | applicationConfig.coordinator.coordinatorConcurrency 98 | ); 99 | sinon.assert.calledWith( 100 | utilities.invoke, 101 | coordinatorArn = utilities.getLambdaFunctionArn( 102 | constants.coordinator.NAME, 103 | arnMap 104 | ), 105 | event, 106 | sinon.match.func 107 | ); 108 | 109 | done(error); 110 | }); 111 | 112 | clock.tick(applicationConfig.coordinator.minInterval * 1000); 113 | }); 114 | 115 | it('functions as expected when coordinatorConcurrency = 1', function (done) { 116 | var coordinatorConcurrency = applicationConfig.coordinator.coordinatorConcurrency; 117 | applicationConfig.coordinator.coordinatorConcurrency = 1; 118 | 119 | cloudFormationUtilities.startApplication(arnMap, applicationConfig, function (error) { 120 | sinon.assert.callCount( 121 | utilities.invoke, 122 | applicationConfig.coordinator.coordinatorConcurrency 123 | ); 124 | sinon.assert.calledWith( 125 | utilities.invoke, 126 | coordinatorArn = utilities.getLambdaFunctionArn( 127 | constants.coordinator.NAME, 128 | arnMap 129 | ), 130 | event, 131 | sinon.match.func 132 | ); 133 | 134 | applicationConfig.coordinator.coordinatorConcurrency = coordinatorConcurrency; 135 | 136 | done(error); 137 | }); 138 | 139 | clock.tick(applicationConfig.coordinator.minInterval * 1000); 140 | }); 141 | }); 142 | 143 | describe('awaitApplicationConfirmation', function () { 144 | 145 | it('functions correctly', function (done) { 146 | utilities.applicationConfirmationExists.onCall(0).yields(null, false); 147 | cloudFormationUtilities.awaitApplicationConfirmation(applicationConfig, function (error) { 148 | sinon.assert.calledTwice(utilities.applicationConfirmationExists); 149 | sinon.assert.alwaysCalledWith( 150 | utilities.applicationConfirmationExists, 151 | resources.getConfigMatcher(applicationConfig), 152 | sinon.match.func 153 | ); 154 | 155 | done(error); 156 | }); 157 | 158 | clock.tick(4000); 159 | }); 160 | 161 | it('calls back with error', function (done) { 162 | utilities.applicationConfirmationExists.yields(new Error()); 163 | cloudFormationUtilities.awaitApplicationConfirmation(applicationConfig, function (error) { 164 | expect(error).to.be.instanceOf(Error); 165 | 166 | sinon.assert.calledOnce(utilities.applicationConfirmationExists); 167 | sinon.assert.alwaysCalledWith( 168 | utilities.applicationConfirmationExists, 169 | resources.getConfigMatcher(applicationConfig), 170 | sinon.match.func 171 | ); 172 | 173 | done(); 174 | }); 175 | 176 | clock.tick(2000); 177 | }); 178 | 179 | it('times out', function (done) { 180 | utilities.applicationConfirmationExists.yields(null, false); 181 | cloudFormationUtilities.awaitApplicationConfirmation(applicationConfig, function (error) { 182 | expect(error).to.be.instanceOf(Error); 183 | 184 | sinon.assert.alwaysCalledWith( 185 | utilities.applicationConfirmationExists, 186 | resources.getConfigMatcher(applicationConfig), 187 | sinon.match.func 188 | ); 189 | 190 | done(); 191 | }); 192 | 193 | clock.tick((applicationConfig.coordinator.minInterval + 1) * 2 * 1000); 194 | }); 195 | }); 196 | 197 | describe('getSwitchoverFunction', function () { 198 | var arnMap; 199 | var stackDescription; 200 | var switchoverFn; 201 | 202 | beforeEach(function () { 203 | stackDescription = { 204 | Outputs: [ 205 | { 206 | OutputKey: 'a', 207 | OutputValue: 'b', 208 | Description: 'c' 209 | } 210 | ] 211 | }; 212 | arnMap = { 213 | a: 'b' 214 | }; 215 | 216 | sandbox.stub(s3Utilities, 'uploadArnMap').yields(); 217 | sandbox.stub(cloudFormationUtilities, 'startApplication').yields(); 218 | sandbox.stub(cloudFormationUtilities, 'awaitApplicationConfirmation').yields(); 219 | sandbox.stub(applicationConfig.deployment, 'switchoverFunction').yields(); 220 | 221 | switchoverFn = cloudFormationUtilities.getSwitchoverFunction( 222 | applicationConfig, 223 | applicationConfig.deploymentswitchoverFunction 224 | ); 225 | }); 226 | 227 | function checkCalls () { 228 | sinon.assert.calledWith( 229 | s3Utilities.uploadArnMap, 230 | arnMap, 231 | resources.getConfigMatcher(applicationConfig), 232 | sinon.match.func 233 | ); 234 | sinon.assert.calledWith( 235 | cloudFormationUtilities.startApplication, 236 | arnMap, 237 | resources.getConfigMatcher(applicationConfig), 238 | sinon.match.func 239 | ); 240 | sinon.assert.calledWith( 241 | cloudFormationUtilities.awaitApplicationConfirmation, 242 | resources.getConfigMatcher(applicationConfig), 243 | sinon.match.func 244 | ); 245 | sinon.assert.calledWith( 246 | applicationConfig.deployment.switchoverFunction, 247 | stackDescription, 248 | resources.getConfigMatcher(applicationConfig), 249 | sinon.match.func 250 | ); 251 | sinon.assert.callOrder( 252 | s3Utilities.uploadArnMap, 253 | cloudFormationUtilities.startApplication, 254 | cloudFormationUtilities.awaitApplicationConfirmation, 255 | applicationConfig.deployment.switchoverFunction 256 | ); 257 | } 258 | 259 | it('creates function that behaves correctly', function (done) { 260 | switchoverFn(stackDescription, function (error) { 261 | checkCalls(); 262 | done(error); 263 | }); 264 | 265 | clock.tick(applicationConfig.coordinator.minInterval * 2 * 1000); 266 | }); 267 | 268 | it('creates function that behaves correctly when minInterval = 0', function (done) { 269 | var minInterval = applicationConfig.coordinator.minInterval; 270 | applicationConfig.coordinator.minInterval = 0; 271 | 272 | switchoverFn(stackDescription, function (error) { 273 | checkCalls(); 274 | applicationConfig.coordinator.minInterval = minInterval; 275 | done(error); 276 | }); 277 | }); 278 | 279 | it('creates function that behaves correctly when coordinatorConcurrency = 1', function (done) { 280 | var coordinatorConcurrency = applicationConfig.coordinator.coordinatorConcurrency; 281 | applicationConfig.coordinator.coordinatorConcurrency = 1; 282 | 283 | switchoverFn(stackDescription, function (error) { 284 | checkCalls(); 285 | applicationConfig.coordinator.coordinatorConcurrency = coordinatorConcurrency; 286 | done(error); 287 | }); 288 | }); 289 | 290 | it('works when user switchover function is not a function', function (done) { 291 | var fn = applicationConfig.deployment.switchoverFunction; 292 | applicationConfig.deployment.switchoverFunction = undefined; 293 | 294 | switchoverFn(stackDescription, function (error) { 295 | sinon.assert.calledWith( 296 | s3Utilities.uploadArnMap, 297 | arnMap, 298 | resources.getConfigMatcher(applicationConfig), 299 | sinon.match.func 300 | ); 301 | sinon.assert.calledWith( 302 | cloudFormationUtilities.startApplication, 303 | arnMap, 304 | resources.getConfigMatcher(applicationConfig), 305 | sinon.match.func 306 | ); 307 | sinon.assert.calledWith( 308 | cloudFormationUtilities.awaitApplicationConfirmation, 309 | resources.getConfigMatcher(applicationConfig), 310 | sinon.match.func 311 | ); 312 | 313 | // Put it back the way it was. 314 | applicationConfig.deployment.switchoverFunction = fn; 315 | 316 | done(error); 317 | }); 318 | }); 319 | 320 | it('skips user switchover function on uploadArnMap error', function (done) { 321 | s3Utilities.uploadArnMap.yields(new Error()); 322 | 323 | switchoverFn(stackDescription, function (error) { 324 | expect(error).to.be.instanceOf(Error); 325 | sinon.assert.notCalled(applicationConfig.deployment.switchoverFunction); 326 | 327 | done(); 328 | }); 329 | }); 330 | 331 | it('skips user switchover function on startApplication error', function (done) { 332 | cloudFormationUtilities.startApplication.yields(new Error()); 333 | 334 | switchoverFn(stackDescription, function (error) { 335 | expect(error).to.be.instanceOf(Error); 336 | sinon.assert.notCalled(applicationConfig.deployment.switchoverFunction); 337 | 338 | done(); 339 | }); 340 | }); 341 | 342 | it('skips user switchover function on awaitApplicationConfirmation error', function (done) { 343 | cloudFormationUtilities.awaitApplicationConfirmation.yields(new Error()); 344 | 345 | switchoverFn(stackDescription, function (error) { 346 | expect(error).to.be.instanceOf(Error); 347 | sinon.assert.notCalled(applicationConfig.deployment.switchoverFunction); 348 | 349 | done(); 350 | }); 351 | }); 352 | 353 | }); 354 | 355 | describe('generateCloudFormationDeployConfig', function () { 356 | 357 | it('creates correct configuration object', function () { 358 | var obj = cloudFormationUtilities.generateCloudFormationDeployConfig( 359 | applicationConfig 360 | ); 361 | 362 | // Check the function. 363 | expect(obj.postCreationFn).to.be.a('function'); 364 | delete obj.postCreationFn; 365 | 366 | // Compare the rest. 367 | expect(obj).to.eql({ 368 | baseName: applicationConfig.name, 369 | version: applicationConfig.version, 370 | deployId: applicationConfig.deployId, 371 | createStackTimeoutInMinutes: 10, 372 | tags: applicationConfig.deployment.tags, 373 | progressCheckIntervalInSeconds: 10, 374 | priorInstance: cloudFormationDeploy.priorInstance.DELETE, 375 | onDeployFailure: cloudFormationDeploy.onDeployFailure.DELETE 376 | }); 377 | }); 378 | 379 | it('creates correct configuration object for skipPriorCloudFormationStackDeletion', function () { 380 | applicationConfig.deployment.skipPriorCloudFormationStackDeletion = true; 381 | var obj = cloudFormationUtilities.generateCloudFormationDeployConfig( 382 | applicationConfig 383 | ); 384 | 385 | // Check the function. 386 | expect(obj.postCreationFn).to.be.a('function'); 387 | delete obj.postCreationFn; 388 | 389 | // Compare the rest. 390 | expect(obj).to.eql({ 391 | baseName: applicationConfig.name, 392 | version: applicationConfig.version, 393 | deployId: applicationConfig.deployId, 394 | createStackTimeoutInMinutes: 10, 395 | tags: applicationConfig.deployment.tags, 396 | progressCheckIntervalInSeconds: 10, 397 | priorInstance: cloudFormationDeploy.priorInstance.DO_NOTHING, 398 | onDeployFailure: cloudFormationDeploy.onDeployFailure.DELETE 399 | }); 400 | }); 401 | 402 | it('creates correct configuration object for skipCloudFormationStackDeletionOnFailure', function () { 403 | applicationConfig.deployment.skipCloudFormationStackDeletionOnFailure = true; 404 | var obj = cloudFormationUtilities.generateCloudFormationDeployConfig( 405 | applicationConfig 406 | ); 407 | 408 | // Check the function. 409 | expect(obj.postCreationFn).to.be.a('function'); 410 | delete obj.postCreationFn; 411 | 412 | // Compare the rest. 413 | expect(obj).to.eql({ 414 | baseName: applicationConfig.name, 415 | version: applicationConfig.version, 416 | deployId: applicationConfig.deployId, 417 | createStackTimeoutInMinutes: 10, 418 | tags: applicationConfig.deployment.tags, 419 | progressCheckIntervalInSeconds: 10, 420 | priorInstance: cloudFormationDeploy.priorInstance.DELETE, 421 | onDeployFailure: cloudFormationDeploy.onDeployFailure.DO_NOTHING 422 | }); 423 | }); 424 | }); 425 | 426 | describe('deployStack', function () { 427 | var cloudFormationDeployConfig; 428 | var results; 429 | var template; 430 | 431 | beforeEach(function () { 432 | cloudFormationDeployConfig = {}; 433 | results = {}; 434 | template = JSON.stringify({}); 435 | 436 | sandbox.stub( 437 | cloudFormationUtilities, 438 | 'generateCloudFormationDeployConfig' 439 | ).returns(cloudFormationDeployConfig); 440 | 441 | sandbox.stub(fs, 'readFile').yields(null, template); 442 | sandbox.stub(cloudFormationDeploy, 'deploy').yields(null, results); 443 | }); 444 | 445 | it('functions correctly', function (done) { 446 | cloudFormationUtilities.deployStack(applicationConfig, function (error, obtainedResults) { 447 | expect(obtainedResults).to.equal(results); 448 | 449 | sinon.assert.calledWith( 450 | fs.readFile, 451 | common.getCloudFormationTemplatePath(applicationConfig), 452 | { 453 | encoding: 'utf8' 454 | }, 455 | sinon.match.func 456 | ); 457 | 458 | sinon.assert.calledWith( 459 | cloudFormationDeploy.deploy, 460 | cloudFormationDeployConfig, 461 | template, 462 | sinon.match.func 463 | ); 464 | 465 | done(error); 466 | }); 467 | }); 468 | 469 | it('calls back with error on read file error', function (done) { 470 | fs.readFile.yields(new Error()); 471 | 472 | cloudFormationUtilities.deployStack(applicationConfig, function (error) { 473 | expect(error).to.be.instanceOf(Error); 474 | done(); 475 | }); 476 | }); 477 | 478 | it('calls back with error on deploy error', function (done) { 479 | cloudFormationDeploy.deploy.yields(new Error()); 480 | 481 | cloudFormationUtilities.deployStack(applicationConfig, function (error) { 482 | expect(error).to.be.instanceOf(Error); 483 | done(); 484 | }); 485 | }); 486 | 487 | }); 488 | 489 | }); 490 | -------------------------------------------------------------------------------- /test/lib/deploy/s3Utilities.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Tests for lib/deploy/s3Utilities. 3 | */ 4 | 5 | var common = require('../../../lib/build/common'); 6 | var s3Utilities = require('../../../lib/deploy/s3Utilities'); 7 | var utilities = require('../../../lib/shared/utilities'); 8 | 9 | var resources = require('../../resources'); 10 | var applicationConfig = require('../../resources/mockApplication/applicationConfig'); 11 | 12 | describe('lib/deploy/s3Utilities', function () { 13 | 14 | var sandbox; 15 | 16 | beforeEach(function () { 17 | sandbox = sinon.sandbox.create(); 18 | 19 | // Make sure the relevant methods on the S3 client are stubbed. 20 | sandbox.stub(s3Utilities.s3Client, 'putObject').yields(); 21 | }); 22 | 23 | afterEach(function () { 24 | sandbox.restore(); 25 | }); 26 | 27 | describe('uploadArnMap', function () { 28 | var arnMap; 29 | 30 | beforeEach(function () { 31 | arnMap = {}; 32 | }); 33 | 34 | it('correctly invokes the client function', function (done) { 35 | s3Utilities.uploadArnMap(arnMap, applicationConfig, function (error) { 36 | sinon.assert.calledOnce(s3Utilities.s3Client.putObject); 37 | sinon.assert.calledWith( 38 | s3Utilities.s3Client.putObject, 39 | { 40 | Body: JSON.stringify(arnMap), 41 | Bucket: applicationConfig.deployment.s3Bucket, 42 | ContentType: 'application/json', 43 | Key: utilities.getArnMapS3Key(applicationConfig) 44 | }, 45 | sinon.match.func 46 | ); 47 | 48 | done(error); 49 | }); 50 | }); 51 | 52 | it('retries on failure', function (done) { 53 | s3Utilities.s3Client.putObject.onCall(0).yields(new Error()); 54 | 55 | s3Utilities.uploadArnMap(arnMap, applicationConfig, function (error) { 56 | sinon.assert.calledTwice(s3Utilities.s3Client.putObject); 57 | sinon.assert.calledWith( 58 | s3Utilities.s3Client.putObject, 59 | { 60 | Body: JSON.stringify(arnMap), 61 | Bucket: applicationConfig.deployment.s3Bucket, 62 | ContentType: 'application/json', 63 | Key: utilities.getArnMapS3Key(applicationConfig) 64 | }, 65 | sinon.match.func 66 | ); 67 | 68 | done(error); 69 | }); 70 | }); 71 | }); 72 | 73 | describe('uploadConfig', function () { 74 | 75 | it('correctly invokes the client function', function (done) { 76 | s3Utilities.uploadConfig(applicationConfig, function (error) { 77 | sinon.assert.calledOnce(s3Utilities.s3Client.putObject); 78 | sinon.assert.calledWith( 79 | s3Utilities.s3Client.putObject, 80 | { 81 | Body: common.generateConfigContents(applicationConfig), 82 | Bucket: applicationConfig.deployment.s3Bucket, 83 | ContentType: 'application/javascript', 84 | Key: utilities.getConfigS3Key(applicationConfig) 85 | }, 86 | sinon.match.func 87 | ); 88 | 89 | done(error); 90 | }); 91 | }); 92 | 93 | it('retries on failure', function (done) { 94 | s3Utilities.s3Client.putObject.onCall(0).yields(new Error()); 95 | 96 | s3Utilities.uploadConfig(applicationConfig, function (error) { 97 | sinon.assert.calledTwice(s3Utilities.s3Client.putObject); 98 | sinon.assert.calledWith( 99 | s3Utilities.s3Client.putObject, 100 | { 101 | Body: common.generateConfigContents(applicationConfig), 102 | Bucket: applicationConfig.deployment.s3Bucket, 103 | ContentType: 'application/javascript', 104 | Key: utilities.getConfigS3Key(applicationConfig) 105 | }, 106 | sinon.match.func 107 | ); 108 | 109 | done(error); 110 | }); 111 | }); 112 | }); 113 | 114 | describe('uploadLambdaFunction', function () { 115 | var component; 116 | 117 | beforeEach(function () { 118 | component = applicationConfig.components[0]; 119 | }); 120 | 121 | it('correctly invokes the client function', function (done) { 122 | s3Utilities.uploadLambdaFunction(component, applicationConfig, function (error) { 123 | sinon.assert.calledOnce(s3Utilities.s3Client.putObject); 124 | sinon.assert.calledWith( 125 | s3Utilities.s3Client.putObject, 126 | { 127 | // Should be a read stream. 128 | Body: sinon.match.object, 129 | Bucket: applicationConfig.deployment.s3Bucket, 130 | Key: common.getComponentS3Key(component, applicationConfig) 131 | }, 132 | sinon.match.func 133 | ); 134 | 135 | done(error); 136 | }); 137 | }); 138 | 139 | it('retries on failure', function (done) { 140 | s3Utilities.s3Client.putObject.onCall(0).yields(new Error()); 141 | 142 | s3Utilities.uploadLambdaFunction(component, applicationConfig, function (error) { 143 | sinon.assert.calledTwice(s3Utilities.s3Client.putObject); 144 | sinon.assert.calledWith( 145 | s3Utilities.s3Client.putObject, 146 | { 147 | // Should be a read stream. 148 | Body: sinon.match.object, 149 | Bucket: applicationConfig.deployment.s3Bucket, 150 | Key: common.getComponentS3Key(component, applicationConfig) 151 | }, 152 | sinon.match.func 153 | ); 154 | 155 | done(error); 156 | }); 157 | }); 158 | }); 159 | 160 | describe('uploadLambdaFunctions', function () { 161 | it('functions correctly', function (done) { 162 | sandbox.stub(s3Utilities, 'uploadLambdaFunction').yields(); 163 | 164 | s3Utilities.uploadLambdaFunctions(applicationConfig, function (error) { 165 | sinon.assert.callCount(s3Utilities.uploadLambdaFunction, 4); 166 | sinon.assert.alwaysCalledWith( 167 | s3Utilities.uploadLambdaFunction, 168 | // A component; these functions run concurrently, so can't easily 169 | // specify which component per call. 170 | sinon.match.object, 171 | resources.getConfigMatcher(applicationConfig), 172 | sinon.match.func 173 | ); 174 | 175 | done(error); 176 | }); 177 | }); 178 | }); 179 | }); 180 | -------------------------------------------------------------------------------- /test/lib/grunt/common.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Tests for lib/grunt/common. 3 | */ 4 | 5 | // Core. 6 | var path = require('path'); 7 | 8 | // Local. 9 | var applicationConfig = require('../../resources/mockApplication/applicationConfig'); 10 | var gruntCommon = require('../../../lib/grunt/common'); 11 | var resources = require('../../resources'); 12 | 13 | describe('lib/grunt/common', function () { 14 | var grunt; 15 | var relativeConfigPath; 16 | var absoluteConfigPath; 17 | var sandbox; 18 | 19 | beforeEach(function () { 20 | grunt = { 21 | fail: { 22 | fatal: function () {} 23 | }, 24 | option: function () {} 25 | }; 26 | sandbox = sinon.sandbox.create(); 27 | 28 | // Relative to the top level directory, which should be process.cwd() when 29 | // Grunt is running. 30 | relativeConfigPath = './test/resources/mockApplication/applicationConfig'; 31 | absoluteConfigPath = path.resolve(process.cwd(), relativeConfigPath); 32 | 33 | sandbox.stub(grunt.fail, 'fatal'); 34 | }); 35 | 36 | afterEach(function () { 37 | sandbox.restore(); 38 | }); 39 | 40 | describe('getConfigurationFromOptionOrFail', function () { 41 | it('functions as expected for relative path', function () { 42 | sandbox.stub(grunt, 'option').returns(relativeConfigPath); 43 | 44 | var config = gruntCommon.getConfigurationFromOptionOrFail(grunt); 45 | 46 | sinon.assert.calledWith(grunt.option, 'config-path'); 47 | 48 | // Hijack the Sinon matcher to make the comparison, since that is what it 49 | // is for. 50 | var matcher = resources.getConfigMatcher(applicationConfig); 51 | 52 | expect(matcher.test(config)).to.equal(true); 53 | }); 54 | 55 | it('functions as expected for absolute path', function () { 56 | sandbox.stub(grunt, 'option').returns(absoluteConfigPath); 57 | 58 | var config = gruntCommon.getConfigurationFromOptionOrFail(grunt); 59 | 60 | sinon.assert.calledWith(grunt.option, 'config-path'); 61 | 62 | // Hijack the Sinon matcher to make the comparison, since that is what it 63 | // is for. 64 | var matcher = resources.getConfigMatcher(applicationConfig); 65 | 66 | expect(matcher.test(config)).to.equal(true); 67 | }); 68 | 69 | it('invokes grunt.fail.fatal for missing option', function () { 70 | sandbox.stub(grunt, 'option').returns(undefined); 71 | 72 | gruntCommon.getConfigurationFromOptionOrFail(grunt); 73 | 74 | sinon.assert.calledWith(grunt.option, 'config-path'); 75 | sinon.assert.calledWith(grunt.fail.fatal, sinon.match.instanceOf(Error)); 76 | }); 77 | 78 | it('invokes grunt.fail.fatal for bad path', function () { 79 | sandbox.stub(grunt, 'option').returns('/no/such/path'); 80 | 81 | gruntCommon.getConfigurationFromOptionOrFail(grunt); 82 | 83 | sinon.assert.calledWith(grunt.option, 'config-path'); 84 | sinon.assert.calledWith(grunt.fail.fatal, sinon.match.instanceOf(Error)); 85 | }); 86 | }) 87 | 88 | }); 89 | -------------------------------------------------------------------------------- /test/lib/lambdaFunctions/coordinator/index.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Tests for lib/lambdaFunctions/coordinator/index.js. 3 | */ 4 | 5 | // Core. 6 | var path = require('path'); 7 | 8 | // Local. 9 | var applicationConfig = require('../../../resources/mockApplication/applicationConfig'); 10 | var resources = require('../../../resources'); 11 | var scratchDir = resources.getScratchDirectory(); 12 | var mockApplicationDir = path.join(scratchDir, applicationConfig.name); 13 | 14 | describe('lib/lambdaFunctions/coordinator/coordinator', function () { 15 | var constants; 16 | var coordinator; 17 | var index; 18 | var invoker; 19 | var utilities; 20 | 21 | before(function (done) { 22 | // Needs time to set up the mock application as there are npm install 23 | // commands in there. 24 | this.timeout(30000); 25 | // Set up the mock application. 26 | resources.setUpMockApplication(applicationConfig, done); 27 | }); 28 | 29 | beforeEach(function () { 30 | constants = require(path.join( 31 | mockApplicationDir, 32 | 'lambdaComplexCoordinator', 33 | '_constants' 34 | )); 35 | coordinator = require(path.join( 36 | mockApplicationDir, 37 | 'lambdaComplexCoordinator', 38 | 'coordinator' 39 | )); 40 | index = require(path.join( 41 | mockApplicationDir, 42 | 'lambdaComplexCoordinator', 43 | 'index' 44 | )); 45 | invoker = require(path.join( 46 | mockApplicationDir, 47 | 'lambdaComplexCoordinator', 48 | 'invoker' 49 | )); 50 | utilities = require(path.join( 51 | mockApplicationDir, 52 | 'lambdaComplexCoordinator', 53 | '_utilities' 54 | )); 55 | }); 56 | 57 | it('exports expected handlers', function () { 58 | var coordinatorHandlerFnName = utilities.getFunctionNameFromHandle( 59 | constants.coordinator.HANDLER 60 | ); 61 | var invokerHandlerFnName = utilities.getFunctionNameFromHandle( 62 | constants.invoker.HANDLER 63 | ); 64 | 65 | expect(index[coordinatorHandlerFnName]).to.equal(coordinator.handler); 66 | expect(index[invokerHandlerFnName]).to.equal(invoker.handler); 67 | }); 68 | }); 69 | -------------------------------------------------------------------------------- /test/lib/lambdaFunctions/coordinator/invoker.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Tests for lib/lambdaFunctions/coordinator/invoker.js. 3 | */ 4 | 5 | // Core. 6 | var path = require('path'); 7 | 8 | // Local. 9 | 10 | 11 | var resources = require('../../../resources'); 12 | var applicationConfig = require('../../../resources/mockApplication/applicationConfig'); 13 | var scratchDir = resources.getScratchDirectory(); 14 | var mockApplicationDir = path.join(scratchDir, applicationConfig.name); 15 | 16 | describe('lib/lambdaFunctions/coordinator/invoker', function () { 17 | 18 | var arnMap; 19 | var common; 20 | var invoker; 21 | var sandbox; 22 | var utilities; 23 | 24 | before(function (done) { 25 | // Needs time to set up the mock application as there are npm install 26 | // commands in there. 27 | this.timeout(30000); 28 | // Set up the mock application. 29 | resources.setUpMockApplication(applicationConfig, done); 30 | }); 31 | 32 | beforeEach(function () { 33 | sandbox = sinon.sandbox.create(); 34 | 35 | common = require(path.join( 36 | mockApplicationDir, 37 | 'lambdaComplexCoordinator', 38 | 'common' 39 | )); 40 | constants = require(path.join( 41 | mockApplicationDir, 42 | 'lambdaComplexCoordinator', 43 | '_constants' 44 | )); 45 | invoker = require(path.join( 46 | mockApplicationDir, 47 | 'lambdaComplexCoordinator', 48 | 'invoker' 49 | )); 50 | utilities = require(path.join( 51 | mockApplicationDir, 52 | 'lambdaComplexCoordinator', 53 | '_utilities' 54 | )); 55 | 56 | sandbox.stub(console, 'info'); 57 | 58 | arnMap = resources.getMockArnMap(applicationConfig); 59 | invoker.arnMap = arnMap; 60 | }); 61 | 62 | afterEach(function () { 63 | sandbox.restore(); 64 | }); 65 | 66 | describe('handler', function () { 67 | var event; 68 | var context; 69 | 70 | beforeEach(function () { 71 | event = { 72 | components: [ 73 | { 74 | name: 'componentName', 75 | count: 10 76 | } 77 | ] 78 | }; 79 | context = { 80 | done: sandbox.stub() 81 | }; 82 | 83 | // This function is expected to set the ARN map. 84 | invoker.arnMap = undefined; 85 | 86 | sandbox.stub(utilities, 'loadArnMap').yields(null, arnMap); 87 | sandbox.stub(utilities, 'incrementConcurrencyCount').yields(); 88 | sandbox.stub(utilities, 'decrementConcurrencyCount').yields(); 89 | sandbox.stub(common, 'invokeApplicationLambdaFunctions').yields(); 90 | }); 91 | 92 | it('calls expected functions', function (done) { 93 | invoker.handler(event, context); 94 | 95 | setTimeout(function () { 96 | sinon.assert.calledWith( 97 | utilities.loadArnMap, 98 | resources.getConfigMatcher(applicationConfig), 99 | sinon.match.func 100 | ); 101 | sinon.assert.calledWith( 102 | utilities.incrementConcurrencyCount, 103 | constants.invoker.COMPONENT, 104 | arnMap, 105 | sinon.match.func 106 | ); 107 | sinon.assert.calledWith( 108 | common.invokeApplicationLambdaFunctions, 109 | event.components, 110 | arnMap, 111 | sinon.match.func 112 | ); 113 | sinon.assert.calledWith( 114 | utilities.decrementConcurrencyCount, 115 | constants.invoker.COMPONENT, 116 | arnMap, 117 | sinon.match.func 118 | ); 119 | sinon.assert.calledWith( 120 | context.done, 121 | null, 122 | event.components 123 | ); 124 | 125 | done(); 126 | }, 20); 127 | }); 128 | 129 | it('defaults to empty component array', function (done) { 130 | event = {}; 131 | invoker.handler(event, context); 132 | 133 | setTimeout(function () { 134 | sinon.assert.calledWith( 135 | common.invokeApplicationLambdaFunctions, 136 | [], 137 | arnMap, 138 | sinon.match.func 139 | ); 140 | 141 | done(); 142 | }, 20); 143 | }); 144 | 145 | it('errors on failure of loadArnMap', function (done) { 146 | sandbox.stub(console, 'error'); 147 | utilities.loadArnMap.yields(new Error()); 148 | 149 | invoker.handler(event, context); 150 | 151 | setTimeout(function () { 152 | sinon.assert.calledWith( 153 | context.done, 154 | sinon.match.instanceOf(Error), 155 | event.components 156 | ); 157 | 158 | done(); 159 | }, 20); 160 | }); 161 | 162 | it('errors on increment error', function (done) { 163 | sandbox.stub(console, 'error'); 164 | utilities.incrementConcurrencyCount.yields(new Error()); 165 | 166 | invoker.handler(event, context); 167 | 168 | setTimeout(function () { 169 | sinon.assert.calledWith( 170 | context.done, 171 | sinon.match.instanceOf(Error), 172 | event.components 173 | ); 174 | 175 | done(); 176 | }, 20); 177 | }); 178 | 179 | 180 | it('errors on failure of invokeApplicationLambdaFunctions', function (done) { 181 | sandbox.stub(console, 'error'); 182 | common.invokeApplicationLambdaFunctions.yields(new Error()); 183 | 184 | invoker.handler(event, context); 185 | 186 | setTimeout(function () { 187 | sinon.assert.calledWith( 188 | context.done, 189 | sinon.match.instanceOf(Error), 190 | event.components 191 | ); 192 | 193 | done(); 194 | }, 20); 195 | }); 196 | 197 | it('errors on failure of decrementConcurrencyCount', function (done) { 198 | sandbox.stub(console, 'error'); 199 | utilities.decrementConcurrencyCount.yields(new Error()); 200 | 201 | invoker.handler(event, context); 202 | 203 | setTimeout(function () { 204 | sinon.assert.calledWith( 205 | context.done, 206 | sinon.match.instanceOf(Error), 207 | event.components 208 | ); 209 | 210 | done(); 211 | }, 20); 212 | }); 213 | }); 214 | 215 | }); 216 | -------------------------------------------------------------------------------- /test/mochaInit.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Set up some of the globals for testing. 3 | */ 4 | 5 | var chai = require('chai'); 6 | var sinon = require('sinon'); 7 | 8 | global.sinon = sinon; 9 | 10 | global.assert = chai.assert; 11 | global.expect = chai.expect; 12 | global.should = chai.should(); 13 | -------------------------------------------------------------------------------- /test/resources/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Helpers for testing. 3 | */ 4 | 5 | // Core. 6 | var path = require('path'); 7 | var util = require('util'); 8 | 9 | // NPM. 10 | var fs = require('fs-extra'); 11 | var _ = require('lodash'); 12 | 13 | // Local. 14 | var index = require('../../index'); 15 | var buildCommon = require('../../lib/build/common'); 16 | var utilities = require('../../lib/shared/utilities'); 17 | var constants = require('../../lib/shared/constants'); 18 | 19 | // --------------------------------------------------------------------------- 20 | // Exported functions. 21 | // --------------------------------------------------------------------------- 22 | 23 | exports.getScratchDirectory = function () { 24 | return path.resolve(__dirname, '../scratch'); 25 | }; 26 | 27 | /** 28 | * Obtain a mock ARN map for an application. 29 | * 30 | * @param {Object} config The mock application config. 31 | * @return {Object} The ARN map. 32 | */ 33 | exports.getMockArnMap = function (config) { 34 | var arnMap = {}; 35 | var prop; 36 | 37 | // Ensure that this includes values for the internal components not specified 38 | // explicitly in the config. 39 | _.each(buildCommon.getAllComponents(config), function (component) { 40 | // Queues for event from message type components. 41 | if (component.type === constants.componentType.EVENT_FROM_MESSAGE) { 42 | prop = utilities.getQueueArnOutputName(component.name); 43 | arnMap[prop] = util.format( 44 | 'arn:aws:sqs:%s:444555666777:%s', 45 | config.deployment.region, 46 | utilities.getQueueName(component.name) 47 | ); 48 | } 49 | 50 | // Concurrency queues for all components. 51 | prop = utilities.getConcurrencyQueueArnOutputName(component.name); 52 | arnMap[prop] = util.format( 53 | 'arn:aws:sqs:%s:444555666777:%s', 54 | config.deployment.region, 55 | utilities.getConcurrencyQueueName(component.name) 56 | ); 57 | 58 | // Lambda functions. 59 | prop = utilities.getLambdaFunctionArnOutputName(component.name); 60 | arnMap[prop] = util.format( 61 | 'arn:aws:sqs:%s:444555666777:%s', 62 | config.deployment.region, 63 | // Can't produce a real thing here as it is auto-generated by AWS. 64 | 'lambdaFunctionAutoGeneratedName' 65 | ); 66 | }); 67 | 68 | return arnMap; 69 | }; 70 | 71 | /** 72 | * Set up a mock application in the scratch directory. 73 | * 74 | * @param {Object} config The mock application config. 75 | * @param {Function} callback Of the form function (error). 76 | */ 77 | exports.setUpMockApplication = function (config, callback) { 78 | // We only want to set this up once per test run to save time, but multiple 79 | // suites request it. Hence set a global and check it to ensure it runs only 80 | // once. 81 | if (global.mockApplicationCreated) { 82 | return callback(); 83 | } 84 | 85 | var mockApplicationDir = path.resolve( 86 | exports.getScratchDirectory(), 87 | config.name 88 | ); 89 | 90 | // Start by changing the build directory to the test scratch directory. 91 | sinon.stub( 92 | buildCommon, 93 | 'getApplicationBuildDirectory' 94 | ).returns(mockApplicationDir); 95 | 96 | // Get rid of what was there. 97 | fs.removeSync(mockApplicationDir); 98 | 99 | // Run this through the normal build process, but with the altered directory. 100 | index.build(config, function (error) { 101 | buildCommon.getApplicationBuildDirectory.restore(); 102 | global.mockApplicationCreated = true; 103 | callback(error); 104 | }); 105 | }; 106 | 107 | /** 108 | * Load the expected CloudFormation template. 109 | * 110 | * @return {Object} The expected CloudFormation template. 111 | */ 112 | exports.getExpectedCloudFormationTemplate = function () { 113 | return require(path.resolve( 114 | __dirname, 115 | 'mockApplication/cloudFormation' 116 | )); 117 | }; 118 | 119 | /** 120 | * Obtain a Sinon matcher that can compare an original configuration object with 121 | * one loaded from the copied configuration file. 122 | * 123 | * This requires matching the possible functions in the config object, which 124 | * can't be done by a straight equality since they'll be different function 125 | * instances. 126 | * 127 | * @param {Object} config A configuration object. 128 | * @return {Object} A matcher. 129 | */ 130 | exports.getConfigMatcher = function (expectedConfig) { 131 | 132 | function compareFns (fn1, fn2) { 133 | if (typeof fn1 === 'function' && typeof fn2 === 'function') { 134 | return fn1.toString() === fn2.toString(); 135 | } 136 | 137 | return fn1 === fn2; 138 | } 139 | 140 | // The function provided must return true on a match, false on no match. 141 | return sinon.match(function (actualConfig) { 142 | // This covers most of it, but functions are not stringified. 143 | if (JSON.stringify(expectedConfig) !== JSON.stringify(actualConfig)) { 144 | return false; 145 | } 146 | 147 | // Check the switchover function. 148 | if (!compareFns( 149 | expectedConfig.deployment.switchoverFunction, 150 | actualConfig.deployment.switchoverFunction 151 | )) { 152 | return false; 153 | } 154 | 155 | // Check the routing functions. 156 | return _.every(expectedConfig.components, function (component, index) { 157 | return compareFns( 158 | component.routing, 159 | actualConfig.components[index].routing 160 | ); 161 | }); 162 | }, 'Configuration object does not match.'); 163 | }; 164 | -------------------------------------------------------------------------------- /test/resources/mockApplication/applicationConfig.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview A configuration for the mock application. 3 | */ 4 | 5 | // NPM. 6 | var path = require('path'); 7 | 8 | module.exports = { 9 | name: 'mock', 10 | version: '0.1.0', 11 | deployId: 15, 12 | 13 | deployment: { 14 | region: 'us-east-1', 15 | s3Bucket: 'lambda-complex', 16 | s3KeyPrefix: 'applications/', 17 | // No additional tags. 18 | tags: {}, 19 | // No additional duties for the switchover between versions of the 20 | // deployed application. 21 | switchoverFunction: function (stackDescription, config, callback) { 22 | callback(); 23 | }, 24 | skipPriorCloudFormationStackDeletion: false, 25 | skipPriorCloudWatchLogGroupsDeletion: false, 26 | skipCloudFormationStackDeletionOnFailure: false 27 | }, 28 | 29 | coordinator: { 30 | coordinatorConcurrency: 2, 31 | // Keep these low to make inspection of test data easier. 32 | maxApiConcurrency: 4, 33 | maxInvocationCount: 6, 34 | minInterval: 10 35 | }, 36 | 37 | roles: [ 38 | // Not really used, just here to exercise code more completely. 39 | { 40 | name: 's3ReadA', 41 | statements: [ 42 | { 43 | effect: 'Allow', 44 | action: 's3:get*', 45 | resource: [ 46 | 'arn:aws:s3:::exampleBucket1/*' 47 | ] 48 | } 49 | ] 50 | }, 51 | { 52 | name: 's3ReadB', 53 | statements: [ 54 | { 55 | effect: 'Allow', 56 | action: 's3:get*', 57 | resource: [ 58 | 'arn:aws:s3:::exampleBucket2/*' 59 | ] 60 | } 61 | ] 62 | } 63 | ], 64 | 65 | components: [ 66 | { 67 | name: 'message', 68 | type: 'eventFromMessage', 69 | maxConcurrency: 10, 70 | queueWaitTime: 0, 71 | // Routing will be defined in tests as needed. 72 | // routing: undefined, 73 | lambda: { 74 | npmPackage: path.join(__dirname, 'mockLambdaFunction'), 75 | handler: 'index.handler', 76 | memorySize: 128, 77 | timeout: 60, 78 | role: 's3ReadA' 79 | } 80 | }, 81 | 82 | { 83 | name: 'invocation', 84 | type: 'eventFromInvocation', 85 | // Routing will be defined in tests as needed. 86 | // routing: undefined, 87 | lambda: { 88 | npmPackage: path.join(__dirname, 'mockLambdaFunction'), 89 | handler: 'index.handler', 90 | memorySize: 128, 91 | timeout: 60, 92 | role: 's3ReadB' 93 | } 94 | } 95 | ] 96 | }; 97 | -------------------------------------------------------------------------------- /test/resources/mockApplication/mockLambdaFunction/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview A dummy Lambda function for testing purposes. 3 | */ 4 | 5 | exports.handler = function (event, context) {}; 6 | -------------------------------------------------------------------------------- /test/resources/mockApplication/mockLambdaFunction/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mock-lambda-function", 3 | "description": "Part of the lambda-complex mock application.", 4 | "private": true, 5 | "version": "0.1.0", 6 | "author": { 7 | "name": "Reason", 8 | "email": "reason@exratione.com" 9 | }, 10 | "engines": { 11 | "node": ">= 0.10" 12 | }, 13 | "dependencies": {} 14 | } 15 | -------------------------------------------------------------------------------- /test/resources/uncaughtExceptionTest.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview A test case for handling uncaught exceptions. 3 | * 4 | * Load this in its own process after the mock application has been built. 5 | */ 6 | 7 | // Core. 8 | var path = require('path'); 9 | 10 | // Local. 11 | var resources = require('./index'); 12 | var applicationConfig = require('./mockApplication/applicationConfig'); 13 | 14 | // Get the various test globals set up. 15 | require('../mochaInit'); 16 | 17 | var invocationPackageDir = path.resolve( 18 | resources.getScratchDirectory(), 19 | applicationConfig.name, 20 | 'invocation' 21 | ); 22 | 23 | var invocationOriginalIndex = require(path.resolve( 24 | invocationPackageDir, 25 | '_index' 26 | )); 27 | 28 | // Since we're loading this here, it will attempt to catch any exception that 29 | // happens due to errors in this test code. That will still effectively fail 30 | // the test, but the errors may be misleading. 31 | var invocationIndex = require(path.resolve( 32 | invocationPackageDir, 33 | 'index' 34 | )); 35 | 36 | // Make the original handler throw. 37 | sinon.stub(invocationOriginalIndex, 'handler').throws(); 38 | 39 | // Stub everything relevant to this experiment in the wrapper. 40 | sinon.stub(invocationIndex.lc, 'sendData').yields(); 41 | sinon.stub(invocationIndex.lc, 'deleteMessageFromInputQueue').yields(); 42 | sinon.stub(invocationIndex.lc.utilities, 'loadArnMap').yields( 43 | null, 44 | resources.getMockArnMap(applicationConfig) 45 | ); 46 | sinon.stub(invocationIndex.lc.utilities, 'incrementConcurrencyCount').yields(); 47 | sinon.stub(invocationIndex.lc.utilities, 'decrementConcurrencyCount').yields(); 48 | sinon.stub(invocationIndex.lc.utilities, 'receiveMessage').yields(null, { 49 | message: '{}', 50 | receiptHandle: 'receiptHandle' 51 | }); 52 | 53 | var event = {}; 54 | var context = { 55 | done: sinon.stub(), 56 | fail: sinon.stub(), 57 | getRemainingTimeInMillis: sinon.stub(), 58 | succeed: sinon.stub() 59 | }; 60 | 61 | setTimeout(function () { 62 | // Did the finalizing functions still get called? 63 | sinon.assert.calledOnce(invocationIndex.lc.sendData); 64 | sinon.assert.calledOnce(invocationIndex.lc.utilities.decrementConcurrencyCount); 65 | sinon.assert.calledWith( 66 | context.fail, 67 | sinon.match.instanceOf(Error) 68 | ); 69 | 70 | // This is useful for pattern matching in stdout. It will only appear if 71 | // everything worked according to plan - errors will derail the flow of 72 | // execution before it gets to this point. 73 | console.info('SUCCESS'); 74 | }, 20); 75 | 76 | // This should throw, but trigger the actions that are checked above. 77 | invocationIndex.handler(event, context); 78 | -------------------------------------------------------------------------------- /test/tasks/build.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Tests for lib/grunt/tasks/build. 3 | */ 4 | 5 | // Local. 6 | var build = require('../../tasks/build') 7 | var gruntCommon = require('../../lib/grunt/common'); 8 | var index = require('../../index'); 9 | 10 | describe('lib/grunt/tasks/build', function () { 11 | var callback; 12 | var config; 13 | var grunt; 14 | var sandbox; 15 | var taskFn; 16 | var taskFnContext; 17 | 18 | beforeEach(function () { 19 | sandbox = sinon.sandbox.create(); 20 | 21 | callback = sandbox.stub(); 22 | grunt = { 23 | registerTask: function (name, description, fn) { 24 | taskFn = fn; 25 | } 26 | }; 27 | taskFn = undefined; 28 | taskFnContext = { 29 | async: function () { 30 | return callback; 31 | } 32 | }; 33 | 34 | sandbox.stub(index, 'build').yields(); 35 | sandbox.stub(gruntCommon, 'getConfigurationFromOptionOrFail').returns(config); 36 | sandbox.spy(grunt, 'registerTask'); 37 | sandbox.spy(taskFnContext, 'async'); 38 | 39 | // This should wind up getting taskFn assigned. 40 | build(grunt); 41 | 42 | sinon.assert.calledWith( 43 | grunt.registerTask, 44 | 'lambda-complex-build', 45 | 'Build a Lambda Complex application.', 46 | taskFn 47 | ); 48 | expect(taskFn).to.be.instanceOf(Function); 49 | }); 50 | 51 | afterEach(function () { 52 | sandbox.restore(); 53 | }); 54 | 55 | it('task function functions as expected', function (done) { 56 | taskFn.call(taskFnContext); 57 | 58 | setTimeout(function () { 59 | sinon.assert.calledWith(taskFnContext.async); 60 | sinon.assert.calledWith( 61 | index.build, 62 | config, 63 | sinon.match.func 64 | ); 65 | sinon.assert.calledWith(callback); 66 | 67 | done(); 68 | }, 10); 69 | }); 70 | }); 71 | -------------------------------------------------------------------------------- /test/tasks/deploy.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileOverview Tests for lib/grunt/tasks/build. 3 | */ 4 | 5 | // Local. 6 | var deploy = require('../../tasks/deploy') 7 | var gruntCommon = require('../../lib/grunt/common'); 8 | var index = require('../../index'); 9 | 10 | describe('lib/grunt/tasks/build', function () { 11 | var callback; 12 | var config; 13 | var grunt; 14 | var sandbox; 15 | var taskFn; 16 | var taskFnContext; 17 | 18 | beforeEach(function () { 19 | sandbox = sinon.sandbox.create(); 20 | 21 | callback = sandbox.stub(); 22 | grunt = { 23 | registerTask: function (name, description, fn) { 24 | taskFn = fn; 25 | } 26 | }; 27 | taskFn = undefined; 28 | taskFnContext = { 29 | async: function () { 30 | return callback; 31 | } 32 | }; 33 | 34 | sandbox.stub(index, 'deploy').yields(); 35 | sandbox.stub(gruntCommon, 'getConfigurationFromOptionOrFail').returns(config); 36 | sandbox.spy(grunt, 'registerTask'); 37 | sandbox.spy(taskFnContext, 'async'); 38 | 39 | // This should wind up getting taskFn assigned. 40 | deploy(grunt); 41 | 42 | sinon.assert.calledWith( 43 | grunt.registerTask, 44 | 'lambda-complex-deploy', 45 | 'Build and deploy a Lambda Complex application as a CloudFormation stack.', 46 | taskFn 47 | ); 48 | expect(taskFn).to.be.instanceOf(Function); 49 | }); 50 | 51 | afterEach(function () { 52 | sandbox.restore(); 53 | }); 54 | 55 | it('task function functions as expected', function (done) { 56 | taskFn.call(taskFnContext); 57 | 58 | setTimeout(function () { 59 | sinon.assert.calledWith(taskFnContext.async); 60 | sinon.assert.calledWith( 61 | index.deploy, 62 | config, 63 | sinon.match.func 64 | ); 65 | sinon.assert.calledWith(callback); 66 | 67 | done(); 68 | }, 10); 69 | }); 70 | }); 71 | --------------------------------------------------------------------------------