├── .gitignore ├── .npmignore ├── .travis.yml ├── CHANGELOG ├── Gruntfile.js ├── LICENSE-2.0.txt ├── ProcessKinesisRecords.js ├── README.md ├── Vagrantfile ├── dist └── .gitignore ├── docs └── dynamodb-table-image.png ├── package.json ├── resources └── lambda-admin.template ├── tasks ├── associateStream.js ├── cloudformation.js ├── deployLambda.js ├── dynamo.js ├── generateEvents.js ├── iam.js ├── kinesis.js └── packaging.js └── vagrant ├── .gitignore ├── ansible.hosts ├── peru.yaml ├── up.bash ├── up.guidance └── up.playbooks /.gitignore: -------------------------------------------------------------------------------- 1 | # Only project- and language-specific ignores in here. Use global .gitignore for editors etc 2 | 3 | # Vagrant 4 | .vagrant/ 5 | 6 | # node.js 7 | lib-cov 8 | *.seed 9 | *.log 10 | *.csv 11 | *.dat 12 | *.out 13 | *.pid 14 | *.gz 15 | 16 | pids 17 | logs 18 | results 19 | 20 | npm-debug.log 21 | node_modules 22 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | Gruntfile.js 2 | dist 3 | *.iml 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - :0.12.5 4 | 5 | before_install: npm install -g grunt-cli 6 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | Version 0.1.0 (2015-07-10) 2 | -------------------------- 3 | Initial release 4 | -------------------------------------------------------------------------------- /Gruntfile.js: -------------------------------------------------------------------------------- 1 | var grunt = require('grunt'); 2 | var async = require('async'); 3 | 4 | grunt.initConfig({ 5 | dynamo: { 6 | default: { 7 | function: 'dynamo' 8 | } 9 | }, 10 | createRole: { 11 | default: { 12 | function: 'createRole' 13 | } 14 | }, 15 | kinesis: { 16 | default: { 17 | function: 'kinesis' 18 | } 19 | }, 20 | attachRole: { 21 | default: { 22 | function: 'attachRole' 23 | } 24 | }, 25 | packaging: { 26 | default: { 27 | function: 'packaging' 28 | } 29 | }, 30 | deployLambda: { 31 | default: { 32 | function: 'deployLambda' 33 | } 34 | }, 35 | associateStream: { 36 | default: { 37 | function: 'associateStream' 38 | } 39 | }, 40 | generateEvents: { 41 | default: { 42 | function: 'generateEvents' 43 | } 44 | } 45 | }); 46 | 47 | grunt.loadTasks('tasks'); 48 | grunt.registerTask('init', ['dynamo','createRole','kinesis']); 49 | grunt.registerTask('role', ['attachRole','packaging']); 50 | grunt.registerTask('deploy', ['deployLambda']); 51 | grunt.registerTask('connect', ['associateStream']); 52 | grunt.registerTask('events', ['generateEvents']); 53 | 54 | 55 | -------------------------------------------------------------------------------- /LICENSE-2.0.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /ProcessKinesisRecords.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | 14 | console.log('Loading ProcessKinesisRecordsDynamoDB function'); 15 | 16 | var aws = require('aws-sdk'); 17 | var _ = require('lodash'); 18 | var table = new aws.DynamoDB({params: {TableName: 'my-table'}}); 19 | 20 | // first record processing function called by export.handler 21 | exports.kinesisHandler = function(records, context) { 22 | 23 | // process kinesis records 24 | var data = records 25 | .map(function(record) { 26 | return new Buffer(record.kinesis.data, 'base64').toString('utf8'); 27 | }); 28 | // downsample aggregate 29 | var aggData = _.chain(data) 30 | .map(aggregateData) 31 | .groupBy(aggData, function(singleRecord) { return singleRecord.Timestamp }) 32 | .sortBy('Timestamp') 33 | .value(); 34 | // count events 35 | var countedObject = _.chain(aggData[0]) 36 | .map(function(item) { 37 | return item.Timestamp + ' ' + item.EventType; 38 | }) 39 | .countBy(_.identity) 40 | .value(); 41 | // write to dynamodb using strings _NOT_ATOMIC 42 | _.forEach(countedObject, function(count, keys) { 43 | var params = transformData(keys.split(" ")[0], keys.split(" ")[1], count); 44 | updateDynamoDB(params, function () { 45 | context.done(null, 'Added to DynamoDB'); 46 | }); 47 | }); 48 | 49 | ///////////////////////////////helper functions //////////////////////////////// 50 | 51 | // update item to DynamoDB 52 | function updateDynamoDB (params, callback) { 53 | table.updateItem(params, function (err, data) { 54 | if (err) console.log(err, err.stack); 55 | else console.log(data); 56 | callback(); 57 | }); 58 | } 59 | 60 | // creating records objects to sort 61 | function aggregateData(payload) { 62 | var datum = JSON.parse(payload); 63 | var timestampItem = datum.timestamp; 64 | var typeItem = datum.type; 65 | var parsedDate = new Date(timestampItem); 66 | var singleRecord = { 67 | 'Timestamp': downsample(parsedDate), 68 | 'EventType': typeItem, 69 | 'CreatedAt': timestampItem, 70 | 'UpdatedAt': new Date().toISOString(), 71 | 'Count': '1' 72 | } 73 | return singleRecord; 74 | } 75 | 76 | // create records for insert into dynamodb 77 | function transformData(itemTimestamp, itemEventType, count) { 78 | 79 | var params = { 80 | Key: { 81 | 'Timestamp': {'S': itemTimestamp}, 82 | 'EventType': {'S': itemEventType} 83 | }, 84 | AttributeUpdates: { 85 | 'CreatedAt': {'Value': {'S': new Date().toISOString() },'Action':'PUT'}, 86 | 'UpdatedAt': {'Value': {'S': new Date().toISOString() },'Action':'PUT'}, 87 | 'Count': {'Value': {'N': count.toString() },'Action':'ADD'} 88 | } 89 | } 90 | return params; 91 | } 92 | 93 | // manual check of UTC build zero adder 94 | function pad(number) { 95 | var r = String(number); 96 | if ( r.length === 1 ) { 97 | r = '0' + r; 98 | } 99 | return r; 100 | } 101 | 102 | // downsample function for creating metadata 103 | function downsample(dateObject) { 104 | return dateObject.getUTCFullYear() 105 | + '-' + pad( dateObject.getUTCMonth() + 1 ) 106 | + '-' + pad( dateObject.getUTCDate() ) 107 | + 'T' + pad( dateObject.getUTCHours() ) 108 | + ':' + pad( dateObject.getUTCMinutes() ) 109 | + ':' + '00.000'; 110 | } 111 | context.done(); 112 | }; 113 | 114 | // main function 115 | exports.handler = function(event, context) { 116 | var record = event.Records[0]; 117 | if (record.kinesis) { 118 | exports.kinesisHandler(event.Records, context); 119 | } 120 | }; 121 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AWS Lambda Node.js Example Project 2 | 3 | **DEPRECATED**: This project is no longer maintained. If you wish to process a Kinesis stream of Snowplow events using an AWS Lambda application, we recommend using the [Snowplow JavaScript and TypeScript Analytics SDK][analytics-sdk]. 4 | 5 | [![Build Status][travis-image]][travis] [![Release][release-image]][releases] [![License][license-image]][license] 6 | 7 | ## Introduction 8 | 9 | This is an example [AWS Lambda][aws-lambda] application for processing a [Kinesis][aws-kinesis] stream of events ([introductory blog post][blog-post]). It reads the stream of simple JSON events generated by our event generator. Our AWS Lambda function aggregates and buckets events and stores them in [DynamoDB][aws-dynamodb]. 10 | 11 | This was built by the Data Science team at [Snowplow Analytics][snowplow], who use AWS Lambda in their projects. 12 | 13 | **Running this requires an Amazon AWS account, and will incur charges.** 14 | 15 | _See also:_ [Spark Streaming Example Project][spark-streaming-example-project] | [Spark Example Project][spark-example-project] 16 | 17 | ## Overview 18 | 19 | We have implemented a super-simple analytics-on-write stream processing job using AWS Lambda. Our AWS Lambda function, written in JavaScript, reads a Kinesis stream containing events in a JSON format: 20 | 21 | ```json 22 | { 23 | "timestamp": "2015-06-05T12:54:43.064528", 24 | "type": "Green", 25 | "id": "4ec80fb1-0963-4e35-8f54-ce760499d974" 26 | } 27 | ``` 28 | 29 | Our job counts the events by `type` and aggregates these counts into 1 minute buckets. The job then takes these aggregates and saves them into a table in DynamoDB: 30 | 31 | ![dynamodb-table-image][dynamodb-table-image] 32 | 33 | ## Developer Quickstart 34 | 35 | Assuming git, [Vagrant][vagrant-install] and [VirtualBox][virtualbox-install] installed: 36 | 37 | ```bash 38 | host$ git clone https://github.com/snowplow/aws-lambda-nodejs-example-project.git 39 | host$ cd aws-lambda-example-project 40 | host$ vagrant up && vagrant ssh 41 | guest$ cd /vagrant 42 | guest# npm install grunt 43 | guest$ npm install 44 | guest$ grunt --help 45 | ``` 46 | 47 | ## Tutorial 48 | 49 | You can follow along in [the release blog post][blog-post] to get the project up and running yourself. 50 | 51 | The following steps assume that you are running inside Vagrant, as per the Developer Quickstart above. 52 | 53 | ### 1. Setting up AWS 54 | 55 | First we need to configure a default AWS profile: 56 | 57 | ```bash 58 | $ aws configure 59 | AWS Access Key ID [None]: ... 60 | AWS Secret Access Key [None]: ... 61 | Default region name [None]: us-east-1 62 | Default output format [None]: json 63 | ``` 64 | 65 | Now we can create our DynamoDB table, Kinesis stream, and IAM role. We will be using [CloudFormation](http://aws.amazon.com/cloudformation) to make our new role. Using Grunt, we can create all like so: 66 | 67 | ```bash 68 | $ grunt init 69 | Running "dynamo:default" (dynamo) task 70 | { TableDescription: 71 | { AttributeDefinitions: [ [Object], [Object], [Object] ], 72 | CreationDateTime: Sun Jun 28 2015 13:04:02 GMT-0700 (PDT), 73 | ItemCount: 0, 74 | KeySchema: [ [Object], [Object] ], 75 | LocalSecondaryIndexes: [ [Object] ], 76 | ProvisionedThroughput: 77 | { NumberOfDecreasesToday: 0, 78 | ReadCapacityUnits: 20, 79 | WriteCapacityUnits: 20 }, 80 | TableName: 'my-table', 81 | TableSizeBytes: 0, 82 | TableStatus: 'CREATING' } } 83 | 84 | Running "createRole:default" (createRole) task 85 | { ResponseMetadata: { RequestId: 'd29asdff0-1dd0-11e5-984e-35a24700edda' }, 86 | StackId: 'arn:aws:cloudformation:us-east-1:84asdf429716:stack/kinesisDynamo/d2af8730-1dd0-11e5-854a-50d5017c76e0' } 87 | 88 | Running "kinesis:default" (kinesis) task 89 | {} 90 | 91 | Done, without errors. 92 | ``` 93 | 94 | ### 2. Connect AWS Lambda service with the new role and building the project 95 | 96 | Wait a minute to ensure our IAM service role gets created. Now we connect the new service role to access Kinesis, CloudWatch, Lambda, and DynamoDB. We will attach an admin policy to the lambda exec role to easily access the services. Using Grunt, our AWS Lambda function gets assembled into a zip file for upload to the AWS Lambda service. Once it's zipped, we attach a service role to it: 97 | 98 | ```bash 99 | $ grunt role 100 | Running "attachRole:default" (attachRole) task 101 | { ResponseMetadata: { RequestId: '36ac7877-1dca-11e5-b439-d1da60d122be' } } 102 | 103 | Running "packaging:default" (packaging) task 104 | aws-lambda-example-project@0.1.0 ../../../../var/folders/3t/7nlz8rzs2mq5fg_sf3x4j7_m0000gn/T/1435519004662.0046/node_modules/aws-lambda-example-project 105 | ├── rimraf@2.2.8 106 | ├── async@0.9.2 107 | ├── temporary@0.0.8 (package@1.0.1) 108 | ├── mkdirp@0.5.1 (minimist@0.0.8) 109 | ├── glob@4.3.5 (inherits@2.0.1, once@1.3.2, inflight@1.0.4, minimatch@2.0.8) 110 | ├── lodash@3.9.3 111 | ├── archiver@0.14.4 (buffer-crc32@0.2.5, lazystream@0.1.0, readable-stream@1.0.33, tar-stream@1.1.5, zip-stream@0.5.2, lodash@3.2.0) 112 | └── aws-sdk@2.1.23 (xmlbuilder@0.4.2, xml2js@0.2.8, sax@0.5.3) 113 | Created package at dist/aws-lambda-example-project_0-1-0_latest.zip 114 | ... 115 | ``` 116 | 117 | ### 3. Deploy zip file to AWS Lambda service and connect Kinesis to Lambda 118 | 119 | In deploy this project to Lambda with the `grunt deploy` command: 120 | 121 | ```bash 122 | $ grunt deploy 123 | Running "deployLambda:default" (deployLambda) task 124 | Trying to create AWS Lambda Function... 125 | Created AWS Lambda Function... 126 | ``` 127 | 128 | ### 4. Connect Kinesis to Lambda 129 | 130 | The final step to getting this projected ready to start processing events is to associate our Kinesis stream to the Lambda function with this command: 131 | 132 | ```bash 133 | $ grunt connect 134 | Running "associateStream:default" (associateStream) task 135 | arn:aws:kinesis:us-east-1:844709429716:stream/my-stream 136 | { BatchSize: 100, 137 | EventSourceArn: 'arn:aws:kinesis:us-east-1:2349429716:stream/my-stream', 138 | FunctionArn: 'arn:aws:lambda:us-east-1:2349429716:function:ProcessKinesisRecordsDynamo', 139 | LastModified: Sun Jun 28 2015 12:38:37 GMT-0700 (PDT), 140 | LastProcessingResult: 'No records processed', 141 | State: 'Creating', 142 | StateTransitionReason: 'User action', 143 | UUID: 'f4efc-fe72-4337-9907-89d4e64c' } 144 | 145 | Done, without errors. 146 | ``` 147 | 148 | ### 5. Sending events to Kinesis 149 | 150 | We need to start sending events to our new Kinesis stream. We have created a helper method to do this - run the below and leave it running in a tab: 151 | 152 | ```bash 153 | $ grunt events 154 | Writing Kineis Event: {"timestamp":"2015-06-29T20:12:21.625Z","type":"Red"} 155 | { SequenceNumber: '49552099319153062484931809176874704852938278389141209090', 156 | ShardId: 'shardId-000000000000' } 157 | Writing Kineis Event: {"timestamp":"2015-06-29T20:12:22.200Z","type":"Red"} 158 | { SequenceNumber: '49552099319153062484931809176875913778757893018315915266', 159 | ShardId: 'shardId-000000000000' } 160 | Writing Kineis Event: {"timestamp":"2015-06-29T20:12:22.708Z","type":"Green"} 161 | { SequenceNumber: '49552099319153062484931809176877122704577507716210098178', 162 | ShardId: 'shardId-000000000000' } 163 | ... 164 | ``` 165 | 166 | ### 6. Monitoring your job 167 | 168 | First head over to the AWS Lambda service console, then review the logs in CloudWatch. 169 | 170 | Finally, let's check the data in our DynamoDB table. Make sure you are in the correct AWS region, then click on `my-table` and hit the `Explore Table` button: 171 | 172 | ![dynamodb-table-image][dynamodb-table-image] 173 | 174 | For each **BucketStart** and **EventType** pair, we see a **Count**, plus some **CreatedAt** and **UpdatedAt** metadata for debugging purposes. Our bucket size is 1 minute, and we have 5 discrete event types, hence the matrix of rows that we see. 175 | 176 | ## Roadmap 177 | 178 | * Various improvements for the [0.2.0 release][020-milestone] 179 | * Expanding our analytics-on-write thinking into our new [Icebucket][icebucket] project 180 | 181 | ## Credits 182 | 183 | * [Tim Bell] [tim-b] for his blog post [Writing Functions for AWS Lambda Using NPM and Grunt][tim-b-post] 184 | * Ian Meyers and his [Amazon-Kinesis-Aggregators-Project][amazon-kinesis-aggregators], a true inspiration for streaming analytics-on-write 185 | 186 | ## Copyright and license 187 | 188 | AWS Lambda Example Project is copyright 2015 Snowplow Analytics Ltd. 189 | 190 | Licensed under the **[Apache License, Version 2.0][license]** (the "License"); 191 | you may not use this software except in compliance with the License. 192 | 193 | Unless required by applicable law or agreed to in writing, software 194 | distributed under the License is distributed on an "AS IS" BASIS, 195 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 196 | See the License for the specific language governing permissions and 197 | limitations under the License. 198 | 199 | [analytics-sdk]: https://github.com/snowplow-incubator/snowplow-js-analytics-sdk 200 | [travis]: https://travis-ci.org/snowplow/aws-lambda-nodejs-example-project 201 | [travis-image]: https://travis-ci.org/snowplow/aws-lambda-nodejs-example-project.png?branch=master 202 | [license-image]: https://img.shields.io/badge/license-Apache--2-blue.svg?style=flat 203 | [license]: https://www.apache.org/licenses/LICENSE-2.0 204 | [release-image]: https://img.shields.io/badge/release-0.1.0-blue.svg?style=flat 205 | [releases]: https://github.com/snowplow/aws-lambda-nodejs-example-project/releases 206 | [grunt-image]: https://cdn.gruntjs.com/builtwith.png 207 | 208 | [spark-example-project]: https://github.com/snowplow/spark-example-project 209 | [spark-streaming-example-project]: https://github.com/snowplow/spark-streaming-example-project 210 | 211 | [vagrant-install]: http://docs.vagrantup.com/v2/installation/index.html 212 | [virtualbox-install]: https://www.virtualbox.org/wiki/Downloads 213 | 214 | [blog-post]: http://snowplowanalytics.com/blog/2015/07/11/aws-lambda-nodejs-example-project-0.1.0-released/ 215 | [020-milestone]: https://github.com/snowplow/aws-lambda-nodejs-example-project/milestones/Version%200.2.0 216 | [dynamodb-table-image]: /docs/dynamodb-table-image.png?raw=true 217 | 218 | [aws-lambda]: http://aws.amazon.com/lambda/ 219 | [aws-kinesis]: http://aws.amazon.com/kinesis/ 220 | [aws-dynamodb]: http://aws.amazon.com/dynamodb 221 | [vagrant-install]: http://docs.vagrantup.com/v2/installation/index.html 222 | [virtualbox-install]: https://www.virtualbox.org/wiki/Downloads 223 | [tim-b]: https://github.com/Tim-B 224 | [tim-b-post]: http://hipsterdevblog.com/blog/2014/12/07/writing-functions-for-aws-lambda-using-npm-and-grunt/ 225 | [amazon-kinesis-aggregators]: https://github.com/awslabs/amazon-kinesis-aggregators 226 | 227 | [snowplow]: http://snowplowanalytics.com 228 | [icebucket]: https://github.com/snowplow/icebucket 229 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | Vagrant.configure("2") do |config| 2 | 3 | config.vm.box = "ubuntu/trusty64" 4 | config.vm.hostname = "aws-lambda-example-project" 5 | config.ssh.forward_agent = true 6 | 7 | config.vm.provider :virtualbox do |vb| 8 | vb.name = Dir.pwd().split("/")[-1] + "-" + Time.now.to_f.to_i.to_s 9 | vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] 10 | vb.customize [ "guestproperty", "set", :id, "--timesync-threshold", 10000 ] 11 | # Don't need much memory for Node.js 12 | vb.memory = 1024 13 | end 14 | 15 | config.vm.provision :shell do |sh| 16 | sh.path = "vagrant/up.bash" 17 | end 18 | 19 | end 20 | -------------------------------------------------------------------------------- /dist/.gitignore: -------------------------------------------------------------------------------- 1 | # Nothing in dist gets saved to version control 2 | * 3 | !.gitignore 4 | -------------------------------------------------------------------------------- /docs/dynamodb-table-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snowplow-archive/aws-lambda-nodejs-example-project/9191e038474daacc5f8b012b4cf192c644dcd8c4/docs/dynamodb-table-image.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aws-lambda-example-project", 3 | "version": "0.1.0", 4 | "description": "Example AWS Lambda application for processing a Kinesis stream of events and aggregating counts to DynamoDB", 5 | "private": "true", 6 | "devDependencies": { 7 | "grunt": "^0.4.5", 8 | "adm-zip": "~0.4.4", 9 | "npm": "^2.10.0" 10 | }, 11 | "homepage": "https://github.com/snowplow/aws-lambda-example-project/", 12 | "keywords": [ 13 | "kinesis", 14 | "dynamodb" 15 | ], 16 | "author": "Alexander Dean ", 17 | "license": "Apache-2", 18 | "main": "ProcessKinesisRecords.js", 19 | "repository": { 20 | "type": "git", 21 | "url": "https://github.com:snowplow/aws-lambda-example-project.git" 22 | }, 23 | "dependencies": { 24 | "async": "^0.9.0", 25 | "lodash": "^3.9.3", 26 | "temporary": "~0.0.8", 27 | "archiver": "~0.14.4", 28 | "mkdirp": "~0.5.0", 29 | "rimraf": "~2.2.8", 30 | "glob": "~4.3.0", 31 | "aws-sdk": "2.1.23" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /resources/lambda-admin.template: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Resources": { 4 | "LambdaExecRole": { 5 | "Type": "AWS::IAM::Role", 6 | "Properties": { 7 | "AssumeRolePolicyDocument": { 8 | "Statement": [{ 9 | "Effect": "Allow", 10 | "Principal": { 11 | "Service": [ 12 | "lambda.amazonaws.com" 13 | ] 14 | }, 15 | "Action": "sts:AssumeRole" 16 | }] 17 | }, 18 | "Policies": [{ 19 | "PolicyName": "LambdaExecute", 20 | "PolicyDocument": { 21 | "Statement": [{ 22 | "Effect": "Allow", 23 | "Action": [ 24 | "logs:*" 25 | ], 26 | "Resource": [ 27 | "arn:aws:logs:*:*:*" 28 | ] 29 | }] 30 | } 31 | }] 32 | } 33 | }, 34 | 35 | "LambdaInvokeRole": { 36 | "Type": "AWS::IAM::Role", 37 | "Properties": { 38 | "AssumeRolePolicyDocument": { 39 | "Statement": [{ 40 | "Effect": "Allow", 41 | "Principal": { 42 | "Service": [ 43 | "lambda.amazonaws.com" 44 | ] 45 | }, 46 | "Action": "sts:AssumeRole" 47 | }] 48 | }, 49 | "Policies": [{ 50 | "PolicyName": "LambdaInvoke", 51 | "PolicyDocument": { 52 | "Statement": [{ 53 | "Effect": "Allow", 54 | "Action": [ 55 | "lambda:InvokeFunction", 56 | "kinesis:GetRecords", 57 | "kinesis:GetShardIterator", 58 | "kinesis:DescribeStream", 59 | "kinesis:ListStreams" 60 | ], 61 | "Resource": [ 62 | "*" 63 | ] 64 | }] 65 | } 66 | }] 67 | } 68 | } 69 | }, 70 | 71 | "Outputs" : { 72 | "ExecutionRole" : { 73 | "Description" : "An IAM role that Lambda can assume 'adminuser' to access your AWS resources.", 74 | "Value" : { "Fn::GetAtt" : ["LambdaExecRole", "Arn"] } 75 | }, 76 | 77 | "PushInvocationRole" : { 78 | "Description" : "An IAM role that Amazon S3 can assume 'adminuser' to invoke your Lambda function.", 79 | "Value" : { "Fn::GetAtt" : ["LambdaInvokeRole", "Arn"] } 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /tasks/associateStream.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var aws = require('aws-sdk'); 4 | var async = require('async'); 5 | 6 | module.exports = function (grunt) { 7 | 8 | var DESC = 'describe kinesis stream called my-stream'; 9 | var params = { 10 | StreamName: 'my-stream', 11 | }; 12 | var DEFAULTS = { 13 | endpoint: "kinesis.us-east-1.amazonaws.com", 14 | lambdaEndpoint: "lambda.us-east-1.amazonaws.com", 15 | region: "us-east-1", 16 | params: params, 17 | streamARN: null 18 | }; 19 | 20 | grunt.registerMultiTask('associateStream', DESC, function () { 21 | 22 | var done = this.async(); 23 | var opts = this.options(DEFAULTS); 24 | 25 | var credentials = new aws.SharedIniFileCredentials({profile: 'default'}); 26 | aws.config.credentials = credentials; 27 | aws.config.apiVersions = { 28 | kinesis: '2013-12-02', 29 | }; 30 | 31 | var kinesis = new aws.Kinesis(); 32 | kinesis.config.region = DEFAULTS.region; 33 | kinesis.config.endpoint = DEFAULTS.endpoint; 34 | kinesis.region = DEFAULTS.region; 35 | kinesis.endpoint = DEFAULTS.endpoint; 36 | 37 | var lambda = new aws.Lambda(); 38 | lambda.config.region = DEFAULTS.region; 39 | lambda.config.endpoint = DEFAULTS.lambdaEndpoint; 40 | lambda.region = DEFAULTS.region; 41 | lambda.endpoint = DEFAULTS.lambdaEndpoint; 42 | 43 | 44 | var subtasks = []; 45 | subtasks.push(describeStream); 46 | subtasks.push(associateStream); 47 | async.series(subtasks, done); 48 | 49 | function describeStream(callback) { 50 | var params = { 51 | 'StreamName': 'my-stream' 52 | }; 53 | kinesis.describeStream(params, function(err, data) { 54 | if (err) console.log(err, err.stack); 55 | else { 56 | console.log(data.StreamDescription.StreamARN); 57 | DEFAULTS.streamARN = data.StreamDescription.StreamARN; 58 | } 59 | callback(err); 60 | }); 61 | } 62 | 63 | 64 | function associateStream(callback) { 65 | var params = { 66 | EventSourceArn: DEFAULTS.streamARN, 67 | FunctionName: 'ProcessKinesisRecordsDynamo', 68 | StartingPosition: 'TRIM_HORIZON', 69 | BatchSize: 100 70 | }; 71 | lambda.createEventSourceMapping(params, function(err, data) { 72 | if (err) console.log(err, err.stack); 73 | else { 74 | console.log(data); 75 | } 76 | callback(err); 77 | }); 78 | } 79 | 80 | 81 | function taskComplete(err) { 82 | if(err) { 83 | grunt.fail.warn(err); 84 | return done(false); 85 | } 86 | } 87 | }); 88 | } 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /tasks/cloudformation.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var aws = require('aws-sdk'); 4 | var async = require('async'); 5 | 6 | 7 | module.exports = function (grunt) { 8 | 9 | var DESC = 'creates AWS lambda admin role using CloudFormation'; 10 | var DEFAULTS = { 11 | endpoint: "cloudformation.us-east-1.amazonaws.com", 12 | region: "us-east-1", 13 | desc:'creates AWS lambda admin role using CloudFormation' 14 | }; 15 | 16 | grunt.registerMultiTask('createRole', DEFAULTS.desc, function () { 17 | 18 | var done = this.async(); 19 | var opts = this.options(DEFAULTS); 20 | 21 | var credentials = new aws.SharedIniFileCredentials({profile: 'default'}); 22 | aws.config.credentials = credentials; 23 | var cloudformation = new aws.CloudFormation({apiVersion: '2010-05-15'}); 24 | cloudformation.region = DEFAULTS.region; 25 | cloudformation.endpoint = DEFAULTS.endpoint; 26 | cloudformation.config.region = DEFAULTS.region; 27 | cloudformation.config.endpoint = DEFAULTS.endpoint; 28 | 29 | var subtasks = []; 30 | subtasks.push(createRole); 31 | async.series(subtasks, done); 32 | 33 | function createRole(callback) { 34 | 35 | var params = { 36 | StackName: 'kinesisDynamo', 37 | Capabilities: [ 38 | 'CAPABILITY_IAM', 39 | ], 40 | TemplateURL: 'https://snowplow-hosted-assets.s3.amazonaws.com/third-party/aws-lambda/lambda-admin.template' 41 | }; 42 | 43 | cloudformation.createStack(params, function (err, data) { 44 | if (err) console.log(err, err.stack); 45 | else console.log(data); 46 | callback(err); 47 | }); 48 | } 49 | }); 50 | } 51 | 52 | -------------------------------------------------------------------------------- /tasks/deployLambda.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'); 2 | var AWS = require('aws-sdk'); 3 | var extend = require('util')._extend; 4 | var async = require('async'); 5 | 6 | 7 | module.exports = function (grunt) { 8 | 9 | var DESC = 'Uploads a package to lambda'; 10 | 11 | grunt.registerMultiTask('deployLambda', DESC, function () { 12 | 13 | var DEFAULTS = { 14 | endpoint: 'iam.amazonaws.com' 15 | }; 16 | var credentials = new AWS.SharedIniFileCredentials({profile: 'default'}); 17 | AWS.config.credentials = credentials; 18 | 19 | var done = this.async(); 20 | 21 | AWS.config.update({region: 'us-east-1'}); 22 | var lambda = new AWS.Lambda({ 23 | apiVersion: '2015-03-31' 24 | }); 25 | 26 | var iam = new AWS.IAM({apiVersion: '2010-05-08'}); 27 | iam.config.endpoint = DEFAULTS.endpoint; 28 | iam.endpoint = DEFAULTS.endpoint; 29 | 30 | // main 31 | var subtasks = []; 32 | subtasks.push(listRoles); 33 | subtasks.push(createFunction); 34 | async.series(subtasks, done); 35 | 36 | function listRoles(callback) { 37 | paramsRole = {} 38 | iam.listRoles(paramsRole, function(err, data) { 39 | if (err) { 40 | console.log(err, err.stack); 41 | } else { 42 | DEFAULTS['arnRoles'] = data.Roles; 43 | for (var i = 0; i < DEFAULTS.arnRoles.length; i++) { 44 | var lambdaRole = data.Roles[i].Arn; 45 | if (lambdaRole.indexOf("LambdaExecRole") > 38 && lambdaRole.indexOf("LambdaExecRole") < 48) { 46 | DEFAULTS['arn'] = data.Roles[i].Arn; 47 | console.log("Found"); 48 | console.log(DEFAULTS.arn); 49 | } else { 50 | console.log("Looking for ... kinesisDynamo-LambdaExecRole"); 51 | } 52 | } 53 | } 54 | callback(err); 55 | }); 56 | } 57 | 58 | 59 | function createFunction(callback) { 60 | 61 | var params = { 62 | FunctionName: 'ProcessKinesisRecordsDynamo', 63 | Handler: 'ProcessKinesisRecords.handler', 64 | Role: DEFAULTS.arn, 65 | Timeout: 3 66 | }; 67 | console.log("Polling for ARN"); 68 | console.log(DEFAULTS.arn); 69 | grunt.log.writeln('Trying to create AWS Lambda Function...'); 70 | 71 | fs.readFile('dist/aws-lambda-example-project_0-1-0_latest.zip', function(err, data) { 72 | 73 | if (err) { 74 | return callback('Error reading specified package "'+ 'dist/aws-lambda-example-project_0-1-0_latest.zip' + '"'); 75 | } 76 | 77 | params['Code'] = { ZipFile: data }; 78 | params['Runtime'] = "nodejs"; 79 | 80 | lambda.createFunction(params, function(err, data) { 81 | if (err) { 82 | var warning = 'Create function failed. ' 83 | warning += 'Check your iam:PassRole permissions.' 84 | callback(err); 85 | } else { 86 | grunt.log.writeln('Created AWS Lambda Function...'); 87 | } 88 | }); 89 | 90 | }); 91 | 92 | }; 93 | 94 | }); 95 | } 96 | 97 | 98 | -------------------------------------------------------------------------------- /tasks/dynamo.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var aws = require('aws-sdk'); 4 | var async = require('async'); 5 | 6 | module.exports = function (grunt) { 7 | 8 | var DESC = 'creates DynamoDB table called my-table'; 9 | var params = { 10 | "AttributeDefinitions": [ 11 | { 12 | "AttributeName": "Timestamp", 13 | "AttributeType": "S" 14 | }, 15 | { 16 | "AttributeName": "EventType", 17 | "AttributeType": "S" 18 | }, 19 | { 20 | "AttributeName": "CreatedAt", 21 | "AttributeType": "S" 22 | } 23 | ], 24 | "TableName": "my-table", 25 | "KeySchema": [ 26 | { 27 | "AttributeName": "Timestamp", 28 | "KeyType": "HASH" 29 | }, 30 | { 31 | "AttributeName": "EventType", 32 | "KeyType": "RANGE" 33 | } 34 | ], 35 | "LocalSecondaryIndexes": [ 36 | { 37 | "IndexName": "LastPostIndex", 38 | "KeySchema": [ 39 | { 40 | "AttributeName": "Timestamp", 41 | "KeyType": "HASH" 42 | }, 43 | { 44 | "AttributeName": "CreatedAt", 45 | "KeyType": "RANGE" 46 | } 47 | ], 48 | "Projection": { 49 | "ProjectionType": "KEYS_ONLY" 50 | } 51 | } 52 | ], 53 | "ProvisionedThroughput": { 54 | "ReadCapacityUnits": 20, 55 | "WriteCapacityUnits": 20 56 | } 57 | }; 58 | var DEFAULTS = { 59 | endpoint: "dynamodb.us-east-1.amazonaws.com", 60 | region: "us-east-1", 61 | params: params 62 | }; 63 | 64 | grunt.registerMultiTask('dynamo', DESC, function () { 65 | 66 | var done = this.async(); 67 | var opts = this.options(DEFAULTS); 68 | 69 | var credentials = new aws.SharedIniFileCredentials({profile: 'default'}); 70 | aws.config.credentials = credentials; 71 | var dynamodb = new aws.DynamoDB(); 72 | dynamodb.config.region = DEFAULTS.region; 73 | dynamodb.config.endpoint = DEFAULTS.endpoint; 74 | dynamodb.region = DEFAULTS.region; 75 | dynamodb.endpoint = DEFAULTS.endpoint; 76 | 77 | var subtasks = []; 78 | subtasks.push(createDynamoTable); 79 | async.series(subtasks, done); 80 | 81 | function createDynamoTable(callback) { 82 | 83 | var params = { 84 | "AttributeDefinitions": [ 85 | { 86 | "AttributeName": "Timestamp", 87 | "AttributeType": "S" 88 | }, 89 | { 90 | "AttributeName": "EventType", 91 | "AttributeType": "S" 92 | }, 93 | { 94 | "AttributeName": "CreatedAt", 95 | "AttributeType": "S" 96 | } 97 | ], 98 | "TableName": "my-table", 99 | "KeySchema": [ 100 | { 101 | "AttributeName": "Timestamp", 102 | "KeyType": "HASH" 103 | }, 104 | { 105 | "AttributeName": "EventType", 106 | "KeyType": "RANGE" 107 | } 108 | ], 109 | "LocalSecondaryIndexes": [ 110 | { 111 | "IndexName": "LastPostIndex", 112 | "KeySchema": [ 113 | { 114 | "AttributeName": "Timestamp", 115 | "KeyType": "HASH" 116 | }, 117 | { 118 | "AttributeName": "CreatedAt", 119 | "KeyType": "RANGE" 120 | } 121 | ], 122 | "Projection": { 123 | "ProjectionType": "KEYS_ONLY" 124 | } 125 | } 126 | ], 127 | "ProvisionedThroughput": { 128 | "ReadCapacityUnits": 20, 129 | "WriteCapacityUnits": 20 130 | } 131 | }; 132 | dynamodb.createTable(params, function (err, data) { 133 | if (err) console.log(err, err.stack); 134 | else console.log(data); 135 | callback(err); 136 | }); 137 | } 138 | 139 | function taskComplete(err) { 140 | if(err) { 141 | grunt.fail.warn(err); 142 | return done(false); 143 | } 144 | } 145 | }); 146 | } 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | -------------------------------------------------------------------------------- /tasks/generateEvents.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var aws = require('aws-sdk'); 4 | var async = require('async'); 5 | 6 | module.exports = function (grunt) { 7 | 8 | var DESC = 'generate simple events for kinesis stream called my-stream'; 9 | var DEFAULTS = { 10 | endpoint: "kinesis.us-east-1.amazonaws.com", 11 | region: "us-east-1" 12 | }; 13 | var STREAMNAME = 'my-stream'; 14 | var COLORS = ['Red','Orange','Yellow','Green','Blue']; 15 | 16 | grunt.registerMultiTask('generateEvents', DESC, function () { 17 | 18 | var done = this.async(); 19 | var opts = this.options(DEFAULTS); 20 | 21 | var credentials = new aws.SharedIniFileCredentials({profile: 'default'}); 22 | aws.config.credentials = credentials; 23 | aws.config.apiVersions = { 24 | kinesis: '2013-12-02', 25 | }; 26 | 27 | var kinesis = new aws.Kinesis(); 28 | kinesis.config.region = DEFAULTS.region; 29 | kinesis.config.endpoint = DEFAULTS.endpoint; 30 | kinesis.region = DEFAULTS.region; 31 | kinesis.endpoint = DEFAULTS.endpoint; 32 | 33 | var subtasks = []; 34 | for (var i=1; i < 10000; i += 1){ 35 | subtasks.push(writeToKinesis); 36 | } 37 | async.series(subtasks, done); 38 | 39 | 40 | function SimpleEvent() { 41 | this.timestamp = new Date().toISOString(); 42 | this.type = COLORS[Math.floor(Math.random() * COLORS.length)]; 43 | } 44 | 45 | function guid() { 46 | function s4() { 47 | return Math.floor((1 + Math.random()) * 0x10000).toString(16).substring(1); 48 | } 49 | return s4() + s4() + '-' + s4() + '-' + s4() + '-' + 50 | s4() + '-' + s4() + s4() + s4(); 51 | } 52 | 53 | function writeToKinesis(callback) { 54 | 55 | var instance = new SimpleEvent(); 56 | var data = JSON.stringify(instance); 57 | var partitionKey = 'pk-' + guid(); 58 | var recordParams = { 59 | Data: data, 60 | PartitionKey: partitionKey, 61 | StreamName: 'my-stream' 62 | }; 63 | console.log("Writing Kineis Event: " + data); 64 | kinesis.putRecord(recordParams, function(err, data) { 65 | if (err) console.log(err, err.stack); 66 | else console.log(data); 67 | callback(err); 68 | }); 69 | } 70 | 71 | 72 | function taskComplete(err) { 73 | if(err) { 74 | grunt.fail.warn(err); 75 | return done(false); 76 | } 77 | } 78 | }); 79 | } 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /tasks/iam.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var aws = require('aws-sdk'); 4 | var async = require('async'); 5 | 6 | module.exports = function (grunt) { 7 | 8 | var DESC = 'attaches ADMIN policy to newly created Lambda exec role'; 9 | var params = { 10 | 'PolicyArn': 'arn:aws:iam::aws:policy/AdministratorAccess', 11 | 'RoleName': null 12 | }; 13 | var DEFAULTS = { 14 | endpoint: 'iam.amazonaws.com', 15 | params: params 16 | }; 17 | 18 | grunt.registerMultiTask('attachRole', DESC, function () { 19 | 20 | var done = this.async(); 21 | var opts = this.options(DEFAULTS); 22 | 23 | var credentials = new aws.SharedIniFileCredentials({profile: 'default'}); 24 | aws.config.credentials = credentials; 25 | var iam = new aws.IAM({apiVersion: '2010-05-08'}); 26 | iam.config.endpoint = DEFAULTS.endpoint; 27 | iam.endpoint = DEFAULTS.endpoint; 28 | 29 | var subtasks = []; 30 | subtasks.push(listRoles); 31 | subtasks.push(attachRole); 32 | async.series(subtasks, done); 33 | 34 | function listRoles(callback) { 35 | params = {} 36 | iam.listRoles(params, function(err, data) { 37 | if (err) { 38 | console.log(err, err.stack); 39 | } else { 40 | opts.params['arnRoles'] = data.Roles; 41 | for (var i = 0; i < opts.params.arnRoles.length; i++) { 42 | //opts.params.RoleName = data.Roles[0].RoleName; 43 | var lambdaRole = data.Roles[i].RoleName; 44 | if (lambdaRole.indexOf("LambdaExecRole") > 0) { 45 | opts.params.RoleName = data.Roles[i].RoleName; 46 | console.log("Found"); 47 | console.log(opts.params.RoleName); 48 | } else { 49 | console.log("Looking for ... kinesisDynamo-LambdaExecRole"); 50 | } 51 | } 52 | callback(err); 53 | } 54 | }); 55 | } 56 | 57 | 58 | function attachRole(callback) { 59 | var params = { 60 | 'PolicyArn': 'arn:aws:iam::aws:policy/AdministratorAccess', 61 | 'RoleName': opts.params.RoleName 62 | }; 63 | iam.attachRolePolicy(params, function(err, data) { 64 | if (err) console.log(err, err.stack); 65 | else console.log(data); 66 | callback(err); 67 | }); 68 | } 69 | }); 70 | } 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /tasks/kinesis.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var aws = require('aws-sdk'); 4 | var async = require('async'); 5 | 6 | module.exports = function (grunt) { 7 | 8 | var DESC = 'creates kinesis stream called my-stream'; 9 | var params = { 10 | 'ShardCount': 1, 11 | 'StreamName': 'my-stream' 12 | }; 13 | var DEFAULTS = { 14 | endpoint: "kinesis.us-east-1.amazonaws.com", 15 | region: "us-east-1", 16 | params: params 17 | }; 18 | 19 | grunt.registerMultiTask('kinesis', DESC, function () { 20 | 21 | var done = this.async(); 22 | var opts = this.options(DEFAULTS); 23 | 24 | var credentials = new aws.SharedIniFileCredentials({profile: 'default'}); 25 | aws.config.credentials = credentials; 26 | aws.config.apiVersions = { 27 | kinesis: '2013-12-02', 28 | }; 29 | 30 | var kinesis = new aws.Kinesis(); 31 | kinesis.config.region = DEFAULTS.region; 32 | kinesis.config.endpoint = DEFAULTS.endpoint; 33 | kinesis.region = DEFAULTS.region; 34 | kinesis.endpoint = DEFAULTS.endpoint; 35 | 36 | var subtasks = []; 37 | subtasks.push(createStream); 38 | async.series(subtasks, done); 39 | 40 | function createStream(callback) { 41 | var params = { 42 | 'ShardCount': 1, 43 | 'StreamName': 'my-stream' 44 | }; 45 | kinesis.createStream(params, function(err, data) { 46 | if (err) console.log(err, err.stack); 47 | else console.log(data); 48 | callback(err); 49 | }); 50 | 51 | } 52 | 53 | function taskComplete(err) { 54 | if(err) { 55 | grunt.fail.warn(err); 56 | return done(false); 57 | } 58 | } 59 | }); 60 | } 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /tasks/packaging.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var async = require('async'); 4 | var path = require('path'); 5 | var npm = require('npm'); 6 | var tmp = require('temporary'); 7 | var archive = require('archiver'); 8 | var fs = require('fs'); 9 | var mkdirp = require('mkdirp'); 10 | var rimraf = require('rimraf'); 11 | 12 | module.exports = function (grunt) { 13 | 14 | var DESC = 'creates package for aws lambda service'; 15 | var DEFAULTS = { 16 | 'dist_folder': 'dist', 17 | 'include_time': true, 18 | 'package_folder': './' 19 | }; 20 | 21 | grunt.registerMultiTask('packaging', DESC, function () { 22 | 23 | var task = this; 24 | var opts = this.options(DEFAULTS); 25 | var done = this.async(); 26 | var pkg = grunt.file.readJSON(path.resolve(DEFAULTS.package_folder + '/package.json')); 27 | var dir = { path: './dist/' }; 28 | var now = new Date(); 29 | var time_string = 'latest'; 30 | var file_version = pkg.version.replace(/\./g, '-'); 31 | var archive_name = pkg.name + '_' + file_version + '_' + time_string; 32 | 33 | npm.load([], function (err, npm) { 34 | 35 | npm.config.set('loglevel', 'silent'); 36 | var install_location = dir.path; 37 | 38 | npm.commands.install(install_location, opts.package_folder, function () { 39 | 40 | var output = fs.createWriteStream(install_location + '/' + archive_name + '.zip'); 41 | var zipArchive = archive('zip'); 42 | zipArchive.pipe(output); 43 | 44 | zipArchive.bulk([ 45 | { 46 | src: ['./**'], 47 | expand: true, 48 | cwd: install_location + '/node_modules/' + pkg.name 49 | } 50 | ]); 51 | 52 | zipArchive.finalize(); 53 | 54 | output.on('close', function () { 55 | mkdirp('./' + opts.dist_folder, function (err) { 56 | fs.createReadStream(install_location + '/' + archive_name + '.zip').pipe( 57 | fs.createWriteStream('./' + opts.dist_folder + '/' + archive_name + '.zip') 58 | ); 59 | 60 | rimraf(install_location, function () { 61 | 62 | grunt.config.set('deployLambda.' + task.target + '.package', 63 | './' + opts.dist_folder + '/' + archive_name + '.zip'); 64 | 65 | grunt.log.writeln('Created package at ' + opts.dist_folder + '/' + archive_name + '.zip'); 66 | done(true); 67 | }); 68 | }); 69 | }); 70 | }); 71 | }); 72 | }); 73 | }; 74 | -------------------------------------------------------------------------------- /vagrant/.gitignore: -------------------------------------------------------------------------------- 1 | .peru 2 | oss-playbooks 3 | ansible 4 | -------------------------------------------------------------------------------- /vagrant/ansible.hosts: -------------------------------------------------------------------------------- 1 | [vagrant] 2 | 127.0.0.1:2222 3 | -------------------------------------------------------------------------------- /vagrant/peru.yaml: -------------------------------------------------------------------------------- 1 | imports: 2 | ansible: ansible 3 | ansible_playbooks: oss-playbooks 4 | 5 | curl module ansible: 6 | # Equivalent of git cloning tags/v1.6.6 but much, much faster 7 | url: https://codeload.github.com/ansible/ansible/zip/69d85c22c7475ccf8169b6ec9dee3ee28c92a314 8 | # Unzip the archive after fetching. 9 | unpack: zip 10 | export: ansible-69d85c22c7475ccf8169b6ec9dee3ee28c92a314 11 | 12 | 13 | 14 | git module ansible_playbooks: 15 | url: https://github.com/snowplow/ansible-playbooks.git 16 | # Comment out to fetch a specific rev instead of master: 17 | # rev: xxx 18 | -------------------------------------------------------------------------------- /vagrant/up.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | vagrant_dir=/vagrant/vagrant 5 | bashrc=/home/vagrant/.bashrc 6 | 7 | echo "========================================" 8 | echo "INSTALLING PERU AND ANSIBLE DEPENDENCIES" 9 | echo "----------------------------------------" 10 | apt-get update 11 | apt-get install -y language-pack-en git unzip libyaml-dev python3-pip python-yaml python-paramiko python-jinja2 12 | 13 | echo "===============" 14 | echo "INSTALLING PERU" 15 | echo "---------------" 16 | sudo pip3 install peru 17 | 18 | echo "=======================================" 19 | echo "CLONING ANSIBLE AND PLAYBOOKS WITH PERU" 20 | echo "---------------------------------------" 21 | cd ${vagrant_dir} && peru sync -v 22 | echo "... done" 23 | 24 | env_setup=${vagrant_dir}/ansible/hacking/env-setup 25 | hosts=${vagrant_dir}/ansible.hosts 26 | 27 | echo "===================" 28 | echo "CONFIGURING ANSIBLE" 29 | echo "-------------------" 30 | touch ${bashrc} 31 | echo "source ${env_setup}" >> ${bashrc} 32 | echo "export ANSIBLE_HOSTS=${hosts}" >> ${bashrc} 33 | echo "... done" 34 | 35 | echo "==========================================" 36 | echo "RUNNING PLAYBOOKS WITH ANSIBLE*" 37 | echo "* no output while each playbook is running" 38 | echo "------------------------------------------" 39 | while read pb; do 40 | su - -c "source ${env_setup} && ${vagrant_dir}/ansible/bin/ansible-playbook ${vagrant_dir}/${pb} --connection=local --inventory-file=${hosts}" vagrant 41 | done <${vagrant_dir}/up.playbooks 42 | 43 | guidance=${vagrant_dir}/up.guidance 44 | 45 | if [ -f ${guidance} ]; then 46 | echo "===========" 47 | echo "PLEASE READ" 48 | echo "-----------" 49 | cat $guidance 50 | fi 51 | -------------------------------------------------------------------------------- /vagrant/up.guidance: -------------------------------------------------------------------------------- 1 | To get started: 2 | vagrant ssh 3 | cd /vagrant 4 | npm install grunt 5 | npm install 6 | grunt --help 7 | -------------------------------------------------------------------------------- /vagrant/up.playbooks: -------------------------------------------------------------------------------- 1 | oss-playbooks/nodejs.yml 2 | oss-playbooks/aws-cli-and-psql.yml 3 | oss-playbooks/grunt.yml 4 | --------------------------------------------------------------------------------