├── .gitignore ├── LICENSE ├── Makefile ├── MyLambda.js ├── README.md ├── demo-invoke-kinesis-payload.json ├── demo-invoke-s3-payload.json ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | *.zip 2 | node_modules 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2015 Localytics℠ http://www.localytics.com/ 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | delete: 2 | aws lambda delete-function \ 3 | --region us-east-1 \ 4 | --function-name s3-kinesis-lambda-starter 5 | 6 | get: 7 | aws lambda get-function \ 8 | --region us-east-1 \ 9 | --function-name s3-kinesis-lambda-starter 10 | 11 | invoke: 12 | aws lambda invoke-async \ 13 | --region us-east-1 \ 14 | --function-name s3-kinesis-lambda-starter \ 15 | --invoke-args $(payload) 16 | 17 | list: 18 | aws lambda list-functions \ 19 | --region us-east-1 20 | 21 | list-event-sources: 22 | aws lambda list-event-sources \ 23 | --region us-east-1 24 | 25 | update: 26 | @npm install 27 | @zip -r ./MyLambda.zip * -x *.json *.zip test.js 28 | aws lambda update-function-code \ 29 | --region us-east-1 \ 30 | --function-name s3-kinesis-lambda-starter \ 31 | --zip-file fileb://MyLambda.zip 32 | 33 | upload: 34 | @npm install 35 | @zip -r ./MyLambda.zip * -x *.json *.zip test.js 36 | aws lambda create-function \ 37 | --region us-east-1 \ 38 | --function-name s3-kinesis-lambda-starter \ 39 | --zip-file fileb://MyLambda.zip \ 40 | --handler MyLambda.handler \ 41 | --runtime nodejs \ 42 | --role arn:aws:iam::$(shell echo $(AWS_ACCOUNT_ID)):role/lambda_basic_execution \ 43 | 44 | test: 45 | @npm test 46 | 47 | .PROXY: delete get invoke list list-event-sources update upload test 48 | -------------------------------------------------------------------------------- /MyLambda.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var async = require('async'); 4 | var AWS = require('aws-sdk'); 5 | var zlib = require('zlib'); 6 | 7 | var s3 = new AWS.S3(); 8 | 9 | exports.kinesisHandler = function(records, context) { 10 | var data = records 11 | .map(function(record) { 12 | return new Buffer(record.kinesis.data, 'base64').toString('utf8'); 13 | }) 14 | .join(); 15 | console.log(data); 16 | context.done(); 17 | }; 18 | exports.s3Handler = function(record, context) { 19 | async.waterfall([ 20 | function download(next) { 21 | s3.getObject({ 22 | Bucket: record.s3.bucket.name, 23 | Key: record.s3.object.key 24 | }, function(err, data) { 25 | next(err, data); 26 | }); 27 | }, 28 | function gunzip(response, next) { 29 | var buffer = new Buffer(response.Body); 30 | zlib.gunzip(buffer, function(err, decoded) { 31 | next(err, decoded && decoded.toString()); 32 | }); 33 | }, 34 | function doSomething(data, next) { 35 | console.log(data); 36 | context.done(); 37 | } 38 | ], function(err) { 39 | if (err) throw err; 40 | }); 41 | }; 42 | 43 | exports.handler = function(event, context) { 44 | var record = event.Records[0]; 45 | if (record.kinesis) { 46 | exports.kinesisHandler(event.Records, context); 47 | } else if (record.s3) { 48 | exports.s3Handler(record, context); 49 | } 50 | }; 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | S3 / Kinesis Lambda Starter 2 | --------------------------- 3 | 4 | A starter module for an AWS Lambda that can handle S3 PUT notifications and 5 | Kinesis stream events. 6 | 7 | This module is intended to be copied and edited as your own Lambda function. 8 | The utilities within this repo are focused around creating and maintaining 9 | _a single_ Lambda function. 10 | 11 | [Read the accompanying blog post here](http://eng.localytics.com/taming-aws-lambda-for-s3-and-kinesis-at-localytics/). 12 | 13 | #### Install 14 | 15 | - `npm install` 16 | 17 | #### Test 18 | 19 | - `npm test` 20 | 21 | #### Deploy 22 | 23 | First, set your `AWS_ACCOUNT_ID` environment variable: 24 | 25 | - bash: `export AWS_ACCOUNT_ID=123` 26 | - fish: `set -x AWS_ACCOUNT_ID 123` 27 | 28 | **Permissions:** 29 | 30 | There are two specific permissions that you'll likely need. 31 | 32 | - User permission for `iam:PassRole`. This policy needs to be applied to the 33 | user who is _creating_ the Lambda: 34 | 35 | ``` 36 | { 37 | "Version": "2012-10-17", 38 | "Statement": [ 39 | { 40 | "Sid": "Stmt1429124462000", 41 | "Effect": "Allow", 42 | "Action": [ 43 | "iam:PassRole" 44 | ], 45 | "Resource": [ 46 | "arn:aws:iam:::role/lambda_basic_execution" 47 | ] 48 | } 49 | ] 50 | } 51 | ``` 52 | 53 | - Lambda execution role. You need to create a new role that the Lambda will 54 | run as. We assume that role is named `lambda_basic_execution` for the purposes 55 | of this project. That role must have (at least) this policy applied: 56 | 57 | ``` 58 | { 59 | "Version": "2012-10-17", 60 | "Statement": [ 61 | { 62 | "Effect": "Allow", 63 | "Action": [ 64 | "logs:*" 65 | ], 66 | "Resource": "arn:aws:logs:*:*:*" 67 | } 68 | ] 69 | } 70 | ``` 71 | 72 | This policy allows the `lambda_basic_execution` role to manage CloudWatch logs 73 | for our Lambda's execution. 74 | 75 | **Upload:** 76 | 77 | - `make upload` 78 | 79 | **Connect an S3 bucket:** 80 | 81 | To connect an S3 bucket to the Lambda function, you must use the 82 | [AWS UI](https://console.aws.amazon.com/lambda/home?region=us-east-1#/functions). 83 | 84 | **Connect a Kinesis stream:** 85 | 86 | To connect a Kinesis stream to the Lambda function, run the following: 87 | 88 | ``` 89 | aws lambda add-event-source \ 90 | --region \ 91 | --function-name \ 92 | --role arn:aws:iam:::role/ \ 93 | --event-source arn:aws:kinesis:::stream/ \ 94 | --batch-size 100 \ 95 | --profile 96 | ``` 97 | 98 | *Note: `cli-profile-name` is whatever block you've named your credentials in 99 | `~/.aws/credentials`. It is `default` by default.* 100 | 101 | #### Utilities 102 | 103 | We've defined some useful utilities in the Makefile which can make uploading, 104 | updating, and invoking this Lambda a little easier. 105 | 106 | *Note: We have hardcoded the region and function name into the Makefile for this 107 | module. You'll want to modify this if you plan to change the function name or 108 | use a region other than `us-east-1`.* 109 | 110 | - `make delete` - will remove this Lambda from AWS. 111 | - `make get` - will retrieve the details of this function on AWS. 112 | - `make invoke` - will invoke this Lambda on AWS with the provided data file. 113 | - `make list` - will list all of the Lambda functions on this account for the 114 | given region. 115 | - `make list-event-sources` - will list event sources for this Lambda. 116 | - `make update` - will re-upload this Lambda package without overwriting the 117 | function's configuration details. 118 | - `make upload` - will upload this Lambda function for the first time. 119 | -------------------------------------------------------------------------------- /demo-invoke-kinesis-payload.json: -------------------------------------------------------------------------------- 1 | { 2 | "Records": [ 3 | { 4 | "kinesis": { 5 | "partitionKey": "partitionKey-3", 6 | "kinesisSchemaVersion": "1.0", 7 | "data": "TG9jYWx5dGljcyBFbmdpbmVlcmluZyBpcyBoaXJpbmchIGh0dHA6Ly9iaXQubHkvMURqN2N1bA==", 8 | "sequenceNumber": "49545115243490985018280067714973144582180062593244200961" 9 | }, 10 | "eventSource": "aws:kinesis", 11 | "eventID": "shardId-000000000000:49545115243490985018280067714973144582180062593244200961", 12 | "invokeIdentityArn": "arn:aws:iam::EXAMPLE", 13 | "eventVersion": "1.0", 14 | "eventName": "aws:kinesis:record", 15 | "eventSourceARN": "arn:aws:kinesis:EXAMPLE", 16 | "awsRegion": "us-east-1" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /demo-invoke-s3-payload.json: -------------------------------------------------------------------------------- 1 | { 2 | "Records": [ 3 | { 4 | "eventVersion": "2.0", 5 | "eventSource": "aws:s3", 6 | "awsRegion": "us-east-1", 7 | "eventTime": "1970-01-01T00:00:00.000Z", 8 | "eventName": "ObjectCreated:Put", 9 | "userIdentity": { 10 | "principalId": "EXAMPLE" 11 | }, 12 | "requestParameters": { 13 | "sourceIPAddress": "127.0.0.1" 14 | }, 15 | "responseElements": { 16 | "x-amz-request-id": "C3D13FE58DE4C810", 17 | "x-amz-id-2": "FMyUVURIY8/IgAtTv8xRjskZQpcIZ9KG4V5Wp6S7S/JRWeUWerMUE5JgHvANOjpD" 18 | }, 19 | "s3": { 20 | "s3SchemaVersion": "1.0", 21 | "configurationId": "testConfigRule", 22 | "bucket": { 23 | "name": "sourcebucket", 24 | "ownerIdentity": { 25 | "principalId": "EXAMPLE" 26 | }, 27 | "arn": "arn:aws:s3:::mybucket" 28 | }, 29 | "object": { 30 | "key": "HappyFace.jpg", 31 | "size": 1024, 32 | "eTag": "d41d8cd98f00b204e9800998ecf8427e" 33 | } 34 | } 35 | } 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "s3-kinesis-lambda-starter", 3 | "version": "1.0.0", 4 | "description": "A starter module for an AWS Lambda that can handle S3 PUT notifications and Kinesis stream events.", 5 | "main": "MyLambda.js", 6 | "scripts": { 7 | "test": "mocha" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/nicksergeant/s3-kinesis-lambda-starter.git" 12 | }, 13 | "author": "Localytics", 14 | "license": "ISC", 15 | "bugs": { 16 | "url": "https://github.com/nicksergeant/s3-kinesis-lambda-starter/issues" 17 | }, 18 | "homepage": "https://github.com/nicksergeant/s3-kinesis-lambda-starter", 19 | "dependencies": { 20 | "async": "^0.9.0", 21 | "aws-sdk": "^2.1.18" 22 | }, 23 | "devDependencies": { 24 | "chai": "^2.1.2", 25 | "mocha": "^2.2.1", 26 | "proxyquire": "^1.4.0", 27 | "sinon": "^1.14.1", 28 | "sinon-chai": "^2.7.0" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var chai = require('chai'); 4 | var myLambda = require('./MyLambda'); 5 | var sinon = require('sinon'); 6 | var sinonChai = require('sinon-chai'); 7 | var proxyquire = require('proxyquire').noCallThru(); 8 | 9 | var kinesisHandler; 10 | var s3Handler; 11 | 12 | global.expect = chai.expect; 13 | chai.use(sinonChai); 14 | 15 | describe('MyLambda', function() { 16 | describe('exports.handler()', function() { 17 | beforeEach(function() { 18 | kinesisHandler = sinon.stub(myLambda, 'kinesisHandler'); 19 | s3Handler = sinon.stub(myLambda, 's3Handler'); 20 | }); 21 | afterEach(function() { 22 | kinesisHandler.restore(); 23 | s3Handler.restore(); 24 | }); 25 | describe('with kinesis input', function() { 26 | it('calls kinesis handler', function() { 27 | var event = { Records: [ 28 | { kinesis: { data: 'TG9jYWx5dGljcyBFbmdpbmVlcmluZyBpcyBoaXJpbmchIGh0dHA6Ly9iaXQubHkvMURqN2N1bA==' }} 29 | ]}; 30 | myLambda.handler(event); 31 | expect(s3Handler).to.not.have.been.called; 32 | expect(kinesisHandler).to.be.calledWith(event.Records); 33 | }); 34 | }); 35 | describe('with s3 input', function() { 36 | it('calls s3 handler', function() { 37 | var event = { Records: [ 38 | { s3: { bucket: { name: '' }, object: { key: 'path/to/file.gz' } }} 39 | ]}; 40 | myLambda.handler(event); 41 | expect(kinesisHandler).to.not.have.been.called; 42 | expect(s3Handler).to.be.calledWith(event.Records[0]); 43 | }); 44 | }); 45 | }); 46 | }); 47 | --------------------------------------------------------------------------------