├── .github └── FUNDING.yml ├── .gitignore ├── README.md ├── cloudformation.yml ├── consumer ├── .jshintignore ├── .jshintrc ├── index.js └── package.json └── worker ├── .jshintignore ├── .jshintrc ├── index.js └── package.json /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: widdix 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | output.yml 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Example: SQS and Lambda 2 | 3 | This repository contains example code which is integrating SQS and Lambda as described in our blog post [Integrate SQS and Lambda: serverless architecture for asynchronous workloads](https://cloudonaut.io/integrate-sqs-and-lambda-serverless-architecture-for-asynchronous-workloads/). 4 | 5 | ## Deploy 6 | 7 | Use the following command to deploy the example. Replace `` with your nickname. 8 | 9 | ``` 10 | aws s3 mb s3://sqs-lambda- 11 | 12 | aws cloudformation package --template-file cloudformation.yml --s3-bucket sqs-lambda- --output-template-file output.yml && aws cloudformation deploy --template-file output.yml --stack-name sqs-lambda-example --capabilities CAPABILITY_IAM 13 | ``` 14 | -------------------------------------------------------------------------------- /cloudformation.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Transform: 'AWS::Serverless-2016-10-31' 3 | Description: 'SQS and Lambda example.' 4 | Resources: 5 | TaskQueue: 6 | Type: AWS::SQS::Queue 7 | Properties: 8 | VisibilityTimeout: 60 9 | RedrivePolicy: 10 | deadLetterTargetArn: !Sub ${DeadLetterQueue.Arn} 11 | maxReceiveCount: 10 12 | DeadLetterQueue: 13 | Type: AWS::SQS::Queue 14 | ConsumerLambdaRole: 15 | Type: AWS::IAM::Role 16 | Properties: 17 | AssumeRolePolicyDocument: 18 | Version: '2012-10-17' 19 | Statement: 20 | - Effect: Allow 21 | Principal: 22 | Service: lambda.amazonaws.com 23 | Action: 24 | - sts:AssumeRole 25 | Path: '/' 26 | Policies: 27 | - PolicyName: logs 28 | PolicyDocument: 29 | Statement: 30 | - Effect: Allow 31 | Action: 32 | - logs:CreateLogGroup 33 | - logs:CreateLogStream 34 | - logs:PutLogEvents 35 | Resource: arn:aws:logs:*:*:* 36 | - PolicyName: sqs 37 | PolicyDocument: 38 | Statement: 39 | - Effect: Allow 40 | Action: 41 | - sqs:ReceiveMessage 42 | Resource: !Sub ${TaskQueue.Arn} 43 | - PolicyName: lambda 44 | PolicyDocument: 45 | Statement: 46 | - Effect: Allow 47 | Action: 48 | - lambda:InvokeFunction 49 | Resource: !Sub ${WorkerLambda.Arn} 50 | ConsumerLambda: 51 | Type: AWS::Serverless::Function 52 | Properties: 53 | CodeUri: ./consumer 54 | Handler: index.handler 55 | MemorySize: 128 56 | Role: !Sub ${ConsumerLambdaRole.Arn} 57 | Runtime: nodejs6.10 58 | Timeout: 60 59 | Environment: 60 | Variables: 61 | TASK_QUEUE_URL: !Ref TaskQueue 62 | WORKER_LAMBDA_NAME: !Ref WorkerLambda 63 | Events: 64 | Timer: 65 | Type: Schedule 66 | Properties: 67 | Schedule: rate(1 minute) 68 | WorkerLambdaRole: 69 | Type: AWS::IAM::Role 70 | Properties: 71 | AssumeRolePolicyDocument: 72 | Version: '2012-10-17' 73 | Statement: 74 | - Effect: Allow 75 | Principal: 76 | Service: lambda.amazonaws.com 77 | Action: 78 | - sts:AssumeRole 79 | Path: '/' 80 | Policies: 81 | - PolicyName: logs 82 | PolicyDocument: 83 | Statement: 84 | - Effect: Allow 85 | Action: 86 | - logs:CreateLogGroup 87 | - logs:CreateLogStream 88 | - logs:PutLogEvents 89 | Resource: arn:aws:logs:*:*:* 90 | - PolicyName: sqs 91 | PolicyDocument: 92 | Statement: 93 | - Effect: Allow 94 | Action: 95 | - sqs:DeleteMessage 96 | Resource: !Sub ${TaskQueue.Arn} 97 | WorkerLambda: 98 | Type: AWS::Serverless::Function 99 | Properties: 100 | CodeUri: ./worker 101 | Handler: index.handler 102 | MemorySize: 128 103 | Role: !Sub ${WorkerLambdaRole.Arn} 104 | Runtime: nodejs6.10 105 | Timeout: 60 106 | Environment: 107 | Variables: 108 | TASK_QUEUE_URL: !Ref TaskQueue 109 | -------------------------------------------------------------------------------- /consumer/.jshintignore: -------------------------------------------------------------------------------- 1 | node_modules/** 2 | -------------------------------------------------------------------------------- /consumer/.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "maxerr" : 500, 3 | "bitwise" : true, 4 | "camelcase" : false, 5 | "curly" : true, 6 | "eqeqeq" : true, 7 | "forin" : true, 8 | "immed" : false, 9 | "indent" : 4, 10 | "latedef" : false, 11 | "newcap" : false, 12 | "noarg" : true, 13 | "noempty" : true, 14 | "nonew" : false, 15 | "plusplus" : false, 16 | "quotmark" : false, 17 | "undef" : true, 18 | "unused" : false, 19 | "strict" : true, 20 | "trailing" : true, 21 | "maxparams" : false, 22 | "maxdepth" : false, 23 | "maxstatements" : false, 24 | "maxcomplexity" : false, 25 | "maxlen" : false, 26 | "asi" : false, 27 | "boss" : false, 28 | "debug" : false, 29 | "eqnull" : false, 30 | "es5" : false, 31 | "esnext" : false, 32 | "moz" : false, 33 | "evil" : false, 34 | "expr" : false, 35 | "funcscope" : false, 36 | "globalstrict" : false, 37 | "iterator" : false, 38 | "lastsemic" : false, 39 | "laxbreak" : false, 40 | "laxcomma" : false, 41 | "loopfunc" : false, 42 | "multistr" : false, 43 | "proto" : false, 44 | "scripturl" : false, 45 | "shadow" : false, 46 | "sub" : false, 47 | "supernew" : false, 48 | "validthis" : false, 49 | "browser" : false, 50 | "couch" : false, 51 | "devel" : false, 52 | "dojo" : false, 53 | "jquery" : false, 54 | "mootools" : false, 55 | "node" : true, 56 | "nonstandard" : false, 57 | "prototypejs" : false, 58 | "rhino" : false, 59 | "worker" : false, 60 | "wsh" : false, 61 | "yui" : false, 62 | "globals" : {}, 63 | "predef" : ["describe", "it", "before", "after"] 64 | } 65 | -------------------------------------------------------------------------------- /consumer/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var AWS = require('aws-sdk'); 4 | var async = require('async'); 5 | 6 | var TASK_QUEUE_URL = process.env.TASK_QUEUE_URL; 7 | var WORKER_LAMBDA_NAME = process.env.WORKER_LAMBDA_NAME; 8 | var AWS_REGION = process.env.AWS_REGION; 9 | 10 | var sqs = new AWS.SQS({region: AWS_REGION}); 11 | var lambda = new AWS.Lambda({region: AWS_REGION}); 12 | 13 | function receiveMessages(callback) { 14 | var params = { 15 | QueueUrl: TASK_QUEUE_URL, 16 | MaxNumberOfMessages: 10 17 | }; 18 | sqs.receiveMessage(params, function(err, data) { 19 | if (err) { 20 | console.error(err, err.stack); 21 | callback(err); 22 | } else { 23 | callback(null, data.Messages); 24 | } 25 | }); 26 | } 27 | 28 | function invokeWorkerLambda(task, callback) { 29 | var params = { 30 | FunctionName: WORKER_LAMBDA_NAME, 31 | InvocationType: 'Event', 32 | Payload: JSON.stringify(task) 33 | }; 34 | lambda.invoke(params, function(err, data) { 35 | if (err) { 36 | console.error(err, err.stack); 37 | callback(err); 38 | } else { 39 | callback(null, data); 40 | } 41 | }); 42 | } 43 | 44 | function handleSQSMessages(context, callback) { 45 | receiveMessages(function(err, messages) { 46 | if (messages && messages.length > 0) { 47 | var invocations = []; 48 | messages.forEach(function(message) { 49 | invocations.push(function(callback) { 50 | invokeWorkerLambda(message, callback); 51 | }); 52 | }); 53 | async.parallel(invocations, function(err) { 54 | if (err) { 55 | console.error(err, err.stack); 56 | callback(err); 57 | } else { 58 | if (context.getRemainingTimeInMillis() > 20000) { 59 | handleSQSMessages(context, callback); 60 | } else { 61 | callback(null, 'PAUSE'); 62 | } 63 | } 64 | }); 65 | } else { 66 | callback(null, 'DONE'); 67 | } 68 | }); 69 | } 70 | 71 | exports.handler = function(event, context, callback) { 72 | handleSQSMessages(context, callback); 73 | }; 74 | -------------------------------------------------------------------------------- /consumer/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "scheduler-lambda", 3 | "version": "0.1.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "./node_modules/.bin/jshint ." 8 | }, 9 | "dependencies": { 10 | "async": "2.0.1" 11 | }, 12 | "devDependencies": { 13 | "jshint": "2.9.1" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /worker/.jshintignore: -------------------------------------------------------------------------------- 1 | node_modules/** 2 | -------------------------------------------------------------------------------- /worker/.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "maxerr" : 500, 3 | "bitwise" : true, 4 | "camelcase" : false, 5 | "curly" : true, 6 | "eqeqeq" : true, 7 | "forin" : true, 8 | "immed" : false, 9 | "indent" : 4, 10 | "latedef" : false, 11 | "newcap" : false, 12 | "noarg" : true, 13 | "noempty" : true, 14 | "nonew" : false, 15 | "plusplus" : false, 16 | "quotmark" : false, 17 | "undef" : true, 18 | "unused" : false, 19 | "strict" : true, 20 | "trailing" : true, 21 | "maxparams" : false, 22 | "maxdepth" : false, 23 | "maxstatements" : false, 24 | "maxcomplexity" : false, 25 | "maxlen" : false, 26 | "asi" : false, 27 | "boss" : false, 28 | "debug" : false, 29 | "eqnull" : false, 30 | "es5" : false, 31 | "esnext" : false, 32 | "moz" : false, 33 | "evil" : false, 34 | "expr" : false, 35 | "funcscope" : false, 36 | "globalstrict" : false, 37 | "iterator" : false, 38 | "lastsemic" : false, 39 | "laxbreak" : false, 40 | "laxcomma" : false, 41 | "loopfunc" : false, 42 | "multistr" : false, 43 | "proto" : false, 44 | "scripturl" : false, 45 | "shadow" : false, 46 | "sub" : false, 47 | "supernew" : false, 48 | "validthis" : false, 49 | "browser" : false, 50 | "couch" : false, 51 | "devel" : false, 52 | "dojo" : false, 53 | "jquery" : false, 54 | "mootools" : false, 55 | "node" : true, 56 | "nonstandard" : false, 57 | "prototypejs" : false, 58 | "rhino" : false, 59 | "worker" : false, 60 | "wsh" : false, 61 | "yui" : false, 62 | "globals" : {}, 63 | "predef" : ["describe", "it", "before", "after"] 64 | } 65 | -------------------------------------------------------------------------------- /worker/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var AWS = require("aws-sdk"); 4 | 5 | var TASK_QUEUE_URL = process.env.TASK_QUEUE_URL; 6 | var AWS_REGION = process.env.AWS_REGION; 7 | 8 | var sqs = new AWS.SQS({region: AWS_REGION}); 9 | var s3 = new AWS.S3({region: AWS_REGION}); 10 | 11 | function deleteMessage(receiptHandle, cb) { 12 | sqs.deleteMessage({ 13 | ReceiptHandle: receiptHandle, 14 | QueueUrl: TASK_QUEUE_URL 15 | }, cb); 16 | } 17 | 18 | function work(task, cb) { 19 | console.log(task); 20 | // TODO implement 21 | cb(); 22 | } 23 | 24 | exports.handler = function(event, context, callback) { 25 | work(event.Body, function(err) { 26 | if (err) { 27 | callback(err); 28 | } else { 29 | deleteMessage(event.ReceiptHandle, callback); 30 | } 31 | }); 32 | }; 33 | -------------------------------------------------------------------------------- /worker/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "worker-lambda", 3 | "version": "0.1.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "./node_modules/.bin/jshint ." 8 | }, 9 | "dependencies": { 10 | 11 | }, 12 | "devDependencies": { 13 | "jshint": "2.9.1" 14 | } 15 | } 16 | --------------------------------------------------------------------------------