├── .gitignore
├── README.md
├── build.js
├── package.json
├── sqs-to-lambda.yml
└── src
    ├── cloudformation.yml
    └── sqs-to-lambda.js


/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # SQS to Lambda (via Lambda)
 2 | 
 3 | There is currently not a native SQS event source for Lambda. This is
 4 | unfortunate. You can run something like https://github.com/robinjmurphy/sqs-to-lambda
 5 | but that requires running an instance, and who wants to do that?
 6 | 
 7 | Running a Lambda function (128MB) full-time costs about $5.39/month, and a t2.nano
 8 | (on demand) costs $4.68. Surely $0.71 is worth the coolness of not running a
 9 | boring EC2 instance. (There's a cheaper option too, keep reading.)
10 | 
11 | The supplied CloudFormation template will set up a Lambda function that reads
12 | from SQS queues, and invokes Lambda functions with the payloads. By default it will
13 | run a Lambda function continuously, polling your queues constantly for new items.
14 | 
15 | You can also configure it to poll your queues once a minute, reducing Lambda
16 | costs to almost nothing, at the expense of waiting up to 60s to receive new queue
17 | items.
18 | 
19 | One day, hopefully soon, Lambda will likely support SQS as a native event source,
20 | and then this will be completely unnecessary. :fingerscrossed:
21 | 
22 | ## Getting Started
23 | 
24 | First, create the CloudFormation stack. Inside a clone of this project:
25 | 
26 | ```bash
27 | aws --region us-east-1 cloudformation create-stack                            \
28 |   --stack-name sqs-to-lambda                                                  \
29 |   --template-body file://sqs-to-lambda.yml                                    \
30 |   --parameters 'ParameterKey=QueueToFunctionMapping,ParameterValue="<queue url 1>,<function 1>,<queue url 2>,<function 2>,..."' \
31 |   --capabilities CAPABILITY_IAM
32 | ```
33 | 
34 | If you'd like the every-minute, cheaper option, add `ParameterKey=Frequency,ParameterValue=1Minute`
35 | to the `--parameters` option.
36 | 
37 | You can also use the console, if you are so inclined.
38 | 
39 | Your functions specified in `QueueToFunctionMapping` will now receive everything
40 | sent to their respective queues. The payload is an object with `source` set to
41 | `aws.sqs` and `Message` as the actual message from SQS:
42 | 
43 | ```json
44 | {
45 |     "source": "aws.sqs",
46 |     "QueueUrl": "https://sqs.us-east-1.amazonaws.com/1234567/my-queue",
47 |     "Message": {
48 |         "MessageId": "2b2ea032-5d3d-4a17-b38c-92bece3ad7ce",
49 |         "ReceiptHandle": "AQEB8t7sz7fgeAalKraYO3brB2+r0d3p18RE3G6J9k9GmRFODibL64oget5R6NaRJDoYrwHNtLutKOiY3Ggls2F6LRJFKLZhLbr3fSd+Hg6KiECu4tfdyAZxAwj2/X5QIieu0dtCMIEujHSDn7Xzz9L5hNW/uCB7Tx7Km0Sal077KE4h4CCHMvZDza8bNzmFTXvfRj5+odG80oLtir0w+lwx+DQYnkIZJxvVRLkfOspU2/84/ye4VZkr8pOD7xIGtgzU/Z7pdzTXeKw0WSfHQoQ661qBcqBHhMTjXXZ0WzsYHW1HPqtSwqA760nZfh0RXRjo9AGFsXYmtnQoFs64PCJ1hZ2u+N+azHChx4Ma+PtT6pgUfkCzrYG5Gq/BaR+RmPsW",
50 |         "MD5OfBody": "abecffaa52f529a2b83b6612a7964b02",
51 |         "Body": "{\"foo\":\"bar\"}"
52 |     }
53 | }
54 | ```
55 | 
56 | *NOTE:* Your Lambda function will need to delete the item from the queue using the given
57 | `ReceiptHandle`! Otherwise, it will keep getting delivered.
58 | 
59 | ## Questions
60 | 
61 | ### How does it work?
62 | 
63 | The supplied CloudFormation template will create a Lambda function that knows
64 | how to pull from queues, and invoke other Lambda functions. The function is
65 | triggered by CloudWatch Events, also set up by the CloudFormation template.
66 | 
67 | If you choose Continuous frequency, the function is invoked every 5 minutes, and
68 | runs for almost 5 minutes, constantly polling.
69 | 
70 | If you choose 1Minute frequency, the function is invoked every minute, but exits
71 | once the queue returns 0 items to process.
72 | 
73 | ### What's up with the Javascript in `sqs-to-lambda.yml`?
74 | 
75 | Embedding Javascript in the CloudFormation template is an easy way to make the
76 | template self-contained. It also allows us to dynamically inject the configuration
77 | into the SQS poller function. The javascript in the `sqs-to-lambda.yml` file
78 | is minimized from `sqs-to-lambda.js`.
79 | 
80 | We minimize the Javascript because there is a 2KB limit to inline code in
81 | CloudFormation templates.
82 | 
83 | ## Building
84 | 
85 | The `sqs-to-lambda.yml` file is a build artifact - don't edit it directly!
86 | The original is in `src/cloudformation.yml`
87 | To build:
88 | 
89 | ```
90 | $ npm install
91 | $ npm run build
92 | ```
93 | 
94 | ## Contributing
95 | 
96 | Make your changes, build a new `sqs-to-lambda.yml` file (see above), and
97 | submit a pull request.
98 | 


--------------------------------------------------------------------------------
/build.js:
--------------------------------------------------------------------------------
 1 | var UglifyJS = require("uglify-js");
 2 | var fs = require('fs');
 3 | 
 4 | var sqsToLambdaCode = UglifyJS.minify('./src/sqs-to-lambda.js', {
 5 |   mangle: { 'toplevel': true }
 6 | }).code;
 7 | 
 8 | var templateBody = fs.readFileSync('./src/cloudformation.yml', 'utf-8');
 9 | var builtTemplate = templateBody.replace('{SQSToLambdaCode}', sqsToLambdaCode);
10 | fs.writeFileSync('./sqs-to-lambda.yml', builtTemplate);
11 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 |   "scripts": {
3 |     "build": "node build.js"
4 |   },
5 |   "dependencies": {
6 |     "uglify-js": "^2.6.2"
7 |   }
8 | }
9 | 


--------------------------------------------------------------------------------
/sqs-to-lambda.yml:
--------------------------------------------------------------------------------
  1 | Description: |
  2 |   Creates a Lambda function that invokes other Lambda functions with
  3 |   SQS events.
  4 | 
  5 | Parameters:
  6 |   QueueToFunctionMapping:
  7 |     Description: |
  8 |       The list of SQS queue URLs and corresponding Lambda function
  9 |       names. Separate queue urls, function names, and pairs with commas:
 10 |       <queue_url_1>,<function_1>,<queue_url_2>,<function_2>.
 11 |       There must be an odd number of commas.
 12 |     Type: String
 13 | 
 14 |   Frequency:
 15 |     Description: |
 16 |       How often you want the specified SQS queues checked. If you choose
 17 |       "Continuous", a Lambda function will be constantly running, waiting for
 18 |       items in the queue. If you choose "1Minute", the Lambda function will
 19 |       only run every minute and check every queue. This can result in drastically
 20 |       reduced Lambda cost, at the expense of waiting up to a minute for items
 21 |       to become visibile in the queue before being processed by your functions.
 22 |     Type: String
 23 |     Default: Continuous
 24 |     AllowedValues:
 25 |       - Continuous
 26 |       - 1Minute
 27 | 
 28 | Outputs: {}
 29 | 
 30 | Conditions:
 31 |   Enable1MinutePoll: !Equals [!Ref Frequency, 1Minute]
 32 |   Enable5MinutePoll: !Equals [!Ref Frequency, Continuous]
 33 | 
 34 | Resources:
 35 | 
 36 |   ### IAM Setup
 37 | 
 38 |   SQSToLambdaRole:
 39 |     Type: AWS::IAM::Role
 40 |     Properties:
 41 |       Path: /
 42 |       AssumeRolePolicyDocument:
 43 |         Version: "2012-10-17"
 44 |         Statement:
 45 |           - Effect: Allow
 46 |             Principal:
 47 |               Service: lambda.amazonaws.com
 48 |             Action: sts:AssumeRole
 49 |       Policies:
 50 |         - PolicyName: lambda-to-sqs-policy
 51 |           PolicyDocument:
 52 |             Version: "2012-10-17"
 53 |             Statement:
 54 |               - Effect: Allow
 55 |                 Action:
 56 |                   - sqs:ReceiveMessage
 57 |                 Resource:
 58 |                   # TODO: Should we list the configured queues and functions here?
 59 |                   # Will blank ones cause any problems? One problem is that queues
 60 |                   # are supplied with URLs, and functions with names. (No ARNs.)
 61 |                   - "*"
 62 |               - Effect: Allow
 63 |                 Action:
 64 |                   - lambda:Invoke*
 65 |                 Resource:
 66 |                   - "*"
 67 |               - Effect: Allow
 68 |                 Action:
 69 |                   - logs:CreateLogGroup
 70 |                   - logs:CreateLogStream
 71 |                   - logs:PutLogEvents
 72 |                 Resource: arn:aws:logs:*:*:*
 73 | 
 74 |   ### CloudWatch Event
 75 | 
 76 |   CloudWatchEvent1Minute:
 77 |     Type: AWS::Events::Rule
 78 |     Condition: Enable1MinutePoll
 79 |     Properties:
 80 |       ScheduleExpression: cron(* * * * ? *)
 81 |       Targets:
 82 |         - Arn: !GetAtt [SQSToLambdaFunction, Arn]
 83 |           Id: !Ref SQSToLambdaFunction
 84 | 
 85 |   CloudWatchEvent5Minute:
 86 |     Type: AWS::Events::Rule
 87 |     Condition: Enable5MinutePoll
 88 |     Properties:
 89 |       ScheduleExpression: cron(0/5 * * * ? *)
 90 |       Targets:
 91 |         - Arn: !GetAtt [SQSToLambdaFunction, Arn]
 92 |           Id: !Ref SQSToLambdaFunction
 93 | 
 94 |   ### Lambda Function
 95 | 
 96 |   SQSToLambdaFunctionPermission1Minute:
 97 |     Type: AWS::Lambda::Permission
 98 |     Condition: Enable1MinutePoll
 99 |     Properties:
100 |       Action: lambda:InvokeFunction
101 |       FunctionName:
102 |         Ref: SQSToLambdaFunction
103 |       Principal: events.amazonaws.com
104 |       SourceArn: !GetAtt [CloudWatchEvent1Minute, Arn]
105 | 
106 |   SQSToLambdaFunctionPermission5Minute:
107 |     Type: AWS::Lambda::Permission
108 |     Condition: Enable5MinutePoll
109 |     Properties:
110 |       Action: lambda:InvokeFunction
111 |       FunctionName:
112 |         Ref: SQSToLambdaFunction
113 |       Principal: events.amazonaws.com
114 |       SourceArn: !GetAtt [CloudWatchEvent5Minute, Arn]
115 | 
116 |   SQSToLambdaFunction:
117 |     Type: AWS::Lambda::Function
118 |     Properties:
119 |       Description: Invokes Lambda functions with items from SQS queues.
120 |       Handler: index.handler
121 |       MemorySize: 128
122 |       Role: !GetAtt [SQSToLambdaRole, Arn]
123 |       Runtime: nodejs
124 |       Timeout: 300
125 |       Code:
126 |         ZipFile:
127 |           Fn::Sub: |
128 |             var CONFIG = "${QueueToFunctionMapping}".split(",");
129 |             var ONCE = "${Frequency}" === "1Minute";
130 |             function e(n,i,t,a){return t()<5e3?a():""==n||""==i?a():void s.receiveMessage({QueueUrl:n,MaxNumberOfMessages:1,WaitTimeSeconds:1},function(s,g){return s?(console.log(s),a()):g.Messages&&0!==g.Messages.length?void o.invoke({FunctionName:i,InvocationType:"Event",Payload:JSON.stringify({source:"aws.sqs",QueueUrl:n,Message:g.Messages[0]})},function(s){return s?(console.log(s),a()):e(n,i,t,a)}):r?a():e(n,i,t,a)})}var n=require("aws-sdk"),s=new n.SQS,o=new n.Lambda,i=CONFIG,r=ONCE;exports.handler=function(n,s){if(0===i.length)return s.done();for(var o=i.length/2,r=function(){o-=1,0==o&&(console.log("exiting"),s.done())},t=0;t<i.length;t+=2)e(i[t],i[t+1],s.getRemainingTimeInMillis,r)};
131 | 


--------------------------------------------------------------------------------
/src/cloudformation.yml:
--------------------------------------------------------------------------------
  1 | Description: |
  2 |   Creates a Lambda function that invokes other Lambda functions with
  3 |   SQS events.
  4 | 
  5 | Parameters:
  6 |   QueueToFunctionMapping:
  7 |     Description: |
  8 |       The list of SQS queue URLs and corresponding Lambda function
  9 |       names. Separate queue urls, function names, and pairs with commas:
 10 |       <queue_url_1>,<function_1>,<queue_url_2>,<function_2>.
 11 |       There must be an odd number of commas.
 12 |     Type: String
 13 | 
 14 |   Frequency:
 15 |     Description: |
 16 |       How often you want the specified SQS queues checked. If you choose
 17 |       "Continuous", a Lambda function will be constantly running, waiting for
 18 |       items in the queue. If you choose "1Minute", the Lambda function will
 19 |       only run every minute and check every queue. This can result in drastically
 20 |       reduced Lambda cost, at the expense of waiting up to a minute for items
 21 |       to become visibile in the queue before being processed by your functions.
 22 |     Type: String
 23 |     Default: Continuous
 24 |     AllowedValues:
 25 |       - Continuous
 26 |       - 1Minute
 27 | 
 28 | Outputs: {}
 29 | 
 30 | Conditions:
 31 |   Enable1MinutePoll: !Equals [!Ref Frequency, 1Minute]
 32 |   Enable5MinutePoll: !Equals [!Ref Frequency, Continuous]
 33 | 
 34 | Resources:
 35 | 
 36 |   ### IAM Setup
 37 | 
 38 |   SQSToLambdaRole:
 39 |     Type: AWS::IAM::Role
 40 |     Properties:
 41 |       Path: /
 42 |       AssumeRolePolicyDocument:
 43 |         Version: "2012-10-17"
 44 |         Statement:
 45 |           - Effect: Allow
 46 |             Principal:
 47 |               Service: lambda.amazonaws.com
 48 |             Action: sts:AssumeRole
 49 |       Policies:
 50 |         - PolicyName: lambda-to-sqs-policy
 51 |           PolicyDocument:
 52 |             Version: "2012-10-17"
 53 |             Statement:
 54 |               - Effect: Allow
 55 |                 Action:
 56 |                   - sqs:ReceiveMessage
 57 |                 Resource:
 58 |                   # TODO: Should we list the configured queues and functions here?
 59 |                   # Will blank ones cause any problems? One problem is that queues
 60 |                   # are supplied with URLs, and functions with names. (No ARNs.)
 61 |                   - "*"
 62 |               - Effect: Allow
 63 |                 Action:
 64 |                   - lambda:Invoke*
 65 |                 Resource:
 66 |                   - "*"
 67 |               - Effect: Allow
 68 |                 Action:
 69 |                   - logs:CreateLogGroup
 70 |                   - logs:CreateLogStream
 71 |                   - logs:PutLogEvents
 72 |                 Resource: arn:aws:logs:*:*:*
 73 | 
 74 |   ### CloudWatch Event
 75 | 
 76 |   CloudWatchEvent1Minute:
 77 |     Type: AWS::Events::Rule
 78 |     Condition: Enable1MinutePoll
 79 |     Properties:
 80 |       ScheduleExpression: cron(* * * * ? *)
 81 |       Targets:
 82 |         - Arn: !GetAtt [SQSToLambdaFunction, Arn]
 83 |           Id: !Ref SQSToLambdaFunction
 84 | 
 85 |   CloudWatchEvent5Minute:
 86 |     Type: AWS::Events::Rule
 87 |     Condition: Enable5MinutePoll
 88 |     Properties:
 89 |       ScheduleExpression: cron(0/5 * * * ? *)
 90 |       Targets:
 91 |         - Arn: !GetAtt [SQSToLambdaFunction, Arn]
 92 |           Id: !Ref SQSToLambdaFunction
 93 | 
 94 |   ### Lambda Function
 95 | 
 96 |   SQSToLambdaFunctionPermission1Minute:
 97 |     Type: AWS::Lambda::Permission
 98 |     Condition: Enable1MinutePoll
 99 |     Properties:
100 |       Action: lambda:InvokeFunction
101 |       FunctionName:
102 |         Ref: SQSToLambdaFunction
103 |       Principal: events.amazonaws.com
104 |       SourceArn: !GetAtt [CloudWatchEvent1Minute, Arn]
105 | 
106 |   SQSToLambdaFunctionPermission5Minute:
107 |     Type: AWS::Lambda::Permission
108 |     Condition: Enable5MinutePoll
109 |     Properties:
110 |       Action: lambda:InvokeFunction
111 |       FunctionName:
112 |         Ref: SQSToLambdaFunction
113 |       Principal: events.amazonaws.com
114 |       SourceArn: !GetAtt [CloudWatchEvent5Minute, Arn]
115 | 
116 |   SQSToLambdaFunction:
117 |     Type: AWS::Lambda::Function
118 |     Properties:
119 |       Description: Invokes Lambda functions with items from SQS queues.
120 |       Handler: index.handler
121 |       MemorySize: 128
122 |       Role: !GetAtt [SQSToLambdaRole, Arn]
123 |       Runtime: nodejs
124 |       Timeout: 300
125 |       Code:
126 |         ZipFile:
127 |           Fn::Sub: |
128 |             var CONFIG = "${QueueToFunctionMapping}".split(",");
129 |             var ONCE = "${Frequency}" === "1Minute";
130 |             {SQSToLambdaCode}
131 | 


--------------------------------------------------------------------------------
/src/sqs-to-lambda.js:
--------------------------------------------------------------------------------
 1 | /* Two global variables should already be injected by the CloudFormation template:
 2 |  *
 3 |  * CONFIG (String) The comma-separated list of queue url/lambda function pairs.
 4 |  * ONCE (Bool) True if the function should exit after polling each queue a single
 5 |  *             time. If False, the function will keep polling until it nears timeout.
 6 |  */
 7 | 
 8 | var AWS = require('aws-sdk');
 9 | var sqs = new AWS.SQS();
10 | var lambda = new AWS.Lambda();
11 | var config = CONFIG;
12 | var once = ONCE;
13 | 
14 | function pollQueue(queueUrl, functionName, remaining, done) {
15 |   if (remaining() < 5000) {
16 |     return done();
17 |   }
18 | 
19 |   if (queueUrl == "" || functionName == "") {
20 |     return done();
21 |   }
22 | 
23 |   sqs.receiveMessage({
24 |     QueueUrl: queueUrl,
25 |     MaxNumberOfMessages: 1,
26 |     WaitTimeSeconds: 1
27 |   }, function(err, data) {
28 |     if (err) {
29 |       console.log(err);
30 |       return done();
31 |     }
32 | 
33 |     if (!data.Messages || data.Messages.length === 0) {
34 |       if (once) {
35 |         return done();
36 |       }
37 | 
38 |       return pollQueue(queueUrl, functionName, remaining, done);
39 |     }
40 | 
41 |     lambda.invoke({
42 |       FunctionName: functionName,
43 |       InvocationType: "Event",
44 |       Payload: JSON.stringify({
45 |         source: "aws.sqs",
46 |         QueueUrl: queueUrl,
47 |         Message: data.Messages[0]
48 |       })
49 |     }, function(err) {
50 |       if (err) {
51 |         console.log(err);
52 |         return done();
53 |       }
54 | 
55 |       return pollQueue(queueUrl, functionName, remaining, done);
56 |     });
57 |   });
58 | }
59 | 
60 | exports.handler = function(event, context) {
61 |   if (config.length === 0) {
62 |     return context.done();
63 |   }
64 | 
65 |   var remainingWorkers = config.length / 2;
66 |   var done = function() {
67 |     remainingWorkers = remainingWorkers - 1;
68 |     if (remainingWorkers == 0) {
69 |       console.log('exiting');
70 |       context.done();
71 |     }
72 |   }
73 | 
74 |   for (var i = 0; i < config.length; i += 2) {
75 |     pollQueue(config[i], config[i+1], context.getRemainingTimeInMillis, done);
76 |   }
77 | }
78 | 


--------------------------------------------------------------------------------