├── .editorconfig ├── .env ├── .gitignore ├── .nodemonignore ├── Gruntfile.coffee ├── LICENSE ├── Procfile ├── README.md ├── dist └── sqs-queue-parallel.js ├── package.json ├── src └── sqs-queue-parallel.coffee └── test └── app.coffee /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org/ 2 | root = true 3 | 4 | [*] 5 | indent_style = tab 6 | end_of_line = lf 7 | insert_final_newline = true 8 | charset = utf-8 9 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | #AWS_ACCESS_KEY= 2 | #AWS_SECRET_KEY= 3 | AWS_REGION=us-east-1 4 | 5 | NODE_ENV=development 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .env.test 3 | -------------------------------------------------------------------------------- /.nodemonignore: -------------------------------------------------------------------------------- 1 | /dist/* 2 | -------------------------------------------------------------------------------- /Gruntfile.coffee: -------------------------------------------------------------------------------- 1 | module.exports = (grunt) -> 2 | grunt.initConfig 3 | pkg: grunt.file.readJSON 'package.json' 4 | usebanner: 5 | options: 6 | banner: """ 7 | /** 8 | * <%= pkg.name %> <%= pkg.version %> 9 | * <%= pkg.description %> 10 | * 11 | * Available under MIT license 12 | */ 13 | """ 14 | position: 'top' 15 | linkbreak: true 16 | dist: 17 | files: 18 | 'dist/sqs-queue-parallel.js': 'dist/sqs-queue-parallel.js' 19 | coffee: 20 | dist: 21 | files: 22 | 'dist/sqs-queue-parallel.js': 'src/sqs-queue-parallel.coffee' 23 | 24 | grunt.loadNpmTasks 'grunt-contrib-coffee' 25 | grunt.loadNpmTasks 'grunt-banner' 26 | 27 | grunt.registerTask 'default', [ 28 | 'coffee' 29 | 'usebanner' 30 | ] 31 | grunt.registerTask 'dist', [ 32 | 'coffee' 33 | 'usebanner' 34 | ] 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Luca Bigon 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: nodemon test/app.coffee -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sqs-queue-parallel.js 2 | 3 | sqs-queue-parallel is a **node.js** library build on top of **Amazon AWS SQS** with **concurrency and parallel** message poll support. 4 | 5 | You can create a poll of SQS queue watchers, each one can receive 1 or more messages from Amazon SQS. 6 | 7 | With sqs-queue-parallel you need just to configure your AWS private keys, setup your one o more `message` event callbacks and wait for new messages to be processed. 8 | 9 | 10 | 11 | # Example 12 | 13 | ```javascript 14 | var SqsQueueParallel = require('sqs-queue-parallel'); 15 | 16 | // Simple configuration: 17 | // - 2 concurrency listeners 18 | // - each listener can receive up to 4 messages 19 | // With this configuration you could receive and parse 8 `message` events in parallel 20 | var queue = new SqsQueueParallel({ 21 | name: "sqs-test", 22 | maxNumberOfMessages: 4, 23 | concurrency: 2 24 | }); 25 | queue.on('message', function (e) 26 | { 27 | console.log('New message: ', e.metadata, e.data.MessageId) 28 | e.deleteMessage(function(err, data) { 29 | e.next(); 30 | }); 31 | }); 32 | queue.on('error', function (err) 33 | { 34 | console.log('There was an error: ', err); 35 | }); 36 | ``` 37 | 38 | 39 | # Download 40 | 41 | You can download and install this library using Node Package Manager (npm): 42 | 43 | ```bash 44 | npm install sqs-queue-parallel --save 45 | ``` 46 | 47 | 48 | # Summary 49 | 50 | * [Constructor](#constructor): 51 | * new SqsQueueParallel(options = {}) 52 | * [Methods](#methods): 53 | * sendMessage(message = {}, callback) 54 | * deleteMessage(receiptHandle, callback) 55 | * changeMessageVisibility(receiptHandle, timeout, callback) 56 | * [Properties](#properties): 57 | * client 58 | * url 59 | * [Events](#events): 60 | * connection 61 | * connect 62 | * message 63 | * error 64 | 65 | * Global env: 66 | * AWS_REGION 67 | * AWS_ACCESS_KEY 68 | * AWS_SECREY_KEY 69 | 70 | 71 | 72 | # Constructor 73 | 74 | 75 | ## new SqsQueueParallel(options = {}) 76 | 77 | First you need to initialize a new object instance with a configuration. 78 | 79 | **Examples:** 80 | 81 | Constructing an object 82 | ```javascript 83 | var queue = new SqsQueueParallel({ name: 'sqs-test' }); 84 | ``` 85 | 86 | **Options Hash (options):** 87 | 88 | * **name** (String) — **_Required_**: name of the remote queue to be watched 89 | * **region** (String) — the region to send/read service requests. Default is `process.env.AWS_REGION` 90 | * **accessKeyId** (String) — your AWS access key ID. Default is `process.env.AWS_ACCESS_KEY` 91 | * **secretAccessKey** (String) — your AWS secret access key. Default is `process.env.AWS_SECRET_KEY` 92 | * **visibilityTimeout** (Integer) — duration (in seconds) that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. 93 | * **waitTimeSeconds** (Integer) — duration (in seconds) for which the call will wait for a message to arrive in the queue before returning. If a message is available, the call will return sooner than WaitTimeSeconds. Default is 20 94 | * **maxNumberOfMessages** (Integer) — maximum number of messages to return. Amazon SQS never returns more messages than this value but may return fewer. Default is 1 95 | * **concurrency** (Integer) — number of concurrency fetcher to start. Default is 1 96 | * **debug** (Boolean) — enable debug mode. Default is false 97 | 98 | 99 | **Important**: 100 | 101 | Each `concurrency` queue can read `maxNumberOfMessages` messages from Amazon SQS. 102 | 103 | For example, **2** `concurrency` queue with **5** `maxNumberOfMessages` can trigger a max of **5 * 2 = 10** `message` events; so it's very important to be carefull, expecially if you're working with I/O streams. 104 | 105 | 106 | 107 | # Properties 108 | 109 | 110 | ## queue.client 111 | 112 | Returns the SQS client object used by the queue. 113 | 114 | 115 | ## queue.url 116 | 117 | Url of the connected queue. 118 | 119 | 120 | 121 | # Methods 122 | 123 | 124 | ## queue.sendMessage(params = {}, callback) 125 | 126 | Build on the top of `SQS.sendMessage()` allow you to easly push a message to the connected queue. 127 | 128 | **Parameters:** 129 | 130 | * **params** (Object) 131 | * body (Any type) — default to {} 132 | 133 | An arbitrary message, could be a string, a number or a object. 134 | * delay (Integer) 135 | 136 | The number of seconds (0 to 900 - 15 minutes) to delay a specific message. Messages with a positive DelaySeconds value become available for processing after the delay time is finished. If you don't specify a value, the default value for the queue applies 137 | 138 | **Callback (callback):** 139 | 140 | ```javascript 141 | function(err, data) {} 142 | ``` 143 | 144 | For more information take checkout the [official AWS documentation](http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/SQS.html#sendMessage-property). 145 | 146 | **Esample:** 147 | 148 | ```javascript 149 | var SqsQueueParallel = require('src/sqs-queue-parallel'); 150 | 151 | var queue = new SqsQueueParallel({ name: "sqs-test" }); 152 | queue.sendMessage({ 153 | body: 'my message', 154 | delay: 10 155 | }); 156 | queue.sendMessage({ 157 | body: [1, 2, 3] 158 | }, function (err, data) 159 | { 160 | if (err) 161 | console.log('There was a problem: ', err); 162 | else 163 | console.log('Item pushed', data); 164 | }); 165 | ``` 166 | 167 | 168 | ## queue.changeMessageVisibility(receiptHandle, timeout, callback) 169 | 170 | Build on the top of `SQS.changeMessageVisibility()` allow you to easly delay a message from the connected queue. 171 | 172 | **Parameters:** 173 | 174 | * **receipHandler** (String) 175 | 176 | The receipt handle associated with the message to delay. 177 | 178 | * **timeout** (Integer) 179 | 180 | The new value (in seconds - from 0 to 43200 - maximum 12 hours) for the message's visibility timeout. 181 | 182 | **Callback (callback):** 183 | 184 | ```javascript 185 | function(err, data) {} 186 | ``` 187 | 188 | For more information take checkout the official [AWS documentation](http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/SQS.html#changeMessageVisibility-property). 189 | 190 | **Esample:** 191 | 192 | ```javascript 193 | var SqsQueueParallel = require('src/sqs-queue-parallel'); 194 | 195 | var queue = new SqsQueueParallel({ name: "sqs-test" }); 196 | queue.changeMessageVisibility('receipt-handle-to-delay-1', 30); 197 | queue.on('message', function (job) 198 | { 199 | if (myTest is true) 200 | job.deleteMessage(); 201 | else 202 | job.changeMessageVisibility(30); 203 | job.next(); 204 | }); 205 | ``` 206 | 207 | 208 | ## queue.deleteMessage(receiptHandle, callback) 209 | 210 | Build on the top of `SQS.deleteMessage()` allow you to easly delete a message from the connected queue. 211 | 212 | **Parameters:** 213 | 214 | * **receipHandler** (String) 215 | 216 | The receipt handle associated with the message to delete. 217 | 218 | **Callback (callback):** 219 | 220 | ```javascript 221 | function(err, data) {} 222 | ``` 223 | 224 | For more information take checkout the official [AWS documentation](http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/SQS.html#deleteMessage-property). 225 | 226 | **Example:** 227 | 228 | ```javascript 229 | var SqsQueueParallel = require('src/sqs-queue-parallel'); 230 | 231 | var queue = new SqsQueueParallel({ name: "sqs-test" }); 232 | queue.deleteMessage('receipt-handle-to-delete-1'); 233 | queue.on('message', function (job) 234 | { 235 | if (myTest is true) 236 | job.deleteMessage(function(err, data) { 237 | job.next(); 238 | }); 239 | }); 240 | ``` 241 | 242 | 243 | 244 | # Events 245 | 246 | 247 | ## connection 248 | 249 | ```javascript 250 | function(urls) { } 251 | ``` 252 | 253 | Triggered when a connection is established with the remote server. 254 | 255 | * **urls** (Array): list of all remotes urls 256 | 257 | 258 | ## connect 259 | 260 | ```javascript 261 | function(url) { } 262 | ``` 263 | 264 | Triggered when the required queue `name` is found in the remote list of queues. 265 | 266 | * **url** (Object): url of the connected queue 267 | 268 | 269 | ## message 270 | 271 | ```javascript 272 | function(message) { } 273 | ``` 274 | 275 | Event triggered each time a new message has been received from the remote queue. 276 | 277 | * **message** (Object) 278 | * type (String): default is "Message" 279 | * data (Unknown): JSON.parsed message.Body or a string (if could not be parsed) 280 | * message (Object): reference to the received message 281 | * metadata (Object): reference to the metadata of the received message 282 | * name (String): name of the remote queue 283 | * url (String): url of the connected queue 284 | * **deleteMessage(callback)** (Function): 285 | 286 | Helper to deleteMessage (or `SQS.deleteMessage()`) when the job is completed; `callback` is the same of the public `deleteMessage()` method 287 | * **changeMessageVisibility(timeout, callback)** (Function): 288 | 289 | Helper to changeMessageVisibility (or `SQS.changeMessageVisibility()`) when the job is completed; `callback` is the same of the public `changeMessageVisibility()` method 290 | * **delay(timeout, callback)** (Function): 291 | 292 | Helper to changeMessageVisibility (or `SQS.changeMessageVisibility()`) without completing the job; `callback` is the same of the public `changeMessageVisibility()` method 293 | * **sendMessage(params = {}, callback)** (Function): send a new message in the queue 294 | * **next()** (Function): call this method when you've completed your jobs in the event callback. 295 | 296 | 297 | ## error 298 | 299 | ```javascript 300 | function(error) { } 301 | ``` 302 | 303 | 304 | 305 | # License 306 | 307 | (The MIT License) 308 | 309 | Copyright (c) 2014 Luca Bigon 310 | 311 | Permission is hereby granted, free of charge, to any person obtaining a copy 312 | of this software and associated documentation files (the "Software"), to deal 313 | in the Software without restriction, including without limitation the rights 314 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 315 | copies of the Software, and to permit persons to whom the Software is 316 | furnished to do so, subject to the following conditions: 317 | 318 | The above copyright notice and this permission notice shall be included in 319 | all copies or substantial portions of the Software. 320 | 321 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 322 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 323 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 324 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 325 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 326 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 327 | THE SOFTWARE. 328 | -------------------------------------------------------------------------------- /dist/sqs-queue-parallel.js: -------------------------------------------------------------------------------- 1 | /** 2 | * sqs-queue-parallel 0.1.6 3 | * Create a poll of Amazon SQS queue watchers and each one can receive 1+ messages 4 | * 5 | * Available under MIT license 6 | */ 7 | (function() { 8 | var AWS, SqsQueueParallel, async, events, globalConfig, _, 9 | __hasProp = {}.hasOwnProperty, 10 | __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; }, 11 | __slice = [].slice; 12 | 13 | AWS = require('aws-sdk'); 14 | 15 | events = require('events'); 16 | 17 | async = require('async'); 18 | 19 | _ = require('lodash'); 20 | 21 | globalConfig = {}; 22 | 23 | module.exports = SqsQueueParallel = (function(_super) { 24 | __extends(SqsQueueParallel, _super); 25 | 26 | SqsQueueParallel.configure = function(config) { 27 | if (config == null) { 28 | config = {}; 29 | } 30 | return globalConfig = _.extend(globalConfig, config); 31 | }; 32 | 33 | function SqsQueueParallel(config) { 34 | var readQueue, self; 35 | if (config == null) { 36 | config = {}; 37 | } 38 | this.config = _.extend({ 39 | region: process.env.AWS_REGION, 40 | accessKeyId: process.env.AWS_ACCESS_KEY, 41 | secretAccessKey: process.env.AWS_SECRET_KEY, 42 | visibilityTimeout: null, 43 | waitTimeSeconds: 20, 44 | maxNumberOfMessages: 1, 45 | name: '', 46 | concurrency: 1, 47 | debug: false 48 | }, globalConfig, config); 49 | this.client = null; 50 | this.url = null; 51 | self = this; 52 | readQueue = function(index) { 53 | if (!(self.listeners("message").length && self.url)) { 54 | return; 55 | } 56 | return async.waterfall([ 57 | function(next) { 58 | var options; 59 | if (self.config.debug) { 60 | console.log("SqsQueueParallel " + self.config.name + "[" + index + "]: waiting messages"); 61 | } 62 | return self.client.receiveMessage((options = { 63 | QueueUrl: self.url, 64 | AttributeNames: ["All"], 65 | MaxNumberOfMessages: self.config.maxNumberOfMessages, 66 | WaitTimeSeconds: self.config.waitTimeSeconds 67 | }, self.config.visibilityTimeout != null ? options.VisibilityTimeout = self.config.visibilityTimeout : void 0, options), next); 68 | }, function(queue, next) { 69 | var _ref; 70 | if (!((_ref = queue.Messages) != null ? _ref[0] : void 0)) { 71 | return next(null); 72 | } 73 | if (self.config.debug) { 74 | console.log("SqsQueueParallel " + self.config.name + "[" + index + "]: " + queue.Messages.length + " new messages"); 75 | } 76 | return async.eachSeries(queue.Messages, function(message, next) { 77 | return self.emit("message", { 78 | type: 'message', 79 | data: JSON.parse(message.Body) || message.Body, 80 | message: message, 81 | metadata: queue.ResponseMetadata, 82 | url: self.url, 83 | name: self.config.name, 84 | next: next, 85 | deleteMessage: function(cb) { 86 | return self.deleteMessage(message.ReceiptHandle, cb); 87 | }, 88 | delay: function(timeout, cb) { 89 | return self.changeMessageVisibility(message.ReceiptHandle, timeout, cb); 90 | }, 91 | changeMessageVisibility: function(timeout, cb) { 92 | return self.changeMessageVisibility(message.ReceiptHandle, timeout, cb); 93 | } 94 | }); 95 | }, function() { 96 | return next(null); 97 | }); 98 | } 99 | ], function(err) { 100 | if (err) { 101 | self.emit.apply(self, ["error"].concat(__slice.call(arguments))); 102 | } 103 | return process.nextTick(function() { 104 | return readQueue(index); 105 | }); 106 | }); 107 | }; 108 | this.addListener('newListener', function(name) { 109 | if (name !== 'message') { 110 | return; 111 | } 112 | if (self.config.debug) { 113 | console.info("SqsQueueParallel " + self.config.name + ": new listener"); 114 | } 115 | if (!this.client || this.listeners("message").length === 1) { 116 | return this.connect(function(err) { 117 | if (err || !self.url || !self.listeners("message").length) { 118 | return; 119 | } 120 | return _.times(self.config.concurrency || 1, function(index) { 121 | return readQueue(index); 122 | }); 123 | }); 124 | } 125 | }); 126 | if (this.config.debug) { 127 | this.on('connection', function(urls) { 128 | return console.log("SqsQueueParallel: connection to SQS", urls); 129 | }); 130 | this.on('connect', function() { 131 | return console.log("SqsQueueParallel " + self.config.name + ": connected with url `" + self.url + "`"); 132 | }); 133 | this.on('error', function(e) { 134 | return console.log("SqsQueueParallel " + self.config.name + ": connection failed", e); 135 | }); 136 | } 137 | } 138 | 139 | SqsQueueParallel.prototype.connect = function(cb) { 140 | var self; 141 | if (!(this.client && this.url)) { 142 | this.once('connect', function() { 143 | return cb(null); 144 | }); 145 | if (this.client && !this.url) { 146 | return; 147 | } 148 | } 149 | if (this.client) { 150 | return cb(null); 151 | } 152 | self = this; 153 | this.client = new AWS.SQS({ 154 | region: this.config.region, 155 | accessKeyId: this.config.accessKeyId, 156 | secretAccessKey: this.config.secretAccessKey 157 | }); 158 | async.waterfall([ 159 | function(next) { 160 | return self.client.listQueues({ 161 | QueueNamePrefix: self.config.name 162 | }, next); 163 | }, function(data, next) { 164 | var re, url, _i, _len, _ref; 165 | re = new RegExp("/[\\d]+/" + self.config.name + "$"); 166 | self.emit('connection', data.QueueUrls); 167 | _ref = data.QueueUrls; 168 | for (_i = 0, _len = _ref.length; _i < _len; _i++) { 169 | url = _ref[_i]; 170 | if (re.test(url)) { 171 | self.emit('connect', self.url = url); 172 | } 173 | } 174 | if (!self.url) { 175 | self.emit('error', new Error('Queue not found')); 176 | return next('Queue not found'); 177 | } 178 | } 179 | ], function(err) { 180 | if (!err) { 181 | return; 182 | } 183 | self.emit('error', err); 184 | return cb.apply(null, arguments); 185 | }); 186 | return this; 187 | }; 188 | 189 | SqsQueueParallel.prototype.sendMessage = function(message, cb) { 190 | var self; 191 | if (message == null) { 192 | message = {}; 193 | } 194 | if (cb == null) { 195 | cb = function() {}; 196 | } 197 | self = this; 198 | this.connect(function(err) { 199 | var params; 200 | if (err) { 201 | return cb.apply(null, arguments); 202 | } 203 | if (self.config.debug) { 204 | console.log("SqsQueueParallel " + self.config.name + ": before sendMessage with url `" + self.url + "`"); 205 | } 206 | params = { 207 | MessageBody: JSON.stringify(message.body || {}), 208 | QueueUrl: self.url 209 | }; 210 | if (message.delay != null) { 211 | params.DelaySeconds = message.delay; 212 | } 213 | return self.client.sendMessage(params, cb); 214 | }); 215 | return this; 216 | }; 217 | 218 | SqsQueueParallel.prototype.deleteMessage = function(receiptHandle, cb) { 219 | var self; 220 | if (cb == null) { 221 | cb = function() {}; 222 | } 223 | self = this; 224 | this.connect(function(err) { 225 | if (err) { 226 | return cb.apply(null, arguments); 227 | } 228 | if (self.config.debug) { 229 | console.log("SqsQueueParallel " + self.config.name + ": before deleteMessage " + receiptHandle + " with url `" + self.url + "`"); 230 | } 231 | return self.client.deleteMessage({ 232 | QueueUrl: self.url, 233 | ReceiptHandle: receiptHandle 234 | }, cb); 235 | }); 236 | return this; 237 | }; 238 | 239 | SqsQueueParallel.prototype.changeMessageVisibility = function(receiptHandle, timeout, cb) { 240 | var self; 241 | if (timeout == null) { 242 | timeout = 30; 243 | } 244 | if (cb == null) { 245 | cb = function() {}; 246 | } 247 | self = this; 248 | this.connect(function(err) { 249 | if (err) { 250 | return cb.apply(null, arguments); 251 | } 252 | if (self.config.debug) { 253 | console.log("SqsQueueParallel " + self.config.name + ": before changeMessageVisibility " + receiptHandle + " with url `" + self.url + "`"); 254 | } 255 | return self.client.changeMessageVisibility({ 256 | QueueUrl: self.url, 257 | ReceiptHandle: receiptHandle, 258 | VisibilityTimeout: timeout 259 | }, cb); 260 | }); 261 | return this; 262 | }; 263 | 264 | return SqsQueueParallel; 265 | 266 | })(events.EventEmitter); 267 | 268 | }).call(this); 269 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sqs-queue-parallel", 3 | "description": "Create a poll of Amazon SQS queue watchers and each one can receive 1+ messages", 4 | "main": "dist/sqs-queue-parallel", 5 | "homepage": "https://github.com/bigluck/sqs-queue-parallel", 6 | "author": "Luca Bigon", 7 | "version": "0.1.6", 8 | "license": "MIT", 9 | "licenses": [ 10 | { 11 | "type": "MIT", 12 | "url": "https://github.com/bigluck/sqs-queue-parallel/raw/master/LICENSE" 13 | } 14 | ], 15 | "keywords": [ 16 | "sqs", 17 | "queue", 18 | "poll", 19 | "amazon", 20 | "aws" 21 | ], 22 | "maintainers": [ 23 | { 24 | "name": "Luca Bigon", 25 | "website": "http://www.exit4web.net/" 26 | } 27 | ], 28 | "scripts": { 29 | "build": "grunt dist" 30 | }, 31 | "repository": { 32 | "type": "git", 33 | "url": "https://github.com/bigluck/sqs-queue-parallel.git" 34 | }, 35 | "bugs": { 36 | "url": "https://github.com/bigluck/sqs-queue-parallel/issues" 37 | }, 38 | "dependencies": { 39 | "aws-sdk": "^2.0.0-rc11", 40 | "async": "^0.2.10", 41 | "lodash": "^2.4.1" 42 | }, 43 | "devDependencies": { 44 | "grunt": "^0.4.4", 45 | "coffee-script": "^1.7.1", 46 | "nodemon": "^1.0.15", 47 | "grunt-contrib-coffee": "^0.10.1", 48 | "grunt-banner": "^0.2.2" 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/sqs-queue-parallel.coffee: -------------------------------------------------------------------------------- 1 | AWS = require 'aws-sdk' 2 | events = require 'events' 3 | async = require 'async' 4 | _ = require 'lodash' 5 | 6 | globalConfig = {} 7 | 8 | module.exports = class SqsQueueParallel extends events.EventEmitter 9 | @configure: (config={}) -> 10 | globalConfig = _.extend globalConfig, config 11 | constructor: (config={}) -> 12 | @config = _.extend 13 | region: process.env.AWS_REGION 14 | accessKeyId: process.env.AWS_ACCESS_KEY 15 | secretAccessKey: process.env.AWS_SECRET_KEY 16 | visibilityTimeout: null 17 | waitTimeSeconds: 20 18 | maxNumberOfMessages: 1 19 | name: '' 20 | concurrency: 1 21 | debug: false 22 | , globalConfig, config 23 | @client = null 24 | @url = null 25 | self = @ 26 | 27 | readQueue = (index) -> 28 | return unless self.listeners("message").length and self.url 29 | async.waterfall [ 30 | (next) -> 31 | console.log "SqsQueueParallel #{ self.config.name }[#{ index }]: waiting messages" if self.config.debug 32 | self.client.receiveMessage ( 33 | options = 34 | QueueUrl: self.url 35 | AttributeNames: ["All"] 36 | MaxNumberOfMessages: self.config.maxNumberOfMessages 37 | WaitTimeSeconds: self.config.waitTimeSeconds 38 | options.VisibilityTimeout = self.config.visibilityTimeout if self.config.visibilityTimeout? 39 | options 40 | ) 41 | , next 42 | (queue, next) -> 43 | return next null unless queue.Messages?[0] 44 | console.log "SqsQueueParallel #{ self.config.name }[#{ index }]: #{ queue.Messages.length } new messages" if self.config.debug 45 | async.eachSeries queue.Messages, (message, next) -> 46 | self.emit "message", 47 | type: 'message' 48 | data: JSON.parse(message.Body) or message.Body 49 | message: message 50 | metadata: queue.ResponseMetadata 51 | url: self.url 52 | name: self.config.name 53 | next: next 54 | deleteMessage: (cb) -> 55 | self.deleteMessage message.ReceiptHandle, cb 56 | delay: (timeout, cb) -> 57 | self.changeMessageVisibility message.ReceiptHandle, timeout, cb 58 | changeMessageVisibility: (timeout, cb) -> 59 | self.changeMessageVisibility message.ReceiptHandle, timeout, cb 60 | , -> 61 | next null 62 | ], (err) -> 63 | self.emit "error", arguments... if err 64 | process.nextTick -> 65 | readQueue index 66 | 67 | @addListener 'newListener', (name) -> 68 | return unless name is 'message' 69 | console.info "SqsQueueParallel #{ self.config.name }: new listener" if self.config.debug 70 | if not @client or @listeners("message").length is 1 71 | @connect (err) -> 72 | return if err or not self.url or not self.listeners("message").length 73 | _.times self.config.concurrency or 1, (index) -> 74 | readQueue index 75 | if @config.debug 76 | @on 'connection', (urls) -> 77 | console.log "SqsQueueParallel: connection to SQS", urls 78 | @on 'connect', -> 79 | console.log "SqsQueueParallel #{ self.config.name }: connected with url `#{ self.url }`" 80 | @on 'error', (e) -> 81 | console.log "SqsQueueParallel #{ self.config.name }: connection failed", e 82 | connect: (cb) -> 83 | unless @client and @url 84 | @once 'connect', -> 85 | cb null 86 | return if @client and not @url 87 | return cb null if @client 88 | self = @ 89 | @client = new AWS.SQS 90 | region: @config.region 91 | accessKeyId: @config.accessKeyId 92 | secretAccessKey: @config.secretAccessKey 93 | async.waterfall [ 94 | (next) -> 95 | self.client.listQueues 96 | QueueNamePrefix: self.config.name 97 | , next 98 | (data, next) -> 99 | re = new RegExp "/[\\d]+/#{ self.config.name }$" 100 | self.emit 'connection', data.QueueUrls 101 | self.emit 'connect', self.url = url for url in data.QueueUrls when re.test url 102 | unless self.url 103 | self.emit 'error', new Error 'Queue not found' 104 | next 'Queue not found' 105 | ], (err) -> 106 | return unless err 107 | self.emit 'error', err 108 | cb arguments... 109 | @ 110 | sendMessage: (message={}, cb=->) -> 111 | self = @ 112 | @connect (err) -> 113 | return cb arguments... if err 114 | console.log "SqsQueueParallel #{ self.config.name }: before sendMessage with url `#{ self.url }`" if self.config.debug 115 | params = 116 | MessageBody: JSON.stringify message.body or {} 117 | QueueUrl: self.url 118 | params.DelaySeconds = message.delay if message.delay? 119 | self.client.sendMessage params, cb 120 | @ 121 | deleteMessage: (receiptHandle, cb=->) -> 122 | self = @ 123 | @connect (err) -> 124 | return cb arguments... if err 125 | console.log "SqsQueueParallel #{ self.config.name }: before deleteMessage #{ receiptHandle } with url `#{ self.url }`" if self.config.debug 126 | self.client.deleteMessage 127 | QueueUrl: self.url 128 | ReceiptHandle: receiptHandle 129 | , cb 130 | @ 131 | changeMessageVisibility: (receiptHandle, timeout=30, cb=->) -> 132 | self = @ 133 | @connect (err) -> 134 | return cb arguments... if err 135 | console.log "SqsQueueParallel #{ self.config.name }: before changeMessageVisibility #{ receiptHandle } with url `#{ self.url }`" if self.config.debug 136 | self.client.changeMessageVisibility 137 | QueueUrl: self.url 138 | ReceiptHandle: receiptHandle 139 | VisibilityTimeout: timeout 140 | , cb 141 | @ 142 | -------------------------------------------------------------------------------- /test/app.coffee: -------------------------------------------------------------------------------- 1 | SqsQueueParallel = require 'src/sqs-queue-parallel' 2 | 3 | queue = new SqsQueueParallel 4 | name: "sqs-test" 5 | maxNumberOfMessages: 4 6 | concurrency: 2 7 | 8 | queue.on 'message', (e) -> 9 | console.log 'New message: ', e.metadata, e.data.MessageId 10 | e.deleteMessage() 11 | e.next() 12 | 13 | queue.on 'error', (err) -> 14 | console.log 'There was an error: ', err 15 | --------------------------------------------------------------------------------