├── example.js ├── package.json ├── LICENSE ├── Readme.md └── index.js /example.js: -------------------------------------------------------------------------------- 1 | 2 | var SQSWorker = require('./') 3 | 4 | var options = 5 | { url: 'https://sqs.eu-west-1.amazonaws.com/001123456789/my-queue' 6 | } 7 | 8 | var queue = new SQSWorker(options, worker) 9 | 10 | function worker(notifi, done) { 11 | var message 12 | try { 13 | message = JSON.parse(notifi.Data) 14 | } catch (err) { 15 | throw err 16 | } 17 | 18 | console.log(message) 19 | 20 | done() 21 | } 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sqs-worker", 3 | "version": "0.0.5", 4 | "description": "Handle messages in Amazon Simple Queue Service easily", 5 | "main": "index.js", 6 | "dependencies": {}, 7 | "optionalDependencies": { 8 | "aws-sdk": "^2.0.15" 9 | }, 10 | "devDependencies": {}, 11 | "scripts": {}, 12 | "repository": { 13 | "type": "git", 14 | "url": "https://github.com/tellnes/sqs-worker.git" 15 | }, 16 | "keywords": [ 17 | "aws", 18 | "amazon", 19 | "sqs", 20 | "simple", 21 | "queue", 22 | "service" 23 | ], 24 | "author": "Christian Tellnes (http://christian.tellnes.com)", 25 | "license": "MIT", 26 | "bugs": { 27 | "url": "https://github.com/tellnes/sqs-worker/issues" 28 | }, 29 | "homepage": "https://github.com/tellnes/sqs-worker", 30 | "publishConfig": { 31 | "registry": "https://registry.npmjs.org/" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Christian Tellnes 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # sqs-worker 2 | 3 | sqs-worker reads messages from Amazon Simple Queue Service and call a function 4 | provided by you. It also deletes the message from the queue if you signals 5 | that everything went successfully. 6 | 7 | Note: It does not guarantee that you don's sees the same message twice, 8 | therefore should you design your application to handle that. 9 | 10 | 11 | ## Usage 12 | 13 | ```js 14 | var SQSWorker = require('sqs-worker') 15 | 16 | var options = 17 | { url: 'https://sqs.eu-west-1.amazonaws.com/001123456789/my-queue' 18 | } 19 | 20 | var queue = new SQSWorker(options, worker) 21 | 22 | function worker(notifi, done) { 23 | var message 24 | try { 25 | message = JSON.parse(notifi.Data) 26 | } catch (err) { 27 | throw err 28 | } 29 | 30 | // Do something with `message` 31 | 32 | var success = true 33 | 34 | // Call `done` when you are done processing a message. 35 | // If everything went successfully and you don't want to see it any more, 36 | // set the second parameter to `true`. 37 | done(null, success) 38 | } 39 | 40 | ``` 41 | 42 | ## Install 43 | 44 | npm install sqs-worker 45 | 46 | ## License 47 | 48 | MIT 49 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 2 | module.exports = SQSWorker 3 | 4 | function SQSWorker(options, fn) { 5 | if (!(this instanceof SQSWorker)) return new SQSWorker(options, fn) 6 | 7 | this.client = options.client || new (require('aws-sdk').SQS)(options) 8 | 9 | // Five seconds more than WaitTimeSeconds 10 | this.client.config.httpOptions.timeout = 25000 11 | 12 | this.url = options.url 13 | this.parallel = options.parallel || 1 14 | this.timeout = options.timeout || void 0 15 | this.parse = options.parse 16 | this.log = options.log || console 17 | this.attributes = Array.isArray(options.attributes) ? options.attributes.slice() : [] 18 | this.attempts = options.attempts || 3 19 | 20 | if (this.attempts && !~this.attributes.indexOf('ApproximateReceiveCount')) { 21 | this.attributes.push('ApproximateReceiveCount') 22 | } 23 | 24 | this.fn = fn 25 | 26 | this.handling = 0 27 | this.receiving = 0 28 | 29 | this.maybeMore() 30 | } 31 | 32 | SQSWorker.prototype.maybeMore = function (retries) { 33 | if (this.receiving) return 34 | if (this.handling >= this.parallel) return 35 | 36 | this.receiving++ 37 | 38 | retries = retries || 0 39 | retries++ 40 | 41 | var self = this 42 | 43 | var params = 44 | { 'QueueUrl': this.url 45 | , 'MaxNumberOfMessages': Math.min(this.parallel, 10) 46 | , 'VisibilityTimeout': this.timeout 47 | , 'WaitTimeSeconds': 20 48 | , 'AttributeNames': this.attributes 49 | } 50 | 51 | this.client.receiveMessage(params, function (err, data) { 52 | self.receiving-- 53 | 54 | if (err) { 55 | self.log.error({ err: err, params: params }, 'failed to receive messages') 56 | setTimeout(self.maybeMore.bind(self), Math.min(Math.pow(2, retries), 300) * 1000, retries) 57 | return 58 | } 59 | 60 | // self.log.info({ params: params, response: data }, 'receiveMessage response') 61 | 62 | if (Array.isArray(data.Messages)) { 63 | data.Messages.map(self.handleMessage, self) 64 | } 65 | 66 | self.maybeMore() 67 | }) 68 | } 69 | 70 | SQSWorker.prototype.handleMessage = function (message) { 71 | this.handling++ 72 | 73 | var payload = message.Body 74 | 75 | if (this.parse) { 76 | try { 77 | payload = this.parse(payload) 78 | } catch (err) { 79 | this.log.error({ err: err, message: message }, 'error parsing message body') 80 | return 81 | } 82 | } 83 | 84 | var self = this 85 | 86 | if (this.fn.length === 2) { 87 | this.fn(payload, callback) 88 | } else { 89 | this.fn(payload, message, callback) 90 | } 91 | 92 | function callback(err, del) { 93 | self.handling-- 94 | 95 | if (err) { 96 | self.log.error({ err: err, message: message, payload: payload }, 'error handling message') 97 | del = (self.attempts && Number(message['Attributes']['ApproximateReceiveCount']) >= self.attempts) 98 | } 99 | 100 | var params = 101 | { 'QueueUrl': self.url 102 | , 'ReceiptHandle': message.ReceiptHandle 103 | } 104 | 105 | if (!del) { 106 | params['VisibilityTimeout'] = 0 107 | } 108 | 109 | self.client[del ? 'deleteMessage' : 'changeMessageVisibility'](params, function (err) { 110 | if (err) { 111 | self.log.error( 112 | { err: err, message: message, payload: payload } 113 | , 'failed to ' + (del ? 'delete message' : 'change visibility timeout') 114 | ) 115 | // Keep retrying even if updating the props fails 116 | // return 117 | } 118 | 119 | self.maybeMore() 120 | }) 121 | } 122 | } 123 | --------------------------------------------------------------------------------