├── .gitignore ├── .npmrc ├── .git-blame-ignore-revs ├── eslint.config.cjs ├── package.json ├── LICENSE ├── README.md └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | example.js 3 | pnpm-lock.yaml 4 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | @clear:registry=https://npm.pkg.github.com/clear 2 | legacy-peer-deps=true -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Introduced eslint and prettier, and applied format 2 | f574bd2627370d6736c892bbecfb02a34cc44a75 3 | -------------------------------------------------------------------------------- /eslint.config.cjs: -------------------------------------------------------------------------------- 1 | const clearConfig = require("@clear/style/eslint/js"); 2 | const globals = require("globals"); 3 | 4 | module.exports = [ 5 | // Add additional globals for mocha and node support 6 | { 7 | languageOptions: { 8 | globals: { 9 | ...globals.node, 10 | 11 | should: false, 12 | }, 13 | }, 14 | }, 15 | ...clearConfig, 16 | ]; 17 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@clearlrs/sqs", 3 | "version": "2.1.0", 4 | "dependencies": { 5 | "request": "~2.88.2" 6 | }, 7 | "repository": "git://github.com/clear/sqs", 8 | "description": "Forked from mafintosh/sqs to fix unmaintained bugs – A message queue using Amazon Simple Queue Service.", 9 | "keywords": [ 10 | "sqs", 11 | "aws", 12 | "amazon", 13 | "queue", 14 | "message" 15 | ], 16 | "author": "Mathias Buus Madsen ", 17 | "prettier": "@clear/style/prettier", 18 | "devDependencies": { 19 | "@clear/style": "^0.3.0", 20 | "eslint": "^9.0.0", 21 | "globals": "^15.11.0", 22 | "prettier": "^3.3.3" 23 | }, 24 | "scripts": { 25 | "lint": "eslint .", 26 | "format": "eslint --fix ." 27 | }, 28 | "pnpm": { 29 | "overrides": { 30 | "request>tough-cookie@<4.1.3": "4.1.3" 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sqs 2 | 3 | A message queue using [Amazon Simple Queue Service](http://docs.amazonwebservices.com/AWSSimpleQueueService/latest/APIReference/Welcome.html). 4 | 5 | npm install sqs 6 | 7 | Usage is simple 8 | 9 | ``` js 10 | var sqs = require('sqs'); 11 | 12 | var queue = sqs({ 13 | access:'my-aws-access-key', 14 | secret:'my-aws-secret-key', 15 | region:'us-east-1' // defaults to us-east-1 16 | }); 17 | 18 | // push some data to the test queue 19 | queue.push('test', { 20 | some:'data' 21 | }, function () { 22 | // complete 23 | }); 24 | 25 | // pull messages from the test queue 26 | queue.pull('test', function(message, callback) { 27 | console.log('someone pushed', message); 28 | callback(); // we are done with this message - pull a new one 29 | // calling the callback will also delete the message from the queue 30 | }); 31 | ``` 32 | 33 | ## API 34 | 35 | var queue = sqs(options) 36 | 37 | Create a queue instance. Available options are 38 | 39 | * `options.access` - Your AWS access key (required) 40 | * `options.secret` - Your AWS secret key (required) 41 | * `options.region` - The AWS region for the queue. Defaults to us-east-1 42 | * `options.namespace` - Prefix all queues with `namespace`. Defaults to empty string. 43 | * `options.https` - If true use https. Defaults to false. 44 | * `options.raw` - If true it doesn't parse message's body to JSON. Defaults to false. 45 | * `options.proxy` - If set to a URL it will overwrite the default AWS SQS url. 46 | 47 | Some of the options can be configured using env vars. See below for more. 48 | 49 | queue.push(name, message, [callback]) 50 | 51 | Push a new message to the queue defined by name. If the queue doesn't exist sqs will create it. Optional callback called once request is completed. 52 | 53 | queue.pull(name, [workers=1], onmessage) 54 | 55 | Pull messages from the queue defined by name. 56 | 57 | The pull flow is as follows: 58 | 59 | 1. A message is pulled and is passed to `onmessage(message, callback)` 60 | 2. You process the message 61 | 3. Call `callback` when you are done and the message will be deleted from the queue. 62 | 4. Goto 1 63 | 64 | If for some reason the callback is not called amazon sqs will re-add the message to the queue after 30s. 65 | 66 | Delete a queue. 67 | 68 | queue.delete(name, [callback]) 69 | 70 | ## Fault tolerance 71 | 72 | Both `pull` and `push` will retry multiple times if a network error occurs or if amazon sqs is temporary unavailable. 73 | 74 | ## Env config 75 | 76 | You can use env variables to configure `sqs` as well 77 | 78 | ``` 79 | SQS_ACCESS_KEY=my-access-key 80 | SQS_SECRET_KEY=my-secret-key 81 | SQS_REGION=us-east-1 82 | ``` 83 | 84 | In your application you can just call an empty constructor 85 | 86 | ``` js 87 | var queue = sqs(); 88 | ``` 89 | 90 | This is very useful if you dont want to hardcode your keys in the application. 91 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var request = require("request"); 2 | var crypto = require("crypto"); 3 | var qs = require("querystring"); 4 | var http = require("http"); 5 | var https = require("https"); 6 | var events = require("events"); 7 | 8 | var SIGNATURE_METHOD = "HmacSHA256"; 9 | var SIGNATURE_VERSION = "2"; 10 | var SIGNATURE_TTL = 150 * 1000; 11 | var VERSION = "2012-11-05"; 12 | var DEFAULT_REGION = "us-east-1"; 13 | 14 | var text = function (xml, tag) { 15 | var i = xml.indexOf("<" + tag + ">"); 16 | if (i === -1) return null; 17 | i += tag.length + 2; 18 | 19 | return xml.substring(i, xml.indexOf("") 28 | .replace(/&/g, "&"); 29 | }; 30 | 31 | var range = function (num) { 32 | return Array(num).join(",").split(","); 33 | }; 34 | 35 | module.exports = function (options) { 36 | options = options || {}; 37 | 38 | options.access = options.access || process.env.SQS_ACCESS_KEY || process.env.AWS_ACCESS_KEY_ID; 39 | options.secret = 40 | options.secret || process.env.SQS_SECRET_KEY || process.env.AWS_SECRET_ACCESS_KEY; 41 | options.region = 42 | options.region || process.env.SQS_REGION || process.env.AWS_REGION || DEFAULT_REGION; 43 | options.raw = options.raw || false; 44 | options.proxy = options.proxy || false; 45 | 46 | if (!options.access || !options.secret) 47 | throw new Error("options.access and options.secret are required"); 48 | 49 | var queues = {}; 50 | var closed = false; 51 | var proto = options.https ? "https://" : "http://"; 52 | 53 | var host = "sqs." + options.region + ".amazonaws.com"; 54 | if (options.proxy) host = options.proxy; 55 | 56 | var namespace = options.namespace ? options.namespace + "-" : ""; 57 | 58 | namespace = namespace.replace(/[^a-zA-Z0-9]/g, "-").replace(/-+/g, "-"); 59 | 60 | var queryURL = function (action, path, params) { 61 | params = params || {}; 62 | 63 | params.Action = action; 64 | params.AWSAccessKeyId = options.access; 65 | params.SignatureMethod = SIGNATURE_METHOD; 66 | params.SignatureVersion = SIGNATURE_VERSION; 67 | params.Expires = new Date(Date.now() + SIGNATURE_TTL).toISOString(); 68 | params.Version = VERSION; 69 | 70 | var stringToSign = 71 | "GET\n" + 72 | host + 73 | "\n" + 74 | path + 75 | "\n" + 76 | Object.keys(params) 77 | .sort() 78 | .map(function (name) { 79 | return ( 80 | name + 81 | "=" + 82 | encodeURIComponent(params[name]) 83 | .replace(/[!'()]/g, escape) 84 | .replace(/\*/g, "%2A") 85 | ); 86 | }) 87 | .join("&"); 88 | 89 | params.Signature = crypto 90 | .createHmac("sha256", options.secret) 91 | .update(stringToSign) 92 | .digest("base64"); 93 | 94 | return proto + host + path + "?" + qs.stringify(params); 95 | }; 96 | 97 | var retry = function (req, url, callback) { 98 | var retries = 0; 99 | var action = function () { 100 | req(url, { timeout: 10000 }, function (err, res) { 101 | if (!err && res.statusCode >= 500) 102 | err = new Error("invalid status-code: " + res.statusCode); 103 | if (!err) { 104 | if (callback) return callback(err, res); 105 | return; 106 | } 107 | retries++; 108 | if (retries > 15) { 109 | if (callback) return callback(err); 110 | return that.emit("error", new Error("could not send " + url)); 111 | } 112 | setTimeout(action, retries * 1000); 113 | }); 114 | }; 115 | 116 | action(); 117 | }; 118 | 119 | var queueURL = function (name, callback) { 120 | if (queues[name]) return queues[name](callback); 121 | 122 | var stack = [callback]; 123 | 124 | queues[name] = function (callback) { 125 | stack.push(callback); 126 | }; 127 | 128 | var onresult = function (err, url) { 129 | if (err) return that.emit("error", err); 130 | 131 | queues[name] = function (callback) { 132 | callback(url); 133 | }; 134 | 135 | while (stack.length) { 136 | stack.shift()(url); 137 | } 138 | }; 139 | 140 | retry(request, queryURL("CreateQueue", "/", { QueueName: name }), function (err) { 141 | if (err) return onresult(err); 142 | retry(request, queryURL("GetQueueUrl", "/", { QueueName: name }), function (err, res) { 143 | if (err || res.statusCode !== 200) { 144 | var errMessage = res.body ? text(res.body, "Message") : ""; 145 | return onresult( 146 | err || 147 | new Error( 148 | "invalid status code from GetQueueUrl: " + 149 | res.statusCode + 150 | (errMessage ? ". " + errMessage : ""), 151 | ), 152 | ); 153 | } 154 | onresult(null, "/" + text(res.body, "QueueUrl").split("/").slice(3).join("/")); 155 | }); 156 | }); 157 | }; 158 | 159 | var that = new events.EventEmitter(); 160 | 161 | that.push = function (name, message, callback) { 162 | name = namespace + name; 163 | 164 | queueURL(name, function (url) { 165 | var body = options.raw ? message : JSON.stringify(message); 166 | retry(request, queryURL("SendMessage", url, { MessageBody: body }), callback); 167 | }); 168 | }; 169 | 170 | that.delete = that.del = function (name, callback) { 171 | name = namespace + name; 172 | 173 | queueURL(name, function (url) { 174 | retry(request, queryURL("DeleteQueue", url), callback); 175 | }); 176 | }; 177 | 178 | var wait = options.wait || 2000; 179 | 180 | that.pull = function (name, workers, onmessage) { 181 | if (typeof workers === "function") return that.pull(name, options.workers || 1, workers); 182 | 183 | name = namespace + name; 184 | 185 | var agent = options.https 186 | ? new https.Agent({ maxSockets: workers }) 187 | : new http.Agent({ maxSockets: workers }); // long poll should use its own agent 188 | var req = request.defaults({ agent: agent }); 189 | 190 | range(workers).forEach(function () { 191 | var next = function () { 192 | if (closed) return; 193 | 194 | queueURL(name, function (url) { 195 | req( 196 | queryURL("ReceiveMessage", url, { WaitTimeSeconds: 20 }), 197 | function (err, res) { 198 | if (err || res.statusCode !== 200) return setTimeout(next, wait); 199 | 200 | var body = text(res.body, "Body"); 201 | 202 | if (!body) return options.wait ? setTimeout(next, wait) : next(); 203 | 204 | var receipt = text(res.body, "ReceiptHandle"); 205 | 206 | try { 207 | body = options.raw ? unscape(body) : JSON.parse(unscape(body)); 208 | } catch (err) { 209 | return next(); 210 | } 211 | 212 | // Callback takes an arbitrary error and a `stop` attribute to stop 213 | // polling for messages 214 | onmessage(body, function (err, ctx) { 215 | if (err) return next(); 216 | retry( 217 | request, 218 | queryURL("DeleteMessage", url, { ReceiptHandle: receipt }), 219 | ); 220 | if (ctx && ctx.stop === true) return; 221 | next(); 222 | }); 223 | }, 224 | ); 225 | }); 226 | }; 227 | 228 | next(); 229 | }); 230 | }; 231 | 232 | that.close = function () { 233 | closed = true; 234 | }; 235 | 236 | return that; 237 | }; 238 | --------------------------------------------------------------------------------