├── .editorconfig ├── test ├── lib │ └── helper.js ├── fixtures │ ├── PostgresAdapter.js │ └── SqliteAdapter.js ├── stats.js ├── ticket.js ├── store.js ├── tickets.js ├── complex.js └── basic.js ├── .gitignore ├── lib ├── tickets.js ├── ticket.js ├── worker.js └── queue.js ├── package.json ├── .github └── workflows │ └── node.js.yml ├── LICENSE ├── README.md └── yarn.lock /.editorconfig: -------------------------------------------------------------------------------- 1 | [*.{js,json}] 2 | indent_style = space 3 | indent_size = 2 -------------------------------------------------------------------------------- /test/lib/helper.js: -------------------------------------------------------------------------------- 1 | 2 | exports.destroyQueues = function () { 3 | [this.q, this.q1, this.q2].forEach(function (q) { 4 | if (!q) return; 5 | setTimeout(function () { 6 | q.destroy(); 7 | }, 15); 8 | }); 9 | }; 10 | -------------------------------------------------------------------------------- /test/fixtures/PostgresAdapter.js: -------------------------------------------------------------------------------- 1 | var PostgresAdapter = require('../../lib/stores/PostgresAdapter'); 2 | 3 | function MockPostgresAdapter(opts) { 4 | opts.verbose = false; 5 | opts.username = 'diamond'; 6 | opts.dbname = 'diamond'; 7 | PostgresAdapter.call(this, opts); 8 | } 9 | 10 | MockPostgresAdapter.prototype = Object.create(PostgresAdapter.prototype); 11 | 12 | module.exports = MockPostgresAdapter; 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # node-waf configuration 20 | .lock-wscript 21 | 22 | # Compiled binary addons (http://nodejs.org/api/addons.html) 23 | build/Release 24 | 25 | # Dependency directory 26 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 27 | node_modules 28 | .DS_Store 29 | 30 | # Test file 31 | test.js 32 | *.test.js -------------------------------------------------------------------------------- /test/fixtures/SqliteAdapter.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs-extra'); 2 | var uuid = require('uuid'); 3 | var SqliteAdapter = require('../../lib/stores/SqliteAdapter'); 4 | 5 | function MockSqliteAdapter(opts) { 6 | opts.verbose = false; 7 | opts.path = opts.path || uuid.v4() + '.sqlite'; 8 | SqliteAdapter.call(this, opts); 9 | } 10 | 11 | MockSqliteAdapter.prototype = Object.create(SqliteAdapter.prototype); 12 | 13 | MockSqliteAdapter.prototype.close = function (cb) { 14 | var after = function () { 15 | SqliteAdapter.prototype.close.call(this, cb) 16 | } 17 | if (this.path === ':memory:') return after(); 18 | fs.unlink(this.path, function (err) { 19 | after(); 20 | }); 21 | } 22 | 23 | module.exports = MockSqliteAdapter; 24 | -------------------------------------------------------------------------------- /lib/tickets.js: -------------------------------------------------------------------------------- 1 | 2 | var Ticket = require('./ticket'); 3 | 4 | function Tickets() { 5 | this.tickets = []; 6 | } 7 | 8 | Tickets.prototype._apply = function (fn, args) { 9 | this.tickets.forEach(function (ticket) { 10 | ticket[fn].apply(ticket, args); 11 | }) 12 | } 13 | 14 | Tickets.prototype.push = function (ticket) { 15 | var self = this; 16 | if (ticket instanceof Tickets) { 17 | return ticket.tickets.forEach(function (ticket) { 18 | self.push(ticket) 19 | }) 20 | } 21 | if (ticket instanceof Ticket) { 22 | if (self.tickets.indexOf(ticket) === -1) { 23 | self.tickets.push(ticket); 24 | } 25 | } 26 | } 27 | 28 | Object.keys(Ticket.prototype).forEach(function (method) { 29 | Tickets.prototype[method] = function () { 30 | this._apply(method, arguments); 31 | } 32 | }) 33 | 34 | module.exports = Tickets; 35 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "better-queue", 3 | "version": "3.8.12", 4 | "description": "Better Queue for NodeJS", 5 | "main": "lib/queue.js", 6 | "directories": { 7 | "test": "test" 8 | }, 9 | "scripts": { 10 | "test": "mocha" 11 | }, 12 | "repository": { 13 | "type": "git", 14 | "url": "https://github.com/diamondio/better-queue.git" 15 | }, 16 | "keywords": [ 17 | "queue", 18 | "cargo", 19 | "async", 20 | "timeout", 21 | "priority" 22 | ], 23 | "author": "Diamond Inc. ", 24 | "license": "MIT", 25 | "bugs": { 26 | "url": "https://github.com/diamondio/better-queue/issues" 27 | }, 28 | "homepage": "https://github.com/diamondio/better-queue", 29 | "devDependencies": { 30 | "mocha": "^10.0.0", 31 | "mocha-junit-reporter": "^1.12.1" 32 | }, 33 | "dependencies": { 34 | "better-queue-memory": "^1.0.1", 35 | "node-eta": "^0.9.0", 36 | "uuid": "^9.0.0" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /.github/workflows/node.js.yml: -------------------------------------------------------------------------------- 1 | # This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 3 | 4 | name: Node.js CI 5 | 6 | on: 7 | push: 8 | branches: [ "master" ] 9 | pull_request: 10 | branches: [ "master" ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | strategy: 18 | matrix: 19 | node-version: [14.x] 20 | # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ 21 | 22 | steps: 23 | - uses: actions/checkout@v3 24 | - name: Use Node.js ${{ matrix.node-version }} 25 | uses: actions/setup-node@v3 26 | with: 27 | node-version: ${{ matrix.node-version }} 28 | cache: 'npm' 29 | - run: npm ci 30 | - run: npm run build --if-present 31 | - run: npm test 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Leander 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test/stats.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var helper = require('./lib/helper'); 3 | var Queue = require('../lib/queue'); 4 | 5 | describe('Stats', function() { 6 | afterEach(helper.destroyQueues); 7 | 8 | it('should get stat', function (done) { 9 | var completed = 0; 10 | var elapsedTotals = 0; 11 | var q = new Queue(function (wait, cb) { 12 | setTimeout(function () { 13 | cb() 14 | }, wait) 15 | }) 16 | q.on('task_finish', function (id, result, stat) { 17 | completed++; 18 | elapsedTotals += stat.elapsed; 19 | }) 20 | q.on('drain', function () { 21 | var stats = q.getStats(); 22 | assert.ok(stats.peak); 23 | assert.equal(3, stats.total); 24 | assert.equal(Math.round(elapsedTotals / 3 * 1000) / 1000, Math.round(stats.average * 1000) / 1000); 25 | done(); 26 | }) 27 | q.push(1); 28 | q.push(1); 29 | q.push(1); 30 | this.q = q; 31 | }) 32 | 33 | it('should reset stat', function (done) { 34 | var queued = 0; 35 | var elapsedTotal = 0; 36 | var q = new Queue(function (wait, cb) { 37 | setTimeout(function () { 38 | cb() 39 | }, wait) 40 | }, { id: function (n, cb) { cb(null, n) } }) 41 | q.push(1, function () { 42 | q.push(1, function () { 43 | q.resetStats(); 44 | q.on('task_finish', function (id, result, stat) { 45 | if (id !== '2') return; 46 | assert.ok(stat.elapsed > 0); 47 | var stats = q.getStats(); 48 | assert.equal(1, stats.peak); 49 | assert.equal(1, stats.total); 50 | assert.equal(stats.average, stat.elapsed); 51 | done(); 52 | }) 53 | q.push(2); 54 | }); 55 | }); 56 | this.q = q; 57 | }) 58 | 59 | 60 | }) 61 | -------------------------------------------------------------------------------- /test/ticket.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var Ticket = require('../lib/ticket'); 3 | 4 | describe('Ticket', function() { 5 | var t; 6 | 7 | before(function () { 8 | t = new Ticket(); 9 | }) 10 | 11 | it('should instantiate', function () { 12 | assert.ok(t); 13 | }) 14 | 15 | it('should accept', function () { 16 | assert.ok(!t.isAccepted, 'ticket is not accepted'); 17 | t.accept(); 18 | assert.ok(t.isAccepted, 'ticket is accepted'); 19 | }) 20 | 21 | it('should queue', function () { 22 | assert.ok(!t.isQueued, 'ticket is not queued'); 23 | t.queued(); 24 | assert.ok(t.isQueued, 'ticket is queued'); 25 | }) 26 | 27 | it('should start and stop', function () { 28 | assert.ok(!t.isStarted, 'ticket is not started'); 29 | t.started(); 30 | assert.ok(t.isStarted, 'ticket is started'); 31 | t.stopped(); 32 | assert.ok(!t.isStarted, 'ticket is stopped'); 33 | }) 34 | 35 | it('should finish and emit', function (done) { 36 | assert.ok(!t.isFinished, 'ticket is not finished'); 37 | t.once('finish', function (result) { 38 | assert.deepEqual(result, { x: 1 }); 39 | assert.ok(t.isFinished, 'ticket is finished'); 40 | done(); 41 | }) 42 | t.finish({ x: 1 }); 43 | }) 44 | 45 | it('should fail and emit', function (done) { 46 | assert.ok(!t.isFailed, 'ticket not failed'); 47 | t.once('failed', function (err) { 48 | assert.equal(err, 'some_error'); 49 | assert.ok(t.isFailed, 'ticket failed'); 50 | done(); 51 | }) 52 | t.failed('some_error'); 53 | }) 54 | 55 | it('should progress and emit', function (done) { 56 | t.started(); 57 | t.once('progress', function (progress) { 58 | assert.equal(progress.pct, 50); 59 | assert.equal(progress.complete, 1); 60 | assert.equal(progress.total, 2); 61 | assert.equal(progress.message, 'test'); 62 | assert.equal(typeof progress.eta, 'string'); 63 | done() 64 | }); 65 | t.progress({ 66 | complete: 1, 67 | total: 2, 68 | message: 'test' 69 | }); 70 | }) 71 | 72 | 73 | }) 74 | -------------------------------------------------------------------------------- /lib/ticket.js: -------------------------------------------------------------------------------- 1 | 2 | var util = require('util'); 3 | var EE = require('events').EventEmitter; 4 | var ETA = require('node-eta'); 5 | 6 | function Ticket(opts) { 7 | this.isAccepted = false; 8 | this.isQueued = false; 9 | this.isStarted = false; 10 | this.isFailed = false; 11 | this.isFinished = false; 12 | this.result = null; 13 | this.status = 'created'; 14 | this.eta = new ETA(); 15 | } 16 | 17 | util.inherits(Ticket, EE); 18 | 19 | Ticket.prototype.accept = function () { 20 | this.status = 'accepted'; 21 | this.isAccepted = true; 22 | this.emit('accepted'); 23 | } 24 | 25 | Ticket.prototype.queued = function () { 26 | this.status = 'queued'; 27 | this.isQueued = true; 28 | this.emit('queued'); 29 | } 30 | 31 | Ticket.prototype.unqueued = function () { 32 | this.status = 'accepted'; 33 | this.isQueued = false; 34 | this.emit('unqueued'); 35 | } 36 | 37 | Ticket.prototype.started = function () { 38 | this.eta.count = 1; 39 | this.eta.start(); 40 | this.isStarted = true; 41 | this.status = 'in-progress'; 42 | this.emit('started'); 43 | } 44 | 45 | Ticket.prototype.failed = function (msg) { 46 | this.isFailed = true; 47 | this.isFinished = true; 48 | this.status = 'failed'; 49 | this.emit('failed', msg); 50 | } 51 | 52 | Ticket.prototype.finish = function (result) { 53 | this.eta.done = this.eta.count; 54 | this.isFinished = true; 55 | this.status = 'finished'; 56 | this.result = result; 57 | this.emit('finish', this.result); 58 | } 59 | 60 | Ticket.prototype.stopped = function () { 61 | this.eta = new ETA(); 62 | this.isFinished = false; 63 | this.isStarted = false; 64 | this.status = 'queued'; 65 | this.result = null; 66 | this.emit('stopped'); 67 | } 68 | 69 | Ticket.prototype.progress = function (progress) { 70 | this.eta.done = progress.complete; 71 | this.eta.count = progress.total; 72 | this.emit('progress', { 73 | complete: this.eta.done, 74 | total: this.eta.count, 75 | pct: (this.eta.done/this.eta.count)*100, 76 | eta: this.eta.format('{{etah}}'), 77 | message: progress.message 78 | }); 79 | } 80 | 81 | module.exports = Ticket; 82 | -------------------------------------------------------------------------------- /test/store.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var Queue = require('../lib/queue'); 3 | 4 | describe('Store Usage', function() { 5 | 6 | it('should retry connect', function (done) { 7 | var tries = 0; 8 | var s = { 9 | connect: function (cb) { 10 | tries++; 11 | if (tries < 3) { 12 | return cb('failed'); 13 | } 14 | done(); 15 | }, 16 | getTask: function (taskId, cb) { cb() }, 17 | putTask: function (taskId, task, priority, cb) { cb() }, 18 | takeFirstN: function (n, cb) { cb() }, 19 | takeLastN: function (n, cb) { cb() } 20 | } 21 | var q = new Queue(function (batch, cb) { cb() }, { 22 | storeMaxRetries: 5, 23 | storeRetryTimeout: 1, 24 | store: s 25 | }) 26 | }) 27 | 28 | it('should fail retry', function (done) { 29 | var tries = 0; 30 | var s = { 31 | connect: function (cb) { 32 | tries++; 33 | cb('failed'); 34 | }, 35 | getTask: function (taskId, cb) { cb() }, 36 | putTask: function (taskId, task, priority, cb) { cb() }, 37 | takeFirstN: function (n, cb) { cb() }, 38 | takeLastN: function (n, cb) { cb() } 39 | } 40 | var q = new Queue(function (batch, cb) { cb() }, { 41 | storeMaxRetries: 2, 42 | storeRetryTimeout: 1, 43 | store: s 44 | }) 45 | .on('error', function (e) { 46 | assert.ok(e); 47 | done(); 48 | }) 49 | }) 50 | 51 | it('should queue length', function (done) { 52 | var queued = false; 53 | var s = { 54 | connect: function (cb) { cb(null, 5) }, 55 | getTask: function (taskId, cb) { cb() }, 56 | putTask: function (taskId, task, priority, cb) { cb() }, 57 | getLock: function (lockId, cb) { cb(null, { 'task-id': queued ? 2 : 1 }) }, 58 | getRunningTasks: function (cb) { cb(null, {}) }, 59 | takeFirstN: function (n, cb) { cb(null, 'lock-id') }, 60 | takeLastN: function (n, cb) { cb() }, 61 | releaseLock: function (lockId, cb) { cb(null) }, 62 | } 63 | var q = new Queue(function (n, cb) { 64 | if (n === 2) { 65 | assert.equal(q.length, 5); 66 | done(); 67 | } 68 | cb(); 69 | }, { store: s, autoResume: false }) 70 | q.push(1).on('queued', function (e) { 71 | queued = true; 72 | assert.equal(q.length, 6); 73 | }) 74 | }) 75 | 76 | it('should fail if there is no length on connect', function (done) { 77 | var queued = false; 78 | var s = { 79 | connect: function (cb) { cb() } 80 | } 81 | try { 82 | var q = new Queue(function (n, cb) {}, { store: s }) 83 | } catch (e) { 84 | done(); 85 | } 86 | }) 87 | 88 | // TODO: Test progress 89 | 90 | }) 91 | -------------------------------------------------------------------------------- /test/tickets.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var Ticket = require('../lib/ticket'); 3 | var Tickets = require('../lib/tickets'); 4 | 5 | describe('Tickets', function() { 6 | var ts, t1, t2; 7 | 8 | before(function () { 9 | t1 = new Ticket(); 10 | t2 = new Ticket(); 11 | ts = new Tickets(); 12 | ts.push(t1); 13 | ts.push(t2); 14 | }) 15 | 16 | it('should accept', function () { 17 | assert.ok(!t1.isAccepted, 'ticket 1 is not accepted'); 18 | assert.ok(!t2.isAccepted, 'ticket 2 is not accepted'); 19 | ts.accept(); 20 | assert.ok(t1.isAccepted, 'ticket 1 is accepted'); 21 | assert.ok(t2.isAccepted, 'ticket 2 is accepted'); 22 | }) 23 | 24 | it('should queue', function () { 25 | assert.ok(!t1.isQueued, 'ticket 1 is not queued'); 26 | assert.ok(!t2.isQueued, 'ticket 2 is not queued'); 27 | ts.queued(); 28 | assert.ok(t1.isQueued, 'ticket 1 is queued'); 29 | assert.ok(t2.isQueued, 'ticket 2 is queued'); 30 | }) 31 | 32 | it('should start and stop', function () { 33 | assert.ok(!t1.isStarted, 'ticket 1 is not started'); 34 | assert.ok(!t2.isStarted, 'ticket 2 is not started'); 35 | ts.started(); 36 | assert.ok(t1.isStarted, 'ticket 1 is started'); 37 | assert.ok(t2.isStarted, 'ticket 2 is started'); 38 | ts.stopped(); 39 | assert.ok(!t1.isStarted, 'ticket 1 is stopped'); 40 | assert.ok(!t2.isStarted, 'ticket 2 is stopped'); 41 | }) 42 | 43 | it('should finish and emit', function (done) { 44 | assert.ok(!t1.isFinished, 'ticket 1 is not finished'); 45 | assert.ok(!t2.isFinished, 'ticket 2 is not finished'); 46 | t2.once('finish', function (result) { 47 | assert.deepEqual(result, { x: 1 }); 48 | assert.ok(t2.isFinished, 'ticket 2 is finished'); 49 | done(); 50 | }) 51 | ts.finish({ x: 1 }); 52 | }) 53 | 54 | it('should fail and emit', function (done) { 55 | assert.ok(!t1.isFailed, 'ticket 1 not failed'); 56 | assert.ok(!t2.isFailed, 'ticket 2 not failed'); 57 | var called = 0; 58 | t1.once('failed', function (err) { 59 | assert.equal(err, 'some_error'); 60 | assert.ok(t1.isFailed, 'ticket 1 failed'); 61 | called++; 62 | if (called == 2) { done() } 63 | }) 64 | t2.once('failed', function (err) { 65 | assert.equal(err, 'some_error'); 66 | assert.ok(t2.isFailed, 'ticket 2 failed'); 67 | called++; 68 | if (called == 2) { done() } 69 | }) 70 | ts.failed('some_error'); 71 | }) 72 | 73 | it('should progress and emit', function (done) { 74 | t1.once('progress', function (progress) { 75 | assert.equal(progress.pct, 50); 76 | assert.equal(progress.complete, 1); 77 | assert.equal(progress.total, 2); 78 | assert.equal(progress.message, 'test'); 79 | assert.equal(typeof progress.eta, 'string'); 80 | done() 81 | }); 82 | ts.progress({ 83 | complete: 1, 84 | total: 2, 85 | message: 'test' 86 | }); 87 | }) 88 | 89 | 90 | }) 91 | -------------------------------------------------------------------------------- /lib/worker.js: -------------------------------------------------------------------------------- 1 | 2 | var util = require('util'); 3 | var EE = require('events').EventEmitter; 4 | var ETA = require('node-eta'); 5 | 6 | function Worker(opts) { 7 | this.fn = opts.fn; 8 | this.batch = opts.batch; 9 | this.single = opts.single; 10 | this.active = false; 11 | this.cancelled = false; 12 | this.failTaskOnProcessException = opts.failTaskOnProcessException; 13 | } 14 | 15 | util.inherits(Worker, EE); 16 | 17 | Worker.prototype.setup = function () { 18 | var self = this; 19 | 20 | // Internal 21 | self._taskIds = Object.keys(self.batch); 22 | self._process = {}; 23 | self._waiting = {}; 24 | self._eta = new ETA(); 25 | 26 | // Task counts 27 | self.counts = { 28 | finished: 0, 29 | failed: 0, 30 | completed: 0, 31 | total: self._taskIds.length, 32 | }; 33 | 34 | // Progress 35 | self.status = 'ready'; 36 | self.progress = { 37 | tasks: {}, 38 | complete: 0, 39 | total: self._taskIds.length, 40 | eta: '', 41 | }; 42 | 43 | // Setup 44 | self._taskIds.forEach(function (taskId, id) { 45 | self._waiting[id] = true; 46 | self.progress.tasks[id] = { 47 | pct: 0, 48 | complete: 0, 49 | total: 1, 50 | } 51 | }) 52 | } 53 | 54 | Worker.prototype.start = function () { 55 | var self = this; 56 | if (self.active) return; 57 | 58 | self.setup(); 59 | self._eta.count = self.progress.total; 60 | self._eta.start(); 61 | 62 | self.active = true; 63 | self.status = 'in-progress'; 64 | var tasks = self._taskIds.map(function (taskId) { return self.batch[taskId] }); 65 | if (self.single) { 66 | tasks = tasks[0] 67 | } 68 | try { 69 | self._process = self.fn.call(self, tasks, function (err, result) { 70 | if (!self.active) return; 71 | if (err) { 72 | self.failedBatch(err); 73 | } else { 74 | self.finishBatch(result); 75 | } 76 | }) 77 | } catch (err) { 78 | if (self.failTaskOnProcessException) { 79 | self.failedBatch(err); 80 | } else { 81 | throw new Error(err); 82 | } 83 | } 84 | self._process = self._process || {}; 85 | } 86 | 87 | Worker.prototype.end = function () { 88 | if (!this.active) return; 89 | this.status = 'finished'; 90 | this.active = false; 91 | this.emit('end'); 92 | } 93 | 94 | Worker.prototype.resume = function () { 95 | if (typeof this._process.resume === 'function') { 96 | this._process.resume(); 97 | } 98 | this.status = 'in-progress'; 99 | } 100 | 101 | Worker.prototype.pause = function () { 102 | if (typeof this._process.pause === 'function') { 103 | this._process.pause(); 104 | } 105 | this.status = 'paused'; 106 | } 107 | 108 | Worker.prototype.cancel = function () { 109 | this.cancelled = true; 110 | if (typeof this._process.cancel === 'function') { 111 | this._process.cancel(); 112 | } 113 | if (typeof this._process.abort === 'function') { 114 | this._process.abort(); 115 | } 116 | this.failedBatch('cancelled'); 117 | } 118 | 119 | Worker.prototype.failedBatch = function (msg) { 120 | var self = this; 121 | if (!self.active) return; 122 | Object.keys(self._waiting).forEach(function (id) { 123 | if (!self._waiting[id]) return; 124 | self.failedTask(id, msg); 125 | }) 126 | self.emit('failed', msg); 127 | self.end(); 128 | } 129 | 130 | Worker.prototype.failedTask = function (id, msg) { 131 | var self = this; 132 | if (!self.active) return; 133 | if (self._waiting[id]) { 134 | self._waiting[id] = false; 135 | self.counts.failed++; 136 | self.counts.completed++; 137 | self.emit('task_failed', id, msg); 138 | } 139 | } 140 | 141 | Worker.prototype.finishBatch = function (result) { 142 | var self = this; 143 | if (!self.active) return; 144 | Object.keys(self._waiting).forEach(function (id) { 145 | if (!self._waiting[id]) return; 146 | self.finishTask(id, result); 147 | }) 148 | self.emit('finish', result); 149 | self.end(); 150 | } 151 | 152 | Worker.prototype.finishTask = function (id, result) { 153 | var self = this; 154 | if (!self.active) return; 155 | if (self._waiting[id]) { 156 | self._waiting[id] = false; 157 | self.counts.finished++; 158 | self.counts.completed++; 159 | self.emit('task_finish', id, result); 160 | } 161 | } 162 | 163 | Worker.prototype.progressBatch = function (complete, total, msg) { 164 | var self = this; 165 | if (!self.active) return; 166 | Object.keys(self._waiting).forEach(function (id) { 167 | if (!self._waiting[id]) return; 168 | self.progressTask(id, complete, total, msg); 169 | }) 170 | self.progress.complete = 0; 171 | self._taskIds.forEach(function (taskId, id) { 172 | self.progress.complete += self.progress.tasks[id].pct; 173 | }) 174 | self._eta.done = self.progress.complete; 175 | self.progress.eta = self._eta.format('{{etah}}') 176 | self.progress.message = msg || ''; 177 | self.emit('progress', self.progress); 178 | } 179 | 180 | Worker.prototype.progressTask = function (id, complete, total, msg) { 181 | var self = this; 182 | if (!self.active) return; 183 | if (self._waiting[id]) { 184 | self.progress.tasks[id].complete = complete; 185 | self.progress.tasks[id].total = self.progress.tasks[id].total || total; 186 | self.progress.tasks[id].message = self.progress.tasks[id].message || msg; 187 | self.progress.tasks[id].pct = Math.max(0, Math.min(1, complete/total)); 188 | self.emit('task_progress', id, self.progress.tasks[id]); 189 | } 190 | } 191 | 192 | module.exports = Worker; 193 | -------------------------------------------------------------------------------- /test/complex.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var helper = require('./lib/helper'); 3 | 4 | var Queue = require('../lib/queue'); 5 | var MemoryStore = require('better-queue-memory'); 6 | 7 | describe('Complex Queue', function() { 8 | afterEach(helper.destroyQueues); 9 | 10 | it('should run in batch mode', function (done) { 11 | var q = new Queue({ 12 | batchSize: 3, 13 | process: function (batch, cb) { 14 | assert.equal(batch.length, 3); 15 | var total = 0; 16 | batch.forEach(function (task) { 17 | total += task; 18 | }) 19 | cb(null, total); 20 | }, 21 | }) 22 | var queued = 0; 23 | q.on('task_queued', function () { 24 | queued++; 25 | if (queued >= 3) { 26 | q.resume(); 27 | } 28 | }) 29 | q.pause(); 30 | q.push(1, function (err, total) { 31 | assert.equal(total, 6); 32 | }) 33 | q.push(2, function (err, total) { 34 | assert.equal(total, 6); 35 | }) 36 | q.push(3, function (err, total) { 37 | assert.equal(total, 6); 38 | done(); 39 | }) 40 | this.q = q; 41 | }) 42 | 43 | it('should store properly', function (done) { 44 | var self = this; 45 | var s = new MemoryStore(); 46 | var finished = 0; 47 | var queued = 0; 48 | var q1 = new Queue(function (n, cb) { throw new Error('failed') }, { store: s }) 49 | q1.on('task_queued', function () { 50 | queued++; 51 | if (queued >= 3) { 52 | var q2 = new Queue(function (n, cb) { 53 | finished++; 54 | cb(); 55 | if (finished === 3) { 56 | done(); 57 | } 58 | }, { store: s }); 59 | self.q2 = q2; 60 | } 61 | }) 62 | q1.pause(); 63 | q1.push(1); 64 | q1.push(2); 65 | q1.push(3); 66 | this.q1 = q1; 67 | }) 68 | 69 | it('should retry', function (done) { 70 | var tries = 0; 71 | var q = new Queue(function (n, cb) { 72 | tries++; 73 | if (tries === 3) { 74 | cb(); 75 | done(); 76 | } else { 77 | cb('fail'); 78 | } 79 | }, { maxRetries: 3 }); 80 | q.push(1); 81 | this.q = q; 82 | }) 83 | 84 | it('should retry jobs with same id', function (done) { 85 | const maxRetries = 3 86 | const getJob = (id = 1) => ({ id, attempt: 0 }) 87 | 88 | const q = new Queue((job, cb) => { 89 | job.attempt++ 90 | cb(job) 91 | }, { maxRetries }) 92 | 93 | let job = getJob(1) 94 | q.push(job) 95 | .on('failed', err => { 96 | assert.equal(err.id, job.id) 97 | assert.equal(err.attempt, maxRetries) 98 | job = getJob(2) 99 | q.push(job) 100 | .on('failed', err => { 101 | assert.equal(err.id, job.id) 102 | assert.equal(err.attempt,maxRetries) 103 | job = getJob(1) 104 | q.push(job) 105 | .on('failed', err => { 106 | assert.equal(err.id, job.id) 107 | assert.equal(err.attempt, maxRetries, 'job with id 1 shall fail after maxRetries') 108 | done() 109 | }) 110 | }) 111 | }) 112 | }) 113 | 114 | it('should fail retry', function (done) { 115 | var tries = 0; 116 | var q = new Queue(function (n, cb) { 117 | tries++; 118 | if (tries === 3) { 119 | cb(); 120 | } else { 121 | cb('fail'); 122 | } 123 | }, { maxRetries: 2, autoResume: true }) 124 | q.on('task_failed', function () { 125 | done(); 126 | }); 127 | q.push(1); 128 | this.q = q; 129 | }) 130 | 131 | it('should respect afterProcessDelay', function (done) { 132 | var delay = 100; 133 | var finished = 0; 134 | var startTime; 135 | var q = new Queue(function (task, cb) { 136 | finished++; 137 | cb(); 138 | if (finished === 1) { 139 | startTime = +(new Date()); 140 | } else if (finished === 2) { 141 | var endTime = +(new Date()); 142 | var elapsedTime = endTime - startTime; 143 | assert(elapsedTime >= delay); 144 | done(); 145 | } 146 | }, { batchSize: 1, afterProcessDelay: delay }); 147 | var queued = 0; 148 | q.on('task_queued', function () { 149 | queued++; 150 | if (queued >= 2) { 151 | q.resume(); 152 | } 153 | }) 154 | q.pause(); 155 | q.push(1); 156 | q.push(2); 157 | this.q = q; 158 | }) 159 | 160 | it('should max timeout', function (done) { 161 | var q = new Queue(function (tasks, cb) {}, { maxTimeout: 1 }) 162 | q.on('task_failed', function (taskId, msg) { 163 | assert.equal(msg, 'task_timeout'); 164 | done(); 165 | }); 166 | q.push(1, function (err, r) { 167 | assert.equal(err, 'task_timeout'); 168 | }); 169 | this.q = q; 170 | }) 171 | 172 | it('should merge tasks', function (done) { 173 | var q = new Queue(function (o, cb) { 174 | if (o.id === 1) { 175 | assert.equal(o.x, 3); 176 | cb(); 177 | } else { 178 | cb(); 179 | } 180 | }, { 181 | id: 'id', 182 | merge: function (a, b, cb) { 183 | a.x += b.x; 184 | cb(null, a); 185 | } 186 | }) 187 | var queued = 0; 188 | q.on('task_queued', function () { 189 | queued++; 190 | if (queued >= 2) { 191 | q.resume(); 192 | } 193 | }) 194 | q.on('task_finish', function (taskId, r) { 195 | if (taskId === '1') { 196 | done(); 197 | } 198 | }) 199 | q.pause() 200 | q.push({ id: '0', x: 4 }); 201 | q.push({ id: '1', x: 1 }, function (err, r) { 202 | assert.ok(!err) 203 | }); 204 | q.push({ id: '1', x: 2 }, function (err, r) { 205 | assert.ok(!err); 206 | }); 207 | this.q = q; 208 | }) 209 | 210 | it('should respect id property (string)', function (done) { 211 | var q = new Queue(function (o, cb) { 212 | if (o.name === 'john') { 213 | assert.equal(o.x, 4); 214 | cb(); 215 | } 216 | if (o.name === 'mary') { 217 | assert.equal(o.x, 5); 218 | cb(); 219 | } 220 | if (o.name === 'jim') { 221 | assert.equal(o.x, 2); 222 | cb(); 223 | } 224 | }, { 225 | id: 'name', 226 | merge: function (a, b, cb) { 227 | a.x += b.x; 228 | cb(null, a); 229 | } 230 | }) 231 | var finished = 0; 232 | var queued = 0; 233 | q.on('task_finish', function (taskId, r) { 234 | finished++; 235 | if (finished >= 3) done(); 236 | }) 237 | q.on('task_queued', function (taskId, r) { 238 | queued++; 239 | if (queued >= 3) { 240 | q.resume(); 241 | } 242 | }) 243 | q.pause(); 244 | q.push({ name: 'john', x: 4 }); 245 | q.push({ name: 'mary', x: 3 }); 246 | q.push({ name: 'jim', x: 1 }); 247 | q.push({ name: 'jim', x: 1 }); 248 | q.push({ name: 'mary', x: 2 }); 249 | this.q = q; 250 | }) 251 | 252 | it('should respect id property (function)', function (done) { 253 | var finished = 0; 254 | var q = new Queue(function (n, cb) { 255 | cb(null, n) 256 | }, { 257 | batchDelay: 3, 258 | id: function (n, cb) { 259 | cb(null, n % 2 === 0 ? 'even' : 'odd'); 260 | }, 261 | merge: function (a, b, cb) { 262 | cb(null, a+b); 263 | } 264 | }) 265 | var finished = 0; 266 | var queued = 0; 267 | q.on('task_queued', function (taskId, r) { 268 | }) 269 | q.on('task_finish', function (taskId, r) { 270 | finished++; 271 | if (taskId === 'odd') { 272 | assert.equal(r, 9); 273 | } 274 | if (taskId === 'even') { 275 | assert.equal(r, 6); 276 | } 277 | if (finished >= 2) { 278 | done(); 279 | } 280 | }) 281 | q.push(1); 282 | q.push(2); 283 | q.push(3); 284 | q.push(4); 285 | q.push(5); 286 | this.q = q; 287 | }) 288 | 289 | it('should cancel if running', function (done) { 290 | var ran = 0; 291 | var cancelled = false; 292 | var q = new Queue(function (n, cb) { 293 | ran++; 294 | if (ran >= 2) { 295 | cb(); 296 | } 297 | if (ran === 3) { 298 | assert.ok(cancelled); 299 | done(); 300 | } 301 | return { 302 | cancel: function () { 303 | cancelled = true; 304 | } 305 | } 306 | }, { id: 'id', cancelIfRunning: true }) 307 | q.push({ id: 1 }) 308 | .on('started', function () { 309 | q.push({ id: 2 }); 310 | setTimeout(function () { 311 | q.push({ id: 1 }); 312 | }, 1) 313 | }); 314 | this.q = q; 315 | }) 316 | 317 | it('failed task should not stack overflow', function (done) { 318 | var count = 0; 319 | var q = new Queue(function (n, cb) { 320 | count++ 321 | if (count > 100) { 322 | cb(); 323 | done(); 324 | } else { 325 | cb('fail'); 326 | } 327 | }, { 328 | maxRetries: Infinity 329 | }) 330 | q.push(1); 331 | this.q = q; 332 | }) 333 | 334 | // it('drain should still work with persistent queues', function (done) { 335 | // var q = new Queue(function (n, cb) { 336 | // setTimeout(cb, 1); 337 | // }, { 338 | // store: { 339 | // type: 'sql', 340 | // dialect: 'sqlite', 341 | // path: 'testqueue.sql' 342 | // } 343 | // }) 344 | // var drained = false; 345 | // q.on('drain', function () { 346 | // drained = true; 347 | // done(); 348 | // }); 349 | // q.push(1); 350 | // this.q = q; 351 | // }) 352 | 353 | // it('drain should still work when there are persisted items at load time', function (done) { 354 | // var initialQueue = new Queue(function (n, cb) { 355 | // setTimeout(cb, 100); 356 | // }, { 357 | // store: { 358 | // type: 'sql', 359 | // dialect: 'sqlite', 360 | // path: 'testqueue.sql' 361 | // } 362 | // }); 363 | // initialQueue.push('' + 1); 364 | // initialQueue.push('' + 2); 365 | // setTimeout(function () { 366 | // // This effectively captures the queue in a state where there were unprocessed items 367 | // fs.copySync('testqueue.sql', 'testqueue2.sql'); 368 | // initialQueue.destroy(); 369 | // var persistedQueue = new Queue(function (n, cb) { 370 | // setTimeout(cb, 1); 371 | // }, { 372 | // store: { 373 | // type: 'sql', 374 | // dialect: 'sqlite', 375 | // path: 'testqueue2.sql' 376 | // } 377 | // }) 378 | // var drained = false; 379 | // persistedQueue.on('drain', function () { 380 | // drained = true; 381 | // }); 382 | // persistedQueue.push(2); 383 | // setTimeout(function () { 384 | // persistedQueue.destroy(); 385 | 386 | // assert.ok(drained); 387 | // done(); 388 | // }, 140) 389 | // }, 40) 390 | // }) 391 | }) 392 | -------------------------------------------------------------------------------- /test/basic.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var helper = require('./lib/helper'); 3 | var Queue = require('../lib/queue'); 4 | 5 | describe('Basic Queue', function() { 6 | afterEach(helper.destroyQueues); 7 | 8 | it('should succeed', function (done) { 9 | var q = new Queue(function (n, cb) { 10 | cb(null, n+1) 11 | }, { autoResume: true }) 12 | q.on('task_finish', function (taskId, r) { 13 | assert.equal(r, 2); 14 | done(); 15 | }) 16 | q.push(1, function (err, r) { 17 | assert.equal(r, 2); 18 | }) 19 | this.q = q; 20 | }); 21 | 22 | it('should fail task if failTaskOnProcessException is true', function (done) { 23 | var q = new Queue(function (n, cb) { 24 | throw new Error("failed"); 25 | }, { autoResume: true }) 26 | q.on('task_failed', function (taskId, err) { 27 | assert.equal(err.message, "failed"); 28 | done(); 29 | }) 30 | q.push(1) 31 | this.q = q; 32 | }); 33 | 34 | it('should emit an error if failTaskOnProcessException is false', function (done) { 35 | var q = new Queue(function (n, cb) { 36 | throw new Error("failed"); 37 | }, { failTaskOnProcessException: false, autoResume: true }) 38 | q.on('error', function () { 39 | done(); 40 | }) 41 | q.push(1) 42 | this.q = q; 43 | }); 44 | 45 | it('should fail', function (done) { 46 | var q = new Queue(function (n, cb) { 47 | cb('nope') 48 | }, { autoResume: true }) 49 | q.on('task_failed', function (taskId, msg) { 50 | assert.equal(msg, 'nope'); 51 | done(); 52 | }) 53 | q.push(1, function (err, r) { 54 | assert.equal(err, 'nope'); 55 | }) 56 | this.q = q; 57 | }); 58 | 59 | it('should run fifo', function (done) { 60 | var finished = 0; 61 | var queued = 0; 62 | var q = new Queue(function (num, cb) { cb() }) 63 | q.on('task_finish', function () { 64 | if (finished >= 3) { 65 | done(); 66 | } 67 | }) 68 | q.on('task_queued', function () { 69 | queued++; 70 | if (queued >= 3) { 71 | q.resume(); 72 | } 73 | }) 74 | q.pause(); 75 | q.push(1, function (err, r) { 76 | assert.equal(finished, 0); 77 | finished++; 78 | }) 79 | q.push(2, function (err, r) { 80 | assert.equal(finished, 1); 81 | finished++; 82 | }) 83 | q.push(3, function (err, r) { 84 | assert.equal(finished, 2); 85 | finished++; 86 | }) 87 | this.q = q; 88 | }) 89 | 90 | it('should prioritize', function (done) { 91 | var q = new Queue(function (num, cb) { cb() }, { 92 | priority: function (n, cb) { 93 | if (n === 2) return cb(null, 10); 94 | if (n === 1) return cb(null, 5); 95 | return cb(null, 1); 96 | } 97 | }) 98 | q.pause(); 99 | var finished = 0; 100 | var queued = 0; 101 | q.on('task_queued', function () { 102 | queued++; 103 | if (queued === 3) { 104 | q.resume(); 105 | } 106 | }) 107 | q.push(3, function (err, r) { 108 | assert.equal(finished, 2); 109 | finished++; 110 | }); 111 | q.push(2, function (err, r) { 112 | assert.equal(finished, 0); 113 | finished++; 114 | }); 115 | q.push(1, function (err, r) { 116 | assert.equal(finished, 1); 117 | finished++; 118 | done() 119 | }); 120 | this.q = q; 121 | }) 122 | 123 | it('should run filo', function (done) { 124 | var finished = 0; 125 | var queued = 0; 126 | var q = new Queue(function (num, cb) { 127 | cb(); 128 | }, { filo: true }) 129 | q.on('task_finish', function () { 130 | if (finished >= 3) { 131 | done(); 132 | } 133 | }) 134 | q.on('task_queued', function () { 135 | queued++; 136 | if (queued >= 3) { 137 | q.resume(); 138 | } 139 | }) 140 | q.pause(); 141 | q.push(1, function (err, r) { 142 | assert.equal(finished, 2); 143 | finished++; 144 | }) 145 | q.push(2, function (err, r) { 146 | assert.equal(finished, 1); 147 | finished++; 148 | }) 149 | q.push(3, function (err, r) { 150 | assert.equal(finished, 0); 151 | finished++; 152 | }) 153 | this.q = q; 154 | }) 155 | 156 | it('should filter before process', function (done) { 157 | var q = new Queue(function (n, cb) { cb(null, n) }, { 158 | filter: function (n, cb) { 159 | cb(null, n === 2 ? false : n); 160 | } 161 | }) 162 | q.push(2, function (err, r) { 163 | assert.equal(err, 'input_rejected'); 164 | }) 165 | q.push(3, function (err, r) { 166 | assert.equal(r, 3); 167 | done(); 168 | }) 169 | this.q = q; 170 | }) 171 | 172 | it('should batch delay', function (done) { 173 | var batches = 0; 174 | var q = new Queue(function (batch, cb) { 175 | batches++; 176 | if (batches === 1) { 177 | assert.equal(batch.length, 2); 178 | return cb(); 179 | } 180 | if (batches === 2) { 181 | assert.equal(batch.length, 1); 182 | cb(); 183 | return done(); 184 | } 185 | }, { batchSize: 2, batchDelay: 5, failTaskOnProcessException: false }); 186 | q.push(1); 187 | q.push(2); 188 | q.push(3); 189 | this.q = q; 190 | }) 191 | 192 | it('should batch 2', function (done) { 193 | var finished = 0; 194 | var q = new Queue(function (batch, cb) { 195 | finished++; 196 | assert.equal(batch.length, 1); 197 | if (finished >= 2) { 198 | done(); 199 | } 200 | cb(); 201 | }, { batchSize: 2, batchDelay: 1, autoResume: true }); 202 | q.push(1) 203 | .on('queued', function () { 204 | setTimeout(function () { 205 | q.push(2); 206 | }, 2) 207 | }) 208 | this.q = q; 209 | }) 210 | 211 | it('should drain and empty', function (done) { 212 | var emptied = false; 213 | var q = new Queue(function (n, cb) { cb() }) 214 | q.on('empty', function () { 215 | emptied = true; 216 | }, { autoResume: true }) 217 | q.on('drain', function () { 218 | assert.ok(emptied); 219 | done(); 220 | }); 221 | var queued = 0; 222 | q.on('task_queued', function () { 223 | queued++; 224 | if (queued >= 3) { 225 | q.resume(); 226 | } 227 | }) 228 | q.pause(); 229 | q.push(1) 230 | q.push(2) 231 | q.push(3) 232 | this.q = q; 233 | }) 234 | 235 | it('should drain only once the task is complete', function (done) { 236 | var finished_task = false; 237 | var q = new Queue(function (n, cb) { 238 | finished_task = true; 239 | cb(); 240 | }, { concurrent: 2 }); 241 | q.on('drain', function () { 242 | assert.ok(finished_task); 243 | done(); 244 | }); 245 | q.push(1); 246 | this.q = q; 247 | }); 248 | 249 | it('should queue 50 things', function (done) { 250 | var q = new Queue(function (n, cb) { 251 | cb(null, n+1); 252 | }) 253 | var finished = 0; 254 | for (var i = 0; i < 50; i++) { 255 | (function (n) { 256 | q.push(n, function (err, r) { 257 | assert.equal(r, n+1); 258 | finished++; 259 | if (finished === 50) { 260 | done(); 261 | } 262 | }) 263 | })(i) 264 | } 265 | this.q = q; 266 | }); 267 | 268 | it('should concurrently handle tasks', function (done) { 269 | var concurrent = 0; 270 | var ok = false; 271 | var q = new Queue(function (n, cb) { 272 | var wait = function () { 273 | if (concurrent === 3) { 274 | ok = true; 275 | } 276 | if (ok) return cb(); 277 | setImmediate(function () { 278 | wait(); 279 | }) 280 | } 281 | concurrent++; 282 | wait(); 283 | }, { concurrent: 3 }) 284 | var finished = 0; 285 | var finish = function () { 286 | finished++; 287 | if (finished >= 4) { 288 | done(); 289 | } 290 | } 291 | q.push(0, finish); 292 | q.push(1, finish); 293 | q.push(2, finish); 294 | q.push(3, finish); 295 | this.q = q; 296 | }) 297 | 298 | it('should pause and resume', function (done) { 299 | var running = false; 300 | var q = new Queue(function (n, cb) { 301 | running = true; 302 | return { 303 | pause: function () { 304 | running = false; 305 | }, 306 | resume: function () { 307 | running = true; 308 | cb(); 309 | done(); 310 | } 311 | } 312 | }) 313 | q.pause(); 314 | q.push(1) 315 | .on('started', function () { 316 | setTimeout(function () { 317 | assert.ok(running); 318 | q.pause(); 319 | assert.ok(!running); 320 | q.resume(); 321 | }, 1) 322 | }) 323 | assert.ok(!running); 324 | q.resume(); 325 | this.q = q; 326 | }) 327 | 328 | it('should timeout and fail', function (done) { 329 | var tries = 0; 330 | var q = new Queue(function (n, cb) { 331 | tries++; 332 | setTimeout(function () { 333 | cb(null, 'done!') 334 | }, 3) 335 | }, { maxTimeout: 1, maxRetries: 2 }) 336 | q.push(1) 337 | .on('finish', function (result) { 338 | assert.ok(false) 339 | }) 340 | .on('failed', function (err) { 341 | assert.equal(tries, 2); 342 | setTimeout(function () { 343 | done(); 344 | }, 5) 345 | }) 346 | this.q = q; 347 | }) 348 | 349 | it('should cancel while running and in queue', function (done) { 350 | var q = new Queue(function (task, cb) { 351 | assert.ok(task.n, 2) 352 | setTimeout(function () { 353 | q.cancel(1); 354 | }, 1) 355 | return { 356 | cancel: function () { 357 | done(); 358 | } 359 | } 360 | }, { 361 | id: 'id', 362 | merge: function (a,b) { 363 | assert.ok(false); 364 | } 365 | }) 366 | q.push({ id: 1, n: 1 }) 367 | .on('queued', function () { 368 | q.cancel(1, function () { 369 | q.push({ id: 1, n: 2 }); 370 | }) 371 | }); 372 | this.q = q; 373 | }) 374 | 375 | it('should stop if precondition fails', function (done) { 376 | var retries = 0; 377 | var q = new Queue(function (n) { 378 | assert.equal(retries, 2); 379 | done(); 380 | }, { 381 | precondition: function (cb) { 382 | retries++; 383 | cb(null, retries === 2) 384 | }, 385 | preconditionRetryTimeout: 1 386 | }) 387 | q.push(1); 388 | this.q = q; 389 | }) 390 | 391 | it('should call cb on throw', function (done) { 392 | var called = false; 393 | var q = new Queue(function (task, cb) { 394 | throw new Error('fail'); 395 | }); 396 | q.push(1, function (err) { 397 | called = true; 398 | assert.ok(err); 399 | }); 400 | q.on('drain', function () { 401 | assert.ok(called); 402 | done(); 403 | }); 404 | this.q = q; 405 | }) 406 | 407 | it('should respect batchDelayTimeout', function (done) { 408 | var q = new Queue(function (arr) { 409 | assert.equal(arr.length, 2); 410 | done(); 411 | }, { 412 | batchSize: 3, 413 | batchDelay: Infinity, 414 | batchDelayTimeout: 5 415 | }) 416 | q.push(1); 417 | setTimeout(function () { 418 | q.push(2); 419 | }, 1) 420 | this.q = q; 421 | }) 422 | 423 | it('should merge but not batch until the delay has happened', function (done) { 424 | var running = false; 425 | var q = new Queue(function (arr) { 426 | running = true; 427 | }, { 428 | autoResume: true, 429 | batchSize: 2, 430 | batchDelay: Infinity, 431 | id: 'id' 432 | }) 433 | setTimeout(function () { 434 | q.push({ id: 'a', x: 1 }); 435 | q.push({ id: 'a', x: 2 }); 436 | }, 1) 437 | setTimeout(function () { 438 | assert.ok(!running); 439 | done(); 440 | }, 10) 441 | this.q = q; 442 | }) 443 | 444 | it('merge batches should call all push callbacks', function (done) { 445 | var count = 0 446 | function finish() { 447 | count++ 448 | if (count === 2) done() 449 | } 450 | var q = new Queue(function (arr, cb) { 451 | cb() 452 | }, { 453 | autoResume: true, 454 | batchSize: 2, 455 | id: 'id' 456 | }) 457 | q.push({ id: 'a', x: 1 }, finish) 458 | q.push({ id: 'a', x: 2 }, finish) 459 | this.q = q; 460 | }) 461 | 462 | it('cancel should not retry', function (done) { 463 | var count = 0; 464 | var q = new Queue(function (n, cb) { 465 | count++; 466 | if (count === 2) { 467 | q.cancel('a', function () { 468 | cb('failed again'); 469 | setTimeout(function () { 470 | if (count === 2) { 471 | done(); 472 | } 473 | }, 100) 474 | }) 475 | } else { 476 | cb('failed'); 477 | } 478 | }, { 479 | autoResume: true, 480 | failTaskOnProcessException: true, 481 | maxRetries: Infinity, 482 | id: 'id' 483 | }) 484 | q.push({ id: 'a', x: 1 }); 485 | this.q = q; 486 | }) 487 | 488 | }) 489 | -------------------------------------------------------------------------------- /lib/queue.js: -------------------------------------------------------------------------------- 1 | var uuid = require('uuid'); 2 | var util = require('util'); 3 | var EE = require('events').EventEmitter; 4 | var Ticket = require('./ticket'); 5 | var Worker = require('./worker'); 6 | var Tickets = require('./tickets'); 7 | 8 | function Queue(process, opts) { 9 | var self = this; 10 | opts = opts || {}; 11 | if (typeof process === 'object') { 12 | opts = process || {}; 13 | } 14 | if (typeof process === 'function') { 15 | opts.process = process; 16 | } 17 | if (!opts.process) { 18 | throw new Error("Queue has no process function."); 19 | } 20 | 21 | opts = opts || {}; 22 | 23 | self.process = opts.process || function (task, cb) { cb(null, {}) }; 24 | self.filter = opts.filter || function (input, cb) { cb(null, input) }; 25 | self.merge = opts.merge || function (oldTask, newTask, cb) { cb(null, newTask) }; 26 | self.precondition = opts.precondition || function (cb) { cb(null, true) }; 27 | self.setImmediate = opts.setImmediate || setImmediate; 28 | self.id = opts.id || 'id'; 29 | self.priority = opts.priority || null; 30 | 31 | self.cancelIfRunning = (opts.cancelIfRunning === undefined ? false : !!opts.cancelIfRunning); 32 | self.autoResume = (opts.autoResume === undefined ? true : !!opts.autoResume); 33 | self.failTaskOnProcessException = (opts.failTaskOnProcessException === undefined ? true : !!opts.failTaskOnProcessException); 34 | self.filo = opts.filo || false; 35 | self.batchSize = opts.batchSize || 1; 36 | self.batchDelay = opts.batchDelay || 0; 37 | self.batchDelayTimeout = opts.batchDelayTimeout || Infinity; 38 | self.afterProcessDelay = opts.afterProcessDelay || 0; 39 | self.concurrent = opts.concurrent || 1; 40 | self.maxTimeout = opts.maxTimeout || Infinity; 41 | self.maxRetries = opts.maxRetries || 0; 42 | self.retryDelay = opts.retryDelay || 0; 43 | self.storeMaxRetries = opts.storeMaxRetries || Infinity; 44 | self.storeRetryTimeout = opts.storeRetryTimeout || 1000; 45 | self.preconditionRetryTimeout = opts.preconditionRetryTimeout || 1000; 46 | 47 | // Statuses 48 | self._queuedPeak = 0; 49 | self._queuedTime = {}; 50 | self._processedTotalElapsed = 0; 51 | self._processedAverage = 0; 52 | self._processedTotal = 0; 53 | self._failedTotal = 0; 54 | self.length = 0; 55 | self._stopped = false; 56 | self._saturated = false; 57 | 58 | self._preconditionRetryTimeoutId = null; 59 | self._batchTimeoutId = null; 60 | self._batchDelayTimeoutId = null; 61 | self._connected = false; 62 | self._storeRetries = 0; 63 | 64 | // Locks 65 | self._hasMore = false; 66 | self._isWriting = false; 67 | self._writeQueue = []; 68 | self._writing = {}; 69 | self._tasksWaitingForConnect = []; 70 | 71 | self._calledDrain = true; 72 | self._calledEmpty = true; 73 | self._fetching = 0; 74 | self._running = 0; // Active running tasks 75 | self._retries = {}; // Map of taskId => retries 76 | self._workers = {}; // Map of taskId => active job 77 | self._tickets = {}; // Map of taskId => tickets 78 | 79 | // Initialize Storage 80 | self.use(opts.store || 'memory'); 81 | if (!self._store) { 82 | throw new Error('Queue cannot continue without a valid store.') 83 | } 84 | } 85 | 86 | util.inherits(Queue, EE); 87 | 88 | Queue.prototype.destroy = function (cb) { 89 | cb = cb || function () {}; 90 | var self = this; 91 | 92 | // Statuses 93 | self._hasMore = false; 94 | self._isWriting = false; 95 | self._writeQueue = []; 96 | self._writing = {}; 97 | self._tasksWaitingForConnect = []; 98 | 99 | // Clear internals 100 | self._tickets = {}; 101 | self._workers = {}; 102 | self._fetching = 0; 103 | self._running = {}; 104 | self._retries = {}; 105 | self._calledEmpty = true; 106 | self._calledDrain = true; 107 | self._connected = false; 108 | self.pause(); 109 | 110 | if (typeof self._store.close === 'function') { 111 | self._store.close(cb); 112 | } else { 113 | cb(); 114 | } 115 | } 116 | 117 | Queue.prototype.resetStats = function () { 118 | this._queuedPeak = 0; 119 | this._processedTotalElapsed = 0; 120 | this._processedAverage = 0; 121 | this._processedTotal = 0; 122 | this._failedTotal = 0; 123 | } 124 | 125 | Queue.prototype.getStats = function () { 126 | var successRate = this._processedTotal === 0 ? 0 : (1 - (this._failedTotal / this._processedTotal)); 127 | return { 128 | successRate: successRate, 129 | peak: this._queuedPeak, 130 | average: this._processedAverage, 131 | total: this._processedTotal 132 | } 133 | } 134 | 135 | Queue.prototype.use = function (store, opts) { 136 | var self = this; 137 | var loadStore = function (store) { 138 | var Store; 139 | try { 140 | Store = require('better-queue-' + store); 141 | } catch (e) { 142 | throw new Error('Attempting to require better-queue-' + store + ', but failed.\nPlease ensure you have this store installed via npm install --save better-queue-' + store) 143 | } 144 | return Store; 145 | } 146 | if (typeof store === 'string') { 147 | var Store = loadStore(store); 148 | self._store = new Store(opts); 149 | } else if (typeof store === 'object' && typeof store.type === 'string') { 150 | var Store = loadStore(store.type); 151 | self._store = new Store(store); 152 | } else if (typeof store === 'object' && store.putTask && store.getTask && ((self.filo && store.takeLastN) || (!self.filo && store.takeFirstN))) { 153 | self._store = store; 154 | } else { 155 | throw new Error('unknown_store'); 156 | } 157 | self._connected = false; 158 | self._tasksWaitingForConnect = []; 159 | self._connectToStore(); 160 | } 161 | 162 | Queue.prototype._connectToStore = function () { 163 | var self = this; 164 | if (self._connected) return; 165 | if (self._storeRetries >= self.storeMaxRetries) { 166 | return self.emit('error', new Error('failed_connect_to_store')); 167 | } 168 | self._storeRetries++; 169 | self._store.connect(function (err, len) { 170 | if (err) return setTimeout(function () { 171 | self._connectToStore(); 172 | }, self.storeRetryTimeout); 173 | if (len === undefined || len === null) throw new Error("store_not_returning_length"); 174 | self.length = parseInt(len); 175 | if (isNaN(self.length)) throw new Error("length_is_not_a_number"); 176 | if (self.length) self._calledDrain = false; 177 | self._connected = true; 178 | self._storeRetries = 0; 179 | self._store.getRunningTasks(function (err, running) { 180 | if (!self._stopped && self.autoResume) { 181 | Object.keys(running).forEach(function (lockId) { 182 | self._running++; 183 | self._startBatch(running[lockId], {}, lockId); 184 | }) 185 | self.resume(); 186 | } 187 | for (var i = 0; i < self._tasksWaitingForConnect.length; i++) { 188 | self.push(self._tasksWaitingForConnect[i].input, self._tasksWaitingForConnect[i].ticket); 189 | } 190 | }) 191 | }) 192 | 193 | } 194 | 195 | Queue.prototype.resume = function () { 196 | var self = this; 197 | self._stopped = false; 198 | self._getWorkers().forEach(function (worker) { 199 | if (typeof worker.resume === 'function') { 200 | worker.resume(); 201 | } 202 | }) 203 | setTimeout(function () { 204 | self._processNextAfterTimeout(); 205 | }, 0) 206 | } 207 | 208 | Queue.prototype.pause = function () { 209 | this._stopped = true; 210 | this._getWorkers().forEach(function (worker) { 211 | if (typeof worker.pause === 'function') { 212 | worker.pause(); 213 | } 214 | }) 215 | } 216 | 217 | Queue.prototype.cancel = function (taskId, cb) { 218 | cb = cb || function(){}; 219 | var self = this; 220 | var worker = self._workers[taskId]; 221 | if (worker) { 222 | worker.cancel(); 223 | } 224 | self._store.deleteTask(taskId, cb); 225 | } 226 | 227 | Queue.prototype.push = function (input, cb) { 228 | var self = this; 229 | var ticket = new Ticket(); 230 | if (cb instanceof Ticket) { 231 | ticket = cb; 232 | } else if (cb) { 233 | ticket 234 | .on('finish', function (result) { cb(null, result) }) 235 | .on('failed', function (err) { cb(err) }) 236 | } 237 | if (!self._connected) { 238 | self._tasksWaitingForConnect.push({ input: input, ticket: ticket }); 239 | return ticket; 240 | } 241 | 242 | self.filter(input, function (err, task) { 243 | if (err || task === undefined || task === false || task === null) { 244 | return ticket.failed('input_rejected'); 245 | } 246 | var acceptTask = function (taskId) { 247 | setTimeout(function () { 248 | self._queueTask(taskId, task, ticket); 249 | }, 0) 250 | } 251 | if (typeof self.id === 'function') { 252 | self.id(task, function (err, id) { 253 | if (err) return ticket.failed('id_error'); 254 | acceptTask(id); 255 | }) 256 | } else if (typeof self.id === 'string' && typeof task === 'object') { 257 | acceptTask(task[self.id]) 258 | } else { 259 | acceptTask(); 260 | } 261 | }) 262 | return ticket; 263 | } 264 | 265 | Queue.prototype._getWorkers = function () { 266 | var self = this; 267 | var workers = []; 268 | Object.keys(self._workers).forEach(function (taskId) { 269 | var worker = self._workers[taskId]; 270 | if (worker && workers.indexOf(worker) === -1) { 271 | workers.push(worker); 272 | } 273 | }) 274 | return workers; 275 | } 276 | 277 | Queue.prototype._writeNextTask = function () { 278 | var self = this; 279 | if (self._isWriting) return; 280 | if (!self._writeQueue.length) return; 281 | self._isWriting = true; 282 | 283 | var taskId = self._writeQueue.shift(); 284 | var finishedWrite = function () { 285 | self._isWriting = false; 286 | self.setImmediate(function () { 287 | self._writeNextTask(); 288 | }) 289 | } 290 | 291 | if (!self._writing[taskId]) { 292 | delete self._writing[taskId]; 293 | return finishedWrite(); 294 | } 295 | 296 | var task = self._writing[taskId].task; 297 | var priority = self._writing[taskId].priority; 298 | var isNew = self._writing[taskId].isNew; 299 | var writeId = self._writing[taskId].id; 300 | var tickets = self._writing[taskId].tickets; 301 | 302 | self._store.putTask(taskId, task, priority, function (err) { 303 | 304 | // Check if task has changed since put 305 | if (self._writing[taskId] && self._writing[taskId].id !== writeId) { 306 | self._writeQueue.unshift(taskId); 307 | return finishedWrite(); 308 | } 309 | delete self._writing[taskId]; 310 | 311 | // If something else has written to taskId, then wait. 312 | if (err) { 313 | tickets.failed('failed_to_put_task'); 314 | return finishedWrite(); 315 | } 316 | 317 | // Task is in the queue -- update stats 318 | if (isNew) { 319 | self.length++; 320 | if (self._queuedPeak < self.length) { 321 | self._queuedPeak = self.length; 322 | } 323 | self._queuedTime[taskId] = new Date().getTime(); 324 | } 325 | 326 | // Notify the ticket 327 | if (self._tickets[taskId]) { 328 | self._tickets[taskId].push(tickets); 329 | } else { 330 | self._tickets[taskId] = tickets; 331 | } 332 | self.emit('task_queued', taskId, task); 333 | tickets.queued(); 334 | 335 | // If it's a new task, make sure to call drain after. 336 | if (isNew) { 337 | self._calledDrain = false; 338 | self._calledEmpty = false; 339 | } 340 | 341 | // If already fetching, mark that there are additions to the queue 342 | if (self._fetching > 0) { 343 | self._hasMore = true; 344 | } 345 | 346 | // Clear batchDelayTimeout 347 | if (self.batchDelayTimeout < Infinity) { 348 | if (self._batchDelayTimeoutId) clearTimeout(self._batchDelayTimeoutId) 349 | self._batchDelayTimeoutId = setTimeout(function () { 350 | self._batchDelayTimeoutId = null; 351 | if (self._batchTimeoutId) clearTimeout(self._batchTimeoutId); 352 | self._batchTimeoutId = null; 353 | self._processNextIfAllowed(); 354 | }, self.batchDelayTimeout) 355 | } 356 | 357 | // Finish writing 358 | finishedWrite(); 359 | self._processNextAfterTimeout(); 360 | }) 361 | } 362 | 363 | Queue.prototype._queueTask = function (taskId, newTask, ticket) { 364 | var self = this; 365 | var emptyTicket = new Ticket(); 366 | ticket = ticket || emptyTicket; 367 | var isUUID = false; 368 | if (!taskId) { 369 | taskId = uuid.v4(); 370 | isUUID = true; 371 | } 372 | var priority; 373 | var oldTask = null; 374 | var isNew = true; 375 | var putTask = function () { 376 | if (!self._connected) return; 377 | 378 | // Save ticket 379 | var tickets = (self._writing[taskId] && self._writing[taskId].tickets) || new Tickets(); 380 | if (ticket !== emptyTicket) { 381 | tickets.push(ticket); 382 | } 383 | 384 | // Add to queue 385 | var alreadyQueued = !!self._writing[taskId]; 386 | self._writing[taskId] = { 387 | id: uuid.v4(), 388 | isNew: isNew, 389 | task: newTask, 390 | priority: priority, 391 | tickets: tickets 392 | }; 393 | if (!alreadyQueued) { 394 | self._writeQueue.push(taskId); 395 | } 396 | 397 | self._writeNextTask(); 398 | } 399 | var updateTask = function () { 400 | ticket.accept(); 401 | self.emit('task_accepted', taskId, newTask); 402 | 403 | if (!self.priority) return putTask(); 404 | self.priority(newTask, function (err, p) { 405 | if (err) return ticket.failed('failed_to_prioritize'); 406 | priority = p; 407 | putTask(); 408 | }) 409 | } 410 | var mergeTask = function () { 411 | if (!oldTask) return updateTask(); 412 | self.merge(oldTask, newTask, function (err, mergedTask) { 413 | if (err) return ticket.failed('failed_task_merge'); 414 | if (mergedTask === undefined) return; 415 | newTask = mergedTask; 416 | updateTask(); 417 | }) 418 | } 419 | 420 | if (isUUID) { 421 | return updateTask(); 422 | } 423 | 424 | var worker = self._workers[taskId]; 425 | if (self.cancelIfRunning && worker) { 426 | worker.cancel(); 427 | } 428 | 429 | // Check if task is writing 430 | if (self._writing[taskId]) { 431 | oldTask = self._writing[taskId].task; 432 | return mergeTask(); 433 | } 434 | 435 | // Check store for task 436 | self._store.getTask(taskId, function (err, savedTask) { 437 | if (err) return ticket.failed('failed_to_get'); 438 | 439 | // Check if it's already in the store 440 | if (savedTask !== undefined) { 441 | isNew = false; 442 | } 443 | 444 | // Check if task is writing 445 | if (self._writing[taskId]) { 446 | oldTask = self._writing[taskId].task; 447 | return mergeTask(); 448 | } 449 | 450 | // No task before 451 | if (savedTask === undefined) { 452 | return updateTask(); 453 | } 454 | 455 | oldTask = savedTask; 456 | mergeTask(); 457 | }) 458 | } 459 | 460 | Queue.prototype._emptied = function () { 461 | if (this._calledEmpty) return; 462 | this._calledEmpty = true; 463 | this.emit('empty'); 464 | } 465 | 466 | Queue.prototype._drained = function () { 467 | if (this._calledDrain) return; 468 | this._calledDrain = true; 469 | this.emit('drain'); 470 | } 471 | 472 | Queue.prototype._getNextBatch = function (cb) { 473 | this._store[this.filo ? 'takeLastN' : 'takeFirstN'](this.batchSize, cb) 474 | } 475 | 476 | Queue.prototype._processNextAfterTimeout = function () { 477 | var self = this; 478 | if (self.length >= self.batchSize) { 479 | if (self._batchTimeoutId) { 480 | clearTimeout(self._batchTimeoutId); 481 | self._batchTimeoutId = null; 482 | } 483 | self.setImmediate(function () { 484 | self._processNextIfAllowed(); 485 | }) 486 | } else if (!self._batchTimeoutId && self.batchDelay < Infinity) { 487 | self._batchTimeoutId = setTimeout(function () { 488 | self._batchTimeoutId = null; 489 | self._processNextIfAllowed(); 490 | }, self.batchDelay) 491 | } 492 | } 493 | 494 | Queue.prototype._processNextIfAllowed = function () { 495 | var self = this; 496 | if (!self._connected) return; 497 | if (self._stopped) return; 498 | 499 | self._saturated = (self._running + self._fetching >= self.concurrent); 500 | if (self._saturated) return; 501 | if (!self.length) { 502 | if (!self._hasMore) { 503 | self._emptied(); 504 | if (!self._running) { 505 | self._drained(); 506 | } 507 | } 508 | return; 509 | } 510 | 511 | self.precondition(function (err, pass) { 512 | if (err || !pass) { 513 | if (!self._preconditionRetryTimeoutId && self.preconditionRetryTimeout) { 514 | self._preconditionRetryTimeoutId = setTimeout(function () { 515 | self._preconditionRetryTimeoutId = null; 516 | self._processNextIfAllowed(); 517 | }, self.preconditionRetryTimeout) 518 | } 519 | } else { 520 | self._processNext(); 521 | } 522 | }) 523 | } 524 | 525 | Queue.prototype._processNext = function () { 526 | var self = this; 527 | // FIXME: There may still be things writing 528 | self._hasMore = false; 529 | self._fetching++; 530 | self._getNextBatch(function (err, lockId) { 531 | self._fetching--; 532 | if (err || lockId === undefined) return; 533 | self._store.getLock(lockId, function (err, batch) { 534 | if (err || !batch) return; 535 | var batchSize = Object.keys(batch).length; 536 | var isEmpty = (batchSize === 0); 537 | 538 | if (self.length < batchSize) { 539 | self.length = batchSize; 540 | } 541 | 542 | if (!self._hasMore && isEmpty) { 543 | self._emptied(); 544 | if (!self._running) { 545 | self._drained(); 546 | } 547 | return; 548 | } 549 | 550 | // The write queue wasn't empty on fetch, so we should fetch more. 551 | if (self._hasMore && isEmpty) { 552 | return self._processNextAfterTimeout() 553 | } 554 | 555 | var tickets = {}; 556 | Object.keys(batch).forEach(function (taskId) { 557 | var ticket = self._tickets[taskId]; 558 | if (ticket) { 559 | ticket.started(); 560 | tickets[taskId] = ticket; 561 | delete self._tickets[taskId]; 562 | } 563 | }) 564 | 565 | // Acquire lock on process 566 | self._running++; 567 | self._startBatch(batch, tickets, lockId); 568 | 569 | if (self.concurrent - self._running > 1) { 570 | // Continue processing until saturated 571 | self._processNextIfAllowed(); 572 | } 573 | }); 574 | }); 575 | } 576 | 577 | Queue.prototype._startBatch = function (batch, tickets, lockId) { 578 | var self = this; 579 | var taskIds = Object.keys(batch); 580 | var timeout = null; 581 | var worker = new Worker({ 582 | fn: self.process, 583 | batch: batch, 584 | single: (self.batchSize === 1), 585 | failTaskOnProcessException: self.failTaskOnProcessException 586 | }) 587 | var updateStatsForEndedTask = function (taskId) { 588 | self._processedTotal++; 589 | var stats = {}; 590 | if (!self._queuedTime[taskId]) return stats; 591 | 592 | var elapsed = (new Date().getTime() - self._queuedTime[taskId]); 593 | delete self._queuedTime[taskId]; 594 | 595 | if (elapsed > 0) { 596 | stats.elapsed = elapsed; 597 | self._processedTotalElapsed += elapsed; 598 | self._processedAverage = self._processedTotalElapsed/self._processedTotal; 599 | } 600 | return stats; 601 | } 602 | 603 | if (self.maxTimeout < Infinity) { 604 | timeout = setTimeout(function () { 605 | worker.failedBatch('task_timeout'); 606 | }, self.maxTimeout); 607 | } 608 | worker.on('task_failed', function (id, msg) { 609 | var taskId = taskIds[id]; 610 | self._retries[taskId] = self._retries[taskId] || 0; 611 | self._retries[taskId]++; 612 | if (worker.cancelled || self._retries[taskId] >= self.maxRetries) { 613 | var stats = updateStatsForEndedTask(taskId); 614 | if (tickets[taskId]) { 615 | // Mark as a failure 616 | tickets[taskId].failed(msg); 617 | delete tickets[taskId]; 618 | } 619 | self._failedTotal++; 620 | self._retries[taskId] = undefined 621 | self.emit('task_failed', taskId, msg, stats); 622 | } else { 623 | if (self.retryDelay) { 624 | // Pop back onto queue and retry 625 | setTimeout(function () { 626 | self.emit('task_retry', taskId, self._retries[taskId]); 627 | self._queueTask(taskId, batch[taskId], tickets[taskId]); 628 | }, self.retryDelay) 629 | } else { 630 | self.setImmediate(function () { 631 | self.emit('task_retry', taskId, self._retries[taskId]); 632 | self._queueTask(taskId, batch[taskId], tickets[taskId]); 633 | }); 634 | } 635 | } 636 | }) 637 | worker.on('task_finish', function (id, result) { 638 | var taskId = taskIds[id]; 639 | var stats = updateStatsForEndedTask(taskId); 640 | if (tickets[taskId]) { 641 | tickets[taskId].finish(result); 642 | delete tickets[taskId]; 643 | } 644 | self.emit('task_finish', taskId, result, stats); 645 | }) 646 | worker.on('task_progress', function (id, progress) { 647 | var taskId = taskIds[id]; 648 | if (tickets[taskId]) { 649 | tickets[taskId].progress(progress); 650 | } 651 | self.emit('task_progress', taskId, progress); 652 | }) 653 | worker.on('progress', function (progress) { 654 | self.emit('batch_progress', progress); 655 | }) 656 | worker.on('finish', function (result) { 657 | self.emit('batch_finish', result); 658 | }) 659 | worker.on('failed', function (err) { 660 | self.emit('batch_failed', err); 661 | }) 662 | worker.on('end', function () { 663 | self.length -= Object.keys(batch).length; 664 | if (timeout) { 665 | clearTimeout(timeout); 666 | } 667 | var finishAndGetNext = function () { 668 | if (!self._connected) return; 669 | self._store.releaseLock(lockId, function (err) { 670 | if (err) { 671 | // If we cannot release the lock then retry 672 | return setTimeout(function () { 673 | finishAndGetNext(); 674 | }, 1) 675 | } 676 | self._running--; 677 | taskIds.forEach(function (taskId) { 678 | if (self._workers[taskId] && !self._workers[taskId].active) { 679 | delete self._workers[taskId]; 680 | } 681 | }); 682 | self._processNextAfterTimeout(); 683 | }) 684 | } 685 | if (self.afterProcessDelay) { 686 | setTimeout(function () { 687 | finishAndGetNext() 688 | }, self.afterProcessDelay); 689 | } else { 690 | self.setImmediate(function () { 691 | finishAndGetNext() 692 | }) 693 | } 694 | }) 695 | 696 | taskIds.forEach(function (taskId) { 697 | self._workers[taskId] = worker; 698 | }); 699 | 700 | try { 701 | worker.start(); 702 | taskIds.forEach(function (taskId) { 703 | self.emit('task_started', taskId, batch[taskId]) 704 | }); 705 | } catch (e) { 706 | self.emit('error', e); 707 | } 708 | 709 | } 710 | 711 | module.exports = Queue; 712 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Better Queue - Powerful flow control 2 | 3 | [![npm package](https://nodei.co/npm/better-queue.png?downloads=true&downloadRank=true&stars=true)](https://nodei.co/npm/better-queue/) 4 | 5 | [![Build status](https://img.shields.io/travis/diamondio/better-queue.svg?style=flat-square)](https://travis-ci.org/diamondio/better-queue) 6 | [![Dependency Status](https://img.shields.io/david/diamondio/better-queue.svg?style=flat-square)](https://david-dm.org/diamondio/better-queue) 7 | [![Known Vulnerabilities](https://snyk.io/test/npm/better-queue/badge.svg?style=flat-square)](https://snyk.io/test/npm/better-queue) 8 | [![Gitter](https://img.shields.io/badge/gitter-join_chat-blue.svg?style=flat-square)](https://gitter.im/leanderlee/better-queue?utm_source=badge) 9 | 10 | 11 | ## Super simple to use 12 | 13 | Better Queue is designed to be simple to set up but still let you do complex things. 14 | 15 | - Persistent (and extendable) storage 16 | - Batched processing 17 | - Prioritize tasks 18 | - Merge/filter tasks 19 | - Progress events (with ETA!) 20 | - Fine-tuned timing controls 21 | - Retry on fail 22 | - Concurrent batch processing 23 | - Task statistics (average completion time, failure rate and peak queue size) 24 | - ... and more! 25 | 26 | --- 27 | 28 | #### Install (via npm) 29 | 30 | ```bash 31 | npm install --save better-queue 32 | ``` 33 | 34 | --- 35 | 36 | #### Quick Example 37 | 38 | ```js 39 | var Queue = require('better-queue'); 40 | 41 | var q = new Queue(function (input, cb) { 42 | 43 | // Some processing here ... 44 | 45 | cb(null, result); 46 | }) 47 | 48 | q.push(1) 49 | q.push({ x: 1 }) 50 | ``` 51 | 52 | ## Table of contents 53 | 54 | - [Queuing](#queuing) 55 | - [Task Management](#task-management) 56 | - [Queue Management](#queue-management) 57 | - [Advanced](#advanced) 58 | - [Storage](#storage) 59 | - [Using with Webpack](#using-with-webpack) 60 | - [Full Documentation](#full-documentation) 61 | 62 | --- 63 | 64 | You will be able to combine any (and all) of these options 65 | for your queue! 66 | 67 | 68 | ## Queuing 69 | 70 | It's very easy to push tasks into the queue. 71 | 72 | ```js 73 | var q = new Queue(fn); 74 | q.push(1); 75 | q.push({ x: 1, y: 2 }); 76 | q.push("hello"); 77 | ``` 78 | 79 | You can also include a callback as a second parameter to the push 80 | function, which would be called when that task is done. For example: 81 | 82 | ```js 83 | var q = new Queue(fn); 84 | q.push(1, function (err, result) { 85 | // Results from the task! 86 | }); 87 | ``` 88 | 89 | You can also listen to events on the results of the `push` call. 90 | 91 | ```js 92 | var q = new Queue(fn); 93 | q.push(1) 94 | .on('finish', function (result) { 95 | // Task succeeded with {result}! 96 | }) 97 | .on('failed', function (err) { 98 | // Task failed! 99 | }) 100 | ``` 101 | 102 | Alternatively, you can subscribe to the queue's events. 103 | 104 | ```js 105 | var q = new Queue(fn); 106 | q.on('task_finish', function (taskId, result, stats) { 107 | // taskId = 1, result: 3, stats = { elapsed: