├── .travis.yml ├── LICENSE ├── README.md ├── index.js ├── package.json └── test.js /.travis.yml: -------------------------------------------------------------------------------- 1 | notifications: 2 | email: false 3 | language: node_js 4 | node_js: 5 | - 4 6 | - 6 7 | - 8 8 | - node 9 | deploy: 10 | provider: npm 11 | email: github@tixz.dk 12 | api_key: 13 | secure: B7ED2hIjSVoJM0fgb8hjz2LN2Qa89+XIK5+ZWdwNlCADpIiD2XKdqPQsGaQbPDshU6XEEUPKOYYI5izU+KZ2RgJNdZLGAnfHsxm1fgDrxZ1VpWlBp54Mv/Y5zmTZiGUQmZNKdCmShGkMQKx5NKyq9hPTOSFRVJEQ4BED3uACUym8sGcSNJ4cYxE/VIUTtVQlX/zwbNoce4FiiQ/kQ+vFd7kRHXTHSDEF6kX/s713BWoxSsqFSkf4FlJL5Dcsw9kIacm+joPVlBgXvFE41ifCDTKlfDqNgc4ycLHRMbvi/+3GiOlKO+e8pYGhNuGMQsqdttL7T7H4i0RbN5RRrL3KW7xy58KBvNY82WPiCFXzKAUJGAS5C2w14SzWAKOcIBljUruzH/80PqmE6jRbUjhiJKbi76skeMd3Wj/mIOX2PdDnR97/u9xyjjBoV+PwrbBevusAB8/2M3VBuDEtDk34/B4VT+Q94tbnn1tRzkNfEGzmb1+mtdDZFXETUyIi4tVMVL41N5e8uSy4C+HcDfXTAC+Hyisvm4DLdwABJOLOhcs67juCVK5seHI2GaoBWym+zr5nNSPlwL9xGnj6fL+H1Rf8vYcKYFfO5K7eXbJLkpuXBu03EjSzbmTzYkSdB/v7hjXTbhcjLnN3CNKC4gWSy3Wm340cE2Ig4MTlNlLBErg= 14 | on: 15 | tags: true 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017, Emil Bay 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `parallel-queue` 2 | 3 | [![Build Status](https://travis-ci.org/emilbayes/parallel-queue.svg?branch=master)](https://travis-ci.org/emilbayes/parallel-queue) 4 | 5 | > Queue for parallel tasks that can cancel and is destoryable 6 | 7 | ## Usage 8 | 9 | ```js 10 | var parallelQueue = require('parallel-queue') 11 | 12 | var pq = parallelQueue(4, function (args, done) { 13 | setTimeout(done, args.wait) 14 | }) 15 | 16 | pq.push({wait: 30000}) 17 | 18 | // Cancel all pending tasks, and invoke callback on all running tasks with error 19 | pq.destroy() 20 | ``` 21 | 22 | ## API 23 | 24 | ### `var queue = parallelQueue(parallel, worker)` 25 | 26 | Create a queue that is limited to `parallel` concurrent workers. `worker` will be 27 | invoked with `(task, next)`, where `task` is whatever is `push`ed onto the queue 28 | and `next` is a normal callback function, passed an optional error as the first 29 | argument and up to three return values from succeeding arguments. These 30 | arguments will be passed on to the `done` callback in `push`. 31 | 32 | ### `var cancel = queue.push(task, done)` 33 | 34 | Queue `task` and call `cb` when done. `queue.push` will return a function that 35 | you can invoke to cancel the task, which in turn will invoke `cb` with a `Error` 36 | with `err.cancel === true`. If the task has already started, there is no way 37 | to cancel it, and it will not release a slot in the queue until done. If the 38 | `queue` has been `destroy`ed, tasks will not be queued and `done` will be 39 | invoked immediately with an error. 40 | 41 | **Note**: The task will not be executed before the `nextTick` as to avoid the 42 | `worker` function from plugging the event loop. This means you can `.push` and 43 | `cancel` as many tasks as you want synchronously, also from within the callback. 44 | 45 | **Note 2**: You can cancel a running task, but that does not mean the actual work 46 | done by the `worker` is stopped, just that the `cb` supplied to `.push` is 47 | removed from the queue and called with an `Error`. The slot in the queue will be 48 | released as soon as the running worker calls the supplied `next` callback. 49 | 50 | ### `queue.destroy([err])` 51 | 52 | Cancel all pending tasks, and call the callback of all running tasks with an 53 | error. If `err` is not given, the default error will be used (see above). 54 | 55 | ### `queue.parallel` 56 | 57 | Number of tasks that can run in parallel. This is read-only 58 | 59 | ### `queue.pending` 60 | 61 | Number of tasks pending in the queue. This is excluding the current task 62 | if accessed from within the worker function 63 | 64 | ### `queue.running` 65 | 66 | Number of tasks running in the queue. This is including the current task 67 | if accessed from within the worker function. 68 | 69 | ### `queue.destroyed` 70 | 71 | Boolean set after `destroy` has finished 72 | 73 | ## Install 74 | 75 | ```sh 76 | npm install parallel-queue 77 | ``` 78 | 79 | ## License 80 | 81 | [ISC](LICENSE.md) 82 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | module.exports = ParallelQueue 2 | 3 | function ParallelQueue (parallel, worker) { 4 | if (!(this instanceof ParallelQueue)) return new ParallelQueue(parallel, worker) 5 | 6 | this._worker = worker 7 | this._queue = [] 8 | this._running = [] 9 | this._runningCount = 0 10 | this.parallel = parallel 11 | this.destroyed = false 12 | } 13 | 14 | Object.defineProperty(ParallelQueue.prototype, 'pending', { 15 | enumerable: true, 16 | get: function () { 17 | return this._queue.length 18 | } 19 | }) 20 | 21 | Object.defineProperty(ParallelQueue.prototype, 'running', { 22 | enumerable: true, 23 | get: function () { 24 | return this._runningCount 25 | } 26 | }) 27 | 28 | ParallelQueue.prototype.push = function (task, cb) { 29 | if (this.destroyed === true) return cb(new Error('Already destroyed')) 30 | var args = { 31 | task: task, 32 | cb: cb 33 | } 34 | 35 | this._queue.push(args) 36 | 37 | process.nextTick(this._kick.bind(this)) 38 | 39 | return this._cancel.bind(this, args) 40 | } 41 | 42 | ParallelQueue.prototype.destroy = function (err) { 43 | while (this._queue.length) this._cancel(this._queue[0], err) 44 | while (this._running.length) this._cancel(this._running[0], err) 45 | this.destroyed = true 46 | } 47 | 48 | ParallelQueue.prototype._cancel = function (args, err) { 49 | var qidx = this._queue.indexOf(args) 50 | if (qidx >= 0) { 51 | this._queue.splice(qidx, 1) 52 | } 53 | 54 | var ridx = this._running.indexOf(args) 55 | if (ridx >= 0) { 56 | this._running.splice(ridx, 1) 57 | } 58 | 59 | if (ridx < 0 && qidx < 0) return 60 | 61 | if (err == null) { 62 | err = new Error('Cancelled operation') 63 | err.cancel = true 64 | } 65 | 66 | args.cb(err) 67 | process.nextTick(this._kick.bind(this)) 68 | } 69 | 70 | ParallelQueue.prototype._kick = function () { 71 | var self = this 72 | if (self._runningCount >= self.parallel) return 73 | 74 | var args = self._queue.shift() 75 | if (args == null) return 76 | 77 | self._running.push(args) 78 | 79 | self._runningCount++ 80 | self._worker(args.task, done) 81 | 82 | function done (err, res1, res2, res3) { 83 | self._runningCount-- 84 | 85 | var ridx = self._running.indexOf(args) 86 | if (ridx >= 0) { 87 | self._running.splice(ridx, 1) 88 | args.cb(err, res1, res2, res3) 89 | } 90 | 91 | process.nextTick(self._kick.bind(self)) 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "parallel-queue", 3 | "version": "3.0.1", 4 | "description": "Queue for parallel tasks that can cancel and is destoryable", 5 | "main": "index.js", 6 | "dependencies": {}, 7 | "devDependencies": { 8 | "browserify": "^14.4.0", 9 | "coverify": "^1.4.1", 10 | "tape": "^4.6.3" 11 | }, 12 | "scripts": { 13 | "test": "tape test.js", 14 | "coverage": "browserify -t coverify test.js | node | coverify" 15 | }, 16 | "repository": { 17 | "type": "git", 18 | "url": "git+https://github.com/emilbayes/parallel-queue.git" 19 | }, 20 | "keywords": [], 21 | "author": "Emil Bay ", 22 | "license": "ISC", 23 | "bugs": { 24 | "url": "https://github.com/emilbayes/parallel-queue/issues" 25 | }, 26 | "homepage": "https://github.com/emilbayes/parallel-queue#readme" 27 | } 28 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var queue = require('.') 3 | 4 | test('pending', function (assert) { 5 | assert.plan(4) 6 | var q = queue(1, function pending (wait, cb) { 7 | process.nextTick(cb, null, wait) 8 | }) 9 | 10 | q.push(1, function (err, value) { 11 | assert.error(err) 12 | }) 13 | 14 | q.push(2, function (err) { 15 | assert.error(err) 16 | }) 17 | 18 | assert.equal(q.pending, 2) 19 | assert.equal(q.running, 0) 20 | }) 21 | 22 | test('running', function (assert) { 23 | assert.plan(8) 24 | var q = queue(1, function pending (wait, cb) { 25 | assert.equal(q.pending, 2 - wait) 26 | assert.equal(q.running, 1) 27 | process.nextTick(cb, null, wait) 28 | }) 29 | 30 | q.push(1, function (err, value) { 31 | assert.error(err) 32 | }) 33 | 34 | q.push(2, function (err) { 35 | assert.error(err) 36 | }) 37 | 38 | process.nextTick(function () { 39 | assert.equal(q.pending, 1) 40 | assert.equal(q.running, 1) 41 | }) 42 | }) 43 | 44 | test('run then cancel', function (assert) { 45 | assert.plan(2) 46 | var q = queue(1, function pending (wait, cb) { 47 | process.nextTick(cb, null, wait) 48 | }) 49 | 50 | q.push(1, function (err, value) { 51 | assert.error(err) 52 | 53 | ;(q.push(2, function (err) { 54 | assert.ok(err.cancel) 55 | }))() 56 | }) 57 | }) 58 | 59 | test('push push then cancel', function (assert) { 60 | assert.plan(2) 61 | var q = queue(1, function cancel (wait, cb) { 62 | process.nextTick(cb, null, wait) 63 | }) 64 | 65 | q.push(1, function (err) { 66 | assert.error(err) 67 | }) 68 | 69 | ;(q.push(2, function (err) { 70 | assert.ok(err.cancel) 71 | }))() 72 | }) 73 | 74 | test('cancel immediately', function cancelNow (assert) { 75 | var q = queue(1, function (wait, cb) { 76 | process.nextTick(cb, null, wait) 77 | }) 78 | 79 | ;(q.push(1, function (err) { 80 | assert.ok(err.cancel) 81 | assert.end() 82 | }))() 83 | }) 84 | 85 | test('destory', function (assert) { 86 | assert.plan(5) 87 | 88 | var q = queue(1, function destory (wait, cb) { 89 | process.nextTick(cb, null, wait) 90 | }) 91 | 92 | q.push(1001, function (err) { assert.ok(err.cancel, 1) }) 93 | q.push(1002, function (err) { assert.ok(err.cancel, 2) }) 94 | q.push(1003, function (err) { assert.ok(err.cancel, 3) }) 95 | q.push(1004, function (err) { assert.ok(err.cancel, 4) }) 96 | q.push(1005, function (err) { assert.ok(err.cancel, 5) }) 97 | 98 | q.destroy() 99 | }) 100 | 101 | test('destroy with custom error', function (assert) { 102 | assert.plan(5) 103 | 104 | var q = queue(1, function customError (wait, cb) { 105 | process.nextTick(cb, null, wait) 106 | }) 107 | 108 | q.push(1, function (err) { 109 | assert.ok(err.custom) 110 | }) 111 | q.push(1, function (err) { 112 | assert.ok(err.custom) 113 | }) 114 | q.push(1000, function (err) { 115 | assert.ok(err.custom) 116 | }) 117 | q.push(1000, function (err) { 118 | assert.ok(err.custom) 119 | }) 120 | q.push(1000, function (err) { 121 | assert.ok(err.custom) 122 | }) 123 | 124 | var e = new Error('custom') 125 | e.custom = true 126 | q.destroy(e) 127 | }) 128 | 129 | test('push after destroyed', function (assert) { 130 | var q = queue(1, function pushAfterDestory (wait, cb) { 131 | process.nextTick(cb, null, wait) 132 | }) 133 | 134 | q.destroy() 135 | 136 | q.push(0, function (err) { 137 | assert.ok(err) 138 | assert.end() 139 | }) 140 | }) 141 | 142 | function cancel (id, q, assert, e) { 143 | var completed = false 144 | return q.push(id, function (err, iid) { 145 | if (completed === true) return assert.fail('called twice') 146 | completed = true 147 | if (e) assert.equal(err, e, 'errors should be equal') 148 | else assert.ok(err, 'should get error ' + id) 149 | assert.equal(iid, null, 'ids should be equal ' + iid) 150 | }) 151 | } 152 | 153 | function complete (id, q, assert) { 154 | var completed = false 155 | q.push(id, function (err, iid) { 156 | if (completed === true) return assert.fail('called twice') 157 | completed = true 158 | assert.error(err, 'errored by getting err ' + iid) 159 | assert.equal(iid, id, 'ids should be equal ' + iid) 160 | }) 161 | return function () {} 162 | } 163 | 164 | // I don't think this fuzzing is right 165 | test.skip('fuzz', function (assert) { 166 | var que = queue(25, function fuzz (id, cb) { 167 | process.nextTick(cb, null, id) 168 | }) 169 | 170 | debugger 171 | 172 | ;[ 173 | complete(0, que, assert), 174 | cancel(1, que, assert), 175 | cancel(2, que, assert), 176 | cancel(3, que, assert), 177 | cancel(4, que, assert), 178 | cancel(5, que, assert), 179 | complete(6, que, assert), 180 | complete(7, que, assert), 181 | cancel(8, que, assert), 182 | complete(9, que, assert), 183 | cancel(10, que, assert), 184 | complete(11, que, assert), 185 | cancel(12, que, assert), 186 | cancel(13, que, assert), 187 | cancel(14, que, assert), 188 | complete(15, que, assert), 189 | complete(16, que, assert), 190 | complete(17, que, assert), 191 | complete(18, que, assert), 192 | complete(19, que, assert), 193 | complete(20, que, assert), 194 | complete(21, que, assert), 195 | complete(22, que, assert), 196 | complete(23, que, assert), 197 | complete(24, que, assert), 198 | complete(25, que, assert), 199 | complete(26, que, assert), 200 | complete(27, que, assert), 201 | complete(28, que, assert), 202 | complete(29, que, assert), 203 | complete(30, que, assert), 204 | complete(31, que, assert), 205 | complete(32, que, assert), 206 | complete(33, que, assert), 207 | complete(34, que, assert), 208 | cancel(35, que, assert), 209 | cancel(36, que, assert), 210 | cancel(37, que, assert), 211 | cancel(38, que, assert), 212 | cancel(39, que, assert), 213 | cancel(40, que, assert) 214 | ].forEach(function (fn) { 215 | fn() 216 | }) 217 | 218 | setTimeout(function () { 219 | que.destroy() 220 | 221 | que.push(999, function (err) { 222 | assert.ok(err) 223 | assert.end() 224 | }) 225 | }, 250) // defer at least a couple of ticks 226 | }) 227 | --------------------------------------------------------------------------------