├── .gitignore ├── errors.js ├── .zuul.yml ├── example ├── usage.js └── promise.js ├── test ├── concurrency.js ├── init.js ├── limit.js ├── process-error.js ├── process-noargs.js ├── is-drained.js ├── push.js ├── max-size.js ├── soft-max-size.js ├── process.js ├── events.js └── drain.js ├── .travis.yml ├── LICENSE ├── package.json ├── Makefile ├── index.js └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | coverage 3 | node_modules 4 | -------------------------------------------------------------------------------- /errors.js: -------------------------------------------------------------------------------- 1 | var defineError = require('define-error') 2 | 3 | module.exports.MaxSizeExceededError = defineError('MaxSizeExceededError') 4 | -------------------------------------------------------------------------------- /.zuul.yml: -------------------------------------------------------------------------------- 1 | ui: tape 2 | browsers: 3 | - name: chrome 4 | version: latest 5 | - name: safari 6 | version: 6..latest 7 | - name: ie 8 | version: 9..latest 9 | - name: firefox 10 | version: 30..latest 11 | - name: opera 12 | version: 11..latest 13 | - name: ipad 14 | version: 6.0..latest 15 | - name: iphone 16 | version: 6.0..latest 17 | - name: android 18 | version: 4.0..latest 19 | -------------------------------------------------------------------------------- /example/usage.js: -------------------------------------------------------------------------------- 1 | var cq = require('..') 2 | 3 | var queue = cq().limit({ concurrency: 2 }).process(function (task, cb) { 4 | console.log(task + ' started') 5 | setTimeout(function () { 6 | cb(null, task) 7 | }, 1000) 8 | }) 9 | 10 | for (var i = 1; i <= 10; i++) queue('task ' + i, taskDone) 11 | function taskDone (err, task) { 12 | if (err) return console.error(err) 13 | console.log(task + ' done') 14 | } 15 | -------------------------------------------------------------------------------- /test/concurrency.js: -------------------------------------------------------------------------------- 1 | var test = require('tape'), 2 | cq = require('..') 3 | 4 | test('concurrency', function (t) { 5 | var q = cq() 6 | t.plan(3) 7 | 8 | t.equal(q.concurrency, Infinity, 'returns default concurrency initially') 9 | q.concurrency = 5 10 | t.equal(q.concurrency, 5, 'allows concurrency to be set') 11 | t.throws(function () { 12 | q.concurrency = {} 13 | }, 'must be a number') 14 | }) 15 | -------------------------------------------------------------------------------- /test/init.js: -------------------------------------------------------------------------------- 1 | var test = require('tape'), 2 | cq = require('..') 3 | 4 | test('concurrent-queue', function (t) { 5 | var q = cq() 6 | t.equal(typeof q, 'function', 'returns a function') 7 | t.ok(q.process, 'has a process function') 8 | t.ok(q.processing, 'has a processing property') 9 | t.ok(q.size === 0, 'has a size property') 10 | t.ok(q.pending, 'has a pending property') 11 | t.ok(q.maxSize, 'has a maxSize property') 12 | t.ok(q.concurrency, 'has a concurrency property') 13 | 14 | t.end() 15 | }) 16 | -------------------------------------------------------------------------------- /test/limit.js: -------------------------------------------------------------------------------- 1 | var test = require('tape'), 2 | cq = require('..') 3 | 4 | test('limit', function (t) { 5 | var q = cq() 6 | t.plan(3) 7 | 8 | t.throws(limit('maxSize'), 'maxSize requires a number') 9 | t.throws(limit('softMaxSize'), 'maxSize requires a number') 10 | t.throws(limit('concurrency'), 'maxSize requires a number') 11 | 12 | function limit (l) { 13 | var limits = {} 14 | limits[l] = {} 15 | return function () { 16 | q.limit(limits) 17 | } 18 | } 19 | }) 20 | -------------------------------------------------------------------------------- /example/promise.js: -------------------------------------------------------------------------------- 1 | var cq = require('..'), 2 | Promise = require('promise-polyfill') 3 | 4 | var queue = cq().limit({ concurrency: 2 }).process(function (task) { 5 | return new Promise(function (resolve, reject) { 6 | console.log(task + ' started') 7 | setTimeout(resolve.bind(undefined, task), 1000) 8 | }) 9 | }) 10 | 11 | for (var i = 1; i <= 10; i++) queue('task ' + i).then(taskDone).catch(taskError) 12 | function taskDone (task) { 13 | console.log(task + ' done') 14 | } 15 | function taskError (err) { 16 | console.error(err) 17 | } 18 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: node_js 3 | node_js: 4 | - '0.12' 5 | - '4.0' 6 | - '6.0' 7 | before_script: 8 | - npm install -g coveralls@2.10.0 9 | script: 10 | - make travis-test 11 | - '[ "${TRAVIS_NODE_VERSION}" = "0.12" ] && travis_retry travis_wait make browser-test 12 | || true' 13 | env: 14 | global: 15 | - secure: O6/RTKE37CFuSUrnETwl4gg6Z2OtSyPHb7T9y8ENBy/RkFYOIbjyJTWWEXW37lWB5Rr4im6Qc9z47JimECpHM1GUjIpxgSrKi0aTc1qLziBjFHRbgqu03PG9+sRDEeINjZoGAl6xX/Ft6+Iykj3cmOcYulYinrGbjVWRuu843G0= 16 | - secure: INrhW5OnonTfKBE81CdBWS/UtHru3SyOUu0JV7E0dhNbXpTmUvnNkM5QVqURLSCVyg+3UVxcIpaF3N87i03EOo4+cTyncsZUu0lfCKIWqoqmct7aPvjF3/zX+1hJZO7aMisOlzzVUNB1mFI7201HHZaFks2LGfmlSXCJ/4OsURo= 17 | -------------------------------------------------------------------------------- /test/process-error.js: -------------------------------------------------------------------------------- 1 | var test = require('tape'), 2 | cq = require('..') 3 | 4 | test('processor throws error error', function (t) { 5 | t.plan(2) 6 | 7 | var q = cq().process(function (task, cb) { 8 | throw new Error('busted') 9 | }) 10 | 11 | q('task 1', function (err, resolution) { 12 | t.ok(err, 'should receive error to callback') 13 | t.notOk(resolution, 'calls back with no data') 14 | }) 15 | }) 16 | 17 | test('processor throws error (promise)', function (t) { 18 | t.plan(1) 19 | 20 | var q = cq().process(function () { 21 | throw new Error('busted') 22 | }) 23 | 24 | q('task 1').catch(function (err) { 25 | t.ok(err, 'receives error') 26 | }) 27 | }) 28 | -------------------------------------------------------------------------------- /test/process-noargs.js: -------------------------------------------------------------------------------- 1 | var test = require('tape'), 2 | cq = require('..'), 3 | Promise = require('promise-polyfill') 4 | 5 | test('processor no args', function (t) { 6 | t.plan(2) 7 | 8 | var q = cq().process(function () { 9 | arguments[1](null, true) 10 | }) 11 | 12 | q('task 1', function (err, resolution) { 13 | t.notOk(err, 'calls back with no error') 14 | t.ok(resolution, 'calls back with proper data') 15 | }) 16 | }) 17 | 18 | test('processor no args (promise)', function (t) { 19 | t.plan(1) 20 | 21 | var q = cq().process(function () { 22 | return new Promise(function (resolve, reject) { 23 | resolve(true) 24 | }) 25 | }) 26 | 27 | q('task 1').then(function (resolution) { 28 | t.ok(resolution, 'resolves') 29 | }) 30 | }) 31 | -------------------------------------------------------------------------------- /test/is-drained.js: -------------------------------------------------------------------------------- 1 | var test = require('tape'), 2 | cq = require('..') 3 | 4 | test('isDrained', function (t) { 5 | t.plan(4) 6 | 7 | var q = cq() 8 | t.equal(q.isDrained, true, 'should default to true') 9 | 10 | var tasks = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] 11 | tasks.forEach(function (task) { 12 | q(task) 13 | }) 14 | t.equal(q.isDrained, false, 'should be false after queueing 10 items, before processor set') 15 | 16 | q.limit({ concurrency: 5}).process(function (item, cb) { 17 | setTimeout(function () { 18 | cb(null, item) 19 | }, 100) 20 | }) 21 | t.equal(q.isDrained, false, 'should be false after setting processor but before processing is complete') 22 | 23 | q.drained(function () { 24 | t.equal(q.isDrained, true, 'should be true once drained event occurs') 25 | }) 26 | }) 27 | -------------------------------------------------------------------------------- /test/push.js: -------------------------------------------------------------------------------- 1 | var test = require('tape'), 2 | cq = require('..') 3 | 4 | test('push', function (t) { 5 | var q = cq() 6 | 7 | q('task 1', function (err, data) { 8 | t.equal(data, 'task 1', 'task 1 cb executed with expected data') 9 | t.false(err, 'no error on task 1') 10 | }) 11 | q('task 2', function (err) { 12 | t.true(err, 'task 2 cb executed with error') 13 | }) 14 | var promise = q('task 3') 15 | t.equal(q.pending.length + q.processing.length, 3, 'items.length reflects total queue size') 16 | t.ok(promise.then && promise.catch, 'queue should return promise') 17 | promise.then(function (task) { 18 | t.equal(task, 'task 3', 'promise should resolve') 19 | t.end() 20 | }) 21 | 22 | q.process(function (task, cb) { 23 | if (task === 'task 1') cb(null, task) 24 | if (task === 'task 2') cb(true) 25 | if (task === 'task 3') cb(null, task) 26 | }) 27 | }) 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Jason Pincin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test/max-size.js: -------------------------------------------------------------------------------- 1 | var test = require('tape'), 2 | cq = require('..'), 3 | errors = require('../errors') 4 | 5 | test('max size', function (t) { 6 | var q = cq() 7 | t.plan(9) 8 | 9 | t.equal(q.maxSize, Infinity, 'returns default maxSize initially') 10 | q.maxSize = 5 11 | t.equal(q.maxSize, 5, 'allows maxSize to be set') 12 | t.throws(function () { 13 | q.maxSize = {} 14 | }, 'must be a number') 15 | 16 | q.limit({ maxSize: 1 }).process(function (item, cb) { 17 | setTimeout(function () { 18 | cb(null, item) 19 | }, 10) 20 | }) 21 | q.rejected(function (rejected) { 22 | t.equal(rejected.item, 2, 'should get a rejected message with the item when max size is exceeded') 23 | t.true(rejected.err, 'should get a rejected message with an error when max size exceeded') 24 | }) 25 | 26 | q(1, function (err, result) { 27 | t.false(err, 'does not supply error under maxSize threshold') 28 | t.equal(result, 1, 'queue processes items under maxSize threshold correctly') 29 | }) 30 | q(2, function (err, result) { 31 | t.true(err, 'queue supplies error when maxSize exceeded') 32 | t.ok(err instanceof errors.MaxSizeExceededError, 'should throw the proper error type') 33 | }) 34 | }) 35 | -------------------------------------------------------------------------------- /test/soft-max-size.js: -------------------------------------------------------------------------------- 1 | var test = require('tape'), 2 | cq = require('..') 3 | 4 | test('soft max size', function (t) { 5 | var q = cq() 6 | t.plan(11) 7 | 8 | t.equal(q.softMaxSize, Infinity, 'returns default softMaxSize initially') 9 | q.softMaxSize = 5 10 | t.equal(q.softMaxSize, 5, 'allows softMaxSize to be set') 11 | t.throws(function () { 12 | q.softMaxSize = {} 13 | }, 'must be a number') 14 | 15 | q.limit({ softMaxSize: 1 }).process(function (item, cb) { 16 | setTimeout(function () { 17 | cb(null, item) 18 | }, 10) 19 | }) 20 | q.softLimitReached(function (reached) { 21 | t.equal(reached.size, q.size, 'should get a soft limit reached message with size = ' + q.size) 22 | }) 23 | 24 | q(1, function (err, result) { 25 | t.false(err, 'does not supply error under softMaxSize threshold') 26 | t.equal(result, 1, 'queue processes items under softMaxSize threshold correctly') 27 | }) 28 | q(2, function (err, result) { 29 | t.false(err, 'does not supply error at softMaxSize threshold') 30 | t.equal(result, 2, 'queue processes items at softMaxSize threshold correctly') 31 | }) 32 | q(3, function (err, result) { 33 | t.false(err, 'does not supply error over softMaxSize threshold') 34 | t.equal(result, 3, 'queue processes items over softMaxSize threshold correctly') 35 | }) 36 | }) 37 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "concurrent-queue", 3 | "version": "7.0.2", 4 | "description": "Fifo queue with concurrency control", 5 | "main": "index.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "git://github.com/jasonpincin/concurrent-queue.git" 9 | }, 10 | "keywords": [ 11 | "fifo", 12 | "queue", 13 | "concurrent", 14 | "concurrency", 15 | "callback", 16 | "promise" 17 | ], 18 | "license": "MIT", 19 | "bugs": { 20 | "url": "https://github.com/jasonpincin/concurrent-queue/issues" 21 | }, 22 | "homepage": "https://github.com/jasonpincin/concurrent-queue", 23 | "author": { 24 | "name": "Jason Pincin", 25 | "email": "jason@pincin.com", 26 | "url": "http://jason.pincin.com/" 27 | }, 28 | "contributors": [], 29 | "devDependencies": { 30 | "@jasonpincin/standard": "~5.0.0-8", 31 | "istanbul": "~0.3.17", 32 | "opn": "~1.0.2", 33 | "phantomjs": "~1.9.18", 34 | "tap-dot": "~1.0.0", 35 | "tap-spec": "~4.0.2", 36 | "tape": "~4.2.0", 37 | "zuul": "~3.3.0" 38 | }, 39 | "dependencies": { 40 | "afterward": "~2.0.0", 41 | "define-error": "~1.0.0", 42 | "eventuate": "~4.0.0", 43 | "object-assign": "~4.0.1", 44 | "on-error": "~2.1.0", 45 | "once": "~1.3.2", 46 | "promise-polyfill": "~2.1.0" 47 | }, 48 | "scripts": { 49 | "test": "make npm-test", 50 | "coverage": "make npm-coverage", 51 | "browser-test": "make browser-test" 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /test/process.js: -------------------------------------------------------------------------------- 1 | var test = require('tape'), 2 | cq = require('..'), 3 | Promise = require('promise-polyfill'), 4 | setImmediate = require('timers').setImmediate 5 | 6 | test('processor', function (t) { 7 | t.plan(1) 8 | var q = cq() 9 | function processor (task, cb) { 10 | cb() 11 | } 12 | 13 | t.equal(q.process(processor).processor, processor, 'processor property refers to function passed to process') 14 | }) 15 | 16 | test('process (cb)', function (t) { 17 | t.plan(4) 18 | var q = cq() 19 | function processor (task, cb) { 20 | cb() 21 | } 22 | 23 | t.throws(q.process, 'process requires a processor function') 24 | t.equal(q.process(processor), q, 'process returns reference to queue') 25 | t.throws(function () { 26 | q.process(processor) 27 | }, 'only one processor function may be defined') 28 | 29 | q('task 1') 30 | q('task 2') 31 | q('task 3') 32 | 33 | setImmediate(function () { 34 | 35 | t.equal(q.pending.length + q.processing.length, 0, 'tasks completed as queued with processor already defined') 36 | }) 37 | }) 38 | 39 | test('process (promise)', function (t) { 40 | t.plan(1) 41 | var q = cq().limit({ concurrency: 2 }).process(function (task) { 42 | return new Promise(function (resolve, reject) { 43 | resolve() 44 | }) 45 | }) 46 | 47 | Promise.all([ 48 | q('task 1'), 49 | q('task 2'), 50 | q('task 3') 51 | ]).then(function () { 52 | t.equal(q.pending.length + q.processing.length, 0, 'all tasks complete with promisy processor') 53 | }) 54 | }) 55 | -------------------------------------------------------------------------------- /test/events.js: -------------------------------------------------------------------------------- 1 | var test = require('tape'), 2 | cq = require('..') 3 | 4 | test('enqueued event', function (t) { 5 | t.plan(1) 6 | 7 | var q = cq() 8 | q.enqueued(function (enqueued) { 9 | t.equal(enqueued.item, 'task 1', 'should provide items added to queue') 10 | }) 11 | q('task 1') 12 | }) 13 | 14 | test('processingStarted event', function (t) { 15 | t.plan(1) 16 | 17 | var q = cq().process(function (item, cb) { }) 18 | q.processingStarted(function (started) { 19 | t.equal(started.item, 'task 1', 'should provide items as processing starts') 20 | }) 21 | q('task 1') 22 | }) 23 | 24 | test('processingEnded event', function (t) { 25 | t.plan(1) 26 | 27 | var q = cq().process(function (item, cb) { cb() }) 28 | q.processingEnded(function (completed) { 29 | t.equal(completed.item, 'task 1', 'should provide items ad processing completes') 30 | }) 31 | q('task 1') 32 | }) 33 | 34 | test('processingEnded event (with error)', function (t) { 35 | t.plan(2) 36 | 37 | var q = cq().process(function (item, cb) { 38 | cb(new Error('failed')) 39 | }) 40 | q.processingEnded(function (completed) { 41 | t.equal(completed.err.message, 'failed', 'should provide error when processing fails') 42 | t.equal(completed.item, 'task 1', 'should provide items when processing fails') 43 | }) 44 | q('task 1') 45 | }) 46 | 47 | test('drained event', function (t) { 48 | t.plan(2) 49 | 50 | var q = cq().limit({ concurrency: 3 }).process(function (item, cb) { 51 | setTimeout(function () { 52 | if (item > 5) return cb(new Error('too big')) 53 | else cb(null, item) 54 | }, 10) 55 | }) 56 | 57 | var cycleOneDone = false 58 | q.drained(function () { 59 | t.ok(true, 'only fires once per cycle') 60 | if (!cycleOneDone) seedQueue() 61 | cycleOneDone = true 62 | }) 63 | seedQueue() 64 | 65 | function seedQueue () { 66 | var numbers = [1, 9, 3, 0, 8, 4, 7, 2, 5, 6] 67 | numbers.forEach(q) 68 | } 69 | }) 70 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: help clean coverage-check browse-coverage coverage-report coverage-html-report test test-tap test-dot test-spec npm-test travis-test browser-test 2 | 3 | BIN = ./node_modules/.bin 4 | 5 | all: lint test coverage-html-report coverage-report coverage-check 6 | 7 | help: 8 | @echo 9 | @echo "To run tests:" 10 | @echo " npm test [--dot | --spec] [--phantom] [--grep=]" 11 | @echo 12 | @echo "To run tests in all browsers:" 13 | @echo " npm run browser-test" 14 | @echo 15 | @echo "To see coverage:" 16 | @echo " npm run coverage [--html]" 17 | @echo 18 | 19 | npm-test: 20 | ifdef npm_config_grep 21 | @make lint test 22 | else 23 | ifdef npm_config_phantom 24 | @make lint test 25 | else 26 | @make lint test coverage-check 27 | endif 28 | endif 29 | 30 | travis-test: lint test 31 | @(cat coverage/lcov.info | coveralls) || exit 0 32 | 33 | browser-test: 34 | @$(BIN)/zuul -- test/*.js 35 | 36 | npm-coverage: coverage-report coverage-html-report 37 | ifdef npm_config_html 38 | @make browse-coverage 39 | endif 40 | 41 | lint: 42 | @$(BIN)/standard 43 | 44 | test: 45 | $(if $(npm_config_grep), @echo "Running test files that match pattern: $(npm_config_grep)\n",) 46 | ifdef npm_config_dot 47 | @make test-dot 48 | else 49 | ifdef npm_config_spec 50 | @make test-spec 51 | else 52 | @make test-tap 53 | endif 54 | endif 55 | 56 | test-tap: 57 | ifdef npm_config_phantom 58 | @find ./test -maxdepth 1 -name "*.js" -type f | grep ""$(npm_config_grep) | xargs $(BIN)/zuul --phantom -- 59 | else 60 | @find ./test -maxdepth 1 -name "*.js" -type f | grep ""$(npm_config_grep) | xargs $(BIN)/istanbul cover --report lcovonly --print none $(BIN)/tape -- 61 | endif 62 | 63 | test-dot: 64 | @make test-tap | $(BIN)/tap-dot 65 | 66 | test-spec: 67 | @make test-tap | $(BIN)/tap-spec 68 | 69 | coverage: 70 | @make test 71 | 72 | coverage-check: coverage 73 | @rm -f coverage/error 74 | @$(BIN)/istanbul check-coverage --statements 100 --branches 100 --functions 100 --lines 100 2>&1 | cat > coverage/error 75 | $(if $(npm_config_grep),,@if [ -s coverage/error ]; then echo; grep ERROR coverage/error; echo; exit 1; fi) 76 | 77 | coverage-report: coverage 78 | @$(BIN)/istanbul report text 79 | 80 | coverage-html-report: coverage 81 | @$(BIN)/istanbul report html > /dev/null 82 | 83 | browse-coverage: coverage-html-report 84 | @$(BIN)/opn coverage/index.html 85 | 86 | clean: 87 | @rm -rf coverage 88 | -------------------------------------------------------------------------------- /test/drain.js: -------------------------------------------------------------------------------- 1 | var test = require('tape'), 2 | cq = require('..'), 3 | setImmediate = require('timers').setImmediate 4 | 5 | function processorImmediate (task, cb) { 6 | cb() 7 | } 8 | 9 | function processorFast (task, cb) { 10 | setImmediate(cb) 11 | } 12 | 13 | test('drain infinite concurrency and immediate processor', function (t) { 14 | var q = cq() 15 | 16 | q('task 1') 17 | q('task 2') 18 | q('task 3') 19 | 20 | t.equal(q.size, 3, 'items exist before processor') 21 | t.equal(q.pending.length, 3, 'items exist before processor') 22 | t.equal(q.processing.length, 0, 'nothing in processing state before processor') 23 | 24 | q.limit({ concurrency: Infinity }).process(processorImmediate) 25 | setImmediate(function () { 26 | t.equal(q.pending.length + q.processing.length, 0, 'all items processed immediately') 27 | t.end() 28 | }) 29 | }) 30 | 31 | test('drain with concurrency of 2 and immediate processor', function (t) { 32 | var q = cq() 33 | 34 | q('task 1') 35 | q('task 2') 36 | q('task 3') 37 | 38 | q.limit({ concurrency: 2 }).process(processorImmediate) 39 | setImmediate(function () { 40 | t.equal(q.pending.length + q.processing.length, 0, 'all items processed immediately') 41 | t.end() 42 | }) 43 | }) 44 | 45 | test('drain with concurrency of 2 and fast processor', function (t) { 46 | var q = cq() 47 | 48 | q('task 1') 49 | q('task 2') 50 | q('task 3') 51 | 52 | q.limit({ concurrency: 2 }).process(processorFast) 53 | setImmediate(function () { 54 | t.equal(q.size, 1, 'size = 1 after processor') 55 | t.equal(q.pending.length, 1, '1 item pending after processor') 56 | t.equal(q.processing.length, 2, '2 items processing after processor') 57 | 58 | setImmediate(function () { 59 | 60 | t.equal(q.size, 1, 'size = 1 after 2 ticks') 61 | t.equal(q.pending.length, 1, '1 item pending after 2 ticks') 62 | t.equal(q.processing.length, 0, '0 items processing 2 ticks') 63 | 64 | setImmediate(function () { 65 | 66 | t.equal(q.size, 0, 'size = 0 after 3 ticks') 67 | t.equal(q.pending.length, 0, '0 items pending after 3 ticks') 68 | t.equal(q.processing.length, 1, '1 item processing 3 ticks') 69 | 70 | setImmediate(function () { 71 | 72 | t.equal(q.size, 0, 'size = 0 after 4 ticks') 73 | t.equal(q.pending.length, 0, '0 items pending after 4 ticks') 74 | t.equal(q.processing.length, 0, '0 items processing 4 ticks') 75 | t.end() 76 | }) 77 | }) 78 | }) 79 | }) 80 | 81 | }) 82 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'), 2 | assign = require('object-assign'), 3 | onerr = require('on-error'), 4 | eventuate = require('eventuate'), 5 | once = require('once'), 6 | Promise = require('promise-polyfill'), 7 | after = require('afterward'), 8 | setImmediate = require('timers').setImmediate, 9 | MaxSizeExceededError = require('./errors').MaxSizeExceededError 10 | 11 | module.exports = function () { 12 | var pending = [], 13 | processing = [], 14 | maxSize = Infinity, 15 | softMaxSize = Infinity, 16 | concurrency = Infinity, 17 | drained = true, 18 | processor 19 | 20 | function cq (item, cb) { 21 | var done = new Promise(function (resolve, reject) { 22 | if (pending.length >= maxSize) { 23 | var err = new MaxSizeExceededError('unable to queue item') 24 | reject(err) 25 | return cq.rejected.produce({ item: item, err: err }) 26 | } 27 | if (pending.length >= softMaxSize) cq.softLimitReached.produce({ size: pending.length }) 28 | 29 | drained = false 30 | setImmediate(drain) 31 | pending.push({ 32 | item : item, 33 | resolve: onResolve, 34 | reject : onReject 35 | }) 36 | cq.enqueued.produce({ item: item }) 37 | 38 | function onResolve (value) { 39 | resolve(value) 40 | cq.processingEnded.produce({ item: item, result: value }) 41 | } 42 | 43 | function onReject (err) { 44 | reject(err) 45 | cq.processingEnded.produce({ item: item, err: err }) 46 | } 47 | }) 48 | 49 | return after(done, cb) 50 | } 51 | Object.defineProperties(cq, { 52 | size : { get: getSize, enumerable: true }, 53 | isDrained : { get: getIsDrained, enumerable: true }, 54 | pending : { get: getPending, enumerable: true }, 55 | processing : { get: getProcessing, enumerable: true }, 56 | concurrency : { get: getConcurrency, set: setConcurrency, enumerable: true }, 57 | maxSize : { get: getMaxSize, set: setMaxSize, enumerable: true }, 58 | softMaxSize : { get: getSoftMaxSize, set: setSoftMaxSize, enumerable: true }, 59 | processor : { get: getProcessor }, 60 | limit : { value: limit }, 61 | process : { value: process }, 62 | enqueued : { value: eventuate() }, 63 | rejected : { value: eventuate() }, 64 | softLimitReached : { value: eventuate() }, 65 | processingStarted: { value: eventuate() }, 66 | processingEnded : { value: eventuate() }, 67 | drained : { value: eventuate() } 68 | }) 69 | 70 | return cq 71 | 72 | function drain () { 73 | if (!drained && pending.length === 0 && processing.length === 0) { 74 | drained = true 75 | cq.drained.produce() 76 | } 77 | while (processor && pending.length > 0 && processing.length < concurrency) drainItem() 78 | function drainItem () { 79 | var task = pending.shift() 80 | processing.push(task) 81 | cq.processingStarted.produce({ item: task.item }) 82 | 83 | var reject = once(function reject (err) { 84 | processing.splice(processing.indexOf(task), 1) 85 | task.reject(err) 86 | setImmediate(drain) 87 | }) 88 | var resolve = once(function resolve () { 89 | processing.splice(processing.indexOf(task), 1) 90 | task.resolve.apply(undefined, arguments) 91 | setImmediate(drain) 92 | }) 93 | 94 | var p 95 | try { 96 | p = processor(task.item, onerr(reject).otherwise(resolve)) 97 | } 98 | catch (err) { 99 | return reject(err) 100 | } 101 | if (p && typeof p.then === 'function') p.then(resolve, reject) 102 | } 103 | } 104 | 105 | function process (func) { 106 | if (typeof func !== 'function') throw new TypeError('process requires a processor function') 107 | assert(!processor, 'queue processor already defined') 108 | processor = func 109 | setImmediate(drain) 110 | return cq 111 | } 112 | 113 | function limit (limits) { 114 | limits = assign({ concurrency: Infinity, maxSize: Infinity, softMaxSize: Infinity }, limits) 115 | if (typeof limits.maxSize !== 'number') throw new TypeError('maxSize must be a number') 116 | if (typeof limits.softMaxSize !== 'number') throw new TypeError('softMaxSize must be a number') 117 | if (typeof limits.concurrency !== 'number') throw new TypeError('concurrency must be a number') 118 | maxSize = limits.maxSize 119 | softMaxSize = limits.softMaxSize 120 | concurrency = limits.concurrency 121 | return cq 122 | } 123 | 124 | function getSize () { 125 | return pending.length 126 | } 127 | 128 | function getIsDrained () { 129 | return drained 130 | } 131 | 132 | function getPending () { 133 | return pending.map(function (task) { 134 | return task.item 135 | }) 136 | } 137 | 138 | function getProcessing () { 139 | return processing.map(function (task) { 140 | return task.item 141 | }) 142 | } 143 | 144 | function getConcurrency () { 145 | return concurrency 146 | } 147 | 148 | function setConcurrency (value) { 149 | if (typeof value !== 'number') throw new TypeError('concurrency must be a number') 150 | concurrency = value 151 | } 152 | 153 | function getMaxSize () { 154 | return maxSize 155 | } 156 | 157 | function setMaxSize (value) { 158 | if (typeof value !== 'number') throw new TypeError('maxSize must be a number') 159 | maxSize = value 160 | } 161 | 162 | function getSoftMaxSize () { 163 | return softMaxSize 164 | } 165 | 166 | function setSoftMaxSize (value) { 167 | if (typeof value !== 'number') throw new TypeError('softMaxSize must be a number') 168 | softMaxSize = value 169 | } 170 | 171 | function getProcessor () { 172 | return processor 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # concurrent-queue 2 | 3 | [![NPM version](https://badge.fury.io/js/concurrent-queue.png)](http://badge.fury.io/js/concurrent-queue) 4 | [![Build Status](https://travis-ci.org/jasonpincin/concurrent-queue.svg?branch=master)](https://travis-ci.org/jasonpincin/concurrent-queue) 5 | [![Coverage Status](https://coveralls.io/repos/jasonpincin/concurrent-queue/badge.png?branch=master)](https://coveralls.io/r/jasonpincin/concurrent-queue?branch=master) 6 | [![Sauce Test Status](https://saucelabs.com/browser-matrix/jp-concurrent-queue.svg)](https://saucelabs.com/u/jp-concurrent-queue) 7 | 8 | Fifo queue with concurrency control 9 | 10 | ## example 11 | 12 | ```javascript 13 | var cq = require('concurrent-queue') 14 | 15 | var queue = cq().limit({ concurrency: 2 }).process(function (task, cb) { 16 | console.log(task + ' started') 17 | setTimeout(function () { 18 | cb(null, task) 19 | }, 1000) 20 | }) 21 | 22 | for (var i = 1; i <= 10; i++) queue('task '+i, function (err, task) { 23 | console.log(task + ' done') 24 | }) 25 | ``` 26 | 27 | or with promises: 28 | 29 | ```javascript 30 | var cq = require('concurrent-queue') 31 | 32 | var queue = cq().limit({ concurrency: 2 }).process(function (task) { 33 | return new Promise(function (resolve, reject) { 34 | console.log(task + ' started') 35 | setTimeout(resolve.bind(undefined, task), 1000) 36 | }) 37 | }) 38 | 39 | for (var i = 1; i <= 10; i++) queue('task '+i).then(function (task) { 40 | console.log(task + ' done') 41 | }) 42 | ``` 43 | 44 | ## api 45 | 46 | ```javascript 47 | var cq = require('concurrent-queue') 48 | ``` 49 | 50 | ### var queue = cq() 51 | 52 | Create a queue. 53 | 54 | ### queue(item [, cb]) 55 | 56 | Push an item to the queue. Once the item has been processed, the optional callback will be executed with arguments determined by the processor. 57 | 58 | Returns a promise that will be resolved or rejected once the item is processed. 59 | 60 | ### queue.process(processor) 61 | 62 | Configure the queue's `processor` function, to be invoked as concurrency allows with a queued item to be acted upon. 63 | 64 | The `processor` argument should be a function with signature `function (item [, cb])`. If the processor function signature included a callback, an error-first style callback will be passed which should be executed upon completion. If no callback is provided in the function signature, and the processor function returns a `Promise`, the item will be considered complete once the promise is resolved/rejected. 65 | 66 | This function returns a reference to `queue`. 67 | 68 | ### queue.limit(limitObj) 69 | 70 | Set queue limits with a limits object. Valid limit properties are: 71 | 72 | * `concurrency` - (default: `Infinity`) - determine how many items in the queue will be processed concurrently 73 | * `maxSize` - (default: `Infinity`) - determine how many items may be pending in the queue before additional items are no longer accepted. When an item is added that would exceed this, the `callback` associated with the item will be invoked with an error and/or the `promise` returned by `queue()` will be rejected. 74 | * `softMaxSize` - (default: `Infinity`) - determine how many items may be pending before the queue begins producing warnings on the `softLimitReached` eventuate property. 75 | 76 | This function returns a reference to `queue`. 77 | 78 | ### queue.enqueued(func) 79 | 80 | `enqueued` is an [eventuate](https://github.com/jasonpincin/eventuate). Use this to supply a function that will be executed when an item is added to the queue. The function will be passed an object with the following properties: 81 | 82 | * `item` - The queued item that is being processed 83 | 84 | ### queue.rejected(func) 85 | 86 | `rejected` is an eventuate. Register a function to be executed when an item is rejected from the queue. This can happen, for example, when maxSize is exceeded. The function will be passed an object with the following properties: 87 | 88 | * `item` - The item that was rejected from the queue 89 | * `err` - An error containing the reason for rejection 90 | 91 | ### queue.softLimitReached(func) 92 | 93 | `softLimitReached` is an eventuate. Register a function to be executed when the configured soft size limit has been reached or exceeded. This function will be executed any time an item is added to the `queue` when the `queue.limit` meets or exceeds the `softMaxSize` value. The function will be passed an object with the following properties: 94 | 95 | * `size` - the `queue.size` 96 | 97 | ### queue.processingStarted(func) 98 | 99 | `processingStarted` is an eventuate. Register a function to be executed once an item has transitioned from `pending` to `processing`. The function will be passed an object with the following properties: 100 | 101 | * `item` - The queued item that is being processed 102 | 103 | ### queue.processingEnded(func) 104 | 105 | `processingEnded` is an eventuate. Register a function to be executed once processing of an item has completed or failed. The function will be passed an object with the following properties: 106 | 107 | * `item` - The queued item that was processed 108 | * `err` - Will be present if there was an error while processing the item 109 | 110 | ### queue.drained(func) 111 | 112 | `drained` is an eventuate. Register a function to be executed each time the queue is fully drained (no items pending or processing). 113 | 114 | ### queue.size 115 | 116 | A numeric value representing the number of items in queue, waiting to be processed. 117 | 118 | ### queue.isDrained 119 | 120 | A boolean value indicating whether the queue is in a drained state (no items pending or processing). 121 | 122 | ### queue.pending 123 | 124 | An array of items waiting to be processed. 125 | 126 | ### queue.processor 127 | 128 | The processor function is one has been configured via `queue.process()`, 129 | otherwise `undefined`. This is a read-only (getter) property. 130 | 131 | ### queue.processing 132 | 133 | An array of items currently being processed. 134 | 135 | ### queue.concurrency 136 | 137 | An integer property representing the number of concurrent queue items that will be processed. This defaults to `Infinity`, but may be re-assigned. An integer value must be assigned to this property. This property may also be set by calling the `limit()` function and passing an object with the `concurrency` property. Setting this property to `0` will halt the queue (once all in-process items are complete), while setting it to `Infinity` removes all limits. 138 | 139 | ### queue.maxSize 140 | 141 | An integer property representing the maximum number of items that may be pending in the queue. This defaults to `Infinity`, but may be re-assigned. An integer value must be assigned to this property. This property may also be set by calling the `limit()` function and passing an object with the `maxSize` property. 142 | 143 | ### queue.softMaxSize 144 | 145 | An integer property representing the maximum number of items that may be pending in the queue before warnings are produced. This defaults to `Infinity`, but may be re-assigned. An integer value must be assigned to this property. This property may also be set by calling the `limit()` function and passing an object with the `softMaxSize` property. 146 | 147 | ### errors 148 | 149 | ```javascript 150 | var errors = require('concurrent-queue/errors') 151 | var MaxSizeExceededError = errors.MaxSizeExceededError 152 | ``` 153 | 154 | #### MaxSizeExceededError 155 | 156 | Constructor for errors representing the `queue.maxSize` constraint being exceeded. This is supplied to the callback and/or promise rejection when an item cannot be queued due to `queue.maxSize` constraints. Example: 157 | 158 | ```javascript 159 | var cq = require('concurrent-queue'), 160 | MaxSizeExceededError = require('concurrent-queue/errors').MaxSizeExceededError 161 | 162 | queue = cq().limit({ maxSize: 100, concurrency: 1 }).process(function (item, cb) { 163 | // do something 164 | }) 165 | 166 | queue({}, function (err, result) { 167 | if (err instanceof MaxSizeExceededError) { 168 | // the queue is full 169 | } 170 | else if (err) { 171 | // otherwise an error happened while processing... 172 | } 173 | }) 174 | ``` 175 | 176 | ## install 177 | 178 | With [npm](https://npmjs.org) do: 179 | 180 | ``` 181 | npm install concurrent-queue 182 | ``` 183 | 184 | ## testing 185 | 186 | `npm test [--dot | --spec] [--phantom] [--grep=pattern]` 187 | 188 | Specifying `--dot` or `--spec` will change the output from the default TAP style. 189 | Specifying `--phantom` will cause the tests to run in the headless phantom browser instead of node. 190 | Specifying `--grep` will only run the test files that match the given pattern. 191 | 192 | ### browser test 193 | 194 | `npm run browser-test` 195 | 196 | This will run the tests in all browsers (specified in .zuul.yml). Be sure to [educate zuul](https://github.com/defunctzombie/zuul/wiki/cloud-testing#2-educate-zuul) first. 197 | 198 | ### coverage 199 | 200 | `npm run coverage [--html]` 201 | 202 | This will output a textual coverage report. Including `--html` will also open 203 | an HTML coverage report in the default browser. 204 | --------------------------------------------------------------------------------