├── .gitignore ├── .travis.yml ├── timestamp.js ├── index.js ├── peek.js ├── package.json ├── tests ├── client.js └── test.js ├── client.js ├── README.md └── server.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | tests/db 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.12" 4 | - "4" 5 | - "6" -------------------------------------------------------------------------------- /timestamp.js: -------------------------------------------------------------------------------- 1 | module.exports = timestamp; 2 | 3 | var lastTime; 4 | 5 | function timestamp() { 6 | var t = Date.now() * 1024 + Math.floor(Math.random() * 1024); 7 | if (lastTime) t = lastTime + 1; 8 | lastTime = t; 9 | return t; 10 | } 11 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var Server = require('./server'); 2 | var Client = require('./client'); 3 | 4 | // Combine Server and Client 5 | exports = module.exports = function Jobs(db, worker, options) { 6 | return mixin(Server(db, worker, options), Client.Queue.prototype); 7 | } 8 | 9 | function mixin(a, b) { 10 | var target = Object.create(a); // avoids modifying original 11 | Object.keys(b).forEach(function(key) { 12 | target[key] = b[key]; 13 | }); 14 | return target; 15 | } 16 | -------------------------------------------------------------------------------- /peek.js: -------------------------------------------------------------------------------- 1 | module.exports = peek; 2 | 3 | function peek(db, cb) { 4 | 5 | var calledback = false; 6 | function callback() { 7 | if (! calledback) { 8 | calledback = true; 9 | cb.apply(null, arguments); 10 | } 11 | } 12 | 13 | var s = db.createReadStream({ limit: 1}); 14 | 15 | s.on('error', callback); 16 | s.once('end', callback); 17 | 18 | s.once('data', function(d) { 19 | if (d) callback(null, d.key, d.value); 20 | else callback(); 21 | }); 22 | 23 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "level-jobs", 3 | "version": "2.1.1", 4 | "description": "Job Queue in LevelDB", 5 | "main": "index.js", 6 | "browser": "client.js", 7 | "scripts": { 8 | "test": "tap tests/test.js tests/client.js" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git://github.com/pgte/level-jobs.git" 13 | }, 14 | "keywords": [ 15 | "leveldb", 16 | "job", 17 | "queue" 18 | ], 19 | "author": "pgte", 20 | "contributors": [ 21 | "Pedro Teixeira ", 22 | "Tim Oxley ", 23 | "Lars-Magnus Skog ", 24 | "Jannis Redmann ", 25 | "Ash", 26 | "Mark Vayngrib " 27 | ], 28 | "license": "MIT", 29 | "bugs": { 30 | "url": "https://github.com/pgte/level-jobs/issues" 31 | }, 32 | "dependencies": { 33 | "backoff": "~2.3.0", 34 | "json-stringify-safe": "~5.0.0", 35 | "level-hooks": "~4.5.0", 36 | "level-sublevel": "~6.4.6", 37 | "level-write-stream": "~1.0.0", 38 | "xtend": "~2.0.6" 39 | }, 40 | "devDependencies": { 41 | "level": "~1.3.0", 42 | "tap": "~0.4.4", 43 | "rimraf": "~2.2.2", 44 | "async": "*" 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /tests/client.js: -------------------------------------------------------------------------------- 1 | var test = require('tap').test; 2 | var rimraf = require('rimraf'); 3 | var level = require('level'); 4 | var async = require('async'); 5 | var Jobs = require('../'); 6 | var ClientJobs = require('../client'); 7 | 8 | var dbPath = __dirname + '/db'; 9 | 10 | test('can insert and delete job', function(t) { 11 | rimraf.sync(dbPath); 12 | var db = level(dbPath); 13 | var jobs = Jobs(db, worker); 14 | 15 | var clientQueue = ClientJobs(db); 16 | var processed = 0; 17 | 18 | function worker (id, payload, done) { 19 | processed += 1; 20 | t.ok(processed <= 1, 'worker is not called 2 times'); 21 | 22 | clientQueue.del(job2Id, function(err) { 23 | if (err) throw err; 24 | done(); 25 | }); 26 | 27 | setTimeout(function() { 28 | db.once('closed', t.end.bind(t)); 29 | db.close(); 30 | }, 500); 31 | }; 32 | 33 | var job1Id = clientQueue.push({ foo: 'bar', seq: 1 }); 34 | t.type(job1Id, 'number'); 35 | 36 | var job2Id = clientQueue.push({ foo: 'bar', seq: 2 }); 37 | t.type(job2Id, 'number'); 38 | }); 39 | 40 | test('can insert and delete jobs in batches', function(t) { 41 | rimraf.sync(dbPath); 42 | var db = level(dbPath); 43 | var jobs = Jobs(db, worker); 44 | 45 | var clientQueue = ClientJobs(db); 46 | var processed = 0; 47 | 48 | function worker (id, payload, done) { 49 | processed += 1; 50 | t.ok(processed <= 1, 'worker is not called 2 times'); 51 | 52 | var remainingJobsIds = jobBatchIds.slice(1) 53 | 54 | clientQueue.delBatch(remainingJobsIds, function(err) { 55 | if (err) throw err; 56 | done(); 57 | }); 58 | 59 | setTimeout(function() { 60 | db.once('closed', t.end.bind(t)); 61 | db.close(); 62 | }, 500); 63 | }; 64 | 65 | var jobBatchIds = clientQueue.pushBatch([ 66 | { foo: 'bar', seq: 1 }, 67 | { foo: 'bar', seq: 2 }, 68 | { foo: 'bar', seq: 3 } 69 | ]); 70 | 71 | jobBatchIds.forEach(function(id) { 72 | t.type(id, 'number'); 73 | }) 74 | }); 75 | -------------------------------------------------------------------------------- /client.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var inherits = require('util').inherits; 3 | var EventEmitter = require('events').EventEmitter; 4 | var Sublevel = require('level-sublevel'); 5 | var stringify = require('json-stringify-safe'); 6 | var xtend = require('xtend'); 7 | var timestamp = require('./timestamp'); 8 | 9 | exports = module.exports = ClientQueue; 10 | 11 | function ClientQueue(db, worker, options) { 12 | assert.equal(typeof db, 'object', 'need db'); 13 | assert.equal(arguments.length, 1, 'cannot define worker on client'); 14 | 15 | return new Queue(db); 16 | } 17 | 18 | ClientQueue.Queue = Queue 19 | 20 | function Queue(db) { 21 | EventEmitter.call(this); 22 | 23 | this._db = db = Sublevel(db); 24 | this._pending = db.sublevel('pending'); 25 | this._work = db.sublevel('work'); 26 | } 27 | 28 | inherits(Queue, EventEmitter); 29 | 30 | var Q = Queue.prototype; 31 | 32 | /// push 33 | 34 | Q.push = function push(payload, cb) { 35 | var q = this; 36 | var id = timestamp(); 37 | this._work.put(id, stringify(payload), put); 38 | 39 | return id; 40 | 41 | function put(err) { 42 | if (err) { 43 | if (cb) cb(err); 44 | else q.emit('error', err); 45 | } else if (cb) cb(); 46 | }; 47 | } 48 | 49 | /// pushBatch 50 | 51 | Q.pushBatch = function push(payloads, cb) { 52 | var q = this; 53 | var ids = []; 54 | 55 | var ops = payloads.map(function(payload) { 56 | var id = timestamp(); 57 | ids.push(id) 58 | return { 59 | type: 'put', 60 | key: id, 61 | value: stringify(payload) 62 | } 63 | }) 64 | 65 | this._work.batch(ops, batch); 66 | 67 | return ids; 68 | 69 | function batch(err) { 70 | if (err) { 71 | if (cb) cb(err); 72 | else q.emit('error', err); 73 | } else if (cb) cb(); 74 | }; 75 | } 76 | 77 | /// del 78 | 79 | Q.del = function del(id, cb) { 80 | this._work.del(id, cb); 81 | } 82 | 83 | /// delBatch 84 | 85 | Q.delBatch = function del(ids, cb) { 86 | var ops = ids.map(function(id) { 87 | return { type: 'del', key: id } 88 | }) 89 | this._work.batch(ops, cb); 90 | } 91 | 92 | Q.pendingStream = function pendingStream(options) { 93 | if (!options) options = {}; 94 | else options = xtend({}, options); 95 | options.valueEncoding = 'json'; 96 | return this._pending.createReadStream(options); 97 | }; 98 | 99 | Q.runningStream = function runningStream(options) { 100 | if (!options) options = {}; 101 | else options = xtend({}, options); 102 | options.valueEncoding = 'json'; 103 | return this._work.createReadStream(options); 104 | }; 105 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # level-jobs 2 | 3 | > Job Queue in LevelDB for Node.js 4 | 5 | [![Build Status](https://travis-ci.org/pgte/level-jobs.png?branch=master)](https://travis-ci.org/pgte/level-jobs) 6 | 7 | * Define worker functions 8 | * Persist work units 9 | * Work units are retried when failed 10 | * Define maximum concurrency 11 | 12 | ## Install 13 | 14 | ```bash 15 | $ npm install level-jobs --save 16 | ``` 17 | 18 | ## Use 19 | 20 | ### Create a levelup database 21 | 22 | ```javascript 23 | var levelup = require('levelup'); 24 | var db = levelup('./db') 25 | ``` 26 | 27 | ### Require level-jobs 28 | 29 | ```javascript 30 | var Jobs = require('level-jobs'); 31 | ``` 32 | 33 | ### Define a worker function 34 | 35 | This function will take care of a work unit. 36 | 37 | ```javascript 38 | function worker(id, payload, cb) { 39 | doSomething(cb); 40 | } 41 | ``` 42 | 43 | This function gets 3 arguments: 44 | 45 | - `id` uniquely identifies a job to be executed. 46 | - `payload` contains everyting `worker` need to process the job. 47 | - `cb` is the callback function that must be called when the job is done. 48 | 49 | This callback function accepts an error as the first argument. If an error is provided, the work unit is retried. 50 | 51 | 52 | ### Wrap the database 53 | 54 | ```javascript 55 | var queue = Jobs(db, worker); 56 | ``` 57 | 58 | This database will be at the mercy and control of level-jobs, don't use it for anything else! 59 | 60 | (this database can be a root levelup database or a sublevel) 61 | 62 | You can define a maximum concurrency (the default is `Infinity`): 63 | 64 | ```javascript 65 | var maxConcurrency = 2; 66 | var queue = Jobs(db, worker, maxConcurrency); 67 | ``` 68 | 69 | ### More Options 70 | 71 | As an alternative the third argument can be an options object with these defaults: 72 | 73 | ```javascript 74 | var options = { 75 | maxConcurrency: Infinity, 76 | maxRetries: 10, 77 | backoff: { 78 | randomisationFactor: 0, 79 | initialDelay: 10, 80 | maxDelay: 300 81 | } 82 | }; 83 | 84 | var queue = Jobs(db, worker, options); 85 | ``` 86 | 87 | ### Push work to the queue 88 | 89 | ```javascript 90 | var payload = {what: 'ever'}; 91 | 92 | var jobId = queue.push(payload, function(err) { 93 | if (err) console.error('Error pushing work into the queue', err.stack); 94 | }); 95 | ``` 96 | 97 | or in batch: 98 | ```javascript 99 | var payloads = [ 100 | {what: 'ever'}, 101 | {what: 'ever'} 102 | ]; 103 | 104 | var jobIds = queue.pushBatch(payloads, function(err) { 105 | if (err) console.error('Error pushing works into the queue', err.stack); 106 | }); 107 | ``` 108 | 109 | ### Delete pending job 110 | 111 | (Only works for jobs that haven't started yet!) 112 | 113 | ```javascript 114 | queue.del(jobId, function(err) { 115 | if (err) console.error('Error deleting job', err.stack); 116 | }); 117 | ``` 118 | 119 | or in batch: 120 | ```javascript 121 | queue.delBatch(jobIds, function(err) { 122 | if (err) console.error('Error deleting jobs', err.stack); 123 | }); 124 | ``` 125 | 126 | ### Traverse jobs 127 | 128 | `queue.pendingStream()` emits queued jobs. `queue.runningStream()` emits currently running jobs. 129 | 130 | ```javascript 131 | var stream = queue.pendingStream(); 132 | stream.on('data', function(d) { 133 | var jobId = d.key; 134 | var work = d.value; 135 | console.log('pending job id: %s, work: %j', jobId, work); 136 | }); 137 | ``` 138 | 139 | ### Events 140 | 141 | A queue object emits the following event: 142 | 143 | * `drain` — when there are no more jobs pending. Also happens on startup after consuming the backlog work units. 144 | * `error` - when something goes wrong. 145 | * `retry` - when a job is retried because something goes wrong. 146 | 147 | 148 | ## Client isolated API 149 | 150 | If you simply want a pure queue client that is only able to push jobs into the queue, you can use `level-jobs/client` like this: 151 | 152 | ```javascript 153 | var QueueClient = require('level-jobs/client'); 154 | 155 | var client = QueueClient(db); 156 | 157 | client.push(work, function(err) { 158 | if (err) throw err; 159 | console.log('pushed'); 160 | }); 161 | ``` 162 | 163 | ## License 164 | 165 | MIT 166 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var inherits = require('util').inherits; 3 | var EventEmitter = require('events').EventEmitter; 4 | var Sublevel = require('level-sublevel'); 5 | var stringify = require('json-stringify-safe'); 6 | var backoff = require('backoff'); 7 | var xtend = require('xtend'); 8 | var Hooks = require('level-hooks'); 9 | var WriteStream = require('level-write-stream'); 10 | var peek = require('./peek'); 11 | var timestamp = require('./timestamp'); 12 | 13 | var defaultOptions = { 14 | maxConcurrency: Infinity, 15 | maxRetries: 10, 16 | backoff: { 17 | randomisationFactor: 0, 18 | initialDelay: 10, 19 | maxDelay: 300 20 | } 21 | }; 22 | 23 | exports = module.exports = Jobs; 24 | 25 | function Jobs(db, worker, options) { 26 | assert.equal(typeof db, 'object', 'need db'); 27 | assert.equal(typeof worker, 'function', 'need worker function'); 28 | 29 | return new Queue(db, worker, options); 30 | } 31 | 32 | Jobs.Queue = Queue; 33 | 34 | function Queue(db, worker, options) { 35 | var q = this; 36 | EventEmitter.call(this); 37 | 38 | if (typeof options == 'number') options = { maxConcurrency: options }; 39 | options = xtend(defaultOptions, options); 40 | 41 | this._options = options; 42 | this._db = db = Sublevel(db); 43 | this._work = db.sublevel('work'); 44 | this._workWriteStream = WriteStream(this._work); 45 | this._pending = db.sublevel('pending'); 46 | this._worker = worker; 47 | this._concurrency = 0; 48 | 49 | // flags 50 | this._starting = true; 51 | this._flushing = false; 52 | this._peeking = false; 53 | this._needsFlush = false; 54 | this._needsDrain = true; 55 | 56 | // hooks 57 | Hooks(this._work); 58 | this._work.hooks.post(function() { 59 | maybeFlush(q) 60 | }); 61 | 62 | 63 | start(this); 64 | } 65 | 66 | inherits(Queue, EventEmitter); 67 | 68 | var Q = Queue.prototype; 69 | 70 | /// start 71 | 72 | function start(q) { 73 | var ws = q._workWriteStream(); 74 | q._pending.createReadStream().pipe(ws); 75 | ws.once('finish', done); 76 | 77 | function done() { 78 | q._starting = false; 79 | maybeFlush(q); 80 | } 81 | } 82 | 83 | 84 | /// maybeFlush 85 | 86 | function maybeFlush(q) { 87 | if (! q._starting && ! q._flushing) flush(q); 88 | else q._needsFlush = true; 89 | } 90 | 91 | /// flush 92 | 93 | function flush(q) { 94 | if (q._concurrency < q._options.maxConcurrency && ! q._peeking) { 95 | q._peeking = true; 96 | q._flushing = true; 97 | peek(q._work, poke); 98 | } 99 | 100 | function poke(err, key, work) { 101 | q._peeking = false; 102 | var done = false; 103 | 104 | if (key) { 105 | q._concurrency ++; 106 | q._db.batch([ 107 | { type: 'del', key: key, prefix: q._work }, 108 | { type: 'put', key: key, value: work, prefix: q._pending } 109 | ], transfered); 110 | } else { 111 | q._flushing = false; 112 | if (q._needsFlush) { 113 | q._needsFlush = false; 114 | maybeFlush(q); 115 | } else if (q._needsDrain) { 116 | q._needsDrain = false; 117 | q.emit('drain'); 118 | } 119 | } 120 | 121 | function transfered(err) { 122 | if (err) { 123 | q._needsDrain = true; 124 | q._concurrency --; 125 | q.emit('error', err); 126 | } else { 127 | run(q, key, JSON.parse(work), ran); 128 | } 129 | flush(q); 130 | } 131 | 132 | function ran(err) { 133 | if (!err) { 134 | if (! done) { 135 | done = true; 136 | q._needsDrain = true; 137 | q._concurrency --; 138 | q._pending.del(key, deletedPending); 139 | } 140 | } else handleRunError(err); 141 | } 142 | 143 | function deletedPending(_err) { 144 | if (err) q.emit('error', _err); 145 | flush(q); 146 | } 147 | 148 | function handleRunError(err) { 149 | var errorBackoff = backoff.exponential(q._options.backoff); 150 | errorBackoff.failAfter(q._options.maxRetries); 151 | 152 | errorBackoff.on('ready', function() { 153 | q.emit('retry', err); 154 | run(q, key, JSON.parse(work), ranAgain); 155 | }); 156 | 157 | errorBackoff.once('fail', function() { 158 | q.emit('error', new Error('max retries reached')); 159 | }); 160 | 161 | function ranAgain(err) { 162 | if (err) errorBackoff.backoff(); 163 | else ran(); 164 | } 165 | 166 | errorBackoff.backoff(); 167 | } 168 | } 169 | } 170 | 171 | /// run 172 | 173 | function run(q, id, work, cb) { 174 | q._worker(id, work, cb); 175 | } 176 | 177 | -------------------------------------------------------------------------------- /tests/test.js: -------------------------------------------------------------------------------- 1 | var test = require('tap').test; 2 | var rimraf = require('rimraf'); 3 | var level = require('level'); 4 | var async = require('async'); 5 | var Jobs = require('../'); 6 | 7 | var dbPath = __dirname + '/db'; 8 | 9 | test('passes job id into worker fn', function(t) { 10 | rimraf.sync(dbPath); 11 | var db = level(dbPath); 12 | 13 | var queue = Jobs(db, worker); 14 | var jobId = queue.push({foo: 'bar'}, t.ifError.bind(t)); 15 | 16 | function worker(id, work, cb) { 17 | t.equal(id, jobId + ''); 18 | 19 | db.once('closed', t.end.bind(t)); 20 | db.close(); 21 | }; 22 | }); 23 | 24 | test('infinite concurrency', function(t) { 25 | 26 | rimraf.sync(dbPath); 27 | var db = level(dbPath); 28 | 29 | var max = 10; 30 | var queue = Jobs(db, worker); 31 | 32 | for (var i = 1 ; i <= max ; i ++) { 33 | queue.push({n:i}, pushed); 34 | } 35 | 36 | function pushed(err) { 37 | if (err) throw err; 38 | } 39 | 40 | var count = 0; 41 | var cbs = []; 42 | function worker(id, work, cb) { 43 | count ++; 44 | t.equal(work.n, count); 45 | cbs.push(cb); 46 | if (count == max) callback(); 47 | }; 48 | 49 | function callback() { 50 | while(cbs.length) cbs.shift()(); 51 | } 52 | queue.on('drain', function() { 53 | if (count == max) { 54 | t.equal(cbs.length, 0); 55 | t.equal(queue._concurrency, 0); 56 | db.once('closed', t.end.bind(t)); 57 | console.log('CLOSING'); 58 | db.close(); 59 | } 60 | }); 61 | }); 62 | 63 | test('concurrency of 1', function(t) { 64 | 65 | rimraf.sync(dbPath); 66 | var db = level(dbPath); 67 | 68 | var max = 10; 69 | var concurrency = 1; 70 | var queue = Jobs(db, worker, concurrency); 71 | 72 | for (var i = 1 ; i <= max ; i ++) { 73 | queue.push({n:i}, pushed); 74 | } 75 | 76 | function pushed(err) { 77 | if (err) throw err; 78 | } 79 | 80 | var count = 0; 81 | var working = false; 82 | function worker(id, work, cb) { 83 | t.notOk(working, 'should not be concurrent'); 84 | count ++; 85 | working = true; 86 | t.equal(work.n, count); 87 | setTimeout(function() { 88 | working = false; 89 | cb(); 90 | }, 100); 91 | }; 92 | 93 | queue.on('drain', function() { 94 | if (count == max) { 95 | t.equal(queue._concurrency, 0); 96 | db.once('closed', t.end.bind(t)); 97 | db.close(); 98 | } 99 | }); 100 | }); 101 | 102 | test('retries on error', function(t) { 103 | rimraf.sync(dbPath); 104 | var db = level(dbPath); 105 | 106 | var max = 10; 107 | var queue = Jobs(db, worker); 108 | 109 | for (var i = 1 ; i <= max ; i ++) { 110 | queue.push({n:i}, pushed); 111 | } 112 | 113 | function pushed(err) { 114 | if (err) throw err; 115 | } 116 | 117 | var erroredOn = {}; 118 | var count = 0; 119 | function worker(id, work, cb) { 120 | count ++; 121 | if (!erroredOn[work.n]) { 122 | erroredOn[work.n] = true; 123 | cb(new Error('oops!')); 124 | } else { 125 | working = true; 126 | cb(); 127 | } 128 | }; 129 | 130 | queue.on('drain', function() { 131 | if (count == max * 2) { 132 | t.equal(queue._concurrency, 0); 133 | db.once('closed', t.end.bind(t)); 134 | db.close(); 135 | } 136 | }); 137 | }); 138 | 139 | test('emits retry event on retry', function(t) { 140 | rimraf.sync(dbPath); 141 | var db = level(dbPath); 142 | var queue = Jobs(db, worker); 143 | 144 | queue.on('error', Function()); 145 | 146 | queue.once('retry', function(err) { 147 | db.once('closed', t.end.bind(t)); 148 | db.close(); 149 | }); 150 | 151 | function worker(id, work, cb) { 152 | cb(new Error('oops!')); 153 | }; 154 | 155 | queue.push({ foo: 'bar' }); 156 | }); 157 | 158 | test('works with no push callback', function(t) { 159 | rimraf.sync(dbPath); 160 | var db = level(dbPath); 161 | var jobs = Jobs(db, worker); 162 | 163 | function worker (id, payload, done) { 164 | done(); 165 | process.nextTick(function() { 166 | db.once('closed', t.end.bind(t)); 167 | db.close(); 168 | }); 169 | }; 170 | 171 | jobs.push({ foo: 'bar' }); 172 | }); 173 | 174 | test('has exponential backoff in case of error', function(t) { 175 | rimraf.sync(dbPath); 176 | var db = level(dbPath); 177 | var jobs = Jobs(db, worker); 178 | 179 | function worker (id, payload, done) { 180 | done(new Error('Oh no!')); 181 | }; 182 | 183 | jobs.once('error', function(err) { 184 | t.equal(err.message, 'max retries reached'); 185 | db.once('closed', t.end.bind(t)); 186 | db.close(); 187 | }); 188 | 189 | jobs.push({ foo: 'bar' }); 190 | }); 191 | 192 | test('can delete job', function(t) { 193 | rimraf.sync(dbPath); 194 | var db = level(dbPath); 195 | var jobs = Jobs(db, worker); 196 | 197 | var processed = 0; 198 | 199 | function worker (id, payload, done) { 200 | processed += 1; 201 | t.ok(processed <= 1, 'worker is not called 2 times'); 202 | 203 | jobs.del(job2Id, function(err) { 204 | if (err) throw err; 205 | done(); 206 | }); 207 | 208 | setTimeout(function() { 209 | db.once('closed', t.end.bind(t)); 210 | db.close(); 211 | }, 500); 212 | }; 213 | 214 | var job1Id = jobs.push({ foo: 'bar', seq: 1 }); 215 | t.type(job1Id, 'number'); 216 | var job2Id = jobs.push({ foo: 'bar', seq: 2 }); 217 | t.type(job2Id, 'number'); 218 | }); 219 | 220 | test('can get runningStream & pendingStream', function(t) { 221 | rimraf.sync(dbPath); 222 | var db = level(dbPath); 223 | var jobs = Jobs(db, worker); 224 | 225 | var works = [ 226 | { foo: 'bar', seq: 1 }, 227 | { foo: 'bar', seq: 2 }, 228 | { foo: 'bar', seq: 3 } 229 | ]; 230 | 231 | var workIds = []; 232 | 233 | async.each(works, insert, doneInserting); 234 | 235 | function insert(work, done) { 236 | workIds.push(jobs.push(work, done).toString()); 237 | } 238 | 239 | function doneInserting(err) { 240 | if (err) throw err; 241 | 242 | jobs.runningStream().on('data', onData); 243 | jobs.pendingStream().on('data', onData); 244 | 245 | var seq = -1; 246 | function onData(d) { 247 | seq += 1; 248 | 249 | var id = d.key; 250 | var work = d.value; 251 | t.equal(id, workIds[seq]); 252 | var expected = works[seq]; 253 | 254 | t.deepEqual(work, expected); 255 | if (seq == works.length - 1) { 256 | process.nextTick(function() { 257 | db.once('closed', t.end.bind(t)); 258 | db.close(); 259 | }); 260 | } 261 | } 262 | } 263 | 264 | 265 | function worker (id, payload, done) { 266 | // do nothing 267 | }; 268 | }); 269 | 270 | test('doesn\'t skip past failed tasks', function(t) { 271 | rimraf.sync(dbPath); 272 | var db = level(dbPath); 273 | 274 | var max = 10; 275 | var queue = Jobs(db, worker, 1); 276 | 277 | for (var i = 1; i <= max; i++) { 278 | queue.push({ n: i }, pushed); 279 | } 280 | 281 | function pushed(err) { 282 | if (err) throw err; 283 | } 284 | 285 | var erroredOn = {}; 286 | var count = 0; 287 | var next = 1; 288 | function worker(id, work, cb) { 289 | // fail every other one 290 | if (work.n % 2 && !erroredOn[work.n]) { 291 | erroredOn[work.n] = true; 292 | cb(new Error('oops!')); 293 | } else { 294 | count++; 295 | working = true; 296 | t.equal(next++, work.n) 297 | cb(); 298 | } 299 | }; 300 | 301 | queue.on('drain', function() { 302 | if (count === max) { 303 | t.equal(queue._concurrency, 0); 304 | db.once('closed', t.end.bind(t)); 305 | process.nextTick(function() { 306 | db.close(); 307 | }); 308 | } 309 | }); 310 | }); 311 | 312 | test('continues after close and reopen', function(t) { 313 | rimraf.sync(dbPath); 314 | var db = level(dbPath); 315 | 316 | var max = 10; 317 | var restartAfter = max / 2 | 0 318 | var queue = Jobs(db, worker, 1) 319 | 320 | for (var i = 1; i <= max; i++) { 321 | queue.push({ n: i }, pushed); 322 | } 323 | 324 | function pushed(err) { 325 | if (err) throw err; 326 | } 327 | 328 | var count = 0; 329 | function worker(id, work, cb) { 330 | count++; 331 | t.equal(work.n, count); 332 | cb(); 333 | 334 | if (count === restartAfter) { 335 | db.close(function () { 336 | db = level(dbPath) 337 | queue = Jobs(db, worker, 1) 338 | queue.on('drain', function() { 339 | if (count === max) { 340 | t.equal(queue._concurrency, 0); 341 | db.once('closed', t.end.bind(t)); 342 | process.nextTick(function() { 343 | db.close(); 344 | }); 345 | } 346 | }); 347 | }) 348 | } 349 | }; 350 | }); 351 | --------------------------------------------------------------------------------