├── .gitignore ├── .npmignore ├── LICENSE ├── index.js ├── lib ├── CompoundKeySet.js ├── Qool.js ├── ReadBatch.js ├── ReadOp.js ├── SmartBatch.js └── debug.js ├── notes.md ├── package.json ├── profiling.js ├── readme.md ├── test.js └── test ├── CompoundKeySet.test.js ├── Qool.test.js ├── ReadOp.test.js └── SmartBatch.test.js /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | *.seed 3 | *.log 4 | *.csv 5 | *.dat 6 | *.out 7 | *.pid 8 | *.gz 9 | 10 | pids 11 | logs 12 | results 13 | 14 | npm-debug.log 15 | node_modules 16 | 17 | .yo-rc.json 18 | 19 | db 20 | coverage -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | db 2 | bench.md 3 | profiling.js 4 | *.log 5 | coverage -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 yaniv kessler 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const Qool = require('./lib/Qool') 2 | 3 | module.exports.create = (db, defaultLeaseTimeout) => { 4 | return new Qool(db, defaultLeaseTimeout) 5 | } 6 | 7 | module.exports.createCustom = (db, customDataSublevel, defaultLeaseTimeout) => { 8 | return new Qool(db, customDataSublevel, defaultLeaseTimeout) 9 | } 10 | 11 | module.exports.Queue = Qool 12 | -------------------------------------------------------------------------------- /lib/CompoundKeySet.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | //TODO might be better to subclass Set and override? 4 | //TODO refactoring needed 5 | class CompoundKeySet { 6 | 7 | constructor() { 8 | this._map = new Map() 9 | } 10 | 11 | add(item) { 12 | if (!Array.isArray(item)) throw new Error('must be an array') 13 | if (item.length < 2) throw new Error('array must have a minimal length of 2') 14 | 15 | let prefix = item[0] 16 | let suffix = item[1] 17 | 18 | let entry = this._map.get(prefix) 19 | 20 | if (!entry) { 21 | entry = new Set() 22 | this._map.set(prefix, entry) 23 | } 24 | 25 | entry.add(suffix) 26 | } 27 | 28 | delete(item) { 29 | 30 | let prefix = item[0] 31 | let suffix = item[1] 32 | 33 | let entry = this._map.get(prefix) 34 | 35 | // nothing to delete 36 | if (!entry) { 37 | return false 38 | } 39 | 40 | entry.delete(suffix) 41 | 42 | if (entry.size === 0) { 43 | this._map.delete(prefix) 44 | } 45 | 46 | return true 47 | } 48 | 49 | deleteRange(prefix) { 50 | return this._map.delete(prefix) 51 | } 52 | 53 | has(item) { 54 | 55 | let prefix = item[0] 56 | let suffix = item[1] 57 | 58 | let entry = this._map.get(prefix) 59 | 60 | if (!entry) return false 61 | 62 | return entry.has(suffix) 63 | } 64 | 65 | [Symbol.iterator]() { 66 | return this._map.keys() 67 | } 68 | } 69 | 70 | module.exports = CompoundKeySet -------------------------------------------------------------------------------- /lib/Qool.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('./debug')('main') 4 | const debugSilly = require('./debug')('main:silly') 5 | const SmartBatch = require('./SmartBatch') 6 | const ReadBatch = require('./ReadBatch') 7 | const ReadOp = require('./ReadOp') 8 | const LinkedList = require('digital-chain') 9 | const CompoundKeySet = require('./CompoundKeySet') 10 | 11 | class Qool { 12 | constructor(db, dataSublevel, defaultLeaseTimeout) { 13 | this._db = db 14 | this._data = db.sublevel(dataSublevel || 'data') 15 | this._counter = 0 16 | this._isLooping = false 17 | this._label = Date.now() 18 | this._opsQueue = new LinkedList() 19 | this._currentBatch = undefined 20 | this._defaultLeaseTimeout = defaultLeaseTimeout || (1000 * 10) // default lease is 10 seconds 21 | this._leases = new CompoundKeySet() 22 | 23 | this._initKeyStateGenerator() 24 | this._initLeaseTimeoutMonitor() 25 | } 26 | 27 | /** 28 | * enqueue an item 29 | * 30 | * @param {Object} data 31 | * @param {Function} cb - (err) => {} - this parameter is optional 32 | * 33 | * @returns {Array} a positional key of this item in the queue, this can be used with peek 34 | * 35 | */ 36 | enqueue(value, userCb) { 37 | debugSilly('enqueue()') 38 | let key = this.generateKey() 39 | this.enqueueWithKey(key, value, userCb) 40 | return key 41 | } 42 | 43 | /** 44 | * Same as enqueue but without auto key generation 45 | * 46 | * This method is provided for custom ordering or for 47 | * clients that need to use the default key generated with 48 | * generateKey() 49 | * 50 | * for this queue to function properly, the key must 51 | * be sortable in a way that leveldb will read those keys 52 | * in the order they were inserted. 53 | */ 54 | enqueueWithKey(key, value, userCb) { 55 | debugSilly('enqueueWithKey()') 56 | this._pushToOpsQueue({ type: 'put', key, value, userCb}) 57 | } 58 | 59 | /** 60 | * Dequeue one item from the queue 61 | * 62 | * @param {Function} cb - (err, value) => {} - this parameter is optional 63 | * 64 | */ 65 | dequeue(userCb) { 66 | debugSilly('dequeue()') 67 | this._pushToOpsQueue({ type: 'del', userCb }) 68 | } 69 | 70 | /** 71 | * Leasing an item from the queue, will temporarily dequeue it. 72 | * The item will not be visible to other operations. 73 | * It will also not be delete from the underlying database. 74 | * 75 | * The item will "reappear" after a predefined timeout or once the process that hosts 76 | * the queue restarts. 77 | * 78 | * A lease can be xxx()ed during the lifetime of the queue process, which will make 79 | * it permanent (i.e the item will be removed from the database) 80 | * 81 | */ 82 | lease(userCb) { 83 | debugSilly('lease()') 84 | this.leaseWithTimeout(this._defaultLeaseTimeout, userCb) 85 | } 86 | 87 | _leaseCallback(err, timeout, results, userCb) { 88 | debugSilly('_leaseCallback()') 89 | if (err) return userCb(err) 90 | if (!results) return userCb() 91 | if (results.length === 0) return userCb() 92 | 93 | let result = results[0] 94 | this._addLease(result.key, result.value, timeout) 95 | userCb(null, result.key, result.value) 96 | } 97 | 98 | leaseWithTimeout(timeout, userCb) { 99 | debugSilly('leaseWithTimeout()') 100 | this._pushToOpsQueue({ type: 'read', userCb: (err, results) => { 101 | this._leaseCallback(err, timeout, results, userCb) 102 | }}) 103 | } 104 | 105 | delete(key, userCb) { 106 | debugSilly('delete()') 107 | this._leases.delete(key) 108 | this._pushToOpsQueue({ type: 'del', key: key, userCb }) 109 | } 110 | 111 | peek(userCb) { 112 | debugSilly('peek()') 113 | this.peekMany(1, (err, results) => { 114 | this._peekCallback(err, results, userCb) 115 | }) 116 | } 117 | 118 | _peekCallback(err, results, userCb) { 119 | debugSilly('_peekCallback()') 120 | if (err) return userCb(err) 121 | if (!results) return userCb() 122 | if (results.length === 0) return userCb() 123 | 124 | userCb(null, results[0].value) 125 | } 126 | 127 | peekMany(length, userCb) { 128 | debugSilly('peekMany()') 129 | this._pushToOpsQueue({ type: 'peek', length, userCb }) 130 | } 131 | 132 | _addLease(key, value, timeout) { 133 | debugSilly('_addLease()') 134 | this._leases.add(key) 135 | } 136 | 137 | generateKey() { 138 | debugSilly('generateKey()') 139 | return [this._label, this._counter++] 140 | } 141 | 142 | // TODO need to refactor all these if op === something 143 | // also in smart batch 144 | _pushToOpsQueue(op) { 145 | debugSilly('_pushToOpsQueue() %o', op) 146 | let currentBatch = this._currentBatch 147 | 148 | if ((!currentBatch || currentBatch instanceof SmartBatch) 149 | && (op.type === 'read' || op.type === 'peek')) { 150 | 151 | debugSilly('_pushToOpsQueue() creating new ReadBatch') 152 | this._currentBatch = currentBatch = new ReadBatch(this._db, this._leases) 153 | this._opsQueue.unshift(currentBatch) 154 | 155 | } else if ((!currentBatch || currentBatch instanceof ReadBatch) 156 | && (op.type === 'del' || op.type === 'put') ) { 157 | debugSilly('_pushToOpsQueue() creating new SmartBatch') 158 | this._currentBatch = currentBatch = new SmartBatch(this._db, this._leases) 159 | this._opsQueue.unshift(currentBatch) 160 | } 161 | 162 | this._currentBatch.push(op) 163 | 164 | if (!this._isLooping) { 165 | this._isLooping = true 166 | setImmediate(() => { 167 | this._loop() 168 | }) 169 | } 170 | } 171 | 172 | _loop() { 173 | debug('_loop()') 174 | 175 | let currentBatch = this._opsQueue.pop() 176 | if (this._currentBatch === currentBatch) { 177 | this._currentBatch = undefined 178 | } 179 | 180 | if (currentBatch.length === 0) { 181 | debug('batch empty') 182 | return 183 | } 184 | 185 | debug('_loop() executing, batch: { type: [%s], length [%d] } opsQueue: { length:%d }', 186 | currentBatch.batchType, currentBatch.length, this._opsQueue.length) 187 | 188 | // TODO this error is also forwarded to all user callbacks from enqueue/dequeue operations 189 | // should we also emit an error event here perhaps? 190 | currentBatch.execute((err) => { 191 | debug('_loop() batch done') 192 | 193 | if (this._opsQueue.length > 0) { 194 | return setImmediate(() => { 195 | this._loop() 196 | }) 197 | } 198 | 199 | this._isLooping = false 200 | }) 201 | } 202 | 203 | // TODO improve this, see redis probablistic expiry algorithm 204 | _initLeaseTimeoutMonitor() { 205 | setInterval(() => { 206 | let expire = Date.now() + this._defaultLeaseTimeout 207 | for (let timestamp of this._leases) { 208 | if (timestamp < expire) { 209 | this._leases.deleteRange(timestamp) 210 | } 211 | } 212 | }, 1000).unref() 213 | } 214 | 215 | _initKeyStateGenerator() { 216 | setInterval(() => { 217 | this._counter = 0 218 | this._label = Date.now() 219 | }, 1000).unref() 220 | } 221 | } 222 | 223 | module.exports = Qool 224 | -------------------------------------------------------------------------------- /lib/ReadBatch.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('./debug')('ReadBatch') 4 | const debugSilly = require('./debug')('ReadBatch:silly') 5 | const ReadOp = require('./ReadOp') 6 | const CompoundKeySet = require('./CompoundKeySet') 7 | 8 | // TODO this is in fact lease and peek batch 9 | // lease is only a read operation in relation to the database 10 | // but in fact once a row is leased it should not be visible to 11 | // subsequent peeks 12 | class ReadBatch { 13 | constructor(db, leases) { 14 | this._ops = [] 15 | this._db = db 16 | this._leases = leases || new CompoundKeySet() 17 | this._totalReads = 0 18 | this._peekLength = 0 19 | this._readLength = 0 20 | } 21 | 22 | push(op) { 23 | debugSilly('push()', op) 24 | this._ops.push(op) 25 | let length = (op.length || 1) 26 | 27 | if (isPeek(op)) { 28 | this._peekLength = op.length > this._peekLength ? op.length : this._peekLength 29 | } else { 30 | this._readLength += length 31 | } 32 | } 33 | 34 | execute(cb) { 35 | debugSilly('execute()') 36 | 37 | if (this.length === 0) { 38 | return setImmediate(cb) 39 | } 40 | 41 | let limit = this._readLength + this._peekLength 42 | 43 | let readOp = new ReadOp(this._db, { limit, filter: this._filter() }) 44 | 45 | readOp.execute((err, results) => { 46 | debugSilly('execute() readop finished') 47 | this._readOpDone(err, results) 48 | setImmediate(cb) 49 | }) 50 | } 51 | 52 | get length() { 53 | return this._ops.length 54 | } 55 | 56 | get batchType() { 57 | return 'ReadBatch' 58 | } 59 | 60 | _filter() { 61 | return entry => { 62 | return this._leases.has(entry.key) 63 | } 64 | } 65 | 66 | _readOpDone(err, results) { 67 | if (err || results.length === 0) { 68 | for (let i = 0; i < this._ops.length; i++) { 69 | this._ops[i].userCb(err) 70 | } 71 | 72 | return 73 | } 74 | 75 | for (let i = 0, x = 0; i < this._ops.length; i++) { 76 | let op = this._ops[i] 77 | 78 | if (isPeek(op)) { 79 | op.userCb(null, results.slice(x, op.length)) 80 | continue 81 | } 82 | 83 | if (x > results.length) { 84 | op.userCb() 85 | continue 86 | } 87 | 88 | op.userCb(null, results.slice(x, op.length)) 89 | x += op.length 90 | } 91 | } 92 | } 93 | 94 | // TODO have to refactor the entire code to get read of these if statements all over 95 | // probably with Ops being able to operate on the batch instead of the other way around 96 | function isPeek(op) { 97 | return op.type === 'peek' 98 | } 99 | 100 | function isRead(op) { 101 | return op.type === 'read' 102 | } 103 | 104 | module.exports = ReadBatch 105 | -------------------------------------------------------------------------------- /lib/ReadOp.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('./debug')('ReadOp') 4 | const debugSilly = require('./debug')('ReadOp:silly') 5 | 6 | /** 7 | * an abstraction over db.createReadStream() 8 | * 9 | * ReadOp is a buffering operation, meaning it will hold all the results in memory 10 | * It also supports filtering during the reading of the results stream 11 | * 12 | * All the other createReadStream() options can be specified in the constructor 13 | */ 14 | class ReadOp { 15 | 16 | /** 17 | * @param {Object} db - a reference to a leveldb or sublevel 18 | * @param {Object} opts 19 | * @param {Number} opts.limit - how many rows to read 20 | * @param {Variant} opts.gt - leveldb gt options param 21 | * @param {Variant} opts.lt - leveldb gt options param 22 | * @param {Function} opts.filter - a filtering function of the form function(entry) { return true|false}, 23 | * returning true will excluse the entry from the ReadOp results 24 | * 25 | */ 26 | constructor(db, opts) { 27 | this._db = db 28 | this._opts = opts || {} 29 | this._results = [] 30 | this._filter = this._opts.filter || defaultFilter 31 | } 32 | 33 | execute(cb) { 34 | debugSilly('execute()') 35 | let error 36 | let stream = this._db.createReadStream(this._opts) 37 | let start = Date.now() 38 | 39 | stream.on('data', (entry) => { 40 | debugSilly('readStream data', entry) 41 | if (!this._filter(entry)) { 42 | this._results.push(entry) 43 | } 44 | }) 45 | 46 | stream.once('error', (err) => { 47 | error = err 48 | stream.destroy() 49 | }) 50 | 51 | stream.on('close', () => { 52 | debug('_read() stream close') 53 | 54 | if (error) { 55 | debug('_read error ', error) 56 | return cb(error) 57 | } 58 | 59 | debug('_execute() took %d', Date.now() - start) 60 | return cb(null, this._results) 61 | }) 62 | } 63 | } 64 | 65 | function noop() {} 66 | function defaultFilter() { return false } 67 | 68 | module.exports = ReadOp 69 | -------------------------------------------------------------------------------- /lib/SmartBatch.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('./debug')('SmartBatch') 4 | const debugSilly = require('./debug')('SmartBatch:silly') 5 | const ReadOp = require('./ReadOp') 6 | const CompoundKeySet = require('./CompoundKeySet') 7 | 8 | class SmartBatch { 9 | constructor(db, leases) { 10 | this._db = db 11 | this._leases = leases || new CompoundKeySet() 12 | this._deletions = new CompoundKeySet() 13 | this._batch = [] 14 | 15 | // if we have dequeues, we will store results from the 16 | // database here 17 | this._dequeueBuffer = undefined 18 | this._dequeueOpsCount = 0 19 | this._enqueueOpsCache = [] 20 | this._executeCb = undefined 21 | } 22 | 23 | push(op) { 24 | debugSilly('push()', op) 25 | 26 | this._batch.push(op) 27 | 28 | if (isEnqueue(op)) { 29 | return this._enqueueOpsCache.push(op) 30 | } 31 | 32 | if (isDelete(op)) { 33 | this._deletions.add(op.key) 34 | } 35 | 36 | if (isDequeue(op)) { 37 | return this._dequeueOpsCount++ 38 | } 39 | 40 | throw new Error('invalid operation for this batch [' + op.type + ']') 41 | } 42 | 43 | execute(cb) { 44 | debugSilly('execute()') 45 | this._executeCb = this._executeCb || cb 46 | 47 | // if we have dequeues we're gonna fetch all their data at once 48 | // _prefetchDequeue will call execute again when it's finished 49 | if (!this._dequeueBuffer && this._dequeueOpsCount > 0) { 50 | return this._prefetchDequeue() 51 | } 52 | 53 | let enqueues = this._enqueueOpsCache.length 54 | 55 | debug('execute() enqueues %d', enqueues) 56 | 57 | // database didn't contain enough data to fullfill all dequeues 58 | // so we'll merge the pending enqueues into the dequeue buffer 59 | if (this._dequeueBuffer && this._dequeueBuffer.length < this._dequeueOpsCount) { 60 | debug('execute() not enough data for to fullfill all pending dequeues') 61 | 62 | for (let i = 0; i < this._dequeueOpsCount && i < this._enqueueOpsCache.length; i++) { 63 | let enqueueOp = this._enqueueOpsCache[i] 64 | 65 | // we'll use this to filter enqueue ops from the batch 66 | enqueueOp.ignore = true 67 | 68 | // we'll use this later to determine the size of the final batch 69 | enqueues-- 70 | 71 | this._dequeueBuffer.push(enqueueOp) 72 | } 73 | } 74 | 75 | let finalBatch = [] 76 | 77 | for (let i = 0, di = 0; i < this._batch.length; i++) { 78 | let entry = this._batch[i] 79 | 80 | // take dequeue results from the buffer, but if the buffer is empty 81 | // then there is nothing more to dequeue 82 | // if/when we want waiting dequeues, here might be a good 83 | // spot to put them into a next batch or something 84 | if (isDequeue(entry) && di < this._dequeueBuffer.length) { 85 | let dequeueResult = this._dequeueBuffer[di++] 86 | //debug(entry, di, fi, dequeueResult) 87 | entry.key = dequeueResult.key 88 | entry.value = dequeueResult.value 89 | } 90 | 91 | // an ignored entry is usually an enqueue operation 92 | // that was claimed for a dequeue operation when 93 | // the database is empty 94 | if (!entry.ignore) { 95 | finalBatch.push(entry) 96 | } 97 | } 98 | 99 | debugSilly('execute() finalBatch:', finalBatch) 100 | debug('execute() actual batch size is %d', finalBatch.length) 101 | debug('execute() start batch') 102 | 103 | this._db.batch(finalBatch, (err) => { 104 | debug('execute() batch complete') 105 | this._fireCallbacks(err) 106 | setImmediate(() => { 107 | this._executeCb(err) 108 | }) 109 | }) 110 | } 111 | 112 | get length() { 113 | return this._batch.length 114 | } 115 | 116 | get batchType() { 117 | return 'SmartBatch' 118 | } 119 | 120 | /** 121 | * once we execute the SmartBatch, we're gonna get all the 122 | * data that will be dequeued in a single batch operation 123 | * 124 | */ 125 | _prefetchDequeue() { 126 | debugSilly('_prefetchDequeue()') 127 | let readOp = new ReadOp(this._db, { limit: this._dequeueOpsCount, filter: this._filter() }) 128 | readOp.execute((err, results) => { 129 | if (err) { 130 | debug('_prefetchDequeue error ', err) 131 | this._fireCallbacks(err) 132 | return setImmediate(() => { 133 | this._executeCb(err) 134 | }) 135 | } 136 | 137 | this._dequeueBuffer = results || [] 138 | 139 | this.execute() 140 | }) 141 | } 142 | 143 | _fireCallbacks(err) { 144 | for (let i = 0; i < this._batch.length; i++) { 145 | let entry = this._batch[i] 146 | 147 | if (entry.userCb) { 148 | entry.userCb(err, entry.value) 149 | } 150 | } 151 | } 152 | 153 | _filter() { 154 | return entry => { 155 | return this._leases.has(entry.key) || this._deletions.has(entry.key) 156 | } 157 | } 158 | } 159 | 160 | module.exports = SmartBatch 161 | 162 | function isSameType(a, b) { 163 | return a.constructor === b.constructor 164 | } 165 | 166 | function isDelete(op) { 167 | return op.type === 'del' && op.key 168 | } 169 | 170 | function isDequeue(op) { 171 | return op.type === 'del' 172 | } 173 | 174 | function isEnqueue(op) { 175 | return op.type === 'put' 176 | } -------------------------------------------------------------------------------- /lib/debug.js: -------------------------------------------------------------------------------- 1 | const debug = require('debug') 2 | 3 | module.exports = (name) => { 4 | return debug('qool:' + name) 5 | } -------------------------------------------------------------------------------- /notes.md: -------------------------------------------------------------------------------- 1 | # SmartBatch 2 | An evolutions of UnifiedBatch. 3 | Does not have a fixed length, size of the batch varies on the number of ops performed within a single event loop tick 4 | Every batch first tries to fullfill all dequeues from the database with one read stream. 5 | Remaining dequeues are fullfiled from memory (if there are any) 6 | 7 | # UnifiedBatch 8 | A more complex implementation but seems to be performing better when a mix of operations is performed on the queue (rather than one type) 9 | 10 | ~~In this scenario the batch size is fixed, and it includes both enqueue and dequeue operations. In order to make this work dequeue first runs on batched enqueues (which were not writting to the database yet!) before trying to dequeue from the actual database.~~ 11 | 12 | ~~e.g: ```[enq(a), enq(b), deq(), deq(), deq()]```~~ 13 | 14 | ~~The deq in index 2 will dequeue the value ```a```, the deq in index 3 will dequeue the value ```b``` and the deq in index 4 will try to fetch data from the actual database (in this case it will dequeue nothing)~~ 15 | 16 | This led to non fifo ordering, data from the databased should dequeue first before recent enqueues. 17 | 18 | ## Other optimizations 19 | 20 | ### initializing UnifiedBatch arrays to a predefined length 21 | 22 | This yielded very small but noticeable improvement - but made the code far less readable 23 | 24 | # Dequeue/Enqueue Batch 25 | In this scenario each type of operation is included in it's own batch. The batches are processed serially. e.g 3 enqueus and 2 dequeues will created two batchs. 26 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "qool", 3 | "version": "0.2.1", 4 | "description": "qool is a leveldb backed queue", 5 | "license": "MIT", 6 | "author": { 7 | "name": "yaniv kessler", 8 | "email": "yanivk@gmail.com" 9 | }, 10 | "scripts": { 11 | "test": "mocha test/**/*.js" 12 | }, 13 | "dependencies": { 14 | "bytewise": "^1.1.0", 15 | "debug": "^2.2.0", 16 | "digital-chain": "^2.0.0" 17 | }, 18 | "devDependencies": { 19 | "async": "^2.0.1", 20 | "chai": "^3.5.0", 21 | "cumulative-moving-average": "^1.0.0", 22 | "level-bytewise": "^1.0.0", 23 | "lodash": "^4.13.1", 24 | "mocha": "~2.5.3", 25 | "readable-stream": "^2.1.4", 26 | "rimraf": "^2.5.3", 27 | "stream-array": "^1.1.2" 28 | }, 29 | "keywords": [ 30 | "level", 31 | "leveldb", 32 | "queue" 33 | ], 34 | "engines": { 35 | "node": ">=5.0.0", 36 | "npm": ">=2.0.0" 37 | }, 38 | "repository": { 39 | "type": "git", 40 | "url": "https://github.com/kessler/qool" 41 | }, 42 | "bugs": { 43 | "url": "https://github.com/kessler/qool/issues" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /profiling.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const Qool = require('./lib/Qool') 4 | const level = require('level-bytewise') 5 | const rimraf = require('rimraf') 6 | const path = require('path') 7 | const async = require('async') 8 | const _ = require('lodash') 9 | 10 | let iterator = async.compose( 11 | all, 12 | (index, cb) => { 13 | console.log('starting cycle %d', index) 14 | console.log('deleting database') 15 | rimraf(path.join(__dirname, 'db'), cb) 16 | } 17 | ) 18 | 19 | const ITERATIONS = 100 20 | 21 | async.timesSeries(ITERATIONS, iterator, (err, results) => { 22 | if (err) return console.error(err) 23 | console.log('avg: %d', results.reduce((sum, member) => { return sum + member }) / ITERATIONS) 24 | }) 25 | 26 | function all(cb) { 27 | 28 | const db = level('db') 29 | const queue = new Qool(db) 30 | const SIZE = process.argv[2] || 100000 31 | const ENQUEUE_SIZE = SIZE / 4 32 | 33 | let popCount = 0 34 | 35 | function checkPopDone(err) { 36 | 37 | if (err) { 38 | console.log(err) 39 | return process.exit(1) 40 | } 41 | 42 | if (++popCount === ENQUEUE_SIZE) { 43 | console.timeEnd('populate database') 44 | test() 45 | } 46 | } 47 | 48 | console.time('populate database') 49 | 50 | for (let i = 0; i < ENQUEUE_SIZE; i++) { 51 | queue.enqueue(i + 'zz', checkPopDone) 52 | } 53 | 54 | function test() { 55 | 56 | const stats = { 57 | dequeue: 0, 58 | enqueue: 0 59 | } 60 | 61 | // i % selector to decide if we do an enqueue or dequeue 62 | const selector = 3 63 | 64 | let count = 0 65 | 66 | //console.time('test') 67 | let start = Date.now() 68 | for (let i = 0; i < SIZE; i++) { 69 | if (i % selector === 0) { 70 | enqueue(i) 71 | } else { 72 | dequeue(i) 73 | } 74 | } 75 | 76 | function enqueue(i) { 77 | queue.enqueue(i, (err) => { 78 | stats.enqueue++ 79 | 80 | handleError(err) 81 | checkDone() 82 | }) 83 | } 84 | 85 | function dequeue() { 86 | queue.dequeue((err) => { 87 | stats.dequeue++ 88 | 89 | handleError(err) 90 | checkDone() 91 | }) 92 | } 93 | 94 | function checkDone() { 95 | if (count++ === SIZE - 1) { 96 | let end = Date.now() - start 97 | console.log('took %ds', end / 1000) 98 | console.log('done') 99 | console.log(stats) 100 | db.close() 101 | cb(null, end) 102 | } 103 | } 104 | 105 | function handleError(err) { 106 | if (err) { 107 | console.error(err) 108 | process.exit(1) 109 | } 110 | } 111 | } 112 | } 113 | 114 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # qool 2 | 3 | **a leveldb backed [Queue](https://en.wikipedia.org/wiki/Queue_(abstract_data_type))** 4 | 5 | [![npm status](http://img.shields.io/npm/v/qool.svg?style=flat-square)](https://www.npmjs.org/package/qool) 6 | 7 | - durable 8 | - strict FIFO ordering 9 | - dequeus with timeouts (called lease()) 10 | - embeddable 11 | 12 | ## Example 13 | 14 | ### Simple 15 | ```javascript 16 | const Qool = require('qool') 17 | const level = require('level-bytewise') 18 | 19 | const db = level('db') 20 | const queue = Qool.create(db) 21 | 22 | queue.enqueue('a') 23 | queue.enqueue('b', (err) => {}) 24 | 25 | queue.dequeue() 26 | queue.dequeue((err, value) => {}) 27 | ``` 28 | ##### notes 29 | Enqueue and Dequeue are batched under the hood, so callbacks are invoked synchronously. This means that you can only include a last callback and forgo all the others for Enqueues and Dequeues that happen in the same tick 30 | 31 | ### Lease 32 | ```javascript 33 | const Qool = require('qool') 34 | const level = require('level-bytewise') 35 | 36 | const db = level('db') 37 | const queue = Qool.create(db, 1000 * 10) 38 | 39 | queue.enqueue('a') 40 | queue.enqueue('b', (err) => {}) 41 | 42 | queue.lease((err, leaseKey, value) => { 43 | // do something with the data 44 | 45 | // delete the item permanently from the queue 46 | queue.delete(leaseKey, (err) => { 47 | 48 | }) 49 | }) 50 | 51 | // TBD 52 | // lease with a non default timeout 53 | //queue.leaseWithTimeout(1000 * 2, (err, leaseKey, value) => {}) 54 | ``` 55 | 56 | ### Peek 57 | ```javascript 58 | const Qool = require('qool') 59 | const level = require('level-bytewise') 60 | 61 | const db = level('db') 62 | const queue = Qool.create(db) 63 | 64 | queue.enqueue('a') 65 | queue.enqueue('b', (err) => {}) 66 | 67 | queue.peek((err, value) => { 68 | // value === 'a' 69 | }) 70 | 71 | queue.peekMany((err, results) => { 72 | // results === [{ key: ..., value: 'a' }, { key: ..., value: 'b' }] 73 | }) 74 | ``` 75 | 76 | Some ramblings on internal design are [here](./notes.md) 77 | 78 | ## license 79 | 80 | ### TODO 81 | - enhance tests 82 | - implement length property 83 | - should we have a version of dequeue that "waits" if the queue is empty 84 | - expiry for enqueued items 85 | 86 | [MIT](http://opensource.org/licenses/MIT) © yaniv kessler 87 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const implementation = './index.js' 3 | console.log(implementation) 4 | const Qool = require(implementation) 5 | const expect = require('chai').expect 6 | const level = require('level-bytewise') 7 | const rimraf = require('rimraf') 8 | const memwatch = require('memwatch-next') 9 | const path = require('path') 10 | const cma = require('cumulative-moving-average') 11 | 12 | describe('Qool', () => { 13 | let db, data, size, queue 14 | 15 | describe('tests', () => { 16 | 17 | it.only('is FIFO', (done) => { 18 | queue.enqueue(1) 19 | queue.enqueue(2) 20 | queue.enqueue(3) 21 | queue.dequeue((err, value) => { 22 | if (err) return done(err) 23 | expect(value).to.equal(1) 24 | }) 25 | 26 | queue.dequeue((err, value) => { 27 | if (err) return done(err) 28 | expect(value).to.equal(2) 29 | }) 30 | 31 | queue.dequeue((err, value) => { 32 | if (err) return done(err) 33 | expect(value).to.equal(3) 34 | }) 35 | 36 | queue.enqueue(4) 37 | queue.dequeue((err, value) => { 38 | if (err) return done(err) 39 | expect(value).to.equal(4) 40 | done() 41 | }) 42 | }) 43 | 44 | it('dequeue on an empty queue', (done) => { 45 | queue.dequeue() 46 | queue.enqueue(1) 47 | queue.dequeue((err, value) => { 48 | if (err) return done(err) 49 | done() 50 | }) 51 | }) 52 | 53 | it('forwards any errors to the caller, if a callback is provided', () => { 54 | 55 | }) 56 | }) 57 | 58 | describe.skip('bench', () => { 59 | it('enqueue', function (done) { 60 | this.timeout(100000) 61 | 62 | let count = 0 63 | let avg = cma() 64 | for (let i = 0; i < size; i++) { 65 | enqueue(i) 66 | } 67 | 68 | console.log(1) 69 | 70 | function enqueue(i) { 71 | let start = Date.now() 72 | queue.enqueue(i, (err) => { 73 | avg.push(Date.now() - start) 74 | if (err) return done(err) 75 | count++ 76 | }) 77 | } 78 | 79 | function check() { 80 | if (count === size) { 81 | console.log(avg.value / 1000, avg.length) 82 | let afterCount = 0 83 | db.sublevel('data').createReadStream() 84 | .on('data', (e) => { 85 | expect(e.value).to.equal(afterCount++) 86 | }) 87 | .on('end', () => { 88 | expect(afterCount).to.equal(size) 89 | done() 90 | }) 91 | } else { 92 | setImmediate(check) 93 | } 94 | } 95 | 96 | check() 97 | }) 98 | 99 | it('dequeue', function(done) { 100 | this.timeout(2000000) 101 | let avg = cma() 102 | let count = 0 103 | 104 | for (let i = 0; i < size; i++) { 105 | data.put([Date.now(), i], i + 'xyz') 106 | } 107 | 108 | console.log(1) 109 | 110 | setTimeout(test, 5000) 111 | 112 | function dequeue() { 113 | let start = Date.now() 114 | queue.dequeue((err, item) => { 115 | avg.push(Date.now() - start) 116 | if (err) return done(err) 117 | count++ 118 | }) 119 | } 120 | 121 | function test() { 122 | let testStart = Date.now() 123 | 124 | for (let i = 0; i < size; i++) { 125 | dequeue() 126 | } 127 | 128 | function check() { 129 | if (count === size) { 130 | console.log(avg.value / 1000, avg.length) 131 | let afterCount = 0 132 | db.sublevel('data').createReadStream() 133 | .on('data', () => { 134 | afterCount++ 135 | }) 136 | .on('end', () => { 137 | console.log('test time: %d', Date.now() - testStart) 138 | expect(afterCount).to.equal(0) 139 | done() 140 | }) 141 | } else { 142 | setImmediate(check) 143 | } 144 | } 145 | 146 | check() 147 | } 148 | }) 149 | 150 | it('mixed', function (done) { 151 | this.timeout(100000) 152 | 153 | let count = 0 154 | let dequeueAvg = cma() 155 | let enqueueAvg = cma() 156 | let testStart = Date.now() 157 | 158 | for (let i = 0; i < size; i++) { 159 | if (i % 4 === 0) { 160 | enqueue(i) 161 | } else { 162 | dequeue(i) 163 | } 164 | } 165 | 166 | console.log(1) 167 | 168 | setTimeout(test, 5000) 169 | 170 | function enqueue(i) { 171 | let start = Date.now() 172 | queue.enqueue(i, (err) => { 173 | enqueueAvg.push(Date.now() - start) 174 | if (err) return done(err) 175 | count++ 176 | }) 177 | } 178 | 179 | function dequeue() { 180 | let start = Date.now() 181 | queue.dequeue((err, item) => { 182 | dequeueAvg.push(Date.now() - start) 183 | if (err) return done(err) 184 | count++ 185 | }) 186 | } 187 | 188 | function test() { 189 | function check() { 190 | if (count === size) { 191 | console.log('batch size %d', queue._batchSize) 192 | console.log('dequeue: %d', dequeueAvg.value / 1000, dequeueAvg.length) 193 | console.log('enqueue: %d', enqueueAvg.value / 1000, enqueueAvg.length) 194 | console.log('test end %d', Date.now() - testStart) 195 | done() 196 | } else { 197 | setImmediate(check) 198 | } 199 | } 200 | 201 | check() 202 | } 203 | }) 204 | }) 205 | 206 | beforeEach(() => { 207 | 208 | if (db) { 209 | db.close() 210 | } 211 | 212 | let dbPath = path.join(__dirname, 'db') 213 | 214 | rimraf.sync(dbPath) 215 | db = level(dbPath) 216 | data = db.sublevel('data') 217 | size = 100000 218 | queue = new Qool(db) 219 | }) 220 | }) 221 | -------------------------------------------------------------------------------- /test/CompoundKeySet.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const CompoundKeySet = require('../lib/CompoundKeySet') 3 | const expect = require('chai').expect 4 | 5 | describe('CompoundKeySet', () => { 6 | let set 7 | 8 | it('stores lease keys', () => { 9 | let item = [12312323, 1] 10 | expect(set.has(item)).to.be.false 11 | set.add(item) 12 | expect(set.has(item)).to.be.true 13 | }) 14 | 15 | it('can delete stored keys', () => { 16 | let item = [12312323, 1] 17 | set.add(item) 18 | set.delete(item) 19 | expect(set.has(item)).to.be.false 20 | }) 21 | 22 | it('delete operations return true if the operation actually deleted something, false otherwise', () => { 23 | let item = [12312323, 1] 24 | set.add(item) 25 | expect(set.delete(item)).to.be.true 26 | expect(set.delete(item)).to.be.false 27 | }) 28 | 29 | it('delete a whole range', () => { 30 | set.add([1, 1]) 31 | set.add([2, 1]) 32 | set.add([2, 2]) 33 | set.deleteRange(2) 34 | expect(set.has([2, 1])).to.be.false 35 | expect(set.has([2, 2])).to.be.false 36 | }) 37 | 38 | it('iterations are over the prefixes only', () => { 39 | set.add([1, 1]) 40 | set.add([2, 1]) 41 | set.add([2, 2]) 42 | let iteration = [] 43 | for (let x of set) { 44 | iteration.push(x) 45 | } 46 | 47 | expect(iteration).to.eql([1, 2]) 48 | }) 49 | 50 | beforeEach(() => { 51 | set = new CompoundKeySet() 52 | }) 53 | }) -------------------------------------------------------------------------------- /test/Qool.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const Qool = require('../lib/Qool.js') 4 | const expect = require('chai').expect 5 | const level = require('level-bytewise') 6 | const rimraf = require('rimraf') 7 | const path = require('path') 8 | const cma = require('cumulative-moving-average') 9 | const SmartBatch = require('../lib/SmartBatch') 10 | const ReadOp = require('../lib/ReadOp') 11 | 12 | describe('Qool', () => { 13 | let db, data, size, queue 14 | 15 | it('dequeues in the reverse order items were enqueued', (done) => { 16 | queue.enqueue(1) 17 | queue.enqueue(2) 18 | queue.enqueue(3) 19 | queue.dequeue((err, value) => { 20 | if (err) return done(err) 21 | expect(value).to.equal(1) 22 | }) 23 | 24 | queue.dequeue((err, value) => { 25 | if (err) return done(err) 26 | expect(value).to.equal(2) 27 | }) 28 | 29 | queue.dequeue((err, value) => { 30 | if (err) return done(err) 31 | expect(value).to.equal(3) 32 | }) 33 | 34 | queue.enqueue(4) 35 | queue.dequeue((err, value) => { 36 | if (err) return done(err) 37 | expect(value).to.equal(4) 38 | done() 39 | }) 40 | }) 41 | 42 | it('dequeue on an empty queue', (done) => { 43 | queue.dequeue() 44 | queue.enqueue(1) 45 | queue.dequeue((err, value) => { 46 | if (err) return done(err) 47 | done() 48 | }) 49 | }) 50 | 51 | it('peeking at the top of the queue', (done) => { 52 | queue.enqueue(1) 53 | queue.enqueue(2, (err) => { 54 | if (err) return done(err) 55 | queue.peekMany(2, (err, results) => { 56 | if (err) return done(err) 57 | expect(results[0].value).to.eql(1) 58 | expect(results[1].value).to.eql(2) 59 | done() 60 | }) 61 | }) 62 | }) 63 | 64 | it('lease and peek', (done) => { 65 | queue.enqueue(1) 66 | queue.enqueue(2) 67 | queue.enqueue(3) 68 | queue.enqueue(4, (err) => { 69 | if (err) return done(err) 70 | 71 | queue.peek((err, value) => { 72 | if (err) return done(err) 73 | expect(value).to.equal(1) 74 | }) 75 | 76 | queue.lease((err, key, value) => { 77 | if (err) return done(err) 78 | expect(value).to.eql(1) 79 | done() 80 | }) 81 | 82 | queue.lease((err, key, value) => { 83 | if (err) return done(err) 84 | expect(value).to.eql(2) 85 | done() 86 | }) 87 | 88 | // here we expect to get 3 and 4, because 1 and 2 89 | // were leased 90 | queue.peekMany(2, (err, results) => { 91 | if (err) return done(err) 92 | expect(results[0].value).to.equal(3) 93 | expect(results[1].value).to.equal(4) 94 | done() 95 | }) 96 | }) 97 | }) 98 | 99 | it('delete a specific item from the queue', (done) => { 100 | let key = queue.generateKey() 101 | queue.enqueueWithKey(key, 1, (err) => { 102 | if (err) return done(err) 103 | queue.delete(key, (err) => { 104 | if (err) return done(err) 105 | queue.peek((err, key, value) => { 106 | if (err) return done(err) 107 | expect(value).to.equal(undefined) 108 | done() 109 | }) 110 | }) 111 | }) 112 | }) 113 | 114 | it('leasing an item from the queue will make it invisible to other processes', (done) => { 115 | let key = queue.generateKey() 116 | queue.enqueueWithKey(key, 1, (err) => { 117 | if (err) return done(err) 118 | queue.lease((err, leaseKey, value) => { 119 | if (err) return done(err) 120 | expect(key).to.eql(key) 121 | expect(value).to.equal(1) 122 | queue.dequeue((err, value) => { 123 | if (err) return done(err) 124 | expect(value).to.equal(undefined) 125 | done() 126 | }) 127 | }) 128 | }) 129 | }) 130 | 131 | it('a lease on an item will expire', function(done) { 132 | this.timeout(4000) 133 | 134 | let key = queue.generateKey() 135 | queue.enqueueWithKey(key, 1, (err) => { 136 | if (err) return done(err) 137 | 138 | // lease for 1 second 139 | queue.leaseWithTimeout(1000, (err, leaseKey, value) => { 140 | setTimeout(() => { 141 | queue.dequeue((err, value) => { 142 | if (err) return done(err) 143 | expect(value).to.equal(1) 144 | done() 145 | }) 146 | }, 2000) 147 | }) 148 | }) 149 | }) 150 | 151 | it('a lease on an item will become permanent when calling delete()', function(done) { 152 | this.timeout(4000) 153 | 154 | let key = queue.generateKey() 155 | queue.enqueueWithKey(key, 1, (err) => { 156 | if (err) return done(err) 157 | 158 | // lease for 1 second 159 | queue.leaseWithTimeout(1000, (err, leaseKey, value) => { 160 | queue.delete(key, (err) => { 161 | setTimeout(() => { 162 | queue.dequeue((err, value) => { 163 | if (err) return done(err) 164 | expect(value).to.equal(undefined) 165 | done() 166 | }) 167 | }, 2000) 168 | }) 169 | }) 170 | }) 171 | }) 172 | 173 | it.skip('forwards any errors to the caller, if a callback is provided', () => { 174 | 175 | }) 176 | 177 | 178 | beforeEach(() => { 179 | 180 | if (db) { 181 | db.close() 182 | } 183 | 184 | let dbPath = path.join(__dirname, 'db') 185 | 186 | rimraf.sync(dbPath) 187 | db = level(dbPath) 188 | data = db.sublevel('data') 189 | size = 1000 190 | queue = new Qool(db) 191 | }) 192 | }) 193 | -------------------------------------------------------------------------------- /test/ReadOp.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const expect = require('chai').expect 4 | const ReadOp = require('../lib/ReadOp') 5 | const _ = require('lodash') 6 | 7 | const Readable = require('readable-stream').Readable 8 | 9 | describe('ReadOp', () => { 10 | let db 11 | 12 | it('Performs a read operation', (done) => { 13 | 14 | let readOp = new ReadOp(db) 15 | 16 | readOp.execute((err, results) => { 17 | if (err) return done(err) 18 | 19 | expect(results).to.have.length(4) 20 | 21 | expect(results[0]).to.eql({ key: 'a', value: 1 }) 22 | expect(results[1]).to.eql({ key: 'b', value: 2 }) 23 | expect(results[2]).to.eql({ key: 'c', value: 3 }) 24 | expect(results[3]).to.eql({ key: 'd', value: 4 }) 25 | 26 | done() 27 | }) 28 | }) 29 | 30 | it('limit results', (done) => { 31 | 32 | let readOp = new ReadOp(db, { limit: 2 }) 33 | 34 | readOp.execute((err, results) => { 35 | if (err) return done(err) 36 | 37 | expect(results).to.have.length(2) 38 | expect(results[0]).to.eql({ key: 'a', value: 1 }) 39 | expect(results[1]).to.eql({ key: 'b', value: 2 }) 40 | 41 | done() 42 | }) 43 | }) 44 | 45 | it('filter results', (done) => { 46 | let readOp = new ReadOp(db, { 47 | filter: (entry) => { 48 | if (entry.value === 3) return true 49 | } 50 | }) 51 | 52 | readOp.execute((err, results) => { 53 | if (err) return done(err) 54 | 55 | expect(results).to.have.length(3) 56 | 57 | expect(results[0]).to.eql({ key: 'a', value: 1 }) 58 | expect(results[1]).to.eql({ key: 'b', value: 2 }) 59 | expect(results[2]).to.eql({ key: 'd', value: 4 }) 60 | 61 | done() 62 | }) 63 | }) 64 | 65 | beforeEach(() => { 66 | db = new MockDb() 67 | }) 68 | }) 69 | 70 | class MockDb { 71 | constructor() { 72 | // this is not the best data structure to use 73 | // since in a real queue we can have multiple a:1 for 74 | // example 75 | this.data = { 76 | a: 1, 77 | b: 2, 78 | c: 3, 79 | d: 4 80 | } 81 | } 82 | 83 | createReadStream(opts) { 84 | return new MockStream(this.data, opts) 85 | } 86 | 87 | batch(data, cb) { 88 | 89 | if (this.error) { 90 | return setImmediate(() => { 91 | cb(this.error) 92 | }) 93 | } 94 | 95 | _.forEach(data, (entry) => { 96 | if (entry.type === 'del') { 97 | return delete this.data[entry.key] 98 | } 99 | 100 | if (entry.type === 'put') { 101 | return this.data[entry.key] = entry.value 102 | } 103 | }) 104 | 105 | setImmediate(cb) 106 | } 107 | } 108 | 109 | class MockStream extends Readable { 110 | 111 | constructor(data, opts) { 112 | super({ objectMode: true }) 113 | this._keys = Object.keys(data) 114 | this._limit = opts.limit || this._keys.length 115 | this._data = data 116 | } 117 | 118 | _read(size) { 119 | for (let i = 0; i < this._limit; i++) { 120 | let key = this._keys[i] 121 | let value = this._data[key] 122 | this.push({ key, value }) 123 | } 124 | 125 | this.push(null) 126 | setImmediate(() => { 127 | this.emit('close') 128 | }) 129 | } 130 | 131 | destroy() { 132 | this.emit('close') 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /test/SmartBatch.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const expect = require('chai').expect 4 | const SmartBatch = require('../lib/SmartBatch') 5 | 6 | const Readable = require('readable-stream').Readable 7 | const os = require('os') 8 | const _ = require('lodash') 9 | 10 | describe('SmartBatch', () => { 11 | let batch, db 12 | 13 | it('executes operations in fifo order', (done) => { 14 | let ops = [] 15 | 16 | batch.push({ 17 | type: 'del', 18 | userCb: (err, value) => { 19 | if (err) return done(err) 20 | ops.push(value) 21 | } 22 | }) 23 | 24 | batch.push({ 25 | type: 'put', 26 | key: 'e', 27 | value: 5, 28 | userCb: (err) => { 29 | if (err) return done(err) 30 | ops.push('enqueue') 31 | } 32 | }) 33 | 34 | batch.execute((err) => { 35 | if (err) return done(err) 36 | expect(ops).to.have.length(2) 37 | expect(ops[0]).to.equal(1) 38 | expect(ops[1]).to.equal('enqueue') 39 | expect(db.data).to.have.property('e', 5) 40 | done() 41 | }) 42 | }) 43 | 44 | it('fullfills dequeue requests from memory when database is empty', (done) => { 45 | 46 | batch.push({ 47 | type: 'del', 48 | userCb: (err, value) => { 49 | if (err) return done(err) 50 | expect(value).to.equal(1) 51 | } 52 | }) 53 | 54 | batch.push({ 55 | type: 'del', 56 | userCb: (err, value) => { 57 | if (err) return done(err) 58 | expect(value).to.equal(2) 59 | } 60 | }) 61 | 62 | batch.push({ 63 | type: 'del', 64 | userCb: (err, value) => { 65 | if (err) return done(err) 66 | expect(value).to.equal(3) 67 | } 68 | }) 69 | 70 | batch.push({ 71 | type: 'del', 72 | userCb: (err, value) => { 73 | if (err) return done(err) 74 | expect(value).to.equal(4) 75 | } 76 | }) 77 | 78 | batch.push({ 79 | type: 'del', 80 | userCb: (err, value) => { 81 | if (err) return done(err) 82 | expect(value).to.equal(5) 83 | } 84 | }) 85 | 86 | batch.push({ 87 | type: 'put', 88 | key: 'g', 89 | value: 5, 90 | userCb: (err) => { 91 | if (err) return done(err) 92 | } 93 | }) 94 | 95 | batch.execute((err) => { 96 | if (err) return done(err) 97 | done() 98 | }) 99 | }) 100 | 101 | it('stops if an error occurs and calls all the callbacks with the error', (done) => { 102 | db.error = new Error('test') 103 | 104 | let ops = [] 105 | batch.push({ 106 | type: 'del', 107 | userCb: (err) => { 108 | ops.push({ type: 'dequeue', error: err }) 109 | } 110 | }) 111 | 112 | batch.push({ 113 | type: 'put', 114 | key: 'e', 115 | value: 5, 116 | userCb: (err) => { 117 | ops.push({ type: 'enqueue', error: err }) 118 | } 119 | }) 120 | 121 | batch.execute((err) => { 122 | expect(err).to.equal(db.error) 123 | expect(ops).to.have.length(2) 124 | expect(ops[0]).to.have.property('type', 'dequeue') 125 | expect(ops[0].error).to.equal(db.error) 126 | 127 | expect(ops[1]).to.have.property('type', 'enqueue') 128 | expect(ops[1].error).to.equal(db.error) 129 | 130 | done() 131 | }) 132 | }) 133 | 134 | beforeEach(() => { 135 | db = new MockDb() 136 | batch = new SmartBatch(db) 137 | }) 138 | }) 139 | class MockDb { 140 | constructor() { 141 | // this is not the best data structure to use 142 | // since in a real queue we can have multiple a:1 for 143 | // example 144 | this.data = { 145 | a: 1, 146 | b: 2, 147 | c: 3, 148 | d: 4, 149 | e: 5 150 | } 151 | } 152 | 153 | createReadStream(opts) { 154 | return new MockStream(this.data, opts) 155 | } 156 | 157 | batch(data, cb) { 158 | 159 | if (this.error) { 160 | return setImmediate(() => { 161 | cb(this.error) 162 | }) 163 | } 164 | 165 | _.forEach(data, (entry) => { 166 | if (entry.type === 'del') { 167 | return delete this.data[entry.key] 168 | } 169 | 170 | if (entry.type === 'put') { 171 | return this.data[entry.key] = entry.value 172 | } 173 | }) 174 | 175 | setImmediate(cb) 176 | } 177 | } 178 | 179 | class MockStream extends Readable { 180 | 181 | constructor(data, opts) { 182 | super({ objectMode: true }) 183 | this._keys = Object.keys(data) 184 | this._limit = opts.limit || this._keys.length 185 | this._data = data 186 | } 187 | 188 | _read(size) { 189 | for (let i = 0; i < this._limit; i++) { 190 | let key = this._keys[i] 191 | let value = this._data[key] 192 | this.push({ key, value }) 193 | } 194 | 195 | this.push(null) 196 | setImmediate(() => { 197 | this.emit('close') 198 | }) 199 | } 200 | 201 | destroy() { 202 | this.emit('close') 203 | } 204 | } 205 | --------------------------------------------------------------------------------