├── index.js ├── .travis.yml ├── WORKLOG.md ├── .npmignore ├── thinkerjoblist.png ├── .gitignore ├── docker-compose.yaml ├── tests ├── test-utils.js ├── test.js ├── logger.spec.js ├── test-error.js ├── db-assert.spec.js ├── db-assert-database.spec.js ├── test-options.js ├── db-assert-table.spec.js ├── db-assert-index.spec.js ├── db-driver.spec.js ├── enums.spec.js ├── error-booster.spec.js ├── queue-summary.spec.js ├── queue-reset.spec.js ├── datetime.spec.js ├── queue-drop.spec.js ├── queue-interruption.spec.js ├── queue-state.spec.js ├── test-template.js ├── queue-get-job.spec.js ├── queue-find-job.spec.js ├── queue-find-job-by-name.spec.js ├── queue-stop.spec.js ├── queue-reanimate-job.spec.js ├── queue-change.spec.js ├── queue-cancel-job.spec.js ├── job-update.spec.js ├── queue-add-job.spec.js ├── db-result.spec.js ├── queue-remove-job.spec.js ├── job-progress.spec.js ├── job-parse.spec.js ├── is.spec.js ├── job-options.spec.js └── job-log.spec.js ├── .remarkrc ├── .bithoundrc ├── tsconfig.json ├── src ├── error-booster.js ├── logger.js ├── queue-stop.js ├── queue-state.js ├── queue-find-job.js ├── queue-get-job.js ├── queue-drop.js ├── queue-find-job-by-name.js ├── db-assert-database.js ├── queue-reset.js ├── queue-remove-job.js ├── queue-summary.js ├── db-assert.js ├── db-assert-table.js ├── db-driver.js ├── job-update.js ├── queue-add-job.js ├── queue-reanimate-job.js ├── queue-cancel-job.js ├── db-result.js ├── job-progress.js ├── job-log.js ├── queue-interruption.js ├── job-parse.js ├── queue-get-next-job.js ├── job-completed.js ├── queue-db.js ├── datetime.js ├── job-options.js ├── job-failed.js ├── db-assert-index.js ├── enums.js ├── db-review.js ├── job.js ├── is.js ├── queue-change.js ├── queue.js └── queue-process.js ├── .istanbul.yml ├── .tern-project ├── LICENSE.md ├── package.json ├── CODE_OF_CONDUCT.md └── index.d.ts /index.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./dist/queue') 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: "4.1" 3 | -------------------------------------------------------------------------------- /WORKLOG.md: -------------------------------------------------------------------------------- 1 | # WORKLOG 2 | 3 | ## Working On 4 | 5 | ## Scratch Pad 6 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | # Ignore Folders 2 | src 3 | 4 | # Ignore Files 5 | .remarkrc 6 | thinkerjoblist.png 7 | -------------------------------------------------------------------------------- /thinkerjoblist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grantcarthew/node-rethinkdb-job-queue/HEAD/thinkerjoblist.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore Folders 2 | node_modules 3 | dist 4 | 5 | # Ignore Files 6 | .idea 7 | .DS_STORE 8 | .swp 9 | npm-debug.log 10 | package-lock.json 11 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.2' 2 | services: 3 | rethinkdb: 4 | image: rethinkdb 5 | ports: 6 | - '8080:8080' 7 | - '29015:29015' 8 | - '28015:28015' 9 | -------------------------------------------------------------------------------- /tests/test-utils.js: -------------------------------------------------------------------------------- 1 | module.exports.simulateJobProcessing = function (q) { 2 | q._running = 1 3 | setTimeout(function setRunningToZero () { 4 | q._running = 0 5 | }, 500) 6 | } 7 | -------------------------------------------------------------------------------- /.remarkrc: -------------------------------------------------------------------------------- 1 | { 2 | "plugins": { 3 | "lint": { 4 | "maximum-line-length": false, 5 | "no-html": false 6 | } 7 | }, 8 | "settings": { 9 | "commonmark": true 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /.bithoundrc: -------------------------------------------------------------------------------- 1 | { 2 | "critics": { 3 | "lint": { "engine": "standard" }, 4 | "wc": { "limit": 5000 } 5 | }, 6 | "ignore": [ 7 | "**/coverage/**", 8 | "**/dist/**", 9 | "**/node_modules/**" 10 | ] 11 | } -------------------------------------------------------------------------------- /tests/test.js: -------------------------------------------------------------------------------- 1 | const Queue = require('../src/queue') 2 | const q = new Queue() 3 | 4 | let job = q.createJob() 5 | 6 | // console.log(job) 7 | // console.dir(job) 8 | 9 | q.addJob(job).then((jobs) => { 10 | return q.summary() 11 | }).then((result) => { 12 | console.dir(result) 13 | 14 | return q.stop() 15 | }) 16 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es6", 4 | "module": "commonjs", 5 | "noImplicitAny": true, 6 | "removeComments": true, 7 | "preserveConstEnums": true, 8 | "sourceMap": true, 9 | "typeRoots": [ "node_modules/@types" ] 10 | }, 11 | "include": [ 12 | "index.d.ts", 13 | "src/**/*" 14 | ], 15 | "exclude": [ 16 | "node_modules" 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /src/error-booster.js: -------------------------------------------------------------------------------- 1 | const Promise = require('bluebird') 2 | const enums = require('./enums') 3 | 4 | module.exports = function errorBooster (q, logger, name) { 5 | return function errorBoosterInternal (errObj) { 6 | errObj.queueId = q.id 7 | const message = `Event: ${name} error` 8 | logger(message, q.id, errObj) 9 | q.emit(enums.status.error, errObj) 10 | return Promise.reject(errObj) 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/logger.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const datetime = require('./datetime') 3 | const debug = require('debug') 4 | module.exports = function logger (rjqModule) { 5 | if (process.env.DEBUG) { 6 | const time = datetime.format(new Date()) 7 | const moduleName = path.basename(rjqModule.id, '.js') 8 | let prefix = `[${time}][${moduleName}]` 9 | return debug(prefix) 10 | } else { 11 | return () => {} 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/queue-stop.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const enums = require('./enums') 3 | const queueDb = require('./queue-db') 4 | 5 | module.exports = function queueStop (q) { 6 | logger('queueStop') 7 | logger(`Event: stopping [${q.id}]`) 8 | q.emit(enums.status.stopping, q.id) 9 | return q.pause().then(() => { 10 | return queueDb.detach(q) 11 | }).then(() => { 12 | logger(`Event: stopped [${q.id}]`) 13 | q.emit(enums.status.stopped, q.id) 14 | return true 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /tests/logger.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const testName = 'logger' 3 | let logger = require('../src/logger') 4 | 5 | loggerTests() 6 | function loggerTests () { 7 | test(testName, (t) => { 8 | t.plan(1) 9 | t.comment('logger test') 10 | let originalDebugValue = process.env.DEBUG 11 | process.env.DEBUG = '*' 12 | logger = logger(module) 13 | logger('Is this thing turned on?') 14 | process.env.DEBUG = originalDebugValue 15 | t.pass('test message logged in DEBUG mode') 16 | }) 17 | } 18 | -------------------------------------------------------------------------------- /.istanbul.yml: -------------------------------------------------------------------------------- 1 | verbose: false 2 | instrumentation: 3 | root: . 4 | default-excludes: true 5 | excludes: ['tests/**','dist/**','index.js'] 6 | include-all-sources: true 7 | reporting: 8 | print: summary 9 | reports: 10 | - lcov 11 | dir: ../site-rjq-coverage 12 | watermarks: 13 | statements: [50, 80] 14 | lines: [50, 80] 15 | functions: [50, 80] 16 | branches: [50, 80] 17 | check: 18 | global: 19 | statements: 76 20 | lines: 76 21 | branches: 71 22 | functions: 78 23 | excludes: [] 24 | -------------------------------------------------------------------------------- /src/queue-state.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const enums = require('./enums') 3 | 4 | module.exports = function queueState (q, newState) { 5 | logger('queueState', newState) 6 | return q.ready().then(() => { 7 | return q.r.db(q.db) 8 | .table(q.name) 9 | .insert({ 10 | id: enums.state.docId, 11 | queueId: q.id, 12 | dateChange: q.r.now(), 13 | state: newState 14 | }, { conflict: 'replace' }) 15 | .run(q.queryRunOptions) 16 | }).then((insertResult) => { 17 | logger('insertResult', insertResult) 18 | return true 19 | }) 20 | } 21 | -------------------------------------------------------------------------------- /.tern-project: -------------------------------------------------------------------------------- 1 | { 2 | "ecmaVersion": 7, 3 | "libs": [], 4 | "loadEagerly": [ 5 | "src/*.js", 6 | "tests/*.js" 7 | ], 8 | "plugins": { 9 | "complete_strings": { 10 | "maxLength": 15 11 | }, 12 | "node": { 13 | "dontLoad": "", 14 | "load": "", 15 | "modules": "" 16 | }, 17 | "node_resolve": {}, 18 | "modules": { 19 | "dontLoad": "", 20 | "load": "", 21 | "modules": "" 22 | }, 23 | "es_modules": {}, 24 | "requirejs": { 25 | "baseURL": "", 26 | "paths": "", 27 | "override": "" 28 | }, 29 | "commonjs": {} 30 | } 31 | } -------------------------------------------------------------------------------- /src/queue-find-job.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const dbResult = require('./db-result') 4 | 5 | module.exports = function queueFindJob (q, predicate, raw) { 6 | logger('queueFindJob: ', predicate) 7 | return Promise.resolve().then(() => { 8 | return q.r 9 | .db(q.db) 10 | .table(q.name) 11 | .filter(predicate) 12 | .orderBy('dateCreated') 13 | .run(q.queryRunOptions) 14 | }).then((jobsData) => { 15 | logger('jobsData', jobsData) 16 | if (raw) { return jobsData } 17 | return dbResult.toJob(q, jobsData) 18 | }) 19 | } 20 | -------------------------------------------------------------------------------- /src/queue-get-job.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const dbResult = require('./db-result') 4 | const jobParse = require('./job-parse') 5 | 6 | module.exports = function queueGetJob (q, jobOrId) { 7 | logger('queueGetJob: ', jobOrId) 8 | return Promise.resolve().then(() => { 9 | return jobParse.id(jobOrId) 10 | }).then((ids) => { 11 | return q.r 12 | .db(q.db) 13 | .table(q.name) 14 | .getAll(...ids) 15 | .run(q.queryRunOptions) 16 | }).then((jobsData) => { 17 | logger('jobsData', jobsData) 18 | return dbResult.toJob(q, jobsData) 19 | }) 20 | } 21 | -------------------------------------------------------------------------------- /src/queue-drop.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const enums = require('./enums') 4 | const queueDb = require('./queue-db') 5 | const queueStop = require('./queue-stop') 6 | 7 | module.exports = function queueDrop (q) { 8 | logger('queueDrop') 9 | return queueStop(q).then(() => { 10 | q._ready = Promise.resolve(false) 11 | return queueDb.detach(q) 12 | }).then(() => { 13 | return q.r.db(q.db) 14 | .tableDrop(q.name) 15 | .run(q.queryRunOptions) 16 | }).then(() => { 17 | logger(`Event: dropped [${q.id}]`) 18 | q.emit(enums.status.dropped, q.id) 19 | return queueDb.drain(q) 20 | }) 21 | } 22 | -------------------------------------------------------------------------------- /src/queue-find-job-by-name.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const enums = require('./enums') 4 | const dbResult = require('./db-result') 5 | 6 | module.exports = function queueFindJobByName (q, name, raw) { 7 | logger('queueFindJobByName: ', name, raw) 8 | return Promise.resolve().then(() => { 9 | return q.r 10 | .db(q.db) 11 | .table(q.name) 12 | .getAll(name, { index: enums.index.indexName }) 13 | .orderBy('dateCreated') 14 | .run(q.queryRunOptions) 15 | }).then((jobsData) => { 16 | logger('jobsData', jobsData) 17 | if (raw) { return jobsData } 18 | return dbResult.toJob(q, jobsData) 19 | }) 20 | } 21 | -------------------------------------------------------------------------------- /src/db-assert-database.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | 4 | module.exports = function assertDatabase (q) { 5 | logger('assertDatabase') 6 | return Promise.resolve().then(() => { 7 | return q.r.dbList() 8 | .contains(q.db) 9 | .do((databaseExists) => { 10 | return q.r.branch( 11 | databaseExists, 12 | { dbs_created: 0 }, 13 | q.r.dbCreate(q.db) 14 | ) 15 | }) 16 | .run(q.queryRunOptions) 17 | }).then((dbCreateResult) => { 18 | dbCreateResult.dbs_created > 0 19 | ? logger('Database created: ' + q.db) 20 | : logger('Database exists: ' + q.db) 21 | return true 22 | }) 23 | } 24 | -------------------------------------------------------------------------------- /src/queue-reset.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const enums = require('./enums') 4 | const dbResult = require('./db-result') 5 | 6 | module.exports = function queueReset (q) { 7 | logger('reset') 8 | return Promise.resolve().then(() => { 9 | return q.r.db(q.db) 10 | .table(q.name) 11 | .delete() 12 | .run(q.queryRunOptions) 13 | }).then((resetResult) => { 14 | logger('resetResult', resetResult) 15 | return dbResult.status(resetResult, enums.dbResult.deleted) 16 | }).then((totalRemoved) => { 17 | logger(`Event: reset [${totalRemoved}]`) 18 | q.emit(enums.status.reset, q.id, totalRemoved) 19 | return totalRemoved 20 | }) 21 | } 22 | -------------------------------------------------------------------------------- /tests/test-error.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | module.exports = function (err, callingModule, t) { 3 | const moduleName = path.basename(callingModule.id, '.js') 4 | let errorTitle 5 | let errorMessage 6 | if (!err) { 7 | errorTitle = errorMessage = `Error missing: ${moduleName}` 8 | } else if (typeof err === 'string') { 9 | errorTitle = errorMessage = `${err}: ${moduleName}` 10 | } else { 11 | errorTitle = err.message 12 | errorMessage = ` 13 | Module: ${moduleName} 14 | Name: ${err.name} 15 | Message: ${err.message} 16 | ${err.stack}\n 17 | ` 18 | } 19 | 20 | console.error(errorMessage) 21 | console.error(err) 22 | t.fail(errorTitle) 23 | return errorMessage 24 | } 25 | -------------------------------------------------------------------------------- /src/queue-remove-job.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const enums = require('./enums') 4 | const jobParse = require('./job-parse') 5 | 6 | module.exports = function removeJob (q, jobOrId) { 7 | logger('removeJob: ' + jobOrId) 8 | 9 | return Promise.resolve().then(() => { 10 | return jobParse.id(jobOrId) 11 | }).then((jobIds) => { 12 | return Promise.props({ 13 | jobIds, 14 | removeResult: q.r.db(q.db) 15 | .table(q.name) 16 | .getAll(...jobIds) 17 | .delete() 18 | .run(q.queryRunOptions) 19 | }) 20 | }).then((result) => { 21 | for (let id of result.jobIds) { 22 | logger(`Event: removed`, q.id, id) 23 | q.emit(enums.status.removed, q.id, id) 24 | } 25 | return result.jobIds 26 | }) 27 | } 28 | -------------------------------------------------------------------------------- /src/queue-summary.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | 4 | module.exports = function summary (q) { 5 | logger('summary') 6 | return Promise.resolve().then(() => { 7 | return q.r.db(q.db) 8 | .table(q.name) 9 | .group({index: 'status'}).count() 10 | }).then((reduction) => { 11 | const summary = { 12 | waiting: 0, 13 | active: 0, 14 | completed: 0, 15 | cancelled: 0, 16 | failed: 0, 17 | terminated: 0 18 | } 19 | for (let stat of reduction) { 20 | if (stat.group) { summary[stat.group] = stat.reduction } 21 | } 22 | summary.total = Object.keys(summary).reduce((runningTotal, key) => { 23 | return runningTotal + summary[key] 24 | }, 0) 25 | logger('summary', summary) 26 | return summary 27 | }) 28 | } 29 | -------------------------------------------------------------------------------- /tests/db-assert.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const tError = require('./test-error') 4 | const dbAssert = require('../src/db-assert') 5 | const dbDriver = require('../src/db-driver') 6 | const tOpts = require('./test-options') 7 | 8 | dbAssertTests() 9 | function dbAssertTests () { 10 | const q = { 11 | r: dbDriver(tOpts.cxn()), 12 | db: tOpts.dbName, 13 | name: 'dbAssert', 14 | id: 'mock:queue:id' 15 | } 16 | 17 | return new Promise((resolve, reject) => { 18 | test('db-assert', (t) => { 19 | t.plan(1) 20 | 21 | return dbAssert(q).then((dbResult) => { 22 | t.ok(dbResult, 'All database resources asserted') 23 | q.r.getPoolMaster().drain() 24 | return resolve(t.end()) 25 | }).catch(err => tError(err, module, t)) 26 | }) 27 | }) 28 | } 29 | -------------------------------------------------------------------------------- /tests/db-assert-database.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const tError = require('./test-error') 4 | const dbAssertDatabase = require('../src/db-assert-database') 5 | const dbDriver = require('../src/db-driver') 6 | const tOpts = require('./test-options') 7 | 8 | dbAssertDatabaseTests() 9 | function dbAssertDatabaseTests () { 10 | const q = { 11 | r: dbDriver(tOpts.cxn()), 12 | db: tOpts.dbName, 13 | name: 'dbAssertDatabase', 14 | id: 'mock:queue:id' 15 | } 16 | 17 | return new Promise((resolve, reject) => { 18 | test('db-assert-database', (t) => { 19 | t.plan(1) 20 | 21 | return dbAssertDatabase(q).then((assertDbResult) => { 22 | t.ok(assertDbResult, 'Database asserted') 23 | q.r.getPoolMaster().drain() 24 | return resolve(t.end()) 25 | }).catch(err => tError(err, module, t)) 26 | }) 27 | }) 28 | } 29 | -------------------------------------------------------------------------------- /src/db-assert.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const dbAssertDatabase = require('./db-assert-database') 4 | const dbAssertTable = require('./db-assert-table') 5 | const dbAssertIndex = require('./db-assert-index') 6 | 7 | module.exports = function dbAssert (q) { 8 | logger('dbAssert') 9 | 10 | // The delay algorithm below is to prevent multiple Queue objects 11 | // attempting to create the database/table/indexes at the same time. 12 | // Before the delay was introduced it was possible to end up with two 13 | // databases in RethinkDB with the same name. 14 | let randomDelay = Math.floor(Math.random() * 1000) 15 | if (!q.master) { randomDelay += q._databaseInitDelay } 16 | 17 | return Promise.delay(randomDelay).then(() => { 18 | return dbAssertDatabase(q) 19 | }).then(() => { 20 | return dbAssertTable(q) 21 | }).then(() => { 22 | return dbAssertIndex(q) 23 | }) 24 | } 25 | -------------------------------------------------------------------------------- /src/db-assert-table.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | 4 | module.exports = function assertTable (q) { 5 | logger('assertTable') 6 | return Promise.resolve().then(() => { 7 | return q.r.db(q.db) 8 | .tableList() 9 | .contains(q.name) 10 | .do((tableExists) => { 11 | return q.r.branch( 12 | tableExists, 13 | { tables_created: 0 }, 14 | q.r.db(q.db) 15 | .tableCreate(q.name) 16 | ) 17 | }) 18 | .run(q.queryRunOptions) 19 | }).then((tableCreateResult) => { 20 | tableCreateResult.tables_created > 0 21 | ? logger('Table created: ' + q.name) 22 | : logger('Table exists: ' + q.name) 23 | }).then(() => { 24 | return q.r.db(q.db) 25 | .table(q.name) 26 | .wait() 27 | .run(q.queryRunOptions) 28 | }).then(() => { 29 | logger('Table ready.') 30 | return true 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /tests/test-options.js: -------------------------------------------------------------------------------- 1 | const dbHost = module.exports.dbHost = 'localhost' 2 | const dbPort = module.exports.dbPort = 28015 3 | const dbName = module.exports.dbName = 'rjqJobQueueTests' 4 | 5 | module.exports.tData = 'The quick brown fox jumped over the lazy dog' 6 | module.exports.lData = { one_key: 'The quick brown fox jumped over the lazy dog', some_other_key: 0.2 } 7 | 8 | module.exports.cxn = function () { 9 | return { 10 | host: dbHost, 11 | port: dbPort, 12 | db: dbName 13 | } 14 | } 15 | module.exports.default = function (queueName) { 16 | return { 17 | name: queueName, 18 | concurrency: 3, 19 | masterInterval: false 20 | } 21 | } 22 | module.exports.master = function (queueName, interval = 5000) { 23 | return { 24 | name: queueName, 25 | concurrency: 3, 26 | masterInterval: interval 27 | } 28 | } 29 | module.exports.queueNameOnly = function (queueName) { 30 | return { 31 | name: queueName 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/db-driver.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const rethinkdbdash = require('rethinkdbdash') 3 | const is = require('./is') 4 | const enums = require('./enums') 5 | 6 | module.exports = function dbDriver (cxn) { 7 | logger('dbDriver', cxn) 8 | cxn = cxn !== undefined ? cxn : {} 9 | const cxnCopy = Object.assign({}, cxn) 10 | 11 | if (Object.keys(cxn).length < 1 || 12 | cxn.host != null || 13 | cxn.port != null || 14 | is.string(cxn.db)) { 15 | logger('cxn is an options object') 16 | cxnCopy.silent = true 17 | cxnCopy.host = cxnCopy.host == null 18 | ? enums.options.host : cxnCopy.host 19 | cxnCopy.port = cxnCopy.port == null 20 | ? enums.options.port : cxnCopy.port 21 | cxnCopy.db = cxnCopy.db == null 22 | ? enums.options.db : cxnCopy.db 23 | return rethinkdbdash(cxnCopy) 24 | } 25 | 26 | if (cxn.getPoolMaster) { 27 | logger('cxn is a rethinkdbdash object') 28 | return cxn 29 | } 30 | 31 | throw new Error('Database driver or options invalid') 32 | } 33 | -------------------------------------------------------------------------------- /tests/db-assert-table.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const tError = require('./test-error') 4 | const dbAssertDatabase = require('../src/db-assert-database') 5 | const dbAssertTable = require('../src/db-assert-table') 6 | const dbDriver = require('../src/db-driver') 7 | const tOpts = require('./test-options') 8 | 9 | dbAssertTableTests() 10 | function dbAssertTableTests () { 11 | const q = { 12 | r: dbDriver(tOpts.cxn()), 13 | db: tOpts.dbName, 14 | name: 'dbAssertTable', 15 | id: 'mock:queue:id' 16 | } 17 | 18 | return new Promise((resolve, reject) => { 19 | test('db-assert-table', (t) => { 20 | t.plan(2) 21 | 22 | return dbAssertDatabase(q).then((assertDbResult) => { 23 | t.ok(assertDbResult, 'Database asserted') 24 | return dbAssertTable(q) 25 | }).then((assertDbTable) => { 26 | t.ok(assertDbTable, 'Table asserted') 27 | q.r.getPoolMaster().drain() 28 | return resolve(t.end()) 29 | }).catch(err => tError(err, module, t)) 30 | }) 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Grant Carthew 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/job-update.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const enums = require('./enums') 4 | const queueGetJob = require('./queue-get-job') 5 | const jobLog = require('./job-log') 6 | 7 | module.exports = function jobUpdate (job) { 8 | logger(`jobUpdate: [${job.id}]`) 9 | 10 | return Promise.resolve().then(() => { 11 | return queueGetJob(job.q, job.id) 12 | }).then((oldJobs) => { 13 | let oldJobCopy = oldJobs[0].getCleanCopy() 14 | delete oldJobCopy.log 15 | let log = jobLog.createLogObject(job, 16 | oldJobCopy, 17 | enums.message.jobUpdated, 18 | enums.log.information) 19 | job.log.push(log) 20 | return job.getCleanCopy() 21 | }).then((cleanJob) => { 22 | return job.q.r.db(job.q.db) 23 | .table(job.q.name) 24 | .get(job.id) 25 | .update( 26 | cleanJob, 27 | {returnChanges: false} 28 | ) 29 | .run(job.q.queryRunOptions) 30 | }).then((updateResult) => { 31 | logger(`updateResult`, updateResult) 32 | logger(`Event: updated`, job.q.id, job.id) 33 | job.q.emit(enums.status.updated, job.q.id, job.id) 34 | return job 35 | }) 36 | } 37 | -------------------------------------------------------------------------------- /tests/db-assert-index.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const tError = require('./test-error') 4 | const dbAssertDatabase = require('../src/db-assert-database') 5 | const dbAssertTable = require('../src/db-assert-table') 6 | const dbAssertIndex = require('../src/db-assert-index') 7 | const dbDriver = require('../src/db-driver') 8 | const tOpts = require('./test-options') 9 | 10 | dbAssertIndexTests() 11 | function dbAssertIndexTests () { 12 | const q = { 13 | r: dbDriver(tOpts.cxn()), 14 | db: tOpts.dbName, 15 | name: 'dbAssertIndex', 16 | id: 'mock:queue:id' 17 | } 18 | 19 | return new Promise((resolve, reject) => { 20 | test('db-assert-index', (t) => { 21 | t.plan(3) 22 | 23 | return dbAssertDatabase(q).then((assertDbResult) => { 24 | t.ok(assertDbResult, 'Database asserted') 25 | return dbAssertTable(q) 26 | }).then((assertDbTable) => { 27 | t.ok(assertDbTable, 'Table asserted') 28 | return dbAssertIndex(q) 29 | }).then((assertIndexResult) => { 30 | t.ok(assertIndexResult, 'Indexes asserted') 31 | q.r.getPoolMaster().drain() 32 | return resolve(t.end()) 33 | }).catch(err => tError(err, module, t)) 34 | }) 35 | }) 36 | } 37 | -------------------------------------------------------------------------------- /src/queue-add-job.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const enums = require('./enums') 4 | const queueProcess = require('./queue-process') 5 | const dbResult = require('./db-result') 6 | const jobLog = require('./job-log') 7 | const jobParse = require('./job-parse') 8 | 9 | module.exports = function queueAddJob (q, job) { 10 | logger('addJob', job) 11 | return Promise.resolve().then(() => { 12 | return jobParse.job(job) 13 | }).map((oneJob) => { 14 | if (oneJob.status === enums.status.created) { 15 | oneJob.status = enums.status.waiting 16 | } 17 | const log = jobLog.createLogObject(oneJob, 18 | null, 19 | enums.message.jobAdded, 20 | enums.log.information, 21 | enums.status.waiting) 22 | oneJob.log.push(log) 23 | return oneJob.getCleanCopy() 24 | }).then((cleanJobs) => { 25 | logger(`cleanJobs`, cleanJobs) 26 | return q.r.db(q.db) 27 | .table(q.name) 28 | .insert(cleanJobs, {returnChanges: true}) 29 | .run(q.queryRunOptions) 30 | }).then((saveResult) => { 31 | logger(`saveResult`, saveResult) 32 | queueProcess.restart(q) 33 | return dbResult.toJob(q, saveResult) 34 | }).then((savedJobs) => { 35 | for (let savedjob of savedJobs) { 36 | logger(`Event: added [${savedjob.id}]`) 37 | q.emit(enums.status.added, q.id, savedjob.id) 38 | } 39 | return savedJobs 40 | }) 41 | } 42 | -------------------------------------------------------------------------------- /src/queue-reanimate-job.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const enums = require('./enums') 4 | const dbResult = require('./db-result') 5 | const jobParse = require('./job-parse') 6 | const jobLog = require('./job-log') 7 | 8 | module.exports = function queueReanimateJob (q, 9 | jobOrId, 10 | dateEnable = new Date()) { 11 | logger('queueGetJob: ', jobOrId) 12 | return Promise.resolve().then(() => { 13 | return jobParse.id(jobOrId) 14 | }).then((ids) => { 15 | let log = jobLog.createLogObject( 16 | { q, retryCount: 0 }, 17 | null, 18 | enums.message.jobReanimated, 19 | enums.log.information, 20 | enums.status.waiting 21 | ) 22 | return q.r 23 | .db(q.db) 24 | .table(q.name) 25 | .getAll(...ids) 26 | .update({ 27 | dateEnable, 28 | log: q.r.row('log').append(log), 29 | progress: 0, 30 | queueId: q.id, 31 | retryCount: 0, 32 | status: enums.status.waiting 33 | }, {returnChanges: true}) 34 | .run(q.queryRunOptions) 35 | }).then((jobsResult) => { 36 | logger('jobsResult', jobsResult) 37 | return dbResult.toIds(jobsResult) 38 | }).then((reanimatedJobIds) => { 39 | for (let reanimatedJobId of reanimatedJobIds) { 40 | logger(`Event: reanimated`, q.id, reanimatedJobId) 41 | q.emit(enums.status.reanimated, q.id, reanimatedJobId) 42 | } 43 | return reanimatedJobIds 44 | }) 45 | } 46 | -------------------------------------------------------------------------------- /tests/db-driver.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const tError = require('./test-error') 3 | const tOpts = require('./test-options') 4 | const dbDriver = require('../src/db-driver') 5 | const rethinkdbdash = require('rethinkdbdash') 6 | 7 | dbDriverTests() 8 | function dbDriverTests () { 9 | test('db-driver', (t) => { 10 | t.plan(7) 11 | 12 | function testConnOptions (testOpt) { 13 | const driver = dbDriver(testOpt) 14 | testOpt = testOpt || {} 15 | t.ok(driver.getPoolMaster(), `DB driver option [${Object.keys(testOpt)}] returns rethinkdbdash`) 16 | driver.getPoolMaster().drain() 17 | } 18 | 19 | try { 20 | const options = { 21 | hostOnly: { host: tOpts.dbHost }, 22 | portOnly: { port: tOpts.dbPort }, 23 | dbOnly: { db: tOpts.dbName }, 24 | full: tOpts.cxn() 25 | } 26 | options.full.silent = true 27 | 28 | testConnOptions() 29 | testConnOptions(options.hostObnly) 30 | testConnOptions(options.portOnly) 31 | testConnOptions(options.dbOnly) 32 | testConnOptions(tOpts.cxn()) 33 | 34 | const dash = rethinkdbdash(options.full) 35 | const dashResult = dbDriver(dash) 36 | t.ok(dash === dashResult, 'DB driver rethinkdbdash returns rethinkdbdash') 37 | dash.getPoolMaster().drain() 38 | 39 | t.throws(() => { dbDriver({foo: 'bar'}) }, 'Invalid db driver options throws an error') 40 | } catch (err) { 41 | tError(err, module, t) 42 | } 43 | }) 44 | } 45 | -------------------------------------------------------------------------------- /tests/enums.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const tError = require('./test-error') 3 | const enums = require('../src/enums') 4 | 5 | enumsTest() 6 | function enumsTest () { 7 | test('enums', (t) => { 8 | t.plan(13) 9 | 10 | try { 11 | t.equal(enums.priorityFromValue(60), 'lowest', 'Priority from value 60 returns lowest') 12 | t.equal(enums.priorityFromValue(50), 'low', 'Priority from value 50 returns low') 13 | t.equal(enums.priorityFromValue(40), 'normal', 'Priority from value 40 returns normal') 14 | t.equal(enums.priorityFromValue(30), 'medium', 'Priority from value 30 returns medium') 15 | t.equal(enums.priorityFromValue(20), 'high', 'Priority from value 20 returns high') 16 | t.equal(enums.priorityFromValue(10), 'highest', 'Priority from value 10 returns highest') 17 | t.equal(Object.keys(enums.state).length, 3, 'Enums state has the correct number of keys') 18 | t.equal(Object.keys(enums.priority).length, 6, 'Enums priority has correct number of keys') 19 | t.equal(Object.keys(enums.status).length, 26, 'Enums status has correct number of keys') 20 | t.equal(Object.keys(enums.options).length, 16, 'Enums options has correct number of keys') 21 | t.equal(Object.keys(enums.index).length, 5, 'Enums index has correct number of keys') 22 | t.equal(Object.keys(enums.log).length, 3, 'Enums log has correct number of keys') 23 | t.equal(Object.keys(enums.message).length, 31, 'Enums message has correct number of keys') 24 | } catch (err) { 25 | tError(err, module, t) 26 | } 27 | }) 28 | } 29 | -------------------------------------------------------------------------------- /tests/error-booster.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const errorBooster = require('../src/error-booster') 4 | const EventEmitter = require('events') 5 | const is = require('../src/is') 6 | 7 | errorBoosterTests() 8 | function errorBoosterTests () { 9 | return new Promise((resolve, reject) => { 10 | test('error-booster', (t) => { 11 | t.plan(7) 12 | 13 | const testName = 'function name' 14 | const mockQueue = new EventEmitter() 15 | mockQueue.id = 'mock queue id' 16 | const mockErrorMessage = 'mock error message' 17 | const mockError = new Error(mockErrorMessage) 18 | 19 | function mockLogger (message, queueId, errObj) { 20 | t.ok(message.includes(testName), 'Logger message contains name') 21 | t.ok(is.string(queueId), 'queueId is a string') 22 | t.ok(is.error(errObj), 'error object is valid') 23 | } 24 | 25 | function mockHandler (errObj) { 26 | t.pass('Error event emitted') 27 | t.ok(is.error(errObj), 'Error event object is valid') 28 | t.equal(errObj.queueId, mockQueue.id, 'Error object has queueId property') 29 | } 30 | 31 | const errorBoosterInternal = errorBooster(mockQueue, mockLogger, testName) 32 | mockQueue.on('error', mockHandler) 33 | return errorBoosterInternal(mockError).then(() => { 34 | t.fail('This should never be reached!') 35 | }).catch(err => { 36 | t.ok(err === mockError, 'Error object is passed to the catch') 37 | }).then(() => { 38 | return resolve(t.end()) 39 | }) 40 | }) 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /src/queue-cancel-job.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const is = require('./is') 4 | const enums = require('./enums') 5 | const dbResult = require('./db-result') 6 | const jobParse = require('./job-parse') 7 | 8 | module.exports = function cancelJob (q, jobOrId, reason) { 9 | logger('cancelJob', jobOrId, reason) 10 | 11 | return Promise.resolve().then(() => { 12 | return jobParse.id(jobOrId) 13 | }).then((ids) => { 14 | return q.r.db(q.db) 15 | .table(q.name) 16 | .getAll(...ids) 17 | .update({ 18 | status: enums.status.cancelled, 19 | dateFinished: new Date(), 20 | log: q.r.row('log').append({ 21 | date: new Date(), 22 | queueId: q.id, 23 | type: enums.log.information, 24 | status: enums.status.cancelled, 25 | retryCount: q.r.row('retryCount'), 26 | processCount: q.r.row('processCount'), 27 | message: reason 28 | }), 29 | queueId: q.id 30 | }, {returnChanges: true}) 31 | .run(q.queryRunOptions) 32 | }).then((updateResult) => { 33 | logger('updateResult', updateResult) 34 | return dbResult.toIds(updateResult) 35 | }).then((jobIds) => { 36 | jobIds.forEach((jobId) => { 37 | logger(`Event: cancelled`, q.id, jobId) 38 | q.emit(enums.status.cancelled, q.id, jobId) 39 | }) 40 | if (is.true(q.removeFinishedJobs)) { 41 | return q.removeJob(jobIds).then((deleteResult) => { 42 | logger(`Removed finished jobs on cancel [${deleteResult}]`) 43 | return jobIds 44 | }) 45 | } else { 46 | return jobIds 47 | } 48 | }) 49 | } 50 | -------------------------------------------------------------------------------- /src/db-result.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const is = require('./is') 4 | const enums = require('./enums') 5 | 6 | function getResultError (dbResult) { 7 | logger(`getResultError`, dbResult) 8 | const err = new Error(enums.message.dbError) 9 | err.dbError = dbResult 10 | return Promise.reject(err) 11 | } 12 | 13 | function getJobsData (dbResult) { 14 | logger('getJobsData:', dbResult) 15 | return Promise.resolve().then(() => { 16 | if (!dbResult) { return [] } 17 | if (dbResult.errors > 0) { 18 | return getResultError(dbResult) 19 | } 20 | if (is.array(dbResult)) { 21 | return dbResult 22 | } 23 | if (is.array(dbResult.changes)) { 24 | return dbResult.changes.map((change) => { 25 | return change.new_val 26 | }) 27 | } 28 | if (dbResult.new_val) { 29 | return [dbResult.new_val] 30 | } 31 | if (dbResult.id) { 32 | return [dbResult] 33 | } 34 | return [] 35 | }) 36 | } 37 | 38 | module.exports.toJob = function toJob (q, dbResult) { 39 | logger('toJob:', dbResult) 40 | return getJobsData(dbResult).then((jobsData) => { 41 | return jobsData.map((jobData) => { 42 | return q.createJob(jobData) 43 | }) 44 | }) 45 | } 46 | 47 | module.exports.toIds = function toIds (dbResult) { 48 | logger('toIds', dbResult) 49 | return getJobsData(dbResult).then((jobsData) => { 50 | return jobsData.map((jobData) => { 51 | return jobData.id 52 | }) 53 | }) 54 | } 55 | 56 | module.exports.status = function status (dbResult, prop) { 57 | logger('status:', dbResult, prop) 58 | if (dbResult.errors > 0) { return getResultError(dbResult) } 59 | if (!dbResult[prop]) { dbResult[prop] = 0 } 60 | return Promise.resolve(dbResult[prop]) 61 | } 62 | -------------------------------------------------------------------------------- /src/job-progress.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const is = require('./is') 4 | const enums = require('./enums') 5 | const jobLog = require('./job-log') 6 | const dbResult = require('./db-result') 7 | 8 | module.exports = function jobProgress (job, percent) { 9 | logger('jobProgress: ' + job.id) 10 | if (!is.active(job)) { 11 | logger(`Error: progress called on non-active job`, job) 12 | return Promise.reject(new Error(enums.message.jobNotActive)) 13 | } 14 | if (!percent || !is.number(percent) || percent < 0) { percent = 0 } 15 | if (percent > 100) { percent = 100 } 16 | 17 | return Promise.resolve().then(() => { 18 | return job.q.r.db(job.q.db) 19 | .table(job.q.name) 20 | .get(job.id) 21 | .pluck('progress') 22 | .run(job.q.queryRunOptions) 23 | }).then((pluck) => { 24 | return jobLog.createLogObject(job, 25 | pluck.progress, 26 | enums.message.jobProgress, 27 | enums.log.information) 28 | }).then((newLog) => { 29 | return job.q.r.db(job.q.db) 30 | .table(job.q.name) 31 | .get(job.id) 32 | .update({ 33 | queueId: job.q.id, 34 | progress: percent, 35 | dateEnable: job.q.r.now() 36 | .add( 37 | job.q.r.row('timeout').div(1000) 38 | ) 39 | .add( 40 | job.q.r.row('retryDelay').div(1000).mul(job.q.r.row('retryCount') 41 | )), 42 | log: job.q.r.row('log').append(newLog) 43 | }, { returnChanges: true }) 44 | .run(job.q.queryRunOptions) 45 | }).then((updateResult) => { 46 | return dbResult.toJob(job.q, updateResult) 47 | }).then((updateResult) => { 48 | logger(`Event: progress`, job.q.id, job.id, percent) 49 | job.q.emit(enums.status.progress, job.q.id, job.id, percent) 50 | return updateResult[0] 51 | }) 52 | } 53 | -------------------------------------------------------------------------------- /src/job-log.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const enums = require('./enums') 4 | 5 | module.exports.createLogObject = createLogObject 6 | module.exports.commitLog = commitLog 7 | module.exports.getLastLog = getLastLog 8 | 9 | function createLogObject (job, 10 | data = {}, 11 | message = enums.message.seeLogData, 12 | type = enums.log.information, 13 | status = job.status) { 14 | logger('createLogObject', data, message, type, status) 15 | return { 16 | date: new Date(), 17 | queueId: job.q.id, 18 | message, 19 | data, 20 | type, 21 | status, 22 | retryCount: job.retryCount, 23 | processCount: job.processCount 24 | } 25 | } 26 | 27 | function commitLog (job, 28 | data = {}, 29 | message = enums.message.seeLogData, 30 | type = enums.log.information, 31 | status = job.status) { 32 | logger('commitLog', data, message, type, status) 33 | 34 | const newLog = createLogObject(job, data, message, type, status) 35 | 36 | if (job.status === enums.status.created) { 37 | return Promise.reject(new Error(enums.message.jobNotAdded)) 38 | } 39 | return Promise.resolve().then(() => { 40 | return job.q.r.db(job.q.db) 41 | .table(job.q.name) 42 | .get(job.id) 43 | .update({ 44 | log: job.q.r.row('log').append(newLog), 45 | queueId: job.q.id 46 | }) 47 | }).then((updateResult) => { 48 | job.log.push(newLog) 49 | job.log.sort(compareTime) 50 | logger(`Event: log`, job.q.id, job.id) 51 | job.q.emit(enums.status.log, job.q.id, job.id) 52 | return true 53 | }) 54 | } 55 | 56 | function getLastLog (job) { 57 | job.log.sort(compareTime) 58 | return job.log.slice(-1)[0] 59 | } 60 | 61 | function compareTime (a, b) { 62 | return a.date.getTime() >= b.date.getTime() ? 1 : -1 63 | } 64 | -------------------------------------------------------------------------------- /src/queue-interruption.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const enums = require('./enums') 4 | const is = require('./is') 5 | const queueProcess = require('./queue-process') 6 | const queueState = require('./queue-state') 7 | 8 | module.exports.pause = function interruptionPause (q, source) { 9 | logger(`pause`, source) 10 | q._paused = true 11 | const makeGlobal = is.true(source) 12 | const eventGlobal = makeGlobal || source === enums.state.global 13 | return q.ready().then(() => { 14 | if (makeGlobal) { 15 | return queueState(q, enums.status.paused) 16 | } 17 | }).then(() => { 18 | return new Promise((resolve, reject) => { 19 | logger(`Event: pausing`, q.id, eventGlobal) 20 | q.emit(enums.status.pausing, q.id, eventGlobal) 21 | if (q.running < 1) { return resolve() } 22 | let intId = setInterval(function pausing () { 23 | logger(`Pausing, waiting on running jobs: [${q.running}]`) 24 | if (q.running < 1) { 25 | clearInterval(intId) 26 | resolve() 27 | } 28 | }, 400) 29 | }) 30 | }).then(() => { 31 | logger(`Event: paused`, q.id, eventGlobal) 32 | q.emit(enums.status.paused, q.id, eventGlobal) 33 | return true 34 | }) 35 | } 36 | 37 | module.exports.resume = function interruptionResume (q, source) { 38 | logger(`resume`, source) 39 | q._paused = false 40 | const makeGlobal = is.true(source) 41 | const eventGlobal = makeGlobal || source === enums.state.global 42 | return q.ready().then(() => { 43 | if (makeGlobal) { 44 | return queueState(q, enums.status.active) 45 | } 46 | }).then(() => { 47 | queueProcess.restart(q) 48 | logger(`Event: resumed`, q.id, eventGlobal) 49 | q.emit(enums.status.resumed, q.id, eventGlobal) 50 | return true 51 | }) 52 | } 53 | -------------------------------------------------------------------------------- /src/job-parse.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const is = require('./is') 3 | const enums = require('./enums') 4 | 5 | module.exports.id = function jobParseId (job) { 6 | logger('jobParseId', job) 7 | if (!job) { return [] } 8 | let jobs = is.array(job) ? job : [job] 9 | let validIds = [] 10 | for (let j of jobs) { 11 | if (!is.uuid(j) && !is.uuid(j.id)) { 12 | throw new Error(enums.message.idInvalid) 13 | } 14 | if (is.uuid(j)) { 15 | validIds.push(j) 16 | } 17 | if (is.uuid(j.id)) { 18 | validIds.push(j.id) 19 | } 20 | } 21 | return validIds 22 | } 23 | 24 | module.exports.job = function jobParseJob (job) { 25 | logger('jobParseJob', job) 26 | if (!job) { return [] } 27 | let jobs = is.array(job) ? job : [job] 28 | let validJobs = [] 29 | for (let j of jobs) { 30 | let detail = false 31 | if (!is.uuid(j.id)) { detail = 'Job id: ' + j.id } 32 | if (!j.q) { detail = 'Job q missing' } 33 | if (!j.priority) { detail = 'Job priority missing' } 34 | if (j.timeout < 0) { detail = 'Job timeout: ' + j.timeout } 35 | if (j.retryDelay < 0) { detail = 'Job retryDelay: ' + j.retryDelay } 36 | if (j.retryMax < 0) { detail = 'Job retryMax: ' + j.retryMax } 37 | if (j.retryCount < 0) { detail = 'Job retryCount: ' + j.retryCount } 38 | if (!j.status) { detail = 'Job status missing' } 39 | if (!is.array(j.log)) { detail = 'Job log: ' + j.log } 40 | if (!is.date(j.dateCreated)) { 41 | detail = 'Job dateCreated: ' + j.dateCreated 42 | } 43 | if (!is.date(j.dateEnable)) { 44 | detail = 'Job dateEnable: ' + j.dateEnable 45 | } 46 | if (j.progress < 0 || j.progress > 100) { 47 | detail = 'Job progress: ' + j.progress 48 | } 49 | if (!j.queueId) { detail = 'Job queueId missing' } 50 | if (!detail) { 51 | validJobs.push(j) 52 | } else { 53 | throw new Error(enums.message.jobInvalid + ': ' + detail) 54 | } 55 | } 56 | return validJobs 57 | } 58 | -------------------------------------------------------------------------------- /tests/queue-summary.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const is = require('../src/is') 4 | const enums = require('../src/enums') 5 | const tError = require('./test-error') 6 | const queueSummary = require('../src/queue-summary') 7 | const queueAddJob = require('../src/queue-add-job') 8 | const Queue = require('../src/queue') 9 | const tOpts = require('./test-options') 10 | 11 | queueSummaryTests() 12 | function queueSummaryTests () { 13 | return new Promise((resolve, reject) => { 14 | test('queue-summary', (t) => { 15 | t.plan(9) 16 | 17 | const q = new Queue(tOpts.cxn(), tOpts.default('queueSummary')) 18 | let jobs = [] 19 | for (let i = 0; i < 6; i++) { 20 | jobs.push(q.createJob()) 21 | } 22 | jobs[0].status = enums.status.waiting 23 | jobs[1].status = enums.status.active 24 | jobs[2].status = enums.status.completed 25 | jobs[3].status = enums.status.cancelled 26 | jobs[4].status = enums.status.failed 27 | jobs[5].status = enums.status.terminated 28 | 29 | return q.reset().then((resetResult) => { 30 | t.ok(is.integer(resetResult), 'Queue reset') 31 | return queueAddJob(q, jobs) 32 | }).then(() => { 33 | return queueSummary(q) 34 | }).then((summary) => { 35 | t.equal(summary.waiting, 1, 'Queue status summary includes waiting') 36 | t.equal(summary.active, 1, 'Queue status summary includes active') 37 | t.equal(summary.completed, 1, 'Queue status summary includes completed') 38 | t.equal(summary.cancelled, 1, 'Queue status summary includes cancelled') 39 | t.equal(summary.failed, 1, 'Queue status summary includes failed') 40 | t.equal(summary.terminated, 1, 'Queue status summary includes terminated') 41 | t.equal(summary.total, 6, 'Queue status summary includes total') 42 | return q.reset() 43 | }).then((resetResult) => { 44 | t.ok(resetResult >= 0, 'Queue reset') 45 | q.stop() 46 | return resolve(t.end()) 47 | }).catch(err => tError(err, module, t)) 48 | }) 49 | }) 50 | } 51 | -------------------------------------------------------------------------------- /src/queue-get-next-job.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const enums = require('./enums') 4 | const dbResult = require('./db-result') 5 | 6 | module.exports = function queueGetNextJob (q) { 7 | logger('getNextJob') 8 | logger(`Concurrency: [${q.concurrency}] Running: [${q.running}]`) 9 | let quantity = q.concurrency - q.running 10 | logger(`Query Limit: [${quantity}]`) 11 | if (quantity < 1) { 12 | return Promise.resolve([]) 13 | } 14 | return Promise.resolve().then(() => { 15 | return q.r 16 | .table(q.name) 17 | .orderBy({index: enums.index.indexInactivePriorityDateCreated}) 18 | .limit(quantity) 19 | .filter( 20 | q.r.row('dateEnable').le(q.r.now()) 21 | ) 22 | .update(getJobUpdate(q), {returnChanges: true}) 23 | .default({}) 24 | .run(q.queryRunOptions) 25 | }).then((updateResult) => { 26 | logger('updateResult', updateResult) 27 | return dbResult.toJob(q, updateResult) 28 | }).then((updatedJobs) => { 29 | for (let job of updatedJobs) { 30 | logger(`Event: active [${job.id}]`) 31 | q.emit(enums.status.active, q.id, job.id) 32 | } 33 | return updatedJobs 34 | }) 35 | } 36 | 37 | function getJobUpdate (q) { 38 | return function (job) { 39 | return q.r.branch( 40 | job('status').ne(enums.status.active), 41 | { 42 | status: enums.status.active, 43 | dateStarted: q.r.now(), 44 | dateEnable: q.r.now() 45 | .add( 46 | job('timeout').div(1000) 47 | ) 48 | .add( 49 | job('retryDelay').div(1000).mul(job('retryCount')) 50 | ), 51 | queueId: q.id, 52 | processCount: job('processCount').add(1), 53 | log: job('log').append({ 54 | date: q.r.now(), 55 | queueId: q.id, 56 | type: enums.log.information, 57 | status: enums.status.active, 58 | retryCount: job('retryCount'), 59 | processCount: job('processCount'), 60 | message: enums.message.active 61 | }) 62 | }, 63 | null 64 | ) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rethinkdb-job-queue", 3 | "version": "3.1.7", 4 | "description": "A persistent job or task queue backed by RethinkDB.", 5 | "main": "index.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "git+https://github.com/grantcarthew/node-rethinkdb-job-queue.git" 9 | }, 10 | "keywords": [ 11 | "job", 12 | "jobs", 13 | "queue", 14 | "task", 15 | "tasks", 16 | "rethinkdb", 17 | "asynchronous", 18 | "async", 19 | "background", 20 | "long", 21 | "running", 22 | "service", 23 | "distributed", 24 | "worker", 25 | "processing" 26 | ], 27 | "author": "Grant Carthew", 28 | "license": "MIT", 29 | "bugs": { 30 | "url": "https://github.com/grantcarthew/node-rethinkdb-job-queue/issues" 31 | }, 32 | "homepage": "https://github.com/grantcarthew/node-rethinkdb-job-queue", 33 | "scripts": { 34 | "prepublish": "npm run build", 35 | "clean": "rm -Rf dist", 36 | "build": "npm run clean && babel src --presets babel-preset-latest --out-dir dist", 37 | "test": "tap --timeout 10000 ./tests/*.spec.js", 38 | "tv": "tap --timeout 10000 --reporter tap ./tests/*.spec.js", 39 | "lint": "standard", 40 | "coverage": "npm run coverage:rm && npm run coverage:cover && npm run coverage:report && npm run coverage:check", 41 | "coverage:cover": "istanbul cover tap -- --timeout 10000 ./tests/*.spec.js", 42 | "coverage:rm": "rm -Rf coverage", 43 | "coverage:report": "istanbul report", 44 | "coverage:check": "istanbul check-coverage ../site-rjq-coverage/coverage.json", 45 | "upgrade": "npm run upgrade:rm && npm run upgrade:ncu && npm run upgrade:install && npm run upgrade:finish", 46 | "upgrade:rm": "rm -Rf node_modules", 47 | "upgrade:ncu": "npm-check-updates --upgradeAll", 48 | "upgrade:install": "npm install", 49 | "upgrade:finish": "npm run build" 50 | }, 51 | "standard": { 52 | "ignore": "dist" 53 | }, 54 | "dependencies": { 55 | "bluebird": "^3.5.1", 56 | "debug": "^3.1.0", 57 | "rethinkdbdash": "^2.3.31", 58 | "serialize-error": "^2.1.0", 59 | "uuid": "^3.2.1" 60 | }, 61 | "devDependencies": { 62 | "babel-cli": "^6.26.0", 63 | "babel-preset-latest": "^6.24.1", 64 | "istanbul": "0.4.5", 65 | "npm-check-updates": "^2.14.0", 66 | "proxyquire": "^1.8.0", 67 | "standard": "^11.0.0", 68 | "tap": "^11.1.1", 69 | "tap-spec": "^4.1.1" 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/job-completed.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const is = require('./is') 4 | const enums = require('./enums') 5 | const jobLog = require('./job-log') 6 | const dbResult = require('./db-result') 7 | 8 | module.exports = function completed (job, result) { 9 | logger(`completed: [${job.id}]`, result) 10 | const isRepeating = is.repeating(job) 11 | job.status = isRepeating ? enums.status.waiting : enums.status.completed 12 | job.dateFinished = new Date() 13 | job.progress = isRepeating ? 0 : 100 14 | let duration = job.dateFinished - job.dateStarted 15 | duration = duration >= 0 ? duration : 0 16 | 17 | const logCompleted = jobLog.createLogObject(job, 18 | result, enums.status.completed) 19 | logCompleted.duration = duration 20 | 21 | const sliceLogs = job.log.length >= job.q.limitJobLogs 22 | const logTruncated = jobLog.createLogObject(job, 23 | `Retaining ${job.q.limitJobLogs} log entries`, 24 | enums.message.jobLogsTruncated, 25 | enums.log.information, 26 | job.status) 27 | 28 | return Promise.resolve().then(() => { 29 | return job.q.r.db(job.q.db) 30 | .table(job.q.name) 31 | .get(job.id) 32 | .update({ 33 | status: job.status, 34 | dateEnable: job.q.r.branch( 35 | isRepeating, 36 | job.q.r.now().add( 37 | job.q.r.row('repeatDelay').div(1000) 38 | ), 39 | job.q.r.row('dateEnable') 40 | ), 41 | dateFinished: job.dateFinished, 42 | progress: job.progress, 43 | log: job.q.r.branch( 44 | sliceLogs, 45 | job.q.r.row('log').append(logCompleted).append(logTruncated).slice(-job.q.limitJobLogs), 46 | job.q.r.row('log').append(logCompleted) 47 | ), 48 | queueId: job.q.id 49 | }, { returnChanges: true }) 50 | .run(job.q.queryRunOptions) 51 | }).then((updateResult) => { 52 | logger(`updateResult`, updateResult) 53 | return dbResult.toIds(updateResult) 54 | }).then((jobIds) => { 55 | logger(`Event: completed`, jobIds[0], isRepeating) 56 | job.q.emit(enums.status.completed, job.q.id, jobIds[0], isRepeating) 57 | if (!isRepeating && is.true(job.q.removeFinishedJobs)) { 58 | return job.q.removeJob(job).then((deleteResult) => { 59 | return jobIds 60 | }) 61 | } else { 62 | return jobIds 63 | } 64 | }) 65 | } 66 | -------------------------------------------------------------------------------- /tests/queue-reset.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const is = require('../src/is') 4 | const tError = require('./test-error') 5 | const queueReset = require('../src/queue-reset') 6 | const Queue = require('../src/queue') 7 | const tOpts = require('./test-options') 8 | const eventHandlers = require('./test-event-handlers') 9 | const testName = 'queue-reset' 10 | 11 | queueResetTests() 12 | function queueResetTests () { 13 | return new Promise((resolve, reject) => { 14 | test(testName, (t) => { 15 | t.plan(32) 16 | 17 | const q = new Queue(tOpts.cxn(), tOpts.default('queueReset')) 18 | const jobs = [ 19 | q.createJob(), 20 | q.createJob(), 21 | q.createJob() 22 | ] 23 | 24 | // ---------- Event Handler Setup ---------- 25 | let state = { 26 | testName, 27 | enabled: false, 28 | ready: 0, 29 | processing: 0, 30 | progress: 0, 31 | pausing: 0, 32 | paused: 0, 33 | resumed: 0, 34 | removed: 0, 35 | reset: 1, 36 | error: 0, 37 | reviewed: 0, 38 | detached: 0, 39 | stopping: 0, 40 | stopped: 0, 41 | dropped: 0, 42 | added: 3, 43 | waiting: 0, 44 | active: 0, 45 | completed: 0, 46 | cancelled: 0, 47 | failed: 0, 48 | terminated: 0, 49 | reanimated: 0, 50 | log: 0, 51 | updated: 0 52 | } 53 | 54 | return q.reset().then((removed) => { 55 | t.ok(is.integer(removed), 'Initial reset succeeded') 56 | eventHandlers.add(t, q, state) 57 | return q.addJob(jobs) 58 | }).then((savedJobs) => { 59 | t.equal(savedJobs.length, 3, 'Jobs saved successfully') 60 | return q.summary() 61 | }).then((beforeSummary) => { 62 | t.equal(beforeSummary.waiting, 3, 'Status summary contains correct value') 63 | return queueReset(q) 64 | }).then((total) => { 65 | t.equal(total, 3, 'Queue reset removed valid number of jobs') 66 | return q.summary() 67 | }).then((afterSummary) => { 68 | t.equal(afterSummary.waiting, 0, 'Status summary contains no added jobs') 69 | 70 | // ---------- Event Summary ---------- 71 | eventHandlers.remove(t, q, state) 72 | 73 | q.stop() 74 | return resolve(t.end()) 75 | }).catch(err => tError(err, module, t)) 76 | }) 77 | }) 78 | } 79 | -------------------------------------------------------------------------------- /src/queue-db.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const enums = require('./enums') 4 | const uuid = require('uuid') 5 | const hostname = require('os').hostname() 6 | const dbAssert = require('./db-assert') 7 | const dbReview = require('./db-review') 8 | const queueChange = require('./queue-change') 9 | const dbDriver = require('./db-driver') 10 | 11 | module.exports.attach = function dbAttach (q, cxn) { 12 | logger('attach') 13 | q._r = dbDriver(cxn) 14 | q._host = q.r._poolMaster._options.host 15 | q._port = q.r._poolMaster._options.port 16 | q._db = q.r._poolMaster._options.db 17 | q._id = [ 18 | hostname, 19 | q._db, 20 | q.name, 21 | process.pid, 22 | uuid.v4() 23 | ].join(':') 24 | q._ready = dbAssert(q).then(() => { 25 | if (q.changeFeed) { 26 | return q.r.db(q.db) 27 | .table(q.name) 28 | .changes() 29 | .run(q.queryRunOptions) 30 | .then((changeFeed) => { 31 | q._changeFeedCursor = changeFeed 32 | return q._changeFeedCursor.each((err, change) => { 33 | return queueChange(q, err, change) 34 | }) 35 | }) 36 | } 37 | q._changeFeedCursor = false 38 | return null 39 | }).then(() => { 40 | if (q.master) { 41 | logger('Queue is a master') 42 | return dbReview.enable(q) 43 | } 44 | return null 45 | }).then(() => { 46 | logger(`Event: ready [${q.id}]`) 47 | q.emit(enums.status.ready, q.id) 48 | return true 49 | }) 50 | return q._ready 51 | } 52 | 53 | module.exports.detach = function dbDetach (q) { 54 | logger('detach') 55 | return Promise.resolve().then(() => { 56 | if (q._changeFeedCursor) { 57 | let feed = q._changeFeedCursor 58 | q._changeFeedCursor = false 59 | logger('closing changeFeed') 60 | return feed.close() 61 | } 62 | return true 63 | }).then(() => { 64 | if (q.master) { 65 | logger('disabling dbReview') 66 | return dbReview.disable(q) 67 | } 68 | return true 69 | }) 70 | } 71 | 72 | module.exports.drain = function drain (q) { 73 | return Promise.resolve().then(() => { 74 | q._ready = Promise.resolve(false) 75 | logger('draining connection pool') 76 | return q.r.getPoolMaster().drain() 77 | }).delay(1000).then(() => { 78 | logger(`Event: detached [${q.id}]`) 79 | q.emit(enums.status.detached, q.id) 80 | }).delay(1000).then(() => { 81 | q.eventNames().forEach((key) => { 82 | q.removeAllListeners(key) 83 | }) 84 | return true 85 | }) 86 | } 87 | -------------------------------------------------------------------------------- /tests/datetime.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const is = require('../src/is') 3 | const tError = require('./test-error') 4 | const datetime = require('../src/datetime') 5 | 6 | dateTimeTests() 7 | function dateTimeTests () { 8 | test('datetime', (t) => { 9 | t.plan(23) 10 | 11 | try { 12 | const tDate = new Date('2000-01-02 03:04:05.006') 13 | const tValue = 12345 14 | const dateStings = { 15 | date: '2000-01-02', 16 | time: '03:04:05.006', 17 | datetime: '2000-01-02 03:04:05.006' 18 | } 19 | t.ok(is.date(datetime.add.ms(tDate, tValue)), 'Add ms is a date object') 20 | t.equal(datetime.add.ms(tDate, tValue) - tDate, tValue, 'Add ms is valid') 21 | t.ok(is.date(datetime.add.ms(tValue)), 'Add ms only is a date object') 22 | t.ok(is.dateAfter(datetime.add.ms(tValue)), 'Add ms only is valid') 23 | t.ok(is.date(datetime.add.sec(tDate, tValue)), 'Add sec is a date object') 24 | t.equal(datetime.add.sec(tDate, tValue) - tDate, tValue * 1000, 'Add sec is valid') 25 | t.ok(is.date(datetime.add.sec(tValue)), 'Add sec only is a date object') 26 | t.ok(is.dateAfter(datetime.add.sec(tValue)), 'Add sec only is valid') 27 | t.ok(is.date(datetime.add.min(tDate, tValue)), 'Add min is a date object') 28 | t.equal(datetime.add.min(tDate, tValue) - tDate, tValue * 60000, 'Add min is valid') 29 | t.ok(is.date(datetime.add.min(tValue)), 'Add min only is a date object') 30 | t.ok(is.dateAfter(datetime.add.min(tValue)), 'Add min only is valid') 31 | t.ok(is.date(datetime.add.hours(tDate, tValue)), 'Add hours is a date object') 32 | t.equal(datetime.add.hours(tDate, tValue) - tDate, tValue * 3600000, 'Add hours is valid') 33 | t.ok(is.date(datetime.add.hours(tValue)), 'Add hours only is a date object') 34 | t.ok(is.dateAfter(datetime.add.hours(tValue)), 'Add hours only is valid') 35 | t.ok(is.date(datetime.add.days(tDate, tValue)), 'Add days is a date object') 36 | t.equal(datetime.add.days(tDate, tValue) - tDate, tValue * 86400000, 'Add days is valid') 37 | t.ok(is.date(datetime.add.days(tValue)), 'Add days only is a date object') 38 | t.ok(is.dateAfter(datetime.add.days(tValue)), 'Add days only is valid') 39 | t.equal(datetime.formatDate(tDate), dateStings.date, 'formatDate is valid') 40 | t.equal(datetime.formatTime(tDate), dateStings.time, 'formatTime is valid') 41 | t.equal(datetime.format(tDate), dateStings.datetime, 'format is valid') 42 | } catch (err) { 43 | tError(err, module, t) 44 | } 45 | }) 46 | } 47 | -------------------------------------------------------------------------------- /src/datetime.js: -------------------------------------------------------------------------------- 1 | // The following have been removed due to circular dependencies. 2 | // const logger = require('./logger')(module) 3 | // const enums = require('./enums') 4 | 5 | function isDate (value) { 6 | return value instanceof Date || 7 | Object.prototype.toString.call(value) === '[object Date]' 8 | } 9 | function isInteger (value) { 10 | return Object.prototype.toString.call(value) === '[object Number]' && 11 | !Number.isNaN(value) && 12 | value % 1 === 0 13 | } 14 | 15 | function addMs (dateObject, value, multiplier = 0) { 16 | if (isInteger(dateObject)) { 17 | value = dateObject 18 | dateObject = new Date() 19 | } 20 | if (isDate(dateObject) && isInteger(value)) { 21 | return new Date(dateObject.getTime() + (value * multiplier)) 22 | } 23 | throw new Error('Job data can not be a function') 24 | } 25 | 26 | function addMilliseconds (dateObject, ms) { 27 | return addMs(dateObject, ms, 1) 28 | } 29 | 30 | function addSeconds (dateObject, sec) { 31 | return addMs(dateObject, sec, 1000) 32 | } 33 | 34 | function addMinutes (dateObject, min) { 35 | return addMs(dateObject, min, 60000) 36 | } 37 | 38 | function addHours (dateObject, hours) { 39 | return addMs(dateObject, hours, 3600000) 40 | } 41 | 42 | function addDays (dateObject, days) { 43 | return addMs(dateObject, days, 86400000) 44 | } 45 | 46 | module.exports.add = { 47 | ms: addMilliseconds, 48 | sec: addSeconds, 49 | min: addMinutes, 50 | hours: addHours, 51 | days: addDays 52 | } 53 | 54 | function formatDate (dateObject) { 55 | let year = dateObject.getFullYear().toString() 56 | let month = (dateObject.getMonth() + 1).toString() // zero-based 57 | month = month[1] ? month : `0${month}` 58 | let day = dateObject.getDate().toString() 59 | day = day[1] ? day : `0${day}` 60 | return `${year}-${month}-${day}` 61 | } 62 | 63 | module.exports.formatDate = formatDate 64 | 65 | function formatTime (dateObject) { 66 | let hour = dateObject.getHours().toString() 67 | hour = hour[1] ? hour : `0${hour}` 68 | let min = dateObject.getMinutes().toString() 69 | min = min[1] ? min : `0${min}` 70 | let sec = dateObject.getSeconds().toString() 71 | sec = sec[1] ? sec : `0${sec}` 72 | let ms = dateObject.getMilliseconds().toString() 73 | ms = ms.length > 1 ? ms : `00${ms}` 74 | ms = ms.length > 2 ? ms : `0${ms}` 75 | return `${hour}:${min}:${sec}.${ms}` 76 | } 77 | 78 | module.exports.formatTime = formatTime 79 | 80 | function format (dateObject) { 81 | return `${formatDate(dateObject)} ${formatTime(dateObject)}` 82 | } 83 | 84 | module.exports.format = format 85 | -------------------------------------------------------------------------------- /src/job-options.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const enums = require('./enums') 3 | const is = require('./is') 4 | 5 | module.exports = function jobOptions (newOptions = {}, oldOptions = {}) { 6 | logger('jobOptions', newOptions, oldOptions) 7 | 8 | const finalOptions = {} 9 | finalOptions.name = null 10 | finalOptions.priority = enums.options.priority 11 | finalOptions.timeout = enums.options.timeout 12 | finalOptions.retryMax = enums.options.retryMax 13 | finalOptions.retryDelay = enums.options.retryDelay 14 | finalOptions.repeat = enums.options.repeat 15 | finalOptions.repeatDelay = enums.options.repeatDelay 16 | 17 | if (is.string(oldOptions.name)) { 18 | finalOptions.name = oldOptions.name 19 | } 20 | 21 | if (Object.keys(enums.priority).includes(oldOptions.priority)) { 22 | finalOptions.priority = oldOptions.priority 23 | } 24 | 25 | if (is.integer(oldOptions.timeout) && oldOptions.timeout >= 0) { 26 | finalOptions.timeout = oldOptions.timeout 27 | } 28 | 29 | if (is.integer(oldOptions.retryMax) && oldOptions.retryMax >= 0) { 30 | finalOptions.retryMax = oldOptions.retryMax 31 | } 32 | 33 | if (is.integer(oldOptions.retryDelay) && oldOptions.retryDelay >= 0) { 34 | finalOptions.retryDelay = oldOptions.retryDelay 35 | } 36 | 37 | if (is.true(oldOptions.repeat) || 38 | is.false(oldOptions.repeat) || 39 | (is.integer(oldOptions.repeat) && oldOptions.repeat >= 0)) { 40 | finalOptions.repeat = oldOptions.repeat 41 | } 42 | 43 | if (is.integer(oldOptions.repeatDelay) && oldOptions.repeatDelay >= 0) { 44 | finalOptions.repeatDelay = oldOptions.repeatDelay 45 | } 46 | 47 | if (is.string(newOptions.name)) { 48 | finalOptions.name = newOptions.name 49 | } 50 | 51 | if (Object.keys(enums.priority).includes(newOptions.priority)) { 52 | finalOptions.priority = newOptions.priority 53 | } 54 | 55 | if (is.integer(newOptions.timeout) && newOptions.timeout >= 0) { 56 | finalOptions.timeout = newOptions.timeout 57 | } 58 | 59 | if (is.integer(newOptions.retryMax) && newOptions.retryMax >= 0) { 60 | finalOptions.retryMax = newOptions.retryMax 61 | } 62 | 63 | if (is.integer(newOptions.retryDelay) && newOptions.retryDelay >= 0) { 64 | finalOptions.retryDelay = newOptions.retryDelay 65 | } 66 | 67 | if (is.true(newOptions.repeat) || 68 | is.false(newOptions.repeat) || 69 | (is.integer(newOptions.repeat) && newOptions.repeat >= 0)) { 70 | finalOptions.repeat = newOptions.repeat 71 | } 72 | 73 | if (is.integer(newOptions.repeatDelay) && newOptions.repeatDelay >= 0) { 74 | finalOptions.repeatDelay = newOptions.repeatDelay 75 | } 76 | 77 | return finalOptions 78 | } 79 | -------------------------------------------------------------------------------- /tests/queue-drop.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const is = require('../src/is') 4 | const tError = require('./test-error') 5 | const queueDrop = require('../src/queue-drop') 6 | const Queue = require('../src/queue') 7 | const simulateJobProcessing = require('./test-utils').simulateJobProcessing 8 | const tOpts = require('./test-options') 9 | const rethinkdbdash = require('rethinkdbdash') 10 | const eventHandlers = require('./test-event-handlers') 11 | const testName = 'queue-drop' 12 | 13 | queueDropTests() 14 | function queueDropTests () { 15 | return new Promise((resolve, reject) => { 16 | test(testName, (t) => { 17 | t.plan(33) 18 | 19 | const tableName = 'queueDrop' 20 | const mockQueue = { 21 | r: rethinkdbdash(Object.assign(tOpts.cxn(), { silent: true })), 22 | db: tOpts.dbName, 23 | name: tableName, 24 | id: 'mock:queue:id' 25 | } 26 | 27 | let q = new Queue(tOpts.cxn(), tOpts.default(tableName)) 28 | 29 | // ---------- Event Handler Setup ---------- 30 | let state = { 31 | testName, 32 | enabled: false, 33 | ready: 0, 34 | processing: 0, 35 | progress: 0, 36 | pausing: 1, 37 | paused: 1, 38 | resumed: 0, 39 | removed: 0, 40 | reset: 0, 41 | error: 0, 42 | reviewed: 0, 43 | detached: 1, 44 | stopping: 1, 45 | stopped: 1, 46 | dropped: 1, 47 | added: 0, 48 | waiting: 0, 49 | active: 0, 50 | completed: 0, 51 | cancelled: 0, 52 | failed: 0, 53 | terminated: 0, 54 | reanimated: 0, 55 | log: 0, 56 | updated: 0 57 | } 58 | 59 | return q.reset().then((resetResult) => { 60 | t.ok(is.integer(resetResult), 'Queue reset') 61 | 62 | // ---------- Drop Queue Test ---------- 63 | t.comment('queue-drop: Drop Queue') 64 | eventHandlers.add(t, q, state) 65 | simulateJobProcessing(q) 66 | return queueDrop(q) 67 | }).then((removeResult) => { 68 | t.ok(removeResult, 'Queue dropped') 69 | return q.ready() 70 | }).then((ready) => { 71 | t.notOk(ready, 'Queue ready returns false') 72 | return mockQueue.r.db(mockQueue.db).tableList() 73 | }).then((tableList) => { 74 | t.notOk(tableList.includes(mockQueue.name), 'Table dropped from database') 75 | 76 | // ---------- Event Summary ---------- 77 | eventHandlers.remove(t, q, state) 78 | mockQueue.r.getPoolMaster().drain() 79 | return resolve(t.end()) 80 | }).catch(err => tError(err, module, t)) 81 | }) 82 | }) 83 | } 84 | -------------------------------------------------------------------------------- /tests/queue-interruption.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const tError = require('./test-error') 4 | const Queue = require('../src/queue') 5 | const tOpts = require('./test-options') 6 | const proxyquire = require('proxyquire') 7 | const processStub = {} 8 | const queueInterruption = proxyquire('../src/queue-interruption', 9 | { './queue-process': processStub }) 10 | const eventHandlers = require('./test-event-handlers') 11 | const testName = 'queue-interruption' 12 | 13 | queueInterruptionTests() 14 | function queueInterruptionTests () { 15 | return new Promise((resolve, reject) => { 16 | test(testName, (t) => { 17 | t.plan(33) 18 | 19 | const q = new Queue(tOpts.cxn(), tOpts.default('queueInterruption')) 20 | processStub.restart = function (q) { 21 | t.ok(q.id, 'Queue process restart called') 22 | } 23 | 24 | // ---------- Event Handler Setup ---------- 25 | let state = { 26 | testName, 27 | enabled: false, 28 | ready: 0, 29 | processing: 0, 30 | progress: 0, 31 | pausing: 1, 32 | paused: 1, 33 | resumed: 1, 34 | removed: 0, 35 | reset: 0, 36 | error: 0, 37 | reviewed: 0, 38 | detached: 0, 39 | stopping: 0, 40 | stopped: 0, 41 | dropped: 0, 42 | added: 0, 43 | waiting: 0, 44 | active: 0, 45 | completed: 0, 46 | cancelled: 0, 47 | failed: 0, 48 | terminated: 0, 49 | reanimated: 0, 50 | log: 0, 51 | updated: 0 52 | } 53 | 54 | return q.ready().then((ready) => { 55 | eventHandlers.add(t, q, state) 56 | t.ok(ready, 'Queue is ready') 57 | 58 | // ---------- Pause Test ---------- 59 | t.comment('queue-interruption: Pause') 60 | t.notOk(q.paused, 'Queue is not paused') 61 | // Simulate running jobs 62 | q._running = 1 63 | setTimeout(function setRunningToZero () { 64 | q._running = 0 65 | }, 400) 66 | return queueInterruption.pause(q) 67 | }).then((paused) => { 68 | t.ok(paused, 'Interruption pause returns true') 69 | t.ok(q.paused, 'Queue is paused') 70 | 71 | // ---------- Resume Test ---------- 72 | t.comment('queue-interruption: Resume') 73 | return queueInterruption.resume(q) 74 | }).then((resumed) => { 75 | t.ok(resumed, 'Interruption resume returns true') 76 | t.notOk(q.paused, 'Queue is not paused') 77 | 78 | // ---------- Event Summary ---------- 79 | eventHandlers.remove(t, q, state) 80 | 81 | q.stop() 82 | return resolve(t.end()) 83 | }).catch(err => tError(err, module, t)) 84 | }) 85 | }) 86 | } 87 | -------------------------------------------------------------------------------- /tests/queue-state.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const enums = require('../src/enums') 4 | const tError = require('./test-error') 5 | const tData = require('./test-options').tData 6 | const tOpts = require('./test-options') 7 | const Queue = require('../src/queue') 8 | const queueState = require('../src/queue-state') 9 | const eventHandlers = require('./test-event-handlers') 10 | const testName = 'queue-state' 11 | 12 | queueStateTests() 13 | function queueStateTests () { 14 | return new Promise((resolve, reject) => { 15 | test(testName, (t) => { 16 | t.plan(30) 17 | 18 | const tableName = 'queueState' 19 | const q = new Queue(tOpts.cxn(), tOpts.default(tableName)) 20 | let q2 21 | const job = q.createJob() 22 | job.data = tData 23 | 24 | // ---------- Event Handler Setup ---------- 25 | let state = { 26 | testName, 27 | enabled: false, 28 | ready: 0, 29 | processing: 0, 30 | progress: 0, 31 | pausing: 1, 32 | paused: 1, 33 | resumed: 1, 34 | removed: 0, 35 | reset: 0, 36 | error: 0, 37 | reviewed: 0, 38 | detached: 0, 39 | stopping: 0, 40 | stopped: 0, 41 | dropped: 0, 42 | added: 0, 43 | waiting: 0, 44 | active: 0, 45 | completed: 0, 46 | cancelled: 0, 47 | failed: 0, 48 | terminated: 0, 49 | reanimated: 0, 50 | log: 0, 51 | updated: 0 52 | } 53 | 54 | return Promise.resolve( 55 | q.ready() 56 | ).then(() => { 57 | return q.r.table(tableName).wait() 58 | }).then(() => { 59 | return q.reset() 60 | }).then((resetResult) => { 61 | t.ok(resetResult >= 0, 'Queue reset') 62 | q2 = new Queue(tOpts.cxn(), tOpts.default(tableName)) 63 | return q2.r.table(tableName).wait() 64 | }).then((waitResult) => { 65 | t.ok(waitResult.ready === 1, 'Queue reset') 66 | eventHandlers.add(t, q, state) 67 | }).then((ready) => { 68 | // ---------- Global Pause Test ---------- 69 | t.comment('queue-state: Global Pause') 70 | return queueState(q2, enums.status.paused) 71 | }).then((resetResult) => { 72 | t.ok(q.paused, 'Local queue is paused') 73 | // 74 | // ---------- Global Resume Test ---------- 75 | t.comment('queue-state: Global Resume') 76 | return queueState(q2, enums.status.active) 77 | }).then((resetResult) => { 78 | t.notOk(q.paused, 'Local queue is resumed') 79 | 80 | // ---------- Event Summary ---------- 81 | eventHandlers.remove(t, q, state) 82 | 83 | q.stop() 84 | q2.stop() 85 | return resolve(t.end()) 86 | }).catch(err => tError(err, module, t)) 87 | }) 88 | }) 89 | } 90 | -------------------------------------------------------------------------------- /tests/test-template.js: -------------------------------------------------------------------------------- 1 | // const test = require('tap').test 2 | // const Promise = require('bluebird') 3 | // const datetime = require('../src/datetime') 4 | // const is = require('../src/is') 5 | // const enums = require('../src/enums') 6 | // const tError = require('./test-error') 7 | // const tOpts = require('./test-options') 8 | // const tData = require('./test-options').tData 9 | // const queueProcess = require('../src/queue-process') 10 | // const dbReview = require('../src/db-review') 11 | // const Queue = require('../src/queue') 12 | // const eventHandlers = require('./test-event-handlers') 13 | // const testName = 'XXXXXXXXXXXX' 14 | // 15 | // module.exports = function () { 16 | // return new Promise((resolve, reject) => { 17 | // test(testName, (t) => { 18 | // t.plan(1000) 19 | // 20 | // // ---------- Test Setup ---------- 21 | // const q = new Queue(tOpts.cxn(), tOpts.default()) 22 | // 23 | // let jobs 24 | // let jobDelay = 200 25 | // const noOfJobsToCreate = 10 26 | // 27 | // // ---------- Event Handler Setup ---------- 28 | // let state = { 29 | // testName, 30 | // enabled: false, 31 | // ready: 0, 32 | // processing: 0, 33 | // progress: 0, 34 | // pausing: 0, 35 | // paused: 0, 36 | // resumed: 0, 37 | // removed: 0, 38 | // reset: 0, 39 | // error: 0, 40 | // reviewed: 0, 41 | // detached: 0, 42 | // stopping: 0, 43 | // stopped: 0, 44 | // dropped: 0, 45 | // added: 0, 46 | // waiting: 0, 47 | // active: 0, 48 | // completed: 0, 49 | // cancelled: 0, 50 | // failed: 0, 51 | // terminated: 0, 52 | // reanimated: 0, 53 | // log: 0, 54 | // updated: 0 55 | // } 56 | // 57 | // // ---------- Test Setup ---------- 58 | // jobs = [] 59 | // for (let i = 0; i < noOfJobsToCreate; i++) { 60 | // jobs.push(q.createJob()) 61 | // } 62 | // return q.reset().then((resetResult) => { 63 | // t.ok(is.integer(resetResult), 'Queue reset') 64 | // return q.pause() 65 | // }).then(() => { 66 | // eventHandlers.add(t, q, state) 67 | // 68 | // // ---------- Processing, Pause, and Concurrency Test ---------- 69 | // t.comment('queue-process: Process, Pause, and Concurrency') 70 | // return q.addJob(jobs) 71 | // }).then((savedJobs) => { 72 | // t.equal(savedJobs.length, noOfJobsToCreate, `Jobs saved successfully: [${savedJobs.length}]`) 73 | // 74 | // // ---------- Queue Summary ---------- 75 | // t.comment('queue-process: Queue Summary') 76 | // return q.summary() 77 | // }).then((queueSummary) => { 78 | // 79 | // // ---------- Event Summary ---------- 80 | // eventHandlers.remove(t, q, state) 81 | // 82 | // return q.reset() 83 | // }).then((resetResult) => { 84 | // t.ok(resetResult >= 0, 'Queue reset') 85 | // q.stop() 86 | // return resolve(t.end()) 87 | // }).catch(err => tError(err, module, t)) 88 | // }) 89 | // }) 90 | // } 91 | -------------------------------------------------------------------------------- /tests/queue-get-job.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const is = require('../src/is') 4 | const enums = require('../src/enums') 5 | const tError = require('./test-error') 6 | const queueGetJob = require('../src/queue-get-job') 7 | const Queue = require('../src/queue') 8 | const tOpts = require('./test-options') 9 | 10 | queueGetJobTests() 11 | function queueGetJobTests () { 12 | return new Promise((resolve, reject) => { 13 | test('queue-get-job', (t) => { 14 | t.plan(13) 15 | 16 | const q = new Queue(tOpts.cxn(), tOpts.default('queueGetJob')) 17 | const job1 = q.createJob() 18 | const job2 = q.createJob() 19 | const job3 = q.createJob() 20 | const jobs = [ 21 | job1, 22 | job2, 23 | job3 24 | ] 25 | let jobsSaved 26 | 27 | return q.reset().then((resetResult) => { 28 | t.ok(is.integer(resetResult), 'Queue reset') 29 | return q.addJob(jobs) 30 | }).then((savedJobs) => { 31 | jobsSaved = savedJobs 32 | t.equal(savedJobs.length, 3, 'Job saved successfully') 33 | 34 | // ---------- Undefined Tests ---------- 35 | t.comment('queue-get-job: Undefined Job') 36 | return queueGetJob(q) 37 | }).then((undefinedResult) => { 38 | t.ok(is.array(undefinedResult), 'Undefined returns an Array') 39 | t.equal(undefinedResult.length, 0, 'Undefined returns an empty Array') 40 | 41 | // ---------- Invalid Id Tests ---------- 42 | t.comment('queue-get-job: Invalid Id') 43 | return queueGetJob(q, ['invalid id']).catch((err) => { 44 | t.ok(err.message.includes(enums.message.idInvalid), 'Invalid id returns rejected Promise') 45 | }) 46 | }).then((empty) => { 47 | // 48 | // ---------- Empty Array Tests ---------- 49 | t.comment('queue-get-job: Empty Array') 50 | return queueGetJob(q, []) 51 | }).then((empty) => { 52 | t.equal(empty.length, 0, 'Empty array returns empty array') 53 | 54 | // ---------- Single Id Tests ---------- 55 | t.comment('queue-get-job: Single Id') 56 | return queueGetJob(q, job1.id) 57 | }).then((retrievedJob) => { 58 | t.equal(retrievedJob.length, 1, 'One jobs retrieved') 59 | t.deepEqual(retrievedJob[0], jobsSaved[0], 'Job retrieved successfully') 60 | 61 | // ---------- Id Array Tests ---------- 62 | t.comment('queue-get-job: Array of Ids') 63 | return queueGetJob(q, [job1.id, job2.id, job3.id]) 64 | }).then((retrievedJobs) => { 65 | const retrievedIds = retrievedJobs.map(j => j.id) 66 | t.equal(retrievedJobs.length, 3, 'Three jobs retrieved') 67 | t.ok(retrievedIds.includes(retrievedJobs[0].id), 'Job 1 retrieved successfully') 68 | t.ok(retrievedIds.includes(retrievedJobs[1].id), 'Job 2 retrieved successfully') 69 | t.ok(retrievedIds.includes(retrievedJobs[2].id), 'Job 3 retrieved successfully') 70 | return q.reset() 71 | }).then((resetResult) => { 72 | t.ok(resetResult >= 0, 'Queue reset') 73 | q.stop() 74 | return resolve(t.end()) 75 | }).catch(err => tError(err, module, t)) 76 | }) 77 | }) 78 | } 79 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at codeofconduct@carthew.net. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /src/job-failed.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const is = require('./is') 4 | const datetime = require('./datetime') 5 | const enums = require('./enums') 6 | const jobLog = require('./job-log') 7 | const dbResult = require('./db-result') 8 | const serializeError = require('serialize-error') 9 | 10 | module.exports = function failed (job, err) { 11 | logger(`failed: [${job.id}]`) 12 | logger(`error`, err) 13 | 14 | let logType = enums.log.error 15 | const isRetry = job.retryCount < job.retryMax 16 | const isRepeating = is.repeating(job) 17 | let dateEnable = new Date() 18 | 19 | job.status = enums.status.terminated 20 | 21 | if (isRetry) { 22 | job.status = enums.status.failed 23 | dateEnable = datetime.add.ms(job.retryDelay * job.retryCount) 24 | job.retryCount++ 25 | logType = enums.log.warning 26 | } 27 | 28 | if (!isRetry && isRepeating) { 29 | job.status = enums.status.waiting 30 | dateEnable = datetime.add.ms(job.repeatDelay) 31 | job.retryCount = 0 32 | } 33 | 34 | job.dateFinished = new Date() 35 | job.progress = 0 36 | let duration = job.dateFinished - job.dateStarted 37 | duration = duration >= 0 ? duration : 0 38 | 39 | const errAsString = serializeError(err) 40 | 41 | const logFailed = jobLog.createLogObject(job, 42 | errAsString, 43 | enums.message.failed, 44 | logType, 45 | job.status) 46 | logFailed.duration = duration 47 | logFailed.errorMessage = err && err.message 48 | ? err.message : enums.message.noErrorMessage 49 | logFailed.errorStack = err && err.stack 50 | ? err.stack : enums.message.noErrorStack 51 | 52 | const sliceLogs = job.log.length >= job.q.limitJobLogs 53 | const logTruncated = jobLog.createLogObject(job, 54 | `Retaining ${job.q.limitJobLogs} log entries`, 55 | enums.message.jobLogsTruncated, 56 | enums.log.information, 57 | job.status) 58 | 59 | return Promise.resolve().then(() => { 60 | return job.q.r.db(job.q.db) 61 | .table(job.q.name) 62 | .get(job.id) 63 | .update({ 64 | status: job.status, 65 | retryCount: job.retryCount, 66 | progress: job.progress, 67 | dateFinished: job.dateFinished, 68 | dateEnable, 69 | log: job.q.r.branch( 70 | sliceLogs, 71 | job.q.r.row('log').append(logFailed).append(logTruncated).slice(-job.q.limitJobLogs), 72 | job.q.r.row('log').append(logFailed) 73 | ), 74 | queueId: job.q.id 75 | }, {returnChanges: true}) 76 | .run(job.q.queryRunOptions) 77 | }).then((updateResult) => { 78 | logger(`updateResult`, updateResult) 79 | return dbResult.toIds(updateResult) 80 | }).then((jobIds) => { 81 | if (isRetry || isRepeating) { 82 | logger(`Event: failed`, job.q.id, jobIds[0]) 83 | job.q.emit(enums.status.failed, job.q.id, jobIds[0]) 84 | } else { 85 | logger(`Event: terminated`, job.q.id, jobIds[0]) 86 | job.q.emit(enums.status.terminated, job.q.id, jobIds[0]) 87 | } 88 | if (!isRetry && 89 | !isRepeating && 90 | is.true(job.q.removeFinishedJobs)) { 91 | return job.q.removeJob(job).then((deleteResult) => { 92 | return jobIds 93 | }) 94 | } else { 95 | return jobIds 96 | } 97 | }) 98 | } 99 | -------------------------------------------------------------------------------- /tests/queue-find-job.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const tError = require('./test-error') 4 | const tData = require('./test-options').tData 5 | const tOpts = require('./test-options') 6 | const is = require('../src/is') 7 | const Queue = require('../src/queue') 8 | const queueFindJob = require('../src/queue-find-job') 9 | 10 | queueFindJobTests() 11 | function queueFindJobTests () { 12 | return new Promise((resolve, reject) => { 13 | test('queue-find-job', (t) => { 14 | t.plan(15) 15 | 16 | const q = new Queue(tOpts.cxn(), tOpts.default('queueFindJob')) 17 | const titleText = 'Find Job Test' 18 | let job = q.createJob() 19 | job.data = tData 20 | job.title = titleText 21 | 22 | return q.reset().then((resetResult) => { 23 | t.ok(is.integer(resetResult), 'Queue reset') 24 | return q.addJob(job) 25 | }).then((savedJob1) => { 26 | t.equal(savedJob1[0].id, job.id, 'Job saved successfully') 27 | 28 | // ---------- Single Job Find Test ---------- 29 | t.comment('queue-find-job: Single Job Find') 30 | return queueFindJob(q, { title: titleText }) 31 | }).then((foundJobs1) => { 32 | t.equal(foundJobs1[0].title, titleText, 'Found job successfully') 33 | t.equal(foundJobs1[0].data, tData, 'Job data is valid') 34 | 35 | // ---------- Single Raw Find Test ---------- 36 | t.comment('queue-find-job: Raw Job Find') 37 | return queueFindJob(q, { title: titleText }, true) 38 | }).then((foundJobs2) => { 39 | t.equal(foundJobs2[0].title, titleText, 'Found raw job successfully') 40 | t.equal(foundJobs2[0].data, tData, 'Raw job data is valid') 41 | t.notOk(foundJobs2[0].q, 'Raw result does not have a q property') 42 | 43 | // ---------- Multiple Job Find Test ---------- 44 | t.comment('queue-find-job: Multiple Job Find') 45 | job = q.createJob() 46 | job.data = tData 47 | job.title = titleText 48 | return q.addJob(job) 49 | }).then((savedJob1) => { 50 | t.equal(savedJob1[0].id, job.id, 'Job saved successfully') 51 | return queueFindJob(q, { title: titleText }) 52 | }).then((foundJobs2) => { 53 | t.equal(foundJobs2.length, 2, 'Found two jobs successfully') 54 | t.equal(foundJobs2[0].title, titleText, 'Found first job successfully') 55 | t.equal(foundJobs2[0].data, tData, 'First Job data is valid') 56 | t.equal(foundJobs2[1].title, titleText, 'Found second job successfully') 57 | t.equal(foundJobs2[1].data, tData, 'Second Job data is valid') 58 | 59 | // ---------- Predicate Function Job Find Test ---------- 60 | t.comment('queue-find-job: Predicate Function Job Find') 61 | return queueFindJob(q, (job) => { 62 | return job('title').eq(titleText) 63 | }) 64 | }).then((foundJobs4) => { 65 | t.equal(foundJobs4.length, 2, 'Found two jobs successfully') 66 | 67 | // ---------- Zero Job Find Test ---------- 68 | t.comment('Zero Job Find') 69 | return queueFindJob(q, { abc: '123' }) 70 | }).then((foundJobs3) => { 71 | t.equal(foundJobs3.length, 0, 'Zero jobs found successfully') 72 | 73 | q.stop() 74 | return resolve(t.end()) 75 | }).catch(err => tError(err, module, t)) 76 | }) 77 | }) 78 | } 79 | -------------------------------------------------------------------------------- /tests/queue-find-job-by-name.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const tError = require('./test-error') 4 | const tData = require('./test-options').tData 5 | const tOpts = require('./test-options') 6 | const is = require('../src/is') 7 | const Queue = require('../src/queue') 8 | const queueFindJobByName = require('../src/queue-find-job-by-name') 9 | 10 | queueFindJobByNameTests() 11 | function queueFindJobByNameTests () { 12 | return new Promise((resolve, reject) => { 13 | test('queue-find-job-by-name', (t) => { 14 | t.plan(19) 15 | 16 | const q = new Queue(tOpts.cxn(), tOpts.default('queueFindJobByName')) 17 | const titleText = 'Find Job By Name Test' 18 | const jobName = 'rjqTestJob' 19 | let job = q.createJob().setName(jobName) 20 | job.data = tData 21 | job.title = titleText 22 | 23 | return q.reset().then((resetResult) => { 24 | t.ok(is.integer(resetResult), 'Queue reset') 25 | return q.addJob(job) 26 | }).then((savedJob1) => { 27 | t.equal(savedJob1[0].id, job.id, 'Job saved successfully') 28 | 29 | // ---------- Single Job Find Test ---------- 30 | t.comment('queue-find-job-by-name: Single Job Find') 31 | return queueFindJobByName(q, jobName) 32 | }).then((foundJobs1) => { 33 | t.equal(foundJobs1[0].name, jobName, 'Found job by name successfully') 34 | t.equal(foundJobs1[0].title, titleText, 'Job title is valid') 35 | t.equal(foundJobs1[0].data, tData, 'Job data is valid') 36 | 37 | // ---------- Single Raw Find Test ---------- 38 | t.comment('queue-find-job-by-name: Raw Job Find') 39 | return queueFindJobByName(q, jobName, true) 40 | }).then((foundJobs2) => { 41 | t.equal(foundJobs2[0].name, jobName, 'Raw found job by name successfully') 42 | t.equal(foundJobs2[0].title, titleText, 'Raw job title is valid') 43 | t.equal(foundJobs2[0].data, tData, 'Raw job data is valid') 44 | t.notOk(foundJobs2[0].q, 'Raw result does not have a q property') 45 | 46 | // ---------- Multiple Job Find Test ---------- 47 | t.comment('queue-find-job-by-name: Multiple Job Find') 48 | job = q.createJob().setName(jobName) 49 | job.data = tData 50 | job.title = titleText 51 | return q.addJob(job) 52 | }).then((savedJob2) => { 53 | t.equal(savedJob2[0].id, job.id, 'Job saved successfully') 54 | return queueFindJobByName(q, jobName) 55 | }).then((foundJobs3) => { 56 | t.equal(foundJobs3.length, 2, 'Found two jobs successfully') 57 | t.equal(foundJobs3[0].name, jobName, 'Found first job successfully') 58 | t.equal(foundJobs3[0].title, titleText, 'First job title is valid') 59 | t.equal(foundJobs3[0].data, tData, 'First job data is valid') 60 | t.equal(foundJobs3[1].name, jobName, 'Found second job successfully') 61 | t.equal(foundJobs3[1].title, titleText, 'Second job title is valid') 62 | t.equal(foundJobs3[1].data, tData, 'Second job data is valid') 63 | t.ok(is.dateBefore(foundJobs3[0].dateCreated, foundJobs3[1].dateCreated), 'Jobs are in valid order') 64 | 65 | // ---------- Zero Job Find Test ---------- 66 | t.comment('Zero Job Find') 67 | return queueFindJobByName(q, 'bogus') 68 | }).then((foundJobs4) => { 69 | t.equal(foundJobs4.length, 0, 'Zero jobs found successfully') 70 | 71 | q.stop() 72 | return resolve(t.end()) 73 | }).catch(err => tError(err, module, t)) 74 | }) 75 | }) 76 | } 77 | -------------------------------------------------------------------------------- /src/db-assert-index.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const enums = require('./enums') 4 | 5 | function createIndexActiveDateEnable (q) { 6 | logger('createIndexActiveDateEnable') 7 | const indexName = enums.index.indexActiveDateEnable 8 | return Promise.resolve().then(() => { 9 | return q.r.db(q.db) 10 | .table(q.name) 11 | .indexList() 12 | .contains(indexName) 13 | .run(q.queryRunOptions) 14 | }).then((exists) => { 15 | if (exists) { 16 | return exists 17 | } 18 | return q.r.db(q.db) 19 | .table(q.name) 20 | .indexCreate(indexName, function (row) { 21 | return q.r.branch( 22 | row('status').eq(enums.status.active), 23 | row('dateEnable'), 24 | null 25 | ) 26 | }) 27 | .run(q.queryRunOptions) 28 | }) 29 | } 30 | 31 | function createIndexInactivePriorityDateCreated (q) { 32 | logger('createIndexInactivePriorityDateCreated') 33 | const indexName = enums.index.indexInactivePriorityDateCreated 34 | return Promise.resolve().then(() => { 35 | return q.r.db(q.db) 36 | .table(q.name) 37 | .indexList() 38 | .contains(indexName) 39 | .run(q.queryRunOptions) 40 | }).then((exists) => { 41 | if (exists) { 42 | return exists 43 | } 44 | return q.r.db(q.db) 45 | .table(q.name) 46 | .indexCreate(indexName, function (row) { 47 | return q.r.branch( 48 | row('status').eq(enums.status.waiting), 49 | [ 50 | row('priority'), 51 | row('dateEnable'), 52 | row('dateCreated') 53 | ], 54 | row('status').eq(enums.status.failed), 55 | [ 56 | row('priority'), 57 | row('dateEnable'), 58 | row('dateCreated') 59 | ], 60 | null 61 | ) 62 | }) 63 | .run(q.queryRunOptions) 64 | }) 65 | } 66 | 67 | function createIndexFinishedDateFinished (q) { 68 | logger('createIndexFinishedDateFinished') 69 | const indexName = enums.index.indexFinishedDateFinished 70 | return Promise.resolve().then(() => { 71 | return q.r.db(q.db) 72 | .table(q.name) 73 | .indexList() 74 | .contains(indexName) 75 | .run(q.queryRunOptions) 76 | }).then((exists) => { 77 | if (exists) { 78 | return exists 79 | } 80 | return q.r.db(q.db) 81 | .table(q.name) 82 | .indexCreate(indexName, function (row) { 83 | return q.r.branch( 84 | row('status').eq(enums.status.completed), 85 | row('dateFinished'), 86 | row('status').eq(enums.status.cancelled), 87 | row('dateFinished'), 88 | row('status').eq(enums.status.terminated), 89 | row('dateFinished'), 90 | null 91 | ) 92 | }) 93 | .run(q.queryRunOptions) 94 | }) 95 | } 96 | 97 | function createIndexName (q) { 98 | logger('createIndexName') 99 | const indexName = enums.index.indexName 100 | return Promise.resolve().then(() => { 101 | return q.r.db(q.db) 102 | .table(q.name) 103 | .indexList() 104 | .contains(indexName) 105 | .run(q.queryRunOptions) 106 | }).then((exists) => { 107 | if (exists) { 108 | return exists 109 | } 110 | return q.r.db(q.db) 111 | .table(q.name) 112 | .indexCreate(indexName) 113 | .run(q.queryRunOptions) 114 | }) 115 | } 116 | 117 | function createIndexStatus (q) { 118 | logger('createIndexStatus') 119 | const indexName = enums.index.indexStatus 120 | return Promise.resolve().then(() => { 121 | return q.r.db(q.db) 122 | .table(q.name) 123 | .indexList() 124 | .contains(indexName) 125 | .run(q.queryRunOptions) 126 | }).then((exists) => { 127 | if (exists) { 128 | return exists 129 | } 130 | return q.r.db(q.db) 131 | .table(q.name) 132 | .indexCreate(indexName) 133 | .run(q.queryRunOptions) 134 | }) 135 | } 136 | 137 | module.exports = function assertIndex (q) { 138 | logger('assertIndex') 139 | return Promise.all([ 140 | createIndexActiveDateEnable(q), 141 | createIndexInactivePriorityDateCreated(q), 142 | createIndexFinishedDateFinished(q), 143 | createIndexName(q), 144 | createIndexStatus(q) 145 | ]).then((indexCreateResult) => { 146 | logger('Waiting for index...') 147 | return q.r.db(q.db) 148 | .table(q.name) 149 | .indexWait() 150 | .run(q.queryRunOptions) 151 | }).then(() => { 152 | logger('Indexes ready.') 153 | return true 154 | }) 155 | } 156 | -------------------------------------------------------------------------------- /tests/queue-stop.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const is = require('../src/is') 4 | const tError = require('./test-error') 5 | const queueStop = require('../src/queue-stop') 6 | const queueDb = require('../src/queue-db') 7 | const dbReview = require('../src/db-review') 8 | const Queue = require('../src/queue') 9 | const simulateJobProcessing = require('./test-utils').simulateJobProcessing 10 | const tOpts = require('./test-options') 11 | const eventHandlers = require('./test-event-handlers') 12 | const testName = 'queue-stop' 13 | 14 | queueStopTests() 15 | function queueStopTests () { 16 | return new Promise((resolve, reject) => { 17 | test(testName, (t) => { 18 | t.plan(79) 19 | 20 | const tableName = 'queueStop' 21 | let q = new Queue(tOpts.cxn(), tOpts.master(tableName, 999999)) 22 | 23 | // ---------- Event Handler Setup ---------- 24 | let state = { 25 | testName, 26 | enabled: false, 27 | ready: 0, 28 | processing: 0, 29 | progress: 0, 30 | pausing: 1, 31 | paused: 1, 32 | resumed: 0, 33 | removed: 0, 34 | reset: 0, 35 | error: 0, 36 | reviewed: 0, 37 | detached: 1, 38 | stopping: 1, 39 | stopped: 1, 40 | dropped: 0, 41 | added: 0, 42 | waiting: 0, 43 | active: 0, 44 | completed: 0, 45 | cancelled: 0, 46 | failed: 0, 47 | terminated: 0, 48 | reanimated: 0, 49 | log: 0, 50 | updated: 0 51 | } 52 | 53 | return q.reset().then((resetResult) => { 54 | t.ok(is.integer(resetResult), 'Queue reset') 55 | return q.ready() 56 | }).then((ready) => { 57 | t.ok(ready, 'Queue in a ready state') 58 | t.ok(dbReview.isEnabled(q), 'Review is enabled') 59 | t.ok(q._changeFeedCursor.connection.open, 'Change feed is connected') 60 | t.notOk(q.paused, 'Queue is not paused') 61 | eventHandlers.add(t, q, state) 62 | 63 | // ---------- Stop with Drain ---------- 64 | t.comment('queue-stop: Stop with Drain') 65 | simulateJobProcessing(q) 66 | return queueStop(q) 67 | }).then((stopped) => { 68 | return queueDb.drain(q) 69 | }).then((stopped) => { 70 | t.ok(stopped, 'Queue stopped with pool drain') 71 | t.notOk(dbReview.isEnabled(q), 'Review is disabled') 72 | t.notOk(q._changeFeedCursor, 'Change feed is disconnected') 73 | t.ok(q.paused, 'Queue is paused') 74 | return q.ready() 75 | }).then((ready) => { 76 | t.notOk(ready, 'Queue ready returns false') 77 | 78 | // ---------- Event Summary ---------- 79 | eventHandlers.remove(t, q, state) 80 | 81 | // ---------- Stop without Drain ---------- 82 | t.comment('queue-stop: Stop without Drain') 83 | q = new Queue(tOpts.cxn(), tOpts.master(tableName, 999999)) 84 | return q.ready() 85 | }).then((ready) => { 86 | t.ok(ready, 'Queue in a ready state') 87 | eventHandlers.add(t, q, state) 88 | t.ok(dbReview.isEnabled(q), 'Review is enabled') 89 | t.ok(q._changeFeedCursor.connection.open, 'Change feed is connected') 90 | t.notOk(q.paused, 'Queue is not paused') 91 | simulateJobProcessing(q) 92 | return queueStop(q) 93 | }).then((stopped2) => { 94 | t.ok(stopped2, 'Queue stopped without pool drain') 95 | t.notOk(dbReview.isEnabled(q), 'Review is disabled') 96 | t.notOk(q._changeFeedCursor, 'Change feed is disconnected') 97 | t.ok(q.paused, 'Queue is paused') 98 | return q.ready() 99 | }).then((ready) => { 100 | t.ok(ready, 'Queue is still ready') 101 | // detaching with drain or node will not exit gracefully 102 | return queueDb.detach(q) 103 | }).then(() => { 104 | return queueDb.drain(q) 105 | }).then(() => { 106 | return queueDb.attach(q, tOpts.cxn()) 107 | }).then(() => { 108 | return q.ready() 109 | }).then((ready) => { 110 | t.ok(ready, 'Queue in a ready state') 111 | return q.resume() 112 | }).then(() => { 113 | t.ok(dbReview.isEnabled(q), 'Review is enabled') 114 | t.ok(q._changeFeedCursor.connection.open, 'Change feed is connected') 115 | t.notOk(q.paused, 'Queue is not paused') 116 | 117 | // ---------- Event Summary ---------- 118 | eventHandlers.remove(t, q, state) 119 | q.stop() 120 | return resolve(t.end()) 121 | }).catch(err => tError(err, module, t)) 122 | }) 123 | }) 124 | } 125 | -------------------------------------------------------------------------------- /src/enums.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const state = Object.freeze({ 3 | docId: '86f6ff5b-0c4e-46ad-9a5f-e90eb19c9b00', 4 | global: 'global', 5 | local: 'local' 6 | }) 7 | const priority = Object.freeze({ 8 | lowest: 60, 9 | low: 50, 10 | normal: 40, 11 | medium: 30, 12 | high: 20, 13 | highest: 10 14 | }) 15 | const status = Object.freeze({ 16 | // ---------- Queue Status Values ---------- 17 | ready: 'ready', 18 | processing: 'processing', 19 | progress: 'progress', 20 | pausing: 'pausing', 21 | paused: 'paused', 22 | resumed: 'resumed', 23 | removed: 'removed', 24 | idle: 'idle', 25 | reset: 'reset', 26 | error: 'error', 27 | reviewed: 'reviewed', 28 | detached: 'detached', 29 | stopping: 'stopping', 30 | stopped: 'stopped', 31 | dropped: 'dropped', 32 | // ---------- Job Status Values ---------- 33 | created: 'created', // Non-event, initial create job status 34 | added: 'added', // Event only, not a job status 35 | waiting: 'waiting', // Non-event, job status only 36 | active: 'active', 37 | completed: 'completed', 38 | cancelled: 'cancelled', 39 | failed: 'failed', 40 | terminated: 'terminated', 41 | reanimated: 'reanimated', 42 | log: 'log', // Event only, not a job status 43 | updated: 'updated' // Event only, not a job status 44 | }) 45 | const options = Object.freeze({ 46 | name: 'rjqJobList', 47 | host: 'localhost', 48 | port: 28015, 49 | db: 'rjqJobQueue', 50 | queryRunOptions: { readMode: 'majority' }, 51 | databaseInitDelay: 1000, 52 | masterInterval: 310000, // 5 minutes and 10 seconds 53 | priority: 'normal', 54 | timeout: 300000, // 5 minutes 55 | retryMax: 3, 56 | retryDelay: 600000, // 10 minutes 57 | repeat: false, 58 | repeatDelay: 300000, // 5 minutes 59 | concurrency: 1, 60 | limitJobLogs: 1000, 61 | removeFinishedJobs: 15552000000 // 180 days 62 | }) 63 | const index = Object.freeze({ 64 | indexActiveDateEnable: 'indexActiveDateEnable', 65 | indexInactivePriorityDateCreated: 'indexInactivePriorityDateCreated', 66 | indexFinishedDateFinished: 'indexFinishedDateFinished', 67 | indexName: 'name', 68 | indexStatus: 'status' 69 | }) 70 | const dbResult = Object.freeze({ 71 | deleted: 'deleted', 72 | errors: 'errors', 73 | inserted: 'inserted', 74 | replaced: 'replaced', 75 | skipped: 'skipped', 76 | changes: 'changes', 77 | unchanged: 'unchanged' 78 | }) 79 | const log = Object.freeze({ 80 | information: 'information', 81 | warning: 'warning', 82 | error: 'error' 83 | }) 84 | const message = Object.freeze({ 85 | jobAdded: 'Job added to the queue', 86 | active: 'Job retrieved and active', 87 | completed: 'Job completed successfully', 88 | failed: 'Job processing failed', 89 | cancel: 'Job cancelled by Queue process handler', 90 | seeLogData: 'See the data attached to this log entry', 91 | jobUpdated: 'Job updated. Old values in log data', 92 | jobPassBack: 'Job has not been processed to completion and is being placed back into the queue', 93 | jobProgress: 'Job progress updated. Old value in log data', 94 | jobNotActive: 'Job is not at an active status', 95 | jobNotAdded: 'Job not added to the queue', 96 | jobInvalid: 'Job object is invalid', 97 | jobReanimated: 'Job has been reanimated', 98 | jobLogsTruncated: 'Job logs have been truncated', 99 | processTwice: 'Cannot call queue process twice', 100 | idInvalid: 'The job id is invalid', 101 | nameInvalid: 'The job name must be a string', 102 | priorityInvalid: 'The job priority value is invalid', 103 | timeoutInvalid: 'The job timeout value is invalid', 104 | retryMaxIvalid: 'The job retryMax value is invalid', 105 | retryDelayIvalid: 'The job retryDelay value is invalid', 106 | repeatInvalid: 'The job repeat value is invalid', 107 | repeatDelayInvalid: 'The job repeatDelay value is invalid', 108 | dateEnableIvalid: 'The job dateEnable value is invalid', 109 | dbError: 'RethinkDB returned an error', 110 | concurrencyInvalid: 'Invalid concurrency value', 111 | cancelCallbackInvalid: 'The onCancel callback is not a function', 112 | globalStateError: 'The global state document change feed is invalid', 113 | datetimeInvalid: 'Invalid datetime arguments', 114 | noErrorStack: 'The error has no stack detail', 115 | noErrorMessage: 'The error has no message' 116 | }) 117 | 118 | const enums = module.exports = { 119 | priorityFromValue (value) { 120 | logger(`priorityFromValue: [${value}]`) 121 | return Object.keys(enums.priority).find(key => enums.priority[key] === value) 122 | }, 123 | state, 124 | priority, 125 | status, 126 | options, 127 | index, 128 | dbResult, 129 | log, 130 | message 131 | } 132 | -------------------------------------------------------------------------------- /tests/queue-reanimate-job.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const is = require('../src/is') 4 | const enums = require('../src/enums') 5 | const tError = require('./test-error') 6 | const tOpts = require('./test-options') 7 | const queueReanimateJob = require('../src/queue-reanimate-job') 8 | const Queue = require('../src/queue') 9 | const eventHandlers = require('./test-event-handlers') 10 | const testName = 'queue-reanimate' 11 | 12 | queueReanimateJobTests() 13 | function queueReanimateJobTests () { 14 | return new Promise((resolve, reject) => { 15 | test(testName, (t) => { 16 | t.plan(49) 17 | 18 | // ---------- Test Setup ---------- 19 | const q = new Queue(tOpts.cxn(), tOpts.default('queueReanimateJob')) 20 | 21 | // ---------- Event Handler Setup ---------- 22 | let state = { 23 | testName, 24 | enabled: false, 25 | ready: 0, 26 | processing: 0, 27 | progress: 0, 28 | pausing: 0, 29 | paused: 0, 30 | resumed: 0, 31 | removed: 0, 32 | idle: 0, 33 | reset: 0, 34 | error: 0, 35 | reviewed: 0, 36 | detached: 0, 37 | stopping: 0, 38 | stopped: 0, 39 | dropped: 0, 40 | added: 1, 41 | waiting: 0, 42 | active: 0, 43 | completed: 0, 44 | cancelled: 1, 45 | failed: 0, 46 | terminated: 0, 47 | reanimated: 1, 48 | log: 0, 49 | updated: 1 50 | } 51 | 52 | // ---------- Test Setup ---------- 53 | let job = q.createJob() 54 | let dateEnable 55 | let newDateEnable = new Date() 56 | newDateEnable.setDate(newDateEnable.getDate() + 5) 57 | 58 | return q.reset().then((resetResult) => { 59 | t.ok(is.integer(resetResult), 'Queue reset') 60 | return q.pause() 61 | }).then(() => { 62 | eventHandlers.add(t, q, state) 63 | 64 | // ---------- Reanimate Job Tests ---------- 65 | t.comment('queue-reanimate-job: Reanimate Cancelled Job') 66 | return q.addJob(job) 67 | }).then((result) => { 68 | t.equal(result.length, 1, `Job saved successfully`) 69 | return q.getJob(job.id) 70 | }).then((result) => { 71 | dateEnable = result[0].dateEnable.toString() 72 | result[0].progress = 50 73 | result[0].retryCount = 2 74 | return result[0].update() 75 | }).then((result) => { 76 | return q.cancelJob(job.id) 77 | }).then((result) => { 78 | return q.getJob(job.id) 79 | }).then((result) => { 80 | t.equal(result[0].dateEnable.toString(), dateEnable, 'Job dateEnable is valid') 81 | t.equal(result[0].log.length, 3, 'Job log is valid') 82 | t.equal(result[0].progress, 50, 'Job progress valid') 83 | t.equal(result[0].queueId, q.id, 'Job queueId is valid') 84 | t.equal(result[0].retryCount, 2, 'Job retryCount valid') 85 | t.equal(result[0].status, enums.status.cancelled, 'Job is cancelled') 86 | return queueReanimateJob(q, job.id, newDateEnable) 87 | }).then((result) => { 88 | t.ok(is.uuid(result[0]), 'Reanimate jobs returns job ids') 89 | return q.getJob(job.id) 90 | }).then((result) => { 91 | t.equal(result[0].dateEnable.toString(), newDateEnable.toString(), 'Job reanimate dateEnable is valid') 92 | t.equal(result[0].log.length, 4, 'Job reanimate log is valid') 93 | t.equal(result[0].progress, 0, 'Job reanimate progress valid') 94 | t.equal(result[0].queueId, q.id, 'Job reanimate queueId is valid') 95 | t.equal(result[0].retryCount, 0, 'Job reanimate retryCount valid') 96 | t.equal(result[0].status, enums.status.waiting, 'Job reanimate is waiting') 97 | let lastLog = result[0].getLastLog() 98 | t.ok(is.date(lastLog.date), 'Job reanimate log date is valid') 99 | t.equal(lastLog.message, enums.message.jobReanimated, 'Job reanimate log message is valid') 100 | t.equal(lastLog.queueId, q.id, 'Job reanimate log queueId is valid') 101 | t.equal(lastLog.retryCount, 0, 'Job reanimate log retryCount is valid') 102 | t.equal(lastLog.status, enums.status.waiting, 'Job reanimate log status is valid') 103 | t.equal(lastLog.type, enums.log.information, 'Job reanimate log type is valid') 104 | 105 | // ---------- Event Summary ---------- 106 | eventHandlers.remove(t, q, state) 107 | 108 | return q.reset() 109 | }).then((resetResult) => { 110 | t.ok(resetResult >= 0, 'Queue reset') 111 | q.stop() 112 | return resolve(t.end()) 113 | }).catch(err => tError(err, module, t)) 114 | }) 115 | }) 116 | } 117 | -------------------------------------------------------------------------------- /index.d.ts: -------------------------------------------------------------------------------- 1 | // Type definitions for rethinkdb-job-queue 2 | // Project: https://github.com/grantcarthew/node-rethinkdb-job-queue 3 | import { EventEmitter } from 'events'; 4 | 5 | interface PredicateFunction

{ 6 | (job: P): boolean; 7 | } 8 | 9 | declare class Queue

extends EventEmitter { 10 | public readonly name: string; 11 | public readonly id: string; 12 | public readonly host: string; 13 | public readonly port: number; 14 | public readonly db: string; 15 | public readonly r: any; // Actually RethinkDbDash handle 16 | public readonly changeFeed: boolean; 17 | public readonly master: boolean; 18 | public readonly masterInterval: boolean | number; 19 | public jobOptions: Queue.JobOptions; 20 | public readonly removeFinishedJobs: boolean | number; 21 | public readonly running: number; 22 | public concurrency: number; 23 | public readonly paused: boolean; 24 | public readonly idle: boolean; 25 | constructor(cxOptions?: Queue.ConnectionOptions, qOptions?: Queue.QueueOptions); 26 | public addJob(job: P | P[]): Promise; 27 | public cancelJob(job: string | P | P[]): Promise; 28 | public containsJobByName(name: string, raw?: boolean): Promise; 29 | public createJob(jobData?: object): P; 30 | public drop(): Promise; 31 | public findJob(predicate: Object | PredicateFunction

, raw?: boolean): Promise; 32 | public findJobByName(name: string, raw?: boolean): Promise; 33 | public getJob(job: string | P | P[]): Promise; 34 | public pause(global: boolean): Promise; 35 | public process(handler: Queue.ProcessCallback

): Promise; 36 | public ready(): Promise; 37 | public reanimateJob(job: string | P | P[], dateEnable?: Date): Promise; 38 | public removeJob(job: string | P | P[]): Promise; 39 | public reset(): Promise; 40 | public resume(global: boolean): Promise; 41 | public review(): Promise; 42 | public stop(): Promise; 43 | public summary(): Promise; 44 | } 45 | 46 | declare namespace Queue { 47 | interface ConnectionOptions { 48 | host?: string; 49 | port?: number; 50 | db?: string; 51 | } 52 | 53 | interface QueueOptions { 54 | name?: string; 55 | databaseInitDelay?: number; 56 | queryRunOptions?: boolean; 57 | changeFeed?: boolean; 58 | concurrency?: number; 59 | masterInterval?: number; 60 | limitJobLogs?: number; 61 | removeFinishedJobs?: number | boolean; 62 | } 63 | 64 | export interface NextCallback

{ 65 | (error?: Error, jobResult?: P | string): void; 66 | } 67 | 68 | export interface ProcessCallback

{ 69 | (job: P, 70 | next: NextCallback

, 71 | onCancel: (job: P, cancellationCallback?: () => void) => void): void; 72 | } 73 | 74 | enum Priority { 75 | lowest, 76 | low, 77 | normal, 78 | medium, 79 | high, 80 | highest, 81 | } 82 | 83 | interface JobOptions { 84 | name?: string; 85 | priority?: Priority; 86 | timeout?: number; 87 | retryDelay?: number; 88 | retryMax?: number; 89 | repeat?: boolean | number; 90 | repeatDelay?: number; 91 | dateEnable?: Date; 92 | } 93 | 94 | export abstract class Job { 95 | public id?: string; 96 | public name?: string; 97 | public priority?: Priority; 98 | public timeout?: number; 99 | public retryDelay?: number; 100 | public retryMax?: number; 101 | public retryCount?: number; 102 | public repeat?: boolean | number; 103 | public repeatDelay?: number; 104 | public processCount?: number; 105 | public progress?: number; 106 | public status?: string; 107 | public log?: LogEntry[]; 108 | public dateCreated?: Date; 109 | public dateEnable?: Date; 110 | public dateStarted?: Date; 111 | public dateFinished?: Date; 112 | public queueId?: string; 113 | public setName(name: string): this; 114 | public setPriority(priority: Priority): this; 115 | public setTimeout(timeout: number): this; 116 | public setDateEnable(dateEnable: Date): this; 117 | public setRetryMax(retryMax: number): this; 118 | public setRetryDelay(retryDelay: number): this; 119 | public setRepeat(repeat: boolean | number): this; 120 | public setRepeatDelay(repeatDelay: number): this; 121 | public updateProgress(percent: number): this; 122 | public update(): Promise; 123 | public getCleanCopy(priorityAsString?: boolean): object; 124 | public addLog(data?: object, message?: string, type?: string, status?: string): Promise; 125 | public getLastLog(): LogEntry; 126 | } 127 | 128 | export interface LogEntry { 129 | date: Date; 130 | message?: string; 131 | queueId: string; 132 | status?: string; 133 | type?: string; 134 | data?: Object; 135 | // retryCount 136 | } 137 | } 138 | 139 | export = Queue; 140 | -------------------------------------------------------------------------------- /tests/queue-change.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const is = require('../src/is') 4 | const tError = require('./test-error') 5 | const Queue = require('../src/queue') 6 | const dbReview = require('../src/db-review') 7 | const tOpts = require('./test-options') 8 | const eventHandlers = require('./test-event-handlers') 9 | const testName = 'queue-change' 10 | 11 | queueChangeTests() 12 | function queueChangeTests () { 13 | return new Promise((resolve, reject) => { 14 | test(testName, (t) => { 15 | t.plan(62) 16 | 17 | const tableName = 'queueChange' 18 | const q = new Queue(tOpts.cxn(), tOpts.default(tableName)) 19 | let qPub 20 | 21 | // ---------- Event Handler Setup ---------- 22 | let state = { 23 | testName, 24 | enabled: false, 25 | ready: 1, 26 | processing: 3, 27 | progress: 3, 28 | pausing: 3, 29 | paused: 3, 30 | resumed: 2, 31 | removed: 1, 32 | reset: 1, 33 | error: 0, 34 | reviewed: 2, 35 | detached: 0, 36 | stopping: 0, 37 | stopped: 0, 38 | dropped: 0, 39 | added: 3, 40 | waiting: 0, 41 | active: 3, 42 | completed: 1, 43 | cancelled: 1, 44 | failed: 1, 45 | terminated: 1, 46 | reanimated: 0, 47 | log: 1, 48 | updated: 0 49 | } 50 | 51 | let job 52 | let processDelay = 500 53 | 54 | eventHandlers.add(t, q, state) 55 | 56 | return Promise.resolve( 57 | q.ready() 58 | ).then((readyResult) => { 59 | return q.reset() 60 | }).then((resetResult) => { 61 | t.ok(is.integer(resetResult), 'Queue reset') 62 | qPub = new Queue(tOpts.cxn(), tOpts.default(tableName)) 63 | return qPub.r.table(tableName).wait() 64 | }).then((waitResult) => { 65 | job = qPub.createJob() 66 | return q.pause() 67 | }).then(() => { 68 | q.process((j, next) => { 69 | setTimeout(function jobProcessing () { 70 | t.equal(j.id, job.id, `Job Processed [${j.id}]`) 71 | next(null, 'queue-change') 72 | }, processDelay) 73 | return j.updateProgress(50) 74 | }) 75 | 76 | // ---------- Test added, active, progress completed, removed ---------- 77 | t.comment('queue-change: added, active, progress, completed, and removed change events') 78 | }).then(() => { 79 | return qPub.addJob(job) 80 | }).then((savedJob) => { 81 | t.equal(savedJob[0].id, job.id, 'Job saved successfully') 82 | return q.resume() 83 | }).then(() => { 84 | t.ok(!q.paused, 'Queue not paused') 85 | }).delay(processDelay).then(() => { 86 | return q.pause() 87 | }).delay(processDelay).then(() => { 88 | return q.removeJob(job.id) 89 | }).delay(processDelay).then(() => { 90 | // 91 | // ---------- Test global review ---------- 92 | t.comment('queue-change: global review') 93 | // The following will raise a 'reviewed' event. 94 | return dbReview.runOnce(qPub) 95 | }).then(() => { 96 | // 97 | // ---------- Test failed and terminated ---------- 98 | t.comment('queue-change: failed and terminated change events') 99 | job = qPub.createJob() 100 | job.timeout = processDelay / 2000 101 | job.retryDelay = 0 102 | job.retryMax = 1 103 | return qPub.addJob(job) 104 | }).then((savedJob) => { 105 | t.equal(savedJob[0].id, job.id, 'Job saved successfully') 106 | return q.resume() 107 | }).then(() => { 108 | t.ok(!q.paused, 'Queue not paused') 109 | }).delay(processDelay * 2).then(() => { 110 | return q.pause() 111 | }).delay(processDelay).then(() => { 112 | job = qPub.createJob() 113 | 114 | // ---------- Test log and cancelled ---------- 115 | t.comment('queue-change: log and cancelled change events') 116 | return qPub.addJob(job) 117 | }).then((savedJob) => { 118 | t.equal(savedJob[0].id, job.id, 'Job saved successfully') 119 | return savedJob[0].addLog(null, 'test log') 120 | }).then(() => { 121 | return qPub.cancelJob(job.id, 'testing') 122 | }).delay(processDelay).then(() => { 123 | // 124 | // ---------- Event Summary ---------- 125 | eventHandlers.remove(t, q, state) 126 | 127 | return q.reset() 128 | }).then((resetResult) => { 129 | return Promise.all([ 130 | q.stop(), 131 | qPub.stop() 132 | ]) 133 | }).then(() => { 134 | return resolve(t.end()) 135 | }).catch(err => tError(err, module, t)) 136 | }) 137 | }) 138 | } 139 | -------------------------------------------------------------------------------- /src/db-review.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const datetime = require('./datetime') 4 | const dbResult = require('./db-result') 5 | const queueProcess = require('./queue-process') 6 | const queueState = require('./queue-state') 7 | const is = require('./is') 8 | const enums = require('./enums') 9 | const dbReviewIntervalList = new Map() 10 | 11 | function updateFailedJobs (q) { 12 | logger('updateFailedJobs: ' + datetime.format(new Date())) 13 | 14 | return Promise.resolve().then(() => { 15 | return q.r.db(q.db) 16 | .table(q.name) 17 | .orderBy({index: enums.index.indexActiveDateEnable}) 18 | .filter( 19 | q.r.row('dateEnable').lt(q.r.now()) 20 | ) 21 | .update({ 22 | status: q.r.branch( 23 | q.r.row('retryCount').lt(q.r.row('retryMax')), 24 | enums.status.failed, 25 | enums.status.terminated 26 | ), 27 | dateFinished: q.r.now(), 28 | retryCount: q.r.branch( 29 | q.r.row('retryCount').lt(q.r.row('retryMax')), 30 | q.r.row('retryCount').add(1), 31 | q.r.row('retryCount') 32 | ), 33 | log: q.r.row('log').append({ 34 | date: q.r.now(), 35 | queueId: q.id, 36 | type: q.r.branch( 37 | q.r.row('retryCount').lt(q.r.row('retryMax')), 38 | enums.log.warning, 39 | enums.log.error 40 | ), 41 | status: q.r.branch( 42 | q.r.row('retryCount').lt(q.r.row('retryMax')), 43 | enums.status.failed, 44 | enums.status.terminated 45 | ), 46 | retryCount: q.r.row('retryCount'), 47 | processCount: q.r.row('processCount'), 48 | message: `Master: ${enums.message.failed}`, 49 | dateEnable: q.r.row('dateEnable') 50 | }), 51 | queueId: q.id 52 | }) 53 | .run(q.queryRunOptions) 54 | }).then((updateResult) => { 55 | logger(`updateResult`, updateResult) 56 | return dbResult.status(updateResult, enums.dbResult.replaced) 57 | }) 58 | } 59 | 60 | function removeFinishedJobsBasedOnTime (q) { 61 | logger('removeFinishedJobsBasedOnTime') 62 | return q.r.db(q.db) 63 | .table(q.name) 64 | .orderBy({index: enums.index.indexFinishedDateFinished}) 65 | .filter( 66 | q.r.row('dateFinished').add( 67 | q.r.expr(q.removeFinishedJobs).div(1000) 68 | ).lt(q.r.now()) 69 | ) 70 | .delete() 71 | .run(q.queryRunOptions) 72 | } 73 | 74 | function removeFinishedJobsBasedOnNow (q) { 75 | logger('removeFinishedJobsBasedOnNow') 76 | return q.r.db(q.db) 77 | .table(q.name) 78 | .orderBy({index: enums.index.indexFinishedDateFinished}) 79 | .filter( 80 | q.r.row('dateFinished').lt(q.r.now()) 81 | ) 82 | .delete() 83 | .run(q.queryRunOptions) 84 | } 85 | 86 | function removeFinishedJobs (q) { 87 | logger('removeFinishedJobs: ' + datetime.format(new Date())) 88 | 89 | if (q.removeFinishedJobs < 1 || q.removeFinishedJobs === false) { 90 | return Promise.resolve(0) 91 | } 92 | 93 | return Promise.resolve().then(() => { 94 | if (is.true(q.removeFinishedJobs)) { 95 | return removeFinishedJobsBasedOnNow(q) 96 | } 97 | return removeFinishedJobsBasedOnTime(q) 98 | }).then((deleteResult) => { 99 | logger(`deleteResult`, deleteResult) 100 | return dbResult.status(deleteResult, enums.dbResult.deleted) 101 | }) 102 | } 103 | 104 | function runReviewTasks (q) { 105 | logger(`runReviewTasks`) 106 | return Promise.props({ 107 | reviewed: updateFailedJobs(q), 108 | removed: removeFinishedJobs(q) 109 | }).then((runReviewTasksResult) => { 110 | runReviewTasksResult.local = true 111 | logger(`Event: reviewed`, runReviewTasksResult) 112 | q.emit(enums.status.reviewed, q.id, runReviewTasksResult) 113 | queueProcess.restart(q) 114 | return Promise.props({ 115 | queueStateChange: queueState(q, enums.status.reviewed), 116 | reviewResult: runReviewTasksResult 117 | }) 118 | }).then((stateChangeAndReviewResult) => { 119 | return stateChangeAndReviewResult.reviewResult 120 | }) 121 | } 122 | 123 | module.exports.enable = function enable (q) { 124 | logger('enable', q.masterInterval) 125 | if (!dbReviewIntervalList.has(q.id)) { 126 | const interval = q.masterInterval 127 | dbReviewIntervalList.set(q.id, setInterval(() => { 128 | return runReviewTasks(q) 129 | }, interval)) 130 | } 131 | return true 132 | } 133 | 134 | module.exports.disable = function disable (q) { 135 | logger('disable', q.id) 136 | if (dbReviewIntervalList.has(q.id)) { 137 | clearInterval(dbReviewIntervalList.get(q.id)) 138 | dbReviewIntervalList.delete(q.id) 139 | } 140 | return true 141 | } 142 | 143 | module.exports.runOnce = function run (q) { 144 | logger('runOnce') 145 | return runReviewTasks(q) 146 | } 147 | 148 | module.exports.isEnabled = function reviewIsEnabled (q) { 149 | logger('isEnabled', q.id) 150 | return dbReviewIntervalList.has(q.id) 151 | } 152 | -------------------------------------------------------------------------------- /src/job.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const uuid = require('uuid') 3 | const enums = require('./enums') 4 | const is = require('./is') 5 | const errorBooster = require('./error-booster') 6 | const jobOptions = require('./job-options') 7 | const jobProgress = require('./job-progress') 8 | const jobUpdate = require('./job-update') 9 | const jobLog = require('./job-log') 10 | 11 | class Job { 12 | constructor (q, jobData) { 13 | logger('constructor') 14 | logger('queue id', q.id) 15 | logger('jobData', jobData) 16 | this.q = q 17 | 18 | if (is.job(jobData)) { 19 | logger('Creating job from data') 20 | Object.assign(this, jobData) 21 | this.priority = enums.priorityFromValue(this.priority) 22 | } else { 23 | logger('Creating new job from defaults') 24 | 25 | const options = jobOptions() 26 | const now = new Date() 27 | const newId = uuid.v4() 28 | this.id = newId 29 | this.name = options.name || newId 30 | this.priority = options.priority 31 | this.timeout = options.timeout 32 | this.retryDelay = options.retryDelay 33 | this.retryMax = options.retryMax 34 | this.retryCount = 0 35 | this.repeat = options.repeat 36 | this.repeatDelay = options.repeatDelay 37 | this.processCount = 0 38 | this.progress = 0 39 | this.status = enums.status.created 40 | this.log = [] 41 | this.dateCreated = now 42 | this.dateEnable = now 43 | this.dateStarted = null 44 | this.dateFinished = null 45 | this.queueId = q.id 46 | // Conflicting job options will be overwritten. 47 | if (is.function(jobData)) { 48 | throw new Error(enums.message.jobDataInvalid) 49 | } 50 | if (is.array(jobData) || !is.object(jobData)) { 51 | this.data = jobData 52 | } else { 53 | Object.assign(this, jobData) 54 | } 55 | } 56 | } 57 | 58 | setName (newName) { 59 | if (is.string(newName)) { 60 | this.name = newName 61 | return this 62 | } 63 | throw new Error(enums.message.nameInvalid) 64 | } 65 | 66 | setPriority (newPriority) { 67 | if (Object.keys(enums.priority).includes(newPriority)) { 68 | this.priority = newPriority 69 | return this 70 | } 71 | throw new Error(enums.message.priorityInvalid) 72 | } 73 | 74 | setTimeout (newTimeout) { 75 | if (is.integer(newTimeout) && 76 | newTimeout >= 0) { 77 | this.timeout = newTimeout 78 | return this 79 | } 80 | throw new Error(enums.message.timeoutIvalid) 81 | } 82 | 83 | setRetryMax (newRetryMax) { 84 | if (is.integer(newRetryMax) && 85 | newRetryMax >= 0) { 86 | this.retryMax = newRetryMax 87 | return this 88 | } 89 | throw new Error(enums.message.retryMaxIvalid) 90 | } 91 | 92 | setRetryDelay (newRetryDelay) { 93 | if (is.integer(newRetryDelay) && 94 | newRetryDelay >= 0) { 95 | this.retryDelay = newRetryDelay 96 | return this 97 | } 98 | throw new Error(enums.message.retryDelayIvalid) 99 | } 100 | 101 | setRepeat (newRepeat) { 102 | if (is.boolean(newRepeat) || ( 103 | is.integer(newRepeat) && newRepeat >= 0)) { 104 | this.repeat = newRepeat 105 | return this 106 | } 107 | throw new Error(enums.message.repeatInvalid) 108 | } 109 | 110 | setRepeatDelay (newRepeatDelay) { 111 | if (is.integer(newRepeatDelay) && newRepeatDelay >= 0) { 112 | this.repeatDelay = newRepeatDelay 113 | return this 114 | } 115 | throw new Error(enums.message.repeatDelayInvalid) 116 | } 117 | 118 | setDateEnable (newDateEnable) { 119 | if (is.date(newDateEnable)) { 120 | this.dateEnable = newDateEnable 121 | return this 122 | } 123 | throw new Error(enums.message.dateEnableIvalid) 124 | } 125 | 126 | updateProgress (percent) { 127 | logger(`updateProgress [${percent}]`) 128 | return this.q.ready().then(() => { 129 | return jobProgress(this, percent) 130 | }).catch(errorBooster(this, logger, 'updateProgress')) 131 | } 132 | 133 | update () { 134 | logger(`update`) 135 | return this.q.ready().then(() => { 136 | return jobUpdate(this) 137 | }).catch(errorBooster(this, logger, 'update')) 138 | } 139 | 140 | getCleanCopy (priorityAsString) { 141 | logger('getCleanCopy') 142 | const jobCopy = Object.assign({}, this) 143 | if (!priorityAsString) { 144 | jobCopy.priority = enums.priority[jobCopy.priority] 145 | } 146 | delete jobCopy.q 147 | return jobCopy 148 | } 149 | 150 | addLog (data = {}, 151 | message = enums.message.seeLogData, 152 | type = enums.log.information, 153 | status = this.status) { 154 | logger('addLog', data, message, type, status) 155 | return this.q.ready().then(() => { 156 | return jobLog.commitLog(this, data, message, type, status) 157 | }).catch(errorBooster(this, logger, 'addLog')) 158 | } 159 | 160 | getLastLog () { 161 | return jobLog.getLastLog(this) 162 | } 163 | } 164 | 165 | module.exports = Job 166 | -------------------------------------------------------------------------------- /tests/queue-cancel-job.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const is = require('../src/is') 4 | const tError = require('./test-error') 5 | const enums = require('../src/enums') 6 | const queueCancelJob = require('../src/queue-cancel-job') 7 | const tData = require('./test-options').tData 8 | const Queue = require('../src/queue') 9 | const tOpts = require('./test-options') 10 | const eventHandlers = require('./test-event-handlers') 11 | const testName = 'queue-cancel-job' 12 | 13 | queueCancelJobTests() 14 | function queueCancelJobTests () { 15 | return new Promise((resolve, reject) => { 16 | test(testName, (t) => { 17 | t.plan(70) 18 | 19 | const q = new Queue(tOpts.cxn(), tOpts.default('queueCancelJob')) 20 | 21 | // ---------- Event Handler Setup ---------- 22 | let state = { 23 | testName, 24 | enabled: false, 25 | ready: 0, 26 | processing: 0, 27 | progress: 0, 28 | pausing: 0, 29 | paused: 0, 30 | resumed: 0, 31 | removed: 5, 32 | reset: 1, 33 | error: 0, 34 | reviewed: 0, 35 | detached: 0, 36 | stopping: 0, 37 | stopped: 0, 38 | dropped: 0, 39 | added: 6, 40 | waiting: 0, 41 | active: 0, 42 | completed: 0, 43 | cancelled: 11, 44 | failed: 0, 45 | terminated: 0, 46 | reanimated: 0, 47 | log: 0, 48 | updated: 0 49 | } 50 | 51 | const jobsToCreate = 5 52 | let jobs = [] 53 | for (let i = 0; i < jobsToCreate; i++) { 54 | jobs.push(q.createJob()) 55 | } 56 | return q.reset().then((resetResult) => { 57 | t.ok(is.integer(resetResult), 'Queue reset') 58 | return q.addJob(jobs) 59 | }).then((savedJobs) => { 60 | t.equal(savedJobs.length, jobsToCreate, 'Jobs saved successfully') 61 | 62 | // ---------- Cancel Multiple Jobs Tests ---------- 63 | eventHandlers.add(t, q, state) 64 | t.comment('queue-cancel-job: Cancel Multiple Jobs') 65 | return queueCancelJob(q, savedJobs, tData) 66 | }).then((cancelResult) => { 67 | t.equal(cancelResult.length, jobsToCreate, 'Job cancelled successfully') 68 | jobs = q.createJob() 69 | return q.addJob(jobs) 70 | }).then((singleJob) => { 71 | t.equal(singleJob[0].id, jobs.id, 'Jobs saved successfully') 72 | 73 | // ---------- Cancel Single Job Tests ---------- 74 | t.comment('queue-cancel-job: Cancel Single Job') 75 | return queueCancelJob(q, singleJob[0], tData) 76 | }).then((cancelledJobId) => { 77 | t.ok(is.uuid(cancelledJobId[0]), 'Cancel Job returned Id') 78 | return q.getJob(cancelledJobId) 79 | }).then((cancelledJob) => { 80 | t.equal(cancelledJob[0].status, enums.status.cancelled, 'Job status is cancelled') 81 | t.ok(is.date(cancelledJob[0].dateFinished), 'Job dateFinished is a date') 82 | t.equal(cancelledJob[0].queueId, q.id, 'Job queueId is valid') 83 | t.equal(cancelledJob[0].log.length, 2, 'Job log exists') 84 | t.ok(is.date(cancelledJob[0].log[1].date), 'Log date is a date') 85 | t.equal(cancelledJob[0].log[1].queueId, q.id, 'Log queueId is valid') 86 | t.equal(cancelledJob[0].log[1].type, enums.log.information, 'Log type is information') 87 | t.equal(cancelledJob[0].log[1].status, enums.status.cancelled, 'Log status is cancelled') 88 | t.ok(cancelledJob[0].log[1].retryCount >= 0, 'Log retryCount is valid') 89 | t.equal(cancelledJob[0].log[1].message, tData, 'Log message is present') 90 | 91 | // ---------- Cancel Multiple Jobs with Remove Tests ---------- 92 | t.comment('queue-cancel-job: Cancel Multiple Jobs with Remove') 93 | jobs = [] 94 | for (let i = 0; i < jobsToCreate; i++) { 95 | jobs.push(q.createJob()) 96 | } 97 | q._removeFinishedJobs = true 98 | return q.addJob(jobs) 99 | }).then((savedJobs) => { 100 | t.equal(savedJobs.length, jobsToCreate, 'Jobs saved successfully') 101 | return queueCancelJob(q, savedJobs, tData) 102 | }).then((cancelResult) => { 103 | t.equal(cancelResult.length, 5, 'Cancel Job returned valid number of Ids') 104 | cancelResult.forEach((jobId) => { 105 | t.ok(is.uuid(jobId), 'Cancel job returned item is a valid Id') 106 | }) 107 | return q.getJob(cancelResult) 108 | }).then((cancelledJobs) => { 109 | t.equal(cancelledJobs.length, 0, 'Cancelled jobs not in database') 110 | }).then(() => { 111 | return q.reset() 112 | }).then((resetResult) => { 113 | t.ok(resetResult >= 0, 'Queue reset') 114 | 115 | // ---------- Event Summary ---------- 116 | eventHandlers.remove(t, q, state) 117 | 118 | q.stop() 119 | return resolve(t.end()) 120 | }).catch(err => tError(err, module, t)) 121 | }) 122 | }) 123 | } 124 | -------------------------------------------------------------------------------- /tests/job-update.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const is = require('../src/is') 4 | const tError = require('./test-error') 5 | const enums = require('../src/enums') 6 | const jobUpdate = require('../src/job-update') 7 | const tData = require('./test-options').tData 8 | const Queue = require('../src/queue') 9 | const tOpts = require('./test-options') 10 | const eventHandlers = require('./test-event-handlers') 11 | const testName = 'job-update' 12 | 13 | jobUpdateTests() 14 | function jobUpdateTests () { 15 | return new Promise((resolve, reject) => { 16 | test(testName, (t) => { 17 | t.plan(55) 18 | 19 | const q = new Queue(tOpts.cxn(), tOpts.default('jobUpdate')) 20 | let job = q.createJob() 21 | job.data = tData 22 | let tDate = new Date() 23 | 24 | // ---------- Event Handler Setup ---------- 25 | let state = { 26 | testName, 27 | enabled: false, 28 | ready: 0, 29 | processing: 0, 30 | progress: 0, 31 | pausing: 0, 32 | paused: 0, 33 | resumed: 0, 34 | removed: 0, 35 | reset: 1, 36 | error: 0, 37 | reviewed: 0, 38 | detached: 0, 39 | stopping: 0, 40 | stopped: 0, 41 | dropped: 0, 42 | added: 0, 43 | waiting: 0, 44 | active: 0, 45 | completed: 0, 46 | cancelled: 0, 47 | failed: 0, 48 | terminated: 0, 49 | reanimated: 0, 50 | log: 0, 51 | updated: 1 52 | } 53 | 54 | return q.reset().then((resetResult) => { 55 | t.ok(is.integer(resetResult), 'Queue reset') 56 | return q.addJob(job) 57 | }).then((savedJob) => { 58 | t.equal(savedJob[0].id, job.id, 'Job saved successfully') 59 | return q.getJob(job) 60 | }).then((savedJobs1) => { 61 | t.equal(savedJobs1[0].id, job.id, 'Job retrieved successfully') 62 | t.equal(savedJobs1[0].data, tData, 'Job data is valid') 63 | t.equal(savedJobs1[0].log.length, 1, 'Job log is valid') 64 | 65 | // ---------- Job Update Test ---------- 66 | eventHandlers.add(t, q, state) 67 | t.comment('job-update: Update') 68 | job = savedJobs1[0] 69 | job.newData = tData 70 | job.dateEnable = tDate 71 | return jobUpdate(job) 72 | }).then((updateResult) => { 73 | t.ok(is.job(updateResult), 'Job updated successfully') 74 | return q.getJob(job.id) 75 | }).then((updatedJob) => { 76 | t.equal(updatedJob[0].status, job.status, 'Updated job status is valid') 77 | t.equal(updatedJob[0].progress, job.progress, 'Updated job progress is valid') 78 | t.equal(updatedJob[0].queueId, q.id, 'Updated job queueId is valid') 79 | t.equal(updatedJob[0].log.length, 2, 'Updated job log is valid') 80 | t.ok(is.date(updatedJob[0].log[1].date), 'Updated log date is a date') 81 | t.equal(updatedJob[0].log[1].queueId, q.id, 'Updated log queueId is valid') 82 | t.equal(updatedJob[0].log[1].type, enums.log.information, 'Updated log type is information') 83 | t.equal(updatedJob[0].log[1].status, job.status, 'Updated log status is valid') 84 | t.equal(updatedJob[0].log[1].message, enums.message.jobUpdated, 'Updated log message is present') 85 | t.equal(updatedJob[0].log[1].data.data, tData, 'Updated log data.data is present') 86 | t.ok(is.date(updatedJob[0].log[1].data.dateCreated), 'Updated log data.dateCreated is a date') 87 | t.ok(is.date(updatedJob[0].log[1].data.dateEnable), 'Updated log data.dateEnable is a date') 88 | t.ok(is.uuid(updatedJob[0].log[1].data.id), 'Updated log data.id is a uuid') 89 | t.ok(is.number(updatedJob[0].log[1].data.priority), 'Updated log data.priority is a number') 90 | t.ok(is.number(updatedJob[0].log[1].data.progress), 'Updated log data.progress is a number') 91 | t.ok(is.string(updatedJob[0].log[1].data.queueId), 'Updated log data.queueId is a string') 92 | t.ok(is.number(updatedJob[0].log[1].data.retryCount), 'Updated log data.retryCount is a number') 93 | t.ok(is.number(updatedJob[0].log[1].data.retryDelay), 'Updated log data.retryDelay is a number') 94 | t.ok(is.number(updatedJob[0].log[1].data.retryMax), 'Updated log data.retryMax is a number') 95 | t.ok(is.string(updatedJob[0].log[1].data.status), 'Updated log data.status is a string') 96 | t.ok(is.number(updatedJob[0].log[1].data.timeout), 'Updated log data.timeout is a number') 97 | t.equal(updatedJob[0].newData, tData, 'New job data is valid') 98 | t.equal(updatedJob[0].dateEnable.toString(), tDate.toString(), 'New job dateEnable is valid') 99 | 100 | return q.reset() 101 | }).then((resetResult) => { 102 | t.ok(resetResult >= 0, 'Queue reset') 103 | 104 | // ---------- Event Summary ---------- 105 | eventHandlers.remove(t, q, state) 106 | q.stop() 107 | return resolve(t.end()) 108 | }).catch(err => tError(err, module, t)) 109 | }) 110 | }) 111 | } 112 | -------------------------------------------------------------------------------- /tests/queue-add-job.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const is = require('../src/is') 4 | const tError = require('./test-error') 5 | const enums = require('../src/enums') 6 | const queueAddJob = require('../src/queue-add-job') 7 | const Queue = require('../src/queue') 8 | const tOpts = require('./test-options') 9 | const eventHandlers = require('./test-event-handlers') 10 | const testName = 'queue-add-job' 11 | 12 | queueAddJobTests() 13 | function queueAddJobTests () { 14 | return new Promise((resolve, reject) => { 15 | test(testName, (t) => { 16 | t.plan(51) 17 | 18 | const q = new Queue(tOpts.cxn(), tOpts.default('queueAddJob')) 19 | 20 | // ---------- Event Handler Setup ---------- 21 | let state = { 22 | testName, 23 | enabled: false, 24 | ready: 0, 25 | processing: 0, 26 | progress: 0, 27 | pausing: 0, 28 | paused: 0, 29 | resumed: 0, 30 | removed: 0, 31 | reset: 0, 32 | error: 0, 33 | reviewed: 0, 34 | detached: 0, 35 | stopping: 0, 36 | stopped: 0, 37 | dropped: 0, 38 | added: 3, 39 | waiting: 0, 40 | active: 0, 41 | completed: 0, 42 | cancelled: 0, 43 | failed: 0, 44 | terminated: 0, 45 | reanimated: 0, 46 | log: 0, 47 | updated: 0 48 | } 49 | 50 | const job = q.createJob() 51 | const jobs = [ 52 | q.createJob(), 53 | q.createJob() 54 | ] 55 | 56 | return q.reset().then((resetResult) => { 57 | t.ok(is.integer(resetResult), 'Queue reset') 58 | eventHandlers.add(t, q, state) 59 | 60 | // ---------- Add Single Job Tests ---------- 61 | t.comment('queue-add-job: Add Single Job') 62 | return queueAddJob(q, job) 63 | }).then((savedJob) => { 64 | t.equal(savedJob[0].id, job.id, 'Job 1 saved successfully') 65 | return q.getJob(job.id) 66 | }).then((jobsFromDb) => { 67 | t.ok(is.date(jobsFromDb[0].log[0].date), 'Log job 1 date is a date') 68 | t.equal(jobsFromDb[0].log[0].queueId, q.id, 'Log job 1 queueId is valid') 69 | t.equal(jobsFromDb[0].log[0].type, enums.log.information, 'Log job 1 type is information') 70 | t.equal(jobsFromDb[0].log[0].status, enums.status.waiting, 'Log job 1 status is added') 71 | t.equal(jobsFromDb[0].retryCount, 0, 'Log job 1 retryCount is valid') 72 | t.equal(jobsFromDb[0].log[0].message, enums.message.jobAdded, 'Log job 1 message is valid') 73 | 74 | // ---------- Add Multiple Jobs Tests ---------- 75 | t.comment('queue-add-job: Add Multiple Job') 76 | return queueAddJob(q, jobs) 77 | }).then((savedJobs) => { 78 | t.equal(savedJobs[0].id, jobs[0].id, 'Job 2 saved successfully') 79 | t.equal(savedJobs[1].id, jobs[1].id, 'Job 3 saved successfully') 80 | 81 | return q.getJob(jobs[0].id) 82 | }).then((jobsFromDb2) => { 83 | t.ok(is.date(jobsFromDb2[0].log[0].date), 'Log job 2 date is a date') 84 | t.equal(jobsFromDb2[0].log[0].queueId, q.id, 'Log job 2 queueId is valid') 85 | t.equal(jobsFromDb2[0].log[0].type, enums.log.information, 'Log job 2 type is information') 86 | t.equal(jobsFromDb2[0].log[0].status, enums.status.waiting, 'Log job 2 status is added') 87 | t.equal(jobsFromDb2[0].retryCount, 0, 'Log job 2 retryCount is valid') 88 | t.equal(jobsFromDb2[0].log[0].message, enums.message.jobAdded, 'Log job 2 message is valid') 89 | return q.getJob(jobs[1].id) 90 | }).then((jobsFromDb3) => { 91 | t.ok(is.date(jobsFromDb3[0].log[0].date), 'Log job 3 date is a date') 92 | t.equal(jobsFromDb3[0].log[0].queueId, q.id, 'Log job 3 queueId is valid') 93 | t.equal(jobsFromDb3[0].log[0].type, enums.log.information, 'Log job 3 type is information') 94 | t.equal(jobsFromDb3[0].log[0].status, enums.status.waiting, 'Log job 3 status is added') 95 | t.equal(jobsFromDb3[0].retryCount, 0, 'Log job 3 retryCount is valid') 96 | t.equal(jobsFromDb3[0].log[0].message, enums.message.jobAdded, 'Log job 3 message is valid') 97 | }).then(() => { 98 | // 99 | // ---------- Add Null Job Tests ---------- 100 | t.comment('queue-add-job: Add Null Job') 101 | return queueAddJob(q) 102 | }).then((nullJobResult) => { 103 | t.equal(nullJobResult.length, 0, 104 | 'Job null or undefined returns an empty array') 105 | 106 | // ---------- Add Invalid Job Tests ---------- 107 | t.comment('queue-add-job: Add Invalid Job') 108 | return queueAddJob(q, {}).then(() => { 109 | t.fail('Job invalid is not returning a rejected promise') 110 | }).catch((err) => { 111 | t.ok(err.message.includes(enums.message.jobInvalid), 'Job invalid returns a rejected promise') 112 | }) 113 | }).then(() => { 114 | // 115 | // ---------- Event Summary ---------- 116 | eventHandlers.remove(t, q, state) 117 | return q.reset() 118 | }).then((resetResult) => { 119 | t.ok(resetResult >= 0, 'Queue reset') 120 | q.stop() 121 | return resolve(t.end()) 122 | }).catch(err => tError(err, module, t)) 123 | }) 124 | }) 125 | } 126 | -------------------------------------------------------------------------------- /tests/db-result.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const tError = require('./test-error') 4 | const enums = require('../src/enums') 5 | const dbResult = require('../src/db-result') 6 | const tData = require('./test-options').tData 7 | const Queue = require('../src/queue') 8 | const tOpts = require('./test-options') 9 | 10 | dbResultTests() 11 | function dbResultTests () { 12 | return new Promise((resolve, reject) => { 13 | test('db-result', (t) => { 14 | t.plan(28) 15 | 16 | const q = new Queue(tOpts.cxn(), tOpts.default('dbResult')) 17 | const job1 = q.createJob() 18 | job1.data = tData 19 | const job2 = q.createJob() 20 | job2.data = tData 21 | const job3 = q.createJob() 22 | job3.data = tData 23 | const mockArray = [ 24 | job1.getCleanCopy(), 25 | job2.getCleanCopy(), 26 | job3.getCleanCopy() 27 | ] 28 | const mockChange = { 29 | changes: [ 30 | { 31 | new_val: job1.getCleanCopy() 32 | }, 33 | { 34 | new_val: job2.getCleanCopy() 35 | }, 36 | { 37 | new_val: job3.getCleanCopy() 38 | } 39 | ], 40 | errors: 1 41 | } 42 | const mockSingleChange = { 43 | new_val: job1.getCleanCopy(), 44 | what: '?' 45 | } 46 | 47 | // ---------- Invalid Parameters Tests ---------- 48 | t.comment('db-result: Invalid Parameters') 49 | return dbResult.toJob(q).then((undefinedResult) => { 50 | t.equal(undefinedResult.length, 0, 'Undefined db result returns empty array') 51 | return dbResult.toJob(q, mockChange).then((noErr) => { 52 | t.fail('Does not fail if errors exist in db result') 53 | }).catch((err) => { 54 | t.equal(err.message, enums.message.dbError, 'Returns rejected Promise on error in toJobs') 55 | }) 56 | }).then(() => { 57 | return dbResult.toIds(mockChange).then((noErr) => { 58 | t.fail('Does not fail if errors exist in db result') 59 | }).catch((err) => { 60 | t.equal(err.message, enums.message.dbError, 'Returns rejected Promise on error in toIds') 61 | }) 62 | }).then(() => { 63 | mockChange.errors = 0 64 | 65 | // ---------- toJob Tests ---------- 66 | t.comment('db-result: toJobs') 67 | return dbResult.toJob(q, mockChange) 68 | }).then((changeResult) => { 69 | t.equal(changeResult.length, 3, 'DB result with changes returns jobs array') 70 | t.equal(changeResult[0].id, job1.id, 'First returned job is valid') 71 | t.equal(changeResult[1].id, job2.id, 'Second returned job is valid') 72 | t.equal(changeResult[2].id, job3.id, 'Third returned job is valid') 73 | return dbResult.toJob(q, mockArray) 74 | }).then((arrayResult) => { 75 | t.equal(arrayResult.length, 3, 'Array of data returns jobs array') 76 | t.equal(arrayResult[0].id, job1.id, 'First returned job is valid') 77 | t.equal(arrayResult[1].id, job2.id, 'Second returned job is valid') 78 | t.equal(arrayResult[2].id, job3.id, 'Third returned job is valid') 79 | return dbResult.toJob(q, mockSingleChange) 80 | }).then((singleResult) => { 81 | t.equal(singleResult.length, 1, 'Single change returns jobs array') 82 | t.deepEqual(singleResult[0].getCleanCopy(), job1.getCleanCopy(), 'Single returned job is valid') 83 | return dbResult.toJob(q, job1.getCleanCopy()) 84 | }).then((jobResult) => { 85 | t.equal(jobResult.length, 1, 'Job data returns jobs array') 86 | t.deepEqual(jobResult[0].getCleanCopy(), job1.getCleanCopy(), 'Job data returned job is valid') 87 | 88 | // ---------- toIds Tests ---------- 89 | t.comment('db-result: toIds') 90 | return dbResult.toIds(mockChange) 91 | }).then((changeResult) => { 92 | t.equal(changeResult.length, 3, 'DB result with changes returns ids array') 93 | t.equal(changeResult[0], job1.id, 'First returned id is valid') 94 | t.equal(changeResult[1], job2.id, 'Second returned id is valid') 95 | t.equal(changeResult[2], job3.id, 'Third returned id is valid') 96 | return dbResult.toIds(mockArray) 97 | }).then((arrayResult) => { 98 | t.equal(arrayResult.length, 3, 'Array of data returns ids array') 99 | t.equal(arrayResult[0], job1.id, 'First returned id is valid') 100 | t.equal(arrayResult[1], job2.id, 'Second returned id is valid') 101 | t.equal(arrayResult[2], job3.id, 'Third returned id is valid') 102 | return dbResult.toIds(mockSingleChange) 103 | }).then((singleResult) => { 104 | t.equal(singleResult.length, 1, 'Single change returns ids array') 105 | t.deepEqual(singleResult[0], job1.id, 'Single returned id is valid') 106 | return dbResult.toIds(job1.getCleanCopy()) 107 | }).then((jobResult) => { 108 | t.equal(jobResult.length, 1, 'Job data returns ids array') 109 | t.deepEqual(jobResult[0], job1.id, 'Job data returned id is valid') 110 | 111 | return q.reset() 112 | }).then((resetResult) => { 113 | t.ok(resetResult >= 0, 'Queue reset') 114 | q.stop() 115 | return resolve(t.end()) 116 | }).catch(err => tError(err, module, t)) 117 | }) 118 | }) 119 | } 120 | -------------------------------------------------------------------------------- /tests/queue-remove-job.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const is = require('../src/is') 4 | const enums = require('../src/enums') 5 | const tError = require('./test-error') 6 | const queueRemoveJob = require('../src/queue-remove-job') 7 | const Queue = require('../src/queue') 8 | const tOpts = require('./test-options') 9 | const eventHandlers = require('./test-event-handlers') 10 | const testName = 'queue-remove-job' 11 | 12 | queueRemoveJobTests() 13 | function queueRemoveJobTests () { 14 | return new Promise((resolve, reject) => { 15 | test(testName, (t) => { 16 | t.plan(55) 17 | 18 | const q = new Queue(tOpts.cxn(), tOpts.default('queueRemoveJob')) 19 | let jobs = [] 20 | for (let i = 0; i < 3; i++) { 21 | jobs.push(q.createJob()) 22 | } 23 | 24 | // ---------- Event Handler Setup ---------- 25 | let state = { 26 | testName, 27 | enabled: false, 28 | ready: 0, 29 | processing: 0, 30 | progress: 0, 31 | pausing: 0, 32 | paused: 0, 33 | resumed: 0, 34 | removed: 8, 35 | reset: 0, 36 | error: 0, 37 | reviewed: 0, 38 | detached: 0, 39 | stopping: 0, 40 | stopped: 0, 41 | dropped: 0, 42 | added: 8, 43 | waiting: 0, 44 | active: 0, 45 | completed: 0, 46 | cancelled: 0, 47 | failed: 0, 48 | terminated: 0, 49 | reanimated: 0, 50 | log: 0, 51 | updated: 0 52 | } 53 | 54 | return q.reset().then((resetResult) => { 55 | t.ok(is.integer(resetResult), 'Queue reset') 56 | eventHandlers.add(t, q, state) 57 | return q.addJob(jobs) 58 | }).then((savedJobs) => { 59 | t.equal(savedJobs.length, 3, 'Jobs saved successfully') 60 | 61 | // ---------- Remove by Job Object Tests ---------- 62 | t.comment('queue-remove-job: Remove by Jobs') 63 | return queueRemoveJob(q, savedJobs) 64 | }).then((removeResult) => { 65 | t.equal(removeResult.length, 3, 'Jobs removed successfully') 66 | return q.getJob(jobs.map(j => j.id)) 67 | }).then((getResult) => { 68 | t.equal(getResult.length, 0, 'Jobs no longer in database') 69 | let jobs = [] 70 | for (let i = 0; i < 3; i++) { 71 | jobs.push(q.createJob()) 72 | } 73 | return q.addJob(jobs) 74 | }).then((savedAgain) => { 75 | t.equal(savedAgain.length, 3, 'Jobs saved successfully (again)') 76 | 77 | // ---------- Remove by Ids Tests ---------- 78 | t.comment('queue-remove-job: Remove by Ids') 79 | return queueRemoveJob(q, savedAgain.map(j => j.id)) 80 | }).then((removeIdResult) => { 81 | t.equal(removeIdResult.length, 3, 'Jobs removed by id successfully') 82 | return q.getJob(jobs.map(j => j.id)) 83 | }).then((getResult2) => { 84 | t.equal(getResult2.length, 0, 'Jobs no longer in database') 85 | jobs = q.createJob() 86 | return q.addJob(jobs) 87 | }).then((saveSingle) => { 88 | t.equal(saveSingle.length, 1, 'Single job saved successfully') 89 | 90 | // ---------- Remove Single Job Tests ---------- 91 | t.comment('queue-remove-job: Remove Single Job') 92 | return queueRemoveJob(q, saveSingle) 93 | }).then((removeSingleResult) => { 94 | t.equal(removeSingleResult.length, 1, 'Single job removed successfully') 95 | return q.getJob(jobs.id) 96 | }).then((getResult3) => { 97 | t.equal(getResult3.length, 0, 'Single job no longer in database') 98 | jobs = q.createJob() 99 | return q.addJob(jobs) 100 | }).then((saveSingle2) => { 101 | t.equal(saveSingle2.length, 1, 'Single job saved successfully (again)') 102 | 103 | // ---------- Remove Single Job by Id Tests ---------- 104 | t.comment('queue-remove-job: Remove Single Job by Id') 105 | return queueRemoveJob(q, saveSingle2[0].id) 106 | }).then((removeSingleResult2) => { 107 | t.equal(removeSingleResult2.length, 1, 'Single job removed by id successfully') 108 | return q.getJob(jobs.id) 109 | }).then((getResult4) => { 110 | t.equal(getResult4.length, 0, 'Single job no longer in database') 111 | 112 | // ---------- Remove Undefined Job Tests ---------- 113 | t.comment('queue-remove-job: Remove Undefined Job') 114 | return queueRemoveJob(q) 115 | }).then((undefinedResult) => { 116 | t.equal(undefinedResult.length, 0, 'Remove undefined job returns 0 result') 117 | 118 | // ---------- Remove Invalid Job Tests ---------- 119 | t.comment('queue-remove-job: Remove Invalid Job') 120 | return queueRemoveJob(q, ['not a job']).then(() => { 121 | t.fail('queue-remove-job is not failing on an invalid job') 122 | }).catch((err) => { 123 | t.ok(err.message.includes(enums.message.idInvalid), 'Invalid job returns a rejected Promise') 124 | }) 125 | }).then(() => { 126 | // 127 | // ---------- Event Summary ---------- 128 | eventHandlers.remove(t, q, state) 129 | return q.reset() 130 | }).then((resetResult) => { 131 | t.ok(resetResult >= 0, 'Queue reset') 132 | q.stop() 133 | return resolve(t.end()) 134 | }).catch(err => tError(err, module, t)) 135 | }) 136 | }) 137 | } 138 | -------------------------------------------------------------------------------- /tests/job-progress.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const datetime = require('../src/datetime') 4 | const is = require('../src/is') 5 | const tError = require('./test-error') 6 | const enums = require('../src/enums') 7 | const jobProgress = require('../src/job-progress') 8 | const Queue = require('../src/queue') 9 | const tOpts = require('./test-options') 10 | const eventHandlers = require('./test-event-handlers') 11 | const testName = 'job-progress' 12 | 13 | jobProgressTests() 14 | function jobProgressTests () { 15 | return new Promise((resolve, reject) => { 16 | test(testName, (t) => { 17 | t.plan(47) 18 | 19 | const q = new Queue(tOpts.cxn(), tOpts.default('jobProgress')) 20 | const job = q.createJob() 21 | job.timeout = enums.options.timeout 22 | job.retryDelay = enums.options.retryDelay 23 | job.retryCount = 0 24 | 25 | // ---------- Event Handler Setup ---------- 26 | let state = { 27 | testName, 28 | enabled: false, 29 | ready: 0, 30 | processing: 0, 31 | progress: 6, 32 | pausing: 0, 33 | paused: 0, 34 | resumed: 0, 35 | removed: 0, 36 | idle: 0, 37 | reset: 0, 38 | error: 0, 39 | reviewed: 0, 40 | detached: 0, 41 | stopping: 0, 42 | stopped: 0, 43 | dropped: 0, 44 | added: 0, 45 | waiting: 0, 46 | active: 0, 47 | completed: 0, 48 | cancelled: 0, 49 | failed: 0, 50 | terminated: 0, 51 | reanimated: 0, 52 | log: 0, 53 | updated: 0 54 | } 55 | 56 | let tempDateEnable = job.dateEnable 57 | return q.addJob(job).then((savedJob) => { 58 | t.equal(savedJob[0].id, job.id, 'Job saved successfully') 59 | eventHandlers.add(t, q, state) 60 | savedJob[0].retryCount = 1 61 | savedJob[0].status = enums.status.active 62 | return jobProgress(savedJob[0]) 63 | }).then((updatedJob) => { 64 | t.ok(is.job(updatedJob), 'Job updated successfully') 65 | return job.q.r.db(job.q.db).table(job.q.name).get(job.id).update({ 66 | retryCount: 1 67 | }).run() 68 | }).then(() => { 69 | return q.getJob(job.id) 70 | }).then((updatedJob) => { 71 | t.equal(updatedJob[0].progress, 0, 'Job progress is 0 when updated with a null value') 72 | t.ok( 73 | is.dateBetween(updatedJob[0].dateEnable, 74 | tempDateEnable, 75 | datetime.add.ms(new Date(), updatedJob[0].timeout + 2000)), 76 | 'Job dateEnable updated successfully' 77 | ) 78 | updatedJob[0].status = enums.status.active 79 | return jobProgress(updatedJob[0], -10) 80 | }).then((updatedJob) => { 81 | t.ok(is.job(updatedJob), 'Job updated successfully') 82 | return q.getJob(job.id) 83 | }).then((updatedJob) => { 84 | t.equal(updatedJob[0].progress, 0, 'Job progress is 0 when updated with negative value') 85 | t.ok( 86 | is.dateBetween(updatedJob[0].dateEnable, 87 | tempDateEnable, 88 | datetime.add.ms(new Date(), updatedJob[0].timeout + 2000 + updatedJob[0].retryDelay)), 89 | 'Job dateEnable updated successfully' 90 | ) 91 | updatedJob[0].status = enums.status.active 92 | return jobProgress(updatedJob[0], 1) 93 | }).then((updatedJob) => { 94 | t.ok(is.job(updatedJob), 'Job updated successfully') 95 | return q.getJob(job.id) 96 | }).then((updatedJob) => { 97 | t.equal(updatedJob[0].progress, 1, 'Job progress is 1 percent') 98 | updatedJob[0].status = enums.status.active 99 | return jobProgress(updatedJob[0], 50) 100 | }).then((updatedJob) => { 101 | t.ok(is.job(updatedJob), 'Job updated successfully') 102 | return q.getJob(job.id) 103 | }).then((updatedJob) => { 104 | t.equal(updatedJob[0].progress, 50, 'Job progress is 50 percent') 105 | updatedJob[0].status = enums.status.active 106 | return jobProgress(updatedJob[0], 100) 107 | }).then((updatedJob) => { 108 | t.ok(is.job(updatedJob), 'Job updated successfully') 109 | return q.getJob(job.id) 110 | }).then((updatedJob) => { 111 | t.equal(updatedJob[0].progress, 100, 'Job progress is 100 percent') 112 | t.equal(updatedJob[0].getLastLog().data, 50, 'Job progress log contains old percent') 113 | updatedJob[0].status = enums.status.active 114 | return jobProgress(updatedJob[0], 101) 115 | }).then((updatedJob) => { 116 | t.ok(is.job(updatedJob), 'Job updated successfully') 117 | return q.getJob(job.id) 118 | }).then((updatedJob) => { 119 | t.equal(updatedJob[0].progress, 100, 'Job progress is 100 when updated with larger value') 120 | updatedJob[0].status = enums.status.failed 121 | return jobProgress(updatedJob[0], 101).catch((err) => { 122 | t.ok(is.error(err), 'Inactive job rejects Promise') 123 | }) 124 | }).then(() => { 125 | // 126 | // ---------- Event Summary ---------- 127 | eventHandlers.remove(t, q, state) 128 | return q.reset() 129 | }).then((resetResult) => { 130 | t.ok(resetResult >= 0, 'Queue reset') 131 | q.stop() 132 | return resolve(t.end()) 133 | }).catch(err => tError(err, module, t)) 134 | }) 135 | }) 136 | } 137 | -------------------------------------------------------------------------------- /tests/job-parse.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const is = require('../src/is') 3 | const jobParse = require('../src/job-parse') 4 | const Job = require('../src/job') 5 | const uuid = require('uuid') 6 | 7 | jobParseTests() 8 | function jobParseTests () { 9 | test('job-parse', (t) => { 10 | t.plan(40) 11 | 12 | const ids = [ 13 | uuid.v4(), 14 | uuid.v4(), 15 | uuid.v4() 16 | ] 17 | const idsResult = jobParse.id(ids) 18 | const objWithIds = [ 19 | {id: uuid.v4()}, 20 | {id: uuid.v4()}, 21 | {id: uuid.v4()} 22 | ] 23 | const objWithIdsResult = jobParse.id(ids) 24 | const mix = [...ids, ...objWithIds] 25 | const mixResult = jobParse.id(mix) 26 | 27 | // ---------- Parse Ids ---------- 28 | t.comment('job-parse: Parse Ids') 29 | t.throws(() => { jobParse.id({}) }, 'Throws error if not a valid job for id') 30 | t.ok(is.array(jobParse.id()), 'Null or undefined returns an array') 31 | t.equal(jobParse.id().length, 0, 'Null or undefined returns an empty array') 32 | t.ok(is.array(jobParse.id(uuid.v4())), 'Single id returns an array') 33 | t.equal(jobParse.id(uuid.v4()).length, 1, 'Single id returns one item in an array') 34 | t.ok(is.uuid(jobParse.id(uuid.v4())[0]), 'Single id returns one valid id in an array') 35 | t.ok(is.array(jobParse.id({id: uuid.v4()})), 'Single object.id returns an array') 36 | t.equal(jobParse.id({id: uuid.v4()}).length, 1, 'Single object.id returns one item in an array') 37 | t.ok(is.uuid(jobParse.id({id: uuid.v4()})[0]), 'Single object.id returns one valid id in an array') 38 | t.ok(is.array(idsResult), 'Array of ids returns an array') 39 | t.equal(idsResult.length, 3, 'Array of ids returns valid number of items') 40 | t.ok(is.uuid(idsResult[0]), 'Array of ids returns valid ids') 41 | t.ok(is.array(objWithIdsResult), 'Array of object.ids returns an array') 42 | t.equal(objWithIdsResult.length, 3, 'Array of object.ids returns valid number of items') 43 | t.ok(is.uuid(objWithIdsResult[0]), 'Array of object.ids returns valid ids') 44 | t.ok(is.array(mixResult), 'Array of mixed objects and ids returns an array') 45 | t.equal(mixResult.length, 6, 'Array of mixed objects and ids returns valid number of items') 46 | t.ok(is.uuid(mixResult[0]), 'Array of mixed objects and ids returns valid ids') 47 | 48 | // ---------- Parse Single Job ---------- 49 | t.comment('job-parse: Parse Single Job') 50 | const mockQueue = { id: uuid.v4(), name: 'fake' } 51 | let job = new Job(mockQueue) 52 | t.throws(() => { jobParse.job({}) }, 'Throws error if not a valid job for id') 53 | t.ok(is.array(jobParse.job()), 'Null or undefined returns an array') 54 | t.equal(jobParse.job().length, 0, 'Null or undefined returns an empty array') 55 | t.ok(is.array(jobParse.job(job)), 'Single job returns an Array') 56 | t.equal(jobParse.job(job).length, 1, 'Single job returns one item in an array') 57 | t.ok(is.uuid(jobParse.job(job)[0].id), 'Single job returns one valid job in an Array') 58 | 59 | // ---------- Parse Array of Jobs ---------- 60 | t.comment('job-parse: Parse Job Array') 61 | let jobs = [ 62 | new Job(mockQueue), 63 | new Job(mockQueue), 64 | new Job(mockQueue) 65 | ] 66 | let jobsResult = jobParse.job(jobs) 67 | t.ok(is.array(jobsResult), 'Array of jobs returns an array') 68 | t.equal(jobsResult.length, 3, 'Array of jobs returns valid number of items') 69 | t.ok(is.uuid(jobsResult[0].id), 'Array of jobs returns valid jobs') 70 | 71 | // ---------- Parse Invalid Job ---------- 72 | t.comment('job-parse: Parse Invalid Job') 73 | job = new Job(mockQueue) 74 | job.id = 'not an id' 75 | t.throws(() => { jobParse.job(job) }, 'Invalid job id throws an exception') 76 | job = new Job(mockQueue) 77 | job.q = null 78 | t.throws(() => { jobParse.job(job) }, 'Invalid job queue throws an exception') 79 | job = new Job(mockQueue) 80 | job.priority = null 81 | t.throws(() => { jobParse.job(job) }, 'Invalid job priority throws an exception') 82 | job = new Job(mockQueue) 83 | job.timeout = -1 84 | t.throws(() => { jobParse.job(job) }, 'Invalid job timeout throws an exception') 85 | job = new Job(mockQueue) 86 | job.retryDelay = -1 87 | t.throws(() => { jobParse.job(job) }, 'Invalid job retryDelay throws an exception') 88 | job = new Job(mockQueue) 89 | job.retryMax = -1 90 | t.throws(() => { jobParse.job(job) }, 'Invalid job retryMax throws an exception') 91 | job = new Job(mockQueue) 92 | job.retryCount = -1 93 | t.throws(() => { jobParse.job(job) }, 'Invalid job retryCount throws an exception') 94 | job = new Job(mockQueue) 95 | job.status = null 96 | t.throws(() => { jobParse.job(job) }, 'Invalid job status throws an exception') 97 | job = new Job(mockQueue) 98 | job.log = {} 99 | t.throws(() => { jobParse.job(job) }, 'Invalid job log throws an exception') 100 | job = new Job(mockQueue) 101 | job.dateCreated = {} 102 | t.throws(() => { jobParse.job(job) }, 'Invalid job dateCreated throws an exception') 103 | job = new Job(mockQueue) 104 | job.dateEnable = {} 105 | t.throws(() => { jobParse.job(job) }, 'Invalid job dateEnable throws an exception') 106 | job = new Job(mockQueue) 107 | job.progress = 101 108 | t.throws(() => { jobParse.job(job) }, 'Invalid job progress throws an exception') 109 | job = new Job(mockQueue) 110 | job.queueId = null 111 | t.throws(() => { jobParse.job(job) }, 'Invalid job queueId throws an exception') 112 | }) 113 | } 114 | -------------------------------------------------------------------------------- /src/is.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const enums = require('./enums') 3 | const uuidRegExp = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i 4 | 5 | function isObject (value) { 6 | logger(`isObject`, value) 7 | return Object.prototype.toString.call(value) === '[object Object]' 8 | } 9 | module.exports.object = isObject 10 | 11 | function isFunction (value) { 12 | logger(`isFunction`, value) 13 | return Object.prototype.toString.call(value) === '[object Function]' 14 | } 15 | module.exports.function = isFunction 16 | 17 | function isString (value) { 18 | logger(`isString`, value) 19 | return Object.prototype.toString.call(value) === '[object String]' 20 | } 21 | module.exports.string = isString 22 | 23 | function isNumber (value) { 24 | logger(`isNumber`, value) 25 | return Object.prototype.toString.call(value) === '[object Number]' 26 | } 27 | module.exports.number = isNumber 28 | 29 | function isBoolean (value) { 30 | logger(`isBoolean`, value) 31 | return Object.prototype.toString.call(value) === '[object Boolean]' 32 | } 33 | module.exports.boolean = isBoolean 34 | 35 | function isTrue (value) { 36 | logger(`isTrue`, value) 37 | return isBoolean(value) && value === true 38 | } 39 | module.exports.true = isTrue 40 | 41 | function isFalse (value) { 42 | logger(`isFalse`, value) 43 | return isBoolean(value) && value === false 44 | } 45 | module.exports.false = isFalse 46 | 47 | function isDate (value) { 48 | logger(`isDate`, value) 49 | return value instanceof Date || 50 | Object.prototype.toString.call(value) === '[object Date]' 51 | } 52 | module.exports.date = isDate 53 | 54 | function ensureDate (value) { 55 | logger(`ensureDate`, value) 56 | return isDate(value) ? value : new Date() 57 | } 58 | 59 | function isDateBefore (testDate, refDate) { 60 | logger('isDateBefore', testDate, refDate) 61 | testDate = ensureDate(testDate) 62 | refDate = ensureDate(refDate) 63 | return refDate.valueOf() > testDate.valueOf() 64 | } 65 | module.exports.dateBefore = isDateBefore 66 | 67 | function isDateAfter (testDate, refDate) { 68 | logger('isDateAfter', testDate, refDate) 69 | testDate = ensureDate(testDate) 70 | refDate = ensureDate(refDate) 71 | return refDate.valueOf() < testDate.valueOf() 72 | } 73 | module.exports.dateAfter = isDateAfter 74 | 75 | function isDateBetween (testDate, aDate, bDate) { 76 | logger('isDateBetween', testDate, aDate, bDate) 77 | aDate = ensureDate(aDate) 78 | bDate = ensureDate(bDate) 79 | const earlyDate = aDate > bDate ? bDate : aDate 80 | const laterDate = aDate > bDate ? aDate : bDate 81 | return isDateAfter(testDate, earlyDate) && isDateBefore(testDate, laterDate) 82 | } 83 | module.exports.dateBetween = isDateBetween 84 | 85 | function isUuid (value) { 86 | logger(`isUuid`, value) 87 | return uuidRegExp.test(value) 88 | } 89 | module.exports.uuid = isUuid 90 | 91 | function isNan (value) { 92 | logger(`isNan`, value) 93 | return Number.isNaN(value) 94 | } 95 | module.exports.nan = isNan 96 | 97 | function isInteger (value) { 98 | logger(`isInteger`, value) 99 | return isNumber(value) && !isNan(value) && value % 1 === 0 100 | } 101 | module.exports.integer = isInteger 102 | 103 | function isArray (value) { 104 | logger(`isArray`, value) 105 | return Array.isArray(value) 106 | } 107 | module.exports.array = isArray 108 | 109 | function isError (value) { 110 | logger('isError', value) 111 | return value instanceof Error 112 | } 113 | module.exports.error = isError 114 | 115 | function isLog (value) { 116 | logger('isLog', value) 117 | if (!value) { return false } 118 | if (!isDate(value.date)) { return false } 119 | if (!isString(value.queueId)) { return false } 120 | if (!isString(value.type)) { return false } 121 | if (!isString(value.status)) { return false } 122 | return true 123 | } 124 | module.exports.log = isLog 125 | 126 | function isJob (value) { 127 | logger(`isJob`, value) 128 | if (!value) { return false } 129 | if (!value.id) { return false } 130 | if (!isUuid(value.id)) { return false } 131 | if (!value.queueId) { return false } 132 | if (!isDate(value.dateCreated)) { return false } 133 | if (!isNumber(value.priority) && 134 | !Object.keys(enums.priority).includes(value.priority)) { return false } 135 | if (!Object.keys(enums.status).includes(value.status)) { return false } 136 | return true 137 | } 138 | module.exports.job = isJob 139 | 140 | function isStatus (job, status) { 141 | logger('isStatus', job, status) 142 | if (!isJob(job)) { return false } 143 | if (job.status === status) { return true } 144 | return false 145 | } 146 | 147 | module.exports.repeating = function isRepeating (job) { 148 | logger('isRepeating', job) 149 | if (isTrue(job.repeat)) { 150 | return true 151 | } 152 | if (isInteger(job.repeat) && 153 | job.repeat > 0 && 154 | job.processCount <= job.repeat) { 155 | return true 156 | } 157 | return false 158 | } 159 | 160 | module.exports.active = function isActive (job) { 161 | logger('isActive', job) 162 | return isStatus(job, enums.status.active) 163 | } 164 | 165 | module.exports.completed = function isCompleted (job) { 166 | logger('isCompleted', job) 167 | return isStatus(job, enums.status.completed) 168 | } 169 | 170 | module.exports.cancelled = function isCancelled (job) { 171 | logger('isCancelled', job) 172 | return isStatus(job, enums.status.cancelled) 173 | } 174 | 175 | module.exports.failed = function isFailed (job) { 176 | logger('isFailed', job) 177 | return isStatus(job, enums.status.failed) 178 | } 179 | 180 | module.exports.terminated = function isTerminated (job) { 181 | logger('isTerminated', job) 182 | return isStatus(job, enums.status.terminated) 183 | } 184 | -------------------------------------------------------------------------------- /src/queue-change.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const is = require('./is') 3 | const util = require('util') 4 | const enums = require('./enums') 5 | const queueProcess = require('./queue-process') 6 | const queueInterruption = require('./queue-interruption') 7 | 8 | // Following is the list of supported change feed events; 9 | // paused - Global event 10 | // resumed - Global event 11 | // reviewed - Global event 12 | // added 13 | // active 14 | // progress 15 | // completed 16 | // cancelled 17 | // failed 18 | // terminated 19 | // removed 20 | // log 21 | 22 | function restartProcessing (q) { 23 | logger('restartProcessing') 24 | setTimeout(function randomRestart () { 25 | queueProcess.restart(q) 26 | }, Math.floor(Math.random() * 1000)) 27 | } 28 | 29 | module.exports = function queueChange (q, err, change = {}) { 30 | logger('queueChange', change) 31 | 32 | const newVal = change.new_val 33 | const oldVal = change.old_val 34 | 35 | let queueId = false 36 | if (newVal && newVal.queueId) { queueId = newVal.queueId } 37 | if (!newVal && oldVal && oldVal.queueId) { queueId = oldVal.queueId } 38 | if (!queueId) { 39 | logger(`Change feed and queueId missing`, change) 40 | return 41 | } 42 | 43 | // Prevent any change processing if change is caused by this queue 44 | if (queueId === q.id) { 45 | logger('Change feed by self, skipping events') 46 | return 47 | } 48 | 49 | if (err) { throw new Error(err) } 50 | 51 | logger('------------- QUEUE CHANGE -------------') 52 | logger(util.inspect(change, {colors: true})) 53 | logger(queueId) 54 | logger('----------------------------------------') 55 | 56 | // Queue global state change 57 | if (!newVal && oldVal && oldVal.id && oldVal.id === enums.state.docId) { 58 | // Ignoring state document deletion. 59 | return enums.status.active 60 | } 61 | if (newVal && newVal.id && newVal.id === enums.state.docId) { 62 | logger('State document changed') 63 | if (newVal && newVal.state) { 64 | if (newVal.state === enums.status.paused) { 65 | logger('Global queue state paused') 66 | return queueInterruption.pause(q, enums.state.global) 67 | } 68 | if (newVal.state === enums.status.active) { 69 | logger('Global queue state active') 70 | return queueInterruption.resume(q, enums.state.global) 71 | } 72 | if (newVal.state === enums.status.reviewed) { 73 | logger('Global queue state reviewed') 74 | logger(`Event: reviewed [local: false]`) 75 | q.emit(enums.status.reviewed, newVal.queueId, { 76 | local: false, 77 | reviewed: null, 78 | removed: null 79 | }) 80 | if (q.running < q.concurrency) { 81 | return restartProcessing(q) 82 | } 83 | return enums.status.reviewed 84 | } 85 | } 86 | const err = new Error(enums.message.globalStateError) 87 | err.queueId = q.id 88 | logger('Event: State document change error', err, q.id) 89 | q.emit(enums.status.error, err) 90 | return enums.status.error 91 | } 92 | 93 | // Job added 94 | if (is.job(newVal) && 95 | !is.job(oldVal)) { 96 | logger(`Event: added [${newVal.id}]`) 97 | q.emit(enums.status.added, newVal.queueId, newVal.id) 98 | restartProcessing(q) 99 | return enums.status.added 100 | } 101 | 102 | // Job active 103 | if (is.active(newVal) && 104 | !is.active(oldVal)) { 105 | logger(`Event: active [${newVal.id}]`) 106 | q.emit(enums.status.active, newVal.queueId, newVal.id) 107 | return enums.status.active 108 | } 109 | 110 | // Job progress 111 | if (is.job(newVal) && 112 | is.job(oldVal) && 113 | newVal.progress !== oldVal.progress) { 114 | logger(`Event: progress [${newVal.progress}]`) 115 | q.emit(enums.status.progress, newVal.queueId, newVal.id, newVal.progress) 116 | return enums.status.progress 117 | } 118 | 119 | // Job completed 120 | if (is.completed(newVal) && 121 | !is.completed(oldVal)) { 122 | let isRepeating = is.repeating(newVal) 123 | logger(`Event: completed`, newVal.queueId, newVal.id, isRepeating) 124 | q.emit(enums.status.completed, newVal.queueId, newVal.id, isRepeating) 125 | return enums.status.completed 126 | } 127 | 128 | // Job cancelled 129 | if (is.cancelled(newVal) && 130 | !is.cancelled(oldVal)) { 131 | logger(`Event: cancelled`, newVal.queueId, newVal.id) 132 | q.emit(enums.status.cancelled, newVal.queueId, newVal.id) 133 | return enums.status.cancelled 134 | } 135 | 136 | // Job failed 137 | if (is.failed(newVal) && 138 | !is.failed(oldVal)) { 139 | logger(`Event: failed`, newVal.queueId, newVal.id) 140 | q.emit(enums.status.failed, newVal.queueId, newVal.id) 141 | return enums.status.failed 142 | } 143 | 144 | // Job terminated 145 | if (is.terminated(newVal) && 146 | !is.terminated(oldVal)) { 147 | logger(`Event: terminated`, newVal.queueId, newVal.id) 148 | q.emit(enums.status.terminated, newVal.queueId, newVal.id) 149 | return enums.status.terminated 150 | } 151 | 152 | // Job removed 153 | if (!is.job(newVal) && 154 | is.job(oldVal)) { 155 | logger(`Event: removed`, oldVal.id) 156 | q.emit(enums.status.removed, null, oldVal.id) 157 | return enums.status.removed 158 | } 159 | 160 | // Job log 161 | if (is.job(newVal) && 162 | is.job(oldVal) && 163 | is.array(newVal.log) && 164 | is.array(oldVal.log) && 165 | newVal.log.length > oldVal.log.length) { 166 | logger(`Event: log`, newVal.queueId, newVal.log) 167 | q.emit(enums.status.log, newVal.queueId, newVal.id) 168 | return enums.status.log 169 | } 170 | 171 | logger('Unknown database change', change) 172 | } 173 | -------------------------------------------------------------------------------- /tests/is.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const uuid = require('uuid') 3 | const is = require('../src/is') 4 | const enums = require('../src/enums') 5 | 6 | isTests() 7 | function isTests () { 8 | test('is', (t) => { 9 | t.plan(71) 10 | 11 | const ms = 5000 12 | const tDate = new Date() 13 | const earlyDate = new Date(tDate.getTime() - ms) 14 | const laterDate = new Date(tDate.getTime() + ms) 15 | const job = { 16 | id: uuid.v4(), 17 | queueId: 'queue id string', 18 | dateCreated: new Date(), 19 | priority: enums.priority.normal, 20 | status: enums.status.created, 21 | repeat: false, 22 | processCount: 0 23 | } 24 | const log = { 25 | date: new Date(), 26 | queueId: 'queue id string', 27 | type: 'type string', 28 | status: 'status string' 29 | } 30 | 31 | t.ok(is.object({}), 'Is object true with object') 32 | t.notOk(is.object(''), 'Is object false with string') 33 | t.ok(is.function(function () {}), 'Is function true with function') 34 | t.notOk(is.function({}), 'Is function false with object') 35 | t.ok(is.string(''), 'Is string true with string') 36 | t.notOk(is.string({}), 'Is string false with object') 37 | t.ok(is.number(1.1), 'Is number true with decimal') 38 | t.ok(is.number(1), 'Is number true with integer') 39 | t.notOk(is.number({}), 'Is number false with object') 40 | t.ok(is.boolean(true), 'Is boolean true with true') 41 | t.ok(is.boolean(false), 'Is boolean true with false') 42 | t.notOk(is.boolean(''), 'Is boolean false with string') 43 | t.ok(is.true(true), 'Is true true with true') 44 | t.notOk(is.true(false), 'Is true false with false') 45 | t.notOk(is.true(1), 'Is true false with integer 1') 46 | t.ok(is.false(false), 'Is false true with false') 47 | t.notOk(is.false(true), 'Is false false with true') 48 | t.notOk(is.false(0), 'Is false false with integer 0') 49 | t.ok(is.date(new Date()), 'Is date true with new Date()') 50 | t.notOk(is.date({}), 'Is date false with object') 51 | t.ok(is.uuid(uuid.v4()), 'Is uuid true with uuid') 52 | t.notOk(is.uuid('1234'), 'Is uuid false with string of numbers') 53 | t.notOk(is.uuid({}), 'Is uuid false with object') 54 | t.ok(is.nan(Number.NaN), 'Is nan true with Number.NaN') 55 | t.notOk(is.nan(1), 'Is nan false with integer 1') 56 | t.notOk(is.nan({}), 'Is nan false with object') 57 | t.ok(is.integer(1), 'Is integer true with integer 1') 58 | t.notOk(is.integer(1.1), 'Is integer false with decimal') 59 | t.notOk(is.integer({}), 'Is integer false with object') 60 | t.ok(is.array([]), 'Is array true with array') 61 | t.notOk(is.array(1), 'Is array false with integer 1') 62 | t.notOk(is.array({}), 'Is array false with object') 63 | t.ok(is.error(new Error()), 'Is error true with new Error') 64 | t.ok(is.error(Error()), 'Is error true with Error') 65 | t.notOk(is.error('not an error'), 'Is error false with string') 66 | t.ok(is.job(job), 'Is job true with mock job') 67 | t.notOk(is.job(), 'Is job false with null') 68 | job.id = null 69 | t.notOk(is.job(job), 'Is job false with null job id') 70 | job.id = '1234' 71 | t.notOk(is.job(job), 'Is job false with invalid job id') 72 | job.id = uuid.v4() 73 | job.queueId = null 74 | t.notOk(is.job(job), 'Is job false with null queueId') 75 | job.queueId = '1234' 76 | job.dateCreated = {} 77 | t.notOk(is.job(job), 'Is job false with invalid dateCreated') 78 | job.dateCreated = new Date() 79 | job.priority = 40 80 | t.ok(is.job(job), 'Is job true with priority a number') 81 | job.priority = enums.priority.normal 82 | job.status = 'not a real status' 83 | t.notOk(is.job(job), 'Is job false with invalid status') 84 | job.repeat = true 85 | t.ok(is.repeating(job), 'Is job repeating true when repeat is true') 86 | job.repeat = 5 87 | t.ok(is.repeating(job), 'Is job repeating true when repeat is integer') 88 | job.processCount = 5 89 | t.ok(is.repeating(job), 'Is job repeating true when processCount equals repeat') 90 | job.processCount = 6 91 | t.notOk(is.repeating(job), 'Is job repeating false when processCount > repeat') 92 | job.repeat = 0 93 | t.notOk(is.repeating(job), 'Is job repeating false when repeat is 0') 94 | job.repeat = false 95 | t.notOk(is.repeating(job), 'Is job repeating false when repeat is false') 96 | job.status = enums.status.created 97 | t.notOk(is.active(job), 'Is active false with invalid status') 98 | job.status = enums.status.active 99 | t.ok(is.active(job), 'Is active true with active status') 100 | t.notOk(is.completed(job), 'Is completed false with invalid status') 101 | job.status = enums.status.completed 102 | t.ok(is.completed(job), 'Is completed true with completed status') 103 | t.notOk(is.cancelled(job), 'Is cancelled false with invalid status') 104 | job.status = enums.status.cancelled 105 | t.ok(is.cancelled(job), 'Is cancelled true with cancelled status') 106 | t.notOk(is.failed(job), 'Is failed false with invalid status') 107 | job.status = enums.status.failed 108 | t.ok(is.failed(job), 'Is failed true with failed status') 109 | t.notOk(is.terminated(job), 'Is terminated false with invalid status') 110 | job.status = enums.status.terminated 111 | t.ok(is.terminated(job), 'Is terminated true with terminated status') 112 | t.notOk(is.dateBefore(tDate, earlyDate), 'Is dateBefore false when after') 113 | t.ok(is.dateBefore(tDate, laterDate), 'Is dateBefore true when before') 114 | t.notOk(is.dateAfter(tDate, laterDate), 'Is dateAfter false when before') 115 | t.ok(is.dateAfter(tDate, earlyDate), 'Is dateAfter true when after') 116 | t.notOk(is.dateBetween(earlyDate, tDate, laterDate), 'Is dateBetween false when before dates') 117 | t.notOk(is.dateBetween(laterDate, earlyDate, tDate), 'Is dateBetween false when after dates') 118 | t.ok(is.dateBetween(tDate, earlyDate, laterDate), 'Is dateBetween true when between dates') 119 | t.ok(is.log(log), 'Is log true with mock log') 120 | log.date = 'not a date' 121 | t.notOk(is.log(log), 'Is log false with invalid date') 122 | log.date = new Date() 123 | delete log.queueId 124 | t.notOk(is.log(log), 'Is log false with no queueId') 125 | log.queueId = 'queue id string' 126 | delete log.type 127 | t.notOk(is.log(log), 'Is log false with no type') 128 | log.type = 'type string' 129 | delete log.status 130 | t.notOk(is.log(log), 'Is log false with no status`') 131 | }) 132 | } 133 | -------------------------------------------------------------------------------- /tests/job-options.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const tError = require('./test-error') 3 | const enums = require('../src/enums') 4 | const jobOptions = require('../src/job-options') 5 | 6 | jobOptionsTests() 7 | function jobOptionsTests () { 8 | test('job-options', (t) => { 9 | t.plan(70) 10 | 11 | try { 12 | let to = jobOptions() 13 | t.equal(to.name, null, 'Job default name option is null') 14 | t.equal(to.priority, 'normal', 'Job default priority option is normal') 15 | t.equal(to.timeout, enums.options.timeout, 'Job default timeout option is valid') 16 | t.equal(to.retryMax, enums.options.retryMax, 'Job default retryMax option is valid') 17 | t.equal(to.retryDelay, enums.options.retryDelay, 'Job default retryDelay option is valid') 18 | t.equal(to.repeat, enums.options.repeat, 'Job default repeat option is valid') 19 | t.equal(to.repeatDelay, enums.options.repeatDelay, 'Job default repeatDelay option is valid') 20 | to = jobOptions({ 21 | name: 'one', 22 | priority: 'high', 23 | timeout: 100000, 24 | retryMax: 8, 25 | retryDelay: 200000, 26 | repeat: 4, 27 | repeatDelay: 1000 28 | }, to) 29 | t.equal(to.name, 'one', 'Job custom name option is valid') 30 | t.equal(to.priority, 'high', 'Job custom priority option is valid') 31 | t.equal(to.timeout, 100000, 'Job custom timeout option is valid') 32 | t.equal(to.retryMax, 8, 'Job custom retryMax option is valid') 33 | t.equal(to.retryDelay, 200000, 'Job custom retryDelay option is valid') 34 | t.equal(to.repeat, 4, 'Job custom repeat option is valid') 35 | t.equal(to.repeatDelay, 1000, 'Job custom repeatDelay option is valid') 36 | 37 | to = jobOptions({ name: 'two' }, to) 38 | t.equal(to.name, 'two', 'Job name custom name option is valid') 39 | t.equal(to.priority, 'high', 'Job priority custom priority option is valid') 40 | t.equal(to.timeout, 100000, 'Job priority custom timeout option is valid') 41 | t.equal(to.retryMax, 8, 'Job priority custom retryMax option is valid') 42 | t.equal(to.retryDelay, 200000, 'Job priority custom retryDelay option is valid') 43 | t.equal(to.repeat, 4, 'Job priority custom repeat option is valid') 44 | t.equal(to.repeatDelay, 1000, 'Job priority custom repeatDelay option is valid') 45 | 46 | to = jobOptions({ priority: 'lowest' }, to) 47 | t.equal(to.name, 'two', 'Job name custom name option is valid') 48 | t.equal(to.priority, 'lowest', 'Job priority custom priority option is valid') 49 | t.equal(to.timeout, 100000, 'Job priority custom timeout option is valid') 50 | t.equal(to.retryMax, 8, 'Job priority custom retryMax option is valid') 51 | t.equal(to.retryDelay, 200000, 'Job priority custom retryDelay option is valid') 52 | t.equal(to.repeat, 4, 'Job priority custom repeat option is valid') 53 | t.equal(to.repeatDelay, 1000, 'Job priority custom repeatDelay option is valid') 54 | 55 | to = jobOptions({ timeout: 700000 }, to) 56 | t.equal(to.name, 'two', 'Job name custom name option is valid') 57 | t.equal(to.priority, 'lowest', 'Job timeout custom priority option is valid') 58 | t.equal(to.timeout, 700000, 'Job timeout custom timeout option is valid') 59 | t.equal(to.retryMax, 8, 'Job timeout custom retryMax option is valid') 60 | t.equal(to.retryDelay, 200000, 'Job timeout custom retryDelay option is valid') 61 | t.equal(to.repeat, 4, 'Job timeout custom repeat option is valid') 62 | t.equal(to.repeatDelay, 1000, 'Job timeout custom repeatDelay option is valid') 63 | 64 | to = jobOptions({ retryMax: 2 }, to) 65 | t.equal(to.name, 'two', 'Job name custom name option is valid') 66 | t.equal(to.priority, 'lowest', 'Job retryMax custom priority option is valid') 67 | t.equal(to.timeout, 700000, 'Job retryMax custom timeout option is valid') 68 | t.equal(to.retryMax, 2, 'Job retryMax custom retryMax option is valid') 69 | t.equal(to.retryDelay, 200000, 'Job retryMax custom retryDelay option is valid') 70 | t.equal(to.repeat, 4, 'Job retryMax custom repeat option is valid') 71 | t.equal(to.repeatDelay, 1000, 'Job retryMax custom repeatDelay option is valid') 72 | 73 | to = jobOptions({ retryDelay: 800000 }, to) 74 | t.equal(to.name, 'two', 'Job name custom name option is valid') 75 | t.equal(to.priority, 'lowest', 'Job retryDelay custom priority option is valid') 76 | t.equal(to.timeout, 700000, 'Job retryDelay custom timeout option is valid') 77 | t.equal(to.retryMax, 2, 'Job retryDelay custom retryMax option is valid') 78 | t.equal(to.retryDelay, 800000, 'Job retryDelay custom retryDelay option is valid') 79 | t.equal(to.repeat, 4, 'Job retryDelay custom repeat option is valid') 80 | t.equal(to.repeatDelay, 1000, 'Job retryDelay custom repeatDelay option is valid') 81 | 82 | to = jobOptions({ repeat: false }, to) 83 | t.equal(to.name, 'two', 'Job name custom name option is valid') 84 | t.equal(to.priority, 'lowest', 'Job repeat custom priority option is valid') 85 | t.equal(to.timeout, 700000, 'Job repeat custom timeout option is valid') 86 | t.equal(to.retryMax, 2, 'Job repeat custom retryMax option is valid') 87 | t.equal(to.retryDelay, 800000, 'Job repeat custom retryDelay option is valid') 88 | t.equal(to.repeat, false, 'Job repeat custom repeat option is valid') 89 | t.equal(to.repeatDelay, 1000, 'Job repeat custom repeatDelay option is valid') 90 | 91 | to = jobOptions({ repeatDelay: 2000 }, to) 92 | t.equal(to.name, 'two', 'Job name custom name option is valid') 93 | t.equal(to.priority, 'lowest', 'Job repeatDelay custom priority option is valid') 94 | t.equal(to.timeout, 700000, 'Job repeatDelay custom timeout option is valid') 95 | t.equal(to.retryMax, 2, 'Job repeatDelay custom retryMax option is valid') 96 | t.equal(to.retryDelay, 800000, 'Job repeatDelay custom retryDelay option is valid') 97 | t.equal(to.repeat, false, 'Job repeatDelay custom repeat option is valid') 98 | t.equal(to.repeatDelay, 2000, 'Job repeatDelay custom repeatDelay option is valid') 99 | 100 | to = jobOptions({ 101 | name: true, 102 | priority: 'oops', 103 | timeout: -20, 104 | retryMax: -30, 105 | retryDelay: -40, 106 | repeat: -50, 107 | repeatDelay: -60 108 | }, to) 109 | t.equal(to.name, 'two', 'Job invalid name option is reverted') 110 | t.equal(to.priority, 'lowest', 'Job invalid priority option is reverted') 111 | t.equal(to.timeout, 700000, 'Job invalid timeout option is reverted') 112 | t.equal(to.retryMax, 2, 'Job invalid retryMax option is reverted') 113 | t.equal(to.retryDelay, 800000, 'Job invalid retryDelay option is reverted') 114 | t.equal(to.repeat, false, 'Job invalid repeat option is reverted') 115 | t.equal(to.repeatDelay, 2000, 'Job invalid repeatDelay option is reverted') 116 | } catch (err) { 117 | tError(err, module, t) 118 | } 119 | }) 120 | } 121 | -------------------------------------------------------------------------------- /tests/job-log.spec.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | const Promise = require('bluebird') 3 | const is = require('../src/is') 4 | const tError = require('./test-error') 5 | const enums = require('../src/enums') 6 | const jobLog = require('../src/job-log') 7 | const tData = require('./test-options').tData 8 | const Queue = require('../src/queue') 9 | const tOpts = require('./test-options') 10 | const eventHandlers = require('./test-event-handlers') 11 | const testName = 'job-log' 12 | 13 | jogLogTests() 14 | function jogLogTests () { 15 | return new Promise((resolve, reject) => { 16 | test(testName, (t) => { 17 | t.plan(76) 18 | 19 | const q = new Queue(tOpts.cxn(), tOpts.default('jobLog')) 20 | let job = q.createJob() 21 | job.detail = tData 22 | let logObject = { foo: 'bar' } 23 | 24 | // ---------- Event Handler Setup ---------- 25 | let state = { 26 | testName, 27 | enabled: false, 28 | ready: 0, 29 | processing: 0, 30 | progress: 0, 31 | pausing: 0, 32 | paused: 0, 33 | resumed: 0, 34 | removed: 0, 35 | reset: 1, 36 | error: 0, 37 | reviewed: 0, 38 | detached: 0, 39 | stopping: 0, 40 | stopped: 0, 41 | dropped: 0, 42 | added: 1, 43 | waiting: 0, 44 | active: 0, 45 | completed: 0, 46 | cancelled: 0, 47 | failed: 0, 48 | terminated: 0, 49 | reanimated: 0, 50 | log: 4, 51 | updated: 0 52 | } 53 | 54 | return q.reset().then((resetResult) => { 55 | t.ok(is.integer(resetResult), 'Queue reset') 56 | eventHandlers.add(t, q, state) 57 | return q.addJob(job) 58 | }).then((newJob) => { 59 | job = newJob[0] 60 | t.equal(job.status, enums.status.waiting, 'New job added successfully') 61 | 62 | // ---------- Add First Log Tests ---------- 63 | t.comment('job-log: Add First Log') 64 | return jobLog.commitLog(job, tData, tData) 65 | }).then((updateResult1) => { 66 | t.ok(updateResult1, 'Log 1 added to job successfully') 67 | return q.getJob(job.id) 68 | }).then((jobWithLog1) => { 69 | t.equal(jobWithLog1[0].log.length, 2, 'Log 1 exists on retrieved job') 70 | t.ok(is.date(jobWithLog1[0].log[1].date), 'Log 1 date is a date') 71 | t.equal(jobWithLog1[0].log[1].queueId, q.id, 'Log 1 queueId is valid') 72 | t.equal(jobWithLog1[0].log[1].type, enums.log.information, 'Log 1 type is information') 73 | t.equal(jobWithLog1[0].log[1].status, enums.status.waiting, 'Log 1 status is added') 74 | t.ok(jobWithLog1[0].log[1].retryCount >= 0, 'Log retryCount is valid') 75 | t.ok(jobWithLog1[0].log[1].processCount >= 0, 'Log processCount is valid') 76 | t.equal(jobWithLog1[0].log[1].message, tData, 'Log 1 message is valid') 77 | t.equal(jobWithLog1[0].log[1].data, tData, 'Log 1 detail is valid') 78 | t.equal(jobWithLog1[0].getLastLog(), jobWithLog1[0].log[1], 'Last log entry is correctly retrieved') 79 | 80 | // ---------- Add Second Log Tests ---------- 81 | t.comment('job-log: Add Second Log') 82 | return jobLog.commitLog(job, tData, tData) 83 | }).then((updateResult2) => { 84 | t.ok(updateResult2, 'Log 2 added to job successfully') 85 | return q.getJob(job.id) 86 | }).then((jobWithLog2) => { 87 | t.equal(jobWithLog2[0].log.length, 3, 'Log 2 exists on retrieved job') 88 | t.ok(is.date(jobWithLog2[0].log[2].date), 'Log 2 date is a date') 89 | t.equal(jobWithLog2[0].log[2].queueId, q.id, 'Log 2 queueId is valid') 90 | t.equal(jobWithLog2[0].log[2].type, enums.log.information, 'Log 2 type is information') 91 | t.equal(jobWithLog2[0].log[2].status, enums.status.waiting, 'Log 2 status is waiting') 92 | t.ok(jobWithLog2[0].log[2].retryCount >= 0, 'Log retryCount is valid') 93 | t.ok(jobWithLog2[0].log[2].processCount >= 0, 'Log processCount is valid') 94 | t.equal(jobWithLog2[0].log[2].message, tData, 'Log 2 message is valid') 95 | t.equal(jobWithLog2[0].log[2].data, tData, 'Log 2 data is valid') 96 | t.equal(jobWithLog2[0].getLastLog(), jobWithLog2[0].log[2], 'Last log entry is correctly retrieved') 97 | 98 | // ---------- Add Log with Defaults Tests ---------- 99 | t.comment('job-log: Add Log with Defaults') 100 | return jobLog.commitLog(job) 101 | }).then((updateResult3) => { 102 | t.ok(updateResult3, 'Log 3 added to job successfully') 103 | return q.getJob(job.id) 104 | }).then((jobWithLog3) => { 105 | t.equal(jobWithLog3[0].log.length, 4, 'Log 3 exists on retrieved job') 106 | t.ok(is.date(jobWithLog3[0].log[3].date), 'Log 3 date is a date') 107 | t.equal(jobWithLog3[0].log[3].queueId, q.id, 'Log 3 queueId is valid') 108 | t.equal(jobWithLog3[0].log[3].type, enums.log.information, 'Log 3 type is information') 109 | t.equal(jobWithLog3[0].log[3].status, enums.status.waiting, 'Log 3 status is waiting') 110 | t.ok(jobWithLog3[0].log[3].retryCount >= 0, 'Log retryCount is valid') 111 | t.ok(jobWithLog3[0].log[3].processCount >= 0, 'Log processCount is valid') 112 | t.equal(jobWithLog3[0].log[3].message, enums.message.seeLogData, 'Log 3 message is valid') 113 | t.ok(is.object(jobWithLog3[0].log[3].data), 'Log 3 data is valid') 114 | t.equal(jobWithLog3[0].getLastLog(), jobWithLog3[0].log[3], 'Last log entry is correctly retrieved') 115 | 116 | // ---------- Add Object Log Tests ---------- 117 | t.comment('job-log: Add Object Log') 118 | return jobLog.commitLog(job, logObject) 119 | }).then((updateResult4) => { 120 | t.ok(updateResult4, 'Log 4 added to job successfully') 121 | return q.getJob(job.id) 122 | }).then((jobWithLog4) => { 123 | t.equal(jobWithLog4[0].log.length, 5, 'Log 4 exists on retrieved job') 124 | t.ok(is.date(jobWithLog4[0].log[4].date), 'Log 4 date is a date') 125 | t.equal(jobWithLog4[0].log[4].queueId, q.id, 'Log 4 queueId is valid') 126 | t.equal(jobWithLog4[0].log[4].type, enums.log.information, 'Log 4 type is information') 127 | t.equal(jobWithLog4[0].log[4].status, enums.status.waiting, 'Log 4 status is added') 128 | t.ok(jobWithLog4[0].log[4].retryCount >= 0, 'Log retryCount is valid') 129 | t.ok(jobWithLog4[0].log[4].processCount >= 0, 'Log processCount is valid') 130 | t.equal(jobWithLog4[0].log[4].message, enums.message.seeLogData, 'Log 4 message is valid') 131 | t.equal(jobWithLog4[0].log[4].data.foo, 'bar', 'Log 4 data object is valid') 132 | t.equal(jobWithLog4[0].getLastLog(), jobWithLog4[0].log[4], 'Last log entry is correctly retrieved') 133 | 134 | return q.reset() 135 | }).then((resetResult) => { 136 | t.ok(resetResult >= 0, 'Queue reset') 137 | 138 | // ---------- Event Summary ---------- 139 | eventHandlers.remove(t, q, state) 140 | q.stop() 141 | return resolve(t.end()) 142 | }).catch(err => tError(err, module, t)) 143 | }) 144 | }) 145 | } 146 | -------------------------------------------------------------------------------- /src/queue.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const EventEmitter = require('events').EventEmitter 3 | const is = require('./is') 4 | const enums = require('./enums') 5 | const errorBooster = require('./error-booster') 6 | const Job = require('./job') 7 | const dbReview = require('./db-review') 8 | const queueDb = require('./queue-db') 9 | const queueProcess = require('./queue-process') 10 | const queueAddJob = require('./queue-add-job') 11 | const queueGetJob = require('./queue-get-job') 12 | const queueFindJob = require('./queue-find-job') 13 | const queueFindJobByName = require('./queue-find-job-by-name') 14 | const queueInterruption = require('./queue-interruption') 15 | const queueCancelJob = require('./queue-cancel-job') 16 | const queueReanimateJob = require('./queue-reanimate-job') 17 | const queueRemoveJob = require('./queue-remove-job') 18 | const queueReset = require('./queue-reset') 19 | const queueSummary = require('./queue-summary') 20 | const queueStop = require('./queue-stop') 21 | const queueDrop = require('./queue-drop') 22 | const jobOptions = require('./job-options') 23 | 24 | class Queue extends EventEmitter { 25 | constructor (cxn, options) { 26 | super() 27 | logger('Queue Constructor', options) 28 | 29 | options = options || {} 30 | this._name = options.name || enums.options.name 31 | 32 | // The following properties are Populated 33 | // by the queueDb.attach(this, cxn) call below. 34 | this._r = false 35 | this._host = '' 36 | this._port = 0 37 | this._db = '' 38 | this._id = '' 39 | this._ready = false 40 | 41 | this._queryRunOptions = options.queryRunOptions == null 42 | ? enums.options.queryRunOptions : options.queryRunOptions 43 | this._masterInterval = options.masterInterval == null 44 | ? enums.options.masterInterval : options.masterInterval 45 | this._databaseInitDelay = options.databaseInitDelay == null 46 | ? enums.options.databaseInitDelay : options.databaseInitDelay 47 | this._jobOptions = jobOptions() 48 | this._changeFeedCursor = false 49 | this._paused = false 50 | this._running = 0 51 | this._changeFeed = options.changeFeed == null 52 | ? true : options.changeFeed 53 | this._concurrency = options.concurrency > 1 54 | ? options.concurrency : enums.options.concurrency 55 | this._limitJobLogs = options.limitJobLogs == null 56 | ? enums.options.limitJobLogs : options.limitJobLogs 57 | this._removeFinishedJobs = options.removeFinishedJobs == null 58 | ? enums.options.removeFinishedJobs : options.removeFinishedJobs 59 | this._handler = false 60 | queueDb.attach(this, cxn) 61 | } 62 | 63 | get name () { return this._name } 64 | get id () { return this._id } 65 | get host () { return this._host } 66 | get port () { return this._port } 67 | get db () { return this._db } 68 | get r () { return this._r } 69 | get changeFeed () { return this._changeFeed } 70 | get master () { return this._masterInterval > 0 } 71 | get masterInterval () { return this._masterInterval } 72 | get jobOptions () { return this._jobOptions } 73 | get limitJobLogs () { return this._limitJobLogs } 74 | get removeFinishedJobs () { return this._removeFinishedJobs } 75 | get running () { return this._running } 76 | get concurrency () { return this._concurrency } 77 | get paused () { return this._paused } 78 | get idle () { return this._running < 1 } 79 | 80 | set jobOptions (options) { 81 | logger('set jobOptions', options) 82 | this._jobOptions = jobOptions(options, this._jobOptions) 83 | } 84 | 85 | set concurrency (value) { 86 | logger('set concurrency', value) 87 | if (!is.integer(value) || value < 1) { 88 | const err = new Error(enums.message.concurrencyInvalid) 89 | err.queueId = this.id 90 | logger('concurrency', this.id, err) 91 | this.emit(enums.status.error, err) 92 | return 93 | } 94 | this._concurrency = value 95 | } 96 | 97 | createJob (jobData) { 98 | logger('createJob', jobData) 99 | jobData = jobData == null ? this.jobOptions : jobData 100 | return new Job(this, jobData) 101 | } 102 | 103 | addJob (job) { 104 | logger('addJob', job) 105 | return this.ready().then(() => { 106 | return queueAddJob(this, job) 107 | }).catch(errorBooster(this, logger, 'addJob')) 108 | } 109 | 110 | getJob (jobOrId) { 111 | logger('getJob', jobOrId) 112 | return this.ready().then(() => { 113 | return queueGetJob(this, jobOrId) 114 | }).catch(errorBooster(this, logger, 'getJob')) 115 | } 116 | 117 | findJob (predicate, raw) { 118 | logger('findJob', predicate, raw) 119 | return this.ready().then(() => { 120 | return queueFindJob(this, predicate, raw) 121 | }).catch(errorBooster(this, logger, 'findJob')) 122 | } 123 | 124 | findJobByName (name, raw) { 125 | logger('findJobByName', name, raw) 126 | return this.ready().then(() => { 127 | return queueFindJobByName(this, name, raw) 128 | }).catch(errorBooster(this, logger, 'findJobByName')) 129 | } 130 | 131 | containsJobByName (name) { 132 | logger('containsJobByName', name) 133 | return this.ready().then(() => { 134 | return queueFindJobByName(this, name, true) 135 | }).then((namedJobs) => { 136 | return namedJobs.length > 0 137 | }).catch(errorBooster(this, logger, 'containsJobByName')) 138 | } 139 | 140 | cancelJob (jobOrId, reason) { 141 | logger('cancelJob', jobOrId, reason) 142 | return this.ready().then(() => { 143 | return queueCancelJob(this, jobOrId, reason) 144 | }).catch(errorBooster(this, logger, 'cancelJob')) 145 | } 146 | 147 | reanimateJob (jobOrId, dateEnable) { 148 | logger('reanimateJob', jobOrId, dateEnable) 149 | return this.ready().then(() => { 150 | return queueReanimateJob(this, jobOrId, dateEnable) 151 | }).catch(errorBooster(this, logger, 'reanimateJob')) 152 | } 153 | 154 | removeJob (jobOrId) { 155 | logger('removeJob', jobOrId) 156 | return this.ready().then(() => { 157 | return queueRemoveJob(this, jobOrId) 158 | }).catch(errorBooster(this, logger, 'removeJob')) 159 | } 160 | 161 | process (handler) { 162 | logger('process', handler) 163 | return this.ready().then(() => { 164 | return queueProcess.addHandler(this, handler) 165 | }).catch(errorBooster(this, logger, 'process')) 166 | } 167 | 168 | review () { 169 | logger('review') 170 | return this.ready().then(() => { 171 | return dbReview.runOnce(this) 172 | }).catch(errorBooster(this, logger, 'review')) 173 | } 174 | 175 | summary () { 176 | logger('summary') 177 | return this.ready().then(() => { 178 | return queueSummary(this) 179 | }).catch(errorBooster(this, logger, 'summary')) 180 | } 181 | 182 | ready () { 183 | logger('ready') 184 | return this._ready 185 | } 186 | 187 | pause (global) { 188 | logger(`pause`) 189 | return this.ready().then(() => { 190 | return queueInterruption.pause(this, global) 191 | }).catch(errorBooster(this, logger, 'pause')) 192 | } 193 | 194 | resume (global) { 195 | logger(`resume`) 196 | return this.ready().then(() => { 197 | return queueInterruption.resume(this, global) 198 | }).catch(errorBooster(this, logger, 'resume')) 199 | } 200 | 201 | reset () { 202 | logger('reset') 203 | return this.ready().then(() => { 204 | return queueReset(this) 205 | }).catch(errorBooster(this, logger, 'reset')) 206 | } 207 | 208 | stop () { 209 | logger('stop') 210 | return queueStop(this).then(() => { 211 | return queueDb.drain(this) 212 | }).catch(errorBooster(this, logger, 'stop')) 213 | } 214 | 215 | drop () { 216 | logger('drop') 217 | return queueDrop(this) 218 | .catch(errorBooster(this, logger, 'drop')) 219 | } 220 | } 221 | 222 | // Make TypeScript compiler happy. Needed because Job is an abstract class. 223 | Queue.Job = Job 224 | 225 | module.exports = Queue 226 | -------------------------------------------------------------------------------- /src/queue-process.js: -------------------------------------------------------------------------------- 1 | const logger = require('./logger')(module) 2 | const Promise = require('bluebird') 3 | const enums = require('./enums') 4 | const is = require('./is') 5 | const errorBooster = require('./error-booster') 6 | const queueGetNextJob = require('./queue-get-next-job') 7 | const jobLog = require('./job-log') 8 | const jobCompleted = require('./job-completed') 9 | const jobUpdate = require('./job-update') 10 | const queueCancelJob = require('./queue-cancel-job') 11 | const jobFailed = require('./job-failed') 12 | const jobTimeouts = new Map() 13 | const jobOnCancelHandlers = new Map() 14 | 15 | function addJobTimeout (job, timeoutHandler) { 16 | logger('addJobTimeout', job) 17 | const timeoutValue = job.timeout 18 | let jobTimeout = { 19 | timeoutHandler, 20 | timeoutValue, 21 | timeoutId: setTimeout(timeoutHandler, timeoutValue) 22 | } 23 | jobTimeouts.set(job.id, jobTimeout) 24 | } 25 | 26 | function addOnCancelHandler (job, cancellationCallback) { 27 | logger('addJobCancellation', job.id) 28 | if (is.function(cancellationCallback)) { 29 | jobOnCancelHandlers.set(job.id, cancellationCallback) 30 | } else { 31 | let err = new Error(enums.message.cancelCallbackInvalid) 32 | err.queueId = job.q.id 33 | logger(`Event: addOnCancelHandler error`, err, job.q.id) 34 | job.q.emit(enums.status.error, err) 35 | throw err 36 | } 37 | } 38 | 39 | function removeJobTimeoutAndOnCancelHandler (jobId) { 40 | logger('removeJobTimeoutAndOnCancelHandler', jobId) 41 | if (jobTimeouts.has(jobId)) { 42 | const jobTimeout = jobTimeouts.get(jobId) 43 | clearTimeout(jobTimeout.timeoutId) 44 | jobTimeouts.delete(jobId) 45 | } 46 | jobOnCancelHandlers.delete(jobId) 47 | } 48 | 49 | function onCancelJob (jobId, q) { 50 | logger('onCancelJob', jobId) 51 | if (jobOnCancelHandlers.has(jobId)) { 52 | const onCancelHandler = jobOnCancelHandlers.get(jobId) 53 | removeJobTimeoutAndOnCancelHandler(jobId) 54 | q._running-- 55 | // Calling the user defined cancel function 56 | onCancelHandler() 57 | setImmediate(jobTick, q) 58 | } 59 | } 60 | 61 | function restartJobTimeout (queueId, jobId) { 62 | logger('resetJobTimeout', queueId, jobId) 63 | let jobTimeout 64 | if (jobTimeouts.has(jobId)) { 65 | jobTimeout = jobTimeouts.get(jobId) 66 | clearTimeout(jobTimeout.timeoutId) 67 | jobTimeout.timeoutId = setTimeout( 68 | jobTimeout.timeoutHandler, 69 | jobTimeout.timeoutValue) 70 | } 71 | } 72 | 73 | function jobRun (job) { 74 | logger('jobRun', `Running: [${job.q.running}]`) 75 | let handled = false 76 | 77 | function nextHandler (err, jobResult) { 78 | logger('nextHandler', `Running: [${job.q.running}]`, err, jobResult) 79 | logger('handled', handled) 80 | // Ignore mulpiple calls to next() 81 | if (handled) { 82 | return Promise.resolve(job.q.running) 83 | } 84 | handled = true 85 | 86 | removeJobTimeoutAndOnCancelHandler(job.id) 87 | 88 | function cancelJobAction () { 89 | logger('Job is being cancelled') 90 | return queueCancelJob(job.q, job, err.cancelJob) 91 | } 92 | 93 | function errAction (err) { 94 | logger('jobResult is an error') 95 | const isCancelJob = (is.object(err) || is.error(err)) && err.cancelJob 96 | return isCancelJob ? cancelJobAction() 97 | : jobFailed(job, err) 98 | } 99 | 100 | function setJobStatusToWaiting () { 101 | logger('Invalid job status', job.status) 102 | logger('Setting job status to: ' + enums.status.waiting) 103 | job.status = enums.status.waiting 104 | } 105 | 106 | function updateJobAction (jobToUpdate) { 107 | logger('Job is being updated') 108 | jobToUpdate.status === enums.status.active && setJobStatusToWaiting() 109 | let newLog = jobLog.createLogObject(job, {}, enums.message.jobPassBack) 110 | job.log.push(newLog) 111 | return jobUpdate(jobToUpdate) 112 | } 113 | 114 | function resultAction (validJobResult) { 115 | logger('jobResult is valid') 116 | let resultIsJob = is.job(validJobResult) && is.object(validJobResult.q) 117 | return resultIsJob 118 | ? updateJobAction(validJobResult) 119 | : jobCompleted(job, validJobResult) 120 | } 121 | 122 | let returnPromise = err ? errAction(err) : resultAction(jobResult) 123 | 124 | return returnPromise.then((finalResult) => { 125 | job.q._running-- 126 | setImmediate(jobTick, job.q) 127 | return job.q.running 128 | }).catch(errorBooster(job.q, logger, 'next() Promise')) 129 | } 130 | 131 | function timeoutHandler () { 132 | logger('timeoutHandler called, job timeout value exceeded', job.timeout) 133 | const timedOutMessage = `Job timed out (run time > ${job.timeout} ms)` 134 | nextHandler(new Error(timedOutMessage)) 135 | } 136 | 137 | addJobTimeout(job, timeoutHandler) 138 | logger(`Event: processing [${job.id}]`) 139 | job.q.emit(enums.status.processing, job.q.id, job.id) 140 | logger('calling handler function') 141 | job.q._handler(job, nextHandler, addOnCancelHandler) 142 | } 143 | 144 | const jobTick = function jobTick (q) { 145 | logger('jobTick') 146 | logger(`Running: [${q.running}]`) 147 | logger(`_getNextJobActive [${q._getNextJobActive}]`) 148 | logger(`_getNextJobCalled [${q._getNextJobCalled}]`) 149 | if (q._getNextJobActive) { q._getNextJobCalled = true } 150 | if (q.paused || q._getNextJobActive) { return } 151 | 152 | function getNextJobCleanup (runAgain) { 153 | logger(`getNextJobCleanup`) 154 | logger(`runAgain: [${runAgain}]`) 155 | logger(`Running: [${q.running}]`) 156 | q._getNextJobActive = false 157 | q._getNextJobCalled = false 158 | if (q.running < q.concurrency && runAgain) { 159 | // q._running has been decremented whilst talking to the database. 160 | setImmediate(jobTick, q) 161 | return 162 | } 163 | if (q.idle && !runAgain) { 164 | // No running jobs and no jobs in the database, we are idle. 165 | logger(`Event: idle [${q.id}]`) 166 | q.emit(enums.status.idle, q.id) 167 | } 168 | } 169 | 170 | // q._getNextJobActive stops jobs that finish at the same time causing 171 | // multiple database queries and breaking concurrency. 172 | // This is an issue because the q._running++ is not incremented until 173 | // after the async database query has finished. 174 | // If a call to jobTick is made whilst the getNextJob query is active, 175 | // then q._getNextJobCalled is flagged to initiate another call 176 | // on completion of the getNextJob database query. 177 | q._getNextJobActive = true 178 | return queueGetNextJob(q).then((jobsToDo) => { 179 | logger('jobsToDo', `Retrieved: [${jobsToDo.length}]`) 180 | if (jobsToDo.length > 0) { 181 | q._running += jobsToDo.length 182 | jobsToDo.forEach(j => jobRun(j)) 183 | } 184 | getNextJobCleanup(q._getNextJobCalled) 185 | return null 186 | }).catch((err) => { 187 | getNextJobCleanup(q._getNextJobCalled) 188 | return errorBooster(q, logger, 'queueGetNextJob')(err) 189 | }) 190 | } 191 | 192 | module.exports.addHandler = function queueProcessAddHandler (q, handler) { 193 | logger('addHandler') 194 | 195 | if (q._handler) { 196 | return Promise.reject(new Error(enums.message.processTwice)) 197 | } 198 | 199 | q._handler = handler 200 | q._running = 0 201 | q.on(enums.status.progress, restartJobTimeout) 202 | q.on(enums.status.cancelled, (queueId, jobId) => onCancelJob(jobId, q)) 203 | 204 | // Returning a Promise so the jobTick is initiated 205 | // after the dbReview process. The Promise can be ignored. 206 | return Promise.resolve().then(() => { 207 | if (q.master) { return true } 208 | return q.review() 209 | }).then(() => { 210 | setImmediate(jobTick, q) 211 | return true 212 | }) 213 | } 214 | 215 | module.exports.restart = function queueProcessRestart (q) { 216 | logger('restart', `Running: [${q.running}]`) 217 | if (!q._handler) { return } 218 | if (q.running < q.concurrency) { 219 | setImmediate(jobTick, q) 220 | } 221 | } 222 | --------------------------------------------------------------------------------