├── .travis.yml ├── lib ├── errors.js ├── schema.sql ├── sql │ └── nextJob.sql ├── job.js └── index.js ├── .jshintrc ├── .gitignore ├── test ├── support │ ├── common.js │ └── db.js ├── interface.js └── queue.js ├── bin ├── install-schema ├── add-job └── process-job-queue ├── LICENSE ├── package.json └── README.md /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "4" 4 | - "5" 5 | - "6" 6 | addons: 7 | postgresql: "9.4" 8 | -------------------------------------------------------------------------------- /lib/errors.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const util = require('util') 3 | 4 | function JobQueueEmpty() { 5 | Error.captureStackTrace(this, this.constructor) 6 | this.name = this.constructor.name 7 | } 8 | 9 | util.inherits(JobQueueEmpty, Error) 10 | 11 | exports.JobQueueEmpty = JobQueueEmpty 12 | 13 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "esnext": true, 3 | "moz": true, 4 | "boss": true, 5 | "node": true, 6 | "validthis": true, 7 | "globals": { 8 | "EventEmitter": true, 9 | "Promise": true 10 | }, 11 | "asi": true, 12 | "globals": { 13 | "Promise": true 14 | }, 15 | "unused" : "vars" 16 | } 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # Dependency directories 18 | node_modules 19 | 20 | # Optional npm cache directory 21 | .npm -------------------------------------------------------------------------------- /test/support/common.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | global.Promise = require('bluebird') 4 | global.sinon = require('sinon') 5 | global.chai = require("chai") 6 | global.expect = chai.expect 7 | global.assert = chai.assert 8 | 9 | var chaiAsPromised = require('chai-as-promised') 10 | chai.use(chaiAsPromised) 11 | 12 | require('sinon-as-promised')(Promise) 13 | 14 | require('co-mocha') 15 | chai.use(require('chai-datetime')) -------------------------------------------------------------------------------- /lib/schema.sql: -------------------------------------------------------------------------------- 1 | CREATE TYPE jobState AS ENUM ('waiting', 'processing', 'failed', 'finished'); 2 | 3 | CREATE TABLE "JobQueue" 4 | ( 5 | "id" serial NOT NULL, 6 | "state" jobState NOT NULL DEFAULT 'waiting', 7 | "type" character varying NOT NULL, 8 | "data" json, 9 | "scheduledFor" timestamp with time zone NOT NULL, 10 | "failedAttempts" integer DEFAULT 0, 11 | "lastFailureMessage" character varying, 12 | "maxAttempts" integer DEFAULT 1, 13 | "createdAt" timestamp with time zone NOT NULL DEFAULT now(), 14 | "lastRun" timestamp with time zone, 15 | CONSTRAINT "JobQueue_pkey" PRIMARY KEY (id) 16 | ) 17 | WITH ( 18 | OIDS=FALSE 19 | ); 20 | -------------------------------------------------------------------------------- /test/support/db.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | var jobqueue = require('../../lib') 3 | 4 | const connectionString = 'postgres://postgres@localhost/job-queue-test' 5 | 6 | var pgDestroyCreate = require('pg-destroy-create-db')(connectionString) 7 | var destroyCreate = Promise.promisify(pgDestroyCreate.destroyCreate, {context: pgDestroyCreate}) 8 | 9 | 10 | exports.destroyAndCreate = function() { 11 | // destroy and create database 12 | return destroyCreate() 13 | .then(() => { 14 | var queue = new jobqueue(connectionString) 15 | // import the schema 16 | return queue.installSchema().then(() => { 17 | return queue 18 | }) 19 | }) 20 | } 21 | 22 | exports.connectionString = connectionString -------------------------------------------------------------------------------- /bin/install-schema: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 'use strict' 3 | require('string-format').extend(String.prototype) 4 | const program = require('commander') 5 | const jobqueue = require('../lib') 6 | const cluster = require('cluster') 7 | 8 | program 9 | .version('0.0.1') 10 | .option( 11 | '-c, --connection-string ', 12 | 'postgresql connection string. defaults to postgres://postgres@localhost/pg-job-queue', 13 | 'postgres://postgres@localhost/pg-job-queue' 14 | ) 15 | // .option( 16 | // '-c, --create-database', 17 | // 'create the database if it does not exist', 18 | // false 19 | // ) 20 | 21 | program.parse(process.argv) 22 | 23 | 24 | var queue = new jobqueue(program.connectionString) 25 | queue.installSchema() 26 | .then(() => { 27 | return queue.disconnect() 28 | }) 29 | .catch((e) => { 30 | console.error(e.stack) 31 | queue.disconnect() 32 | }) 33 | -------------------------------------------------------------------------------- /bin/add-job: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 'use strict' 3 | require('string-format').extend(String.prototype) 4 | const program = require('commander') 5 | const jobqueue = require('../lib') 6 | 7 | program 8 | .version('0.0.1') 9 | .option( 10 | '--type ', 11 | 'job type' 12 | ) 13 | .option( 14 | '--data ', 15 | 'job data (json)', 16 | {} 17 | ) 18 | .option( 19 | '-c, --connection-string ', 20 | 'postgresql connection string. defaults to postgres://postgres@localhost/pg-job-queue', 21 | 'postgres://postgres@localhost/pg-job-queue' 22 | ) 23 | 24 | program.parse(process.argv) 25 | 26 | 27 | if (!program.type) { 28 | return program.outputHelp() 29 | } 30 | 31 | const queue = new jobqueue(program.connectionString) 32 | 33 | var job = { 34 | type: program.type, 35 | data: program.data 36 | } 37 | 38 | queue.addJob(job).then(() => { 39 | queue.disconnect() 40 | }) -------------------------------------------------------------------------------- /lib/sql/nextJob.sql: -------------------------------------------------------------------------------- 1 | WITH RECURSIVE jobs AS ( 2 | SELECT (j).*, pg_try_advisory_lock((j).id) AS locked 3 | FROM ( 4 | SELECT j 5 | FROM "JobQueue" AS j 6 | WHERE "state" = 'waiting' AND 7 | (${types} = '{}' OR "type" = ANY(${types})) AND 8 | ("scheduledFor" IS NULL OR "scheduledFor" <= NOW()) 9 | ORDER BY "scheduledFor", "id" 10 | LIMIT 1 11 | ) AS t1 12 | UNION ALL ( 13 | SELECT (j).*, pg_try_advisory_lock((j).id) AS locked 14 | FROM ( 15 | SELECT ( 16 | SELECT j 17 | FROM "JobQueue" AS j 18 | WHERE "state" = 'waiting' 19 | AND (${types} = '{}' OR "type" = ANY(${types})) 20 | AND ("scheduledFor" IS NULL OR "scheduledFor" <= NOW()) 21 | AND ("scheduledFor", "id") > (jobs."scheduledFor", jobs."id") 22 | ORDER BY "scheduledFor", "id" 23 | LIMIT 1 24 | ) AS j 25 | FROM jobs 26 | WHERE jobs.id IS NOT NULL 27 | LIMIT 1 28 | ) AS t1 29 | ) 30 | ) 31 | SELECT * FROM jobs WHERE locked LIMIT 1; -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2016 James Hutchby 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 5 | 6 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 9 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pg-job-queue", 3 | "version": "0.3.3", 4 | "description": "A job queue for node.js based on PostgreSQL", 5 | "main": "lib/index.js", 6 | "bin": { 7 | "process-job-queue": "./bin/process-job-queue" 8 | }, 9 | "scripts": { 10 | "test": "mocha test" 11 | }, 12 | "repository": { 13 | "type": "git", 14 | "url": "git+https://github.com/jameshy/pg-job-queue.git" 15 | }, 16 | "keywords": [ 17 | "postgres", 18 | "postgresql", 19 | "job", 20 | "queue" 21 | ], 22 | "author": "James Hutchby", 23 | "license": "MIT", 24 | "bugs": { 25 | "url": "https://github.com/jameshy/pg-job-queue/issues" 26 | }, 27 | "homepage": "https://github.com/jameshy/pg-job-queue#readme", 28 | "dependencies": { 29 | "bluebird": "^3.3.4", 30 | "chai": "^3.5.0", 31 | "chai-as-promised": "^5.2.0", 32 | "chai-datetime": "^1.4.1", 33 | "co-mocha": "^1.1.2", 34 | "commander": "^2.9.0", 35 | "lodash": "^4.6.1", 36 | "mocha": "^2.4.5", 37 | "pg-destroy-create-db": "^1.0.2", 38 | "pg-promise": "^3.3.1", 39 | "sinon": "^1.17.3", 40 | "sinon-as-promised": "^4.0.0", 41 | "string-format": "^0.5.0" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /test/interface.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | require('./support/common') 3 | 4 | const jobqueue = require('../lib') 5 | const db = require('./support/db') 6 | 7 | 8 | describe('Interface', function() { 9 | beforeEach(function() { 10 | this.instance = new jobqueue(db.connectionString) 11 | }) 12 | it('must be possible to instantiate a new instance', function() { 13 | expect(jobqueue instanceof Function).to.be.true 14 | }) 15 | 16 | it("must have function 'disconnect'", function() { 17 | expect(this.instance.disconnect instanceof Function).to.be.true 18 | }) 19 | 20 | it("must have function 'setHandlers'", function() { 21 | expect(this.instance.setHandlers instanceof Function).to.be.true 22 | }) 23 | 24 | it("must have function 'startProcessing'", function() { 25 | expect(this.instance.startProcessing instanceof Function).to.be.true 26 | }) 27 | 28 | it("must have function 'processNextJob'", function() { 29 | expect(this.instance.processNextJob instanceof Function).to.be.true 30 | }) 31 | 32 | it("must have function 'stopProcessing'", function() { 33 | expect(this.instance.stopProcessing instanceof Function).to.be.true 34 | }) 35 | 36 | it("must have function 'addJob'", function() { 37 | expect(this.instance.addJob instanceof Function).to.be.true 38 | }) 39 | 40 | it("must have function 'installSchema'", function() { 41 | expect(this.instance.installSchema instanceof Function).to.be.true 42 | }) 43 | 44 | it("must have function 'getFailedJobs'", function() { 45 | expect(this.instance.getFailedJobs instanceof Function).to.be.true 46 | }) 47 | 48 | it("must have function 'processAllJobs'", function() { 49 | expect(this.instance.processAllJobs instanceof Function).to.be.true 50 | }) 51 | 52 | it("must have function 'waitingCount'", function() { 53 | expect(this.instance.waitingCount instanceof Function).to.be.true 54 | }) 55 | 56 | it("must export 'pgp'", function() { 57 | expect(this.instance.pgp instanceof require('pg-promise')).to.be.an.object 58 | }) 59 | 60 | }) 61 | -------------------------------------------------------------------------------- /lib/job.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const _ = require('lodash') 3 | 4 | function Job(row, queue) { 5 | _.extend(this, row) 6 | this.queue = queue 7 | this.db = queue.db 8 | this.handled = false 9 | } 10 | 11 | Job.prototype.finish = function() { 12 | this.handled = true 13 | this.state = 'finished' 14 | return this.db.none('UPDATE "JobQueue" SET "state"=${state}, "lastRun"=NOW() WHERE "id"=${id}', this) 15 | .then(() => this.queue.logEvent('finished', this)) 16 | } 17 | 18 | Job.prototype.destroy = function() { 19 | this.handled = true 20 | this.state = 'destroyed' 21 | return this.db.none('DELETE FROM "JobQueue" WHERE "id"=${id}', this) 22 | .then(() => this.queue.logEvent('destroyed', this)) 23 | } 24 | 25 | Job.prototype.reschedule = function(date) { 26 | this.handled = true 27 | this.state = 'waiting' 28 | this.scheduledFor = date 29 | return this.db.none('UPDATE "JobQueue" SET "state"=${state}, "scheduledFor"=${scheduledFor}, "lastRun"=NOW() WHERE "id"=${id}', this) 30 | .then(() => this.queue.logEvent('rescheduled', this)) 31 | .then(() => this.queue.logEvent('finished', this)) 32 | } 33 | 34 | Job.prototype.fail = function (error, rescheduleFor) { 35 | let self = this; 36 | self.handled = true; 37 | return self.db.tx(function*(t) { 38 | yield t.none('UPDATE "JobQueue" SET "lastFailureMessage"=${failureMessage}, "failedAttempts"="failedAttempts"+1, "lastRun"=NOW() WHERE "id"=${id}', { 39 | id: self.id, 40 | failureMessage: error.message 41 | }); 42 | let JobQueue = yield t.one('SELECT * FROM "JobQueue" WHERE "id"=${id}', self); 43 | if (JobQueue.failedAttempts >= JobQueue.maxAttempts) { 44 | // complete failure, ensure the job isn't run again 45 | self.state = 'failed'; 46 | return yield t.none('UPDATE "JobQueue" SET "state"=${state} WHERE "id"=${id}', self); 47 | } 48 | // reschedule the job to run in 3 minute 49 | let now = new Date(); 50 | let minute = 3 * 60 * 1000; 51 | self.scheduledFor = rescheduleFor || new Date(now.getTime() + minute); 52 | self.state = 'waiting'; 53 | return yield t.none('UPDATE "JobQueue" SET "state"=${state}, "scheduledFor"=${scheduledFor} WHERE "id"=${id}', self); 54 | }) 55 | .then(() => self.queue.logEvent('failed', self)) 56 | .then(() => self.queue.logError(error, self)); 57 | }; 58 | 59 | module.exports = Job 60 | -------------------------------------------------------------------------------- /bin/process-job-queue: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 'use strict' 3 | require('string-format').extend(String.prototype) 4 | const program = require('commander') 5 | const jobqueue = require('../lib') 6 | const cluster = require('cluster') 7 | 8 | program 9 | .version('0.0.1') 10 | .option( 11 | '-f, --handlers-file ', 12 | 'set path to handlers file. defaults to ./handlers.js', 13 | './handlers.js' 14 | ) 15 | .option( 16 | '-c, --connection-string ', 17 | 'postgresql connection string. defaults to postgres://postgres@localhost/pg-job-queue', 18 | false 19 | ) 20 | .option( 21 | '-m, --multi-process', 22 | 'spawn a process for each cpu available', 23 | false 24 | ) 25 | 26 | program.parse(process.argv) 27 | 28 | if (cluster.isMaster) { 29 | console.log("loading handlers from '{}'..".format(program.handlersFile)) 30 | } 31 | 32 | var path = require('path') 33 | var handlers = require(path.join(process.cwd(), program.handlersFile)) 34 | 35 | if (cluster.isMaster && program.multiProcess) { 36 | const numCPUs = require('os').cpus().length; 37 | console.log('launching {} workers'.format(numCPUs)) 38 | for (var i =0; i < numCPUs; i++) { 39 | cluster.fork() 40 | } 41 | } 42 | else { 43 | function log(s) { 44 | if (program.multiProcess) { 45 | console.log('worker {} - {}'.format(cluster.worker.id, s)) 46 | } 47 | else { 48 | console.log(s) 49 | } 50 | } 51 | function gracefulShutdown() { 52 | log('waiting for jobs to finish before shutdown') 53 | queue.stopProcessing().then(() => { 54 | queue.disconnect() 55 | }) 56 | } 57 | function getConnectionDetailsFromEnv() { 58 | return { 59 | host: process.env.PGHOST, 60 | port: process.env.PGPORT, 61 | database: process.env.PGDATABASE , 62 | user: process.env.PGUSER , 63 | password: process.env.PGPASSWORD 64 | } 65 | } 66 | 67 | // when the process is politely killed, do it gracefully (wait for the current job to finish processing) 68 | process.on('SIGINT', gracefulShutdown) 69 | process.on('SIGTERM', gracefulShutdown) 70 | 71 | var queue = new jobqueue(program.connectionString || getConnectionDetailsFromEnv()) 72 | queue.setHandlers(handlers) 73 | log('processing jobs..') 74 | queue.startProcessing().catch((e) => { 75 | console.error(e.stack) 76 | }) 77 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pg-job-queue 2 | 3 | A job queue for node.js based on PostgreSQL. 4 | 5 | [![npm version](https://badge.fury.io/js/pg-job-queue.svg)](https://badge.fury.io/js/pg-job-queue) 6 | [![Build Status](https://travis-ci.org/jameshy/pg-job-queue.svg?branch=master)](https://travis-ci.org/jameshy/pg-job-queue) 7 | 8 | 9 | ## Installation 10 | ```bash 11 | npm install pg-job-queue 12 | ``` 13 | 14 | ## Create a job 15 | 16 | ```javascript 17 | const jobqueue = require('pg-job-queue') 18 | const queue = new jobqueue('postgres://postgres@localhost/my-job-queue') 19 | 20 | queue.addJob({ 21 | type: 'sendmail.welcome', 22 | data: { 23 | toAddress: 'demo@example.com', 24 | message: 'hello' 25 | } 26 | }) 27 | ``` 28 | 29 | 30 | ## Processing jobs 31 | 32 | The tool `process-job-queue` is provided for the continuous processing of jobs. It will loop forever, polling the database for new jobs, until the process receives either SIGINT or SIGTERM. If it's terminated while processing a job, it will finish that job before terminating. 33 | 34 | The idea is that you define all your job handlers in a standard javascript module, and `process-job-queue` will call your handlers function when it processes a job. `process-job-queue` will only process jobs that have a matching handler. 35 | 36 | ##### 1. Create handlers.js file 37 | ```javascript 38 | module.exports = { 39 | sendmail: { 40 | welcome: function(job) { 41 | return sendMail(job.data.toAddress, job.data.message) 42 | .then(() => { 43 | return job.finish() 44 | }) 45 | } 46 | } 47 | } 48 | ``` 49 | 50 | Notice the handler above is nested, so it will be invoked when a job is added with type 'sendmail.welcome'. 51 | 52 | ##### 2. Run process-job-queue 53 | ```bash 54 | node_modules/pg-job-queue/bin/process-job-queue -f ./handlers.js -c postgres://postgres@localhost/my-job-queue 55 | ``` 56 | 57 | ##### Special handler methods 58 | You can define the following special handler methods: 59 | 60 | * `$logHandler(action, job)` - for general purpose logging, it's called whenever something is about to happen on a job. possible actions are: 'starting', 'destroyed', 'rescheduled', 'failed', 'finished' 61 | 62 | * `$errorHandler(error, job)` - called whenever an uncaught exception occurs while running a job. 63 | 64 | * `$shutdownHandler()` - called when we are stopping processing. This is useful to close handles that would prevent the node process from terminating. 65 | 66 | For example: 67 | ```javascript 68 | module.exports = { 69 | $errorHandler: function(e, job) { 70 | console.error(e.stack) 71 | }, 72 | $shutdownHandler: function() { 73 | return closeDatabaseConnection() 74 | }, 75 | $logHandler: function(action, job) { 76 | console.log('job #{} ({}) - {}'.format(job.id, job.type, action)) 77 | }, 78 | normalJob: function(job) { 79 | return job.finish() 80 | } 81 | } 82 | ``` 83 | 84 | ## License 85 | [MIT](LICENSE) 86 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const path = require('path') 3 | const Promise = require('bluebird') 4 | const pgpOptions = {promiseLib: Promise} 5 | const pgp = require('pg-promise')(pgpOptions) 6 | const Job = require('./job') 7 | const _ = require('lodash') 8 | const errors = require('./errors') 9 | 10 | // var monitor = require('pg-monitor') 11 | // monitor.attach(pgpOptions) 12 | // monitor.setTheme('matrix') 13 | 14 | const nextJobSQL = new pgp.QueryFile(path.join(__dirname, './sql/nextJob.sql')) 15 | var schemaPath = path.join(__dirname, 'schema.sql') 16 | const schemaFile = new pgp.QueryFile(schemaPath, {minify: true}) 17 | 18 | function JobQueue(connectionString) { 19 | this.db = pgp(connectionString) 20 | this.jobHandlers = {} 21 | this.shutdown = false 22 | this.stopProcessingCallback = null 23 | } 24 | 25 | /* 26 | Close all connections to the database. 27 | This is useful because a node process won't gracefully end if there are connections open. 28 | */ 29 | JobQueue.prototype.disconnect = function() { 30 | pgp.end() 31 | } 32 | 33 | /* Install schema to the database. */ 34 | JobQueue.prototype.installSchema = function() { 35 | return this.db.none(schemaFile) 36 | } 37 | 38 | /* Check that we can connect to the database and the table exists */ 39 | JobQueue.prototype.checkDatabase = function() { 40 | return this.db.oneOrNone('SELECT "id" FROM "JobQueue" LIMIT 1') 41 | } 42 | 43 | /* Clear the entire job queue. */ 44 | JobQueue.prototype.clearAllJobs = function() { 45 | return this.db.none('TRUNCATE table "JobQueue"') 46 | } 47 | 48 | /* 49 | Set the job handlers, replacing any previous handlers. 50 | This should be an object with a key of the job type, and value being the function for processing the job. 51 | e.g. 52 | { sendEmail: (job) => { sendmail() } 53 | */ 54 | JobQueue.prototype.setHandlers = function(handlers) { 55 | this.jobHandlers = handlers 56 | } 57 | 58 | /* Add a job to the queue. */ 59 | JobQueue.prototype.addJob = function(job) { 60 | if (!_.isObject(job)) { 61 | throw new TypeError('job is not an object') 62 | } 63 | 64 | job.scheduledFor = job.scheduledFor || new Date() 65 | job.maxAttempts = job.maxAttempts || 1 66 | job.data = job.data || {} 67 | 68 | if (!job.type) { 69 | throw new TypeError("property 'type' not specified") 70 | } 71 | if (!_.isString(job.type)) { 72 | throw new TypeError("property 'type' is not a string") 73 | } 74 | if (!_.isDate(job.scheduledFor)) { 75 | throw new TypeError("property 'scheduledFor' is not a date") 76 | } 77 | if (!_.isNumber(job.maxAttempts)) { 78 | throw new TypeError("property 'maxAttempts' is not a number") 79 | } 80 | else if (job.maxAttempts < 0) { 81 | throw new TypeError("property 'maxAttempts' is negative") 82 | } 83 | 84 | var query = ` 85 | INSERT INTO "JobQueue" 86 | ("type", "data", "scheduledFor", "maxAttempts", "createdAt") 87 | VALUES ($[type], $[data], $[scheduledFor], $[maxAttempts], NOW() )` 88 | 89 | return this.db.none(query, job) 90 | } 91 | 92 | /* 93 | Determine the job types we can process by scanning handlers. 94 | */ 95 | JobQueue.prototype.getAvailableJobTypes = function() { 96 | function recurse(val, key) { 97 | 98 | if (_.isObject(val) && !_.isFunction(val)) { 99 | return _.map(val, (subval, subkey) => { 100 | var path = key + '.' + subkey 101 | return recurse(subval, path) 102 | }) 103 | } 104 | else { 105 | return key 106 | } 107 | } 108 | 109 | return _(this.jobHandlers).map(recurse).flattenDeep().value() 110 | } 111 | 112 | /* 113 | Resolve a job type to a job handler. 114 | 'job.path' will resolve to handlers.job.path if it exists 115 | otherwise it will fallback to handlers['job.path'] 116 | */ 117 | JobQueue.prototype.resolveHandler = function(path) 118 | { 119 | // try and return the resolved path (e.g. handlers.job.path 120 | var byPath = _(this.jobHandlers).at(path).compact().head() 121 | if (byPath) { 122 | return byPath 123 | } 124 | else { 125 | // maybe there is a handler defined with the path (e.g. handlers['job.path']) 126 | return this.jobHandlers[path] 127 | } 128 | } 129 | 130 | /* 131 | If $logHandler is a defined handler, call it. 132 | */ 133 | JobQueue.prototype.logEvent = function(event, job) { 134 | // if $logHandler is defined, call it 135 | if (this.jobHandlers.$logHandler) { 136 | this.jobHandlers.$logHandler(event, job) 137 | } 138 | } 139 | 140 | /* 141 | If $errorHandler is a defined handler, call it. 142 | */ 143 | JobQueue.prototype.logError = function(error, job) { 144 | // if $errorHandler is defined, call it 145 | if (this.jobHandlers.$errorHandler) { 146 | this.jobHandlers.$errorHandler(error, job) 147 | } 148 | } 149 | 150 | /* 151 | Grab the next job with a known handler, process it. 152 | If no jobs are found, throw errors.JobQueueEmpty 153 | */ 154 | JobQueue.prototype.processNextJob = function() { 155 | // we only process jobs that we have a handler defined for 156 | // so get all these types ready to pass into the SQL query 157 | var types = this.getAvailableJobTypes() 158 | 159 | // we use a task, to ensure the same connection is used 160 | // this is important because we use a session-level advisory lock 161 | return this.db.task((t) => { 162 | // nextJobSQL acquires a session-level pg_advisory_lock 163 | return t.oneOrNone(nextJobSQL, {types: types}) 164 | .then((result) => { 165 | if (!result) { 166 | throw new errors.JobQueueEmpty() 167 | } 168 | var job = new Job(result, this) 169 | 170 | return Promise.try(() => { 171 | var handler = this.resolveHandler(result.type) 172 | this.logEvent('starting', job) 173 | return handler(job) 174 | }).then(() => { 175 | // the job was not destroyed, finished or rescheduled 176 | // we mark the job as finished 177 | if (!job.handled) { 178 | return job.finish() 179 | } 180 | }) 181 | .catch((e) => { 182 | return job.fail(e) 183 | }) 184 | .finally(() => { 185 | // release the advisory lock 186 | return t.one('SELECT pg_advisory_unlock($1)', result.id) 187 | }) 188 | }) 189 | }) 190 | } 191 | 192 | /* 193 | Process all jobs (one at a time). 194 | When no jobs are available, throws errors.JobQueueEmpty 195 | */ 196 | JobQueue.prototype.processAllJobs = function() { 197 | return this.processNextJob().then(() => { 198 | return this.processAllJobs() 199 | }) 200 | } 201 | 202 | /* 203 | Begin an infinite loop of job-processing. 204 | We continually poll for new jobs, waiting for $delay milliseconds between each poll. (default 500 milliseconds) 205 | */ 206 | JobQueue.prototype.startProcessing = function(delay) { 207 | var loop = function() { 208 | if (this.shutdown) { 209 | this.processing = false 210 | if (this.stopProcessingCallback) { 211 | if (this.jobHandlers.$shutdownHandler) { 212 | return this.jobHandlers.$shutdownHandler().then(this.stopProcessingCallback) 213 | } 214 | else { 215 | return this.stopProcessingCallback() 216 | } 217 | } 218 | return Promise.resolve() 219 | } 220 | return this.processAllJobs().catch((e) => { 221 | // ignore JobQueueEmpty exceptions, we must continue our loop 222 | if (!(e instanceof errors.JobQueueEmpty)) { 223 | throw e 224 | } 225 | }) 226 | .then(() => { 227 | return Promise.delay(delay || 500).then(loop) 228 | }) 229 | } 230 | loop = _.bind(loop, this) 231 | if (this.processing) { 232 | throw new Error("already processing") 233 | } 234 | this.processing = true 235 | return loop() 236 | } 237 | 238 | /* 239 | Set a sentinel value to trigger termination of the startProcessing loop 240 | */ 241 | JobQueue.prototype.stopProcessing = function() { 242 | if (!this.processing) { 243 | return Promise.resolve() 244 | } 245 | this.shutdown = true 246 | 247 | return new Promise((fulfill) => { 248 | this.stopProcessingCallback = fulfill 249 | }) 250 | } 251 | 252 | /* Returns all failed jobs (without locking) */ 253 | JobQueue.prototype.getFailedJobs = function() { 254 | return this.db.manyOrNone('SELECT * FROM "JobQueue" WHERE "state"=$1', 'failed') 255 | } 256 | 257 | /* Returns all jobs (without locking) */ 258 | JobQueue.prototype.getAllJobs = function() { 259 | return this.db.manyOrNone('SELECT * FROM "JobQueue"') 260 | } 261 | 262 | /* Returns current job queue length */ 263 | JobQueue.prototype.waitingCount = function() { 264 | return this.db.one('SELECT COUNT(*) FROM "JobQueue" WHERE "state"=$1', 'waiting').then((result) => { 265 | return _.parseInt(result.count) 266 | }) 267 | } 268 | 269 | /* Returns number of failed jobs */ 270 | JobQueue.prototype.failedCount = function() { 271 | return this.db.one('SELECT COUNT(*) FROM "JobQueue" WHERE "state"=$1', 'failed').then((result) => { 272 | return _.parseInt(result.count) 273 | }) 274 | } 275 | 276 | /* 277 | Simplify access to our custome error types. 278 | */ 279 | JobQueue.prototype.errors = require('./errors') 280 | 281 | module.exports = JobQueue 282 | 283 | -------------------------------------------------------------------------------- /test/queue.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | require('./support/common') 3 | const _ = require('lodash'); 4 | const db = require('./support/db') 5 | const jobqueue = require('../lib') 6 | const Job = require('../lib/job') 7 | 8 | describe('Job Queue', function() { 9 | 10 | before(function() { 11 | // drop and create the database and install schema 12 | return db.destroyAndCreate().then((queue) => { 13 | this.queue = queue 14 | }) 15 | }) 16 | 17 | beforeEach(function() { 18 | return this.queue.clearAllJobs() 19 | }) 20 | 21 | describe('checkDatabase', function() { 22 | it('should succeed when the database is OK', function() { 23 | return this.queue.checkDatabase() 24 | }) 25 | it('should fail when the database not OK', function() { 26 | var queue = new jobqueue('postgres://127.0.0.1:61548/unknown') 27 | return expect(queue.checkDatabase()).to.eventually.be.rejected 28 | }) 29 | }) 30 | 31 | describe('addJob should throw an exception when called with invalid arguments', function() { 32 | this.validJob = { 33 | type: 'sendMail', 34 | scheduledFor: new Date(), 35 | maxAttempts: 1, 36 | data: {} 37 | } 38 | 39 | it('should reject non-objects', function() { 40 | expect(() => this.queue.addJob(1)).to.throw(TypeError) 41 | }) 42 | 43 | it('should reject scheduledFor specified with non-date', function() { 44 | var job = _.extend({}, this.validJob, {scheduledFor: 123}) 45 | expect(() => this.queue.addJob(job)).to.throw(TypeError) 46 | }) 47 | 48 | it('should reject invalid type', function() { 49 | var job = _.extend({}, this.validJob, {type: 123}) 50 | expect(() => this.queue.addJob(job)).to.throw(TypeError) 51 | }) 52 | 53 | it('should reject invalid maxAttempts', function() { 54 | var job = _.extend({}, this.validJob, {maxAttempts: 123}) 55 | expect(() => this.queue.addJob(job)).to.throw(TypeError) 56 | 57 | job = _.extend({}, this.validJob, {maxAttempts: -123}) 58 | expect(() => this.queue.addJob(job)).to.throw(TypeError) 59 | }) 60 | }) 61 | 62 | it('should accept a new job and then process it once', function() { 63 | 64 | function jobHandler(job, queue) { 65 | // send email to job.data.recipient, message=job.data.message 66 | return job.finish() 67 | } 68 | 69 | var spy = sinon.spy(jobHandler) 70 | 71 | // setup a single job handler 72 | this.queue.setHandlers({ 73 | sendEmail: spy 74 | }) 75 | 76 | var job = { 77 | type: 'sendEmail', 78 | data: { 79 | recipient: 'user@example.com', 80 | message: 'HELLO' 81 | } 82 | } 83 | 84 | 85 | // add a job 86 | return this.queue.addJob(job).then(() => { 87 | // process the job 88 | return this.queue.processNextJob().then(() => { 89 | 90 | // check the handler was called correctly 91 | expect(spy.calledOnce).to.be.true 92 | expect(spy.getCall(0).args[0].data).to.deep.equal(job.data) 93 | 94 | // try and process the job again (should fail) 95 | return expect(this.queue.processNextJob()).to.eventually.be.rejected 96 | }) 97 | }) 98 | }) 99 | 100 | it('should resolve job types as paths', function() { 101 | function jobHandler(job, queue) { 102 | // send email to job.data.recipient, message=job.data.message 103 | return job.finish() 104 | } 105 | 106 | var spy = sinon.spy(jobHandler) 107 | 108 | // setup a single job handler 109 | this.queue.setHandlers({ 110 | emails: { 111 | subgroup: { 112 | welcome: spy 113 | } 114 | } 115 | }) 116 | 117 | // define the job 118 | var job = { 119 | type: 'emails.subgroup.welcome', 120 | data: { 121 | recipient: 'user@example.com', 122 | message: 'HELLO' 123 | } 124 | } 125 | // add the job 126 | return this.queue.addJob(job).then(() => { 127 | // process the job 128 | return this.queue.processNextJob().then(() => { 129 | 130 | // check the handler was called correctly 131 | expect(spy.calledOnce).to.be.true 132 | expect(spy.getCall(0).args[0].data).to.deep.equal(job.data) 133 | 134 | // try and process the job again (should fail) 135 | return expect(this.queue.processNextJob()).to.eventually.be.rejected 136 | }) 137 | }) 138 | }) 139 | 140 | it('should resolve a job handler from a path', function() { 141 | var handlers = { 142 | email: { 143 | subgroup: { 144 | welcome: () => {} 145 | } 146 | }, 147 | sendEmail: () => {} 148 | } 149 | this.queue.setHandlers(handlers) 150 | var welcome = this.queue.resolveHandler('email.subgroup.welcome') 151 | expect(welcome).to.equal(handlers.email.subgroup.welcome) 152 | var sendEmail = handlers.sendEmail 153 | expect(sendEmail).to.equal(handlers.sendEmail) 154 | }) 155 | 156 | it('should determine available job types', function() { 157 | var handlers = { 158 | testJob: () => {}, 159 | emails: { 160 | subgroup: { 161 | welcome: () => {}, 162 | goodbye: () => {}, 163 | } 164 | } 165 | } 166 | this.queue.setHandlers(handlers) 167 | var expected = [ 168 | 'testJob', 169 | 'emails.subgroup.welcome', 170 | 'emails.subgroup.goodbye', 171 | ] 172 | var types = this.queue.getAvailableJobTypes() 173 | 174 | // sort them both 175 | types = _.sortBy(types) 176 | expected = _.sortBy(expected) 177 | expect(_.isEqual(types, expected)) 178 | }) 179 | 180 | 181 | it('should mark a job as failed if it throws an exception', function() { 182 | this.queue.setHandlers({ 183 | failingJob: function() { 184 | throw new Error('error message') 185 | } 186 | }) 187 | 188 | var job = { 189 | type: 'failingJob', 190 | maxAttempts: 1 191 | } 192 | 193 | 194 | return this.queue.addJob(job) 195 | .then(() => this.queue.processNextJob()) 196 | .then(() => this.queue.getFailedJobs()) 197 | .then(function(jobs) { 198 | expect(jobs.length).to.equal(1) 199 | var job = jobs[0] 200 | expect(job.failedAttempts).to.equal(1) 201 | expect(job.lastFailureMessage).to.equal('error message') 202 | }) 203 | }) 204 | 205 | it('should retry a failed job `maxAttempts` times', function() { 206 | this.queue.setHandlers({ 207 | failingJob: function(job) { 208 | return job.fail(new Error('error message'), new Date()) 209 | } 210 | }) 211 | var job = { 212 | type: 'failingJob', 213 | maxAttempts: 5 214 | } 215 | return this.queue.addJob(job) 216 | .then(() => { 217 | var loop = () => { 218 | return this.queue.processNextJob().then(() => { 219 | return loop() 220 | }) 221 | } 222 | return loop() 223 | }).catch((e) => { 224 | if (!(e instanceof this.queue.errors.JobQueueEmpty)) { 225 | throw e 226 | } 227 | }).then(() => { 228 | // job has been run many times and should have reached complete failure 229 | // check that is true 230 | return this.queue.getFailedJobs().then(function(jobs) { 231 | expect(jobs.length).to.equal(1) 232 | var job = jobs[0] 233 | expect(job.state).to.equal('failed') 234 | expect(job.failedAttempts).to.equal(5) 235 | expect(job.maxAttempts).to.equal(5) 236 | expect(job.lastFailureMessage).to.equal('error message') 237 | }) 238 | }) 239 | }) 240 | 241 | it('should fail a job when job.fail() is called', function() { 242 | this.queue.setHandlers({ 243 | failingJob: function(job) { 244 | return job.fail(new Error('error message')) 245 | } 246 | }) 247 | var job = { 248 | type: 'failingJob' 249 | } 250 | return this.queue.addJob(job) 251 | .then(() => { 252 | var loop = () => { 253 | return this.queue.processNextJob().then(() => loop()) 254 | } 255 | return loop() 256 | }).catch((e) => { 257 | if (!(e instanceof this.queue.errors.JobQueueEmpty)) { 258 | throw e 259 | } 260 | }).then(() => { 261 | // job has been run many times and should have reached complete failure 262 | // check that is true 263 | return this.queue.getFailedJobs().then((jobs) => { 264 | expect(jobs.length).to.equal(1) 265 | var job = jobs[0] 266 | expect(job.state).to.equal('failed') 267 | expect(job.failedAttempts).to.equal(1) 268 | expect(job.maxAttempts).to.equal(1) 269 | expect(job.lastFailureMessage).to.equal('error message') 270 | }) 271 | }) 272 | 273 | }) 274 | 275 | it('should correctly reschedule a job', function*() { 276 | this.queue.setHandlers({ 277 | rescheduleJob: function(job) { 278 | return job.reschedule(new Date()) 279 | } 280 | }) 281 | 282 | var job = { 283 | type: 'rescheduleJob', 284 | } 285 | yield this.queue.addJob(job) 286 | yield this.queue.processNextJob() 287 | yield this.queue.processNextJob() 288 | 289 | 290 | var count = yield this.queue.waitingCount() 291 | expect(count).to.equal(1) 292 | }) 293 | 294 | describe('should update lastRun', function() { 295 | it('for a rescheduled job', function*() { 296 | this.queue.setHandlers({ 297 | rescheduleJob: function(job) { 298 | return job.reschedule(new Date()) 299 | } 300 | }) 301 | var job = { 302 | type: 'rescheduleJob', 303 | } 304 | 305 | 306 | var start = new Date() 307 | // deduct 1 second, because 'pg' module cannot handle microseconds 308 | start.setSeconds(start.getSeconds() - 1) 309 | 310 | // add the job to the queue and process it 311 | yield this.queue.addJob(job) 312 | yield this.queue.processNextJob() 313 | 314 | // verify that lastRun column has been correctly updated 315 | var allJobs = yield this.queue.getAllJobs() 316 | expect(allJobs).to.have.length(1) 317 | var _job = allJobs[0] 318 | expect(_job.lastRun).afterTime(start) 319 | expect(_job.lastRun).beforeTime(new Date()) 320 | }) 321 | 322 | it('for a finished job', function*() { 323 | this.queue.setHandlers({ 324 | sendmail: function(job) { } 325 | }) 326 | var job = { 327 | type: 'sendmail', 328 | } 329 | 330 | 331 | var start = new Date() 332 | // deduct 1 second, because 'pg' module cannot handle microseconds 333 | start.setSeconds(start.getSeconds() - 1) 334 | 335 | // add the job to the queue and process it 336 | yield this.queue.addJob(job) 337 | yield this.queue.processNextJob() 338 | 339 | // verify that lastRun column has been correctly updated 340 | var allJobs = yield this.queue.getAllJobs() 341 | expect(allJobs).to.have.length(1) 342 | var _job = allJobs[0] 343 | expect(_job.lastRun).afterTime(start) 344 | expect(_job.lastRun).beforeTime(new Date()) 345 | }) 346 | 347 | it('for a failed job', function*() { 348 | this.queue.setHandlers({ 349 | failingJob: function(job) { 350 | throw new Error("error") 351 | } 352 | }) 353 | var job = { 354 | type: 'failingJob', 355 | } 356 | 357 | 358 | var start = new Date() 359 | // deduct 1 second, because 'pg' module cannot handle microseconds 360 | start.setSeconds(start.getSeconds() - 1) 361 | 362 | // add the job to the queue and process it 363 | yield this.queue.addJob(job) 364 | yield this.queue.processNextJob() 365 | 366 | // verify that lastRun column has been correctly updated 367 | var allJobs = yield this.queue.getFailedJobs() 368 | expect(allJobs).to.have.length(1) 369 | var _job = allJobs[0] 370 | expect(_job.lastRun).afterTime(start) 371 | expect(_job.lastRun).beforeTime(new Date()) 372 | }) 373 | }) 374 | 375 | 376 | it('should only process jobs with handlers available', function() { 377 | var job = { 378 | type: 'sendmail', 379 | } 380 | return this.queue.addJob(job) 381 | .then(() => { 382 | // processNextJob should throw the error JobQueueEmpty 383 | // because we haven't setup any handlers, so it doesn't see the job 384 | return expect(this.queue.processNextJob()).to.eventually.be.rejectedWith(this.queue.errors.JobQueueEmpty) 385 | }) 386 | }) 387 | 388 | it('should not allow multiple threads to acquire the same job', function() { 389 | // initialize 2 jobqueue instances to the same database 390 | var queue1 = new jobqueue(db.connectionString) 391 | var queue2 = new jobqueue(db.connectionString) 392 | 393 | // the job we will use to test 394 | var job = { 395 | type: 'slowJob', 396 | } 397 | var handlers = { 398 | slowJob: function(job) { 399 | return Promise.delay(100).then(() => { 400 | return job.finish() 401 | }) 402 | } 403 | } 404 | 405 | // a wrapper around processNextJob(), that returns false on error 406 | function catchProcessJobError(queue) { 407 | return queue.processNextJob().then(() => { 408 | return true 409 | }).catch((e) => { 410 | return false 411 | }) 412 | } 413 | 414 | // use the same handlers for both jobqueues 415 | queue1.setHandlers(handlers) 416 | queue2.setHandlers(handlers) 417 | 418 | // add the test job 419 | return queue1.addJob(job).then(() => { 420 | // try and process the job with 2 threads 421 | return Promise.join( 422 | catchProcessJobError(queue1), 423 | catchProcessJobError(queue2), 424 | function(result1, result2) { 425 | // one of the queues should succeeed (it acquires the job and processes it successfully) 426 | // the other queue should fail (it cannot acquire the already acquired job) 427 | expect(result1 != result2).to.be.true 428 | }) 429 | }) 430 | }) 431 | 432 | it('should allow jobs to destroy themselves', function() { 433 | this.queue.setHandlers({ 434 | testjob: function(job) { 435 | return job.destroy() 436 | } 437 | }) 438 | var job = { 439 | type: 'testjob' 440 | } 441 | 442 | return this.queue.addJob(job) 443 | .then(() => { 444 | return this.queue.waitingCount().then((count) => { 445 | expect(count).to.equal(1) 446 | }) 447 | }) 448 | .then(() => this.queue.processNextJob()) 449 | .then(() => { 450 | return this.queue.failedCount().then((count) => { 451 | expect(count).to.equal(0) 452 | }) 453 | }) 454 | .then(() => { 455 | return this.queue.waitingCount().then((count) => { 456 | expect(count).to.equal(0) 457 | }) 458 | }) 459 | }) 460 | 461 | it('should call the configured error handler', function() { 462 | var errorHandler = sinon.spy() 463 | 464 | this.queue.setHandlers({ 465 | // special error handler method 466 | $errorHandler: errorHandler, 467 | failingJob: function(job) { 468 | throw new Error("error") 469 | } 470 | }) 471 | var job = { 472 | type: 'failingJob' 473 | } 474 | return this.queue.addJob(job) 475 | .then(() => this.queue.processNextJob()) 476 | .then(() => { 477 | // check the error handler was called correctly 478 | expect(errorHandler.calledOnce).to.be.true 479 | var args = errorHandler.getCall(0).args 480 | expect(args[0]).to.be.an.instanceof(Error) 481 | expect(args[1]).to.be.an.instanceof(Job) 482 | }) 483 | }) 484 | 485 | describe('$logHandler should be called', function() { 486 | 487 | it('for a failing job', function* () { 488 | var logHandler = sinon.spy() 489 | 490 | this.queue.setHandlers({ 491 | // special error handler method 492 | $logHandler: logHandler, 493 | failingJob: function(job) { 494 | throw new Error("error") 495 | } 496 | }) 497 | 498 | /* for a failing job */ 499 | yield this.queue.addJob({type: 'failingJob'}) 500 | yield this.queue.processNextJob() 501 | 502 | expect(logHandler.calledTwice).to.be.true 503 | 504 | var firstCallArgs = logHandler.getCall(0).args 505 | expect(firstCallArgs[0]).to.equal('starting') 506 | expect(firstCallArgs[1]).to.be.an.instanceof(Job) 507 | 508 | var secondCallArgs = logHandler.getCall(1).args 509 | expect(secondCallArgs[0]).to.equal('failed') 510 | expect(secondCallArgs[1]).to.be.an.instanceof(Job) 511 | 512 | }) 513 | 514 | it('for a healthy job', function*() { 515 | var logHandler = sinon.spy() 516 | 517 | this.queue.setHandlers({ 518 | // special error handler method 519 | $logHandler: logHandler, 520 | 521 | healthyJob: function(job) { 522 | } 523 | }) 524 | 525 | yield this.queue.addJob({type: 'healthyJob'}) 526 | yield this.queue.processNextJob() 527 | 528 | expect(logHandler.calledTwice).to.be.true 529 | 530 | var firstCallArgs = logHandler.getCall(0).args 531 | expect(firstCallArgs[0]).to.equal('starting') 532 | expect(firstCallArgs[1]).to.be.an.instanceof(Job) 533 | 534 | var secondCallArgs = logHandler.getCall(1).args 535 | expect(secondCallArgs[0]).to.equal('finished') 536 | expect(secondCallArgs[1]).to.be.an.instanceof(Job) 537 | }) 538 | 539 | it('for a rescheduling job', function*() { 540 | var logHandler = sinon.spy() 541 | 542 | this.queue.setHandlers({ 543 | // special error handler method 544 | $logHandler: logHandler, 545 | 546 | reschedulingJob: function(job) { 547 | return job.reschedule(new Date()) 548 | } 549 | }) 550 | 551 | yield this.queue.addJob({type: 'reschedulingJob'}) 552 | yield this.queue.processNextJob() 553 | 554 | expect(logHandler.calledThrice).to.be.true 555 | 556 | var firstCallArgs = logHandler.getCall(0).args 557 | expect(firstCallArgs[0]).to.equal('starting') 558 | expect(firstCallArgs[1]).to.be.an.instanceof(Job) 559 | 560 | var secondCallArgs = logHandler.getCall(1).args 561 | expect(secondCallArgs[0]).to.equal('rescheduled') 562 | expect(secondCallArgs[1]).to.be.an.instanceof(Job) 563 | 564 | var thirdCallArgs = logHandler.getCall(2).args 565 | expect(thirdCallArgs[0]).to.equal('finished') 566 | expect(thirdCallArgs[1]).to.be.an.instanceof(Job) 567 | }) 568 | }) 569 | }) 570 | 571 | --------------------------------------------------------------------------------