├── .gitignore ├── .npmignore ├── .travis.yml ├── Makefile ├── README.md ├── examples ├── perf-test.coffee └── simple.coffee ├── package.json ├── src └── queue.coffee └── test ├── connection.coffee ├── template.coffee ├── tests.coffee ├── utils └── db.coffee └── worker.coffee /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | lib 3 | coverage 4 | .c9 -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .c9 2 | .npmignore 3 | Makefile 4 | examples/ 5 | src/ 6 | test/ 7 | coverage/ 8 | node_modules/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: node_js 3 | 4 | node_js: 5 | - "0.10" 6 | - "0.12" 7 | - iojs 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test compile-test compile clean publish install 2 | 3 | test: compile compile-test run-test clean 4 | 5 | compile-test: 6 | @./node_modules/.bin/coffee -c test/*.coffee test/**/*.coffee 7 | 8 | clean-test: 9 | @rm -fr test/*.js test/**/*.js 10 | 11 | run-test: 12 | @node test/tests.js 13 | 14 | compile: 15 | @./node_modules/.bin/coffee -c -o lib src/*.coffee 16 | 17 | clean: clean-test 18 | @rm -fr lib/ 19 | 20 | publish: compile 21 | npm publish 22 | 23 | install: compile 24 | npm install 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A MongoDB job queue 2 | 3 | [![Build Status](https://secure.travis-ci.org/kof/node-mongo-queue.svg)](http://travis-ci.org/kof/node-mongo-queue) 4 | 5 | Its is a job queue inspired by Resque, but is trying to improve certain 6 | shortcomings of Resque's design. In particular, mongo-queue makes it impossible 7 | to lose jobs. Jobs are kept in the database until workers have successfully 8 | completed them and only then are they removed from the database. 9 | 10 | Even if your host or application crashes hard (without chance to catch the 11 | exception in the application's runtime), no jobs are lost. 12 | 13 | 14 | # API 15 | 16 | ## Connection 17 | 18 | An object that will hold the MongoDB connection and allow manipulation of the job queue. 19 | 20 | ### constructor([options]) 21 | 22 | Options are : 23 | 24 | - `expires`: time in milliseconds until a job is no more valid. Defaults to 1h. 25 | - `timeout`: time in milliseconds allowed to workers before a job goes back to the queue to be processed by another worker. Defaults to 10s. 26 | - `maxAttempts`: maximum number of attempts to run the job. Defaults to 5. 27 | - `host`: hostname for the MongoDB database. Defaults to '127.0.0.1'. 28 | - `port`: port for the MongoDB database. Defaults to 27017. 29 | - `db`: either a [MongoDB connection](http://mongodb.github.io/node-mongodb-native/1.4/api-generated/db.html) or a string for the database name. Defaults to 'queue'. 30 | - `username`: login for an authenticated MongoDB database. 31 | - `password`: password for an authenticated MongoDB database. 32 | 33 | ### Methods 34 | 35 | #### enqueue(queue, args..., callback) 36 | 37 | Adds a job to the queue. 38 | 39 | - `queue`: either a string with the name of the queue, or an object containing : 40 | - `queue`: the name of the queue. 41 | - `expires`: override the `expires` of the connection for this job. 42 | - `startDate`: a Date object defining a future point in time when the job should execute. 43 | - `args...`: any number of arguments your job should take. 44 | - `callback(err, job)`: provides the feedback from the database after the job is inserted. 45 | 46 | ### Events 47 | 48 | #### error(err) 49 | 50 | Emitted in case of a connection error. 51 | 52 | #### connected 53 | 54 | Emitted when the [`Connection`](#connection) object is ready to work. 55 | 56 | ## Template 57 | 58 | Base class to create your own job templates, you need to inherit from it. 59 | 60 | ### Methods 61 | 62 | #### perform([...]) 63 | 64 | Function containing your job's code. You can define any arguments you need. 65 | 66 | Any synchronous exception thrown will be caught and the job will go back to the queue. 67 | 68 | #### complete([err]) 69 | 70 | You need to call this function when you are done with the job. 71 | 72 | You can provide an error in case anything went wrong. 73 | 74 | ## Worker 75 | 76 | Worker object that will watch out for new jobs and run them. 77 | 78 | ### constructor(connection, templates, options) 79 | 80 | You need to provide a [`Connection`](#connection) object and an array of [`Worker`](#worker) classes. 81 | 82 | Options are : 83 | 84 | - `timeout`: time in milliseconds for the database polling. Defaults to 1s. 85 | - `rotate`: boolean, indicates if you want job types to be processed in a round-robin fashion or sequentially, meaning all jobs of type A would have to done before jobs of type B. Defaults to false. 86 | - `workers`: integer indicating the maximum number of jobs to execute in parallel. Defaults to 3. 87 | 88 | ### Methods 89 | 90 | #### poll() 91 | 92 | Starts the polling of the database. 93 | 94 | #### stop() 95 | 96 | Stops the worker, it will not interrupt an ongoing job. 97 | 98 | ### Events 99 | 100 | #### error(err) 101 | 102 | Emitted on polling errors, unknown template definitions or job error. 103 | 104 | #### drained 105 | 106 | Emitted when the queue has no more job to process or if there is no job to run on a poll. 107 | 108 | #### stopped 109 | 110 | Emitted after a call to `stop()` and once all the jobs are stopped. 111 | 112 | # Example 113 | 114 | queue = require 'mongo-queue' 115 | 116 | # First declare your job by extending queue.Template 117 | class Addition extends queue.Template 118 | perform: (a, b) -> 119 | console.log a + ' + ' + b + ' = ' + (a + b) 120 | @complete() 121 | 122 | # Create a connection to the database 123 | options = host: 'localhost', port: 27017, db: 'test' 124 | connection = new queue.Connection options 125 | 126 | # Listen to connection errors 127 | connection.on 'error', console.error 128 | 129 | # Put some jobs into the queue 130 | connection.enqueue Addition.name, 1, 1, (err, job) -> if err then console.log(err) else console.log('Job added :', job) 131 | connection.enqueue Addition.name, 2, 4, (err, job) -> if err then console.log(err) else console.log('Job added :', job) 132 | connection.enqueue Addition.name, 3, 6, (err, job) -> if err then console.log(err) else console.log('Job added :', job) 133 | connection.enqueue Addition.name, 4, 8, (err, job) -> if err then console.log(err) else console.log('Job added :', job) 134 | 135 | # Now you need a worker who will process the jobs 136 | worker = new queue.Worker connection, [ Addition ] 137 | worker.on 'error', console.error 138 | worker.poll() 139 | -------------------------------------------------------------------------------- /examples/perf-test.coffee: -------------------------------------------------------------------------------- 1 | 2 | queue = require '../src/queue' 3 | connection = new queue.Connection db: 'test' 4 | 5 | # Keep track of the jobs / s we process 6 | numCompleted = 0 7 | lastReport = new Date().getTime() 8 | 9 | class Noop extends queue.Template 10 | perform: -> 11 | @complete() 12 | 13 | ++numCompleted 14 | now = new Date().getTime() 15 | diff = now - lastReport 16 | if diff > 1000 17 | console.log(Math.round(numCompleted / diff * 1000) + ' jobs/s') 18 | numCompleted = 0 19 | lastReport = now 20 | 21 | # Add 100k jobs to the queue 22 | numAdded = 0 23 | producer = -> 24 | numThisRound = 0 25 | while numThisRound < 100 and ++numAdded < 100000 26 | connection.enqueue Noop.name, null, -> 27 | ++numThisRound 28 | 29 | if numAdded < 100000 30 | process.nextTick producer 31 | 32 | producer() 33 | 34 | # Create a worker which will process the jobs 35 | worker = new queue.Worker connection, [ Noop ], workers: 9 36 | worker.poll() 37 | 38 | -------------------------------------------------------------------------------- /examples/simple.coffee: -------------------------------------------------------------------------------- 1 | 2 | queue = require '../src/queue' 3 | 4 | # Declare the job. Each time there is work to do, a new instance of this 5 | # class will be created and the method `perform` called. 6 | class Addition extends queue.Template 7 | perform: (args...) -> 8 | console.log arguments 9 | 10 | if Math.random() < 0.1 11 | @complete() 12 | else 13 | @release() 14 | 15 | 16 | connection = new queue.Connection 17 | 18 | # Insert the Addition job into the queue 19 | connection.enqueue Addition.name, 3, 2, -> 20 | connection.enqueue Addition.name, 3, 2, -> 21 | connection.enqueue Addition.name, 3, 2, -> 22 | connection.enqueue Addition.name, 3, 2, -> 23 | connection.enqueue Addition.name, 3, 2, -> 24 | 25 | # Create a worker which will process the Addition jobs 26 | worker = new queue.Worker connection, [ Addition ] 27 | worker.poll() 28 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mongo-queue", 3 | "description": "Node.js job queue backed by MongoDB", 4 | "version": "1.0.0", 5 | "keywords": [ 6 | "queue", 7 | "jobqueue", 8 | "mongo", 9 | "mongodb" 10 | ], 11 | "author": "Tomas Carnecky", 12 | "repository": { 13 | "type": "git", 14 | "url": "git://github.com/kof/node-mongo-queue" 15 | }, 16 | "licenses": [ 17 | { 18 | "type": "Unlicense", 19 | "url": "http://unlicense.org/" 20 | } 21 | ], 22 | "engines": { 23 | "node": ">=0.4.0" 24 | }, 25 | "directories": { 26 | "lib": "./lib" 27 | }, 28 | "main": "./lib/queue", 29 | "dependencies": { 30 | "mongodb": "^2.0.33" 31 | }, 32 | "devDependencies": { 33 | "coffee-script": "^1.9.3", 34 | "lodash": "^3.9.3", 35 | "qunit": "^0.7.6" 36 | }, 37 | "scripts": { 38 | "prepublish": "make compile", 39 | "test": "make test" 40 | }, 41 | "contributors": [ 42 | { 43 | "name": "Oleg Slobodksoi", 44 | "email": "oleg008@gmail.com" 45 | }, 46 | { 47 | "name": "Hannes Gassert" 48 | } 49 | ] 50 | } 51 | -------------------------------------------------------------------------------- /src/queue.coffee: -------------------------------------------------------------------------------- 1 | # **mongo-queue** - a MongoDB job queue 2 | # 3 | # Jobs are stored in a collection and retrieved or updated using the 4 | # findAndModify command. This allows multiple workers to concurrently 5 | # access the queue. Each job has an expiration date, and once acquired by 6 | # a worker, also a timeout. Old jobs and stuck workers can so be identified 7 | # and dealt with appropriately. 8 | 9 | 10 | #### Connection 11 | 12 | # mongo-queue is backed by MongoDB 13 | mongodb = require 'mongodb' 14 | EventEmitter = require('events').EventEmitter 15 | 16 | # The **Connection** class wraps the connection to MongoDB. It includes 17 | # methods to manipulate (add, remove, clear, ...) jobs in the queues. 18 | class exports.Connection extends EventEmitter 19 | 20 | # Initialize with a reference to the MongoDB and optional options 21 | # hash. Two options are currently supported: expires and timeout. 22 | constructor: (options) -> 23 | options or options = {} 24 | @expires = options.expires or 60 * 60 * 1000 25 | @timeout = options.timeout or 10 * 1000 26 | @maxAttempts = options.maxAttempts or 5 27 | @queue = [] 28 | setImmediate () => 29 | @ensureConnection options 30 | 31 | # Open a connection to the MongoDB server. Queries created while we are 32 | # connecting are queued and executed after the connection is established. 33 | ensureConnection: (opt) -> 34 | afterConnectionEstablished = (err) => 35 | return @emit('error', err) if err 36 | 37 | # Make a lame read request to the database. This will return an error 38 | # if the client is not authorized to access it. 39 | db.collections (err) => 40 | return @emit('error', err) if err 41 | 42 | db.collection 'queue', (err, collection) => 43 | return @emit('error', err) if err 44 | 45 | @collection = collection 46 | fn(collection) for fn in @queue if @queue 47 | delete @queue 48 | 49 | collection.ensureIndex [ ['expires'], ['owner'], ['queue'] ], (err) => 50 | if err then @emit('error', err) else @emit('connected') 51 | 52 | # Use an existing database connection if one is passed 53 | if opt.db instanceof mongodb.Db 54 | db = opt.db; 55 | return db.once('open', afterConnectionEstablished) unless db.serverConfig.isConnected() 56 | return afterConnectionEstablished null 57 | 58 | # TODO: support replica sets 59 | # TODO: support connection URIs 60 | 61 | url = "mongodb://" 62 | 63 | if opt.username and opt.password 64 | url += encodeURIComponent("#{opt.username}:#{opt.password}") + '@' 65 | 66 | url += "#{opt.host || '127.0.0.1'}:#{opt.port || 27017}/#{opt.db || 'queue'}?w=1" 67 | 68 | mongodb.MongoClient.connect url, (err, _db) => 69 | @emit('error', err) if err 70 | db = _db if _db 71 | afterConnectionEstablished null 72 | 73 | 74 | # Execute the given function if the connection to the database has been 75 | # established. If not, put it into a queue so it can be executed later. 76 | exec: (fn) -> 77 | @queue and @queue.push(fn) or fn(@collection) 78 | 79 | 80 | # Remove all jobs from the queue. This is a brute-force method, useful if 81 | # you want to reset queue, for example in a test environment. Note that it 82 | # resets only a single queue, and not all. 83 | clear: (queue, callback) -> 84 | @exec (collection) -> 85 | collection.remove { queue }, callback 86 | 87 | 88 | # Insert a new job into the queue. A job is just an array of arguments 89 | # which are inserted into a queue. What these arguments mean is entirely 90 | # up to the individual workers. 91 | enqueue: (queue, args..., callback)-> 92 | @exec (collection) => 93 | return callback(new Error('Last argument must be a callback')) if typeof callback isnt 'function' 94 | scheduledDate = queue.startDate if queue.startDate 95 | startDate = scheduledDate or Date.now() 96 | expires = new Date(+startDate + (queue.expires or @expires)) 97 | attempts = 0 98 | queue = queue.queue or queue 99 | 100 | task = { queue, expires, args, attempts } 101 | task.startDate = scheduledDate if scheduledDate 102 | collection.insertOne task, callback 103 | 104 | 105 | # Fetch the next job from the queue. The owner argument is used to identify 106 | # the worker which acquired the job. If you use a combination of hostname 107 | # and process ID, you can later identify stuck workers. 108 | next: (queue, owner, callback) -> 109 | now = new Date; timeout = new Date(now.getTime() + @timeout) 110 | query = 111 | expires: { $gt: now } 112 | $or: [ 113 | { startDate: { $lte: now } }, 114 | { startDate: { $exists: false } }, 115 | ] 116 | owner: null 117 | attempts: 118 | $lt: @maxAttempts 119 | 120 | update = { $set: { timeout, owner } } 121 | options = { sort: { expires: 1 }, returnOriginal: false } 122 | 123 | if queue then query.queue = queue 124 | @exec (collection) -> 125 | collection.findOneAndUpdate query, update, options, (err, result) -> callback err, result?.value 126 | 127 | 128 | # After you are done with the job, mark it as completed. This will remove 129 | # the job from MongoDB. 130 | complete: (doc, callback) -> 131 | @exec (collection) -> 132 | query = { _id: doc._id } 133 | options = { sort: { expires: 1 } } 134 | 135 | collection.findOneAndDelete query, options, (err, result) -> callback err, result?.value 136 | 137 | 138 | # You can also refuse to complete the job and leave it in the database 139 | # so that other workers can pick it up. 140 | release: (doc, callback) -> 141 | @exec (collection) -> 142 | query = { _id: doc._id } 143 | update = { $unset: { timeout: 1, owner: 1 }, $inc: {attempts: 1} } 144 | options = { sort: { expires: 1 }, returnOriginal: false } 145 | 146 | collection.findOneAndUpdate query, update, options, (err, result) -> callback err, result?.value 147 | 148 | 149 | # Release all timed out jobs, this makes them available for future 150 | # clients again. You should call this method regularly, possibly from 151 | # within the workers after every couple completed jobs. 152 | cleanup: (callback) -> 153 | @exec (collection) -> 154 | query = { timeout: { $lt: new Date } } 155 | update = { $unset: { timeout: 1, owner: 1 } } 156 | options = { multi: 1 } 157 | 158 | collection.update query, update, options, callback 159 | 160 | 161 | 162 | #### Template 163 | 164 | # Extend the **Template** class to define a job. You need to implement the 165 | # `perform` method. That method will be called when there is work to be done. 166 | # After you are done with the job, call `@complete` to signal the worker that 167 | # it can process the next job. 168 | class exports.Template 169 | constructor: (@worker, @doc) -> 170 | 171 | # Bind `this` to this instance in @perform and catch any exceptions. 172 | invoke: -> 173 | try 174 | @perform.apply @, @doc.args 175 | catch err 176 | @complete err 177 | 178 | # Implement this method. If you don't, kittens will die! 179 | perform: (args...) -> 180 | throw new Error 'Yo, you need to implement me!' 181 | 182 | 183 | # As per unwritten standard, first argument in callbacks is an error 184 | # indicator. So you can pass this method around as a completion callback. 185 | complete: (err) -> 186 | @worker.complete err, @doc 187 | 188 | 189 | 190 | #### Worker 191 | 192 | # A worker polls for new jobs and executes them. 193 | class exports.Worker extends require('events').EventEmitter 194 | constructor: (@connection, @templates, options) -> 195 | options or options = {} 196 | 197 | @name = [ require('os').hostname(), process.pid ].join ':' 198 | @timeout = options.timeout or 1000 199 | @rotate = options.rotate or false 200 | @workers = options.workers or 3 201 | @pending = 0 202 | 203 | 204 | poll: -> 205 | return if @stopped 206 | 207 | # If there are too many pending jobs, sleep for a bit. 208 | if @pending >= @workers 209 | return @sleep() 210 | 211 | Template = @getTemplate() 212 | templateName = if Template then Template.name 213 | 214 | @connection.next templateName, @name, (err, doc) => 215 | if err? and err.message isnt 'No matching object found' 216 | @emit 'error', err 217 | else if doc? 218 | ++@pending 219 | if !Template then Template = @getTemplate(doc.queue) 220 | if Template 221 | new Template(@, doc).invoke() 222 | else 223 | @emit 'error', new Error("Unknown template '#{ @name }'") 224 | process.nextTick => 225 | @poll() 226 | else 227 | @emit 'drained' if @pending is 0 228 | 229 | @sleep() 230 | 231 | getTemplate: (name) -> 232 | Template = null 233 | if name 234 | @templates.some (_Template) => 235 | if _Template.name == name 236 | Template = _Template 237 | true 238 | # Check the templates in round-robin order, one in each iteration. 239 | else if @rotate 240 | Template = @templates.shift() 241 | @templates.push Template 242 | Template 243 | 244 | # Sleep for a bit and then try to poll the queue again. If a timeout is 245 | # already active make sure to clear it first. 246 | sleep: -> 247 | clearTimeout @pollTimeout if @pollTimeout 248 | 249 | if not @stopped 250 | @pollTimeout = setTimeout => 251 | @pollTimeout = null 252 | @poll() 253 | , @timeout 254 | 255 | 256 | complete: (err, doc) -> 257 | cb = => 258 | --@pending 259 | @poll() if not @stopped 260 | @emit 'stopped' if @pending is 0 261 | 262 | if err? 263 | @emit 'error', err 264 | @connection.release doc, cb 265 | else 266 | @connection.complete doc, cb 267 | 268 | stop: () -> 269 | @stopped = true 270 | clearTimeout @pollTimeout 271 | @emit 'stopped' if @pending is 0 272 | 273 | -------------------------------------------------------------------------------- /test/connection.coffee: -------------------------------------------------------------------------------- 1 | _ = require 'lodash' 2 | queue = require '..' 3 | db = require './utils/db' 4 | 5 | QUnit.module 'Connection', 6 | teardown: () -> 7 | stop() 8 | db.teardown () -> 9 | start() 10 | 11 | test 'test connection with defaults', () -> 12 | expect 3 13 | 14 | stop() 15 | 16 | db.connect 'mongodb://localhost:27017/queue', (err) -> 17 | ok !err, 'connection to mongodb failed' 18 | conn = new queue.Connection 19 | conn.once 'connected', () -> 20 | db.conn.collection('queue').indexes (err, indexes) -> 21 | ok !err, 'getting indexes failed' 22 | equal indexes.length, 2 23 | start() 24 | 25 | test 'providing a connection', () -> 26 | expect 3 27 | 28 | stop() 29 | 30 | db.connect 'mongodb://localhost:27017/test-queue', (err) -> 31 | ok !err, 'connection to mongodb failed' 32 | conn = new queue.Connection(db: db.conn) 33 | conn.once 'connected', () -> 34 | db.conn.collection('queue').indexes (err, indexes) -> 35 | ok !err, 'getting indexes failed' 36 | equal indexes.length, 2 37 | start() 38 | 39 | test 'enqueue a task', () -> 40 | expect 7 41 | 42 | stop() 43 | 44 | db.connect 'mongodb://localhost:27017/test-queue', (err) -> 45 | ok !err, 'connection to mongodb failed' 46 | conn = new queue.Connection(db: db.conn) 47 | conn.once 'connected', () -> 48 | conn.enqueue 'Addition', () -> 49 | db.conn.collection('queue').find().toArray (err, docs) -> 50 | ok !err, 'error retrieving documents' 51 | equal docs.length, 1 52 | doc = docs[0] 53 | equal doc.queue, 'Addition' 54 | ok _.isDate(doc.expires) 55 | equal doc.args.length, 0 56 | equal doc.attempts, 0 57 | start() 58 | 59 | test 'enqueue a task overriding its settings', () -> 60 | expect 5 61 | 62 | stop() 63 | 64 | db.connect 'mongodb://localhost:27017/test-queue', (err) -> 65 | ok !err, 'connection to mongodb failed' 66 | conn = new queue.Connection(db: db.conn) 67 | conn.once 'connected', () -> 68 | task = 69 | queue: 'Testing', 70 | startDate: new Date(0), 71 | expires: 90 * 60 * 1000 72 | conn.enqueue task, () -> 73 | db.conn.collection('queue').find().toArray (err, docs) -> 74 | ok !err, 'error retrieving documents' 75 | equal docs.length, 1 76 | doc = docs[0] 77 | equal doc.queue, 'Testing' 78 | equal +doc.expires, +new Date(90 * 60 * 1000) 79 | start() -------------------------------------------------------------------------------- /test/template.coffee: -------------------------------------------------------------------------------- 1 | queue = require '..' 2 | 3 | QUnit.module 'Template' 4 | 5 | test 'perform throws if not implemented', () -> 6 | expect 1 7 | 8 | class Empty extends queue.Template 9 | 10 | empty = new Empty 11 | throws( 12 | () -> empty.perform() 13 | 'Unimplemented template should throw an error on perform' 14 | ) 15 | 16 | test 'executes an addition', () -> 17 | expect 3 18 | 19 | class Addition extends queue.Template 20 | perform: (a, b) -> 21 | @worker.result = a + b 22 | @complete() 23 | 24 | worker = 25 | complete: (err, doc) -> 26 | equal err, undefined 27 | deepEqual doc.args, [1, 2] 28 | equal @result, 3 29 | 30 | addition = new Addition(worker, args: [1, 2]) 31 | addition.invoke() 32 | 33 | test 'crash the worker', () -> 34 | expect 2 35 | 36 | class Crash extends queue.Template 37 | perform: () -> 38 | ok true, 'should go through here' 39 | throw new Error 40 | 41 | worker = 42 | complete: (err) -> 43 | ok !!err 44 | 45 | crash = new Crash(worker, args: []) 46 | crash.invoke() 47 | -------------------------------------------------------------------------------- /test/tests.coffee: -------------------------------------------------------------------------------- 1 | testRunner = require 'qunit' 2 | fs = require 'fs' 3 | path = require 'path' 4 | 5 | tests = fs.readdirSync(__dirname). 6 | map((file) -> path.join __dirname, file). 7 | filter((file) -> /\.js/.test(file) and file isnt __filename) 8 | 9 | options = testRunner.options 10 | options.coverage = true 11 | 12 | log = options.log 13 | log.assertions = false 14 | log.tests = false 15 | log.summary = false 16 | 17 | testRunner.run( 18 | { code: path.join(__dirname, '../lib/queue.js'), tests } 19 | (err) -> console.error(err) if err 20 | ) -------------------------------------------------------------------------------- /test/utils/db.coffee: -------------------------------------------------------------------------------- 1 | mongodb = require 'mongodb' 2 | 3 | exports.conn = null 4 | 5 | exports.connect = (connStr, callback) -> 6 | mongodb.MongoClient.connect connStr, (err, db) -> 7 | callback(err) if err 8 | db.dropDatabase (err) -> 9 | callback(err) if err 10 | exports.conn = db 11 | callback() 12 | 13 | exports.teardown = (callback) -> 14 | exports.conn.dropDatabase () -> 15 | exports.conn.close true, () -> 16 | exports.conn = null 17 | callback() -------------------------------------------------------------------------------- /test/worker.coffee: -------------------------------------------------------------------------------- 1 | _ = require 'lodash' 2 | queue = require '..' 3 | db = require './utils/db' 4 | 5 | QUnit.module 'Worker', 6 | teardown: () -> 7 | stop() 8 | db.teardown () -> 9 | start() 10 | 11 | test 'test worker with one task', () -> 12 | expect 12 13 | 14 | stop() 15 | 16 | db.connect 'mongodb://localhost:27017/test-worker', (err) -> 17 | ok !err, 'connection to mongodb failed' 18 | 19 | class Addition extends queue.Template 20 | perform: (a, b) -> 21 | ok true 22 | db.conn.collection('queue').findOne {}, (err, doc) => 23 | ok !err, 'error finding the task' 24 | equal doc.queue, 'Addition' 25 | ok _.isDate(doc.expires) 26 | deepEqual doc.args, [1, 1] 27 | equal doc.attempts, 0 28 | ok _.isDate(doc.timeout) 29 | ok _.isString(doc.owner) 30 | @complete() 31 | 32 | conn = new queue.Connection(db: db.conn) 33 | conn.on 'error', (err) -> 34 | ok false, err 35 | 36 | conn.enqueue Addition.name, 1, 1, () -> 37 | ok true, 'should be called when enqueued' 38 | 39 | worker = new queue.Worker conn, [ Addition ] 40 | worker.on 'error', () -> 41 | ok false, 'there was an error on the worker' 42 | 43 | worker.once 'drained', () -> 44 | worker.once 'stopped', () -> 45 | db.conn.collection('queue').count (err, count) -> 46 | ok !err, 'error counting' 47 | equal count, 0 48 | start() 49 | worker.stop() 50 | 51 | worker.poll() 52 | 53 | test 'test worker with a task erroring', () -> 54 | expect 9 55 | 56 | stop() 57 | 58 | db.connect 'mongodb://localhost:27017/test-worker', (err) -> 59 | ok !err, 'connection to mongodb failed' 60 | 61 | class Addition extends queue.Template 62 | perform: (a, b) -> 63 | @complete new Error('task error') 64 | 65 | conn = new queue.Connection(db: db.conn) 66 | conn.on 'error', (err) -> 67 | ok false, err 68 | 69 | conn.enqueue Addition.name, 1, 2, () -> 70 | ok true, 'should be called when enqueued' 71 | 72 | worker = new queue.Worker conn, [ Addition ] 73 | worker.on 'error', (err) -> 74 | equal err.message, 'task error' 75 | 76 | worker.once 'drained', () -> 77 | worker.once 'stopped', () -> 78 | db.conn.collection('queue').findOne {}, (err, doc) -> 79 | ok !err, 'error finding task' 80 | equal doc.attempts, 5 81 | start() 82 | worker.stop() 83 | 84 | worker.poll() 85 | 86 | test 'scheduled task', () -> 87 | expect 3 88 | 89 | stop() 90 | 91 | db.connect 'mongodb://localhost:27017/test-worker', (err) -> 92 | ok !err, 'connection to mongodb failed' 93 | 94 | conn = new queue.Connection(db: db.conn) 95 | conn.on 'error', (err) -> 96 | ok false, err 97 | 98 | conn.once 'connected', () -> 99 | class Addition extends queue.Template 100 | perform: (a, b) -> 101 | ok new Date >= startDate, 'task started too soon' 102 | @complete() 103 | worker.once 'stopped', () -> 104 | start() 105 | worker.stop() 106 | 107 | startDate = new Date 108 | startDate.setSeconds(startDate.getSeconds() + 5) 109 | task = 110 | queue: Addition.name 111 | startDate: startDate 112 | 113 | conn.enqueue task, 1, 3, () -> 114 | ok true, 'should be called when enqueued' 115 | 116 | worker = new queue.Worker conn, [ Addition ] 117 | worker.on 'error', () -> 118 | ok false, 'there was an error on the worker' 119 | 120 | worker.poll() 121 | --------------------------------------------------------------------------------