├── index.js ├── .npmignore ├── lib ├── http │ ├── views │ │ ├── _row.pug │ │ ├── _search.pug │ │ ├── _sort.pug │ │ ├── _filter.pug │ │ ├── job │ │ │ └── list.pug │ │ ├── _menu.pug │ │ ├── layout.pug │ │ └── _job.pug │ ├── public │ │ ├── images │ │ │ └── bg.jpg │ │ ├── stylesheets │ │ │ ├── error.styl │ │ │ ├── mixins.styl │ │ │ ├── actions.styl │ │ │ ├── config.styl │ │ │ ├── context-menu.styl │ │ │ ├── main.styl │ │ │ ├── scrollbar.styl │ │ │ ├── menu.styl │ │ │ ├── job.styl │ │ │ └── main.css │ │ └── javascripts │ │ │ ├── search.js │ │ │ ├── jquery.ext.js │ │ │ ├── utils.js │ │ │ ├── loading.js │ │ │ ├── progress.js │ │ │ ├── main.js │ │ │ ├── job.js │ │ │ └── caustic.js │ ├── middleware │ │ └── provides.js │ ├── routes │ │ ├── index.js │ │ └── json.js │ └── index.js ├── queue │ ├── test_mode.js │ ├── events.js │ └── worker.js ├── redis.js └── kue.js ├── .gitignore ├── test ├── mocha.opts ├── test_mode.js ├── prefix.coffee ├── shutdown.coffee ├── tdd │ ├── redis.spec.js │ └── kue.spec.js ├── test.js ├── jsonapi.js └── test.coffee ├── .travis.yml ├── bin └── kue-dashboard ├── Makefile ├── examples ├── shutdown.js ├── delayed.js ├── many.js ├── video.js ├── events.js └── stale.js ├── LICENSE ├── package.json └── History.md /index.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./lib/kue'); -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | support 2 | test 3 | examples 4 | *.sock 5 | -------------------------------------------------------------------------------- /lib/http/views/_row.pug: -------------------------------------------------------------------------------- 1 | tr 2 | td.title 3 | td.value -------------------------------------------------------------------------------- /lib/http/views/_search.pug: -------------------------------------------------------------------------------- 1 | input#search(type='text', placeholder='Search') -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .DS_Store 3 | node_modules 4 | *.sock 5 | *.rdb 6 | test/incomplete 7 | *.swp 8 | -------------------------------------------------------------------------------- /lib/http/public/images/bg.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Automattic/kue/HEAD/lib/http/public/images/bg.jpg -------------------------------------------------------------------------------- /test/mocha.opts: -------------------------------------------------------------------------------- 1 | --compilers coffee:coffee-script 2 | --require should 3 | --reporter spec 4 | --ui bdd 5 | --timeout 10000 -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "4" 4 | - "6" 5 | - "8" 6 | - "node" 7 | services: 8 | - redis-server 9 | -------------------------------------------------------------------------------- /lib/http/views/_sort.pug: -------------------------------------------------------------------------------- 1 | select#sort 2 | option(value='asc') sort 3 | option(value='asc') asc 4 | option(value='desc') desc -------------------------------------------------------------------------------- /lib/http/views/_filter.pug: -------------------------------------------------------------------------------- 1 | select#filter 2 | option(value='') filter by 3 | each type in types 4 | option(value=type)= type 5 | -------------------------------------------------------------------------------- /lib/http/views/job/list.pug: -------------------------------------------------------------------------------- 1 | extends ../layout 2 | 3 | block body 4 | h1 #{state} 5 | 6 | script. 7 | o(function(){ 8 | init('#{state}'); 9 | }); 10 | 11 | 12 | #jobs 13 | #loading: canvas(width=50, height=50) 14 | -------------------------------------------------------------------------------- /lib/http/public/stylesheets/error.styl: -------------------------------------------------------------------------------- 1 | 2 | #error 3 | fixed: top -50px right 15px 4 | padding: 20px 5 | transition: top 500ms, opacity 500ms 6 | opacity: 0 7 | background: rgba(dark, .2) 8 | border: 1px solid rgba(dark, .3) 9 | border-radius: 5px 10 | color: dark 11 | &.show 12 | top: 15px 13 | opacity: 1 14 | -------------------------------------------------------------------------------- /lib/http/middleware/provides.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Specify that the route provides `type`. 3 | * 4 | * @param {String} type 5 | * @return {Function} 6 | * @api private 7 | */ 8 | 9 | module.exports = function( type ) { 10 | return function( req, res, next ) { 11 | if( req.accepts(type) ) return next(); 12 | next('route'); 13 | } 14 | }; -------------------------------------------------------------------------------- /lib/http/public/stylesheets/mixins.styl: -------------------------------------------------------------------------------- 1 | reset-list() 2 | margin: 0 3 | padding: 0 4 | li 5 | margin: 0 6 | list-style: none 7 | 8 | decorated-box() 9 | border: 1px solid #eee 10 | border-bottom-color: rgba(black, .25) 11 | border-left-color: rgba(black, .2) 12 | border-right-color: rgba(black, .2) 13 | box-shadow: 0 2px 2px 0 rgba(black, .1) 14 | border-radius: 4px 15 | -------------------------------------------------------------------------------- /lib/http/public/stylesheets/actions.styl: -------------------------------------------------------------------------------- 1 | 2 | #actions 3 | fixed: top -2px right -2px 4 | z-index: 20 5 | 6 | #sort 7 | #filter 8 | #search 9 | float: left 10 | margin: 0 11 | padding: 5px 10px 12 | border: 1px solid #eee 13 | border-radius: 0 0 0 5px 14 | -webkit-appearance: none 15 | color: dark 16 | outline: none 17 | &:hover 18 | border-color: #eee - 10% 19 | 20 | #sort 21 | #filter 22 | cursor: pointer 23 | 24 | #sort 25 | #filter 26 | border-radius: 0 27 | border-left: none -------------------------------------------------------------------------------- /lib/http/views/_menu.pug: -------------------------------------------------------------------------------- 1 | ul#menu 2 | li.inactive 3 | a(href='./inactive') 4 | .count 0 5 | | Queued 6 | li.active 7 | a.active(href='./active') 8 | .count 0 9 | | Active 10 | li.failed 11 | a(href='./failed') 12 | .count 0 13 | | Failed 14 | li.complete 15 | a(href='./complete') 16 | .count 0 17 | | Complete 18 | li.delayed 19 | a(href='./delayed') 20 | .count 0 21 | | Delayed 22 | -------------------------------------------------------------------------------- /lib/http/public/stylesheets/config.styl: -------------------------------------------------------------------------------- 1 | // general colors 2 | 3 | dark = #3b3b3b 4 | light = #666 5 | lighter = #777 6 | 7 | bg = #fff 8 | 9 | // status colors 10 | 11 | inactive-color = #00CCCC 12 | complete-color = #00CC7A 13 | active-color = #CCC500 14 | failed-color = #c00 15 | 16 | // menu config 17 | 18 | menu-bg = dark 19 | menu-fg = lighter 20 | 21 | menu-intensity = 13% 22 | menu-colored = false 23 | 24 | // job config 25 | 26 | job-bg = white 27 | 28 | // scrollbar 29 | 30 | scroll-bg = transparent 31 | scroll-thumb = menu-bg 32 | scroll-width = 6px 33 | -------------------------------------------------------------------------------- /bin/kue-dashboard: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | var kue = require('kue'); 3 | var argv = require('yargs') 4 | .usage('Usage: $0 [options]') 5 | .example('$0 -p 3050 -r redis://10.0.0.4:6379 -q q') 6 | .describe('r', 'Redis url') 7 | .describe('p', 'Dashboard port') 8 | .describe('q', 'Prefix to use') 9 | .default('p', 3000) 10 | .default('r', 'redis://127.0.0.1:6379') 11 | .default('q', 'q') 12 | .help('h') 13 | .alias('h', 'help') 14 | .argv 15 | ; 16 | 17 | kue.createQueue({ 18 | redis: argv.r, 19 | prefix: argv.q 20 | }); 21 | 22 | 23 | kue.app.listen(argv.p); 24 | console.log("Running on http://127.0.0.1:" + argv.p); 25 | -------------------------------------------------------------------------------- /lib/http/routes/index.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * kue - http - routes 3 | * Copyright (c) 2011 LearnBoost 4 | * MIT Licensed 5 | */ 6 | 7 | /** 8 | * Module dependencies. 9 | */ 10 | 11 | var Queue = require('../../kue') 12 | , Job = require('../../queue/job') 13 | , queue = Queue.createQueue(); 14 | 15 | /** 16 | * Serve the index page. 17 | */ 18 | 19 | exports.jobs = function( state ) { 20 | return function( req, res ) { 21 | queue.types(function( err, types ) { 22 | res.render('job/list', { 23 | state: state, types: types, title: req.app.get('title') 24 | }); 25 | }); 26 | }; 27 | }; 28 | -------------------------------------------------------------------------------- /lib/http/public/javascripts/search.js: -------------------------------------------------------------------------------- 1 | o(function () { 2 | var search = o('#search'); 3 | search.keyup(function () { 4 | var val = search.val().trim() 5 | , jobs = o('#jobs .job'); 6 | 7 | // show all 8 | if (val.length < 2) return jobs.show(); 9 | 10 | // query 11 | o.get('./job/search?q=' + encodeURIComponent(val), function (ids) { 12 | jobs.each(function (i, el) { 13 | var id = el.id.replace('job-', ''); 14 | if (~ids.indexOf(id)) { 15 | o(el).show(); 16 | } else { 17 | o(el).hide(); 18 | } 19 | }); 20 | }); 21 | }); 22 | }); 23 | -------------------------------------------------------------------------------- /lib/http/public/stylesheets/context-menu.styl: -------------------------------------------------------------------------------- 1 | @import 'mixins' 2 | 3 | highlight-color = #00B3E9 4 | 5 | .context-menu 6 | display: none 7 | reset-list() 8 | decorated-box() 9 | li 10 | &:last-child a 11 | border-bottom: none 12 | a 13 | display: block 14 | background: white 15 | padding: 5px 10px 16 | border: 1px solid transparent 17 | border-bottom: 1px solid #eee 18 | font-size: 12px 19 | &:hover 20 | background: linear-gradient(bottom, highlight-color, highlight-color + 50%) 21 | color: white 22 | border: 1px solid white 23 | &:active 24 | background: linear-gradient(bottom, highlight-color + 10%, highlight-color + 10% + 50%) 25 | -------------------------------------------------------------------------------- /lib/http/public/javascripts/jquery.ext.js: -------------------------------------------------------------------------------- 1 | // proxy to allow formatting 2 | // and because $ is ugly 3 | 4 | var o = function (val) { 5 | var args = arguments 6 | , options = args[1] 7 | , i = 0; 8 | 9 | if ('string' != typeof val) return $(val); 10 | if (!~val.indexOf('<')) return $(val); 11 | 12 | val = val.replace(/%([sd])/g, function (_, specifier) { 13 | var arg = args[++i]; 14 | switch (specifier) { 15 | case 's': 16 | return String(arg) 17 | case 'd': 18 | return arg | 0; 19 | } 20 | }); 21 | 22 | val = val.replace(/\{(\w+)\}/g, function (_, name) { 23 | return options[name]; 24 | }); 25 | 26 | return $(val); 27 | }; 28 | 29 | for (var key in $) o[key] = $[key]; -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | REPORTER = spec 2 | 3 | all: build 4 | 5 | build: 6 | @./node_modules/coffee-script/bin/coffee \ 7 | -c \ 8 | -o lib src 9 | 10 | test-tdd: 11 | @./node_modules/.bin/mocha \ 12 | --reporter $(REPORTER) \ 13 | --require should \ 14 | --require sinon \ 15 | --ui tdd \ 16 | test/tdd/*.js 17 | 18 | test-bdd: 19 | @./node_modules/.bin/mocha \ 20 | --reporter $(REPORTER) \ 21 | --require should \ 22 | --ui bdd \ 23 | test/*.js 24 | 25 | test-bdd-coffee: 26 | @./node_modules/.bin/mocha \ 27 | --compilers coffee:coffee-script \ 28 | --reporter $(REPORTER) \ 29 | --require should \ 30 | --require coffee-script/register \ 31 | --ui bdd \ 32 | test/*.coffee 33 | 34 | 35 | test-all: test-tdd test-bdd test-bdd-coffee 36 | 37 | .PHONY: test-all 38 | -------------------------------------------------------------------------------- /examples/shutdown.js: -------------------------------------------------------------------------------- 1 | var kue = require( '../' ) 2 | 3 | var jobs = kue.createQueue() 4 | 5 | 6 | function generateJobs() { 7 | for ( var i = 0; i < 12; i++ ) { 8 | console.log( 'Creating Job #' + i ); 9 | jobs.create( 'long render', { 10 | title: 'rendering frame #' + i 11 | } ).save(); 12 | } 13 | } 14 | 15 | 16 | jobs.process( 'long render', 4, function ( job, done ) { 17 | console.log( 'Starting ' + job.data.title ); 18 | setTimeout( function () { 19 | console.log( 'Finished ' + job.data.title ); 20 | done(); 21 | }, 3000 ); 22 | } ) 23 | 24 | 25 | generateJobs(); 26 | 27 | setTimeout( function () { 28 | console.log( '[ Shutting down when all jobs finish... ]' ); 29 | jobs.shutdown( function ( err ) { 30 | console.log( '[ All jobs finished. Kue is shut down. ]' ); 31 | process.exit( 0 ); 32 | } ) 33 | }, 4200 ) 34 | 35 | -------------------------------------------------------------------------------- /lib/http/views/layout.pug: -------------------------------------------------------------------------------- 1 | html 2 | head 3 | title= title 4 | link(rel='stylesheet', href='./stylesheets/main.css') 5 | script(src='./javascripts/utils.js') 6 | script(src='./javascripts/jquery.min.js') 7 | script(src='./javascripts/jquery.ext.js') 8 | script(src='./javascripts/caustic.js') 9 | script(src='./javascripts/progress.js') 10 | script(src='./javascripts/loading.js') 11 | script(src='./javascripts/job.js') 12 | script(src='./javascripts/search.js') 13 | script(src='./javascripts/main.js') 14 | body 15 | include _menu 16 | #actions 17 | include _search 18 | include _filter 19 | include _sort 20 | #content 21 | block body 22 | script(type='text/template')#job-template 23 | include _job 24 | script(type='text/template')#row-template 25 | include _row 26 | #error 27 | -------------------------------------------------------------------------------- /lib/http/public/stylesheets/main.styl: -------------------------------------------------------------------------------- 1 | font-smoothing() 2 | -webkit-font-smoothing: arguments 3 | 4 | @import 'nib' 5 | @import 'config' 6 | @import 'scrollbar' 7 | @import 'menu' 8 | @import 'context-menu' 9 | @import 'job' 10 | @import 'actions' 11 | @import 'error' 12 | 13 | body 14 | font: 13px "helvetica neue", helvetica, arial, sans-serif 15 | font-smoothing: antialiased 16 | background: bg 17 | color: light 18 | 19 | h1, h2, h3 20 | margin: 0 0 25px 0 21 | padding: 0 22 | font-weight: normal 23 | text-transform: capitalize 24 | color: light 25 | 26 | h2 27 | font-size: 16px 28 | margin-top: 20px 29 | 30 | button 31 | a.button 32 | input[type='submit'] 33 | bold-button(glow:#00ABFA) 34 | 35 | pre 36 | margin-top: 20px 37 | 38 | a 39 | text-decoration: none 40 | cursor: pointer 41 | 42 | table 43 | reset-table() 44 | tr td 45 | padding: 2px 5px 46 | 47 | #loading 48 | width: 100% 49 | text-align: center 50 | margin-top: 40px 51 | margin-left: 20px 52 | canvas 53 | margin: 0 auto 54 | -------------------------------------------------------------------------------- /examples/delayed.js: -------------------------------------------------------------------------------- 1 | var kue = require( '../' ); 2 | 3 | // create our job queue 4 | 5 | var jobs = kue.createQueue(); 6 | 7 | // one minute 8 | 9 | var minute = 60000; 10 | 11 | var email = jobs.create( 'email', { 12 | title: 'Account renewal required', to: 'tj@learnboost.com', template: 'renewal-email' 13 | } ).delay( minute ) 14 | .priority( 'high' ) 15 | .save(); 16 | 17 | 18 | email.on( 'promotion', function () { 19 | console.log( 'renewal job promoted' ); 20 | } ); 21 | 22 | email.on( 'complete', function () { 23 | console.log( 'renewal job completed' ); 24 | } ); 25 | 26 | jobs.create( 'email', { 27 | title: 'Account expired', to: 'tj@learnboost.com', template: 'expired-email' 28 | } ).delay( minute * 10 ) 29 | .priority( 'high' ) 30 | .save(); 31 | 32 | jobs.promote(); 33 | 34 | jobs.process( 'email', 10, function ( job, done ) { 35 | setTimeout( function () { 36 | done(); 37 | }, Math.random() * 5000 ); 38 | } ); 39 | 40 | // start the UI 41 | kue.app.listen( 3000 ); 42 | console.log( 'UI started on port 3000' ); -------------------------------------------------------------------------------- /lib/http/public/stylesheets/scrollbar.styl: -------------------------------------------------------------------------------- 1 | width = scroll-width 2 | pad-x = 60px 3 | pad-y = 40px 4 | 5 | body 6 | padding: 50px 120px 7 | 8 | /* 9 | html 10 | overflow: auto 11 | 12 | body 13 | position: absolute 14 | top: pad-y 15 | left: pad-x 16 | bottom: pad-y 17 | right: pad-x 18 | padding: 0 pad-x 19 | overflow-y: scroll 20 | overflow-x: hidden 21 | 22 | ::-webkit-scrollbar 23 | background: scroll-bg 24 | width: width 25 | 26 | ::-webkit-scrollbar-button:start:decrement 27 | ::-webkit-scrollbar-button:start:increment 28 | display: none 29 | 30 | ::-webkit-scrollbar-track 31 | border-radius: (width / 2) 32 | box-shadow: inset 0 0 1px rgba(black, .2), inset 0 4px 10px rgba(black, .2) 33 | border: 1px solid rgba(white, .5) 34 | 35 | ::-webkit-scrollbar-track-piece 36 | background: transparent 37 | 38 | ::-webkit-scrollbar-thumb:vertical 39 | height: 30px 40 | transition: background-color 300ms ease-out 41 | background: rgba(scroll-thumb, .5) 42 | border-radius: (width / 2) 43 | &:window-inactive 44 | background: rgba(scroll-thumb, .2) 45 | */ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2011 LearnBoost 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /examples/many.js: -------------------------------------------------------------------------------- 1 | var kue = require( '../' ) 2 | , express = require( 'express' ); 3 | 4 | // create our job queue 5 | 6 | var jobs = kue.createQueue(); 7 | 8 | function create() { 9 | var name = [ 'tobi', 'loki', 'jane', 'manny' ][ Math.random() * 4 | 0 ]; 10 | jobs.create( 'video conversion', { 11 | title: 'converting ' + name + '\'s to avi', user: 1, frames: 200 12 | } ).save(); 13 | setTimeout( create, Math.random() * 3000 | 0 ); 14 | } 15 | 16 | create(); 17 | 18 | function create2() { 19 | var name = [ 'tobi', 'loki', 'jane', 'manny' ][ Math.random() * 4 | 0 ]; 20 | jobs.create( 'email', { 21 | title: 'emailing ' + name + '', body: 'hello' 22 | } ).save(); 23 | setTimeout( create2, Math.random() * 1000 | 0 ); 24 | } 25 | 26 | create2(); 27 | 28 | // process video conversion jobs, 2 at a time. 29 | 30 | jobs.process( 'video conversion', 2, function ( job, done ) { 31 | console.log( 'video' ); 32 | setTimeout( done, Math.random() * 5000 ); 33 | } ); 34 | 35 | // process 10 emails at a time 36 | 37 | jobs.process( 'email', 10, function ( job, done ) { 38 | console.log( 'email' ); 39 | setTimeout( done, Math.random() * 2000 ); 40 | } ); 41 | 42 | // start the UI 43 | kue.app.listen( 3000 ); 44 | console.log( 'UI started on port 3000' ); 45 | -------------------------------------------------------------------------------- /lib/http/public/stylesheets/menu.styl: -------------------------------------------------------------------------------- 1 | @import 'mixins' 2 | 3 | #menu 4 | reset-list() 5 | fixed: top left 6 | height: 100% 7 | width: 80px 8 | background: menu-bg 9 | border-right: 1px solid menu-bg - 40% 10 | box-shadow: 0 0 0 1px rgba(white, .5) 11 | li 12 | position: relative 13 | text-align: center 14 | if menu-colored 15 | &.inactive 16 | border-right: 1px solid inactive-color 17 | &.active 18 | border-right: 1px solid active-color 19 | &.complete 20 | border-right: 1px solid complete-color 21 | &.failed 22 | border-right: 1px solid failed-color 23 | .count 24 | absolute: top 15px left 25 | text-shadow: 1px 1px 1px menu-bg - menu-intensity 26 | width: 100% 27 | color: menu-fg + (menu-intensity / 2) 28 | a 29 | display: block 30 | padding: 40px 0 10px 0 31 | color: menu-fg 32 | border-top: 1px solid menu-bg + menu-intensity 33 | border-bottom: 1px solid menu-bg - menu-intensity 34 | font-size: 12px 35 | background: menu-bg -= 2% 36 | &:hover 37 | background: menu-bg + 5% 38 | &:active 39 | &.active 40 | background: #343434 41 | box-shadow: inset 0 0 3px 2px menu-bg - 30%, inset 0 -5px 10px 2px menu-bg - 15% 42 | border-bottom: 1px solid menu-bg - 40% 43 | -------------------------------------------------------------------------------- /lib/http/views/_job.pug: -------------------------------------------------------------------------------- 1 | .job 2 | .block.contents 3 | h2.id 4 | a.remove(title='Delete Job') x 5 | a.restart(title='Restart Job') ↻ 6 | canvas.progress(width=50, height=50) 7 | table.meta 8 | tbody 9 | tr 10 | td Type: 11 | td.type 12 | tr 13 | td Title: 14 | td.title 15 | tr 16 | td Error: 17 | td.errorMessage 18 | .details 19 | .data 20 | table.data 21 | tbody 22 | tr 23 | td State: 24 | td.state 25 | tr 26 | td Priority: 27 | td.priority 28 | tr 29 | td Attempts: 30 | td.attempts 31 | tr.time 32 | td Duration: 33 | td.duration 34 | tr.time 35 | td Created: 36 | td.created_at 37 | tr.time 38 | td Updated: 39 | td.updated_at 40 | tr.time 41 | td Failed: 42 | td.failed_at 43 | .error 44 | pre 45 | ul.log 46 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kue", 3 | "version": "0.11.5", 4 | "description": "Feature rich priority job queue backed by redis", 5 | "homepage": "http://automattic.github.io/kue/", 6 | "keywords": [ 7 | "job", 8 | "queue", 9 | "worker", 10 | "redis" 11 | ], 12 | "license": "MIT", 13 | "author": "TJ Holowaychuk ", 14 | "contributors": [ 15 | { 16 | "name": "Behrad Zari", 17 | "email": "behradz@gmail.com" 18 | } 19 | ], 20 | "repository": { 21 | "type": "git", 22 | "url": "https://github.com/Automattic/kue.git" 23 | }, 24 | "bugs": { 25 | "url": "https://github.com/Automattic/kue/issues" 26 | }, 27 | "dependencies": { 28 | "body-parser": "^1.12.2", 29 | "express": "^4.12.2", 30 | "lodash": "^4.0.0", 31 | "nib": "~1.1.2", 32 | "node-redis-warlock": "~0.2.0", 33 | "pug": "^2.0.0-beta3", 34 | "redis": "~2.6.0-2", 35 | "stylus": "~0.54.5", 36 | "yargs": "^4.0.0" 37 | }, 38 | "devDependencies": { 39 | "async": "^1.4.2", 40 | "chai": "^3.3.0", 41 | "coffee-script": "~1.10.0", 42 | "mocha": "^2.3.3", 43 | "should": "^3.1.0", 44 | "sinon": "^1.17.2", 45 | "supertest": "^1.1.0" 46 | }, 47 | "main": "index", 48 | "bin": { 49 | "kue-dashboard": "bin/kue-dashboard" 50 | }, 51 | "scripts": { 52 | "test": "make test-all" 53 | }, 54 | "optionalDependencies": { 55 | "reds": "^0.2.5" 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /lib/queue/test_mode.js: -------------------------------------------------------------------------------- 1 | var Job = require('./job'), 2 | _ = require('lodash'); 3 | 4 | var originalJobSave = Job.prototype.save, 5 | originalJobUpdate = Job.prototype.update, 6 | processQueue, 7 | jobs; 8 | 9 | function testJobSave( fn ) { 10 | if(processQueue) { 11 | jobs.push(this); 12 | originalJobSave.call(this, fn); 13 | } else { 14 | this.id = _.uniqueId(); 15 | jobs.push(this); 16 | if( _.isFunction(fn) ) fn(); 17 | } 18 | }; 19 | 20 | function testJobUpdate( fn ) { 21 | if(processQueue) { 22 | originalJobUpdate.call(this, fn); 23 | } else { 24 | if( _.isFunction(fn) ) fn(); 25 | } 26 | }; 27 | 28 | /** 29 | * Array of jobs added to the queue 30 | * @api public 31 | */ 32 | 33 | module.exports.jobs = jobs = []; 34 | module.exports.processQueue = processQueue = false; 35 | 36 | /** 37 | * Enable test mode. 38 | * @api public 39 | */ 40 | 41 | module.exports.enter = function(process) { 42 | processQueue = process || false; 43 | Job.prototype.save = testJobSave; 44 | Job.prototype.update = testJobUpdate; 45 | }; 46 | 47 | /** 48 | * Disable test mode. 49 | * @api public 50 | */ 51 | 52 | module.exports.exit = function() { 53 | Job.prototype.save = originalJobSave; 54 | Job.prototype.update = originalJobUpdate; 55 | }; 56 | 57 | /** 58 | * Clear the array of queued jobs 59 | * @api public 60 | */ 61 | 62 | module.exports.clear = function() { 63 | jobs.length = 0; 64 | }; 65 | -------------------------------------------------------------------------------- /examples/video.js: -------------------------------------------------------------------------------- 1 | var kue = require( '../' ) 2 | , express = require( 'express' ); 3 | 4 | // create our job queue 5 | 6 | var jobs = kue.createQueue(); 7 | 8 | // start redis with $ redis-server 9 | 10 | // create some jobs at random, 11 | // usually you would create these 12 | // in your http processes upon 13 | // user input etc. 14 | 15 | function create() { 16 | var name = [ 'tobi', 'loki', 'jane', 'manny' ][ Math.random() * 4 | 0 ]; 17 | console.log( '- creating job for %s', name ); 18 | jobs.create( 'video conversion', { 19 | title: 'converting ' + name + '\'s to avi', user: 1, frames: 200 20 | } ).save(); 21 | setTimeout( create, Math.random() * 3000 | 0 ); 22 | } 23 | 24 | create(); 25 | 26 | // process video conversion jobs, 3 at a time. 27 | 28 | jobs.process( 'video conversion', 3, function ( job, done ) { 29 | var frames = job.data.frames; 30 | console.log( "job process %d", job.id ); 31 | function next( i ) { 32 | // pretend we are doing some work 33 | convertFrame( i, function ( err ) { 34 | if ( err ) return done( err ); 35 | // report progress, i/frames complete 36 | job.progress( i, frames ); 37 | if ( i == frames ) done() 38 | else next( i + 1 ); 39 | } ); 40 | } 41 | 42 | next( 0 ); 43 | } ); 44 | 45 | function convertFrame( i, fn ) { 46 | setTimeout( fn, Math.random() * 100 ); 47 | } 48 | 49 | // start the UI 50 | var app = express.createServer(); 51 | app.use( express.basicAuth( 'foo', 'bar' ) ); 52 | app.use( kue.app ); 53 | app.listen( 3000 ); 54 | console.log( 'UI started on port 3000' ); -------------------------------------------------------------------------------- /examples/events.js: -------------------------------------------------------------------------------- 1 | var kue = require( '../' ); 2 | 3 | // create our job queue 4 | 5 | var jobs = kue.createQueue(); 6 | 7 | // start redis with $ redis-server 8 | 9 | // create some jobs at random, 10 | // usually you would create these 11 | // in your http processes upon 12 | // user input etc. 13 | 14 | function create() { 15 | var name = [ 'tobi', 'loki', 'jane', 'manny' ][ Math.random() * 4 | 0 ]; 16 | var job = jobs.create( 'video conversion', { 17 | title: 'converting ' + name + '\'s to avi', user: 1, frames: 200 18 | } ); 19 | 20 | job.on( 'complete', function () { 21 | console.log( " Job complete" ); 22 | } ).on( 'failed', function () { 23 | console.log( " Job failed" ); 24 | } ).on( 'progress', function ( progress ) { 25 | process.stdout.write( '\r job #' + job.id + ' ' + progress + '% complete' ); 26 | } ); 27 | 28 | job.save(); 29 | 30 | setTimeout( create, Math.random() * 2000 | 0 ); 31 | } 32 | 33 | create(); 34 | 35 | // process video conversion jobs, 1 at a time. 36 | 37 | jobs.process( 'video conversion', 1, function ( job, done ) { 38 | var frames = job.data.frames; 39 | 40 | function next( i ) { 41 | // pretend we are doing some work 42 | convertFrame( i, function ( err ) { 43 | if ( err ) return done( err ); 44 | // report progress, i/frames complete 45 | job.progress( i, frames ); 46 | if ( i >= frames ) done() 47 | else next( i + Math.random() * 10 ); 48 | } ); 49 | } 50 | 51 | next( 0 ); 52 | } ); 53 | 54 | function convertFrame( i, fn ) { 55 | setTimeout( fn, Math.random() * 50 ); 56 | } 57 | 58 | // start the UI 59 | kue.app.listen( 3000 ); 60 | console.log( 'UI started on port 3000' ); 61 | -------------------------------------------------------------------------------- /examples/stale.js: -------------------------------------------------------------------------------- 1 | var kue = require( '../' ) 2 | , express = require( 'express' ); 3 | 4 | // create our job queue 5 | 6 | var jobs = kue.createQueue() 7 | , Job = kue.Job; 8 | 9 | // start redis with $ redis-server 10 | 11 | // create some jobs at random, 12 | // usually you would create these 13 | // in your http processes upon 14 | // user input etc. 15 | 16 | function create() { 17 | var name = [ 'tobi', 'loki', 'jane', 'manny' ][ Math.random() * 4 | 0 ]; 18 | console.log( '- creating job for %s', name ); 19 | jobs.create( 'video conversion', { 20 | title: 'converting ' + name + '\'s to avi', user: 1, frames: 200 21 | } ).save(); 22 | setTimeout( create, Math.random() * 3000 | 0 ); 23 | } 24 | 25 | create(); 26 | 27 | // process video conversion jobs, 3 at a time. 28 | 29 | jobs.process( 'video conversion', 3, function ( job, done ) { 30 | var frames = job.data.frames; 31 | console.log( "job process %d", job.id ); 32 | function next( i ) { 33 | // pretend we are doing some work 34 | convertFrame( i, function ( err ) { 35 | if ( err ) return done( err ); 36 | // report progress, i/frames complete 37 | job.progress( i, frames ); 38 | if ( i == frames ) done() 39 | else next( i + 5 ); 40 | } ); 41 | } 42 | 43 | next( 0 ); 44 | } ); 45 | 46 | function convertFrame( i, fn ) { 47 | setTimeout( fn, Math.random() * 100 ); 48 | } 49 | 50 | // remove stale jobs 51 | jobs.on( 'job complete', function ( id ) { 52 | Job.get( id, function ( err, job ) { 53 | if ( err ) return; 54 | job.remove( function ( err ) { 55 | if ( err ) throw err; 56 | console.log( 'removed completed job #%d', job.id ); 57 | } ); 58 | } ); 59 | } ); 60 | 61 | // start the UI 62 | var app = express.createServer(); 63 | app.use( kue.app ); 64 | app.listen( 3000 ); 65 | console.log( 'UI started on port 3000' ); -------------------------------------------------------------------------------- /lib/http/public/javascripts/utils.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * kue - utils 3 | * Copyright (c) 2010 LearnBoost 4 | * MIT Licensed 5 | */ 6 | 7 | /** 8 | * Format `ms` in words. 9 | * 10 | * @param {Number} ms 11 | * @return {String} 12 | */ 13 | 14 | function relative(ms) { 15 | var sec = 1000 16 | , min = 60 * sec 17 | , hour = 60 * min; 18 | 19 | function n(n, name) { 20 | n = Math.round(n); 21 | return n + ' ' + name + (n > 1 ? 's' : ''); 22 | } 23 | 24 | if (isNaN(ms)) return ''; 25 | if (ms < sec) return 'less than one second'; 26 | if (ms < min) return n(ms / sec, 'second'); 27 | if (ms < hour) return n(ms / min, 'minute'); 28 | return n(ms / hour, 'hour'); 29 | // TODO: larger than an hour or so, we should 30 | // have some nice date formatting 31 | } 32 | 33 | /** 34 | * Default job states. 35 | */ 36 | 37 | var states = { 38 | active: 'active', inactive: 'inactive', failed: 'failed', complete: 'complete', delayed: 'delayed' 39 | }; 40 | 41 | /** 42 | * Default job priority map. 43 | */ 44 | 45 | var priorities = { 46 | '10': 'low', '0': 'normal', '-5': 'medium', '-10': 'high', '-15': 'critical' 47 | }; 48 | 49 | /** 50 | * Return priority string for `job`. 51 | * 52 | * @param {Job} job 53 | * @return {String} 54 | */ 55 | 56 | function priority(job) { 57 | return priorities[job.priority] || job.priority; 58 | } 59 | 60 | /** 61 | * Generate options from `obj`. 62 | * 63 | * @param {Object} obj 64 | * @param {String} selected 65 | * @return {String} 66 | */ 67 | 68 | function options(obj, selected) { 69 | var html = ''; 70 | for (var key in obj) { 71 | html += '\n'; 74 | } 75 | return html; 76 | } 77 | -------------------------------------------------------------------------------- /lib/http/index.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * q - http 3 | * Copyright (c) 2011 LearnBoost 4 | * MIT Licensed 5 | */ 6 | 7 | /** 8 | * Module dependencies. 9 | */ 10 | 11 | var express = require('express'); 12 | 13 | // setup 14 | 15 | var app = express() 16 | , bodyParser = require('body-parser') 17 | , provides = require('./middleware/provides') 18 | , stylus = require('stylus') 19 | , routes = require('./routes') 20 | , pug = require('pug') 21 | , json = require('./routes/json') 22 | , util = require('util') 23 | , nib = require('nib'); 24 | 25 | // expose the app 26 | 27 | module.exports = app; 28 | 29 | // stylus config 30 | 31 | function compile( str, path ) { 32 | return stylus(str) 33 | .set('filename', path) 34 | .use(nib()); 35 | } 36 | 37 | // config 38 | 39 | app.set('view options', { doctype: 'html' }); 40 | app.set('view engine', 'pug'); 41 | app.engine('pug', pug.renderFile); 42 | app.set('views', __dirname + '/views'); 43 | app.set('title', 'Kue'); 44 | app.locals = { inspect: util.inspect }; 45 | 46 | // middlewares 47 | 48 | app.use(stylus.middleware({ src: __dirname + '/public', compile: compile })); 49 | app.use(express.static(__dirname + '/public')); 50 | 51 | // JSON api 52 | 53 | app.get('/stats', provides('json'), json.stats); 54 | app.get('/job/search', provides('json'), json.search); 55 | app.get('/jobs/:from..:to/:order?', provides('json'), json.jobRange); 56 | app.get('/jobs/:type/:state/:from..:to/:order?', provides('json'), json.jobTypeRange); 57 | app.get('/jobs/:type/:state/stats', provides('json'), json.jobTypeStateStats); 58 | app.get('/jobs/:state/:from..:to/:order?', provides('json'), json.jobStateRange); 59 | app.get('/job/types', provides('json'), json.types); 60 | app.get('/job/:id', provides('json'), json.job); 61 | app.get('/job/:id/log', provides('json'), json.log); 62 | app.put('/job/:id/state/:state', provides('json'), json.updateState); 63 | app.put('/job/:id/priority/:priority', provides('json'), json.updatePriority); 64 | app.delete('/job/:id', provides('json'), json.remove); 65 | app.post('/job', provides('json'), bodyParser.json(), json.createJob); 66 | app.get('/inactive/:id', provides('json'), json.inactive); 67 | 68 | // routes 69 | 70 | app.get('/', routes.jobs('active')); 71 | 72 | app.get('/active', routes.jobs('active')); 73 | app.get('/inactive', routes.jobs('inactive')); 74 | app.get('/failed', routes.jobs('failed')); 75 | app.get('/complete', routes.jobs('complete')); 76 | app.get('/delayed', routes.jobs('delayed')); 77 | -------------------------------------------------------------------------------- /lib/http/public/javascripts/loading.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * kue - LoadingIndicator 3 | * Copyright (c) 2011 LearnBoost 4 | * MIT Licensed 5 | */ 6 | 7 | /** 8 | * Initialize a new `LoadingIndicator`. 9 | */ 10 | 11 | function LoadingIndicator() { 12 | this.size(0); 13 | this.fontSize(9); 14 | this.font('helvetica, arial, sans-serif'); 15 | } 16 | 17 | /** 18 | * Set size to `n`. 19 | * 20 | * @param {Number} n 21 | * @return {LoadingIndicator} for chaining 22 | * @api public 23 | */ 24 | 25 | LoadingIndicator.prototype.size = function (n) { 26 | this._size = n; 27 | return this; 28 | }; 29 | 30 | /** 31 | * Set font size to `n`. 32 | * 33 | * @param {Number} n 34 | * @return {LoadingIndicator} for chaining 35 | * @api public 36 | */ 37 | 38 | LoadingIndicator.prototype.fontSize = function (n) { 39 | this._fontSize = n; 40 | return this; 41 | }; 42 | 43 | /** 44 | * Set font `family`. 45 | * 46 | * @param {String} family 47 | * @return {LoadingIndicator} for chaining 48 | */ 49 | 50 | LoadingIndicator.prototype.font = function (family) { 51 | this._font = family; 52 | return this; 53 | }; 54 | 55 | /** 56 | * Update pos to `n`. 57 | * 58 | * @param {Number} n 59 | * @return {LoadingIndicator} for chaining 60 | */ 61 | 62 | LoadingIndicator.prototype.update = function (n) { 63 | this.pos = n; 64 | return this; 65 | }; 66 | 67 | /** 68 | * Draw on `ctx`. 69 | * 70 | * @param {CanvasRenderingContext2d} ctx 71 | * @return {LoadingIndicator} for chaining 72 | */ 73 | 74 | LoadingIndicator.prototype.draw = function (ctx) { 75 | var pos = this.pos % 360 76 | , size = this._size 77 | , half = size / 2 78 | , x = half 79 | , y = half 80 | , rad = half - 1 81 | , fontSize = this._fontSize; 82 | 83 | ctx.font = fontSize + 'px ' + this._font; 84 | 85 | ctx.clearRect(0, 0, size, size); 86 | 87 | // outer circle 88 | ctx.strokeStyle = '#9f9f9f'; 89 | ctx.beginPath(); 90 | ctx.arc(x, y, rad, pos, Math.PI / 2 + pos, false); 91 | ctx.stroke(); 92 | 93 | // inner circle 94 | ctx.strokeStyle = '#eee'; 95 | ctx.beginPath(); 96 | ctx.arc(x, y, rad - 3, -pos, Math.PI / 2 - pos, false); 97 | ctx.stroke(); 98 | 99 | // text 100 | var text = 'Loading' 101 | , w = ctx.measureText(text).width; 102 | 103 | ctx.fillText( 104 | text 105 | , x - w / 2 + 1 106 | , y + fontSize / 2 - 1); 107 | 108 | return this; 109 | }; 110 | -------------------------------------------------------------------------------- /test/test_mode.js: -------------------------------------------------------------------------------- 1 | var kue = require('../'), 2 | _ = require('lodash'), 3 | queue = kue.createQueue(); 4 | 5 | describe('Test Mode', function() { 6 | context('when enabled', function() { 7 | before(function() { 8 | queue.testMode.enter(); 9 | }); 10 | 11 | afterEach(function() { 12 | queue.testMode.clear(); 13 | }); 14 | 15 | it('adds jobs to an array in memory', function() { 16 | queue.createJob('myJob', { foo: 'bar' }).save(); 17 | 18 | var jobs = queue.testMode.jobs; 19 | expect(jobs.length).to.equal(1); 20 | 21 | var job = _.last(jobs); 22 | expect(job.type).to.equal('myJob'); 23 | expect(job.data).to.eql({ foo: 'bar' }); 24 | }); 25 | 26 | it('adds jobs to an array in memory and processes them when processQueue is true', function(done) { 27 | queue.testMode.exit(); 28 | queue.testMode.enter(true); 29 | 30 | queue.createJob('test-testMode-process', { foo: 'bar' }).save(); 31 | 32 | var jobs = queue.testMode.jobs; 33 | expect(jobs.length).to.equal(1); 34 | 35 | var job = _.last(jobs); 36 | expect(job.type).to.equal('test-testMode-process'); 37 | expect(job.data).to.eql({ foo: 'bar' }); 38 | 39 | job.on('complete', function() { 40 | queue.testMode.exit(); 41 | queue.testMode.enter(); 42 | done(); 43 | }); 44 | 45 | queue.process('test-testMode-process', function(job, jdone) { 46 | job.data.should.be.eql({ foo: 'bar' }); 47 | 48 | jdone(); 49 | }); 50 | }); 51 | 52 | describe('#clear', function() { 53 | it('resets the list of jobs', function() { 54 | queue.createJob('myJob', { foo: 'bar' }).save(); 55 | queue.testMode.clear(); 56 | 57 | var jobs = queue.testMode.jobs; 58 | expect(jobs.length).to.equal(0); 59 | }); 60 | }); 61 | }); 62 | 63 | context('when disabled', function() { 64 | before(function() { 65 | // Simulate entering and exiting test mode to ensure 66 | // state is restored correctly. 67 | queue.testMode.enter(); 68 | queue.testMode.exit(); 69 | }); 70 | 71 | it('processes jobs regularly', function(done) { 72 | queue.createJob('myJob', { foo: 'bar' }).save(); 73 | 74 | var jobs = queue.testMode.jobs; 75 | expect(jobs.length).to.equal(0); 76 | 77 | queue.process('myJob', function (job, jdone) { 78 | expect(job.data).to.eql({ foo: 'bar' }); 79 | jdone(); 80 | done(); 81 | }); 82 | }); 83 | }); 84 | }); 85 | -------------------------------------------------------------------------------- /lib/http/public/javascripts/progress.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * kue - Progress 3 | * Copyright (c) 2011 LearnBoost 4 | * MIT Licensed 5 | */ 6 | 7 | /** 8 | * Initialize a new `Progress` indicator. 9 | */ 10 | 11 | function Progress() { 12 | this.percent = 0; 13 | this.size(0); 14 | this.fontSize(12); 15 | this.font('helvetica, arial, sans-serif'); 16 | } 17 | 18 | /** 19 | * Set progress size to `n`. 20 | * 21 | * @param {Number} n 22 | * @return {Progress} for chaining 23 | * @api public 24 | */ 25 | 26 | Progress.prototype.size = function (n) { 27 | this._size = n; 28 | return this; 29 | }; 30 | 31 | /** 32 | * Set text to `str`. 33 | * 34 | * @param {String} str 35 | * @return {Progress} for chaining 36 | * @api public 37 | */ 38 | 39 | Progress.prototype.text = function (str) { 40 | this._text = str; 41 | return this; 42 | }; 43 | 44 | /** 45 | * Set font size to `n`. 46 | * 47 | * @param {Number} n 48 | * @return {Progress} for chaining 49 | * @api public 50 | */ 51 | 52 | Progress.prototype.fontSize = function (n) { 53 | this._fontSize = n; 54 | return this; 55 | }; 56 | 57 | /** 58 | * Set font `family`. 59 | * 60 | * @param {String} family 61 | * @return {Progress} for chaining 62 | */ 63 | 64 | Progress.prototype.font = function (family) { 65 | this._font = family; 66 | return this; 67 | }; 68 | 69 | /** 70 | * Update percentage to `n`. 71 | * 72 | * @param {Number} n 73 | * @return {Progress} for chaining 74 | */ 75 | 76 | Progress.prototype.update = function (n) { 77 | this.percent = n; 78 | return this; 79 | }; 80 | 81 | /** 82 | * Draw on `ctx`. 83 | * 84 | * @param {CanvasRenderingContext2d} ctx 85 | * @return {Progress} for chaining 86 | */ 87 | 88 | Progress.prototype.draw = function (ctx) { 89 | var percent = Math.min(this.percent, 100) 90 | , size = this._size 91 | , half = size / 2 92 | , x = half 93 | , y = half 94 | , rad = half - 1 95 | , fontSize = this._fontSize; 96 | 97 | ctx.font = fontSize + 'px ' + this._font; 98 | 99 | var angle = Math.PI * 2 * (percent / 100); 100 | ctx.clearRect(0, 0, size, size); 101 | 102 | // outer circle 103 | ctx.strokeStyle = '#9f9f9f'; 104 | ctx.beginPath(); 105 | ctx.arc(x, y, rad, 0, angle, false); 106 | ctx.stroke(); 107 | 108 | // inner circle 109 | ctx.strokeStyle = '#eee'; 110 | ctx.beginPath(); 111 | ctx.arc(x, y, rad - 1, 0, angle, true); 112 | ctx.stroke(); 113 | 114 | // text 115 | var text = this._text || (percent | 0) + '%' 116 | , w = ctx.measureText(text).width; 117 | 118 | ctx.fillText( 119 | text 120 | , x - w / 2 + 1 121 | , y + fontSize / 2 - 1); 122 | 123 | return this; 124 | }; 125 | -------------------------------------------------------------------------------- /test/prefix.coffee: -------------------------------------------------------------------------------- 1 | kue = require '../' 2 | 3 | describe 'Kue - Prefix', -> 4 | 5 | makeJobs = (queueName) -> 6 | opts = 7 | prefix: queueName 8 | promotion: 9 | interval: 10 10 | jobs = kue.createQueue opts 11 | return jobs 12 | 13 | stopJobs = (jobs, callback) -> 14 | jobs.shutdown callback 15 | 16 | # expected redis activity 17 | # 18 | # 1397744169.196792 "subscribe" "q:events" 19 | # 1397744169.196852 "unsubscribe" 20 | it 'should use prefix q by default', (done) -> 21 | jobs = kue.createQueue() 22 | jobs.client.prefix.should.equal 'q' 23 | stopJobs jobs, done 24 | 25 | # expected redis activity 26 | # 27 | # 1397744498.330456 "subscribe" "testPrefix1:events" 28 | # 1397744498.330638 "unsubscribe" 29 | # 1397744498.330907 "subscribe" "testPrefix2:events" 30 | # 1397744498.331148 "unsubscribe" 31 | it 'should accept and store prefix', (done) -> 32 | 33 | jobs = makeJobs('testPrefix1') 34 | 35 | jobs.client.prefix.should.equal 'testPrefix1' 36 | 37 | stopJobs jobs, (err) -> 38 | jobs2 = makeJobs('testPrefix2') 39 | jobs2.client.prefix.should.equal 'testPrefix2' 40 | stopJobs jobs2, done 41 | 42 | it 'should process and complete a job using a prefix', (testDone) -> 43 | 44 | jobs = makeJobs('simplePrefixTest') 45 | 46 | job = jobs.create('simplePrefixJob') 47 | job.on 'complete', () -> 48 | stopJobs jobs, testDone 49 | job.save() 50 | jobs.process 'simplePrefixJob', (job, done) -> 51 | done() 52 | 53 | # expected redis activity 54 | # 55 | # 1397744498.333423 "subscribe" "jobCompleteTest:events" 56 | # 1397744498.334002 "info" 57 | # 1397744498.334358 "zcard" "jobCompleteTest:jobs:inactive" 58 | # 1397744498.335262 "info" 59 | # 1397744498.335578 "incr" "jobCompleteTest:ids" 60 | # etc... 61 | it 'store queued jobs in different prefixes', (testDone) -> 62 | jobs = makeJobs('jobCompleteTest') 63 | 64 | jobs.inactiveCount (err, count) -> 65 | prevCount = count 66 | 67 | jobs.create( 'fakeJob', {} ).save() 68 | f = -> 69 | jobs.inactiveCount (err, count) -> 70 | count.should.equal prevCount + 1 71 | stopJobs jobs, testDone 72 | setTimeout f, 10 73 | 74 | it 'should not pick up an inactive job from another prefix', (testDone) -> 75 | jobs = makeJobs('inactiveJobs') 76 | # create a job but do not process 77 | job = jobs.create('inactiveJob', {} ).save (err) -> 78 | # stop the 'inactiveJobs' prefix 79 | stopJobs jobs, (err) -> 80 | jobs = makeJobs('inactiveJobs2') 81 | 82 | # verify count of inactive jobs is 0 for this prefix 83 | jobs.inactiveCount (err, count) -> 84 | count.should.equal 0 85 | 86 | stopJobs jobs, testDone 87 | 88 | 89 | it 'should properly switch back to default queue', (testDone) -> 90 | jobs = makeJobs('notDefault') 91 | stopJobs jobs, (err) -> 92 | jobs = kue.createQueue() 93 | 94 | job = jobs.create('defaultPrefixJob') 95 | job.on 'complete', () -> 96 | stopJobs jobs, testDone 97 | job.save() 98 | 99 | jobs.process 'defaultPrefixJob', (job, done) -> 100 | done() 101 | 102 | -------------------------------------------------------------------------------- /lib/queue/events.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * kue - events 3 | * Copyright (c) 2013 Automattic 4 | * Copyright (c) 2011 LearnBoost 5 | * MIT Licensed 6 | */ 7 | 8 | /** 9 | * Module dependencies. 10 | */ 11 | 12 | var redis = require('../redis'); 13 | 14 | /** 15 | * Job map. 16 | */ 17 | 18 | exports.jobs = {}; 19 | 20 | /** 21 | * Pub/sub key. 22 | */ 23 | 24 | exports.key = 'events'; 25 | 26 | /** 27 | * Add `job` to the jobs map, used 28 | * to grab the in-process object 29 | * so we can emit relative events. 30 | * 31 | * @param {Job} job 32 | * @api private 33 | */ 34 | exports.callbackQueue = []; 35 | 36 | exports.add = function( job, callback ) { 37 | if( job.id ) { 38 | if(!exports.jobs[ job.id ]) 39 | exports.jobs[ job.id ] = []; 40 | 41 | exports.jobs[ job.id ].push(job); 42 | } 43 | // if (!exports.subscribed) exports.subscribe(); 44 | if( !exports.subscribeStarted ) exports.subscribe(); 45 | if( !exports.subscribed ) { 46 | exports.callbackQueue.push(callback); 47 | } else { 48 | callback(); 49 | } 50 | }; 51 | 52 | /** 53 | * Remove `job` from the jobs map. 54 | * 55 | * @param {Job} job 56 | * @api private 57 | */ 58 | 59 | exports.remove = function( job ) { 60 | delete exports.jobs[ job.id ]; 61 | }; 62 | 63 | /** 64 | * Subscribe to "q:events". 65 | * 66 | * @api private 67 | */ 68 | 69 | exports.subscribe = function() { 70 | // if (exports.subscribed) return; 71 | if( exports.subscribeStarted ) return; 72 | var client = redis.pubsubClient(); 73 | client.on('message', exports.onMessage); 74 | client.subscribe(client.getKey(exports.key), function() { 75 | exports.subscribed = true; 76 | while( exports.callbackQueue.length ) { 77 | process.nextTick(exports.callbackQueue.shift()); 78 | } 79 | }); 80 | exports.queue = require('../kue').singleton; 81 | // exports.subscribed = true; 82 | exports.subscribeStarted = true; 83 | }; 84 | 85 | exports.unsubscribe = function() { 86 | var client = redis.pubsubClient(); 87 | client.unsubscribe(); 88 | client.removeAllListeners(); 89 | exports.subscribeStarted = false; 90 | }; 91 | 92 | /** 93 | * Message handler. 94 | * 95 | * @api private 96 | */ 97 | 98 | exports.onMessage = function( channel, msg ) { 99 | // TODO: only subscribe on {Queue,Job}#on() 100 | msg = JSON.parse(msg); 101 | 102 | // map to Job when in-process 103 | var jobs = exports.jobs[ msg.id ]; 104 | if( jobs && jobs.length > 0 ) { 105 | for (var i = 0; i < jobs.length; i++) { 106 | var job = jobs[i]; 107 | job.emit.apply(job, msg.args); 108 | if( [ 'complete', 'failed' ].indexOf(msg.event) !== -1 ) exports.remove(job); 109 | } 110 | } 111 | // emit args on Queues 112 | msg.args[ 0 ] = 'job ' + msg.args[ 0 ]; 113 | msg.args.splice(1, 0, msg.id); 114 | if( exports.queue ) { 115 | exports.queue.emit.apply(exports.queue, msg.args); 116 | } 117 | }; 118 | 119 | /** 120 | * Emit `event` for for job `id` with variable args. 121 | * 122 | * @param {Number} id 123 | * @param {String} event 124 | * @param {Mixed} ... 125 | * @api private 126 | */ 127 | 128 | exports.emit = function( id, event ) { 129 | var client = redis.client() 130 | , msg = JSON.stringify({ 131 | id: id, event: event, args: [].slice.call(arguments, 1) 132 | }); 133 | client.publish(client.getKey(exports.key), msg, function () {}); 134 | }; 135 | -------------------------------------------------------------------------------- /lib/http/public/stylesheets/job.styl: -------------------------------------------------------------------------------- 1 | @import 'mixins' 2 | 3 | #job-template 4 | display: none 5 | 6 | bar(color) 7 | background: linear-gradient(top, color + 20%, color) 8 | border: 1px solid rgba(white, .2) 9 | color: white 10 | 11 | // generic blocks 12 | 13 | .block 14 | decorated-box() 15 | width: 90% 16 | margin: 10px 25px 17 | padding: 20px 25px 18 | h2 19 | margin: 0 20 | absolute: top 5px left -15px 21 | padding: 5px 22 | font-size: 10px 23 | border-radius: left 5px right 2px 24 | background: linear-gradient(left, menu-fg - 10%, 50% menu-fg + 5%) 25 | box-shadow: -1px 0 1px 1px rgba(black, .1) 26 | color: white 27 | text-shadow: 1px 1px 1px #444 28 | .type 29 | color: lighter + 20% 30 | 31 | // job delay 32 | .job td.title em 33 | color: lighter + 20% 34 | 35 | // job blocks 36 | 37 | .job .block 38 | position: relative 39 | background: job-bg 40 | cursor: pointer 41 | table td:first-child 42 | display: none 43 | .progress 44 | absolute: top 15px right 20px 45 | .attempts 46 | display: none 47 | absolute: top right 48 | padding: 5px 8px 49 | border-radius: 2px 50 | font-size: 10px 51 | .remove 52 | absolute: top 30px right -6px 53 | /*background: white*/ 54 | background: #F05151 55 | color: white 56 | display: block 57 | width: size = 20px 58 | height: size 59 | line-height: size 60 | text-align: center 61 | font-size: 12px 62 | font-weight: bold 63 | outline: none 64 | border: 1px solid #eee 65 | border-radius: size 66 | transition: opacity 200ms, top 300ms 67 | opacity: 0 68 | &:hover 69 | border: 1px solid #eee - 10% 70 | &:active 71 | border: 1px solid #eee - 20% 72 | .restart 73 | absolute: top 30px right -6px 74 | /*background: white*/ 75 | background: #00e600 76 | color: white 77 | display: block 78 | width: size = 20px 79 | height: size 80 | line-height: size 81 | text-align: center 82 | font-size: 12px 83 | font-weight: bold 84 | outline: none 85 | border: 1px solid #eee 86 | border-radius: size 87 | transition: opacity 200ms, top 300ms 88 | opacity: 0 89 | &:hover 90 | border: 1px solid #eee - 10% 91 | &:active 92 | border: 1px solid #eee - 20% 93 | &:hover 94 | .remove 95 | opacity: 1 96 | top: -6px 97 | .restart 98 | opacity: 1 99 | top: 16px 100 | 101 | // details 102 | 103 | .job .details 104 | background: dark 105 | width: 89% 106 | margin-top: -10px 107 | margin-left: 35px 108 | border-radius: bottom 5px 109 | box-shadow: inset 0 1px 10px 0 rgba(black, .8) 110 | transition: padding 200ms, height 200ms 111 | height: 0 112 | overflow: hidden 113 | table 114 | width: 100% 115 | td:first-child 116 | width: 60px 117 | color: light + 30% 118 | &.show 119 | padding: 15px 20px 120 | height: auto 121 | 122 | // job log 123 | 124 | .job ul.log 125 | reset-list() 126 | margin: 5px 127 | padding: 10px 128 | max-height: 100px 129 | overflow-y: auto 130 | border-radius: 5px 131 | width: 95% 132 | li 133 | padding: 5px 0 134 | border-bottom: 1px dotted light - 35% 135 | color: light 136 | &:last-child 137 | border-bottom: none 138 | 139 | // scrollbar 140 | 141 | .job .details 142 | ::-webkit-scrollbar 143 | width: 2px 144 | ::-webkit-scrollbar-thumb:vertical 145 | background: light + 20% 146 | ::-webkit-scrollbar-track 147 | border: 1px solid rgba(white, .1) 148 | 149 | // sections 150 | 151 | .job .details > div 152 | padding: 10px 0 153 | border-bottom: 1px solid light - 35% 154 | &:last-child 155 | border-bottom: none 156 | -------------------------------------------------------------------------------- /lib/redis.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * kue - RedisClient factory 3 | * Copyright (c) 2013 Automattic 4 | * Copyright (c) 2011 LearnBoost 5 | * MIT Licensed 6 | * Author: behradz@gmail.com 7 | */ 8 | 9 | /** 10 | * Module dependencies. 11 | */ 12 | 13 | var redis = require('redis'); 14 | var url = require('url'); 15 | 16 | /** 17 | * 18 | * @param options 19 | * @param queue 20 | */ 21 | 22 | exports.configureFactory = function( options, queue ) { 23 | options.prefix = options.prefix || 'q'; 24 | 25 | if( typeof options.redis === 'string' ) { 26 | // parse the url 27 | var conn_info = url.parse(options.redis, true /* parse query string */); 28 | if( conn_info.protocol !== 'redis:' ) { 29 | throw new Error('kue connection string must use the redis: protocol'); 30 | } 31 | 32 | options.redis = { 33 | port: conn_info.port || 6379, 34 | host: conn_info.hostname, 35 | db: (conn_info.pathname ? conn_info.pathname.substr(1) : null) || conn_info.query.db || 0, 36 | // see https://github.com/mranney/node_redis#rediscreateclient 37 | options: conn_info.query 38 | }; 39 | 40 | if( conn_info.auth ) { 41 | options.redis.auth = conn_info.auth.replace(/.*?:/, ''); 42 | } 43 | 44 | } 45 | 46 | options.redis = options.redis || {}; 47 | 48 | // guarantee that redis._client has not been populated. 49 | // may warrant some more testing - i was running into cases where shutdown 50 | // would call redis.reset but an event would be emitted after the reset 51 | // which would re-create the client and cache it in the redis module. 52 | exports.reset(); 53 | 54 | /** 55 | * Create a RedisClient. 56 | * 57 | * @return {RedisClient} 58 | * @api private 59 | */ 60 | exports.createClient = function() { 61 | var clientFactoryMethod = options.redis.createClientFactory || exports.createClientFactory; 62 | var client = clientFactoryMethod(options); 63 | 64 | client.on('error', function( err ) { 65 | queue.emit('error', err); 66 | }); 67 | 68 | client.prefix = options.prefix; 69 | 70 | // redefine getKey to use the configured prefix 71 | client.getKey = function( key ) { 72 | if( client.constructor.name == 'Redis' || client.constructor.name == 'Cluster') { 73 | // {prefix}:jobs format is needed in using ioredis cluster to keep they keys in same node 74 | // otherwise multi commands fail, since they use ioredis's pipeline. 75 | return '{' + this.prefix + '}:' + key; 76 | } 77 | return this.prefix + ':' + key; 78 | }; 79 | 80 | client.createFIFO = function( id ) { 81 | //Create an id for the zset to preserve FIFO order 82 | var idLen = '' + id.toString().length; 83 | var len = 2 - idLen.length; 84 | while (len--) idLen = '0' + idLen; 85 | return idLen + '|' + id; 86 | }; 87 | 88 | // Parse out original ID from zid 89 | client.stripFIFO = function( zid ) { 90 | if ( typeof zid === 'string' ) { 91 | return +zid.substr(zid.indexOf('|')+1); 92 | } else { 93 | // Sometimes this gets called with an undefined 94 | // it seems to be OK to have that not resolve to an id 95 | return zid; 96 | } 97 | }; 98 | 99 | return client; 100 | }; 101 | }; 102 | 103 | /** 104 | * Create a RedisClient from options 105 | * @param options 106 | * @return {RedisClient} 107 | * @api private 108 | */ 109 | 110 | exports.createClientFactory = function( options ) { 111 | var socket = options.redis.socket; 112 | var port = !socket ? (options.redis.port || 6379) : null; 113 | var host = !socket ? (options.redis.host || '127.0.0.1') : null; 114 | var db = !socket ? (options.redis.db || 0) : null; 115 | var client = redis.createClient(socket || port, host, options.redis.options); 116 | if( options.redis.auth ) { 117 | client.auth(options.redis.auth); 118 | } 119 | if( db >= 0 ){ 120 | client.select(db); 121 | } 122 | return client; 123 | }; 124 | 125 | /** 126 | * Create or return the existing RedisClient. 127 | * 128 | * @return {RedisClient} 129 | * @api private 130 | */ 131 | 132 | exports.client = function() { 133 | return exports._client || (exports._client = exports.createClient()); 134 | }; 135 | 136 | /** 137 | * Return the pubsub-specific redis client. 138 | * 139 | * @return {RedisClient} 140 | * @api private 141 | */ 142 | 143 | exports.pubsubClient = function() { 144 | return exports._pubsub || (exports._pubsub = exports.createClient()); 145 | }; 146 | 147 | /** 148 | * Resets internal variables to initial state 149 | * 150 | * @api private 151 | */ 152 | exports.reset = function() { 153 | exports._client && exports._client.quit(); 154 | exports._pubsub && exports._pubsub.quit(); 155 | exports._client = null; 156 | exports._pubsub = null; 157 | }; 158 | -------------------------------------------------------------------------------- /test/shutdown.coffee: -------------------------------------------------------------------------------- 1 | should = require 'should' 2 | 3 | kue = require '../' 4 | 5 | 6 | describe 'Kue', -> 7 | 8 | before (done) -> 9 | jobs = kue.createQueue() 10 | jobs.client.flushdb done 11 | 12 | after (done) -> 13 | jobs = kue.createQueue() 14 | jobs.client.flushdb done 15 | 16 | describe 'Shutdown', -> 17 | it 'should return singleton from createQueue', (done) -> 18 | jobs = kue.createQueue() 19 | jobsToo = kue.createQueue() 20 | jobs.should.equal jobsToo 21 | jobs.shutdown done 22 | 23 | 24 | 25 | it 'should destroy singleton on shutdown', (done) -> 26 | jobs = kue.createQueue() 27 | jobs.shutdown (err) -> 28 | # test that new jobs object is a different reference 29 | newJobs = kue.createQueue() 30 | newJobs.should.not.equal jobs 31 | newJobs.shutdown done 32 | 33 | 34 | 35 | it 'should clear properties on shutdown', (done) -> 36 | jobs = kue.createQueue({promotion:{interval:200}}) 37 | jobs.shutdown (err) -> 38 | should(jobs.workers).be.empty 39 | should(jobs.client).be.empty 40 | should(jobs.promoter).be.empty 41 | done() 42 | 43 | 44 | 45 | it 'should be able to pause/resume the worker', (done) -> 46 | jobs = kue.createQueue() 47 | job_data = 48 | title: 'resumable jobs' 49 | to: 'tj@learnboost.com' 50 | total_jobs = 3 51 | for i in [0...total_jobs] 52 | jobs.create('resumable-jobs', job_data).save() 53 | 54 | jobs.process 'resumable-jobs', 1, (job, ctx, job_done) -> 55 | job_done() 56 | if( !--total_jobs ) 57 | jobs.shutdown 1000, done 58 | else 59 | ctx.pause() 60 | setTimeout ctx.resume, 100 61 | 62 | 63 | 64 | it 'should not clear properties on single type shutdown', (testDone) -> 65 | jobs = kue.createQueue() 66 | fn = (err) -> 67 | jobs.client.should.not.be.empty 68 | jobs.shutdown 10, testDone 69 | 70 | jobs.shutdown 10, 'fooJob', fn 71 | 72 | 73 | 74 | it 'should shutdown one worker type on single type shutdown', (testDone) -> 75 | jobs = kue.createQueue() 76 | # set up two worker types 77 | jobs.process 'runningTask', (job, done) -> 78 | done() 79 | jobs.workers.should.have.length 1 80 | jobs.process 'shutdownTask', (job, done) -> 81 | done() 82 | jobs.workers.should.have.length 2 83 | fn = (err) -> 84 | # verify shutdownTask is not running but runningTask is 85 | for worker in jobs.workers 86 | switch worker.type 87 | when 'shutdownTask' 88 | worker.should.have.property 'running', false 89 | when 'runningTask' 90 | worker.should.have.property 'running', true 91 | 92 | # kue should still be running 93 | jobs.promoter.should.not.be.empty 94 | jobs.client.should.not.be.empty 95 | 96 | jobs.shutdown 10, testDone 97 | jobs.shutdown 10, 'shutdownTask', fn 98 | 99 | 100 | it 'should fail active job when shutdown timer expires', (testDone) -> 101 | jobs = kue.createQueue() 102 | jobId = null 103 | jobs.process 'long-task', (job, done) -> 104 | jobId = job.id 105 | fn = -> 106 | done() 107 | setTimeout fn, 10000 108 | 109 | jobs.create('long-task', {}).save() 110 | # need to make sure long-task has had enough time to get into active state 111 | waitForJobToRun = -> 112 | fn = (err) -> 113 | kue.Job.get jobId, (err, job) -> 114 | job.should.have.property '_state', "failed" 115 | job.should.have.property '_error', "Shutdown" 116 | testDone() 117 | 118 | # shutdown timer is shorter than job length 119 | jobs.shutdown 10, fn 120 | 121 | setTimeout waitForJobToRun, 50 122 | 123 | 124 | 125 | it 'should not call graceful shutdown twice on subsequent calls', (testDone) -> 126 | jobs = kue.createQueue() 127 | jobs.process 'test-subsequent-shutdowns', (job, done) -> 128 | done() 129 | setTimeout ()-> 130 | jobs.shutdown 100, (err)-> 131 | should.not.exist(err) 132 | , 50 133 | 134 | setTimeout ()-> 135 | jobs.shutdown 100, (err)-> 136 | should.exist err, 'expected `err` to exist' 137 | err.should.be.an.instanceOf(Error) 138 | .with.property('message', 'Shutdown already in progress') 139 | testDone() 140 | , 60 141 | 142 | jobs.create('test-subsequent-shutdowns', {}).save() 143 | 144 | 145 | 146 | it 'should fail active re-attemptable job when shutdown timer expires', (testDone) -> 147 | jobs = kue.createQueue() 148 | jobId = null 149 | jobs.process 'shutdown-reattemptable-jobs', (job, done) -> 150 | jobId = job.id 151 | setTimeout done, 500 152 | 153 | jobs.create('shutdown-reattemptable-jobs', { title: 'shutdown-reattemptable-jobs' }).attempts(2).save() 154 | 155 | # need to make sure long-task has had enough time to get into active state 156 | waitForJobToRun = -> 157 | fn = (err) -> 158 | kue.Job.get jobId, (err, job) -> 159 | job.should.have.property '_state', "inactive" 160 | job.should.have.property '_attempts', "1" 161 | job.should.have.property '_error', "Shutdown" 162 | testDone() 163 | 164 | # shutdown timer is shorter than job length 165 | jobs.shutdown 100, fn 166 | 167 | setTimeout waitForJobToRun, 50 168 | -------------------------------------------------------------------------------- /test/tdd/redis.spec.js: -------------------------------------------------------------------------------- 1 | var sinon = require('sinon'); 2 | var r = require('redis'); 3 | var redis = require('../../lib/redis'); 4 | 5 | describe('redis', function() { 6 | 7 | describe('Function: configureFactory', function() { 8 | 9 | beforeEach(function(){ 10 | sinon.stub(redis, 'reset'); 11 | }); 12 | 13 | afterEach(function(){ 14 | redis.reset.restore(); 15 | }); 16 | 17 | it('should parse a url connection string', function () { 18 | var options = { 19 | redis: 'redis://:password@host:1234/db' 20 | }; 21 | redis.configureFactory(options); 22 | options.redis.port.should.equal('1234'); 23 | options.redis.host.should.equal('host'); 24 | options.redis.db.should.equal('db'); 25 | }); 26 | 27 | it('should reset everything', function () { 28 | var options = { 29 | redis: 'redis://:password@host:1234/db' 30 | }; 31 | redis.configureFactory(options); 32 | redis.reset.called.should.be.true; 33 | }); 34 | 35 | it('should export the createClient function', function () { 36 | var options = { 37 | redis: 'redis://:password@host:1234/db' 38 | }; 39 | redis.createClient = null; 40 | redis.configureFactory(options); 41 | (typeof redis.createClient == 'function').should.be.true; 42 | }); 43 | 44 | }); 45 | 46 | describe('Function: createClient', function() { 47 | var options; 48 | beforeEach(function(){ 49 | options = { 50 | prefix: 'prefix', 51 | redis: 'redis://:password@host:1234/db' 52 | }; 53 | redis.configureFactory(options); 54 | sinon.stub(redis, 'createClientFactory').returns({ 55 | on: sinon.stub() 56 | }); 57 | }); 58 | 59 | afterEach(function(){ 60 | redis.createClientFactory.restore(); 61 | }); 62 | 63 | it('should create a client object', function () { 64 | var client = redis.createClient(); 65 | client.prefix.should.equal(options.prefix); 66 | ('function' === typeof client.getKey).should.be.true; 67 | ('function' === typeof client.createFIFO).should.be.true; 68 | ('function' === typeof client.stripFIFO).should.be.true; 69 | }); 70 | 71 | describe('Function: client.getKey', function() { 72 | 73 | it('should return the key with the prefix', function () { 74 | var client = redis.createClient(); 75 | var key = client.getKey('key'); 76 | key.should.equal('prefix:key'); 77 | }); 78 | 79 | it('should return key with prefix and curly braces for ioredis cluster', function () { 80 | var client = redis.createClient(); 81 | client.constructor = { 82 | name: 'Redis' 83 | }; 84 | var key = client.getKey('key'); 85 | key.should.equal('{prefix}:key'); 86 | }); 87 | 88 | }); 89 | 90 | describe('Function: client.createFIFO', function() { 91 | 92 | it('should prefix with the length of the id', function () { 93 | var client = redis.createClient(); 94 | var id = client.createFIFO('12345678910'); 95 | id.should.equal('11|12345678910'); 96 | }); 97 | 98 | it('should pad with a zero for single digit length ids', function () { 99 | var client = redis.createClient(); 100 | var id = client.createFIFO('123'); 101 | id.should.equal('03|123'); 102 | }); 103 | 104 | }); 105 | 106 | describe('Function: client.stripFIFO', function() { 107 | 108 | it('should strip the prefix on the id', function () { 109 | var client = redis.createClient(); 110 | var id = client.stripFIFO( '03|123' ); 111 | id.should.equal(123); 112 | }); 113 | }); 114 | 115 | }); 116 | 117 | describe('Function: createClientFactory', function() { 118 | var options, client; 119 | beforeEach(function(){ 120 | options = { 121 | prefix: 'prefix', 122 | redis: { 123 | port: 'port', 124 | host: 'host', 125 | db: 'db', 126 | options: {} 127 | } 128 | }; 129 | client = { 130 | auth: sinon.stub(), 131 | select: sinon.stub() 132 | }; 133 | sinon.stub(r, 'createClient').returns(client); 134 | }); 135 | 136 | afterEach(function(){ 137 | r.createClient.restore(); 138 | }); 139 | 140 | it('should create a client', function () { 141 | var c = redis.createClientFactory(options); 142 | r.createClient.called.should.be.true; 143 | r.createClient.calledWith(options.redis.port, options.redis.host, options.redis.options).should.be.true; 144 | }); 145 | 146 | it('should authenticate if auth is present', function () { 147 | options.redis.auth = 'auth'; 148 | var c = redis.createClientFactory(options); 149 | client.auth.calledWith(options.redis.auth).should.be.true; 150 | }); 151 | 152 | it('should select the passed in db', function () { 153 | options.redis.db = 1; 154 | var c = redis.createClientFactory(options); 155 | client.select.calledWith(options.redis.db).should.be.true; 156 | }); 157 | 158 | }); 159 | 160 | describe('Function: client', function() { 161 | 162 | it('should return the existing client if there is one', function () { 163 | redis._client = 'client'; 164 | (redis.client()).should.equal('client'); 165 | }); 166 | 167 | it('should create a client if one is not present', function () { 168 | redis._client = null; 169 | sinon.stub(redis, 'createClient'); 170 | redis.client(); 171 | redis.createClient.called.should.be.true; 172 | redis.createClient.restore(); 173 | }); 174 | 175 | }); 176 | 177 | describe('Function: pubsubClient', function() { 178 | 179 | it('should return the existing client if there is one', function () { 180 | redis._pubsub = 'pubsubClient'; 181 | (redis.pubsubClient()).should.equal('pubsubClient'); 182 | }); 183 | 184 | it('should create a pubsubClient if one is not present', function () { 185 | redis._pubsub = null; 186 | sinon.stub(redis, 'createClient'); 187 | redis.pubsubClient(); 188 | redis.createClient.called.should.be.true; 189 | redis.createClient.restore(); 190 | }); 191 | 192 | }); 193 | 194 | describe('Function: reset', function() { 195 | var client, pubsub; 196 | beforeEach(function(){ 197 | client = { 198 | quit: sinon.stub() 199 | }; 200 | pubsub = { 201 | quit: sinon.stub() 202 | }; 203 | redis._client = client; 204 | redis._pubsub = pubsub; 205 | }); 206 | 207 | it('should quit and remove the client', function () { 208 | redis.reset(); 209 | (redis._client == null).should.be.true; 210 | client.quit.called.should.be.true; 211 | }); 212 | 213 | it('should quick and remove the pubsub client', function () { 214 | redis.reset(); 215 | (redis._pubsub == null).should.be.true; 216 | pubsub.quit.called.should.be.true; 217 | }); 218 | 219 | }); 220 | 221 | }); -------------------------------------------------------------------------------- /lib/http/public/javascripts/main.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * kue - http - main 3 | * Copyright (c) 2011 LearnBoost 4 | * MIT Licensed 5 | */ 6 | 7 | // TODO: clean up 8 | // TODO: server-side config for this stuff 9 | // TODO: optimize! many of these jQuery objects can be cached 10 | 11 | /** 12 | * Active state. 13 | */ 14 | 15 | var active; 16 | 17 | /** 18 | * Active type filter. 19 | */ 20 | 21 | var filter; 22 | 23 | /** 24 | * Number of jobs fetched when "more" is clicked. 25 | */ 26 | 27 | var more = 10; 28 | 29 | /** 30 | * Number of jobs shown. 31 | */ 32 | 33 | var to = more; 34 | 35 | /** 36 | * Sort order. 37 | */ 38 | 39 | var sort = 'asc'; 40 | 41 | /** 42 | * Loading indicator. 43 | */ 44 | 45 | var loading; 46 | 47 | /** 48 | * Initialize UI. 49 | */ 50 | 51 | function init(state) { 52 | var canvas = o('#loading canvas').get(0) 53 | , ctx = canvas.getContext('2d'); 54 | 55 | loading = new LoadingIndicator; 56 | loading.ctx = ctx; 57 | loading.size(canvas.width); 58 | 59 | pollStats(1000); 60 | show(state)(); 61 | o('li.inactive a').click(show('inactive')); 62 | o('li.complete a').click(show('complete')); 63 | o('li.active a').click(show('active')); 64 | o('li.failed a').click(show('failed')); 65 | o('li.delayed a').click(show('delayed')); 66 | 67 | o('#filter').change(function () { 68 | filter = $(this).val(); 69 | }); 70 | 71 | o('#sort').change(function () { 72 | sort = $(this).val(); 73 | o('#jobs .job').remove(); 74 | }); 75 | 76 | onpopstate = function (e) { 77 | if (e.state) show(e.state.state)(); 78 | }; 79 | } 80 | 81 | /** 82 | * Show loading indicator. 83 | */ 84 | 85 | function showLoading() { 86 | var n = 0; 87 | o('#loading').show(); 88 | showLoading.timer = setInterval(function () { 89 | loading.update(++n).draw(loading.ctx); 90 | }, 50); 91 | } 92 | 93 | /** 94 | * Hide loading indicator. 95 | */ 96 | 97 | function hideLoading() { 98 | o('#loading').hide(); 99 | clearInterval(showLoading.timer); 100 | } 101 | 102 | /** 103 | * Infinite scroll. 104 | */ 105 | 106 | function infiniteScroll() { 107 | if (infiniteScroll.bound) return; 108 | var body = o('body'); 109 | hideLoading(); 110 | infiniteScroll.bound = true; 111 | 112 | o(window).scroll(function (e) { 113 | var top = body.scrollTop() 114 | , height = body.innerHeight() 115 | , windowHeight = window.innerHeight 116 | , pad = 30; 117 | 118 | if (top + windowHeight + pad >= height) { 119 | to += more; 120 | infiniteScroll.bound = false; 121 | showLoading(); 122 | o(window).unbind('scroll'); 123 | } 124 | }); 125 | } 126 | 127 | /** 128 | * Show jobs with `state`. 129 | * 130 | * @param {String} state 131 | * @param {Boolean} init 132 | * @return {Function} 133 | */ 134 | 135 | function show(state) { 136 | return function () { 137 | active = state; 138 | if (pollForJobs.timer) { 139 | clearTimeout(pollForJobs.timer); 140 | delete pollForJobs.timer; 141 | } 142 | history.pushState({ state: state }, state, state); 143 | o('#jobs .job').remove(); 144 | o('#menu li a').removeClass('active'); 145 | o('#menu li.' + state + ' a').addClass('active'); 146 | pollForJobs(state, 1000); 147 | return false; 148 | } 149 | } 150 | 151 | /** 152 | * Poll for jobs with `state` every `ms`. 153 | * 154 | * @param {String} state 155 | * @param {Number} ms 156 | */ 157 | 158 | function pollForJobs(state, ms) { 159 | o('h1').text(state); 160 | refreshJobs(state, function () { 161 | infiniteScroll(); 162 | if (!pollForJobs.timer) pollForJobs.timer = setTimeout(function () { 163 | delete pollForJobs.timer; 164 | pollForJobs(state, ms); 165 | }, ms); 166 | }); 167 | }; 168 | 169 | /** 170 | * Re-request and refresh job elements. 171 | * 172 | * @param {String} state 173 | * @param {Function} fn 174 | */ 175 | 176 | function refreshJobs(state, fn) { 177 | // TODO: clean this crap up 178 | var jobHeight = o('#jobs .job .block').outerHeight(true) 179 | , top = o(window).scrollTop() 180 | , height = window.innerHeight 181 | , visibleFrom = Math.max(0, Math.floor(top / jobHeight)) 182 | , visibleTo = Math.floor((top + height) / jobHeight) 183 | , url = './jobs/' 184 | + (filter ? filter + '/' : '') 185 | + state + '/0..' + to 186 | + '/' + sort; 187 | 188 | // var color = ['blue', 'red', 'yellow', 'green', 'purple'][Math.random() * 5 | 0]; 189 | 190 | request(url, function (jobs) { 191 | var len = jobs.length 192 | , job 193 | , el; 194 | 195 | // remove jobs which have changed their state 196 | o('#jobs .job').each(function (i, el) { 197 | var el = $(el) 198 | , id = (el.attr('id') || '').replace('job-', '') 199 | , found = jobs.some(function (job) { 200 | return job && id == job.id; 201 | }); 202 | if (!found) el.remove(); 203 | }); 204 | 205 | for (var i = 0; i < len; ++i) { 206 | if (!jobs[i]) continue; 207 | 208 | // exists 209 | if (o('#job-' + jobs[i].id).length) { 210 | if (i < visibleFrom || i > visibleTo) continue; 211 | el = o('#job-' + jobs[i].id); 212 | // el.css('background-color', color); 213 | job = el.get(0).job; 214 | job.update(jobs[i]) 215 | .showProgress('active' == active) 216 | .showErrorMessage('failed' == active) 217 | .render(); 218 | // new 219 | } else { 220 | job = new Job(jobs[i]); 221 | el = job.showProgress('active' == active) 222 | .showErrorMessage('failed' == active) 223 | .render(true); 224 | 225 | el.get(0).job = job; 226 | el.appendTo('#jobs'); 227 | } 228 | } 229 | 230 | fn(); 231 | }); 232 | } 233 | 234 | /** 235 | * Poll for stats every `ms`. 236 | * 237 | * @param {Number} ms 238 | */ 239 | 240 | function pollStats(ms) { 241 | request('./stats', function (data) { 242 | o('li.inactive .count').text(data.inactiveCount); 243 | o('li.active .count').text(data.activeCount); 244 | o('li.complete .count').text(data.completeCount); 245 | o('li.failed .count').text(data.failedCount); 246 | o('li.delayed .count').text(data.delayedCount); 247 | setTimeout(function () { 248 | pollStats(ms); 249 | }, ms); 250 | }); 251 | } 252 | 253 | /** 254 | * Request `url` and invoke `fn(res)`. 255 | * 256 | * @param {String} url 257 | * @param {Function} fn 258 | */ 259 | 260 | function request(url, fn) { 261 | var method = 'GET'; 262 | 263 | if ('string' == typeof fn) { 264 | method = url; 265 | url = fn; 266 | fn = arguments[2]; 267 | } 268 | 269 | fn = fn || function () { 270 | }; 271 | 272 | o.ajax({ type: method, url: url }) 273 | .success(function (res) { 274 | res.error 275 | ? error(res.error) 276 | : fn(res); 277 | }); 278 | } 279 | 280 | /** 281 | * Display error `msg`. 282 | * 283 | * @param {String} msg 284 | */ 285 | 286 | function error(msg) { 287 | o('#error').text(msg).addClass('show'); 288 | setTimeout(function () { 289 | o('#error').removeClass('show'); 290 | }, 4000); 291 | } 292 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | var kue = require( '../' ); 2 | 3 | describe('CONNECTION', function(){ 4 | var jobs = null; 5 | 6 | afterEach( function ( done ) { 7 | jobs.shutdown( 50, function () { 8 | done() 9 | } ); 10 | } ); 11 | 12 | it( 'should configure properly with string', function ( done ) { 13 | jobs = new kue( { 14 | redis: 'redis://localhost:6379/15?foo=bar' 15 | } ); 16 | 17 | jobs.client.options.port.should.be.eql( 6379 ); 18 | jobs.client.options.host.should.be.eql( 'localhost' ); 19 | jobs.client.options.foo.should.be.eql( 'bar' ); 20 | 21 | var jobData = { 22 | title: 'welcome email for tj', 23 | to: '"TJ" ', 24 | template: 'welcome-email' 25 | }; 26 | jobs.create( 'email-should-be-processed-3', jobData ).priority( 'high' ).save(); 27 | jobs.process( 'email-should-be-processed-3', function ( job, jdone ) { 28 | job.data.should.be.eql( jobData ); 29 | job.log( '

This is a formatted log

' ); 30 | // Needs to be here to support the async client.select statement where the return happens sync but the call is async 31 | jobs.client.selected_db.should.be.eql(15); 32 | jdone(); 33 | done(); 34 | } ); 35 | }); 36 | 37 | it( 'should configure properly with dictionary', function ( done ) { 38 | jobs = new kue( { 39 | redis: { 40 | host: 'localhost', 41 | port: 6379, 42 | db: 15, 43 | options: { 44 | foo: 'bar' 45 | } 46 | } 47 | } ); 48 | 49 | jobs.client.options.port.should.be.eql( 6379 ); 50 | jobs.client.options.host.should.be.eql( 'localhost' ); 51 | jobs.client.options.foo.should.be.eql( 'bar' ); 52 | 53 | var jobData = { 54 | title: 'welcome email for tj', 55 | to: '"TJ" ', 56 | template: 'welcome-email' 57 | }; 58 | jobs.create( 'email-should-be-processed-4', jobData ).priority( 'high' ).save(); 59 | jobs.process( 'email-should-be-processed-4', function ( job, jdone ) { 60 | job.data.should.be.eql( jobData ); 61 | job.log( '

This is a formatted log

' ); 62 | // Needs to be here to support the async client.select statement where the return happens sync but the call is async 63 | jobs.client.selected_db.should.be.eql(15); 64 | jdone(); 65 | done(); 66 | } ); 67 | }); 68 | 69 | it( 'should default to 0 db with string', function ( done ) { 70 | var jobs = new kue( { 71 | redis: 'redis://localhost:6379/?foo=bar' 72 | } ); 73 | 74 | jobs.client.options.port.should.be.eql( 6379 ); 75 | jobs.client.options.host.should.be.eql( 'localhost' ); 76 | jobs.client.options.foo.should.be.eql( 'bar' ); 77 | 78 | var jobData = { 79 | title: 'welcome email for tj', 80 | to: '"TJ" ', 81 | template: 'welcome-email' 82 | }; 83 | jobs.create( 'email-should-be-processed-5', jobData ).priority( 'high' ).save(); 84 | jobs.process( 'email-should-be-processed-5', function ( job, jdone ) { 85 | job.data.should.be.eql( jobData ); 86 | job.log( '

This is a formatted log

' ); 87 | jobs.client.selected_db.should.be.eql(0); 88 | jdone(); 89 | done(); 90 | } ); 91 | 92 | }); 93 | 94 | it( 'should default to 0 db with string and no /', function ( done ) { 95 | var jobs = new kue( { 96 | redis: 'redis://localhost:6379?foo=bar' 97 | } ); 98 | 99 | jobs.client.options.port.should.be.eql( 6379 ); 100 | jobs.client.options.host.should.be.eql( 'localhost' ); 101 | jobs.client.options.foo.should.be.eql( 'bar' ); 102 | 103 | var jobData = { 104 | title: 'welcome email for tj', 105 | to: '"TJ" ', 106 | template: 'welcome-email' 107 | }; 108 | jobs.create( 'email-should-be-processed-6', jobData ).priority( 'high' ).save(); 109 | jobs.process( 'email-should-be-processed-6', function ( job, jdone ) { 110 | job.data.should.be.eql( jobData ); 111 | job.log( '

This is a formatted log

' ); 112 | jobs.client.selected_db.should.be.eql(0); 113 | jdone(); 114 | done(); 115 | } ); 116 | 117 | }); 118 | 119 | it( 'should configure properly with dictionary', function ( done ) { 120 | jobs = new kue( { 121 | redis: { 122 | host: 'localhost', 123 | port: 6379, 124 | options: { 125 | foo: 'bar' 126 | } 127 | } 128 | } ); 129 | 130 | jobs.client.options.port.should.be.eql( 6379 ); 131 | jobs.client.options.host.should.be.eql( 'localhost' ); 132 | jobs.client.options.foo.should.be.eql( 'bar' ); 133 | 134 | var jobData = { 135 | title: 'welcome email for tj', 136 | to: '"TJ" ', 137 | template: 'welcome-email' 138 | }; 139 | jobs.create( 'email-should-be-processed-7', jobData ).priority( 'high' ).save(); 140 | jobs.process( 'email-should-be-processed-7', function ( job, jdone ) { 141 | job.data.should.be.eql( jobData ); 142 | job.log( '

This is a formatted log

' ); 143 | // Needs to be here to support the async client.select statement where the return happens sync but the call is async 144 | jobs.client.selected_db.should.be.eql(0); 145 | jdone(); 146 | done(); 147 | } ); 148 | }); 149 | }); 150 | 151 | describe( 'JOBS', function () { 152 | 153 | var jobs = null; 154 | 155 | beforeEach( function ( done ) { 156 | jobs = kue.createQueue( { promotion: { interval: 100 } } ); 157 | done(); 158 | } ); 159 | 160 | afterEach( function ( done ) { 161 | jobs.shutdown( 50, function () { 162 | done() 163 | } ); 164 | } ); 165 | 166 | it( 'should be processed', function ( done ) { 167 | var jobData = { 168 | title: 'welcome email for tj', 169 | to: '"TJ" ', 170 | template: 'welcome-email' 171 | }; 172 | jobs.create( 'email-should-be-processed', jobData ).priority( 'high' ).save(); 173 | jobs.process( 'email-should-be-processed', function ( job, jdone ) { 174 | job.data.should.be.eql( jobData ); 175 | job.log( '

This is a formatted log

' ); 176 | jdone(); 177 | done(); 178 | } ); 179 | } ); 180 | 181 | it( 'should retry on failure if attempts is set', function ( testDone ) { 182 | var job = jobs.create( 'failure-attempts', {} ); 183 | var failures = 0; 184 | job.attempts( 5 ) 185 | .on( 'complete', function () { 186 | attempts.should.be.equal( 5 ); 187 | failures.should.be.equal( 4 ); 188 | testDone(); 189 | } ) 190 | .on( 'failed attempt', function ( attempt ) { 191 | failures++; 192 | } ) 193 | .save(); 194 | var attempts = 0; 195 | jobs.process( 'failure-attempts', function ( job, done ) { 196 | attempts++; 197 | if ( attempts == 5 ) 198 | done(); 199 | else 200 | done( new Error( "error" ) ); 201 | } ); 202 | } ); 203 | 204 | it( 'should accept url strings for redis when making an new queue', function ( done ) { 205 | var jobs = new kue( { 206 | redis: 'redis://localhost:6379/?foo=bar' 207 | } ); 208 | 209 | jobs.client.options.port.should.be.eql( 6379 ); 210 | jobs.client.options.host.should.be.eql( 'localhost' ); 211 | jobs.client.options.foo.should.be.eql( 'bar' ); 212 | 213 | var jobData = { 214 | title: 'welcome email for tj', 215 | to: '"TJ" ', 216 | template: 'welcome-email' 217 | }; 218 | jobs.create( 'email-should-be-processed-2', jobData ).priority( 'high' ).save(); 219 | jobs.process( 'email-should-be-processed-2', function ( job, jdone ) { 220 | job.data.should.be.eql( jobData ); 221 | job.log( '

This is a formatted log

' ); 222 | jdone(); 223 | done(); 224 | } ); 225 | } ); 226 | } ); 227 | -------------------------------------------------------------------------------- /lib/http/public/javascripts/job.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * kue - Job 3 | * Copyright (c) 2011 LearnBoost 4 | * MIT Licensed 5 | */ 6 | 7 | /** 8 | * Initialize a new `Job` with the given `data`. 9 | * 10 | * @param {Object} obj 11 | */ 12 | 13 | function Job(data) { 14 | this.update(data); 15 | } 16 | 17 | /** 18 | * Show progress indicator. 19 | * 20 | * @param {Boolean} val 21 | * @return {Job} for chaining 22 | */ 23 | 24 | Job.prototype.showProgress = function (val) { 25 | this._showProgress = val; 26 | return this; 27 | }; 28 | 29 | /** 30 | * Show error message when `val` is true. 31 | * 32 | * @param {Boolean} val 33 | * @return {Job} for chaining 34 | */ 35 | 36 | Job.prototype.showErrorMessage = function (val) { 37 | this._showError = val; 38 | return this; 39 | }; 40 | 41 | /** 42 | * Remove the job and callback `fn()`. 43 | * 44 | * @param {Function} fn 45 | */ 46 | 47 | Job.prototype.remove = function (fn) { 48 | request('DELETE', './job/' + this.id, fn); 49 | return this; 50 | }; 51 | 52 | /** 53 | * Restart the job and callback `fn()`. 54 | * 55 | * @param {Function} fn 56 | */ 57 | 58 | Job.prototype.restart = function (fn) { 59 | request('GET', './inactive/' + this.id, fn); 60 | return this; 61 | }; 62 | 63 | /** 64 | * Update the job with the given `data`. 65 | * 66 | * @param {Object} data 67 | * @return {Job} for chaining 68 | */ 69 | 70 | Job.prototype.update = function (data) { 71 | for (var key in data) this[key] = data[key]; 72 | if (!this.data) this.data = {}; 73 | return this; 74 | }; 75 | 76 | /** 77 | * Render the job, returning an oQuery object. 78 | * 79 | * @param {Boolean} isNew 80 | * @return {oQuery} 81 | */ 82 | 83 | Job.prototype.render = function (isNew) { 84 | var self = this 85 | , id = this.id 86 | , view = this.view 87 | , keys = Object.keys(this.data).sort() 88 | , data; 89 | 90 | if (isNew) { 91 | view = this.view = View('job'); 92 | 93 | view.remove(function () { 94 | this.remove(); 95 | self.remove(); 96 | }); 97 | 98 | view.restart(function () { 99 | this.restart(); 100 | self.restart(); 101 | }); 102 | 103 | var canvas = view.progress 104 | , ctx = this.ctx = canvas.getContext('2d') 105 | , progress = new Progress; 106 | 107 | progress.size(canvas.width); 108 | this._progress = progress; 109 | 110 | // initially hide the logs 111 | view.log.hide(); 112 | 113 | // populate title and id 114 | view.el.attr('id', 'job-' + id); 115 | view.id(id); 116 | 117 | // show job data 118 | for (var i = 0, len = keys.length; i < len; ++i) { 119 | data = this.data[keys[i]]; 120 | if ('object' == typeof data) data = JSON.stringify(data); 121 | var row = View('row'); 122 | // row.title(keys[i] + ':').value(data); 123 | row.title(keys[i] + ':').value($('

').text(data).html()); 124 | view.data.add(row); 125 | } 126 | 127 | // alter state 128 | view.state(this.state); 129 | view.state().click(function () { 130 | var select = o('', options(states, self.state)); 131 | o(this).replaceWith(select); 132 | select.change(function () { 133 | self.updateState(select.val()); 134 | }); 135 | return false; 136 | }); 137 | 138 | // alter priority 139 | view.priority(priority(this)); 140 | view.priority().click(function () { 141 | var select = o('', options(priorities, self.priority)); 142 | o(this).replaceWith(select); 143 | select.change(function () { 144 | self.updatePriority(select.val()); 145 | }) 146 | return false; 147 | }); 148 | 149 | // show details 150 | view.el.find('.contents').toggle(function () { 151 | view.details().addClass('show'); 152 | self.showDetails = true; 153 | }, function () { 154 | view.details().removeClass('show'); 155 | self.showDetails = false; 156 | }); 157 | } 158 | 159 | this.renderUpdate(); 160 | 161 | return view.el; 162 | }; 163 | 164 | /** 165 | * Update this jobs state to `state`. 166 | * 167 | * @param {String} state 168 | */ 169 | 170 | Job.prototype.updateState = function (state) { 171 | request('PUT', './job/' + this.id + '/state/' + state); 172 | }; 173 | 174 | /** 175 | * Update this jobs priority to `n`. 176 | * 177 | * @param {Number} n 178 | */ 179 | 180 | Job.prototype.updatePriority = function (n) { 181 | request('PUT', './job/' + this.id + '/priority/' + n); 182 | }; 183 | 184 | /** 185 | * Update the job view. 186 | */ 187 | 188 | Job.prototype.renderUpdate = function () { 189 | // TODO: templates 190 | var view = this.view 191 | , showError = this._showError 192 | , showProgress = this._showProgress; 193 | 194 | // type 195 | view.type(this.type); 196 | 197 | // errors 198 | if (showError && this.error) { 199 | view.errorMessage(this.error.split('\n')[0]); 200 | } else { 201 | view.errorMessage().remove(); 202 | } 203 | 204 | // attempts 205 | if (this.attempts.made) { 206 | view.attempts(this.attempts.made + '/' + this.attempts.max); 207 | } else { 208 | view.attempts().parent().remove(); 209 | } 210 | 211 | // title 212 | view.title(this.data.title 213 | ? this.data.title 214 | : 'untitled'); 215 | 216 | // details 217 | this.renderTimestamp('created_at'); 218 | this.renderTimestamp('updated_at'); 219 | this.renderTimestamp('failed_at'); 220 | 221 | // delayed 222 | if ('delayed' == this.state) { 223 | var delay = parseInt(this.delay, 10) 224 | , creation = parseInt(this.created_at, 10) 225 | , remaining = relative(creation + delay - Date.now()); 226 | view.title((this.data.title || '') + ' ( ' + remaining + ' )'); 227 | } 228 | 229 | // inactive 230 | if ('inactive' == this.state) view.log.remove(); 231 | 232 | // completion 233 | if ('complete' == this.state) { 234 | view.duration(relative(this.duration)); 235 | view.updated_at().prev().text('Completed: '); 236 | view.priority().parent().hide(); 237 | } else { 238 | view.duration().parent().remove(); 239 | } 240 | 241 | // error 242 | if ('failed' == this.state) { 243 | view.error().show().find('pre').text(this.error); 244 | } else { 245 | view.error().hide(); 246 | } 247 | 248 | // progress indicator 249 | if (showProgress) this._progress.update(this.progress).draw(this.ctx); 250 | 251 | // logs 252 | if (this.showDetails) { 253 | request('GET', './job/' + this.id + '/log', function (log) { 254 | var ul = view.log.show(); 255 | 256 | // return early if log hasnt changed 257 | if (ul.text() === log) return; 258 | 259 | ul.find('li').remove(); 260 | log.forEach(function (line) { 261 | ul.append(o('
  • %s
    • ', line)); 262 | }); 263 | }); 264 | } 265 | }; 266 | 267 | /** 268 | * Render timestamp for the given `prop`. 269 | * 270 | * @param {String} prop 271 | */ 272 | 273 | Job.prototype.renderTimestamp = function (prop) { 274 | var val = this[prop] 275 | , view = this.view; 276 | 277 | if (val) { 278 | view[prop]().text(relative(Date.now() - val) + ' ago'); 279 | } else { 280 | view[prop]().parent().remove(); 281 | } 282 | }; 283 | -------------------------------------------------------------------------------- /test/jsonapi.js: -------------------------------------------------------------------------------- 1 | var request = require( 'supertest' ), 2 | kue = require( '../index' ), 3 | async = require( 'async' ), 4 | chai = require( 'chai' ), 5 | queue = kue.createQueue( { disableSearch: false } ), //customize queue before accessing kue.app 6 | app = kue.app, 7 | type = 'test:inserts'; 8 | 9 | 10 | expect = chai.expect; 11 | 12 | 13 | function jobsPopulate( count ) { 14 | var priority = [ 10, 0, -5, -10, -15 ], 15 | jobs = []; 16 | 17 | for ( var i = 0; i < count; i++ ) { 18 | jobs.push( { 19 | type: type, 20 | data: { 21 | title: i, 22 | data: type + ':data' 23 | }, 24 | options: { 25 | // random priority 26 | priority: priority[ Math.floor( Math.random() * 5 ) ] 27 | } 28 | } ); 29 | } 30 | 31 | // return array only if length > 1 32 | return jobs.length === 1 ? jobs[ 0 ] : jobs; 33 | } 34 | 35 | 36 | describe( 'JSON API', function () { 37 | var scope = {}; 38 | 39 | 40 | before( function ( done ) { 41 | scope.queue = queue; 42 | 43 | // delete all jobs to get a clean state 44 | kue.Job.rangeByType( type, 'inactive', 0, 100, 'asc', function ( err, jobs ) { 45 | if ( err ) return done( err ); 46 | if ( !jobs.length ) return done(); 47 | async.each( jobs, function ( job, asyncDone ) { 48 | job.remove( asyncDone ); 49 | }, done ); 50 | } ); 51 | } ); 52 | 53 | 54 | after( function ( done ) { 55 | scope.queue.shutdown( 200, function ( err ) { 56 | scope.queue = null; 57 | done( err ); 58 | } ); 59 | } ); 60 | 61 | 62 | describe( 'create, get, update and delete', function () { 63 | it( 'should insert a job and respond with an id', function ( done ) { 64 | request( app ) 65 | .post( '/job' ) 66 | .send( jobsPopulate( 1 ) ) 67 | .expect( 200 ) 68 | .expect( function ( res ) { 69 | res.body.message.should.equal( 'job created' ); 70 | res.body.id.should.be.a.Number; 71 | Object.keys( res.body ).should.have.lengthOf( 2 ); 72 | 73 | scope.jobId = res.body.id; 74 | } ) 75 | .end( done ); 76 | } ); 77 | 78 | 79 | it( 'should insert multiple jobs and respond with ids', function ( done ) { 80 | var jobCount = 5; 81 | 82 | request( app ) 83 | .post( '/job' ) 84 | .send( jobsPopulate( jobCount ) ) 85 | .expect( 200 ) 86 | .expect( function ( res ) { 87 | var created = res.body; 88 | created.should.be.ok; 89 | created.length.should.equal( jobCount ); 90 | 91 | for ( var i = 0; i < jobCount; i++ ) { 92 | var job = created[ i ]; 93 | job.message.should.be.equal( 'job created' ); 94 | job.id.should.be.a.Number; 95 | Object.keys( job ).should.have.lengthOf( 2 ); 96 | } 97 | } ) 98 | .end( done ); 99 | } ); 100 | 101 | 102 | it( 'get job by id: job is inactive', function ( done ) { 103 | request( app ) 104 | .get( '/job/' + scope.jobId ) 105 | .expect( function ( res ) { 106 | res.body.id.should.eql( scope.jobId ); 107 | res.body.type.should.eql( type ); 108 | res.body.state.should.eql( 'inactive' ); 109 | } ) 110 | .end( done ); 111 | } ); 112 | 113 | 114 | it( 'change state', function ( done ) { 115 | request( app ) 116 | .put( '/job/' + scope.jobId + '/state/active' ) 117 | .expect( function ( res ) { 118 | expect( res.body.message ).to.exist; 119 | } ) 120 | .end( done ); 121 | } ); 122 | 123 | 124 | it( 'get job by id: job is now active', function ( done ) { 125 | request( app ) 126 | .get( '/job/' + scope.jobId ) 127 | .expect( function ( res ) { 128 | res.body.id.should.eql( scope.jobId ); 129 | res.body.type.should.eql( type ); 130 | res.body.state.should.eql( 'active' ); 131 | } ) 132 | .end( done ); 133 | } ); 134 | 135 | 136 | it( 'delete job by id', function ( done ) { 137 | request( app ) 138 | .del( '/job/' + scope.jobId ) 139 | .expect( function ( res ) { 140 | expect( res.body.message ).to.contain( scope.jobId ); 141 | } ) 142 | .end( done ); 143 | } ); 144 | } ); 145 | 146 | 147 | describe( 'search', function () { 148 | it( 'search by query: not found', function ( done ) { 149 | request( app ) 150 | .get( '/job/search' ) 151 | .query( {} ) 152 | .expect( function ( res ) { 153 | res.body.length.should.eql( 0 ); 154 | } ) 155 | .end( done ); 156 | } ); 157 | 158 | 159 | it( 'search by query: found', function ( done ) { 160 | request( app ) 161 | .get( '/job/search' ) 162 | .query( { 163 | q: type + ':data' 164 | } ) 165 | .expect( function ( res ) { 166 | // we created 6 jobs, one was deleted, 5 left 167 | res.body.length.should.eql( 5 ); 168 | } ) 169 | .end( done ); 170 | } ); 171 | } ); 172 | 173 | 174 | describe( 'range', function () { 175 | it( 'range from...to', function ( done ) { 176 | request( app ) 177 | .get( '/jobs/0..3' ) 178 | .expect( function ( res ) { 179 | res.body.length.should.eql( 4 ); 180 | } ) 181 | .end( done ); 182 | } ); 183 | 184 | 185 | it( 'range from...to with type and state', function ( done ) { 186 | request( app ) 187 | .get( '/jobs/' + type + '/inactive/0..20/asc' ) 188 | .expect( function ( res ) { 189 | res.body.length.should.eql( 5 ); 190 | } ) 191 | .end( done ); 192 | } ); 193 | } ); 194 | 195 | 196 | describe( 'stats', function () { 197 | it( 'get stats', function ( done ) { 198 | request( app ) 199 | .get( '/stats' ) 200 | .expect( function ( res ) { 201 | expect( res.body.inactiveCount ).to.exist; 202 | expect( res.body.completeCount ).to.exist; 203 | expect( res.body.activeCount ).to.exist; 204 | expect( res.body.delayedCount ).to.exist; 205 | } ) 206 | .end( done ); 207 | } ); 208 | } ); 209 | 210 | 211 | describe( 'error cases', function () { 212 | it( 'should return 204 status code when POST /job body is empty', function ( done ) { 213 | request( app ) 214 | .post( '/job' ) 215 | .send( [] ) 216 | .expect( 204 ) 217 | .expect( function ( res ) { 218 | res.text.should.have.lengthOf( 0 ); 219 | } ) 220 | .end( done ); 221 | } ); 222 | 223 | it( 'should insert jobs including an invalid job, respond with ids and error', function ( done ) { 224 | var jobs = jobsPopulate( 3 ); 225 | delete jobs[ 1 ].type; 226 | 227 | request( app ) 228 | .post( '/job' ) 229 | .send( jobs ) 230 | .expect( 400 ) // Expect a bad request 231 | .expect( function ( res ) { 232 | var created = res.body; 233 | 234 | created.should.be.ok; 235 | created.length.should.equal( 3 ); // should still have 3 objects in the response 236 | 237 | // The first one succeeded 238 | created[ 0 ].message.should.be.equal( 'job created' ); 239 | created[ 0 ].id.should.be.a.Number; 240 | Object.keys( created[ 0 ] ).should.have.lengthOf( 2 ); 241 | 242 | // The second one failed 243 | created[ 1 ].error.should.equal( 'Must provide job type' ); 244 | Object.keys( created[ 1 ] ).should.have.lengthOf( 1 ); 245 | 246 | // The third one succeeded 247 | created[ 2 ].message.should.be.equal( 'job created' ); 248 | created[ 2 ].id.should.be.a.Number; 249 | Object.keys( created[ 2 ] ).should.have.lengthOf( 2 ); 250 | } ) 251 | .end( done ); 252 | } ); 253 | } ); 254 | } ); 255 | -------------------------------------------------------------------------------- /lib/http/routes/json.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * kue - http - routes - json 3 | * Copyright (c) 2011 LearnBoost 4 | * MIT Licensed 5 | */ 6 | 7 | /** 8 | * Module dependencies. 9 | */ 10 | 11 | var Queue = require('../../kue') 12 | , Job = require('../../queue/job') 13 | , lodash = require('lodash') 14 | , queue = Queue.createQueue(); 15 | 16 | /** 17 | * Search instance. 18 | */ 19 | 20 | var search; 21 | function getSearch() { 22 | if( search ) return search; 23 | var reds = require('reds'); 24 | reds.createClient = require('../../redis').createClient; 25 | return search = reds.createSearch(queue.client.getKey('search')); 26 | } 27 | 28 | /** 29 | * Get statistics including: 30 | * 31 | * - inactive count 32 | * - active count 33 | * - complete count 34 | * - failed count 35 | * - delayed count 36 | * 37 | */ 38 | 39 | exports.stats = function( req, res ) { 40 | get(queue) 41 | ('inactiveCount') 42 | ('completeCount') 43 | ('activeCount') 44 | ('failedCount') 45 | ('delayedCount') 46 | ('workTime') 47 | (function( err, obj ) { 48 | if( err ) return res.json({ error: err.message }); 49 | res.json(obj); 50 | }); 51 | }; 52 | 53 | /** 54 | * Get job types. 55 | */ 56 | 57 | exports.types = function( req, res ) { 58 | queue.types(function( err, types ) { 59 | if( err ) return res.json({ error: err.message }); 60 | res.json(types); 61 | }); 62 | }; 63 | 64 | /** 65 | * Get jobs by range :from..:to. 66 | */ 67 | 68 | exports.jobRange = function( req, res ) { 69 | var from = parseInt(req.params.from, 10) 70 | , to = parseInt(req.params.to, 10) 71 | , order = req.params.order; 72 | 73 | Job.range(from, to, order, function( err, jobs ) { 74 | if( err ) return res.json({ error: err.message }); 75 | res.json(jobs); 76 | }); 77 | }; 78 | 79 | /** 80 | * Get jobs by :state, and range :from..:to. 81 | */ 82 | 83 | exports.jobStateRange = function( req, res ) { 84 | var state = req.params.state 85 | , from = parseInt(req.params.from, 10) 86 | , to = parseInt(req.params.to, 10) 87 | , order = req.params.order; 88 | 89 | Job.rangeByState(state, from, to, order, function( err, jobs ) { 90 | if( err ) return res.json({ error: err.message }); 91 | res.json(jobs); 92 | }); 93 | }; 94 | 95 | /** 96 | * Get jobs by :type, :state, and range :from..:to. 97 | */ 98 | 99 | exports.jobTypeRange = function( req, res ) { 100 | var type = req.params.type 101 | , state = req.params.state 102 | , from = parseInt(req.params.from, 10) 103 | , to = parseInt(req.params.to, 10) 104 | , order = req.params.order; 105 | 106 | Job.rangeByType(type, state, from, to, order, function( err, jobs ) { 107 | if( err ) return res.json({ error: err.message }); 108 | res.json(jobs); 109 | }); 110 | }; 111 | 112 | /** 113 | * Get jobs stats by :type and :state 114 | */ 115 | 116 | exports.jobTypeStateStats = function( req, res ) { 117 | var type = req.params.type 118 | , state = req.params.state; 119 | 120 | queue.cardByType(type, state, function( err, count ) { 121 | if( err ) return res.json({ error: err.message }); 122 | res.json({ count: count }); 123 | }); 124 | }; 125 | 126 | /** 127 | * Get job by :id. 128 | */ 129 | 130 | exports.job = function( req, res ) { 131 | var id = req.params.id; 132 | Job.get(id, function( err, job ) { 133 | if( err ) return res.json({ error: err.message }); 134 | res.json(job); 135 | }); 136 | }; 137 | 138 | /** 139 | * Restart job by :id. 140 | */ 141 | 142 | exports.inactive = function( req, res ) { 143 | var id = req.params.id; 144 | Job.get(id, function( err, job ) { 145 | if( err ) return res.json({ error: err.message }); 146 | job.inactive(); 147 | res.json({ message: 'job ' + id + ' inactive' }); 148 | }); 149 | }; 150 | 151 | /** 152 | * Create a job. 153 | */ 154 | 155 | exports.createJob = function( req, res ) { 156 | var body = req.body; 157 | 158 | function _create( args, next ) { 159 | if( !args.type ) return next({ error: 'Must provide job type' }, null, 400); 160 | 161 | var job = new Job(args.type, args.data || {}); 162 | var options = args.options || {}; 163 | if( options.attempts ) job.attempts(parseInt(options.attempts)); 164 | if( options.priority ) job.priority(options.priority); 165 | if( options.delay ) job.delay(options.delay); 166 | if( options.searchKeys ) job.searchKeys(options.searchKeys); 167 | if( options.backoff ) job.backoff(options.backoff); 168 | if( options.removeOnComplete ) job.removeOnComplete(options.removeOnComplete); 169 | if( options.ttl ) job.ttl(options.ttl); 170 | 171 | job.save(function( err ) { 172 | if( err ) { 173 | return next({ error: err.message }, null, 500); 174 | } 175 | else { 176 | return next(null, { message: 'job created', id: job.id }); 177 | } 178 | }); 179 | } 180 | 181 | if( !lodash.isEmpty(body) ) { 182 | if( lodash.isArray(body) ) { 183 | var returnErrorCode = 0; // Default: we don't have any error 184 | var i = 0, len = body.length; 185 | var result = []; 186 | -function _iterate() { 187 | _create(body[ i ], function( err, status, errCode ) { 188 | result.push(err || status); 189 | if( err ) { 190 | // Set an error code for the response 191 | if( !returnErrorCode ) { 192 | returnErrorCode = errCode || 500; 193 | } 194 | } 195 | 196 | // Keep processing even after an error 197 | i++; 198 | if( i < len ) { 199 | _iterate(); 200 | } 201 | else { 202 | // If we had an error code, return it 203 | if( returnErrorCode ) { 204 | res.status(returnErrorCode); 205 | } 206 | 207 | res.json(result); 208 | } 209 | }) 210 | }() 211 | } 212 | else { 213 | _create(body, function( err, status, errCode ) { 214 | if( err ) { 215 | res.status(errCode || 500).json(err); 216 | } 217 | else { 218 | res.json(status); 219 | } 220 | }) 221 | } 222 | } 223 | else { 224 | res.status(204); // "No content" status code 225 | res.end(); 226 | } 227 | }; 228 | 229 | /** 230 | * Remove job :id. 231 | */ 232 | 233 | exports.remove = function( req, res ) { 234 | var id = req.params.id; 235 | Job.remove(id, function( err ) { 236 | if( err ) return res.json({ error: err.message }); 237 | res.json({ message: 'job ' + id + ' removed' }); 238 | }); 239 | }; 240 | 241 | /** 242 | * Update job :id :priority. 243 | */ 244 | 245 | exports.updatePriority = function( req, res ) { 246 | var id = req.params.id 247 | , priority = parseInt(req.params.priority, 10); 248 | 249 | if( isNaN(priority) ) return res.json({ error: 'invalid priority' }); 250 | Job.get(id, function( err, job ) { 251 | if( err ) return res.json({ error: err.message }); 252 | job.priority(priority); 253 | job.save(function( err ) { 254 | if( err ) return res.json({ error: err.message }); 255 | res.json({ message: 'updated priority' }); 256 | }); 257 | }); 258 | }; 259 | 260 | /** 261 | * Update job :id :state. 262 | */ 263 | 264 | exports.updateState = function( req, res ) { 265 | var id = req.params.id 266 | , state = req.params.state; 267 | 268 | Job.get(id, function( err, job ) { 269 | if( err ) return res.json({ error: err.message }); 270 | job.state(state); 271 | job.save(function( err ) { 272 | if( err ) return res.json({ error: err.message }); 273 | res.json({ message: 'updated state' }); 274 | }); 275 | }); 276 | }; 277 | 278 | /** 279 | * Search and respond with ids. 280 | */ 281 | 282 | exports.search = function( req, res ) { 283 | getSearch().query(req.query.q).end(function( err, ids ) { 284 | if( err ) return res.json({ error: err.message }); 285 | res.json(ids); 286 | }); 287 | }; 288 | 289 | /** 290 | * Get log for job :id. 291 | */ 292 | 293 | exports.log = function( req, res ) { 294 | var id = req.params.id; 295 | Job.log(id, function( err, log ) { 296 | if( err ) return res.json({ error: err.message }); 297 | res.json(log); 298 | }); 299 | }; 300 | 301 | /** 302 | * Data fetching helper. 303 | */ 304 | 305 | function get( obj ) { 306 | var pending = 0 307 | , res = {} 308 | , callback 309 | , done; 310 | 311 | return function _( arg ) { 312 | switch(typeof arg) { 313 | case 'function': 314 | callback = arg; 315 | break; 316 | case 'string': 317 | ++pending; 318 | obj[ arg ](function( err, val ) { 319 | if( done ) return; 320 | if( err ) return done = true, callback(err); 321 | res[ arg ] = val; 322 | --pending || callback(null, res); 323 | }); 324 | break; 325 | } 326 | return _; 327 | }; 328 | } 329 | -------------------------------------------------------------------------------- /lib/http/public/javascripts/caustic.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * EventEmitter 3 | * Copyright (c) 2011 TJ Holowaychuk 4 | * MIT Licensed 5 | */ 6 | 7 | /** 8 | * EventEmitter. 9 | */ 10 | 11 | function EventEmitter() { 12 | this.callbacks = {}; 13 | } 14 | 15 | /** 16 | * Listen on the given `event` with `fn`. 17 | * 18 | * @param {String} event 19 | * @param {Function} fn 20 | */ 21 | 22 | EventEmitter.prototype.on = function (event, fn) { 23 | (this.callbacks[event] = this.callbacks[event] || []) 24 | .push(fn); 25 | return this; 26 | }; 27 | 28 | /** 29 | * Emit `event` with the given args. 30 | * 31 | * @param {String} event 32 | * @param {Mixed} ... 33 | */ 34 | 35 | EventEmitter.prototype.emit = function (event) { 36 | var args = Array.prototype.slice.call(arguments, 1) 37 | , callbacks = this.callbacks[event]; 38 | 39 | if (callbacks) { 40 | for (var i = 0, len = callbacks.length; i < len; ++i) { 41 | callbacks[i].apply(this, args) 42 | } 43 | } 44 | 45 | return this; 46 | }; 47 | 48 | /*! 49 | * caustic 50 | * Copyright(c) 2011 TJ Holowaychuk 51 | * MIT Licensed 52 | */ 53 | 54 | // TODO: `make caustic.js` should wrap in an anonymous function 55 | // TODO: `make caustic.min.js` 56 | 57 | // TODO: compile sub-views such as User etc based on the given 58 | // html, as there's no need to keep traversing each time. 59 | 60 | /** 61 | * Convert callback `fn` to a function when a string is given. 62 | * 63 | * @param {Type} name 64 | * @return {Type} 65 | * @api private 66 | */ 67 | 68 | function callback(fn) { 69 | return 'string' == typeof fn 70 | ? function (obj) { 71 | return obj[fn](); 72 | } 73 | : fn; 74 | } 75 | 76 | /** 77 | * Initialize a new view with the given `name` 78 | * or string of html. When a `name` is given an element 79 | * with the id `name + "-template"` will be used. 80 | * 81 | * Examples: 82 | * 83 | * var user = new View('user'); 84 | * var list = new View('
    '); 85 | * 86 | * @param {String} name 87 | * @api public 88 | */ 89 | 90 | function View(name) { 91 | if (!(this instanceof View)) return new View(name); 92 | EventEmitter.call(this); 93 | var html; 94 | if (~name.indexOf('<')) html = name; 95 | else html = $('#' + name + '-template').html(); 96 | this.el = $(html); 97 | this.visit(this.el); 98 | } 99 | 100 | /** 101 | * Inherit from `EventEmitter.prototype`. 102 | */ 103 | 104 | View.prototype.__proto__ = EventEmitter.prototype; 105 | 106 | /** 107 | * Visit `el`. 108 | * 109 | * @param {jQuery} el 110 | * @param {Boolean} ignore 111 | * @api private 112 | */ 113 | 114 | View.prototype.visit = function (el, ignore) { 115 | var self = this 116 | , type = el.get(0).nodeName 117 | , classes = el.attr('class').split(/ +/) 118 | , method = 'visit' + type; 119 | 120 | if (this[method] && !ignore) this[method](el, classes[0]); 121 | 122 | el.children().each(function (i, el) { 123 | self.visit($(el)); 124 | }); 125 | }; 126 | 127 | /** 128 | * Visit INPUT tag. 129 | * 130 | * @param {jQuery} el 131 | * @api public 132 | */ 133 | 134 | View.prototype.visitINPUT = function (el) { 135 | var self = this 136 | , name = el.attr('name') 137 | , type = el.attr('type'); 138 | 139 | switch (type) { 140 | case 'text': 141 | this[name] = function (val) { 142 | if (0 == arguments.length) return el.val(); 143 | el.val(val); 144 | return this; 145 | } 146 | 147 | this[name].isEmpty = function () { 148 | return '' == el.val(); 149 | }; 150 | 151 | this[name].clear = function () { 152 | el.val(''); 153 | return self; 154 | }; 155 | break; 156 | case 'checkbox': 157 | this[name] = function (val) { 158 | if (0 == arguments.length) return el.attr('checked'); 159 | switch (typeof val) { 160 | case 'function': 161 | el.change(function (e) { 162 | val.call(self, el.attr('checked'), e); 163 | }); 164 | break; 165 | default: 166 | el.attr('checked', val 167 | ? 'checked' 168 | : val); 169 | } 170 | return this; 171 | } 172 | break; 173 | } 174 | }; 175 | 176 | /** 177 | * Visit FORM. 178 | * 179 | * @param {jQuery} el 180 | * @api private 181 | */ 182 | 183 | View.prototype.visitFORM = function (el, name) { 184 | var self = this; 185 | this.submit = function (val) { 186 | switch (typeof val) { 187 | case 'function': 188 | el.submit(function (e) { 189 | val.call(self, e, el); 190 | return false; 191 | }); 192 | break; 193 | } 194 | } 195 | }; 196 | 197 | /** 198 | * Visit A tag. 199 | * 200 | * @param {jQuery} el 201 | * @api private 202 | */ 203 | 204 | View.prototype.visitA = function (el, name) { 205 | var self = this; 206 | 207 | el.click(function (e) { 208 | self.emit(name, e, el); 209 | }); 210 | 211 | this[name] = function (fn) { 212 | el.click(function (e) { 213 | fn.call(self, e, el); 214 | return false; 215 | }); 216 | return this; 217 | } 218 | }; 219 | 220 | /** 221 | * Visit P, TD, SPAN, or DIV tag. 222 | * 223 | * @param {jQuery} el 224 | * @api private 225 | */ 226 | 227 | View.prototype.visitP = 228 | View.prototype.visitTD = 229 | View.prototype.visitSPAN = 230 | View.prototype.visitDIV = function (el, name) { 231 | var self = this; 232 | this[name] = function (val) { 233 | if (0 == arguments.length) return el; 234 | el.empty().append(val.el || val); 235 | return this; 236 | }; 237 | }; 238 | 239 | /** 240 | * Visit UL tag. 241 | * 242 | * @param {jQuery} el 243 | * @api private 244 | */ 245 | 246 | View.prototype.visitUL = function (el, name) { 247 | var self = this; 248 | this.children = []; 249 | 250 | this[name] = el; 251 | 252 | // TODO: move these out 253 | 254 | /** 255 | * Add `val` to this list. 256 | * 257 | * @param {String|jQuery|View} val 258 | * @return {View} for chaining 259 | * @api public 260 | */ 261 | 262 | el.add = function (val) { 263 | var li = $('
  • '); 264 | self.children.push(val); 265 | el.append(li.append(val.el || val)); 266 | return this; 267 | }; 268 | 269 | /** 270 | * Return the list item `View`s as an array. 271 | * 272 | * @return {Array} 273 | * @api public 274 | */ 275 | 276 | el.items = function () { 277 | return self.children; 278 | }; 279 | 280 | /** 281 | * Iterate the list `View`s, calling `fn(item, i)`. 282 | * 283 | * @param {Function} fn 284 | * @return {View} for chaining 285 | * @api public 286 | */ 287 | 288 | el.each = function (fn) { 289 | for (var i = 0, len = self.children.length; i < len; ++i) { 290 | fn(self.children[i], i); 291 | } 292 | return this; 293 | }; 294 | 295 | /** 296 | * Map the list `View`s, calling `fn(item, i)`. 297 | * 298 | * @param {String|function} fn 299 | * @return {Array} 300 | * @api public 301 | */ 302 | 303 | el.map = function (fn) { 304 | var ret = [] 305 | , fn = callback(fn); 306 | 307 | for (var i = 0, len = self.children.length; i < len; ++i) { 308 | ret.push(fn(self.children[i], i)); 309 | } 310 | 311 | return ret; 312 | }; 313 | }; 314 | 315 | /** 316 | * Visit TABLE. 317 | * 318 | * @param {jQuery} el 319 | * @api private 320 | */ 321 | 322 | View.prototype.visitTABLE = function (el, name) { 323 | this[name] = el; 324 | 325 | this[name].add = function (val) { 326 | this.append(val.el || val); 327 | }; 328 | }; 329 | 330 | /** 331 | * Visit CANVAS. 332 | * 333 | * @param {jQuery} el 334 | * @api private 335 | */ 336 | 337 | View.prototype.visitCANVAS = function (el, name) { 338 | this[name] = el.get(0); 339 | }; 340 | 341 | /** 342 | * Visit H1-H5 tags. 343 | * 344 | * @param {jQuery} el 345 | * @api private 346 | */ 347 | 348 | View.prototype.visitH1 = 349 | View.prototype.visitH2 = 350 | View.prototype.visitH3 = 351 | View.prototype.visitH4 = 352 | View.prototype.visitH5 = function (el, name) { 353 | var self = this; 354 | this[name] = function (val) { 355 | if (0 == arguments.length) return el.text(); 356 | el.text(val.el || val); 357 | return this; 358 | }; 359 | }; 360 | 361 | /** 362 | * Remove the view from the DOM. 363 | * 364 | * @return {View} 365 | * @api public 366 | */ 367 | 368 | View.prototype.remove = function () { 369 | var parent = this.el.parent() 370 | , type = parent.get(0).nodeName; 371 | if ('LI' == type) parent.remove(); 372 | else this.el.remove(); 373 | return this; 374 | }; 375 | 376 | /** 377 | * Append this view's element to `val`. 378 | * 379 | * @param {String|jQuery} val 380 | * @return {View} 381 | * @api public 382 | */ 383 | 384 | View.prototype.appendTo = function (val) { 385 | this.el.appendTo(val.el || val); 386 | return this; 387 | }; 388 | -------------------------------------------------------------------------------- /lib/http/public/stylesheets/main.css: -------------------------------------------------------------------------------- 1 | body { 2 | padding: 50px 120px; 3 | } 4 | #menu { 5 | margin: 0; 6 | padding: 0; 7 | position: fixed; 8 | top: 0; 9 | left: 0; 10 | height: 100%; 11 | width: 80px; 12 | background: #3b3b3b; 13 | border-right: 1px solid #232323; 14 | -webkit-box-shadow: 0 0 0 1px rgba(255,255,255,0.5); 15 | box-shadow: 0 0 0 1px rgba(255,255,255,0.5); 16 | } 17 | #menu li { 18 | margin: 0; 19 | list-style: none; 20 | } 21 | #menu li { 22 | position: relative; 23 | text-align: center; 24 | } 25 | #menu li .count { 26 | position: absolute; 27 | top: 15px; 28 | left: 0; 29 | text-shadow: 1px 1px 1px #333; 30 | width: 100%; 31 | color: #808080; 32 | } 33 | #menu li a { 34 | display: block; 35 | padding: 40px 0 10px 0; 36 | color: #777; 37 | border-top: 1px solid #545454; 38 | border-bottom: 1px solid #333; 39 | font-size: 12px; 40 | background: #393939; 41 | } 42 | #menu li a:hover { 43 | background: #434343; 44 | } 45 | #menu li a:active, 46 | #menu li a.active { 47 | background: #343434; 48 | -webkit-box-shadow: inset 0 0 3px 2px #282828, inset 0 -5px 10px 2px #303030; 49 | box-shadow: inset 0 0 3px 2px #282828, inset 0 -5px 10px 2px #303030; 50 | border-bottom: 1px solid #222; 51 | } 52 | .context-menu { 53 | display: none; 54 | margin: 0; 55 | padding: 0; 56 | border: 1px solid #eee; 57 | border-bottom-color: rgba(0,0,0,0.25); 58 | border-left-color: rgba(0,0,0,0.2); 59 | border-right-color: rgba(0,0,0,0.2); 60 | -webkit-box-shadow: 0 2px 2px 0 rgba(0,0,0,0.1); 61 | box-shadow: 0 2px 2px 0 rgba(0,0,0,0.1); 62 | -webkit-border-radius: 4px; 63 | border-radius: 4px; 64 | } 65 | .context-menu li { 66 | margin: 0; 67 | list-style: none; 68 | } 69 | .context-menu li:last-child a { 70 | border-bottom: none; 71 | } 72 | .context-menu li a { 73 | display: block; 74 | background: #fff; 75 | padding: 5px 10px; 76 | border: 1px solid transparent; 77 | border-bottom: 1px solid #eee; 78 | font-size: 12px; 79 | } 80 | .context-menu li a:hover { 81 | background: -webkit-linear-gradient(bottom, #00b3e9, #74dfff); 82 | background: -moz-linear-gradient(bottom, #00b3e9, #74dfff); 83 | background: -o-linear-gradient(bottom, #00b3e9, #74dfff); 84 | background: -ms-linear-gradient(bottom, #00b3e9, #74dfff); 85 | background: linear-gradient(to top, #00b3e9, #74dfff); 86 | color: #fff; 87 | border: 1px solid #fff; 88 | } 89 | .context-menu li a:active { 90 | background: -webkit-linear-gradient(bottom, #06c5ff, #83e2ff); 91 | background: -moz-linear-gradient(bottom, #06c5ff, #83e2ff); 92 | background: -o-linear-gradient(bottom, #06c5ff, #83e2ff); 93 | background: -ms-linear-gradient(bottom, #06c5ff, #83e2ff); 94 | background: linear-gradient(to top, #06c5ff, #83e2ff); 95 | } 96 | #job-template { 97 | display: none; 98 | } 99 | .block { 100 | border: 1px solid #eee; 101 | border-bottom-color: rgba(0,0,0,0.25); 102 | border-left-color: rgba(0,0,0,0.2); 103 | border-right-color: rgba(0,0,0,0.2); 104 | -webkit-box-shadow: 0 2px 2px 0 rgba(0,0,0,0.1); 105 | box-shadow: 0 2px 2px 0 rgba(0,0,0,0.1); 106 | -webkit-border-radius: 4px; 107 | border-radius: 4px; 108 | width: 90%; 109 | margin: 10px 25px; 110 | padding: 20px 25px; 111 | } 112 | .block h2 { 113 | margin: 0; 114 | position: absolute; 115 | top: 5px; 116 | left: -15px; 117 | padding: 5px; 118 | font-size: 10px; 119 | -webkit-border-top-left-radius: 5px; 120 | border-top-left-radius: 5px; 121 | -webkit-border-bottom-left-radius: 5px; 122 | border-bottom-left-radius: 5px; 123 | -webkit-border-top-right-radius: 2px; 124 | border-top-right-radius: 2px; 125 | -webkit-border-bottom-right-radius: 2px; 126 | border-bottom-right-radius: 2px; 127 | background: -webkit-linear-gradient(left, #6b6b6b, #7e7e7e 50%); 128 | background: -moz-linear-gradient(left, #6b6b6b, #7e7e7e 50%); 129 | background: -o-linear-gradient(left, #6b6b6b, #7e7e7e 50%); 130 | background: -ms-linear-gradient(left, #6b6b6b, #7e7e7e 50%); 131 | background: linear-gradient(to right, #6b6b6b, #7e7e7e 50%); 132 | -webkit-box-shadow: -1px 0 1px 1px rgba(0,0,0,0.1); 133 | box-shadow: -1px 0 1px 1px rgba(0,0,0,0.1); 134 | color: #fff; 135 | text-shadow: 1px 1px 1px #444; 136 | } 137 | .block .type { 138 | color: #929292; 139 | } 140 | .job td.title em { 141 | color: #929292; 142 | } 143 | .job .block { 144 | position: relative; 145 | background: #fff; 146 | cursor: pointer; 147 | } 148 | .job .block table td:first-child { 149 | display: none; 150 | } 151 | .job .block .progress { 152 | position: absolute; 153 | top: 15px; 154 | right: 20px; 155 | } 156 | .job .block .attempts { 157 | display: none; 158 | position: absolute; 159 | top: 0; 160 | right: 0; 161 | padding: 5px 8px; 162 | -webkit-border-radius: 2px; 163 | border-radius: 2px; 164 | font-size: 10px; 165 | } 166 | .job .block .remove { 167 | position: absolute; 168 | top: 30px; 169 | right: -6px; 170 | /*background: white*/ 171 | background: #f05151; 172 | color: #fff; 173 | display: block; 174 | width: 20px; 175 | height: 20px; 176 | line-height: 20px; 177 | text-align: center; 178 | font-size: 12px; 179 | font-weight: bold; 180 | outline: none; 181 | border: 1px solid #eee; 182 | -webkit-border-radius: 20px; 183 | border-radius: 20px; 184 | -webkit-transition: opacity 200ms, top 300ms; 185 | -moz-transition: opacity 200ms, top 300ms; 186 | -o-transition: opacity 200ms, top 300ms; 187 | -ms-transition: opacity 200ms, top 300ms; 188 | transition: opacity 200ms, top 300ms; 189 | opacity: 0; 190 | -ms-filter: "progid:DXImageTransform.Microsoft.Alpha(Opacity=0)"; 191 | filter: alpha(opacity=0); 192 | } 193 | .job .block .remove:hover { 194 | border: 1px solid #d6d6d6; 195 | } 196 | .job .block .remove:active { 197 | border: 1px solid #bebebe; 198 | } 199 | .job .block .restart { 200 | position: absolute; 201 | top: 30px; 202 | right: -6px; 203 | /*background: white*/ 204 | background: #00e600; 205 | color: #fff; 206 | display: block; 207 | width: 20px; 208 | height: 20px; 209 | line-height: 20px; 210 | text-align: center; 211 | font-size: 12px; 212 | font-weight: bold; 213 | outline: none; 214 | border: 1px solid #eee; 215 | -webkit-border-radius: 20px; 216 | border-radius: 20px; 217 | -webkit-transition: opacity 200ms, top 300ms; 218 | -moz-transition: opacity 200ms, top 300ms; 219 | -o-transition: opacity 200ms, top 300ms; 220 | -ms-transition: opacity 200ms, top 300ms; 221 | transition: opacity 200ms, top 300ms; 222 | opacity: 0; 223 | -ms-filter: "progid:DXImageTransform.Microsoft.Alpha(Opacity=0)"; 224 | filter: alpha(opacity=0); 225 | } 226 | .job .block .restart:hover { 227 | border: 1px solid #d6d6d6; 228 | } 229 | .job .block .restart:active { 230 | border: 1px solid #bebebe; 231 | } 232 | .job .block:hover .remove { 233 | opacity: 1; 234 | -ms-filter: none; 235 | filter: none; 236 | top: -6px; 237 | } 238 | .job .block:hover .restart { 239 | opacity: 1; 240 | -ms-filter: none; 241 | filter: none; 242 | top: 16px; 243 | } 244 | .job .details { 245 | background: #3b3b3b; 246 | width: 89%; 247 | margin-top: -10px; 248 | margin-left: 35px; 249 | -webkit-border-bottom-left-radius: 5px; 250 | border-bottom-left-radius: 5px; 251 | -webkit-border-bottom-right-radius: 5px; 252 | border-bottom-right-radius: 5px; 253 | -webkit-box-shadow: inset 0 1px 10px 0 rgba(0,0,0,0.8); 254 | box-shadow: inset 0 1px 10px 0 rgba(0,0,0,0.8); 255 | -webkit-transition: padding 200ms, height 200ms; 256 | -moz-transition: padding 200ms, height 200ms; 257 | -o-transition: padding 200ms, height 200ms; 258 | -ms-transition: padding 200ms, height 200ms; 259 | transition: padding 200ms, height 200ms; 260 | height: 0; 261 | overflow: hidden; 262 | } 263 | .job .details table { 264 | width: 100%; 265 | } 266 | .job .details table td:first-child { 267 | width: 60px; 268 | color: #949494; 269 | } 270 | .job .details.show { 271 | padding: 15px 20px; 272 | height: auto; 273 | } 274 | .job ul.log { 275 | margin: 0; 276 | padding: 0; 277 | margin: 5px; 278 | padding: 10px; 279 | max-height: 100px; 280 | overflow-y: auto; 281 | -webkit-border-radius: 5px; 282 | border-radius: 5px; 283 | width: 95%; 284 | } 285 | .job ul.log li { 286 | margin: 0; 287 | list-style: none; 288 | } 289 | .job ul.log li { 290 | padding: 5px 0; 291 | border-bottom: 1px dotted #424242; 292 | color: #666; 293 | } 294 | .job ul.log li:last-child { 295 | border-bottom: none; 296 | } 297 | .job .details ::-webkit-scrollbar { 298 | width: 2px; 299 | } 300 | .job .details ::-webkit-scrollbar-thumb:vertical { 301 | background: #858585; 302 | } 303 | .job .details ::-webkit-scrollbar-track { 304 | border: 1px solid rgba(255,255,255,0.1); 305 | } 306 | .job .details > div { 307 | padding: 10px 0; 308 | border-bottom: 1px solid #424242; 309 | } 310 | .job .details > div:last-child { 311 | border-bottom: none; 312 | } 313 | #actions { 314 | position: fixed; 315 | top: -2px; 316 | right: -2px; 317 | z-index: 20; 318 | } 319 | #sort, 320 | #filter, 321 | #search { 322 | float: left; 323 | margin: 0; 324 | padding: 5px 10px; 325 | border: 1px solid #eee; 326 | -webkit-border-radius: 0 0 0 5px; 327 | border-radius: 0 0 0 5px; 328 | -webkit-appearance: none; 329 | color: #3b3b3b; 330 | outline: none; 331 | } 332 | #sort:hover, 333 | #filter:hover, 334 | #search:hover { 335 | border-color: #d6d6d6; 336 | } 337 | #sort, 338 | #filter { 339 | cursor: pointer; 340 | } 341 | #sort, 342 | #filter { 343 | -webkit-border-radius: 0; 344 | border-radius: 0; 345 | border-left: none; 346 | } 347 | #error { 348 | position: fixed; 349 | top: -50px; 350 | right: 15px; 351 | padding: 20px; 352 | -webkit-transition: top 500ms, opacity 500ms; 353 | -moz-transition: top 500ms, opacity 500ms; 354 | -o-transition: top 500ms, opacity 500ms; 355 | -ms-transition: top 500ms, opacity 500ms; 356 | transition: top 500ms, opacity 500ms; 357 | opacity: 0; 358 | -ms-filter: "progid:DXImageTransform.Microsoft.Alpha(Opacity=0)"; 359 | filter: alpha(opacity=0); 360 | background: rgba(59,59,59,0.2); 361 | border: 1px solid rgba(59,59,59,0.3); 362 | -webkit-border-radius: 5px; 363 | border-radius: 5px; 364 | color: #3b3b3b; 365 | } 366 | #error.show { 367 | top: 15px; 368 | opacity: 1; 369 | -ms-filter: none; 370 | filter: none; 371 | } 372 | body { 373 | font: 13px "helvetica neue", helvetica, arial, sans-serif; 374 | -webkit-font-smoothing: antialiased; 375 | background: #fff; 376 | color: #666; 377 | } 378 | h1, 379 | h2, 380 | h3 { 381 | margin: 0 0 25px 0; 382 | padding: 0; 383 | font-weight: normal; 384 | text-transform: capitalize; 385 | color: #666; 386 | } 387 | h2 { 388 | font-size: 16px; 389 | margin-top: 20px; 390 | } 391 | pre { 392 | margin-top: 20px; 393 | } 394 | a { 395 | text-decoration: none; 396 | cursor: pointer; 397 | } 398 | table { 399 | border-collapse: separate; 400 | border-spacing: 0; 401 | vertical-align: middle; 402 | } 403 | table tr td { 404 | padding: 2px 5px; 405 | } 406 | #loading { 407 | width: 100%; 408 | text-align: center; 409 | margin-top: 40px; 410 | margin-left: 20px; 411 | } 412 | #loading canvas { 413 | margin: 0 auto; 414 | } 415 | -------------------------------------------------------------------------------- /lib/queue/worker.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * kue - Worker 3 | * Copyright (c) 2013 Automattic 4 | * Copyright (c) 2011 LearnBoost 5 | * MIT Licensed 6 | * Author: behradz@gmail.com 7 | */ 8 | 9 | /** 10 | * Module dependencies. 11 | */ 12 | 13 | var EventEmitter = require('events').EventEmitter 14 | , redis = require('../redis') 15 | , events = require('./events') 16 | , Job = require('./job') 17 | , noop = function() {}; 18 | 19 | /** 20 | * Expose `Worker`. 21 | */ 22 | 23 | module.exports = Worker; 24 | 25 | /** 26 | * Redis connections used by `getJob()` when blocking. 27 | */ 28 | 29 | var clients = {}; 30 | 31 | /** 32 | * Initialize a new `Worker` with the given Queue 33 | * targetting jobs of `type`. 34 | * 35 | * @param {Queue} queue 36 | * @param {String} type 37 | * @api private 38 | */ 39 | 40 | function Worker( queue, type ) { 41 | this.queue = queue; 42 | this.type = type; 43 | this.client = Worker.client || (Worker.client = redis.createClient()); 44 | this.running = true; 45 | this.job = null; 46 | } 47 | 48 | /** 49 | * Inherit from `EventEmitter.prototype`. 50 | */ 51 | 52 | Worker.prototype.__proto__ = EventEmitter.prototype; 53 | 54 | /** 55 | * Start processing jobs with the given `fn`, 56 | * 57 | * @param {Function} fn 58 | * @return {Worker} for chaining 59 | * @api private 60 | */ 61 | 62 | Worker.prototype.start = function( fn ) { 63 | var self = this; 64 | self.idle(); 65 | if( !self.running ) return; 66 | 67 | if (self.ttlExceededCb) 68 | self.queue.removeListener('job ttl exceeded', self.ttlExceededCb); 69 | 70 | self.ttlExceededCb = function(id) { 71 | if( self.job && self.job.id && self.job.id === id ) { 72 | self.failed( self.job, { error: true, message: 'TTL exceeded' }, fn ); 73 | events.emit(id, 'ttl exceeded ack'); 74 | } 75 | } 76 | 77 | /* 78 | listen if current job ttl received, 79 | so that this worker can fail current stuck job and continue, 80 | in case user's process callback is stuck and done is not called in time 81 | */ 82 | this.queue.on( 'job ttl exceeded', self.ttlExceededCb); 83 | 84 | self.getJob(function( err, job ) { 85 | if( err ) self.error(err, job); 86 | if( !job || err ) return process.nextTick(function() { 87 | self.start(fn); 88 | }); 89 | self.process(job, fn); 90 | }); 91 | return this; 92 | }; 93 | 94 | /** 95 | * Error handler, currently does nothing. 96 | * 97 | * @param {Error} err 98 | * @param {Job} job 99 | * @return {Worker} for chaining 100 | * @api private 101 | */ 102 | 103 | Worker.prototype.error = function( err, job ) { 104 | this.emit('error', err, job); 105 | return this; 106 | }; 107 | 108 | /** 109 | * Process a failed `job`. Set's the job's state 110 | * to "failed" unless more attempts remain, in which 111 | * case the job is marked as "inactive" or "delayed" 112 | * and remains in the queue. 113 | * 114 | * @param {Job} job 115 | * @param {Object} theErr 116 | * @param {Function} fn 117 | * @return {Worker} for chaining 118 | * @api private 119 | */ 120 | 121 | Worker.prototype.failed = function( job, theErr, fn ) { 122 | var self = this; 123 | job.failedAttempt( theErr, function( err, hasAttempts, attempt ) { 124 | if( err ) return self.error(err, job); 125 | if( hasAttempts ) { 126 | self.emitJobEvent( 'failed attempt', job, theErr.message || theErr.toString(), attempt ); 127 | } else { 128 | self.emitJobEvent( 'failed', job, theErr.message || theErr.toString() ); 129 | } 130 | fn && self.start(fn); 131 | }); 132 | return this; 133 | }; 134 | 135 | /** 136 | * Process `job`, marking it as active, 137 | * invoking the given callback `fn(job)`, 138 | * if the job fails `Worker#failed()` is invoked, 139 | * otherwise the job is marked as "complete". 140 | * 141 | * @param {Job} job 142 | * @param {Function} fn 143 | * @return {Worker} for chaining 144 | * @api public 145 | */ 146 | 147 | Worker.prototype.process = function( job, fn ) { 148 | var self = this 149 | , start = new Date(); 150 | 151 | this.job = job; 152 | job.set( 'started_at', job.started_at = start.getTime() ); 153 | job.set( 'workerId', job.workerId = this.id ); 154 | /* 155 | store job.id around given done to the caller, 156 | so that we can later match against it when done is called 157 | */ 158 | var createDoneCallback = function( jobId ) { 159 | return function( err, result ) { 160 | if( self.drop_user_callbacks ) { 161 | //console.warn( 'Worker started to shutdown, ignoring execution of done callback' ); 162 | //job.log( 'Worker started to shutdown, ignoring execution of done callback' ); 163 | return; 164 | } 165 | /* 166 | if no job in hand, or the current job in hand 167 | doesn't match called done callback's jobId 168 | then ignore running callers done. 169 | */ 170 | if( self.job === null || self.job && self.job.id && self.job.id !== jobId ) { 171 | //console.warn( 'This job has already been finished, ignoring execution of done callback' ); 172 | //job.log( 'This job has already been finished, ignoring execution of done callback' ); 173 | return; 174 | } 175 | if( err ) { 176 | return self.failed(job, err, fn); 177 | } 178 | job.set('duration', job.duration = new Date - start); 179 | if( result ) { 180 | try { 181 | job.result = result; 182 | job.set('result', JSON.stringify(result), noop); 183 | } catch(e) { 184 | job.set('result', JSON.stringify({ error: true, message: 'Invalid JSON Result: "' + result + '"' }), noop); 185 | } 186 | } 187 | job.complete(function() { 188 | job.attempt(function() { 189 | if( job.removeOnComplete() ) { 190 | job.remove(); 191 | } 192 | self.emitJobEvent('complete', job, result); 193 | self.start(fn); 194 | }); 195 | }.bind(this)); 196 | }; 197 | }; 198 | 199 | var doneCallback = createDoneCallback( job.id ); 200 | 201 | var workerCtx = { 202 | /** 203 | * @author behrad 204 | * @pause: let the processor to tell worker not to continue processing new jobs 205 | */ 206 | pause: function( timeout, fn ) { 207 | if( arguments.length === 1 ) { 208 | fn = timeout; 209 | timeout = 5000; 210 | } 211 | self.queue.shutdown(Number(timeout), self.type, fn); 212 | }, 213 | /** 214 | * @author behrad 215 | * @pause: let the processor to trigger restart for they job processing 216 | */ 217 | resume: function() { 218 | if( self.resume() ) { 219 | self.start(fn); 220 | } 221 | }, 222 | shutdown: function() { 223 | self.shutdown(); 224 | } 225 | }; 226 | 227 | job.active(function() { 228 | self.emitJobEvent('start', job, job.type); 229 | if( fn.length === 2 ) { // user provided a two argument function, doesn't need workerCtx 230 | fn(job, doneCallback); 231 | } else { // user wants workerCtx parameter, make done callback the last 232 | fn(job, workerCtx, doneCallback); 233 | } 234 | }.bind(this)); 235 | 236 | return this; 237 | }; 238 | 239 | /** 240 | * Atomic ZPOP implementation. 241 | * 242 | * @param {String} key 243 | * @param {Function} fn 244 | * @api private 245 | */ 246 | 247 | Worker.prototype.zpop = function( key, fn ) { 248 | this.client 249 | .multi() 250 | .zrange(key, 0, 0) 251 | .zremrangebyrank(key, 0, 0) 252 | .exec(function( err, res ) { 253 | if( err || !res || !res[ 0 ] || !res[ 0 ].length ) return fn(err); 254 | var id = res[ 0 ][ 0 ] || res[ 0 ][ 1 ][ 0 ]; 255 | fn(null, this.client.stripFIFO(id)); 256 | }.bind(this)); 257 | }; 258 | 259 | /** 260 | * Attempt to fetch the next job. 261 | * 262 | * @param {Function} fn 263 | * @api private 264 | */ 265 | 266 | Worker.prototype.getJob = function( fn ) { 267 | var self = this; 268 | if( !self.running ) { 269 | return fn('Already Shutdown'); 270 | } 271 | // alloc a client for this job type 272 | var client = clients[ self.type ] || (clients[ self.type ] = redis.createClient()); 273 | // BLPOP indicates we have a new inactive job to process 274 | client.blpop(client.getKey(self.type + ':jobs'), 0, function( err ) { 275 | if( err || !self.running ) { 276 | if( self.client && self.client.connected && !self.client.closing ) { 277 | self.client.lpush(self.client.getKey(self.type + ':jobs'), 1, noop); 278 | } 279 | return fn(err); // SAE: Added to avoid crashing redis on zpop 280 | } 281 | // Set job to a temp value so shutdown() knows to wait 282 | self.job = true; 283 | self.zpop(self.client.getKey('jobs:' + self.type + ':inactive'), function( err, id ) { 284 | if( err || !id ) { 285 | self.idle(); 286 | return fn(err /*|| "No job to pop!"*/); 287 | } 288 | Job.get(id, fn); 289 | }); 290 | }); 291 | }; 292 | 293 | /** 294 | * emits worker idle event and nullifies current job in hand 295 | */ 296 | 297 | Worker.prototype.idle = function() { 298 | this.job = null; 299 | this.emit('idle'); 300 | return this; 301 | }; 302 | 303 | /** 304 | * Gracefully shut down the worker 305 | * 306 | * @param {Function} fn 307 | * @param {int} timeout 308 | * @api private 309 | */ 310 | 311 | Worker.prototype.shutdown = function( timeout, fn ) { 312 | var self = this, shutdownTimer = null; 313 | if( arguments.length === 1 ) { 314 | fn = timeout; 315 | timeout = null; 316 | } 317 | 318 | // Wrap `fn` so we don't pass `job` to it 319 | var _fn = function( job ) { 320 | if( job && self.job && job.id != self.job.id ) { 321 | return; // simply ignore older job events currently being received until the right one comes... 322 | } 323 | shutdownTimer && clearTimeout(shutdownTimer); 324 | self.removeAllListeners(); 325 | self.job = null; 326 | //Safeyly kill any blpop's that are waiting. 327 | (self.type in clients) && clients[ self.type ].quit(); 328 | delete clients[ self.type ]; 329 | self.cleaned_up = true; 330 | //fix half-blob job fetches if any 331 | self.client.lpush(self.client.getKey(self.type + ':jobs'), 1, fn || noop); 332 | }; 333 | 334 | if( !this.running ) return _fn(); 335 | this.running = false; 336 | 337 | // As soon as we're free, signal that we're done 338 | if( !this.job ) { 339 | return _fn(); 340 | } 341 | this.on('idle', _fn); 342 | this.on('job complete', _fn); 343 | this.on('job failed', _fn); 344 | this.on('job failed attempt', _fn); 345 | 346 | if( timeout ) { 347 | shutdownTimer = setTimeout(function() { 348 | // shutdown timeout reached... 349 | if( self.job ) { 350 | self.drop_user_callbacks = true; 351 | self.removeAllListeners(); 352 | if( self.job === true ) { 353 | self.once('idle', _fn); 354 | } else { 355 | // a job is running, fail it and call _fn when failed 356 | self.once('job failed', _fn); 357 | self.once('job failed attempt', _fn); 358 | self.failed(self.job, { error: true, message: 'Shutdown' }); 359 | } 360 | } else { 361 | // no job running, just finish immediately 362 | _fn(); 363 | } 364 | }.bind(this), timeout); 365 | } 366 | }; 367 | 368 | Worker.prototype.emitJobEvent = function( event, job, arg1, arg2 ) { 369 | if( this.cleaned_up ) return; 370 | events.emit(job.id, event, arg1, arg2); 371 | this.emit('job ' + event, job); 372 | }; 373 | 374 | Worker.prototype.resume = function() { 375 | if( this.running ) return false; 376 | this.cleaned_up = false; 377 | this.drop_user_callbacks = false; 378 | this.running = true; 379 | return true; 380 | }; 381 | -------------------------------------------------------------------------------- /History.md: -------------------------------------------------------------------------------- 1 | 0.11.5 / 2016-11-05 2 | =================== 3 | 4 | * Fix even more redis command callbacks 5 | * Fix redis commands SLC integration #978 6 | 7 | 8 | 0.11.4 / 2016-10-21 9 | =================== 10 | 11 | * adding reds module to optional dependencies 12 | 13 | 14 | 0.11.3 / 2016-10-21 15 | =================== 16 | 17 | * Fix making reds module optional, #969 18 | 19 | 20 | 0.11.2 / 2016-10-14 21 | =================== 22 | 23 | * Update packages to remove CVEs, #932 24 | * Make reds an optional dependency, #922 25 | * Remove unnecessary dependency to lodash-deep, #921 26 | * Expose shutdown in process worker ctx, #912 27 | * Add ioredis support to watchStuckJobs, #884 28 | 29 | 30 | 0.11.1 / 2016-06-15 31 | =================== 32 | 33 | * Upgrade redis to 2.6 34 | * Add switch for each job event 35 | 36 | 37 | 0.11.0 / 2016-05-13 38 | =================== 39 | 40 | * force node_redis version to 2.4.x, Closes #857 41 | * Converting Job ids back into integers, #855 42 | * Fix LPUSH crash during shutdown, #854 43 | * Install kue-dashboard script, #853 44 | * Add start event to documentation, #841 45 | * Add parameter for testMode.enter to continue processing jobs, #821 46 | * Modern Node.js versions support, #812 47 | * Don't start the next job until the current one is totally finished, Closes #806 48 | * Store multiple instances of jobs in jobs id map to emit events for all, #750 49 | 50 | 51 | 0.10.6 / 2016-04-27 52 | =================== 53 | 54 | * Redis Cluster fix, Closes #861 55 | 56 | 57 | 0.10.5 / 2016-01-14 58 | =================== 59 | 60 | * Attempts surpassing max attempts on delay jobs upon failure, resulting in infinite retries, Fixes #797 61 | * Add yargs dependency for kue-dashboard, #796 62 | 63 | 64 | 0.10.4 / 2016-01-14 65 | =================== 66 | 67 | * fix zpop callback on shutdown 68 | * fix connection_options in test.js 69 | * Unit tests for redis.js #779 70 | * Tests for kue.js #778 71 | 72 | 73 | 0.10.3 / 2015-11-20 74 | =================== 75 | 76 | * Fixing Job processing order without munging the job id, Closes #708, Closes #678 77 | 78 | 79 | 0.10.2 / 2015-11-20 80 | =================== 81 | 82 | * Add support for ioredis, Closes #652 83 | * Add support for Redis Cluster, Closes #642 84 | * Fix `this.state` on refreshTTL 85 | 86 | 87 | 0.10.0 / 2015-11-20 88 | =================== 89 | 90 | * Update TTL on job progress, Closes #694 91 | * Upgrade to node_redis 2.3, #717 92 | * Fix LPUSH vs connection quit race when shutting down 93 | * Restart task btn, #754 94 | * Fix uncaught exception in job.js, #751 95 | * Added kue-dashboard script for conveniently running the dashboard #611 96 | * Fixed invalid CSS on production, #755 97 | * Connection string not supporting DB number #725 98 | * Fix attempts remaining logic, #742 99 | * Update jade, #741 100 | * Properly set job IDs in test mode, #727 101 | * Enhanced Job.log formatting, #630 102 | * Use node's util#format() in Job.log, #724 103 | 104 | 105 | 0.9.6 / 2015-10-06 106 | =================== 107 | 108 | * Fix redirection issue 109 | 110 | 111 | 0.9.5 / 2015-09-16 112 | =================== 113 | 114 | * When no ttl is set for jobs, don't let high priorities to conflict, fixes #697 115 | * Fix redirection issue, closes #685 116 | * Get progress_data along with other redis fields, PR #642 117 | * Grab only password from Redis URL, fixes #681 118 | * Add remove job event, PR #665 119 | 120 | 121 | 0.9.4 / 2015-07-17 122 | =================== 123 | 124 | * Job that doesn't call done() retries twice, fixes #669 125 | 126 | 127 | 0.9.3 / 2015-05-07 128 | =================== 129 | 130 | * Fix unlocking promotion lock, Closes #608 131 | 132 | 133 | 0.9.2 / 2015-05-07 134 | =================== 135 | 136 | * Fix duplicate job promotion/ttl race, Closes #601 137 | 138 | 139 | 0.9.1 / 2015-05-05 140 | =================== 141 | 142 | * Filter only jobs that have ttl set, Fixes #590 143 | 144 | 145 | 0.9.0 / 2015-05-02 146 | =================== 147 | 148 | * Upgrade to express 4.x, Closes #537 149 | * Move `job.reprocess` done callback to the last, Closes #387, Closes #385 150 | * Standardize signature of `.shutdown()` callback, Closes #454 151 | * Turn off search indexes by default, Closes #412 152 | * Improve delayed job promotion feature, Closes #533, fixes #312, closes #352 153 | * Use a distributed redis lock to hide job promotion from user, Closes #556 154 | * Deprecate `.promote` and update documentation 155 | * Document Javascript API to query queue state, Closes #455 156 | * Add jobEvents flag to switch off job events for memory optimization, Closes #401 157 | * Add idle event to capture unsuccessful zpop's in between of worker get Job, should fix #538 158 | * Add TTL for active jobs, Closes #544 159 | * Document `jobEvents` queue config, Closes #557 160 | * Bulk job create API now processes all jobs in case of intermediate errors, Closes #552 161 | * Merge `red job remove buttons and tooltips` PR, Closes #566 162 | * Add a in-memory test Kue mode, Closes #561 163 | * Update reds package to `0.2.5` 164 | * Merge PR #594, bad redirect URL in old express versions, fixes #592 165 | * update dependency to forked warlock repo to fix redis connection cleanup on shutdown, fixes #578 166 | * Update job hash with the worker ID, Closes #580 167 | 168 | 169 | 0.8.12 / 2015-03-22 170 | =================== 171 | 172 | * Bulk job create JSON API, Closes #334, Closes #500, Closes #527 173 | * Add feature to specify redis connection string/url, Closes #540 174 | * Mention kue-ui in readme, Closes #502 175 | * Add an extra parameter to the progress method to notify extra contextual data, Closes #466, Closes #427, Closes #313 176 | * Document job event callback arguments, Closes #542 177 | * Fix typo in documentation, Closes #506 178 | * Document importance of using Kue `error` listeners, Closes #409 179 | * Document Queue maintenance and job.removeOnComplete( true ), Closes #439 180 | * Document how to query all the active jobs programmatically, Closes #418 181 | * Document to explain how "stuck queued jobs" happens, Closes #451 182 | * Document on proper error handling to prevent stuck jobs, Closes #391 183 | 184 | 185 | 0.8.11 / 2014-12-15 186 | =================== 187 | 188 | * Fix shutdown on re-attemptable jobs, Closes #469 189 | * Fix race condition in delaying jobs when re-attempts, Closes #483 190 | * Make `watchStuckJobs` aware of queue prefix, Closes #452 191 | * Send along error message when emitting a failed event, Closes #461 192 | 193 | 194 | 0.8.10 / 2014-12-13 195 | =================== 196 | 197 | * Add more tests, Closes #280 198 | * More atomic job state changes, Closes #411 199 | * Documentation: error passed to done should be string or standard JS error object, Closes #394 200 | * Documentation: backoff documentation, Closes #435 201 | * Documentation: correct `promote` usage, Closes #413 202 | * Add job enqueue event, Closes #458 203 | * Watch for errors with non-string err.stack, Closes #426 204 | * Fix web app redirect path for express 4.0, Closes #393 205 | * `removeBadJob` should do pessimistic job removal from all state ZSETs, Closes #438 206 | * Add stats json api by type and state, Closes #477 207 | * Don't let concurrent graceful shutdowns on subsequent`Queue#shutdown`calls, Closes #479 208 | * Fix `cleanup` global leak, Closes #475 209 | 210 | 211 | 0.8.9 / 2014-10-01 212 | ================== 213 | 214 | * Properly update status flags on resume, Closes #423 215 | 216 | 0.8.8 / 2014-09-12 217 | ================== 218 | 219 | * Fix tests to limited shutdown timeouts 220 | * Add a redis lua watchdog to fix stuck inactive jobs, fixes #130 221 | * Stuck inactive jobs watchdog, Closes #130 222 | 223 | 0.8.7 / 2014-09-12 224 | ================== 225 | 226 | * Shutdown timeout problems and races, fixes #406 227 | 228 | 0.8.6 / 2014-08-30 229 | ================== 230 | 231 | * Quit redis connections on shutdown & let the process exit, closes #398 232 | 233 | 0.8.5 / 2014-08-10 234 | ================== 235 | 236 | * Fix typo in removeOnComplete 237 | 238 | 0.8.4 / 2014-08-08 239 | ================== 240 | 241 | * Emit event 'job failed attempt' after job successfully updated, closes #377 242 | * Fix delaying jobs when failed, closes #384 243 | * Implement `job.removeOnComplete`, closes #383 244 | * Make searchKeys chainable, closes #379 245 | * Add extra job options to JSON API, closes #378 246 | 247 | 0.8.3 / 2014-07-13 248 | ================== 249 | 250 | * Inject other Redis clients compatible with node_redis #344 251 | * Add support to connect to Redis using Linux sockets #362 252 | * Add .save callback sample code in documentation #367 253 | 254 | 0.8.2 / 2014-07-08 255 | ================== 256 | 257 | * Fix broken failure backoff #360 258 | * Merge web console redirection fix #357 259 | * Add db selection option to redis configuration #354 260 | * Get number of jobs with given state and type #349 261 | * Add Queue.prototype.delayed function #351 262 | 263 | 0.8.1 / 2014-06-13 264 | ================== 265 | 266 | * Fix wrong parameter orders in complete event #343s 267 | * Graceful shutdown bug fix #328 268 | 269 | 0.8.0 / 2014-06-11 270 | ================== 271 | 272 | * Implement backoff on failure retries #300 273 | * Allow passing back worker results via done to event handlers #170 274 | * Allow job producer to specify which keys of `job.data` to be indexed for search #284 275 | * Waffle.io Badge #332 276 | * Dropping monkey-patch style redis client connections 277 | * Update docs: Worker Pause/Resume-ability 278 | * Update docs: Reliability of Queue event handlers over Job event handlers 279 | 280 | 0.7.9 / 2014-06-01 281 | ================== 282 | 283 | * Graceful shutdown bug fix #336 284 | * More robust graceful shutdown under heavy load #328 285 | 286 | 0.7.6 / 2014-05-02 287 | ================== 288 | 289 | * Fixed broken monkey-patch style redis connections #323 290 | 291 | 0.7.0 / 2014-01-24 292 | ================== 293 | 294 | * Suppress "undefined" messages on String errors. Closes #230 295 | * Fix cannot read property id of undefined errors. Closes #252 296 | * Parameterize limit of jobs checked in promotion cycles. Closes #244 297 | * Graceful shutdown 298 | * Worker pause/resume ability, Closes #163 299 | * Ensure event subscription before job save. Closes #179 300 | * Fix Queue singleton 301 | * Fix failed event being called in first attempt. Closes #142 302 | * Disable search (Search index memory leaks). See #58 & #218 303 | * Emit error events on both kue and job 304 | * JS/Coffeescript tests added (Mocha+Should) 305 | * Travis support added 306 | 307 | 308 | 0.6.2 / 2013-04-03 309 | ================== 310 | 311 | * Fix redirection to active for mounted apps 312 | 313 | 314 | 0.6.1 / 2013-03-25 315 | ================== 316 | 317 | * Fixed issue preventing polling for new jobs. Closes #192 318 | 319 | 320 | 0.6.0 / 2013-03-20 321 | ================== 322 | 323 | * Make pollForJobs actually use ms argument. Closes #158 324 | * Support delay over HTTP POST. Closes #165 325 | * Fix natural sorting. Closes #174 326 | * Update `updated_at` timestamp during `log`, `progress`, `attempt`, or `state` changes. Closes #188 327 | * Fix redirection to /active. Closes #190 328 | 329 | 0.5.0 / 2012-11-16 330 | ================== 331 | 332 | * add POST /job to create a job 333 | * fix /job/search hang 334 | 335 | 0.4.2 / 2012-11-08 336 | ================== 337 | 338 | * Revert "Fix delay() not really delaying" 339 | * Revert "If a job with a delay has more attempts, honor the original delay" 340 | 341 | 0.4.1 / 2012-09-25 342 | ================== 343 | 344 | * fix: if a job with a delay has more attempts, honor the original delay [mathrawka] 345 | 346 | 0.4.0 / 2012-06-28 347 | ================== 348 | 349 | * Added 0.8.0 support 350 | 351 | 0.3.4 / 2012-02-23 352 | ================== 353 | 354 | * Changed: reduce polling by using BLPOP to notify workers of activity [Davide Bertola] 355 | 356 | 0.3.3 / 2011-11-28 357 | ================== 358 | 359 | * Fixed: use relative stats route to support mounting [alexkwolfe] 360 | * Fixed 0.6.x support 361 | * Removed empty Makefile 362 | 363 | 0.3.2 / 2011-10-04 364 | ================== 365 | 366 | * Removed unnecessary "pooling" 367 | * Fixed multiple event emitting. Closes #73 368 | * Fixed menu styling 369 | 370 | 0.3.1 / 2011-08-25 371 | ================== 372 | 373 | * Fixed auto event subscription. Closes #68 374 | * Changed: one redis connection for all workers 375 | * Removed user-select: none from everything. Closes #50 376 | 377 | 0.3.0 / 2011-08-11 378 | ================== 379 | 380 | * Added search capabilities 381 | * Added `workTime` stat 382 | * Added removal of stale jobs example 383 | * Added Queue-level job events, useful for removing stale jobs etc. Closes * Changed: lazy load reds search [David Wood] 384 | * Fixed `Job#error` for modules that throw strings or emit `error` events with strings [guillermo] #51 385 | * Fixed `Job#remove(fn)` 386 | * Fixed proxy issue with paths, use relative paths [booo] 387 | 388 | 0.2.0 / 2011-07-25 389 | ================== 390 | 391 | * Added infinite scroll 392 | * Added delayed job support 393 | * Added configurable redis support [davidwood] 394 | * Added job windowing. Closes #28 395 | * Added `Job#delay(ms)` 396 | * Removed job scrollIntoView 397 | * Removed fancy scrollbar (for infinite scroll / windowing :( ) 398 | * Removed "More" button 399 | * Fixed z-index for actions 400 | * Fixed job mapping. Closes #43 401 | 402 | 0.1.0 / 2011-07-19 403 | ================== 404 | 405 | * Added exposing of progress via redis pubsub 406 | * Added pubsub job events "complete" and "failed" 407 | * Fixed: capping of progress > 100 == 100 408 | * UI: scroll details into view 409 | 410 | 0.0.3 / 2011-07-07 411 | ================== 412 | 413 | * Added caustic to aid in template management 414 | * Added job attempt support. Closes #31 415 | * Added `Job.attempts(n)` 416 | * Added minified jQuery 417 | * Added cluster integration docs. Closes #13 418 | * Added GET _/jobs/:from..:to_ to JSON API 419 | * Fixed: hide "More" on sort 420 | * Fixed: hide "More" on filter 421 | * Fixed: removed "error" emission, blows up when no one is listening 422 | 423 | 0.0.2 / 2011-07-05 424 | ================== 425 | 426 | * Added support to update state from UI. Closes #26 427 | * Added support to alter priority in UI. Closes #25 428 | * Added filtering by type. Closes #20 429 | 430 | 0.0.1 / 2011-07-04 431 | ================== 432 | 433 | * Initial release 434 | -------------------------------------------------------------------------------- /test/test.coffee: -------------------------------------------------------------------------------- 1 | _ = require 'lodash' 2 | async = require 'async' 3 | should = require 'should' 4 | kue = require '../' 5 | util = require 'util' 6 | 7 | describe 'Kue Tests', -> 8 | 9 | jobs = null 10 | Job = null 11 | 12 | beforeEach -> 13 | jobs = kue.createQueue({promotion:{interval:50}}) 14 | Job = kue.Job 15 | 16 | afterEach (done) -> 17 | jobs.shutdown 50, done 18 | 19 | # before (done) -> 20 | # jobs = kue.createQueue({promotion:{interval:100}}) 21 | # jobs.client.flushdb done 22 | 23 | # after (done) -> 24 | # jobs = kue.createQueue({promotion:{interval:100}}) 25 | # jobs.client.flushdb done 26 | 27 | 28 | describe 'Job Producer', -> 29 | it 'should save jobs having a new id', (done) -> 30 | job_data = 31 | title: 'Test Email Job' 32 | to: 'tj@learnboost.com' 33 | job = jobs.create('email-to-be-saved', job_data) 34 | jobs.process('email-to-be-saved', _.noop) 35 | job.save (err) -> 36 | job.id.should.be.an.instanceOf(Number) 37 | done err 38 | 39 | 40 | it 'should set worker id on job hash', (done) -> 41 | job_data = 42 | title: 'Test workerId Job' 43 | to: 'tj@learnboost.com' 44 | job = jobs.create('worker-id-test', job_data) 45 | jobs.process 'worker-id-test', (job, jdone)-> 46 | jdone() 47 | Job.get job.id, (err, j) -> 48 | j.toJSON().workerId.should.be.not.null; 49 | done() 50 | job.save() 51 | 52 | 53 | it 'should receive job complete event', (done) -> 54 | jobs.process 'email-to-be-completed', (job, done)-> 55 | done() 56 | job_data = 57 | title: 'Test Email Job' 58 | to: 'tj@learnboost.com' 59 | jobs.create('email-to-be-completed', job_data) 60 | .on 'complete', -> 61 | done() 62 | .save() 63 | 64 | 65 | 66 | it 'should receive job result in complete event', (done) -> 67 | jobs.process 'email-with-results', (job, done)-> 68 | done( null, {finalResult:123} ) 69 | job_data = 70 | title: 'Test Email Job With Results' 71 | to: 'tj@learnboost.com' 72 | jobs.create('email-with-results', job_data) 73 | .on 'complete', (result)-> 74 | result.finalResult.should.be.equal 123 75 | done() 76 | .save() 77 | 78 | 79 | 80 | it 'should receive job progress event', (done) -> 81 | jobs.process 'email-to-be-progressed', (job, done)-> 82 | job.progress 1, 2 83 | done() 84 | job_data = 85 | title: 'Test Email Job' 86 | to: 'tj@learnboost.com' 87 | jobs.create('email-to-be-progressed', job_data) 88 | .on 'progress', (progress)-> 89 | progress.should.be.equal 50 90 | done() 91 | .save() 92 | 93 | 94 | 95 | it 'should receive job progress event with extra data', (done) -> 96 | jobs.process 'email-to-be-progressed', (job, done)-> 97 | job.progress 1, 2, 98 | notifyTime : "2014-11-22" 99 | done() 100 | job_data = 101 | title: 'Test Email Job' 102 | to: 'tj@learnboost.com' 103 | jobs.create('email-to-be-progressed', job_data) 104 | .on 'progress', (progress, extraData)-> 105 | progress.should.be.equal 50 106 | extraData.notifyTime.should.be.equal "2014-11-22" 107 | done() 108 | .save() 109 | 110 | 111 | 112 | it 'should receive job failed attempt events', (done) -> 113 | total = 2 114 | errorMsg = 'myError' 115 | jobs.process 'email-to-be-failed', (job, jdone)-> 116 | jdone errorMsg 117 | job_data = 118 | title: 'Test Email Job' 119 | to: 'tj@learnboost.com' 120 | jobs.create('email-to-be-failed', job_data).attempts(2) 121 | .on 'failed attempt', (errMsg,doneAttempts) -> 122 | errMsg.should.be.equal errorMsg 123 | doneAttempts.should.be.equal 1 124 | total-- 125 | .on 'failed', (errMsg)-> 126 | errMsg.should.be.equal errorMsg 127 | (--total).should.be.equal 0 128 | done() 129 | .save() 130 | 131 | 132 | 133 | it 'should receive queue level complete event', (done) -> 134 | jobs.process 'email-to-be-completed', (job, jdone)-> 135 | jdone( null, { prop: 'val' } ) 136 | jobs.on 'job complete', (id, result) -> 137 | id.should.be.equal testJob.id 138 | result.prop.should.be.equal 'val' 139 | done() 140 | job_data = 141 | title: 'Test Email Job' 142 | to: 'tj@learnboost.com' 143 | testJob = jobs.create('email-to-be-completed', job_data).save() 144 | 145 | 146 | 147 | it 'should receive queue level failed attempt events', (done) -> 148 | total = 2 149 | errorMsg = 'myError' 150 | jobs.process 'email-to-be-failed', (job, jdone)-> 151 | jdone errorMsg 152 | job_data = 153 | title: 'Test Email Job' 154 | to: 'tj@learnboost.com' 155 | jobs.on 'job failed attempt', (id, errMsg, doneAttempts) -> 156 | id.should.be.equal newJob.id 157 | errMsg.should.be.equal errorMsg 158 | doneAttempts.should.be.equal 1 159 | total-- 160 | .on 'job failed', (id, errMsg)-> 161 | id.should.be.equal newJob.id 162 | errMsg.should.be.equal errorMsg 163 | (--total).should.be.equal 0 164 | done() 165 | newJob = jobs.create('email-to-be-failed', job_data).attempts(2).save() 166 | 167 | 168 | 169 | 170 | describe 'Job', -> 171 | it 'should be processed after delay', (done) -> 172 | now = Date.now() 173 | jobs.create( 'simple-delay-job', { title: 'simple delay job' } ).delay(300).save() 174 | jobs.process 'simple-delay-job', (job, jdone) -> 175 | processed = Date.now() 176 | (processed - now).should.be.approximately( 300, 100 ) 177 | jdone() 178 | done() 179 | 180 | 181 | it 'should have promote_at timestamp', (done) -> 182 | now = Date.now() 183 | job = jobs.create( 'simple-delayed-job', { title: 'simple delay job' } ).delay(300).save() 184 | jobs.process 'simple-delayed-job', (job, jdone) -> 185 | job.promote_at.should.be.approximately(now + 300, 100) 186 | jdone() 187 | done() 188 | 189 | 190 | it 'should update promote_at after delay change', (done) -> 191 | now = Date.now() 192 | job = jobs.create( 'simple-delayed-job-1', { title: 'simple delay job' } ).delay(300).save() 193 | job.delay(100).save() 194 | jobs.process 'simple-delayed-job-1', (job, jdone) -> 195 | job.promote_at.should.be.approximately(now + 100, 100) 196 | jdone() 197 | done() 198 | 199 | 200 | 201 | it 'should update promote_at after failure with backoff', (done) -> 202 | now = Date.now() 203 | job = jobs.create( 'simple-delayed-job-2', { title: 'simple delay job' } ).delay(100).attempts(2).backoff({delay: 100, type: 'fixed'}).save() 204 | calls = 0 205 | jobs.process 'simple-delayed-job-2', (job, jdone) -> 206 | processed = Date.now() 207 | if calls == 1 208 | (processed - now).should.be.approximately(300, 100) 209 | jdone() 210 | done() 211 | else 212 | (processed - now).should.be.approximately(100, 100) 213 | jdone('error') 214 | 215 | calls++ 216 | 217 | 218 | 219 | it 'should be processed at a future date', (done) -> 220 | now = Date.now() 221 | jobs.create( 'future-job', { title: 'future job' } ).delay(new Date(now + 200)).save() 222 | jobs.process 'future-job', (job, jdone) -> 223 | processed = Date.now() 224 | (processed - now).should.be.approximately( 200, 100 ) 225 | jdone() 226 | done() 227 | 228 | 229 | 230 | it 'should receive promotion event', (done) -> 231 | job_data = 232 | title: 'Test Email Job' 233 | to: 'tj@learnboost.com' 234 | jobs.process('email-to-be-promoted', (job,done)-> ) 235 | jobs.create('email-to-be-promoted', job_data).delay(200) 236 | .on 'promotion', ()-> 237 | done() 238 | .save() 239 | 240 | 241 | 242 | it 'should be re tried after failed attempts', (done) -> 243 | [total, remaining] = [2,2] 244 | jobs.create( 'simple-multi-attempts-job', { title: 'simple-multi-attempts-job' } ).attempts(total).save() 245 | jobs.process 'simple-multi-attempts-job', (job, jdone) -> 246 | job.toJSON().attempts.remaining.should.be.equal remaining 247 | (job.toJSON().attempts.made + job.toJSON().attempts.remaining).should.be.equal total 248 | if( !--remaining ) 249 | jdone() 250 | done() 251 | else 252 | jdone( new Error('reaattempt') ) 253 | 254 | 255 | 256 | it 'should honor original delay at fixed backoff', (done) -> 257 | [total, remaining] = [2,2] 258 | start = Date.now() 259 | jobs.create( 'backoff-fixed-job', { title: 'backoff-fixed-job' } ).delay( 200 ).attempts(total).backoff( true ).save() 260 | jobs.process 'backoff-fixed-job', (job, jdone) -> 261 | if( !--remaining ) 262 | now = Date.now() 263 | (now - start).should.be.approximately(400,120) 264 | jdone() 265 | done() 266 | else 267 | jdone( new Error('reaattempt') ) 268 | 269 | 270 | 271 | it 'should honor original delay at exponential backoff', (done) -> 272 | [total, remaining] = [3,3] 273 | start = Date.now() 274 | jobs.create( 'backoff-exponential-job', { title: 'backoff-exponential-job' } ) 275 | .delay( 50 ).attempts(total).backoff( {type:'exponential', delay: 100} ).save() 276 | jobs.process 'backoff-exponential-job', (job, jdone) -> 277 | job._backoff.type.should.be.equal "exponential" 278 | job._backoff.delay.should.be.equal 100 279 | now = Date.now() 280 | if( !--remaining ) 281 | (now - start).should.be.approximately(350,100) 282 | jdone() 283 | done() 284 | else 285 | jdone( new Error('reaattempt') ) 286 | 287 | it 'should honor max delay at exponential backoff', (done) -> 288 | [total, remaining] = [10,10] 289 | last = Date.now() 290 | jobs.create( 'backoff-exponential-job', { title: 'backoff-exponential-job' } ) 291 | .attempts(total).backoff( {type:'exponential', delay: 50, maxDelay: 100} ).save() 292 | jobs.process 'backoff-exponential-job', (job, jdone) -> 293 | job._backoff.type.should.be.equal "exponential" 294 | job._backoff.delay.should.be.equal 50 295 | job._backoff.maxDelay.should.be.equal 100 296 | now = Date.now() 297 | (now - last).should.be.lessThan 120 298 | if( !--remaining ) 299 | jdone() 300 | done() 301 | else 302 | last = now 303 | jdone( new Error('reaattempt') ) 304 | 305 | 306 | it 'should honor users backoff function', (done) -> 307 | [total, remaining] = [2,2] 308 | start = Date.now() 309 | jobs.create( 'backoff-user-job', { title: 'backoff-user-job' } ) 310 | .delay( 50 ).attempts(total).backoff( ( attempts, delay ) -> 250 ).save() 311 | jobs.process 'backoff-user-job', (job, jdone) -> 312 | now = Date.now() 313 | if( !--remaining ) 314 | (now - start).should.be.approximately(350, 100) 315 | jdone() 316 | done() 317 | else 318 | jdone( new Error('reaattempt') ) 319 | 320 | it 'should log with a sprintf-style string', (done) -> 321 | jobs.create( 'log-job', { title: 'simple job' } ).save() 322 | jobs.process 'log-job', (job, jdone) -> 323 | job.log('this is %s number %d','test',1) 324 | Job.log job.id, (err,logs) -> 325 | logs[0].should.be.equal('this is test number 1'); 326 | done() 327 | jdone() 328 | 329 | 330 | 331 | it 'should log objects, errors, arrays, numbers, etc', (done) -> 332 | jobs.create( 'log-job', { title: 'simple job' } ).save() 333 | jobs.process 'log-job', (job, jdone) -> 334 | testErr = new Error('test error')# to compare the same stack 335 | job.log() 336 | job.log(undefined) 337 | job.log(null) 338 | job.log({test: 'some text'}) 339 | job.log(testErr) 340 | job.log([1,2,3]) 341 | job.log(123) 342 | job.log(1.23) 343 | job.log(0) 344 | job.log(NaN) 345 | job.log(true) 346 | job.log(false) 347 | 348 | Job.log job.id, (err,logs) -> 349 | logs[0].should.be.equal(util.format(undefined)); 350 | logs[1].should.be.equal(util.format(undefined)); 351 | logs[2].should.be.equal(util.format(null)); 352 | logs[3].should.be.equal(util.format({ test: 'some text' })); 353 | logs[4].should.be.equal(util.format(testErr)); 354 | logs[5].should.be.equal(util.format([ 1, 2, 3 ])); 355 | logs[6].should.be.equal(util.format(123)); 356 | logs[7].should.be.equal(util.format(1.23)); 357 | logs[8].should.be.equal(util.format(0)); 358 | logs[9].should.be.equal(util.format(NaN)); 359 | logs[10].should.be.equal(util.format(true)); 360 | logs[11].should.be.equal(util.format(false)); 361 | done() 362 | jdone() 363 | 364 | 365 | 366 | 367 | describe 'Kue Core', -> 368 | 369 | it 'should receive a "job enqueue" event', (done) -> 370 | jobs.on 'job enqueue', (id, type) -> 371 | if type == 'email-to-be-enqueued' 372 | id.should.be.equal job.id 373 | done() 374 | jobs.process 'email-to-be-enqueued', (job, jdone) -> jdone() 375 | job = jobs.create('email-to-be-enqueued').save() 376 | 377 | 378 | it 'should receive a "job remove" event', (done) -> 379 | jobs.on 'job remove', (id, type) -> 380 | if type == 'removable-job' 381 | id.should.be.equal job.id 382 | done() 383 | jobs.process 'removable-job', (job, jdone) -> jdone() 384 | job = jobs.create('removable-job').save().remove() 385 | 386 | 387 | it 'should fail a job with TTL is exceeded', (done) -> 388 | jobs.process('test-job-with-ttl', (job, jdone) -> 389 | # do nothing to sample a stuck worker 390 | ) 391 | jobs.create('test-job-with-ttl', title: 'a ttl job').ttl(500) 392 | .on 'failed', (err) -> 393 | err.should.be.equal 'TTL exceeded' 394 | done() 395 | .save() 396 | 397 | 398 | describe 'Kue Job Concurrency', -> 399 | 400 | it 'should process 2 concurrent jobs at the same time', (done) -> 401 | now = Date.now() 402 | jobStartTimes = [] 403 | jobs.process('test-job-parallel', 2, (job,jdone) -> 404 | jobStartTimes.push Date.now() 405 | if( jobStartTimes.length == 2 ) 406 | (jobStartTimes[0] - now).should.be.approximately( 0, 100 ) 407 | (jobStartTimes[1] - now).should.be.approximately( 0, 100 ) 408 | done() 409 | setTimeout(jdone, 500) 410 | ) 411 | jobs.create('test-job-parallel', title: 'concurrent job 1').save() 412 | jobs.create('test-job-parallel', title: 'concurrent job 2').save() 413 | 414 | it 'should process non concurrent jobs serially', (done) -> 415 | now = Date.now() 416 | jobStartTimes = [] 417 | jobs.process('test-job-serial', 1, (job,jdone) -> 418 | jobStartTimes.push Date.now() 419 | if( jobStartTimes.length == 2 ) 420 | (jobStartTimes[0] - now).should.be.approximately( 0, 100 ) 421 | (jobStartTimes[1] - now).should.be.approximately( 500, 100 ) 422 | done() 423 | setTimeout(jdone, 500) 424 | ) 425 | jobs.create('test-job-serial', title: 'non concurrent job 1').save() 426 | jobs.create('test-job-serial', title: 'non concurrent job 2').save() 427 | 428 | it 'should process a new job after a previous one fails with TTL is exceeded', (done) -> 429 | failures = 0 430 | now = Date.now() 431 | jobStartTimes = [] 432 | jobs.process('test-job-serial-failed', 1, (job,jdone) -> 433 | jobStartTimes.push Date.now() 434 | if( jobStartTimes.length == 2 ) 435 | (jobStartTimes[0] - now).should.be.approximately( 0, 100 ) 436 | (jobStartTimes[1] - now).should.be.approximately( 500, 100 ) 437 | failures.should.be.equal 1 438 | done() 439 | # do not call jdone to simulate a stuck worker 440 | ) 441 | jobs.create('test-job-serial-failed', title: 'a ttl job 1').ttl(500).on( 'failed', ()-> 442 | ++failures 443 | ).save() 444 | jobs.create('test-job-serial-failed', title: 'a ttl job 2').ttl(500).on( 'failed', ()-> 445 | ++failures 446 | ).save() 447 | 448 | it 'should not stuck in inactive mode if one of the workers failed because of ttl', (done) -> 449 | jobs.create('jobsA', 450 | title: 'titleA' 451 | metadata: {}).delay(1000).attempts(3).backoff( 452 | delay: 1 * 1000 453 | type: 'exponential').removeOnComplete(true).ttl(1 * 1000).save() 454 | jobs.create('jobsB', 455 | title: 'titleB' 456 | metadata: {}).delay(1500).attempts(3).backoff( 457 | delay: 1 * 1000 458 | type: 'exponential').removeOnComplete(true).ttl(1 * 1000).save() 459 | 460 | jobs.process 'jobsA', 1, (job, jdone) -> 461 | if job._attempts == '2' 462 | done() 463 | return 464 | jobs.process 'jobsB', 1, (job, jdone) -> 465 | done() 466 | return 467 | 468 | 469 | describe 'Kue Job Removal', -> 470 | 471 | beforeEach (done) -> 472 | jobs.process 'sample-job-to-be-cleaned', (job, jdone) -> jdone() 473 | async.each([1..10], (id, next) -> 474 | jobs.create( 'sample-job-to-be-cleaned', {id: id} ).save(next) 475 | , done) 476 | 477 | 478 | it 'should be able to remove completed jobs', (done) -> 479 | jobs.complete (err, ids) -> 480 | should.not.exist err 481 | async.each(ids, (id, next) -> 482 | Job.remove(id, next) 483 | , done) 484 | 485 | 486 | it 'should be able to remove failed jobs', (done) -> 487 | jobs.failed (err, ids) -> 488 | should.not.exist err 489 | async.each(ids, (id, next) -> 490 | Job.remove(id, next) 491 | , done) 492 | -------------------------------------------------------------------------------- /lib/kue.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * kue 3 | * Copyright (c) 2013 Automattic 4 | * Copyright (c) 2011 LearnBoost 5 | * MIT Licensed 6 | */ 7 | 8 | /** 9 | * Module dependencies. 10 | */ 11 | 12 | var EventEmitter = require('events').EventEmitter 13 | , Worker = require('./queue/worker') 14 | , events = require('./queue/events') 15 | , Job = require('./queue/job') 16 | , Warlock = require('node-redis-warlock') 17 | , _ = require('lodash') 18 | , redis = require('./redis') 19 | , noop = function(){}; 20 | 21 | /** 22 | * Expose `Queue`. 23 | */ 24 | 25 | exports = module.exports = Queue; 26 | 27 | /** 28 | * Library version. 29 | */ 30 | 31 | exports.version = require('../package.json').version; 32 | 33 | /** 34 | * Expose `Job`. 35 | */ 36 | 37 | exports.Job = Job; 38 | 39 | /** 40 | * Server instance (that is lazily required) 41 | */ 42 | 43 | var app; 44 | 45 | /** 46 | * Expose the server. 47 | */ 48 | 49 | Object.defineProperty(exports, 'app', { 50 | get: function() { 51 | return app || (app = require('./http')); 52 | } 53 | }); 54 | 55 | /** 56 | * Expose the RedisClient factory. 57 | */ 58 | 59 | exports.redis = redis; 60 | 61 | /** 62 | * Create a new `Queue`. 63 | * 64 | * @return {Queue} 65 | * @api public 66 | */ 67 | 68 | exports.createQueue = function( options ) { 69 | if( !Queue.singleton ) { 70 | Queue.singleton = new Queue(options); 71 | } 72 | events.subscribe(); 73 | return Queue.singleton; 74 | 75 | }; 76 | 77 | /** 78 | * Store workers 79 | */ 80 | exports.workers = []; 81 | 82 | /** 83 | * Initialize a new job `Queue`. 84 | * 85 | * @api public 86 | */ 87 | 88 | function Queue( options ) { 89 | options = options || {}; 90 | this.name = options.name || 'kue'; 91 | this.id = [ 'kue', require('os').hostname(), process.pid ].join(':'); 92 | this._options = options; 93 | this.promoter = null; 94 | this.workers = exports.workers; 95 | this.shuttingDown = false; 96 | Job.disableSearch = options.disableSearch !== false; 97 | options.jobEvents !== undefined ? Job.jobEvents = options.jobEvents : ''; 98 | redis.configureFactory(options, this); 99 | this.client = Worker.client = Job.client = redis.createClient(); 100 | } 101 | 102 | /** 103 | * Inherit from `EventEmitter.prototype`. 104 | */ 105 | 106 | Queue.prototype.__proto__ = EventEmitter.prototype; 107 | 108 | /** 109 | * Create a `Job` with the given `type` and `data`. 110 | * 111 | * @param {String} type 112 | * @param {Object} data 113 | * @return {Job} 114 | * @api public 115 | */ 116 | 117 | Queue.prototype.create = 118 | Queue.prototype.createJob = function( type, data ) { 119 | return new Job(type, data); 120 | }; 121 | 122 | /** 123 | * Proxy to auto-subscribe to events. 124 | * 125 | * @api public 126 | */ 127 | 128 | var on = EventEmitter.prototype.on; 129 | Queue.prototype.on = function( event ) { 130 | if( 0 == event.indexOf('job') ) events.subscribe(); 131 | return on.apply(this, arguments); 132 | }; 133 | 134 | /** 135 | * Promote delayed jobs, checking every `ms`, 136 | * defaulting to 1 second. 137 | * 138 | * @params {Number} ms 139 | * @deprecated 140 | */ 141 | 142 | Queue.prototype.promote = function( ms, l ) { 143 | console.warn('promote method is deprecated, you don\'t need to call this anymore. You can safely remove it from your code now.'); 144 | }; 145 | 146 | /** 147 | * sets up promotion & ttl timers 148 | */ 149 | 150 | Queue.prototype.setupTimers = function() { 151 | if( this.warlock === undefined ) { 152 | this.lockClient = redis.createClient(); 153 | this.warlock = new Warlock(this.lockClient); 154 | } 155 | this.checkJobPromotion(this._options.promotion); 156 | this.checkActiveJobTtl(this._options.promotion); 157 | }; 158 | 159 | /** 160 | * This new method is called by Kue when created 161 | * 162 | * Promote delayed jobs, checking every `ms`, 163 | * defaulting to 1 second. 164 | * 165 | * @params {Number} ms 166 | */ 167 | 168 | Queue.prototype.checkJobPromotion = function( promotionOptions ) { 169 | promotionOptions = promotionOptions || {}; 170 | var client = this.client 171 | , self = this 172 | , timeout = promotionOptions.interval || 1000 173 | , lockTtl = promotionOptions.lockTtl || 2000 174 | //, lockTtl = timeout 175 | , limit = promotionOptions.limit || 1000; 176 | clearInterval(this.promoter); 177 | this.promoter = setInterval(function() { 178 | self.warlock.lock('promotion', lockTtl, function( err, unlock ) { 179 | if( err ) { 180 | // Something went wrong and we weren't able to set a lock 181 | self.emit('error', err); 182 | return; 183 | } 184 | if( typeof unlock === 'function' ) { 185 | // If the lock is set successfully by this process, an unlock function is passed to our callback. 186 | client.zrangebyscore(client.getKey('jobs:delayed'), 0, Date.now(), 'LIMIT', 0, limit, function( err, ids ) { 187 | if( err || !ids.length ) return unlock(); 188 | //TODO do a ZREMRANGEBYRANK jobs:delayed 0 ids.length-1 189 | var doUnlock = _.after(ids.length, unlock); 190 | ids.forEach(function( id ) { 191 | id = client.stripFIFO(id); 192 | Job.get(id, function( err, job ) { 193 | if( err ) return doUnlock(); 194 | events.emit(id, 'promotion'); 195 | job.inactive(doUnlock); 196 | }); 197 | }); 198 | }); 199 | } else { 200 | // The lock was not established by us, be silent 201 | } 202 | }); 203 | }, timeout); 204 | }; 205 | 206 | 207 | Queue.prototype.checkActiveJobTtl = function( ttlOptions ) { 208 | ttlOptions = ttlOptions || {}; 209 | var client = this.client 210 | , self = this 211 | , timeout = ttlOptions.interval || 1000 212 | , lockTtl = 2000 213 | , limit = ttlOptions.limit || 1000; 214 | clearInterval(this.activeJobsTtlTimer); 215 | this.activeJobsTtlTimer = setInterval(function() { 216 | self.warlock.lock('activeJobsTTL', lockTtl, function( err, unlock ) { 217 | if( err ) { 218 | // Something went wrong and we weren't able to set a lock 219 | self.emit('error', err); 220 | return; 221 | } 222 | if( typeof unlock === 'function' ) { 223 | // If the lock is set successfully by this process, an unlock function is passed to our callback. 224 | // filter only jobs set with a ttl (timestamped) between a large number and current time 225 | client.zrangebyscore(client.getKey('jobs:active'), 100000, Date.now(), 'LIMIT', 0, limit, function( err, ids ) { 226 | if( err || !ids.length ) return unlock(); 227 | 228 | var idsRemaining = ids.slice(); 229 | var doUnlock = _.after(ids.length, function(){ 230 | self.removeAllListeners( 'job ttl exceeded ack' ); 231 | waitForAcks && clearTimeout( waitForAcks ); 232 | unlock && unlock(); 233 | }); 234 | 235 | self.on( 'job ttl exceeded ack', function( id ) { 236 | idsRemaining.splice( idsRemaining.indexOf( id ), 1 ); 237 | doUnlock(); 238 | }); 239 | 240 | var waitForAcks = setTimeout( function(){ 241 | idsRemaining.forEach( function( id ){ 242 | id = client.stripFIFO(id); 243 | Job.get(id, function( err, job ) { 244 | if( err ) return doUnlock(); 245 | job.failedAttempt( { error: true, message: 'TTL exceeded' }, doUnlock ); 246 | }); 247 | }); 248 | }, 1000 ); 249 | 250 | ids.forEach(function( id ) { 251 | id = client.stripFIFO(id); 252 | events.emit(id, 'ttl exceeded'); 253 | }); 254 | }); 255 | } else { 256 | // The lock was not established by us, be silent 257 | } 258 | }); 259 | }, timeout); 260 | }; 261 | 262 | /** 263 | * Runs a LUA script to diff inactive jobs ZSET cardinality 264 | * and helper pop LIST length each `ms` milliseconds and syncs helper LIST. 265 | * 266 | * @param {Number} ms interval for periodical script runs 267 | * @api public 268 | */ 269 | 270 | Queue.prototype.watchStuckJobs = function( ms ) { 271 | var client = this.client 272 | , self = this 273 | , ms = ms || 1000; 274 | var prefix = this.client.prefix; 275 | 276 | if( this.client.constructor.name == 'Redis' || this.client.constructor.name == 'Cluster') { 277 | // {prefix}:jobs format is needed in using ioredis cluster to keep they keys in same node 278 | prefix = '{' + prefix + '}'; 279 | } 280 | var script = 281 | 'local msg = redis.call( "keys", "' + prefix + ':jobs:*:inactive" )\n\ 282 | local need_fix = 0\n\ 283 | for i,v in ipairs(msg) do\n\ 284 | local queue = redis.call( "zcard", v )\n\ 285 | local jt = string.match(v, "' + prefix + ':jobs:(.*):inactive")\n\ 286 | local pending = redis.call( "LLEN", "' + prefix + ':" .. jt .. ":jobs" )\n\ 287 | if queue > pending then\n\ 288 | need_fix = need_fix + 1\n\ 289 | for j=1,(queue-pending) do\n\ 290 | redis.call( "lpush", "' + prefix + ':"..jt..":jobs", 1 )\n\ 291 | end\n\ 292 | end\n\ 293 | end\n\ 294 | return need_fix'; 295 | clearInterval(this.stuck_job_watch); 296 | client.script('LOAD', script, function( err, sha ) { 297 | if( err ) { 298 | return self.emit('error', err); 299 | } 300 | this.stuck_job_watch = setInterval(function() { 301 | client.evalsha(sha, 0, function( err, fixes ) { 302 | if( err ) return clearInterval(this.stuck_job_watch); 303 | }.bind(this)); 304 | }.bind(this), ms); 305 | 306 | }.bind(this)); 307 | }; 308 | 309 | /** 310 | * Get setting `name` and invoke `fn(err, res)`. 311 | * 312 | * @param {String} name 313 | * @param {Function} fn 314 | * @return {Queue} for chaining 315 | * @api public 316 | */ 317 | 318 | Queue.prototype.setting = function( name, fn ) { 319 | fn = fn || noop; 320 | this.client.hget(this.client.getKey('settings'), name, fn); 321 | return this; 322 | }; 323 | 324 | /** 325 | * Process jobs with the given `type`, invoking `fn(job)`. 326 | * 327 | * @param {String} type 328 | * @param {Number|Function} n 329 | * @param {Function} fn 330 | * @api public 331 | */ 332 | 333 | Queue.prototype.process = function( type, n, fn ) { 334 | var self = this; 335 | 336 | if( 'function' == typeof n ) fn = n, n = 1; 337 | 338 | while( n-- ) { 339 | var worker = new Worker(this, type).start(fn); 340 | worker.id = [ self.id, type, self.workers.length + 1 ].join(':'); 341 | worker.on('error', function( err ) { 342 | self.emit('error', err); 343 | }); 344 | worker.on('job complete', function( job ) { 345 | // guard against emit after shutdown 346 | if( self.client ) { 347 | self.client.incrby(self.client.getKey('stats:work-time'), job.duration, noop); 348 | } 349 | }); 350 | // Save worker so we can access it later 351 | self.workers.push(worker); 352 | } 353 | this.setupTimers(); 354 | }; 355 | 356 | /** 357 | * Graceful shutdown 358 | * 359 | * @param {Number} timeout in milliseconds to wait for workers to finish 360 | * @param {String} type specific worker type to shutdown 361 | * @param {Function} fn callback 362 | * @return {Queue} for chaining 363 | * @api public 364 | */ 365 | 366 | Queue.prototype.shutdown = function( timeout, type, fn ) { 367 | var self = this 368 | , n = self.workers.length; 369 | if( arguments.length === 1 ) { 370 | fn = timeout; 371 | type = ''; 372 | timeout = null; 373 | } else if( arguments.length === 2 ) { 374 | fn = type; 375 | type = ''; 376 | } 377 | var origFn = fn || function() { 378 | }; 379 | 380 | if( this.shuttingDown && type === '' ) { // a global shutdown already has been called 381 | return fn(new Error('Shutdown already in progress')); 382 | } 383 | 384 | if( type === '' ) { // this is a global shutdown call 385 | this.shuttingDown = true; 386 | } 387 | 388 | var cleanup = function() { 389 | if( self.shuttingDown ) { 390 | self.workers = []; 391 | exports.workers = []; 392 | self.removeAllListeners(); 393 | Queue.singleton = null; 394 | events.unsubscribe(); 395 | // destroy redis client and pubsub 396 | redis.reset(); 397 | self.client && self.client.quit(); 398 | self.client = null; 399 | self.lockClient && self.lockClient.quit(); 400 | self.lockClient = null; 401 | } 402 | }; 403 | 404 | // Wrap `fn` to only call after all workers finished 405 | fn = function( err ) { 406 | if( err ) { 407 | return origFn(err); 408 | } 409 | if( !--n ) { 410 | cleanup(); 411 | origFn.apply(null, arguments); 412 | } 413 | }; 414 | 415 | // shut down promoter interval 416 | if( self.shuttingDown ) { 417 | if( self.promoter ) { 418 | clearInterval(self.promoter); 419 | self.promoter = null; 420 | } 421 | if( self.activeJobsTtlTimer ) { 422 | clearInterval(self.activeJobsTtlTimer); 423 | self.activeJobsTtlTimer = null; 424 | } 425 | 426 | } 427 | 428 | if( !self.workers.length ) { 429 | cleanup(); 430 | origFn(); 431 | } else { 432 | // Shut down workers 1 by 1 433 | self.workers.forEach(function( worker ) { 434 | if( self.shuttingDown || worker.type == type ) { 435 | worker.shutdown(timeout, fn); 436 | } else { 437 | fn && fn(); 438 | } 439 | }); 440 | } 441 | 442 | return this; 443 | }; 444 | 445 | /** 446 | * Get the job types present and callback `fn(err, types)`. 447 | * 448 | * @param {Function} fn 449 | * @return {Queue} for chaining 450 | * @api public 451 | */ 452 | 453 | Queue.prototype.types = function( fn ) { 454 | fn = fn || noop; 455 | this.client.smembers(this.client.getKey('job:types'), fn); 456 | return this; 457 | }; 458 | 459 | /** 460 | * Return job ids with the given `state`, and callback `fn(err, ids)`. 461 | * 462 | * @param {String} state 463 | * @param {Function} fn 464 | * @return {Queue} for chaining 465 | * @api public 466 | */ 467 | 468 | Queue.prototype.state = function( state, fn ) { 469 | var self = this; 470 | this.client.zrange(this.client.getKey('jobs:' + state), 0, -1, function(err,ids){ 471 | var fixedIds = []; 472 | ids.forEach(function(id){ 473 | fixedIds.push(self.client.stripFIFO(id)); 474 | }); 475 | fn(err,fixedIds); 476 | }); 477 | return this; 478 | }; 479 | 480 | /** 481 | * Get queue work time in milliseconds and invoke `fn(err, ms)`. 482 | * 483 | * @param {Function} fn 484 | * @return {Queue} for chaining 485 | * @api public 486 | */ 487 | 488 | Queue.prototype.workTime = function( fn ) { 489 | this.client.get(this.client.getKey('stats:work-time'), function( err, n ) { 490 | if( err ) return fn(err); 491 | fn(null, parseInt(n, 10)); 492 | }); 493 | return this; 494 | }; 495 | 496 | /** 497 | * Get cardinality of jobs with given `state` and `type` and callback `fn(err, n)`. 498 | * 499 | * @param {String} type 500 | * @param {String} state 501 | * @param {Function} fn 502 | * @return {Queue} for chaining 503 | * @api public 504 | */ 505 | 506 | Queue.prototype.cardByType = function( type, state, fn ) { 507 | fn = fn || noop; 508 | this.client.zcard(this.client.getKey('jobs:' + type + ':' + state), fn); 509 | return this; 510 | }; 511 | 512 | /** 513 | * Get cardinality of `state` and callback `fn(err, n)`. 514 | * 515 | * @param {String} state 516 | * @param {Function} fn 517 | * @return {Queue} for chaining 518 | * @api public 519 | */ 520 | 521 | Queue.prototype.card = function( state, fn ) { 522 | fn = fn || noop; 523 | this.client.zcard(this.client.getKey('jobs:' + state), fn); 524 | return this; 525 | }; 526 | 527 | /** 528 | * Completed jobs. 529 | * @param {Function} fn 530 | * @return {Queue} for chaining 531 | * @api public 532 | */ 533 | 534 | Queue.prototype.complete = function( fn ) { 535 | return this.state('complete', fn); 536 | }; 537 | 538 | /** 539 | * Failed jobs. 540 | * @param {Function} fn 541 | * @return {Queue} for chaining 542 | * @api public 543 | */ 544 | 545 | Queue.prototype.failed = function( fn ) { 546 | return this.state('failed', fn); 547 | }; 548 | 549 | /** 550 | * Inactive jobs (queued). 551 | * @param {Function} fn 552 | * @return {Queue} for chaining 553 | * @api public 554 | */ 555 | 556 | Queue.prototype.inactive = function( fn ) { 557 | return this.state('inactive', fn); 558 | }; 559 | 560 | /** 561 | * Active jobs (mid-process). 562 | * @param {Function} fn 563 | * @return {Queue} for chaining 564 | * @api public 565 | */ 566 | 567 | Queue.prototype.active = function( fn ) { 568 | return this.state('active', fn); 569 | }; 570 | 571 | /** 572 | * Delayed jobs. 573 | * @param {Function} fn 574 | * @return {Queue} for chaining 575 | * @api public 576 | */ 577 | 578 | Queue.prototype.delayed = function( fn ) { 579 | return this.state('delayed', fn); 580 | }; 581 | 582 | /** 583 | * Completed jobs of type `type` count. 584 | * @param {String} type is optional 585 | * @param {Function} fn 586 | * @return {Queue} for chaining 587 | * @api public 588 | */ 589 | 590 | Queue.prototype.completeCount = function( type, fn ) { 591 | if( 1 == arguments.length ) { 592 | fn = type; 593 | return this.card('complete', fn); 594 | } 595 | return this.cardByType(type, 'complete', fn); 596 | }; 597 | 598 | 599 | /** 600 | * Failed jobs of type `type` count. 601 | * @param {String} type is optional 602 | * @param {Function} fn 603 | * @return {Queue} for chaining 604 | * @api public 605 | */ 606 | 607 | Queue.prototype.failedCount = function( type, fn ) { 608 | if( 1 == arguments.length ) { 609 | fn = type; 610 | return this.card('failed', fn); 611 | } 612 | return this.cardByType(type, 'failed', fn); 613 | }; 614 | 615 | /** 616 | * Inactive jobs (queued) of type `type` count. 617 | * @param {String} type is optional 618 | * @param {Function} fn 619 | * @return {Queue} for chaining 620 | * @api public 621 | */ 622 | 623 | Queue.prototype.inactiveCount = function( type, fn ) { 624 | if( 1 == arguments.length ) { 625 | fn = type; 626 | return this.card('inactive', fn); 627 | } 628 | return this.cardByType(type, 'inactive', fn); 629 | }; 630 | 631 | /** 632 | * Active jobs (mid-process) of type `type` count. 633 | * @param {String} type is optional 634 | * @param {Function} fn 635 | * @return {Queue} for chaining 636 | * @api public 637 | */ 638 | 639 | Queue.prototype.activeCount = function( type, fn ) { 640 | if( 1 == arguments.length ) { 641 | fn = type; 642 | return this.card('active', fn); 643 | } 644 | return this.cardByType(type, 'active', fn); 645 | }; 646 | 647 | /** 648 | * Delayed jobs of type `type` count. 649 | * @param {String} type is optional 650 | * @param {Function} fn 651 | * @return {Queue} for chaining 652 | * @api public 653 | */ 654 | 655 | Queue.prototype.delayedCount = function( type, fn ) { 656 | if( 1 == arguments.length ) { 657 | fn = type; 658 | return this.card('delayed', fn); 659 | } 660 | return this.cardByType(type, 'delayed', fn); 661 | }; 662 | 663 | /** 664 | * Test mode for convenience in test suites 665 | * @api public 666 | */ 667 | 668 | Queue.prototype.testMode = require('./queue/test_mode'); 669 | -------------------------------------------------------------------------------- /test/tdd/kue.spec.js: -------------------------------------------------------------------------------- 1 | var sinon = require('sinon'); 2 | var kue = require('../../lib/kue'); 3 | var redis = require('../../lib/redis'); 4 | var events = require('../../lib/queue/events'); 5 | var Job = require('../../lib/queue/job'); 6 | var Worker = require('../../lib/queue/worker'); 7 | var _ = require('lodash'); 8 | var EventEmitter = require('events').EventEmitter; 9 | var redisClient = {}; 10 | 11 | describe('Kue', function () { 12 | 13 | beforeEach(function(){ 14 | sinon.stub(events, 'subscribe'); 15 | sinon.stub(redis, 'configureFactory', function () { 16 | redis.createClient = sinon.stub(); 17 | }); 18 | }); 19 | 20 | afterEach(function(){ 21 | events.subscribe.restore(); 22 | redis.configureFactory.restore(); 23 | }); 24 | 25 | describe('Function: createQueue', function () { 26 | 27 | it('should subscribe to queue events', function () { 28 | var queue = kue.createQueue(); 29 | events.subscribe.called.should.be.true; 30 | }); 31 | 32 | it('should set the correct default values', function () { 33 | var queue = kue.createQueue(); 34 | queue.name.should.equal('kue'); 35 | queue.id.should.equal([ 'kue', require("os").hostname(), process.pid ].join(':')); 36 | (queue.promoter === null).should.be.true; 37 | queue.workers.should.eql(kue.workers); 38 | queue.shuttingDown.should.be.false; 39 | }); 40 | 41 | it('should allow a custom name option', function () { 42 | it('should set the correct default values', function () { 43 | var queue = kue.createQueue({ 44 | name: 'name' 45 | }); 46 | queue.name.should.equal('name'); 47 | }); 48 | }); 49 | }); 50 | 51 | describe('Function: create', function() { 52 | var queue; 53 | beforeEach(function(){ 54 | queue = kue.createQueue(); 55 | }); 56 | 57 | it('should return a new Job instance', function () { 58 | var data = { 59 | key: 'value' 60 | }; 61 | var job = queue.create('type', data); 62 | 63 | job.type.should.equal('type'); 64 | job.data.should.eql(data); 65 | }); 66 | }); 67 | 68 | describe('Function: on', function() { 69 | var queue, noop; 70 | beforeEach(function(){ 71 | queue = kue.createQueue(); 72 | events.subscribe.reset(); 73 | noop = function () {}; 74 | }); 75 | 76 | it('should subscribe to events when subscribing to the job event', function () { 77 | queue.on('job', noop); 78 | events.subscribe.called.should.be.true; 79 | }); 80 | 81 | it('should proxy the event listener', function (done) { 82 | queue.on('event', function (data) { 83 | data.should.equal('data'); 84 | done(); 85 | }); 86 | queue.emit('event', 'data'); 87 | }); 88 | }); 89 | 90 | describe('Function: setupTimers', function() { 91 | var queue; 92 | beforeEach(function(){ 93 | queue = kue.createQueue(); 94 | sinon.stub(queue, 'checkJobPromotion'); 95 | sinon.stub(queue, 'checkActiveJobTtl'); 96 | }); 97 | 98 | afterEach(function(){ 99 | queue.checkJobPromotion.restore(); 100 | queue.checkActiveJobTtl.restore(); 101 | }); 102 | 103 | it('should setup a warlock client if it is not setup yet', function () { 104 | queue.warlock = undefined; 105 | queue.setupTimers(); 106 | queue.warlock.should.exist; 107 | }); 108 | 109 | it('should call checkJobPromotion', function () { 110 | queue.setupTimers(); 111 | queue.checkJobPromotion.called.should.be.true; 112 | }); 113 | 114 | it('should call checkActiveJobTtl', function () { 115 | queue.setupTimers(); 116 | queue.checkActiveJobTtl.called.should.be.true; 117 | }); 118 | }); 119 | 120 | describe('Function: checkJobPromotion', function() { 121 | var queue, unlock, clock, timeout, client, ids, job; 122 | 123 | beforeEach(function(){ 124 | unlock = sinon.spy(); 125 | timeout = 1000; 126 | ids = [1, 2, 3]; 127 | client = { 128 | zrangebyscore: sinon.stub().callsArgWith(6, null, ids), 129 | getKey: sinon.stub().returnsArg(0), 130 | stripFIFO: sinon.stub().returnsArg(0) 131 | }; 132 | job = { 133 | inactive: sinon.stub().callsArg(0) 134 | }; 135 | 136 | queue = kue.createQueue(); 137 | queue.client = client; 138 | 139 | sinon.stub(Job, 'get').callsArgWith(1, null, job); 140 | sinon.stub(queue.warlock, 'lock').callsArgWith(2, null, unlock); 141 | sinon.stub(events, 'emit'); 142 | clock = sinon.useFakeTimers(); 143 | }); 144 | 145 | afterEach(function(){ 146 | Job.get.restore(); 147 | queue.warlock.lock.restore(); 148 | events.emit.restore(); 149 | clock.restore(); 150 | }); 151 | 152 | it('should set the promotion lock', function () { 153 | queue.checkJobPromotion(); 154 | clock.tick(timeout); 155 | queue.warlock.lock.calledWith('promotion', 2000).should.be.true; 156 | }); 157 | 158 | it('should allow an override for the lockTtl', function () { 159 | queue.checkJobPromotion({ lockTtl: 5000 }); 160 | clock.tick(timeout); 161 | queue.warlock.lock.calledWith('promotion', 5000).should.be.true; 162 | }); 163 | 164 | it('should load all delayed jobs that should be run job', function () { 165 | queue.checkJobPromotion(); 166 | clock.tick(timeout); 167 | client.zrangebyscore.calledWith(client.getKey('jobs:delayed'), 0, sinon.match.any, "LIMIT", 0, 1000).should.be.true; 168 | }); 169 | 170 | it('should get each job', function () { 171 | queue.checkJobPromotion(); 172 | clock.tick(timeout); 173 | Job.get.callCount.should.equal(3); 174 | Job.get.calledWith(ids[0]).should.be.true; 175 | Job.get.calledWith(ids[1]).should.be.true; 176 | Job.get.calledWith(ids[2]).should.be.true; 177 | }); 178 | 179 | it('should emit promotion for each job', function () { 180 | queue.checkJobPromotion(); 181 | clock.tick(timeout); 182 | events.emit.callCount.should.equal(3); 183 | events.emit.calledWith(ids[0], 'promotion').should.be.true; 184 | events.emit.calledWith(ids[1], 'promotion').should.be.true; 185 | events.emit.calledWith(ids[2], 'promotion').should.be.true; 186 | }); 187 | 188 | it('should set each job to inactive', function () { 189 | queue.checkJobPromotion(); 190 | clock.tick(timeout); 191 | job.inactive.callCount.should.equal(3); 192 | }); 193 | 194 | it('should unlock promotion', function () { 195 | queue.checkJobPromotion(); 196 | clock.tick(timeout); 197 | unlock.calledOnce.should.be.true; 198 | }); 199 | 200 | }); 201 | 202 | describe('Function: checkActiveJobTtl', function() { 203 | var queue, unlock, clock, timeout, client, ids, job; 204 | 205 | beforeEach(function(){ 206 | unlock = sinon.spy(); 207 | timeout = 1000; 208 | ids = [1, 2, 3]; 209 | client = { 210 | zrangebyscore: sinon.stub().callsArgWith(6, null, ids), 211 | getKey: sinon.stub().returnsArg(0), 212 | stripFIFO: sinon.stub().returnsArg(0) 213 | }; 214 | job = { 215 | failedAttempt: sinon.stub().callsArg(1) 216 | }; 217 | 218 | queue = kue.createQueue(); 219 | queue.client = client; 220 | 221 | sinon.spy(queue, 'removeAllListeners'); 222 | sinon.stub(Job, 'get').callsArgWith(1, null, job); 223 | sinon.stub(queue.warlock, 'lock').callsArgWith(2, null, unlock); 224 | sinon.stub(events, 'emit'); 225 | clock = sinon.useFakeTimers(); 226 | }); 227 | 228 | afterEach(function(){ 229 | queue.removeAllListeners.restore(); 230 | Job.get.restore(); 231 | queue.warlock.lock.restore(); 232 | events.emit.restore(); 233 | clock.restore(); 234 | }); 235 | 236 | it('should set the activeJobsTTL lock', function () { 237 | queue.checkActiveJobTtl(); 238 | clock.tick(timeout); 239 | queue.warlock.lock.calledWith('activeJobsTTL').should.be.true; 240 | }); 241 | 242 | it('should load all expired jobs', function () { 243 | queue.checkActiveJobTtl(); 244 | clock.tick(timeout); 245 | client.zrangebyscore.calledWith(client.getKey('jobs:active'), 100000, sinon.match.any, "LIMIT", 0, 1000).should.be.true; 246 | }); 247 | 248 | it('should emit ttl exceeded for each job', function () { 249 | queue.checkActiveJobTtl(); 250 | clock.tick(timeout); 251 | events.emit.callCount.should.equal(3); 252 | events.emit.calledWith(ids[0], 'ttl exceeded'); 253 | events.emit.calledWith(ids[1], 'ttl exceeded'); 254 | events.emit.calledWith(ids[2], 'ttl exceeded'); 255 | }); 256 | 257 | it('should unlock after all the job ttl exceeded acks have been received', function () { 258 | queue.checkActiveJobTtl('job ttl exceeded ack'); 259 | queue.checkActiveJobTtl(); 260 | clock.tick(timeout); 261 | _.each(ids, function (id) { 262 | // calling queue.emit since queue.on does special logic for events that start with "job" 263 | queue.emit('job ttl exceeded ack', id); 264 | }); 265 | unlock.calledOnce.should.be.true; 266 | queue.removeAllListeners.calledWith('job ttl exceeded ack').should.be.true; 267 | }); 268 | 269 | it('should call job.failedAttempt for each job that did not receive the ack event', function () { 270 | queue.removeAllListeners('job ttl exceeded ack'); 271 | queue.checkActiveJobTtl('job ttl exceeded ack'); 272 | clock.tick(timeout); 273 | var id = ids.splice(0, 1)[0]; 274 | _.each(ids, function (id) { 275 | // calling queue.emit since queue.on does special logic for events that start with "job" 276 | queue.emit('job ttl exceeded ack', id); 277 | }); 278 | clock.tick(timeout); 279 | Job.get.calledWith(id).should.be.true; 280 | job.failedAttempt.calledOnce.should.be.true; 281 | job.failedAttempt.calledWith({ 282 | error: true, 283 | message: 'TTL exceeded' 284 | }).should.be.true; 285 | }); 286 | }); 287 | 288 | describe('Function: watchStuckJobs', function() { 289 | var queue, clock, client, sha; 290 | 291 | beforeEach(function(){ 292 | sha = 'sha'; 293 | client = { 294 | script: sinon.stub().callsArgWith(2, null, sha), 295 | evalsha: sinon.stub().callsArg(2) 296 | }; 297 | 298 | queue = kue.createQueue(); 299 | queue.client = client; 300 | 301 | clock = sinon.useFakeTimers(); 302 | }); 303 | 304 | afterEach(function(){ 305 | clock.restore(); 306 | }); 307 | 308 | it('should load the script', function () { 309 | queue.watchStuckJobs(); 310 | client.script.calledWith('LOAD').should.be.true; 311 | }); 312 | 313 | it('should run the script on an interval', function () { 314 | queue.watchStuckJobs(); 315 | clock.tick(1000); 316 | client.evalsha.calledWith(sha, 0).should.be.true; 317 | client.evalsha.callCount.should.equal(1); 318 | clock.tick(1000); 319 | client.evalsha.callCount.should.equal(2); 320 | }); 321 | 322 | }); 323 | 324 | describe('Function: setting', function() { 325 | var queue, client; 326 | 327 | beforeEach(function(){ 328 | client = { 329 | getKey: sinon.stub().returnsArg(0), 330 | hget: sinon.stub().callsArg(2) 331 | }; 332 | 333 | queue = kue.createQueue(); 334 | queue.client = client; 335 | }); 336 | 337 | it('should get the requested setting', function (done) { 338 | queue.setting('name', function () { 339 | client.hget.calledWith(client.getKey('settings'), 'name').should.be.true; 340 | done(); 341 | }); 342 | }); 343 | 344 | }); 345 | 346 | describe('Function: process', function() { 347 | var queue, client, worker; 348 | 349 | beforeEach(function(){ 350 | client = { 351 | getKey: sinon.stub().returnsArg(0), 352 | incrby: sinon.stub() 353 | }; 354 | worker = new EventEmitter(); 355 | queue = kue.createQueue(); 356 | queue.workers = []; 357 | queue.client = client; 358 | 359 | sinon.stub(queue, 'setupTimers'); 360 | sinon.stub(Worker.prototype, 'start').returns(worker); 361 | }); 362 | 363 | afterEach(function(){ 364 | queue.setupTimers.restore(); 365 | Worker.prototype.start.restore(); 366 | }); 367 | 368 | it('should use 1 as the default number of workers', function () { 369 | queue.process('type', sinon.stub()); 370 | Worker.prototype.start.callCount.should.equal(1); 371 | }); 372 | 373 | it('should accept a number for the number of workers', function () { 374 | queue.process('type', 3, sinon.stub()); 375 | Worker.prototype.start.callCount.should.equal(3); 376 | }); 377 | 378 | it('should add each worker to the queue.workers array', function () { 379 | queue.process('type', 3, sinon.stub()); 380 | queue.workers.length.should.equal(3); 381 | }); 382 | 383 | it('should setup each worker to respond to error events', function () { 384 | sinon.stub(queue, 'emit'); 385 | queue.process('type', 3, sinon.stub()); 386 | worker.emit('error'); 387 | queue.emit.callCount.should.equal(3); 388 | queue.emit.restore(); 389 | }); 390 | 391 | it('should setup each worker to respond to job complete events', function () { 392 | var job = { 393 | duration: 100 394 | }; 395 | queue.process('type', 3, sinon.stub()); 396 | worker.emit('job complete', job); 397 | client.incrby.calledWith(client.getKey('stats:work-time'), job.duration).should.be.true; 398 | }); 399 | 400 | it('should setup timers', function () { 401 | queue.process('type', 3, sinon.stub()); 402 | queue.setupTimers.called.should.be.true; 403 | }); 404 | 405 | }); 406 | 407 | describe('Function: shutdown', function() { 408 | var queue, client, worker, lockClient; 409 | 410 | beforeEach(function(){ 411 | client = { 412 | quit: sinon.stub() 413 | }; 414 | lockClient = { 415 | quit: sinon.stub() 416 | }; 417 | worker = { 418 | shutdown: sinon.stub().callsArg(1) 419 | }; 420 | queue = kue.createQueue(); 421 | queue.shuttingDown = false; 422 | queue.workers = [worker, worker, worker]; 423 | queue.client = client; 424 | queue.lockClient = lockClient; 425 | 426 | sinon.stub(events, 'unsubscribe'); 427 | sinon.stub(redis, 'reset'); 428 | }); 429 | 430 | afterEach(function(){ 431 | events.unsubscribe.restore(); 432 | redis.reset.restore(); 433 | }); 434 | 435 | it('should return an error if it is already shutting down', function (done) { 436 | queue.shuttingDown = true; 437 | queue.shutdown(function(err){ 438 | err.should.exist; 439 | done(); 440 | }); 441 | }); 442 | 443 | it('should shutdown each worker', function (done) { 444 | queue.shutdown(function () { 445 | worker.shutdown.callCount.should.equal(3); 446 | done(); 447 | }); 448 | }); 449 | 450 | it('should clean things up', function (done) { 451 | queue.shutdown(function () { 452 | queue.workers.length.should.equal(0); 453 | events.unsubscribe.called.should.be.true; 454 | redis.reset.called.should.be.true; 455 | client.quit.called.should.be.true; 456 | (queue.client == null).should.be.true; 457 | lockClient.quit.called.should.be.true; 458 | (queue.lockClient == null).should.be.true; 459 | done(); 460 | }); 461 | }); 462 | 463 | }); 464 | 465 | describe('Function: types', function() { 466 | var queue, client, types; 467 | 468 | beforeEach(function(){ 469 | types = ['type1', 'type2']; 470 | client = { 471 | getKey: sinon.stub().returnsArg(0), 472 | smembers: sinon.stub().callsArgWith(1, null, types) 473 | }; 474 | queue = kue.createQueue(); 475 | queue.client = client; 476 | }); 477 | 478 | it('should get the jobs types', function (done) { 479 | queue.types(function(err, tps){ 480 | tps.should.eql(types); 481 | done(); 482 | }); 483 | }); 484 | }); 485 | 486 | describe('Function: state', function() { 487 | var queue, client, jobIds, state; 488 | 489 | beforeEach(function(){ 490 | jobIds = [1, 2]; 491 | state = 'state'; 492 | client = { 493 | getKey: sinon.stub().returnsArg(0), 494 | stripFIFO: sinon.stub().returnsArg(0), 495 | zrange: sinon.stub().callsArgWith(3, null, jobIds) 496 | }; 497 | queue = kue.createQueue(); 498 | queue.client = client; 499 | }); 500 | 501 | it('should get all job ids for the given state', function (done) { 502 | queue.state(state, function (err, ids) { 503 | ids.should.eql(jobIds); 504 | done(); 505 | }); 506 | }); 507 | 508 | }); 509 | 510 | describe('Function: workTime', function() { 511 | var queue, client, n; 512 | 513 | beforeEach(function(){ 514 | n = 20; 515 | client = { 516 | getKey: sinon.stub().returnsArg(0), 517 | get: sinon.stub().callsArgWith(1, null, n) 518 | }; 519 | queue = kue.createQueue(); 520 | queue.client = client; 521 | }); 522 | 523 | it('should load the worktime', function (done) { 524 | queue.workTime(function (err, time) { 525 | time.should.equal(n); 526 | done(); 527 | }); 528 | }); 529 | 530 | }); 531 | 532 | describe('Function: cardByType', function() { 533 | var queue, client, type, state, total; 534 | 535 | beforeEach(function(){ 536 | type = 'type'; 537 | state = 'state'; 538 | total = 20; 539 | client = { 540 | getKey: sinon.stub().returnsArg(0), 541 | zcard: sinon.stub().callsArgWith(1, null, total) 542 | }; 543 | queue = kue.createQueue(); 544 | queue.client = client; 545 | }); 546 | 547 | it('should return the total number of jobs for a given type and state', function (done) { 548 | queue.cardByType(type, state, function (err, card) { 549 | card.should.equal(total); 550 | done(); 551 | }); 552 | }); 553 | }); 554 | 555 | describe('function: card', function() { 556 | var queue, client, state, total; 557 | 558 | beforeEach(function(){ 559 | state = 'state'; 560 | total = 20; 561 | client = { 562 | getKey: sinon.stub().returnsArg(0), 563 | zcard: sinon.stub().callsArgWith(1, null, total) 564 | }; 565 | queue = kue.createQueue(); 566 | queue.client = client; 567 | }); 568 | 569 | it('should return the total number of jobs for a given state', function (done) { 570 | queue.card(state, function (err, card) { 571 | card.should.equal(total); 572 | done(); 573 | }); 574 | }); 575 | }); 576 | 577 | describe('Function: complete', function() { 578 | var queue; 579 | 580 | beforeEach(function(){ 581 | queue = kue.createQueue(); 582 | sinon.stub(queue, 'state').callsArg(1); 583 | }); 584 | 585 | afterEach(function(){ 586 | queue.state.restore(); 587 | }); 588 | 589 | it('should get the completed jobs', function (done) { 590 | queue.complete(function () { 591 | queue.state.calledWith('complete').should.be.true; 592 | done(); 593 | }); 594 | }); 595 | }); 596 | 597 | describe('Function: failed', function() { 598 | var queue; 599 | 600 | beforeEach(function(){ 601 | queue = kue.createQueue(); 602 | sinon.stub(queue, 'state').callsArg(1); 603 | }); 604 | 605 | afterEach(function(){ 606 | queue.state.restore(); 607 | }); 608 | 609 | it('should get the completed jobs', function (done) { 610 | queue.failed(function () { 611 | queue.state.calledWith('failed').should.be.true; 612 | done(); 613 | }); 614 | }); 615 | }); 616 | 617 | describe('Function: inactive', function() { 618 | var queue; 619 | 620 | beforeEach(function(){ 621 | queue = kue.createQueue(); 622 | sinon.stub(queue, 'state').callsArg(1); 623 | }); 624 | 625 | afterEach(function(){ 626 | queue.state.restore(); 627 | }); 628 | 629 | it('should get the completed jobs', function (done) { 630 | queue.inactive(function () { 631 | queue.state.calledWith('inactive').should.be.true; 632 | done(); 633 | }); 634 | }); 635 | }); 636 | 637 | describe('Function: active', function() { 638 | var queue; 639 | 640 | beforeEach(function(){ 641 | queue = kue.createQueue(); 642 | sinon.stub(queue, 'state').callsArg(1); 643 | }); 644 | 645 | afterEach(function(){ 646 | queue.state.restore(); 647 | }); 648 | 649 | it('should get the completed jobs', function (done) { 650 | queue.active(function () { 651 | queue.state.calledWith('active').should.be.true; 652 | done(); 653 | }); 654 | }); 655 | }); 656 | 657 | describe('Function: delayed', function() { 658 | var queue; 659 | 660 | beforeEach(function(){ 661 | queue = kue.createQueue(); 662 | sinon.stub(queue, 'state').callsArg(1); 663 | }); 664 | 665 | afterEach(function(){ 666 | queue.state.restore(); 667 | }); 668 | 669 | it('should get the completed jobs', function (done) { 670 | queue.delayed(function () { 671 | queue.state.calledWith('delayed').should.be.true; 672 | done(); 673 | }); 674 | }); 675 | }); 676 | 677 | describe('Function: completeCount', function() { 678 | var queue; 679 | 680 | beforeEach(function(){ 681 | queue = kue.createQueue(); 682 | sinon.stub(queue, 'card').callsArg(1); 683 | sinon.stub(queue, 'cardByType').callsArg(2); 684 | }); 685 | 686 | afterEach(function(){ 687 | queue.card.restore(); 688 | queue.cardByType.restore(); 689 | }); 690 | 691 | it('should get all completed jobs', function (done) { 692 | queue.completeCount(function () { 693 | queue.card.calledWith('complete').should.be.true; 694 | done(); 695 | }); 696 | }); 697 | 698 | it('should get all completed jobs of a certain type', function (done) { 699 | queue.completeCount('type', function () { 700 | queue.cardByType.calledWith('type', 'complete').should.be.true; 701 | done(); 702 | }); 703 | }); 704 | }); 705 | 706 | describe('Function: failedCount', function() { 707 | var queue; 708 | 709 | beforeEach(function(){ 710 | queue = kue.createQueue(); 711 | sinon.stub(queue, 'card').callsArg(1); 712 | sinon.stub(queue, 'cardByType').callsArg(2); 713 | }); 714 | 715 | afterEach(function(){ 716 | queue.card.restore(); 717 | queue.cardByType.restore(); 718 | }); 719 | 720 | it('should get all completed jobs', function (done) { 721 | queue.failedCount(function () { 722 | queue.card.calledWith('failed').should.be.true; 723 | done(); 724 | }); 725 | }); 726 | 727 | it('should get all completed jobs of a certain type', function (done) { 728 | queue.failedCount('type', function () { 729 | queue.cardByType.calledWith('type', 'failed').should.be.true; 730 | done(); 731 | }); 732 | }); 733 | }); 734 | 735 | describe('Function: inactiveCount', function() { 736 | var queue; 737 | 738 | beforeEach(function(){ 739 | queue = kue.createQueue(); 740 | sinon.stub(queue, 'card').callsArg(1); 741 | sinon.stub(queue, 'cardByType').callsArg(2); 742 | }); 743 | 744 | afterEach(function(){ 745 | queue.card.restore(); 746 | queue.cardByType.restore(); 747 | }); 748 | 749 | it('should get all completed jobs', function (done) { 750 | queue.inactiveCount(function () { 751 | queue.card.calledWith('inactive').should.be.true; 752 | done(); 753 | }); 754 | }); 755 | 756 | it('should get all completed jobs of a certain type', function (done) { 757 | queue.inactiveCount('type', function () { 758 | queue.cardByType.calledWith('type', 'inactive').should.be.true; 759 | done(); 760 | }); 761 | }); 762 | }); 763 | 764 | describe('Function: activeCount', function() { 765 | var queue; 766 | 767 | beforeEach(function(){ 768 | queue = kue.createQueue(); 769 | sinon.stub(queue, 'card').callsArg(1); 770 | sinon.stub(queue, 'cardByType').callsArg(2); 771 | }); 772 | 773 | afterEach(function(){ 774 | queue.card.restore(); 775 | queue.cardByType.restore(); 776 | }); 777 | 778 | it('should get all completed jobs', function (done) { 779 | queue.activeCount(function () { 780 | queue.card.calledWith('active').should.be.true; 781 | done(); 782 | }); 783 | }); 784 | 785 | it('should get all completed jobs of a certain type', function (done) { 786 | queue.activeCount('type', function () { 787 | queue.cardByType.calledWith('type', 'active').should.be.true; 788 | done(); 789 | }); 790 | }); 791 | }); 792 | 793 | describe('Function: delayedCount', function() { 794 | var queue; 795 | 796 | beforeEach(function(){ 797 | queue = kue.createQueue(); 798 | sinon.stub(queue, 'card').callsArg(1); 799 | sinon.stub(queue, 'cardByType').callsArg(2); 800 | }); 801 | 802 | afterEach(function(){ 803 | queue.card.restore(); 804 | queue.cardByType.restore(); 805 | }); 806 | 807 | it('should get all completed jobs', function (done) { 808 | queue.delayedCount(function () { 809 | queue.card.calledWith('delayed').should.be.true; 810 | done(); 811 | }); 812 | }); 813 | 814 | it('should get all completed jobs of a certain type', function (done) { 815 | queue.delayedCount('type', function () { 816 | queue.cardByType.calledWith('type', 'delayed').should.be.true; 817 | done(); 818 | }); 819 | }); 820 | }); 821 | 822 | }); --------------------------------------------------------------------------------