├── .dockerignore ├── .env ├── docker-compose.override.yml ├── Dockerfile ├── docker-compose.yml ├── utils └── config.js ├── app.js ├── package.json ├── .gitignore ├── LICENSE ├── routes └── index.js ├── dq.yml ├── README.md ├── bin └── www └── lib ├── executor.js └── q.js /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | NODE_ENV=development 2 | DEBUG=dq:* 3 | COMPOSE_PROJECT_NAME=dq 4 | -------------------------------------------------------------------------------- /docker-compose.override.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | services: 3 | api: 4 | volumes: 5 | - '.:/dq' 6 | - '/dq/node_modules' 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:5.11.1 2 | 3 | COPY package.json / 4 | RUN npm install 5 | 6 | COPY . /dq 7 | WORKDIR /dq 8 | 9 | CMD ["node", "./bin/www"] 10 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | services: 3 | api: 4 | build: . 5 | command: node ./bin/www 6 | ports: 7 | - "5972:3000" 8 | - "5973:3001" 9 | volumes: 10 | - "/var/run/docker.sock:/var/run/docker.sock" 11 | env_file: 12 | - ./.env 13 | redis: 14 | image: redis:latest 15 | -------------------------------------------------------------------------------- /utils/config.js: -------------------------------------------------------------------------------- 1 | var nconf = require('nconf'); 2 | var yaml = require('js-yaml'); 3 | 4 | var job_config = __dirname + '/../dq.yml'; 5 | 6 | // load cmd line args and environment vars 7 | nconf.argv().env(); 8 | 9 | // load a yaml file using a custom formatter 10 | nconf.file({ 11 | file: job_config, 12 | format: { 13 | parse: yaml.safeLoad, 14 | stringify: yaml.safeDump, 15 | } 16 | }); 17 | 18 | module.exports = nconf; 19 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var path = require('path'); 3 | var logger = require('morgan'); 4 | var cookieParser = require('cookie-parser'); 5 | var bodyParser = require('body-parser'); 6 | 7 | var routes = require('./routes/index'); 8 | 9 | var app = express(); 10 | 11 | app.use(logger('dev')); 12 | app.use(bodyParser.json()); 13 | app.use(bodyParser.urlencoded({ extended: false })); 14 | app.use(cookieParser()); 15 | 16 | app.use('/', routes); 17 | 18 | // catch 404 and forward to error handler 19 | app.use(function(req, res, next) { 20 | res.sendStatus(404); 21 | }); 22 | 23 | module.exports = app; 24 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dq", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "start": "node ./bin/www" 7 | }, 8 | "dependencies": { 9 | "basic-auth-connect": "^1.0.0", 10 | "body-parser": "~1.15.1", 11 | "cookie-parser": "~1.4.3", 12 | "cron-converter": "0.0.11", 13 | "debug": "~2.2.0", 14 | "dockerode": "github:apocas/dockerode", 15 | "express": "~4.13.4", 16 | "js-yaml": "^3.6.1", 17 | "kue": "^0.11.1", 18 | "kue-unique": "^1.0.0", 19 | "morgan": "~1.7.0", 20 | "nconf": "^0.8.4", 21 | "request": "^2.74.0", 22 | "string": "^3.3.1" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # nyc test coverage 18 | .nyc_output 19 | 20 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 21 | .grunt 22 | 23 | # node-waf configuration 24 | .lock-wscript 25 | 26 | # Compiled binary addons (http://nodejs.org/api/addons.html) 27 | build/Release 28 | 29 | # Dependency directories 30 | node_modules 31 | jspm_packages 32 | 33 | # Optional npm cache directory 34 | .npm 35 | 36 | # Optional REPL history 37 | .node_repl_history 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Shrikrishna Holla 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /routes/index.js: -------------------------------------------------------------------------------- 1 | var express = require('express') 2 | , router = express.Router() 3 | , queue = require('./../lib/q'); 4 | 5 | /* GET home page. */ 6 | router.get('/', function(req, res, next) { 7 | res.json({ success: true }); 8 | }); 9 | 10 | /* POST A JOB */ 11 | 12 | router.post('/v1/jobs/:jobType', function(req, res, next){ 13 | console.log("Creating " + req.params.jobType + " job"); 14 | queue.addJob(req.params.jobType, req.body, function(err, jobid){ 15 | if(err){ 16 | console.log("Error occurred:" + err); 17 | next(err); 18 | } 19 | res.json({message: "job created successfully", jobid : jobid}); 20 | }); 21 | }); 22 | 23 | /* GET JOB INFO */ 24 | 25 | router.get('/v1/jobs/:jobid', function(req, res, next){ 26 | console.log("Info for job with id:" + req.params.jobid); 27 | queue.jobState(req.params.jobid, function(err, job){ 28 | log.info("JOb info : " + job); 29 | if(err) { 30 | console.log("Error occurred: " + err); 31 | } 32 | res.json({jobInfo: job}); 33 | }); 34 | }); 35 | 36 | 37 | /* DELETE JOBS */ 38 | 39 | router.delete('/v1/jobs/:jobid', function(req, res, next){ 40 | console.log("Deleting job with id:" + req.params.jobid); 41 | queue.removeJob(req.params.jobid, function(err){ 42 | if(err) { 43 | console.log("Error occurred: " + err) 44 | } 45 | res.json({success: true, message: "job with id:" + req.params.jobid + " has been removed"}); 46 | }); 47 | }); 48 | 49 | 50 | 51 | module.exports = router; 52 | -------------------------------------------------------------------------------- /dq.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | jobs: 3 | mail: 4 | image: shrikrishna/dq_sendmail:latest 5 | command: > 6 | --smtp-user='{{smtp_user}}' 7 | --smtp-pass='{{smtp_pass}}' 8 | --smtp-domain='{{smtp_domain}}' 9 | --from='{{from}}' 10 | --to='{{to}}' 11 | --subject='{{subject}}' 12 | --body='{{body}}' 13 | --tls={{is_tls}} 14 | remove: true 15 | priority: "high" 16 | attempts: 10 17 | backoff: true 18 | ttl: 30000 19 | batch: 5 20 | 21 | serverless_example: 22 | image: shrikrishna/dq_serverless_example:latest 23 | service: 24 | port: 3000 25 | max_replica: 5 26 | endpoints: 27 | vote: 28 | uri: "/" 29 | method: "GET" 30 | 31 | logrotate: 32 | image: logrotater:latest 33 | repeat: "0 0 1 * *" 34 | volumes: 35 | - "/var/log/mylog:/var/log/logrotater" 36 | 37 | mongobackup: 38 | image: mongo:latest 39 | command: "mongodump -o /data/backup" 40 | repeat: "0 0 1 * *" 41 | volumes: 42 | - "/data/db:/data/db" 43 | - "/data/backup:/data/backup" 44 | 45 | 46 | test: 47 | image: test:latest 48 | command: "hello world" 49 | repeat: "*/2 * * * *" 50 | volumes: 51 | - "/data/db:/data/db" 52 | - "/data/backup:/data/backup" 53 | remove: true 54 | priority: "high" 55 | attempts: 10 56 | backoff: true 57 | ttl: 5000 58 | dq: 59 | redis: 60 | port: 6379 61 | host: "redis" 62 | db: 3 63 | queue: 64 | watch_interval: 10000 65 | ui: 66 | enabled: true 67 | title: "DQ" 68 | port: 3001 69 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dq 2 | Task Scheduler for Docker Functions 3 | 4 | DQ is an amalgamation of the best features of a job scheduler like [resque](https://github.com/resque/resque) and a serverless platform like [AWS lambda](https://aws.amazon.com/lambda/details/). It is meant to be self-hosted and called using REST APIs. All tasks run on docker containers defined in a spec similar to compose called `dq.yml` 5 | 6 | [![DQ](http://img.youtube.com/vi/XT2CeC6oMBU/0.jpg)](http://www.youtube.com/watch?v=XT2CeC6oMBU "DQ: Task Scheduler for Docker Functions") 7 | > DQ won [4th place](https://blog.docker.com/2016/08/announcing-docker-1-12-hackathon-winners/) in the Docker 1.12 Hackathon 8 | 9 | ### Features 10 | - Call a background task with arguments [Ex: sending email] 11 | - Schedule a recurring task [Ex: backing up db] 12 | - Run a background service that handles HTTP requests and auto scales based on load [voting, spam checking] 13 | 14 | ## Requirements 15 | To use some features like services, you will need Docker 1.12 or above 16 | 17 | ## Usage 18 | - Create a `dq.yml` based on the [sample](https://github.com/shrikrishnaholla/dq/blob/master/dq.yml) in this repo 19 | - Create a network 20 | ```docker network create dq_net``` 21 | - Start redis 22 | ```docker run -d --name redis --net dq_net redis``` 23 | - Start DQ 24 | ```docker run -it --net dq_net --name dq -p "5972:3000" -p "5973:3001" \ 25 | -v $(pwd)/dq.yml:/dq/dq.yml -v /var/run/docker.sock:/var/run/docker.sock \ 26 | shrikrishna/dq``` 27 | 28 | ## Development 29 | Fork/clone this repo and edit `dq.yml` 30 | - [Fork this repo](https://github.com/shrikrishnaholla/dq/#fork-destination-box) 31 | - ```docker-compose up -d``` 32 | 33 | 34 | -------------------------------------------------------------------------------- /bin/www: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | /** 4 | * Module dependencies. 5 | */ 6 | 7 | var app = require('../app'); 8 | var debug = require('debug')('dq:server'); 9 | var http = require('http'); 10 | 11 | /** 12 | * Get port from environment and store in Express. 13 | */ 14 | 15 | var port = normalizePort(process.env.PORT || '3000'); 16 | app.set('port', port); 17 | 18 | /** 19 | * Create HTTP server. 20 | */ 21 | 22 | var server = http.createServer(app); 23 | 24 | /** 25 | * Listen on provided port, on all network interfaces. 26 | */ 27 | 28 | server.listen(port); 29 | server.on('error', onError); 30 | server.on('listening', onListening); 31 | 32 | /** 33 | * Normalize a port into a number, string, or false. 34 | */ 35 | 36 | function normalizePort(val) { 37 | var port = parseInt(val, 10); 38 | 39 | if (isNaN(port)) { 40 | // named pipe 41 | return val; 42 | } 43 | 44 | if (port >= 0) { 45 | // port number 46 | return port; 47 | } 48 | 49 | return false; 50 | } 51 | 52 | /** 53 | * Event listener for HTTP server "error" event. 54 | */ 55 | 56 | function onError(error) { 57 | if (error.syscall !== 'listen') { 58 | throw error; 59 | } 60 | 61 | var bind = typeof port === 'string' 62 | ? 'Pipe ' + port 63 | : 'Port ' + port; 64 | 65 | // handle specific listen errors with friendly messages 66 | switch (error.code) { 67 | case 'EACCES': 68 | console.error(bind + ' requires elevated privileges'); 69 | process.exit(1); 70 | break; 71 | case 'EADDRINUSE': 72 | console.error(bind + ' is already in use'); 73 | process.exit(1); 74 | break; 75 | default: 76 | throw error; 77 | } 78 | } 79 | 80 | /** 81 | * Event listener for HTTP server "listening" event. 82 | */ 83 | 84 | function onListening() { 85 | var addr = server.address(); 86 | var bind = typeof addr === 'string' 87 | ? 'pipe ' + addr 88 | : 'port ' + addr.port; 89 | debug('Listening on ' + bind); 90 | } 91 | -------------------------------------------------------------------------------- /lib/executor.js: -------------------------------------------------------------------------------- 1 | var Docker = require('dockerode') 2 | , docker = new Docker({socketPath: '/var/run/docker.sock'}) 3 | , S = require('string') 4 | , request = require('request') 5 | , os = require('os') 6 | , debug = require('debug')('dq:executor'); 7 | 8 | var execute = function (job, opts, callback) { 9 | 10 | var command; 11 | if (job.data.command) { 12 | command = opts.command ? S(opts.command).template(job.data.command).s 13 | .split(/(?:\r\n|\r|\n| )/g) : job.data.command; 14 | } 15 | 16 | var volumeArr; 17 | if (opts.volumes) { 18 | volumeArr = opts.volumes.map(function(volumeStr) { return volumeStr.split(':')} ); 19 | } 20 | 21 | var createOpts = { 22 | Image: opts.image, 23 | AttachStdin: false, 24 | AttachStdout: false, 25 | AttachStderr: false, 26 | Tty: false, 27 | }; 28 | if (command) { 29 | createOpts.Cmd = command; 30 | } 31 | if (volumeArr && volumeArr.length > 0) { 32 | createOpts.Volumes = {}; 33 | createOpts.HostConfig = {}; 34 | volumeArr.forEach(function(volumeDetails, index, arr) { 35 | if (volumeDetails.length <= 0) { 36 | return callback(new Error('Invalid volume string: ' + arr[index])); 37 | } 38 | else if (volumeDetails.length == 1) { 39 | createOpts.Volumes[volumeDetails[0]] = {}; 40 | } else { 41 | createOpts.Volumes[volumeDetails[1]] = {}; 42 | if(!createOpts.HostConfig.Binds) createOpts.HostConfig.Binds = []; 43 | createOpts.HostConfig.Binds.push(opts.volumes[index]); 44 | } 45 | }); 46 | } 47 | 48 | createOpts.Labels = { 49 | "DQ": opts.job_name 50 | } 51 | 52 | debug('creating container with opts', createOpts); 53 | 54 | // docker.createContainer({ 55 | // Image: 'ubuntu', 56 | // Cmd: ['/bin/ls', '/stuff'], 57 | // 'Volumes': { 58 | // '/stuff': {} 59 | // } 60 | // }, function(err, container) { 61 | // container.attach({ 62 | // stream: true, 63 | // stdout: true, 64 | // stderr: true, 65 | // tty: true 66 | // }, function(err, stream) { 67 | // stream.pipe(process.stdout); 68 | 69 | // container.start({ 70 | // 'Binds': ['/home/vagrant:/stuff'] 71 | // }, function(err, data) { 72 | // console.log(data); 73 | // }); 74 | // }); 75 | // }); 76 | 77 | docker.run(opts.image, command, undefined, createOpts, function (err, data, container) { 78 | debug('run', opts.image, data); 79 | if (callback) { 80 | if (err) { return callback(err) } 81 | callback(); 82 | } else { 83 | debug(err, data); 84 | } 85 | }); 86 | } 87 | 88 | var scale = function (service_name, service_opts, num_queued, callback) { 89 | 90 | var service = docker.getService(service_name); 91 | 92 | service.inspect(function(insp_err, svc_data) { 93 | // Don't replicate beyond the limits set by max_replica 94 | if (num_queued > 0 && svc_data.Spec.Mode.Replicated.Replicas >= service_opts.service.max_replica) { 95 | return callback(new Error('Service' + service_opts.image + 96 | 'already at max replica capacity')); 97 | } else if (num_queued == 0 && svc_data.Spec.Mode.Replicated.Replicas == 1) { 98 | return callback(new Error('Service' + service_opts.image + 99 | 'already at min replica capacity')); 100 | } 101 | 102 | var updateOpts = { 103 | Name: service_name, // Need name else will be set to random name 104 | version: svc_data.Version.Index, // Needed 105 | TaskTemplate: { 106 | ContainerSpec: { 107 | Image: service_opts.image 108 | } 109 | }, 110 | Mode: { 111 | Replicated: { // Scale by one 112 | Replicas: num_queued == 0 ? svc_data.Spec.Mode.Replicated.Replicas - 1 : // if 0, scale down, else scale up 113 | svc_data.Spec.Mode.Replicated.Replicas + 1 114 | } 115 | }, 116 | 117 | // Networks : [svc_data.Endpoint.VirtualIPs.NetworkID], 118 | 119 | EndpointSpec : { 120 | Ports: [ 121 | { 122 | Protocol: "tcp", 123 | PublishedPort: service_opts.service.port 124 | } 125 | ] 126 | } 127 | 128 | }; 129 | 130 | debug('Scaling with updateOpts', updateOpts); 131 | docker.getService(svc_data.ID).update(updateOpts, function(err, data, service) { 132 | if (callback) { 133 | if (err) return callback(err); 134 | else callback(null, data, num_queued == 0 ? svc_data.Spec.Mode.Replicated.Replicas - 1 : 135 | svc_data.Spec.Mode.Replicated.Replicas + 1); 136 | } else { 137 | debug(err, data); 138 | } 139 | }); 140 | }); 141 | 142 | } 143 | 144 | var startService = function(name, opts, callback) { 145 | 146 | var service = docker.getService(name); 147 | service.inspect(function(insp_err, svc_data) { 148 | if (insp_err && insp_err.statusCode == 404) { 149 | // Service doesn't exist. Create. 150 | var volumeArr; 151 | if (opts.volumes) { 152 | volumeArr = opts.volumes.map(function(volumeStr) { return volumeStr.split(':')} ); 153 | } 154 | 155 | var createOpts = { 156 | Name: name, 157 | TaskTemplate: { 158 | ContainerSpec: { 159 | Image: opts.image 160 | } 161 | }, 162 | Mode: { 163 | Replicated: { 164 | Replicas: 1 165 | } 166 | }, 167 | } 168 | 169 | if (opts.command) { 170 | createOpts.TaskTemplate.ContainerSpec.Command = opts.command; 171 | } 172 | if (volumeArr && volumeArr.length > 0) { 173 | createOpts.TaskTemplate.ContainerSpec.Mounts = []; 174 | 175 | volumeArr.forEach(function(volumeDetails, index, arr) { 176 | var next_volume = {}; 177 | if (volumeDetails.length <= 0) { 178 | return callback(new Error('Invalid volume string: ' + arr[index])); 179 | } 180 | else if (volumeDetails.length == 1) { 181 | next_volume.Target = volumeDetails[0]; 182 | next_volume.Type = 'volume'; 183 | } else { 184 | next_volume.Source = volumeDetails[0]; 185 | next_volume.Target = volumeDetails[1]; 186 | if (volumeDetails.length == 3) { 187 | if (volumeDetails[2] == 'ro') next_volume.ReadOnly = true; 188 | else if (volumeDetails[2] == 'rw') next_volume.ReadOnly = false; 189 | } else { 190 | next_volume.ReadOnly = false; 191 | } 192 | } 193 | 194 | createOpts.TaskTemplate.ContainerSpec.Mounts.push(next_volume); 195 | }); 196 | } 197 | createOpts.Labels = { 198 | "DQ": opts.job_name, 199 | "DQ:Service": "true" 200 | } 201 | 202 | createOpts.EndpointSpec = { 203 | Ports: [ 204 | { 205 | Protocol: "tcp", 206 | PublishedPort: opts.service.port 207 | } 208 | ] 209 | } 210 | 211 | 212 | createService(createOpts, callback); 213 | 214 | } else { 215 | // Service already exists. Return 216 | callback(null, svc_data); 217 | } 218 | }); 219 | 220 | } 221 | 222 | 223 | var createService = function(createOpts, callback, previouslyAttempted) { 224 | docker.createService(createOpts, function(err, data) { 225 | if (callback) { 226 | if (err) { 227 | if (previouslyAttempted) { return callback(err); } // Prevent stack overflow 228 | if (err.statusCode == 406) { 229 | // Init Swarm and try again 230 | initSwarm(function(swarm_err) { 231 | if (swarm_err) { 232 | if (callback) return callback(swarm_err); 233 | } else { 234 | if (!previouslyAttempted) { previouslyAttempted = true } 235 | createService(createOpts, callback, previouslyAttempted); 236 | } 237 | }); 238 | } 239 | return callback(err); 240 | } 241 | else { 242 | // Successful creation of service 243 | var service = docker.getService(createOpts.Name); 244 | service.inspect(function(insp_err, svc_data) { 245 | if (insp_err) debug(insp_err); 246 | // Connect DQ container to service container's Network 247 | debug('svc data before connecting networks', svc_data); 248 | connectNetwork(svc_data.Endpoint.VirtualIPs[0].NetworkID, function(conn_err) { 249 | debug('Connected to service\'s network', conn_err); 250 | callback(null, data); 251 | }); 252 | }); 253 | } 254 | } else { 255 | debug(err, data); 256 | } 257 | }); 258 | } 259 | 260 | var initSwarm = function(callback) { 261 | debug('Initializing swarm'); 262 | docker.swarmInit({ 263 | "ListenAddr": "0.0.0.0:4500", 264 | "AdvertiseAddr": "192.168.1.1:4500", 265 | "ForceNewCluster": false, 266 | "Spec": { 267 | "Orchestration": {}, 268 | "Raft": {}, 269 | "Dispatcher": {}, 270 | "CAConfig": {} 271 | } 272 | }, callback); 273 | } 274 | 275 | 276 | // Executor method that makes request to the service backends 277 | var mkRequest = function(service_name, opts, callback) { 278 | var service = docker.getService(service_name); 279 | 280 | service.inspect(function(insp_err, svc_data) { 281 | if (insp_err) debug(insp_err); 282 | var container_ip = svc_data.Endpoint.VirtualIPs[0].Addr.split('/')[0]; 283 | var base = 'http://' + container_ip + ':' + svc_data.Endpoint.Ports[0].PublishedPort; // base url 284 | debug('Making', opts.method, 'request to', base, 'at', opts.uri, 'with qs', opts.qs, 'and body', opts.body); 285 | 286 | request({ 287 | method: opts.method, 288 | uri: opts.uri, 289 | baseUrl: base, 290 | qs: opts.qs, 291 | body: opts.body 292 | // json: opts.json || false TODO 293 | }, function(error, response, body) { 294 | if (error) return callback(error); 295 | callback(null, response); 296 | }); 297 | 298 | }); 299 | } 300 | 301 | var connectNetwork = function(network_id, callback) { 302 | var network = docker.getNetwork(network_id); 303 | network.connect({ 304 | Container: os.hostname() 305 | }, callback) 306 | } 307 | 308 | exports.execute = execute; 309 | exports.startService = startService; 310 | exports.scale = scale; 311 | exports.mkRequest = mkRequest; 312 | -------------------------------------------------------------------------------- /lib/q.js: -------------------------------------------------------------------------------- 1 | var kue = require('kue') 2 | , basicAuth = require('basic-auth-connect') 3 | , express = require('express') 4 | , Cron = require('cron-converter') 5 | , cronInstance = new Cron() 6 | , debug = require('debug')('dq:q') 7 | , conf = require('../utils/config') 8 | , executor = require('./executor'); 9 | 10 | // Setup start 11 | var q = kue.createQueue({ 12 | prefix: 'dq', 13 | redis: { 14 | port: conf.get('dq:redis:port') || 6379, 15 | host: conf.get('dq:redis:host') || 'localhost', 16 | db: conf.get('dq:redis:db') || 'dq' 17 | } 18 | }); 19 | 20 | q.watchStuckJobs(conf.get('dq:queue:watch_interval') || 10000); 21 | 22 | // Mark incomplete jobs before shutdown as active so that they can be retried 23 | q.active( function( err, ids ) { 24 | ids.forEach( function( id ) { 25 | kue.Job.get( id, function( err, job ) { 26 | job.inactive(); 27 | }); 28 | }); 29 | }); 30 | 31 | 32 | // Start Kue UI server 33 | if ( conf.get('dq:ui:enabled')) { 34 | var app = express(); 35 | if ( conf.get('dq:ui:auth:username') && conf.get('dq:ui:auth:password') ) { 36 | app.use(basicAuth('foo', 'bar')); 37 | } 38 | kue.app.set('title', conf.get('dq:ui:title') || 'DQ'); 39 | app.use(kue.app); 40 | app.listen(conf.get('dq:ui:port') || 3000); 41 | } 42 | 43 | q.on( 'error', function( err ) { 44 | console.log( 'Error in queue: ', err ); 45 | shutdownKue(); 46 | }); 47 | 48 | process.once( 'SIGTERM', function ( sig ) { 49 | shutdownKue(); 50 | }); 51 | 52 | var shutdownKue = function() { 53 | q.shutdown( 5000, function(err) { 54 | console.log( 'Kue shutdown: ', err||'' ); 55 | process.exit( 0 ); 56 | }); 57 | } 58 | 59 | // Setup End 60 | 61 | 62 | var addJob = function(job_type, options, callback) { 63 | if (conf.get('jobs:' + job_type) == undefined) { 64 | return callback(new Error('Incorrect job type ' + job_type)); 65 | } 66 | var job = q.create(job_type, options); // options contains job-specific options 67 | 68 | if (conf.get('jobs:' + job_type + ':priority')) { 69 | // Possible values : { low: 10, normal: 0, medium: -5, high: -10, critical: -15 } 70 | job.priority(conf.get('jobs:' + job_type + ':priority')); 71 | } 72 | 73 | if (conf.get('jobs:' + job_type + ':attempts')) { 74 | job.attempts(conf.get('jobs:' + job_type + ':attempts')); 75 | } 76 | 77 | if (conf.get('jobs:' + job_type + ':backoff')) { 78 | job.backoff(conf.get('jobs:' + job_type + ':backoff')); 79 | } 80 | 81 | if (conf.get('jobs:' + job_type + ':ttl')) { 82 | job.ttl(conf.get('jobs:' + job_type + ':ttl')); 83 | } 84 | 85 | // Recurrent jobs 86 | if (conf.get('jobs:' + job_type + ':repeat')) { 87 | cronInstance.fromString(conf.get('jobs:' + job_type + ':repeat')); 88 | var schedule = cronInstance.schedule(); 89 | var next = schedule.next(); 90 | 91 | // If last executed ts is same as next execution date, increment schedule 92 | // To handle sub-second execution recurrent jobs 93 | kue.redis.client().get('dq:repeat:' + job_type, function(redis_err, reply) { 94 | if (redis_err) { 95 | return callback(redis_err); 96 | } 97 | if (reply) { 98 | last_ts = reply.split('::')[1]; // unix ts of last exec 99 | if (next.valueOf().toString() == last_ts ) { 100 | next = schedule.next(); 101 | } 102 | } 103 | job.delay(next.toDate()); 104 | 105 | // Save ts of the next execution schedule in redis 106 | kue.redis.client().set('dq:repeat:' + job_type, 107 | JSON.stringify(options) + '::' + next.valueOf().toString()); 108 | }); 109 | } 110 | 111 | job.save(function(err) { 112 | if (err) callback(err); 113 | else { 114 | callback(null, job.id); 115 | } 116 | }) 117 | } 118 | 119 | 120 | var removeJob = function(job_id, callback) { 121 | kue.Job.get( job_id, function(err, job) { 122 | if (err) { return callback(err) } 123 | job.remove(callback) 124 | }); // Args to callback - err 125 | } 126 | 127 | 128 | var jobState = function(job_id, callback) { 129 | kue.Job.get( job_id, callback); // Args to callback - err, job 130 | } 131 | 132 | 133 | // Job processing 134 | Object.keys(conf.get('jobs')).forEach( function(job_type) { 135 | q.process(job_type, 136 | conf.get('jobs:' + job_type + ':batch') || 1, 137 | function(job, done) { 138 | 139 | if (conf.get('jobs:' + job_type + ':service')) { 140 | var endpoint_conf_str = 'jobs:' + job_type + ':service:endpoints:' + job.data.endpoint; 141 | executor.mkRequest(job_type, { 142 | method: conf.get(endpoint_conf_str + ':method'), 143 | uri: conf.get(endpoint_conf_str + ':uri'), 144 | qs: job.data.qs, 145 | body: job.data.body 146 | }, done); 147 | 148 | } else { 149 | 150 | // Gather metadata for executor 151 | var opts = conf.get('jobs:' + job_type); 152 | opts.job_name = job_type; 153 | 154 | // Handle uncaught exceptions in executor 155 | var domain = require('domain').create(); 156 | domain.on('error', function(err){ 157 | done(err); 158 | }); 159 | domain.run(function() { 160 | executor.execute(job, opts, function(error) { 161 | if (error) return done(error); 162 | 163 | // If recurring task, create a duplicate job for next execution 164 | if (conf.get('jobs:' + job_type + ':repeat')) { 165 | kue.redis.client().get('dq:repeat:' + job_type, function(redis_err, reply) { 166 | if (redis_err || (reply == null)) { 167 | return done(redis_err || 168 | new Error('Invalid key: dq:repeat:' + job_type)); 169 | } 170 | addJob(job_type, JSON.parse(reply.split('::')[0]), 171 | function(new_job_err, new_job_id) { 172 | done(); 173 | }); 174 | }) 175 | } else { 176 | // Non-recurring, single execution job 177 | done(); 178 | } 179 | }) 180 | }); // done - err or null 181 | } 182 | }); 183 | }); 184 | 185 | 186 | // recurring jobs and services are initialized at process startup 187 | // recurring tasks won’t take per-execution args that would be client-provided 188 | Object.keys(conf.get('jobs')).forEach( function(job_type) { 189 | if(conf.get('jobs:' + job_type + ':repeat')) { 190 | if (conf.get('jobs:' + job_type + ':batch') !== undefined) { 191 | console.log('Cannot enable batch processing for recurrent job', job_type); 192 | shutdownKue(); 193 | } 194 | kue.redis.client().exists('dq:repeat:' + job_type, function(redis_err, reply) { 195 | if (redis_err) { 196 | // If we encounter redis error, we shouldn't risk writes 197 | return debug('Redis error while fetching recurrent jobs'); 198 | } 199 | // Need to add job only if it doesn't already exist 200 | if (reply == null || reply == 0 || reply == '0') { 201 | // add job 202 | addJob(job_type, {}, function(error, job_id) { 203 | if (error) debug('Error while creating recurrent job', error); 204 | else debug('Registered recurrent job', job_type, 'at', job_id); 205 | }); 206 | } else { 207 | debug('Ignored pre-existing recurrent job', job_type); 208 | } 209 | }); 210 | } else if (conf.get('jobs:' + job_type + ':service')) { 211 | executor.startService(job_type, conf.get('jobs:' + job_type), 212 | function(svc_err, data) { 213 | if (svc_err) debug(svc_err); 214 | else { 215 | // Set concurrency equal to number of replicas 216 | if (data.Mode) conf.set('jobs:' + job_type + ':batch', data.Spec.Mode.Replicated.Replicas); 217 | debug('Service', job_type, 'successfully started'); 218 | } 219 | }); 220 | } 221 | }); 222 | 223 | var serviceBacklogTimeout; 224 | var checkServiceJobsBacklog = function () { 225 | Object.keys(conf.get('jobs')).forEach( function(job_type) { 226 | if(conf.get('jobs:' + job_type + ':service')) { 227 | q.inactiveCount(job_type, function(err, total) { 228 | debug('Inactive count', err, total); 229 | 230 | if (err) { 231 | debug('Error getting total queued jobs for service', 232 | job_type, err); 233 | } else if (total > 1000) { // TODO: Justify number or take from user 234 | debug('Number of queued jobs for', job_type, 'reaching', total, '. Scaling up now'); 235 | executor.scale(job_type, conf.get('jobs:' + job_type), total, function(error, data, num_replica) { 236 | if (error) { 237 | debug('Error scaling service', 238 | job_type, error); 239 | } else { 240 | conf.set('jobs:' + job_type + ':batch', num_replica); 241 | debug('Successfully scaled up', job_type, 'to', num_replica, 'replica'); 242 | } 243 | }); 244 | } else if (total == 0) { 245 | debug('Number of queued jobs for', job_type, 'at', total, '. Scaling down now'); 246 | executor.scale(job_type, conf.get('jobs:' + job_type), total, function(error, data, num_replica) { 247 | if (error) { 248 | debug('Error scaling service', 249 | job_type, error); 250 | } else { 251 | conf.set('jobs:' + job_type + ':batch', num_replica); 252 | debug('Successfully scaled down', job_type, 'to', num_replica, 'replica'); 253 | } 254 | }); 255 | } 256 | }); 257 | } 258 | }); 259 | 260 | if (serviceBacklogTimeout) { clearTimeout(serviceBacklogTimeout) } 261 | serviceBacklogTimeout = setTimeout(checkServiceJobsBacklog, 60000); 262 | } 263 | setTimeout(checkServiceJobsBacklog, 60000); 264 | 265 | 266 | exports.addJob = addJob; 267 | exports.removeJob = removeJob; 268 | exports.jobState = jobState; 269 | 270 | 271 | // // Test method 272 | // var executor = { 273 | // execute : function(job, opts, callback) { 274 | // debug('got data', job.data, opts); 275 | // callback(); 276 | // }, 277 | // scale : function(job_name, service_opts, callback) { 278 | // debug('got request to scale', job_name, 'with', service_opts); 279 | // callback(); 280 | // } 281 | // } 282 | 283 | // TODO: Make this module event based 284 | // // - `enqueue` the job is now queued 285 | // // - `start` the job is now running 286 | // // - `promotion` the job is promoted from delayed state to queued 287 | // // - `progress` the job's progress ranging from 0-100 288 | // // - `failed attempt` the job has failed, but has remaining attempts yet 289 | // // - `failed` the job has failed and has no remaining attempts 290 | // // - `complete` the job has completed 291 | // // - `remove` the job has been removed 292 | // queue.on('job enqueue', function(id, type){ 293 | // debug( 'Job %s got queued of type %s', id, type ); 294 | // }) 295 | --------------------------------------------------------------------------------