├── index.js ├── .gitignore ├── lib ├── models │ ├── task_log.js │ ├── task_parameter.js │ ├── task.js │ ├── scheduled_task.js │ └── index.js ├── worker.js └── martinet.js ├── .jshintrc ├── package.json ├── LICENSE ├── test └── test.js └── README.md /index.js: -------------------------------------------------------------------------------- 1 | 2 | module.exports = require('./lib/martinet'); 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | *.seed 3 | *.log 4 | *.csv 5 | *.dat 6 | *.out 7 | *.pid 8 | *.gz 9 | 10 | pids 11 | logs 12 | results 13 | 14 | npm-debug.log 15 | node_modules 16 | 17 | .DS_Store 18 | *.db -------------------------------------------------------------------------------- /lib/models/task_log.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | return sequelize.define('TaskLog', { 5 | 'content': {type: DataTypes.TEXT, required: true } 6 | }); 7 | }; 8 | -------------------------------------------------------------------------------- /lib/models/task_parameter.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | return sequelize.define('TaskParameter', { 5 | 'name': {type: DataTypes.STRING, required: true }, 6 | 'value': {type: DataTypes.TEXT, required: true } 7 | }); 8 | }; 9 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | // Details: https://github.com/victorporof/Sublime-JSHint#using-your-own-jshintrc-options 3 | // Example: https://github.com/jshint/jshint/blob/master/examples/.jshintrc 4 | // Documentation: http://www.jshint.com/docs/ 5 | "browser": true, 6 | "esnext": true, 7 | "globals": { 8 | "require": false, 9 | "define": false, 10 | "console": false, 11 | "module": false, 12 | "process": false, 13 | "__dirname": false, 14 | "exports": false, 15 | "GLOBAL": false 16 | }, 17 | "globalstrict": true, 18 | "quotmark": true, 19 | "smarttabs": true, 20 | "trailing": true, 21 | "undef": true, 22 | "unused": true 23 | } 24 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "martinet", 3 | "description": "Distributed job queue", 4 | "version": "1.0.3", 5 | "author": { 6 | "name": "Matthew Conlen", 7 | "email": "mc@mathisonian.com" 8 | }, 9 | "main": "index", 10 | "engines": { 11 | "node": "0.10.x", 12 | "npm": ">= 1.2.x" 13 | }, 14 | "licenses": [ 15 | { 16 | "type": "MIT" 17 | } 18 | ], 19 | "scripts": { 20 | "start": "./node_modules/.bin/nodemon index.js", 21 | "test": "NODE_ENV=test mocha --timeout 10000" 22 | }, 23 | "dependencies": { 24 | "date.js": "~0.1.1", 25 | "debug": "^2.2.0", 26 | "human-interval": "~0.1.3", 27 | "lodash": "~2.4.1", 28 | "q": "~1.0.0", 29 | "sequelize": "^1.7.10", 30 | "zmq": "^2.14.0" 31 | }, 32 | "devDependencies": { 33 | "expect.js": "^0.3.1", 34 | "mocha": "^1.21.4", 35 | "nodemon": "latest", 36 | "should": "latest", 37 | "sqlite3": "^3.0.0", 38 | "supertest": "latest" 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) Matthew Conlen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /lib/models/task.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | return sequelize.define('Task', { 5 | 'worker': {type: DataTypes.STRING, required: true }, 6 | 'name': {type: DataTypes.STRING, required: true }, 7 | 'description': {type: DataTypes.STRING, required: true }, 8 | 'complete': {type: DataTypes.BOOLEAN, required: true, defaultValue: false }, 9 | 'error': {type: DataTypes.BOOLEAN, required: true, defaultValue: false }, 10 | 'error_message': {type: DataTypes.TEXT, required: false }, 11 | 'progress': {type: DataTypes.DECIMAL, required: false, defaultValue: 0.0 } 12 | }, { 13 | instanceMethods: { 14 | addParameter: function(name, value) { 15 | var TaskParameter = sequelize.import(__dirname + '/task_parameter'); 16 | 17 | return TaskParameter.create({ 18 | name: name, 19 | value: value, 20 | taskId: this.id 21 | }); 22 | } 23 | } 24 | }); 25 | }; 26 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | 2 | var expect = require('expect.js'); 3 | 4 | var Martinet = require('..'); 5 | var Worker = Martinet.Worker; 6 | var _ = require('lodash'); 7 | 8 | 9 | describe('martinet tests', function() { 10 | 11 | var martinet, worker; 12 | var WORKER_PORT = 8010; 13 | 14 | it('should create a new Martinet object', function(done) { 15 | martinet = new Martinet(); 16 | expect(martinet).to.be.a(Martinet); 17 | done(); 18 | }); 19 | 20 | 21 | it('should create a new Worker object', function(done) { 22 | worker = new Worker(WORKER_PORT); 23 | expect(worker).to.be.a(Worker); 24 | done(); 25 | }); 26 | 27 | 28 | it('should register this worker with martinet', function(done) { 29 | martinet.addWorker('test-worker', WORKER_PORT); 30 | expect(martinet.workers).to.have.key('test-worker'); 31 | expect(martinet.workers).to.only.have.keys('test-worker'); 32 | done(); 33 | }); 34 | 35 | it('should execute a simple task', function(done) { 36 | 37 | worker.on('add', function(taskId, data, cb) { 38 | var sum = _.reduce(data.numbers, function(memo, num) { return memo + num; }, 0); 39 | expect(sum).to.be(21); 40 | done(); 41 | }); 42 | 43 | martinet.execute({ 44 | worker: 'test-worker', 45 | name: 'add', 46 | descriptions: 'add some numbers' 47 | }, { 48 | numbers: [1, 2, 3, 4, 5, 6] 49 | }); 50 | 51 | }); 52 | 53 | 54 | }) 55 | -------------------------------------------------------------------------------- /lib/models/scheduled_task.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | return sequelize.define('ScheduledTask', { 5 | 'worker': {type: DataTypes.STRING, required: true }, 6 | 'name': {type: DataTypes.STRING, required: true }, 7 | 'description': {type: DataTypes.STRING, required: true }, 8 | 'run_at': {type: DataTypes.BIGINT, required: true }, 9 | 'is_recurring': {type: DataTypes.BOOLEAN, required: true, defaultValue: false }, 10 | 'interval': {type: DataTypes.INTEGER, required: false } 11 | }, { 12 | instanceMethods: { 13 | addParameter: function(name, value) { 14 | var TaskParameter = sequelize.import(__dirname + '/task_parameter'); 15 | 16 | return TaskParameter.create({ 17 | name: name, 18 | value: value, 19 | ScheduledTaskId: this.id 20 | }); 21 | }, 22 | 23 | createTask: function() { 24 | var Task = sequelize.import(__dirname + '/task'); 25 | 26 | return Task.create({ 27 | worker: this.worker, 28 | name: this.name, 29 | description: this.description 30 | }); 31 | }, 32 | 33 | getParameters: function() { 34 | var TaskParameter = sequelize.import(__dirname + '/task_parameter'); 35 | 36 | return TaskParameter.findAll({ 37 | where: { 38 | ScheduledTaskId: this.id 39 | } 40 | }); 41 | } 42 | } 43 | }); 44 | }; 45 | 46 | -------------------------------------------------------------------------------- /lib/models/index.js: -------------------------------------------------------------------------------- 1 | //'use strict'; 2 | // see https://github.com/JeyDotC/articles/blob/master/EXPRESS%20WITH%20SEQUELIZE.md 3 | // for ways to organize this a little better 4 | var models = {}; 5 | var fs = require('fs'); 6 | var hasSetup = false; 7 | var Sequelize = require('sequelize'); 8 | var sequelize = null; 9 | 10 | var singleton = function() { 11 | 12 | this.setup = function (config){ 13 | 14 | 15 | if(hasSetup) { 16 | return models; 17 | } 18 | 19 | sequelize = new Sequelize(config.database, config.username, config.password, config.options); 20 | 21 | return init(config.sync); 22 | }; 23 | 24 | this.model = function (name){ 25 | return models[name]; 26 | }; 27 | 28 | this.Seq = function (){ 29 | return Sequelize; 30 | }; 31 | 32 | function init(sync) { 33 | 34 | if(sync == null) { 35 | sync = true; 36 | } 37 | 38 | models.sequelize = sequelize; 39 | 40 | // Bootstrap models 41 | fs.readdirSync(__dirname).forEach(function (file) { 42 | if (~file.indexOf('.js') && file.indexOf('index.js') < 0) { 43 | var model = sequelize.import(__dirname + '/' + file); 44 | models[model.name] = model; 45 | } 46 | }); 47 | 48 | 49 | models.TaskParameter.belongsTo(models.Task); 50 | models.TaskParameter.belongsTo(models.ScheduledTask); 51 | 52 | models.Task.hasMany(models.TaskLog); 53 | models.TaskLog.belongsTo(models.Task); 54 | 55 | if(sync) { 56 | sequelize 57 | .sync(); 58 | } 59 | 60 | hasSetup = true; 61 | return models; 62 | } 63 | 64 | if(singleton.caller != singleton.getInstance){ 65 | throw new Error('This object cannot be instantiated'); 66 | } 67 | 68 | }; 69 | 70 | singleton.instance = null; 71 | 72 | singleton.getInstance = function() { 73 | if(this.instance === null) { 74 | this.instance = new singleton(); 75 | } 76 | return this.instance; 77 | }; 78 | 79 | module.exports = singleton.getInstance(); -------------------------------------------------------------------------------- /lib/worker.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var zmq = require('zmq'); 4 | var _ = require('lodash'); 5 | var debug = require('debug')('martinet:worker'); 6 | 7 | function Worker(port, options) { 8 | if (!(this instanceof Worker)) { 9 | return new Worker(port, options); 10 | } 11 | 12 | this.pushSocket = zmq.socket('push'); 13 | this.pullSocket = zmq.socket('pull'); 14 | 15 | var defaults = { 16 | martinet_url: '127.0.0.1', 17 | martinet_port: '8009' 18 | }; 19 | 20 | 21 | 22 | this.options = _.defaults(options || {}, defaults); 23 | 24 | var martinetConnection = 'tcp://' + this.options.martinet_url + ':' + this.options.martinet_port; 25 | var workerConnection = 'tcp://' + this.options.martinet_url + ':' + port; 26 | 27 | debug('Starting push socket on ' + martinetConnection); 28 | this.pushSocket.connect(martinetConnection); 29 | debug('Starting pull socket on ' + workerConnection); 30 | this.pullSocket.connect(workerConnection); 31 | 32 | this.handlers = {}; 33 | var self = this; 34 | 35 | this.pullSocket.on('message', function(data) { 36 | var msg = JSON.parse(data.toString()); 37 | 38 | var handler = self.handlers[msg.name]; 39 | if(handler) { 40 | self._handle(handler, msg); 41 | } 42 | }); 43 | } 44 | 45 | Worker.VERSION = require('../package.json').version; 46 | module.exports = Worker; 47 | 48 | Worker.prototype.on = function(name, f) { 49 | this.handlers[name] = f; 50 | }; 51 | 52 | 53 | Worker.prototype.setComplete = function(task) { 54 | this.pushSocket.send(JSON.stringify({task: task, set: 'complete'})); 55 | }; 56 | 57 | Worker.prototype.setError = function(task, error) { 58 | this.pushSocket.send(JSON.stringify({task: task, set: 'error', error: error})); 59 | }; 60 | 61 | Worker.prototype.setProgress = function(task, progress) { 62 | this.pushSocket.send(JSON.stringify({task: task, set: 'progress', progress: progress})); 63 | }; 64 | 65 | 66 | Worker.prototype._handle = function(handler, task) { 67 | var self = this; 68 | handler(task.id, task.data, function(err) { 69 | if(err) { 70 | return self.setError(task.id, err); 71 | } 72 | self.setComplete(task.id); 73 | }); 74 | }; 75 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Martinet 2 | 3 | Distributed task management. 4 | 5 | Martinet is a database-backed, zeroMQ-based distributed task management system. It is persistent with respect to future and recurring tasks, so if your system goes down, those tasks will be unaffected. Martinet can use any [sequelize.js](github.com/sequelize/sequelize) compatible database as its backing database (SQLite is used by default). 6 | 7 | Martinet uses a push-pull messaging pattern to ensure efficiency when used in a distributed environment. 8 | 9 | ## Installation 10 | 11 | `npm install martinet` 12 | 13 | 14 | ## Usage 15 | 16 | This library is divided into two parts: the `Martinet` object, which 17 | handles dispatching and scheduling tasks, and the `Worker` object 18 | which receives said tasks and defines the actions to take upon being 19 | given certain tasks. 20 | 21 | ### Martinet 22 | 23 | 24 | #### Setup 25 | 26 | ```javascript 27 | var Martinet = require('martinet'); 28 | 29 | var martinet = new Martinet(); 30 | 31 | // Martinet allows you to create multiple workers 32 | // so that you can keep worker code in separate 33 | // logical modules. 34 | 35 | martinet.addWorker('WORKER_NAME_1', 'WORKER_PORT_1'); 36 | martinet.addWorker('WORKER_NAME_2', 'WORKER_PORT_2'); 37 | 38 | ``` 39 | 40 | #### Creating Tasks 41 | 42 | ##### Execute a task immediately 43 | 44 | ```javascript 45 | 46 | martinet.execute({ 47 | worker: 'WORKER_NAME', 48 | name: 'task_name', 49 | description: 'Do a thing' // Used in the backend so it's easier to lookup tasks later 50 | }, args); 51 | 52 | // args JSON object of named arguments, so like 53 | // { 54 | // thing_id: 1 55 | // } 56 | // 57 | // this object gets serialized and passed to the Worker 58 | // 59 | 60 | ``` 61 | 62 | ##### Execute a task in the future 63 | 64 | ```javascript 65 | 66 | martinet.schedule('in 20 minutes', { 67 | worker: 'WORKER_NAME', 68 | name: 'task_name', 69 | description: 'Do a thing in 20 minutes' 70 | }, args); 71 | 72 | ``` 73 | 74 | ##### Create a recurring task 75 | 76 | ```javascript 77 | 78 | martinet.every('30 minutes', { 79 | worker: 'WORKER_NAME', 80 | name: 'task_name', 81 | description: 'Do a thing every half hour', 82 | run_at: 'midnight' // optional time to start the recurring task 83 | }, args); 84 | 85 | ``` 86 | 87 | ### Workers 88 | 89 | 90 | #### Setup 91 | 92 | ```javascript 93 | 94 | var MartinetWorker = require('martinet').Worker; 95 | 96 | var WORKER_PORT = 3000; 97 | var worker = new MartinetWorker(WORKER_PORT, { 98 | martinet_url: '127.0.0.1', 99 | martinet_port: '8089' 100 | }); 101 | ``` 102 | 103 | #### Defining Tasks 104 | 105 | 106 | ```javascript 107 | 108 | worker.on('task_name', function(taskId, data, callback) { 109 | // do a thing. 110 | 111 | // if it's successful, callback(), 112 | // if there's an error, callback(err) 113 | 114 | }); 115 | 116 | ``` 117 | 118 | ## Options 119 | 120 | ### Martinet 121 | 122 | #### Port 123 | 124 | Custom port for martinet's pull socket to listen on. 125 | 126 | ```javascript 127 | var Martinet = require('martinet'); 128 | 129 | var options = { 130 | port: 8009 131 | }; 132 | 133 | var martinet = new Martinet(options); 134 | ``` 135 | 136 | #### DB 137 | 138 | Connection information to the backing database. Uses [sequelize.js options](http://sequelizejs.com/docs/1.7.8/usage#options). 139 | 140 | default is 141 | 142 | 143 | ```javascript 144 | var Martinet = require('martinet'); 145 | 146 | var options = { 147 | db: { 148 | database: 'martinet-db', 149 | username: process.env.USER, 150 | password: null, 151 | options: { 152 | dialect: 'sqlite', 153 | storage: 'martinet.db', 154 | logging: false, 155 | omitNull: true 156 | }, 157 | sync: true 158 | } 159 | }; 160 | 161 | var martinet = new Martinet(options); 162 | ``` 163 | 164 | but for example to use postgres: 165 | 166 | ```javascript 167 | var Martinet = require('martinet'); 168 | 169 | var options = { 170 | db: { 171 | database: 'martinet-db', 172 | username: process.env.USER, 173 | password: null, 174 | options: { 175 | dialect: 'postgres', 176 | port: 5432, 177 | host: 'database.host' 178 | logging: false, 179 | omitNull: true 180 | }, 181 | sync: true 182 | } 183 | }; 184 | 185 | var martinet = new Martinet(options); 186 | ``` 187 | 188 | ### Worker 189 | 190 | #### Martinet URL 191 | 192 | Connection string to connect to martinet. If worker is on the same machine as martinet, this should be 127.0.0.1 193 | 194 | #### Martinet PORT 195 | 196 | The port to connect to martinet on. This should be the same port defined by the martinet object's port option. 197 | -------------------------------------------------------------------------------- /lib/martinet.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * Module dependencies 5 | */ 6 | 7 | 8 | var zmq = require('zmq'); 9 | var _ = require('lodash'); 10 | var humanInterval = require('human-interval'); 11 | var Q = require('q'); 12 | var date = require('date.js'); 13 | var debug = require('debug')('martinet:controller'); 14 | 15 | function Martinet(options) { 16 | if (!(this instanceof Martinet)) { 17 | return new Martinet(options); 18 | } 19 | this.socket = zmq.socket('pull'); 20 | 21 | var defaults = { 22 | port: '8009', 23 | db: { 24 | database: 'martinet-db', 25 | username: process.env.USER, 26 | password: null, 27 | options: { 28 | dialect: 'sqlite', 29 | storage: 'martinet.db', 30 | logging: false, 31 | omitNull: true 32 | }, 33 | sync: true 34 | } 35 | }; 36 | 37 | this.options = _.defaults(options || {}, defaults); 38 | 39 | var connection = 'tcp://*:' + this.options.port; 40 | 41 | debug('Starting pull socket on ' + connection); 42 | this.socket.bindSync(connection); 43 | 44 | var models = require('./models'); 45 | this.models = models.setup(this.options.db); 46 | 47 | this.workers = {}; 48 | var self = this; 49 | 50 | this.socket.on('message', function(data) { 51 | var msg = JSON.parse(data.toString()); 52 | 53 | if(msg.set === 'complete') { 54 | self.setComplete(msg.task); 55 | } else if(msg.set === 'progress') { 56 | self.setProgress(msg.task, msg.progress); 57 | } else if(msg.set === 'error') { 58 | self.setError(msg.task, msg.error); 59 | } 60 | }); 61 | 62 | this._start(); 63 | } 64 | 65 | 66 | Martinet.VERSION = require('../package.json').version; 67 | module.exports = Martinet; 68 | module.exports.Worker = require('./worker'); 69 | 70 | 71 | Martinet.prototype.addWorker = function(name, port) { 72 | // keep a map from worker name -> ZMQ_Client 73 | var connection = 'tcp://*:' + port; 74 | var socket = zmq.socket('push'); 75 | socket.bindSync(connection); 76 | debug('Starting push socket on ' + connection); 77 | this.workers[name] = {socket: socket, connection: connection}; 78 | }; 79 | 80 | 81 | Martinet.prototype.execute = function(taskObj, parameters, cb) { 82 | 83 | // Create a task and then shoot it off 84 | var Task = this.models.Task; 85 | var socket = this.workers[taskObj.worker].socket; 86 | if(!socket) { 87 | return new Error('Target not found'); 88 | } 89 | Task.create(taskObj).success(function(task) { 90 | debug('Dispatching task ' + task.id); 91 | socket.send(JSON.stringify({id: task.id, name: task.name, data: parameters})); 92 | cb && cb(null, task.id); 93 | }).error(function(err) { 94 | cb && cb(err); 95 | }); 96 | }; 97 | 98 | 99 | Martinet.prototype.schedule = function(when, taskObj, parameters, cb) { 100 | var ScheduledTask = this.models.ScheduledTask; 101 | var TaskParameter = this.models.TaskParameter; 102 | var socket = this.workers[taskObj.worker].socket; 103 | 104 | if(!socket) { 105 | return new Error('Worker not found'); 106 | } 107 | 108 | ScheduledTask.create(_.extend(taskObj, { 109 | run_at: (when instanceof Date) ? when : date(when).valueOf() 110 | })).success(function(task) { 111 | _.each(parameters, function(value, key) { 112 | TaskParameter.create({ 113 | name: key, 114 | value: JSON.stringify(value), 115 | ScheduledTaskId: task.id 116 | }); 117 | }); 118 | cb && cb(null, task); 119 | }); 120 | }; 121 | 122 | Martinet.prototype.every = function(interval, taskObj, parameters, cb) { 123 | var ScheduledTask = this.models.ScheduledTask; 124 | var TaskParameter = this.models.TaskParameter; 125 | var socket = this.workers[taskObj.worker].socket; 126 | 127 | if(!socket) { 128 | return new Error('Worker not found'); 129 | } 130 | 131 | return ScheduledTask.create(_.extend(taskObj, { 132 | interval: humanInterval(interval), 133 | is_recurring: true, 134 | run_at: (taskObj.run_at) ? date(taskObj.run_at).valueOf() : Date.now().valueOf() 135 | })).success(function(task) { 136 | _.each(parameters, function(value, key) { 137 | TaskParameter.create({ 138 | name: key, 139 | value: JSON.stringify(value), 140 | ScheduledTaskId: task.id 141 | }); 142 | }); 143 | cb && cb(null, task); 144 | }); 145 | }; 146 | 147 | Martinet.prototype.updateTask = function(taskId, parameters) { 148 | var TaskParameter = this.models.TaskParameter; 149 | 150 | _.each(parameters, function(val, key) { 151 | TaskParameter 152 | .find({ 153 | where: { 154 | ScheduledTaskId: taskId, 155 | name: key 156 | } 157 | }).success(function(param) { 158 | param 159 | .updateAttributes({ 160 | name: key, 161 | value: JSON.stringify(val) 162 | }); 163 | }); 164 | }); 165 | }; 166 | 167 | Martinet.prototype.revoke = function(taskId) { 168 | var TaskParameter = this.models.TaskParameter; 169 | var ScheduledTask = this.models.ScheduledTask; 170 | 171 | Q.all([ 172 | TaskParameter.destroy({ 173 | ScheduledTaskId: taskId 174 | }), 175 | ScheduledTask.destroy({ 176 | id: taskId 177 | }) 178 | ]).spread(function() { 179 | debug('Successfully removed task ' + taskId); 180 | }); 181 | }; 182 | 183 | 184 | Martinet.prototype.setProgress = function(taskId, progress) { 185 | var Task = this.models.Task; 186 | debug('Task ' + taskId + ' progress ' + 100*progress + '%'); 187 | var self = this; 188 | 189 | Task.find(taskId) 190 | .success(function(task) { 191 | task.updateAttributes({ 192 | progress: progress 193 | }).success(function() { 194 | if(self._onProgress) { 195 | self._onProgress(task); 196 | } 197 | }).error(function(err) { 198 | debug(err); 199 | }); 200 | }).error(function(err) { 201 | debug(err); 202 | }); 203 | }; 204 | 205 | 206 | Martinet.prototype.setError = function(taskId, error) { 207 | debug('Task ' + taskId + ' error: ' + error); 208 | var Task = this.models.Task; 209 | var TaskLog = this.models.TaskLog; 210 | var self = this; 211 | 212 | Task.find(taskId) 213 | .success(function(task) { 214 | task.updateAttributes({error: true, error_message: error}) 215 | .success(function() { 216 | TaskLog.create({ 217 | TaskId: taskId, 218 | content: error 219 | }).success(function(){}) 220 | .error(function(err) { 221 | debug(err); 222 | }); 223 | 224 | if(self._onError) { 225 | self._onError(task); 226 | } 227 | }); 228 | }) 229 | .error(function(err) { 230 | debug(err); 231 | }); 232 | }; 233 | 234 | 235 | Martinet.prototype.setComplete = function(taskId) { 236 | debug('Completed task ' + taskId); 237 | var Task = this.models.Task; 238 | var self = this; 239 | Task.find(taskId) 240 | .success(function(task) { 241 | task.updateAttributes({ 242 | complete: true, 243 | progress: 1.0 244 | }).success(function() { 245 | if(self._onComplete) { 246 | self._onComplete(task); 247 | } 248 | }).error(function(err) { 249 | debug(err); 250 | }); 251 | }).error(function(err) { 252 | debug(err); 253 | }); 254 | }; 255 | 256 | Martinet.prototype.onComplete = function(f) { 257 | this._onComplete = f; 258 | }; 259 | 260 | Martinet.prototype.onError = function(f) { 261 | this._onError = f; 262 | }; 263 | 264 | Martinet.prototype.onProgress = function(f) { 265 | this._onProgress = f; 266 | }; 267 | 268 | 269 | Martinet.prototype._start = function() { 270 | // what we need to do: 271 | // 272 | 273 | // periodically watch scheduled tasks and see if they are overdue 274 | 275 | this._scheduledInterval = setInterval(_checkScheduledTasks.bind(this), humanInterval('5 seconds')); 276 | }; 277 | 278 | Martinet.prototype._stop = function() { 279 | clearInterval(this._scheduledInterval); 280 | this._scheduledInterval = undefined; 281 | }; 282 | 283 | 284 | var _checkScheduledTasks = function() { 285 | var ScheduledTask = this.models.ScheduledTask; 286 | var self = this; 287 | 288 | ScheduledTask.findAll({ 289 | where: { 290 | run_at: { 291 | lte: Date.now().valueOf() 292 | } 293 | } 294 | }).success(function(scheduledTasks) { 295 | _.each(scheduledTasks, function(scheduledTask) { 296 | Q.all([ 297 | scheduledTask.createTask(), 298 | scheduledTask.getParameters() 299 | ]).spread(function(task, parameters) { 300 | var socket = self.workers[task.worker].socket; 301 | 302 | var data = {}; 303 | _.each(parameters, function(param) { 304 | data[param.name] = JSON.parse(param.value); 305 | }); 306 | 307 | debug('Dispatching task ' + task.id); 308 | socket.send(JSON.stringify({id: task.id, name: task.name, data: data})); 309 | 310 | // if it was recurring, schedule it again, otherwise destroy it 311 | if(scheduledTask.is_recurring) { 312 | scheduledTask.updateAttributes({ 313 | run_at: Date.now().valueOf() + scheduledTask.interval 314 | }); 315 | } else { 316 | scheduledTask.destroy(); 317 | } 318 | }); 319 | }); 320 | }); 321 | }; 322 | --------------------------------------------------------------------------------