├── .babelrc ├── .codeclimate.yml ├── .coveralls.yml ├── .editorconfig ├── .eslintrc ├── .gitignore ├── .npmignore ├── .travis.yml ├── LICENSE ├── README.md ├── dist ├── bin │ └── dispo.js ├── client.js ├── index.js ├── logger.js ├── mailer.js └── util.js ├── example ├── jobs.json └── jobs │ ├── fail.js │ └── random.js ├── package.json ├── src ├── bin │ └── dispo.js ├── client.js ├── index.js ├── logger.js ├── mailer.js └── util.js ├── test └── index.js └── yarn.lock /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | "es2015" 4 | ], 5 | "plugins": [ 6 | "transform-async-to-generator", 7 | "transform-runtime" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | engines: 2 | eslint: 3 | enabled: true 4 | 5 | ratings: 6 | paths: 7 | - "**.js" 8 | -------------------------------------------------------------------------------- /.coveralls.yml: -------------------------------------------------------------------------------- 1 | service_name: travis-ci 2 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | charset = utf-8 7 | indent_style = space 8 | indent_size = 2 9 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "babel-eslint", 3 | "extends": "standard", 4 | "env": { 5 | "mocha": true, 6 | "node": true 7 | }, 8 | "plugins": [ 9 | "babel", 10 | "import" 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | coverage/ 2 | node_modules/ 3 | *.log 4 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | coverage 2 | src 3 | test 4 | .codeclimate.yml 5 | .coveralls.yml 6 | .travis.yml 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "5" 4 | - "6" 5 | after_script: 6 | - NODE_ENV=test istanbul cover `npm bin`/_mocha --report lcovonly -- -R spec --compilers js:babel-register && cat ./coverage/lcov.info | `npm bin`/coveralls && rm -rf ./coverage 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Christoph Werner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dispo 2 | 3 | [![npm version](https://badge.fury.io/js/dispo.svg)](https://badge.fury.io/js/dispo) [![Build Status](https://travis-ci.org/gonsfx/dispo.svg?branch=master)](https://travis-ci.org/gonsfx/dispo) [![Coverage Status](https://coveralls.io/repos/github/gonsfx/dispo/badge.svg?branch=master)](https://coveralls.io/github/gonsfx/dispo?branch=master) [![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg)](http://standardjs.com/) 4 | 5 | Dispo is a job and cronjob scheduler for Node. 6 | 7 | It uses [Kue](https://github.com/Automattic/kue) as its job queue and [Redis](http://redis.io/) to store job data. Definition of recurring jobs is done using crontab syntax, one-off jobs can be queued using [ZeroMQ](http://zeromq.org/) requests. [nodemailer](https://github.com/nodemailer/nodemailer) and [nodemailer-sendmail-transport](https://github.com/andris9/nodemailer-sendmail-transport) are being used to send emails. 8 | 9 | All Jobs, regardless if running automatically or queued on demand for single runs, have to be defined in a configuration file. Jobs defined as cronjobs with a recurring interval are scheduled and run automatically. 10 | 11 | ## Requirements 12 | 13 | - [Redis](http://redis.io/) 14 | - [ZeroMQ](http://zeromq.org/) 15 | 16 | ## Installation 17 | 18 | ``` 19 | > npm install dispo 20 | ``` 21 | 22 | ## Usage 23 | 24 | Dispo provides a binary to start from the command line. 25 | 26 | ``` 27 | > node_modules/.bin/dispo -h 28 | 29 | Usage: dispo [options] 30 | 31 | Options: 32 | 33 | -h, --help output usage information 34 | -V, --version output the version number 35 | -C, --config config file path, default: `jobs.json` 36 | -B, --basedir directory used as a base for relative config and job paths 37 | ``` 38 | 39 | The `--config` or `-C` option is required and points to the relative path of the configuration file. If no configuration path is given, the configuration is expected to be in `jobs.json` relative to the current working directory. 40 | 41 | ### Job configuration 42 | 43 | Jobs are defined in the `jobs` property of the configuration file. Each job, identified using a name, must point its `file` property to a JavaScript file exporting a function that is run when the job is executed. 44 | 45 | The following configuration example defines a job called `logRandomNumber` that can be queued on-demand and a recurring job called `databaseCleanup` that is defined to run on 01:00 am every day using [crontab syntax](https://en.wikipedia.org/wiki/Cron). 46 | 47 | The `attempts` property on the database cleanup job defines that the job is only attempted to run once. When the property is not explicitely set, it defaults to 3 so a job is retried twice on failure. When a recurringly scheduled job/cronjob reaches fails on each of its attempts, it is not automatically rescheduled. 48 | 49 | The `recipients` property will override the default set in [mailer.js](src/mailer.js). You can also use it to disable sending an email for a job when it is enabled globally. You can set the global configuration in [index.js](src/index.js). 50 | 51 | ```json 52 | { 53 | "jobs": { 54 | "logRandomNumber": { 55 | "file": "jobs/logRandomNumber.js" 56 | }, 57 | "databaseCleanup": { 58 | "file": "jobs/databaseCleanup.js", 59 | "cron": "0 1 * * * *", 60 | "attempts": 1, 61 | "recipients": "example@email.com" 62 | } 63 | } 64 | } 65 | ``` 66 | 67 | #### Send email on job failure 68 | 69 | Dispo supports sending an email when a job fails after all available retries, using [nodemailer](https://github.com/nodemailer/nodemailer) to send the mails. 70 | To do so, simply enable the `mailer` in the configuration file and add email addresses that should be notified whenever a job fails to `notifyOnError` on a per job basis. 71 | 72 | ```json 73 | { 74 | "options": { 75 | "mailer": true 76 | }, 77 | "jobs": { 78 | "mightFail": { 79 | "file": "jobs/mightFail.js", 80 | "cron": "*/1 * * * *", 81 | "attempts": 3, 82 | "notifyOnError": "john.doe@example.com" 83 | } 84 | } 85 | } 86 | ``` 87 | 88 | By default, dispo will use [nodemailer-sendmail-transport](https://github.com/andris9/nodemailer-sendmail-transport) to send your emails, but you're free to add a different nodemailer transport such as smtp. 89 | You can also set mail options, such as `from`. 90 | 91 | ```javascript 92 | import nodemailer from 'nodemailer' 93 | 94 | module.exports = { 95 | options: { 96 | mailer: { 97 | transport: nodemailer.createTransport('smtp://smtp.example.com') 98 | mail: { 99 | from: 'dispo-reporter@example.com' 100 | } 101 | } 102 | }, 103 | jobs: { 104 | mightFail: { 105 | file: 'jobs/mightFail.js', 106 | cron: '*/1 * * * *', 107 | attempts: 3, 108 | notifyOnError: 'john.doe@example.com' 109 | } 110 | } 111 | } 112 | ``` 113 | 114 | #### Attempts with delay and backoff for failing jobs 115 | 116 | Jobs that sometimes fail to execute correctly (or any job in general to be precise) can be configured to restart with a delay after they fail. You can use this feature via the `backoff` property. 117 | Provide `backoff.type = 'fixed'` and the `backoff.delay = ` in milliseconds to set a fixed delay in milliseconds that will be waited after a failed attempt to execute the job. 118 | Provide `backoff.type = 'exponential'` and the `backoff.delay = ` in milliseconds to set a exponential growing delay in milliseconds that will be waited after a failed attempt to execute the job. The base of the expenential growth will be your given delay. 119 | 120 | The following configuration example defines a job called `flakyService` that is defined to run every minute on every day. 121 | `flakyService` will be executed a second, third and fourth time when it fails (`attempts: 4`), but the second, third and fourth try will each wait 3 seconds before re-executing. 122 | 123 | ```json 124 | { 125 | "flakyService": { 126 | "file": "jobs/flakyService.js", 127 | "cron": "*/1 * * * *", 128 | "attempts": 4, 129 | "backoff": { 130 | "delay": 3000, 131 | "type": "fixed" 132 | } 133 | } 134 | } 135 | ``` 136 | 137 | ##### incremental 138 | 139 | The following configuration example defines a job called `flakyServiceWithLongRegenerationTime` that is defined to run every minute on every day. 140 | `flakyServiceWithLongRegenerationTime` will be executed a second, third, fourth, fifth and sixth time when it fails (`attempts: 4`), but: 141 | - the second try will wait 6 seconds, 142 | - the third try will wait 9 seconds, 143 | - the forth try will wait 12 seconds, 144 | before re-executing. 145 | 146 | ```json 147 | { 148 | "flakyServiceWithLongRegenerationTime": { 149 | "file": "jobs/flakyServiceWithLongRegenerationTime.js", 150 | "cron": "*/1 * * * *", 151 | "attempts": 4, 152 | "backoff": { 153 | "delay": 3000, 154 | "type": "incremental" 155 | } 156 | } 157 | } 158 | ``` 159 | 160 | ##### exponential 161 | 162 | The following configuration example defines a job called `anotherFlakyServiceWithLongRegenerationTime` that is defined to run every minute on every day. 163 | `anotherFlakyServiceWithLongRegenerationTime` will be executed a second and third time when it fails (`attempts: 4`), but: 164 | - the second try will wait 4 seconds (= 2000 * 2000 milliseconds), 165 | - the third try will wait 16 seconds (= 4000 * 4000 milliseconds), 166 | before re-executing. 167 | 168 | ```json 169 | { 170 | "anotherFlakyServiceWithLongRegenerationTime": { 171 | "file": "jobs/anotherFlakyServiceWithLongRegenerationTime.js", 172 | "cron": "*/1 * * * *", 173 | "attempts": 3, 174 | "backoff": { 175 | "delay": 2000, 176 | "type": "exponential" 177 | } 178 | } 179 | } 180 | ``` 181 | -------------------------------------------------------------------------------- /dist/bin/dispo.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | 'use strict'; 3 | 4 | var _path = require('path'); 5 | 6 | var _commander = require('commander'); 7 | 8 | var _commander2 = _interopRequireDefault(_commander); 9 | 10 | var _ = require('..'); 11 | 12 | var _2 = _interopRequireDefault(_); 13 | 14 | var _package = require('../../package.json'); 15 | 16 | var _util = require('../util'); 17 | 18 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 19 | 20 | _commander2.default.version(_package.version).option('-C, --config ', 'config file path, default: `jobs.json`').option('-B, --basedir ', 'directory used as a base for relative config and job paths').parse(process.argv); 21 | 22 | _commander2.default.config = _commander2.default.config || 'jobs.json'; 23 | _commander2.default.basedir = _commander2.default.basedir || process.cwd(); 24 | 25 | try { 26 | var config = require((0, _util.getAbsolutePath)((0, _path.resolve)(_commander2.default.basedir, _commander2.default.config))); 27 | config.jobs = (0, _util.parseJobs)(config.jobs, _commander2.default.basedir); 28 | var scheduler = new _2.default(config); 29 | scheduler.init(); 30 | } catch (e) { 31 | console.log('errored', e); 32 | throw e; 33 | } -------------------------------------------------------------------------------- /dist/client.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var _stringify = require('babel-runtime/core-js/json/stringify'); 4 | 5 | var _stringify2 = _interopRequireDefault(_stringify); 6 | 7 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 8 | 9 | var zmq = require('zeromq'); 10 | var requester = zmq.socket('req'); 11 | 12 | var data = { 13 | name: 'randomSingle', 14 | delay: 5000, 15 | meta: { userId: '57bc570c6d505c72cc4d505f' } 16 | }; 17 | 18 | var onMessage = function onMessage(reply) { 19 | console.log(reply.toString()); 20 | }; 21 | 22 | requester.on('message', onMessage); 23 | requester.connect('tcp://localhost:5555'); 24 | 25 | setInterval(function () { 26 | requester.send((0, _stringify2.default)(data)); 27 | }, 1e3); -------------------------------------------------------------------------------- /dist/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | exports.getJob = undefined; 7 | 8 | var _assign = require('babel-runtime/core-js/object/assign'); 9 | 10 | var _assign2 = _interopRequireDefault(_assign); 11 | 12 | var _regenerator = require('babel-runtime/regenerator'); 13 | 14 | var _regenerator2 = _interopRequireDefault(_regenerator); 15 | 16 | var _getIterator2 = require('babel-runtime/core-js/get-iterator'); 17 | 18 | var _getIterator3 = _interopRequireDefault(_getIterator2); 19 | 20 | var _asyncToGenerator2 = require('babel-runtime/helpers/asyncToGenerator'); 21 | 22 | var _asyncToGenerator3 = _interopRequireDefault(_asyncToGenerator2); 23 | 24 | var _classCallCheck2 = require('babel-runtime/helpers/classCallCheck'); 25 | 26 | var _classCallCheck3 = _interopRequireDefault(_classCallCheck2); 27 | 28 | var _createClass2 = require('babel-runtime/helpers/createClass'); 29 | 30 | var _createClass3 = _interopRequireDefault(_createClass2); 31 | 32 | var _kue = require('kue'); 33 | 34 | var _kue2 = _interopRequireDefault(_kue); 35 | 36 | var _later = require('later'); 37 | 38 | var _later2 = _interopRequireDefault(_later); 39 | 40 | var _assert = require('assert'); 41 | 42 | var _assert2 = _interopRequireDefault(_assert); 43 | 44 | var _zeromq = require('zeromq'); 45 | 46 | var _zeromq2 = _interopRequireDefault(_zeromq); 47 | 48 | var _bluebird = require('bluebird'); 49 | 50 | var _lodash = require('lodash'); 51 | 52 | var _logger = require('./logger'); 53 | 54 | var _logger2 = _interopRequireDefault(_logger); 55 | 56 | var _mailer = require('./mailer'); 57 | 58 | var _mailer2 = _interopRequireDefault(_mailer); 59 | 60 | var _util = require('./util'); 61 | 62 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 63 | 64 | var NODE_ENV = process.env.NODE_ENV; 65 | 66 | /** 67 | * Default scheduler config 68 | */ 69 | 70 | var defaultConfig = { 71 | jobs: [], 72 | options: { 73 | port: 5555, 74 | logging: { path: 'log' }, 75 | mailer: null 76 | } 77 | }; 78 | 79 | /** 80 | * Promisified version of `kue.Job.rangeByType` 81 | * @type {Function} 82 | */ 83 | var getJobsByType = (0, _bluebird.promisify)(_kue2.default.Job.rangeByType); 84 | 85 | /** 86 | * Promisified version of `kue.Job.get` 87 | * 88 | * @type {Function} 89 | */ 90 | var getJob = exports.getJob = (0, _bluebird.promisify)(_kue2.default.Job.get); 91 | 92 | /** 93 | * Dispo Scheduler 94 | */ 95 | 96 | var Dispo = function () { 97 | 98 | /** 99 | * Creates an instance of Dispo. 100 | * 101 | * @memberOf Dispo 102 | * @param {Object} [config={}] 103 | */ 104 | function Dispo() { 105 | var config = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {}; 106 | (0, _classCallCheck3.default)(this, Dispo); 107 | 108 | this.config = (0, _lodash.merge)({}, defaultConfig, config); 109 | } 110 | 111 | /** 112 | * Initializes logging, socket bindings and the queue mechanism 113 | * 114 | * @memberOf Dispo 115 | * @return {Promise} 116 | */ 117 | 118 | 119 | (0, _createClass3.default)(Dispo, [{ 120 | key: 'init', 121 | value: function () { 122 | var _ref = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee() { 123 | var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, job; 124 | 125 | return _regenerator2.default.wrap(function _callee$(_context) { 126 | while (1) { 127 | switch (_context.prev = _context.next) { 128 | case 0: 129 | this._logger = new _logger2.default(this.config.options.logging); 130 | this._logger.init(); 131 | 132 | if (this.config.options.mailer) { 133 | this._mailer = new _mailer2.default(this.config.options.mailer); 134 | this._mailer.init(); 135 | } 136 | 137 | this._initSocket(); 138 | this._initQueue(this.config.options.queue); 139 | 140 | _iteratorNormalCompletion = true; 141 | _didIteratorError = false; 142 | _iteratorError = undefined; 143 | _context.prev = 8; 144 | _iterator = (0, _getIterator3.default)(this.config.jobs); 145 | 146 | case 10: 147 | if (_iteratorNormalCompletion = (_step = _iterator.next()).done) { 148 | _context.next = 17; 149 | break; 150 | } 151 | 152 | job = _step.value; 153 | _context.next = 14; 154 | return this.defineJob(job); 155 | 156 | case 14: 157 | _iteratorNormalCompletion = true; 158 | _context.next = 10; 159 | break; 160 | 161 | case 17: 162 | _context.next = 23; 163 | break; 164 | 165 | case 19: 166 | _context.prev = 19; 167 | _context.t0 = _context['catch'](8); 168 | _didIteratorError = true; 169 | _iteratorError = _context.t0; 170 | 171 | case 23: 172 | _context.prev = 23; 173 | _context.prev = 24; 174 | 175 | if (!_iteratorNormalCompletion && _iterator.return) { 176 | _iterator.return(); 177 | } 178 | 179 | case 26: 180 | _context.prev = 26; 181 | 182 | if (!_didIteratorError) { 183 | _context.next = 29; 184 | break; 185 | } 186 | 187 | throw _iteratorError; 188 | 189 | case 29: 190 | return _context.finish(26); 191 | 192 | case 30: 193 | return _context.finish(23); 194 | 195 | case 31: 196 | case 'end': 197 | return _context.stop(); 198 | } 199 | } 200 | }, _callee, this, [[8, 19, 23, 31], [24,, 26, 30]]); 201 | })); 202 | 203 | function init() { 204 | return _ref.apply(this, arguments); 205 | } 206 | 207 | return init; 208 | }() 209 | 210 | /** 211 | * @typedef {Object} DefineJobOptions 212 | * @property {String} name - Job name 213 | * @property {Function} fn - Job method that is executed when the job is run 214 | * @property {Number} attempts - Number of attempts a job is retried until marked as failure 215 | * @property {String} cron - Interval-based scheduling written in cron syntax, ignored when delay is given 216 | */ 217 | /** 218 | * Defines a job 219 | * 220 | * @memberOf Dispo 221 | * @param {DefineJobOptions} options - Job options 222 | * @return {Promise} 223 | */ 224 | 225 | }, { 226 | key: 'defineJob', 227 | value: function () { 228 | var _ref2 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee2(_ref3) { 229 | var attempts = _ref3.attempts, 230 | cron = _ref3.cron, 231 | notifyOnError = _ref3.notifyOnError, 232 | fn = _ref3.fn, 233 | name = _ref3.name, 234 | backoff = _ref3.backoff; 235 | var options; 236 | return _regenerator2.default.wrap(function _callee2$(_context2) { 237 | while (1) { 238 | switch (_context2.prev = _context2.next) { 239 | case 0: 240 | (0, _assert2.default)(name, 'Job must have a name'); 241 | 242 | options = { attempts: attempts, backoff: backoff }; 243 | 244 | this._queue.process(name, function (job, done) { 245 | return fn(job).then(done, done); 246 | }); 247 | 248 | if (notifyOnError) { 249 | options.notifyOnError = notifyOnError; 250 | } 251 | 252 | if (!cron) { 253 | _context2.next = 8; 254 | break; 255 | } 256 | 257 | options.cron = cron; 258 | _context2.next = 8; 259 | return this._queueJob(name, options); 260 | 261 | case 8: 262 | case 'end': 263 | return _context2.stop(); 264 | } 265 | } 266 | }, _callee2, this); 267 | })); 268 | 269 | function defineJob(_x2) { 270 | return _ref2.apply(this, arguments); 271 | } 272 | 273 | return defineJob; 274 | }() 275 | 276 | /** 277 | * Initializes the queue mechanism 278 | * 279 | * This is mostly done to set up queue level logging and to be able to automatically 280 | * queue the next runs of cronjobs after their previous runs have completed. 281 | * 282 | * @memberOf Dispo 283 | * @param {Object} [options={}] 284 | */ 285 | 286 | }, { 287 | key: '_initQueue', 288 | value: function _initQueue() { 289 | var _this = this; 290 | 291 | var options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {}; 292 | 293 | this._queue = _kue2.default.createQueue(options); 294 | this._queue.watchStuckJobs(5e3); 295 | 296 | if (NODE_ENV !== 'test') { 297 | this._queue.on('job start', function () { 298 | var _ref4 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee3(id) { 299 | return _regenerator2.default.wrap(function _callee3$(_context3) { 300 | while (1) { 301 | switch (_context3.prev = _context3.next) { 302 | case 0: 303 | _context3.next = 2; 304 | return _this._handleStart(id); 305 | 306 | case 2: 307 | return _context3.abrupt('return', _context3.sent); 308 | 309 | case 3: 310 | case 'end': 311 | return _context3.stop(); 312 | } 313 | } 314 | }, _callee3, _this); 315 | })); 316 | 317 | return function (_x4) { 318 | return _ref4.apply(this, arguments); 319 | }; 320 | }()); 321 | this._queue.on('job failed attempt', function () { 322 | var _ref5 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee4(id, msg) { 323 | return _regenerator2.default.wrap(function _callee4$(_context4) { 324 | while (1) { 325 | switch (_context4.prev = _context4.next) { 326 | case 0: 327 | _context4.next = 2; 328 | return _this._handleFailedAttempt(id, msg); 329 | 330 | case 2: 331 | return _context4.abrupt('return', _context4.sent); 332 | 333 | case 3: 334 | case 'end': 335 | return _context4.stop(); 336 | } 337 | } 338 | }, _callee4, _this); 339 | })); 340 | 341 | return function (_x5, _x6) { 342 | return _ref5.apply(this, arguments); 343 | }; 344 | }()); 345 | this._queue.on('job failed', function () { 346 | var _ref6 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee5(id, msg) { 347 | return _regenerator2.default.wrap(function _callee5$(_context5) { 348 | while (1) { 349 | switch (_context5.prev = _context5.next) { 350 | case 0: 351 | _context5.next = 2; 352 | return _this._handleFailed(id, msg); 353 | 354 | case 2: 355 | return _context5.abrupt('return', _context5.sent); 356 | 357 | case 3: 358 | case 'end': 359 | return _context5.stop(); 360 | } 361 | } 362 | }, _callee5, _this); 363 | })); 364 | 365 | return function (_x7, _x8) { 366 | return _ref6.apply(this, arguments); 367 | }; 368 | }()); 369 | } 370 | 371 | this._queue.on('job complete', function () { 372 | var _ref7 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee6(id) { 373 | return _regenerator2.default.wrap(function _callee6$(_context6) { 374 | while (1) { 375 | switch (_context6.prev = _context6.next) { 376 | case 0: 377 | _context6.next = 2; 378 | return _this._handleComplete(id); 379 | 380 | case 2: 381 | return _context6.abrupt('return', _context6.sent); 382 | 383 | case 3: 384 | case 'end': 385 | return _context6.stop(); 386 | } 387 | } 388 | }, _callee6, _this); 389 | })); 390 | 391 | return function (_x9) { 392 | return _ref7.apply(this, arguments); 393 | }; 394 | }()); 395 | } 396 | 397 | /** 398 | * Logs job starts 399 | * 400 | * @memberOf Dispo 401 | * @param {Number} id - Job id 402 | */ 403 | 404 | }, { 405 | key: '_handleStart', 406 | value: function () { 407 | var _ref8 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee7(id) { 408 | return _regenerator2.default.wrap(function _callee7$(_context7) { 409 | while (1) { 410 | switch (_context7.prev = _context7.next) { 411 | case 0: 412 | _context7.next = 2; 413 | return this._logger.logStart(id); 414 | 415 | case 2: 416 | case 'end': 417 | return _context7.stop(); 418 | } 419 | } 420 | }, _callee7, this); 421 | })); 422 | 423 | function _handleStart(_x10) { 424 | return _ref8.apply(this, arguments); 425 | } 426 | 427 | return _handleStart; 428 | }() 429 | 430 | /** 431 | * Logs failed attempts 432 | * 433 | * @memberOf Dispo 434 | * @param {Number} id - Job id 435 | * @param {String} msg - Error message 436 | */ 437 | 438 | }, { 439 | key: '_handleFailedAttempt', 440 | value: function () { 441 | var _ref9 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee8(id, msg) { 442 | return _regenerator2.default.wrap(function _callee8$(_context8) { 443 | while (1) { 444 | switch (_context8.prev = _context8.next) { 445 | case 0: 446 | _context8.next = 2; 447 | return this._logger.logFailedAttempt(id, msg); 448 | 449 | case 2: 450 | case 'end': 451 | return _context8.stop(); 452 | } 453 | } 454 | }, _callee8, this); 455 | })); 456 | 457 | function _handleFailedAttempt(_x11, _x12) { 458 | return _ref9.apply(this, arguments); 459 | } 460 | 461 | return _handleFailedAttempt; 462 | }() 463 | 464 | /** 465 | * Logs failed jobs and sends notification emails if configured to do so 466 | * 467 | * @memberOf Dispo 468 | * @param {Number} id - Job id 469 | * @param {String} msg - Error message 470 | */ 471 | 472 | }, { 473 | key: '_handleFailed', 474 | value: function () { 475 | var _ref10 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee9(id, msg) { 476 | var job; 477 | return _regenerator2.default.wrap(function _callee9$(_context9) { 478 | while (1) { 479 | switch (_context9.prev = _context9.next) { 480 | case 0: 481 | _context9.next = 2; 482 | return this._logger.logFailure(id, msg); 483 | 484 | case 2: 485 | _context9.next = 4; 486 | return getJob(id); 487 | 488 | case 4: 489 | job = _context9.sent; 490 | 491 | if (!this._mailer) { 492 | _context9.next = 8; 493 | break; 494 | } 495 | 496 | _context9.next = 8; 497 | return this._mailer.sendMail(id, job.error()); 498 | 499 | case 8: 500 | case 'end': 501 | return _context9.stop(); 502 | } 503 | } 504 | }, _callee9, this); 505 | })); 506 | 507 | function _handleFailed(_x13, _x14) { 508 | return _ref10.apply(this, arguments); 509 | } 510 | 511 | return _handleFailed; 512 | }() 513 | 514 | /** 515 | * Logs completed jobs and re-queues them when defined as a cron 516 | * 517 | * @memberOf Dispo 518 | * @param {Number} id - Job id 519 | */ 520 | 521 | }, { 522 | key: '_handleComplete', 523 | value: function () { 524 | var _ref11 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee10(id) { 525 | var job; 526 | return _regenerator2.default.wrap(function _callee10$(_context10) { 527 | while (1) { 528 | switch (_context10.prev = _context10.next) { 529 | case 0: 530 | if (!(NODE_ENV !== 'test')) { 531 | _context10.next = 3; 532 | break; 533 | } 534 | 535 | _context10.next = 3; 536 | return this._logger.logComplete(id); 537 | 538 | case 3: 539 | _context10.next = 5; 540 | return getJob(id); 541 | 542 | case 5: 543 | job = _context10.sent; 544 | 545 | if (!job.data.cron) { 546 | _context10.next = 9; 547 | break; 548 | } 549 | 550 | _context10.next = 9; 551 | return this._queueJob(job.data.name, job.data); 552 | 553 | case 9: 554 | case 'end': 555 | return _context10.stop(); 556 | } 557 | } 558 | }, _callee10, this); 559 | })); 560 | 561 | function _handleComplete(_x15) { 562 | return _ref11.apply(this, arguments); 563 | } 564 | 565 | return _handleComplete; 566 | }() 567 | 568 | /** 569 | * Initialize ØMQ reply socket 570 | * 571 | * Received messages add new jobs to the queue when the given job is defined in 572 | * the job configuration 573 | * 574 | * @memberOf Dispo 575 | * @param {Number|String} [port=this.config.options.port] 576 | */ 577 | 578 | }, { 579 | key: '_initSocket', 580 | value: function _initSocket() { 581 | var _this2 = this; 582 | 583 | var port = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : this.config.options.port; 584 | 585 | var responder = _zeromq2.default.socket('rep'); 586 | 587 | responder.on('message', function () { 588 | var _ref12 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee11(message) { 589 | var payload, job, data; 590 | return _regenerator2.default.wrap(function _callee11$(_context11) { 591 | while (1) { 592 | switch (_context11.prev = _context11.next) { 593 | case 0: 594 | payload = JSON.parse(message.toString()); 595 | job = _this2.config.jobs.filter(function (job) { 596 | return job.name === payload.name; 597 | }).shift(); 598 | 599 | if (!job) { 600 | _context11.next = 7; 601 | break; 602 | } 603 | 604 | data = (0, _lodash.omit)((0, _assign2.default)(payload, job), 'fn', 'name'); 605 | _context11.next = 6; 606 | return _this2._queueJob(job.name, data); 607 | 608 | case 6: 609 | responder.send('ok'); 610 | 611 | case 7: 612 | case 'end': 613 | return _context11.stop(); 614 | } 615 | } 616 | }, _callee11, _this2); 617 | })); 618 | 619 | return function (_x17) { 620 | return _ref12.apply(this, arguments); 621 | }; 622 | }()); 623 | 624 | responder.bind('tcp://*:' + port, function (err) { 625 | if (err) { 626 | throw new Error('Port binding: ' + err.message); 627 | } else if (NODE_ENV !== 'test') { 628 | _this2._logger.verbose('ZeroMQ rep socket listening on port ' + port); 629 | } 630 | }); 631 | } 632 | 633 | /** 634 | * Checks if a cronjob of the given `name` is already scheduled. 635 | * 636 | * @memberOf Dispo 637 | * @param {String} name - The jobs name 638 | * @return {Promise} 639 | */ 640 | 641 | }, { 642 | key: '_isCronScheduled', 643 | value: function () { 644 | var _ref13 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee12(name) { 645 | var jobsByType, cronjobsByType; 646 | return _regenerator2.default.wrap(function _callee12$(_context12) { 647 | while (1) { 648 | switch (_context12.prev = _context12.next) { 649 | case 0: 650 | _context12.next = 2; 651 | return getJobsByType(name, 'delayed', 0, 10000, 'desc'); 652 | 653 | case 2: 654 | jobsByType = _context12.sent; 655 | cronjobsByType = jobsByType.filter(function (job) { 656 | return !!job.data.cron; 657 | }); 658 | return _context12.abrupt('return', cronjobsByType.length > 0); 659 | 660 | case 5: 661 | case 'end': 662 | return _context12.stop(); 663 | } 664 | } 665 | }, _callee12, this); 666 | })); 667 | 668 | function _isCronScheduled(_x18) { 669 | return _ref13.apply(this, arguments); 670 | } 671 | 672 | return _isCronScheduled; 673 | }() 674 | 675 | /** 676 | * @typedef {Object} QueueJobOptions 677 | * @property {Number} attempts - Number of attempts a job is retried until marked as failure 678 | * @property {Number} delay - Delay job run by the given amount of miliseconds 679 | * @property {String} cron - Interval-based scheduling written in cron syntax, ignored when delay is given 680 | * @property {Boolean|{type:String,delay:Number}} backoff - Interval-based scheduling written in cron syntax, ignored when delay is given 681 | */ 682 | /** 683 | * Queues a job. 684 | * 685 | * @memberOf Dispo 686 | * @param {String} name - Job name 687 | * @param {QueueJobOptions} options - Job options 688 | * @return {Promise} 689 | */ 690 | 691 | }, { 692 | key: '_queueJob', 693 | value: function () { 694 | var _ref14 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee13(name, options) { 695 | var _this3 = this; 696 | 697 | var attempts, cron, delay, backoff, isScheduled; 698 | return _regenerator2.default.wrap(function _callee13$(_context13) { 699 | while (1) { 700 | switch (_context13.prev = _context13.next) { 701 | case 0: 702 | attempts = options.attempts, cron = options.cron, delay = options.delay, backoff = options.backoff; 703 | 704 | (0, _assert2.default)(!!cron || !!delay, 'To queue a job, either `cron` or `delay` needs to be defined'); 705 | 706 | _context13.next = 4; 707 | return this._isCronScheduled(name); 708 | 709 | case 4: 710 | isScheduled = _context13.sent; 711 | 712 | 713 | if (!cron || !isScheduled) { 714 | (function () { 715 | var job = _this3._queue.create(name, (0, _assign2.default)(options, { name: name })).delay(delay || _this3._calculateDelay(cron)).attempts(attempts); 716 | 717 | if (backoff) { 718 | console.log(name, backoff); 719 | job.backoff((0, _util.parseBackoff)(backoff)); 720 | } 721 | 722 | job.save(function (err) { 723 | if (err) { 724 | throw new Error('Job save: ' + err.message); 725 | } else if (NODE_ENV !== 'test') { 726 | _this3._logger.logQueued(job); 727 | } 728 | }); 729 | })(); 730 | } 731 | 732 | case 6: 733 | case 'end': 734 | return _context13.stop(); 735 | } 736 | } 737 | }, _callee13, this); 738 | })); 739 | 740 | function _queueJob(_x19, _x20) { 741 | return _ref14.apply(this, arguments); 742 | } 743 | 744 | return _queueJob; 745 | }() 746 | 747 | /** 748 | * Calculates the delay until a cronjobs next run is due 749 | * 750 | * @memberOf Dispo 751 | * @param {String} cron - Interval-based scheduling written in cron syntax 752 | * @return {Number} Number of miliseconds until next cron run 753 | */ 754 | 755 | }, { 756 | key: '_calculateDelay', 757 | value: function _calculateDelay(cron) { 758 | return _later2.default.schedule(_later2.default.parse.cron(cron)).next(2).map(function (date) { 759 | return date.getTime() - Date.now(); 760 | }).filter(function (msec) { 761 | return msec > 500; 762 | }).shift(); 763 | } 764 | }]); 765 | return Dispo; 766 | }(); 767 | 768 | exports.default = Dispo; -------------------------------------------------------------------------------- /dist/logger.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | 7 | var _regenerator = require('babel-runtime/regenerator'); 8 | 9 | var _regenerator2 = _interopRequireDefault(_regenerator); 10 | 11 | var _asyncToGenerator2 = require('babel-runtime/helpers/asyncToGenerator'); 12 | 13 | var _asyncToGenerator3 = _interopRequireDefault(_asyncToGenerator2); 14 | 15 | var _assign = require('babel-runtime/core-js/object/assign'); 16 | 17 | var _assign2 = _interopRequireDefault(_assign); 18 | 19 | var _classCallCheck2 = require('babel-runtime/helpers/classCallCheck'); 20 | 21 | var _classCallCheck3 = _interopRequireDefault(_classCallCheck2); 22 | 23 | var _createClass2 = require('babel-runtime/helpers/createClass'); 24 | 25 | var _createClass3 = _interopRequireDefault(_createClass2); 26 | 27 | var _stringify = require('babel-runtime/core-js/json/stringify'); 28 | 29 | var _stringify2 = _interopRequireDefault(_stringify); 30 | 31 | var _path = require('path'); 32 | 33 | var _path2 = _interopRequireDefault(_path); 34 | 35 | var _chalk = require('chalk'); 36 | 37 | var _chalk2 = _interopRequireDefault(_chalk); 38 | 39 | var _winston5 = require('winston'); 40 | 41 | var _winston6 = _interopRequireDefault(_winston5); 42 | 43 | var _prettyMs = require('pretty-ms'); 44 | 45 | var _prettyMs2 = _interopRequireDefault(_prettyMs); 46 | 47 | var _dateformat = require('dateformat'); 48 | 49 | var _dateformat2 = _interopRequireDefault(_dateformat); 50 | 51 | var _lodash = require('lodash'); 52 | 53 | var _fs = require('fs'); 54 | 55 | var _ = require('.'); 56 | 57 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 58 | 59 | /** 60 | * Maps log levels to terminal colors 61 | */ 62 | var mapLevelToColor = { 63 | error: 'red', 64 | info: 'white', 65 | verbose: 'cyan', 66 | warn: 'yellow' 67 | }; 68 | 69 | /** 70 | * Default logger options 71 | */ 72 | var defaults = { 73 | winstonConfig: { 74 | level: 'verbose', 75 | transports: [new _winston6.default.transports.Console({ 76 | timestamp: true, 77 | formatter: function formatter(options) { 78 | var color = mapLevelToColor[options.level]; 79 | var result = _chalk2.default[color]('[' + dateTime() + '] ' + options.message); 80 | 81 | if (!(0, _lodash.isEmpty)(options.meta)) { 82 | result += _chalk2.default.dim(_chalk2.default[color]('\n ' + (0, _stringify2.default)(options.meta))); 83 | } 84 | 85 | return result; 86 | } 87 | }), new _winston6.default.transports.File({ filename: 'log/scheduler.log' })] 88 | } 89 | }; 90 | 91 | /** 92 | * Returns human readable time of the next job run based on the jobs creation date and delay 93 | * 94 | * @param {String} createdAt - The jobs createdAt timestamp 95 | * @param {String} delay - The jobs delay 96 | * @return {String} Human readable time of the next job run 97 | */ 98 | var runsIn = function runsIn(createdAt, delay) { 99 | return (0, _prettyMs2.default)(Number(createdAt) + Number(delay) - Date.now()); 100 | }; 101 | 102 | /** 103 | * Returns formatted date 104 | * 105 | * @param {Date} [date=Date.now()] - Date to be formatted 106 | * @param {String} [format=HH:MM:ss] - Date format 107 | * @return {String} Formatted date 108 | */ 109 | var dateTime = function dateTime() { 110 | var date = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : Date.now(); 111 | var format = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 'HH:MM:ss'; 112 | 113 | return (0, _dateformat2.default)(date, format); 114 | }; 115 | 116 | /** 117 | * Logger 118 | */ 119 | 120 | var Logger = function () { 121 | function Logger(options) { 122 | (0, _classCallCheck3.default)(this, Logger); 123 | 124 | this.config = (0, _assign2.default)({}, defaults, options); 125 | } 126 | 127 | /** 128 | * @memberOf Logger 129 | */ 130 | 131 | 132 | (0, _createClass3.default)(Logger, [{ 133 | key: 'init', 134 | value: function init() { 135 | var logpath = _path2.default.resolve(this.config.path); 136 | if (!(0, _fs.existsSync)(logpath)) { 137 | (0, _fs.mkdirSync)(logpath); 138 | } 139 | this.winston = new _winston6.default.Logger(this.config.winstonConfig); 140 | } 141 | 142 | /** 143 | * @param {Array} params 144 | * @memberOf Logger 145 | */ 146 | 147 | }, { 148 | key: 'error', 149 | value: function error() { 150 | var _winston; 151 | 152 | (_winston = this.winston).error.apply(_winston, arguments); 153 | } 154 | /** 155 | * @param {Array} params 156 | * @memberOf Logger 157 | */ 158 | 159 | }, { 160 | key: 'info', 161 | value: function info() { 162 | var _winston2; 163 | 164 | (_winston2 = this.winston).info.apply(_winston2, arguments); 165 | } 166 | /** 167 | * @param {Array} params 168 | * @memberOf Logger 169 | */ 170 | 171 | }, { 172 | key: 'verbose', 173 | value: function verbose() { 174 | var _winston3; 175 | 176 | (_winston3 = this.winston).verbose.apply(_winston3, arguments); 177 | } 178 | /** 179 | * @param {Array} params 180 | * @memberOf Logger 181 | */ 182 | 183 | }, { 184 | key: 'warn', 185 | value: function warn() { 186 | var _winston4; 187 | 188 | (_winston4 = this.winston).warn.apply(_winston4, arguments); 189 | } 190 | 191 | /** 192 | * @param {{data:{name:String},id:String}} job 193 | * @memberOf Logger 194 | */ 195 | 196 | }, { 197 | key: 'logStart', 198 | value: function () { 199 | var _ref = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee(job) { 200 | var _ref2, data, id; 201 | 202 | return _regenerator2.default.wrap(function _callee$(_context) { 203 | while (1) { 204 | switch (_context.prev = _context.next) { 205 | case 0: 206 | _context.next = 2; 207 | return (0, _.getJob)(job); 208 | 209 | case 2: 210 | _ref2 = _context.sent; 211 | data = _ref2.data; 212 | id = _ref2.id; 213 | 214 | this.winston.info('Starting ' + data.name, (0, _assign2.default)({ id: id }, data)); 215 | 216 | case 6: 217 | case 'end': 218 | return _context.stop(); 219 | } 220 | } 221 | }, _callee, this); 222 | })); 223 | 224 | function logStart(_x3) { 225 | return _ref.apply(this, arguments); 226 | } 227 | 228 | return logStart; 229 | }() 230 | 231 | /** 232 | * @param {{_attempts:String,data:{name:String},duration:String,id:String}} job 233 | * @memberOf Logger 234 | */ 235 | 236 | }, { 237 | key: 'logComplete', 238 | value: function () { 239 | var _ref3 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee2(job) { 240 | var _ref4, _attempts, data, duration, id, message; 241 | 242 | return _regenerator2.default.wrap(function _callee2$(_context2) { 243 | while (1) { 244 | switch (_context2.prev = _context2.next) { 245 | case 0: 246 | _context2.next = 2; 247 | return (0, _.getJob)(job); 248 | 249 | case 2: 250 | _ref4 = _context2.sent; 251 | _attempts = _ref4._attempts; 252 | data = _ref4.data; 253 | duration = _ref4.duration; 254 | id = _ref4.id; 255 | message = 'Finished ' + data.name + ' after ' + (0, _prettyMs2.default)(Number(duration)); 256 | 257 | this.winston.info(message, (0, _assign2.default)({ id: id, duration: duration, tries: _attempts }, data)); 258 | 259 | case 9: 260 | case 'end': 261 | return _context2.stop(); 262 | } 263 | } 264 | }, _callee2, this); 265 | })); 266 | 267 | function logComplete(_x4) { 268 | return _ref3.apply(this, arguments); 269 | } 270 | 271 | return logComplete; 272 | }() 273 | 274 | /** 275 | * @param {{data:{attempts:String,name:String},id:String}} job 276 | * @param {String} msg 277 | * @memberOf Logger 278 | */ 279 | 280 | }, { 281 | key: 'logFailure', 282 | value: function () { 283 | var _ref5 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee3(job, msg) { 284 | var _ref6, data, id, message; 285 | 286 | return _regenerator2.default.wrap(function _callee3$(_context3) { 287 | while (1) { 288 | switch (_context3.prev = _context3.next) { 289 | case 0: 290 | _context3.next = 2; 291 | return (0, _.getJob)(job); 292 | 293 | case 2: 294 | _ref6 = _context3.sent; 295 | data = _ref6.data; 296 | id = _ref6.id; 297 | message = 'Failed ' + data.name + ' with message "' + msg + '" after ' + data.attempts + ' tries'; 298 | 299 | this.winston.error(message, (0, _assign2.default)({ id: id, message: msg }, data)); 300 | 301 | case 7: 302 | case 'end': 303 | return _context3.stop(); 304 | } 305 | } 306 | }, _callee3, this); 307 | })); 308 | 309 | function logFailure(_x5, _x6) { 310 | return _ref5.apply(this, arguments); 311 | } 312 | 313 | return logFailure; 314 | }() 315 | 316 | /** 317 | * @param {{data:{name:String},id:String}} job 318 | * @param {String} msg 319 | * @memberOf Logger 320 | */ 321 | 322 | }, { 323 | key: 'logFailedAttempt', 324 | value: function () { 325 | var _ref7 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee4(job, msg) { 326 | var _ref8, data, id; 327 | 328 | return _regenerator2.default.wrap(function _callee4$(_context4) { 329 | while (1) { 330 | switch (_context4.prev = _context4.next) { 331 | case 0: 332 | _context4.next = 2; 333 | return (0, _.getJob)(job); 334 | 335 | case 2: 336 | _ref8 = _context4.sent; 337 | data = _ref8.data; 338 | id = _ref8.id; 339 | 340 | this.winston.warn('Error on ' + data.name, (0, _assign2.default)({ id: id, message: msg }, data)); 341 | 342 | case 6: 343 | case 'end': 344 | return _context4.stop(); 345 | } 346 | } 347 | }, _callee4, this); 348 | })); 349 | 350 | function logFailedAttempt(_x7, _x8) { 351 | return _ref7.apply(this, arguments); 352 | } 353 | 354 | return logFailedAttempt; 355 | }() 356 | 357 | /** 358 | * @param {{data:{name:String},_delay:String,created_at:String,id:String}} job 359 | * @memberOf Logger 360 | */ 361 | 362 | }, { 363 | key: 'logQueued', 364 | value: function () { 365 | var _ref9 = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee5(_ref10) { 366 | var data = _ref10.data, 367 | id = _ref10.id, 368 | createdAt = _ref10.created_at, 369 | delay = _ref10._delay; 370 | var message; 371 | return _regenerator2.default.wrap(function _callee5$(_context5) { 372 | while (1) { 373 | switch (_context5.prev = _context5.next) { 374 | case 0: 375 | message = 'Queued ' + data.name + ' to run in ' + runsIn(createdAt, delay); 376 | 377 | this.winston.info(message, (0, _assign2.default)({ id: id }, data)); 378 | 379 | case 2: 380 | case 'end': 381 | return _context5.stop(); 382 | } 383 | } 384 | }, _callee5, this); 385 | })); 386 | 387 | function logQueued(_x9) { 388 | return _ref9.apply(this, arguments); 389 | } 390 | 391 | return logQueued; 392 | }() 393 | }]); 394 | return Logger; 395 | }(); 396 | 397 | exports.default = Logger; -------------------------------------------------------------------------------- /dist/mailer.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | 7 | var _regenerator = require('babel-runtime/regenerator'); 8 | 9 | var _regenerator2 = _interopRequireDefault(_regenerator); 10 | 11 | var _asyncToGenerator2 = require('babel-runtime/helpers/asyncToGenerator'); 12 | 13 | var _asyncToGenerator3 = _interopRequireDefault(_asyncToGenerator2); 14 | 15 | var _assign = require('babel-runtime/core-js/object/assign'); 16 | 17 | var _assign2 = _interopRequireDefault(_assign); 18 | 19 | var _classCallCheck2 = require('babel-runtime/helpers/classCallCheck'); 20 | 21 | var _classCallCheck3 = _interopRequireDefault(_classCallCheck2); 22 | 23 | var _createClass2 = require('babel-runtime/helpers/createClass'); 24 | 25 | var _createClass3 = _interopRequireDefault(_createClass2); 26 | 27 | var _nodemailer = require('nodemailer'); 28 | 29 | var _nodemailer2 = _interopRequireDefault(_nodemailer); 30 | 31 | var _nodemailerSendmailTransport = require('nodemailer-sendmail-transport'); 32 | 33 | var _nodemailerSendmailTransport2 = _interopRequireDefault(_nodemailerSendmailTransport); 34 | 35 | var _ = require('.'); 36 | 37 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 38 | 39 | /** 40 | * Default logger options 41 | * @type {Object} 42 | */ 43 | var defaults = { 44 | transport: (0, _nodemailerSendmailTransport2.default)(), 45 | mail: { 46 | from: 'Dispo ' 47 | } 48 | }; 49 | 50 | /** 51 | * Mailer 52 | */ 53 | 54 | var Mailer = function () { 55 | 56 | /** 57 | * Creates an instance of Mailer. 58 | * 59 | * @memberOf Mailer 60 | * @param {Object} [config={}] 61 | */ 62 | function Mailer(config) { 63 | (0, _classCallCheck3.default)(this, Mailer); 64 | 65 | this.config = (0, _assign2.default)({}, defaults, config); 66 | } 67 | 68 | /** 69 | * Initializes a nodemailer transport 70 | * 71 | * @memberOf Mailer 72 | * @return {Promise} 73 | */ 74 | 75 | 76 | (0, _createClass3.default)(Mailer, [{ 77 | key: 'init', 78 | value: function init() { 79 | this._mailer = _nodemailer2.default.createTransport(this.config.transport); 80 | } 81 | }, { 82 | key: 'sendMail', 83 | value: function () { 84 | var _ref = (0, _asyncToGenerator3.default)(_regenerator2.default.mark(function _callee(job, message) { 85 | var _ref2, _ref2$data, notifyOnError, name, id; 86 | 87 | return _regenerator2.default.wrap(function _callee$(_context) { 88 | while (1) { 89 | switch (_context.prev = _context.next) { 90 | case 0: 91 | _context.next = 2; 92 | return (0, _.getJob)(job); 93 | 94 | case 2: 95 | _ref2 = _context.sent; 96 | _ref2$data = _ref2.data; 97 | notifyOnError = _ref2$data.notifyOnError; 98 | name = _ref2$data.name; 99 | id = _ref2.id; 100 | 101 | if (notifyOnError) { 102 | _context.next = 9; 103 | break; 104 | } 105 | 106 | return _context.abrupt('return'); 107 | 108 | case 9: 109 | 110 | this._mailer.sendMail((0, _assign2.default)({}, this.config.mail, { 111 | to: notifyOnError, 112 | subject: 'Job "' + name + '" (id ' + id + ') failed', 113 | text: 'Job "' + name + '" (id ' + id + ') failed\n\n' + message 114 | })); 115 | 116 | case 10: 117 | case 'end': 118 | return _context.stop(); 119 | } 120 | } 121 | }, _callee, this); 122 | })); 123 | 124 | function sendMail(_x, _x2) { 125 | return _ref.apply(this, arguments); 126 | } 127 | 128 | return sendMail; 129 | }() 130 | }]); 131 | return Mailer; 132 | }(); 133 | 134 | exports.default = Mailer; -------------------------------------------------------------------------------- /dist/util.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | exports.parseBackoff = exports.parseJobs = exports.getAbsolutePath = undefined; 7 | 8 | var _assign = require('babel-runtime/core-js/object/assign'); 9 | 10 | var _assign2 = _interopRequireDefault(_assign); 11 | 12 | var _keys = require('babel-runtime/core-js/object/keys'); 13 | 14 | var _keys2 = _interopRequireDefault(_keys); 15 | 16 | var _fs = require('fs'); 17 | 18 | var _fs2 = _interopRequireDefault(_fs); 19 | 20 | var _path = require('path'); 21 | 22 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 23 | 24 | /** 25 | * @param {String} path 26 | * @param {Number} [mode=fs.R_OK] 27 | * @return {Boolean|String} 28 | */ 29 | var getAbsolutePath = exports.getAbsolutePath = function getAbsolutePath(path) { 30 | var mode = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : _fs2.default.R_OK; 31 | 32 | var dir = (0, _path.resolve)(path); 33 | return _fs2.default.accessSync(dir, mode) || dir; 34 | }; 35 | 36 | /** 37 | * @param {Array} jobs 38 | * @param {String} basedir 39 | * @return {Array} 40 | */ 41 | var parseJobs = exports.parseJobs = function parseJobs(jobs, basedir) { 42 | return (0, _keys2.default)(jobs).reduce(function (res, name) { 43 | var _jobs$name = jobs[name], 44 | file = _jobs$name.file, 45 | cron = _jobs$name.cron, 46 | attempts = _jobs$name.attempts, 47 | backoff = _jobs$name.backoff, 48 | notifyOnError = _jobs$name.notifyOnError; 49 | 50 | 51 | if (!file) { 52 | throw new Error('no file defined for job "' + name + '"'); 53 | } 54 | 55 | var job = require(getAbsolutePath((0, _path.resolve)(basedir, file))); 56 | 57 | var options = { 58 | attempts: attempts || 3, 59 | fn: job.default || job, 60 | name: name 61 | }; 62 | 63 | if (cron) options.cron = cron; 64 | if (backoff) options.backoff = backoff; 65 | if (notifyOnError) options.notifyOnError = notifyOnError; 66 | 67 | res.push(options); 68 | return res; 69 | }, []); 70 | }; 71 | 72 | /** 73 | * @typedef {Object} BackoffOptions 74 | * @property {String} type 75 | * @property {Number} [exponent=2] 76 | */ 77 | /** 78 | * Parses the given backoff options to iron out some inconveniences in kue.js, like e.g. type === 'exponential' only 79 | * doubling the delay instead of letting it grow exponentially. 80 | * 81 | * @param {Boolean|BackoffOptions} [backoff] Backoff algorithm configuration 82 | * @return {Boolean|BackoffOptions|Function} A backoff configuration that kue.js understands 83 | */ 84 | var parseBackoff = exports.parseBackoff = function parseBackoff(backoff) { 85 | if (!backoff) { 86 | return; 87 | } 88 | 89 | if (backoff.type === 'incremental') { 90 | return (0, _assign2.default)({}, backoff, { type: 'exponential' }); 91 | } else if (backoff.type === 'exponential') { 92 | return function (attempt, delay) { 93 | var range = []; 94 | for (var n = 1; n < attempt; n++) { 95 | range.push(n); 96 | } 97 | return range.reduce(function (result, attempt) { 98 | return Math.pow(result, 2); 99 | }, delay); 100 | }; 101 | } 102 | }; -------------------------------------------------------------------------------- /example/jobs.json: -------------------------------------------------------------------------------- 1 | { 2 | "options": { 3 | "mailer": true 4 | }, 5 | "jobs": { 6 | "random": { "file": "jobs/random.js", "cron": "*/1 * * * *", "attempts": 3 }, 7 | "randomSingle": { "file": "jobs/random.js", "attempts": 2 }, 8 | "flakyService": { 9 | "file": "jobs/fail.js", 10 | "cron": "*/1 * * * *", 11 | "attempts": 4, 12 | "backoff": { 13 | "delay": 3000, 14 | "type": "fixed" 15 | } 16 | }, 17 | "flakyServiceWithLongRegenerationTime": { 18 | "file": "jobs/fail.js", 19 | "cron": "*/1 * * * *", 20 | "attempts": 4, 21 | "backoff": { 22 | "delay": 3000, 23 | "type": "incremental" 24 | } 25 | }, 26 | "anotherFlakyServiceWithLongRegenerationTime": { 27 | "file": "jobs/fail.js", 28 | "cron": "*/1 * * * *", 29 | "attempts": 3, 30 | "backoff": { 31 | "delay": 2000, 32 | "type": "exponential" 33 | } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /example/jobs/fail.js: -------------------------------------------------------------------------------- 1 | var Promise = require('bluebird') 2 | 3 | module.exports = function (job) { 4 | return Promise.resolve().then(function () { 5 | const callMe = {} 6 | callMe.maybe() 7 | }) 8 | } 9 | -------------------------------------------------------------------------------- /example/jobs/random.js: -------------------------------------------------------------------------------- 1 | var Promise = require('bluebird') 2 | 3 | var wait = function (seconds) { 4 | seconds = seconds || Math.random() * 4 + 1 5 | return new Promise(function (resolve) { 6 | setTimeout(function () { resolve() }, 1e3 * seconds) 7 | }) 8 | } 9 | 10 | module.exports = function (job) { 11 | return wait().then(function () { 12 | if (Math.random() > 0.6) { 13 | throw new Error('oh noes') 14 | } 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dispo", 3 | "version": "0.5.0", 4 | "description": "Job and cronjob scheduler for Node", 5 | "main": "dist/index.js", 6 | "author": "Christoph Werner ", 7 | "license": "MIT", 8 | "dependencies": { 9 | "bluebird": "^3.4.6", 10 | "chalk": "^1.1.3", 11 | "commander": "^2.9.0", 12 | "dateformat": "^1.0.12", 13 | "express": "^4.14.0", 14 | "kue": "^0.11.1", 15 | "later": "^1.2.0", 16 | "lodash": "^4.16.1", 17 | "nodemailer": "^2.6.4", 18 | "nodemailer-sendmail-transport": "^1.0.0", 19 | "pretty-ms": "^2.1.0", 20 | "winston": "^2.2.0", 21 | "zeromq": "^3.0.0" 22 | }, 23 | "devDependencies": { 24 | "babel-cli": "^6.14.0", 25 | "babel-eslint": "^6.1.2", 26 | "babel-plugin-transform-async-to-generator": "^6.8.0", 27 | "babel-plugin-transform-runtime": "^6.15.0", 28 | "babel-preset-es2015": "^6.14.0", 29 | "chai": "^3.5.0", 30 | "coveralls": "^2.11.14", 31 | "eslint": "^3.5.0", 32 | "eslint-config-standard": "^6.0.1", 33 | "eslint-plugin-babel": "^3.3.0", 34 | "eslint-plugin-import": "^1.15.0", 35 | "eslint-plugin-promise": "^2.0.1", 36 | "eslint-plugin-standard": "^2.0.0", 37 | "istanbul": "1.1.0-alpha.1", 38 | "mocha": "^3.0.2", 39 | "pre-commit": "^1.1.3", 40 | "sinon": "^1.17.6", 41 | "sinon-chai": "^2.8.0" 42 | }, 43 | "bin": { 44 | "dispo": "dist/bin/dispo.js" 45 | }, 46 | "scripts": { 47 | "addCompile": "git add dist/", 48 | "coverage": "istanbul cover _mocha -- --compilers js:babel-register", 49 | "compile": "rm -rf dist/ && node_modules/.bin/babel -d dist/ src/", 50 | "lint": "eslint {src,test}/**", 51 | "prepublish": "npm run compile", 52 | "test": "NODE_ENV=test npm run lint && NODE_ENV=test mocha --compilers js:babel-register", 53 | "prestart": "npm run compile", 54 | "start": "node dist/bin/dispo.js --basedir example" 55 | }, 56 | "pre-commit": { 57 | "run": "test, compile, addCompile", 58 | "silent": true 59 | }, 60 | "repository": { 61 | "type": "git", 62 | "url": "git://github.com/gonsfx/dispo" 63 | }, 64 | "bugs": "https://github.com/gonsfx/dispo/issues", 65 | "keywords": [ 66 | "kue", 67 | "Kue", 68 | "KUE", 69 | "schedule", 70 | "Schedule", 71 | "SCHEDULE", 72 | "scheduler", 73 | "Scheduler", 74 | "SCHEDULER", 75 | "job", 76 | "Job", 77 | "JOB", 78 | "cron", 79 | "Cron", 80 | "CRON", 81 | "cronjob", 82 | "Cronjob", 83 | "CRONJOB", 84 | "ZeroMQ" 85 | ], 86 | "resolutions": { 87 | "chalk": "1.1.3", 88 | "commander": "2.9.0", 89 | "lodash": "4.16.4", 90 | "supports-color": "3.1.2", 91 | "minimist": "1.2.0", 92 | "camelcase": "2.1.1", 93 | "semver": "5.3.0", 94 | "path-exists": "2.1.0", 95 | "strip-bom": "3.0.0", 96 | "repeating": "2.0.1", 97 | "qs": "6.2.1", 98 | "inherits": "2.0.3", 99 | "redis": "2.6.2", 100 | "stylus": "0.54.5", 101 | "yargs": "3.32.0", 102 | "source-map": "0.5.6", 103 | "acorn": "4.0.3", 104 | "is-promise": "2.1.0", 105 | "promise": "6.1.0", 106 | "uglify-js": "2.7.3", 107 | "glob": "7.1.1", 108 | "minimatch": "3.0.3", 109 | "once": "1.4.0", 110 | "extend": "3.0.0", 111 | "async": "1.5.2", 112 | "npmlog": "4.0.0", 113 | "bl": "1.1.2", 114 | "readable-stream": "2.1.5", 115 | "assert-plus": "1.0.0", 116 | "js-tokens": "2.0.0", 117 | "jsesc": "1.3.0", 118 | "globals": "9.12.0", 119 | "user-home": "2.0.0", 120 | "type-detect": "1.0.0", 121 | "doctrine": "1.4.0", 122 | "estraverse": "4.2.0", 123 | "samsam": "1.1.3", 124 | "css-parse": "1.7.0", 125 | "lru-cache": "4.0.1", 126 | "wordwrap": "1.0.0" 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/bin/dispo.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | import { resolve } from 'path' 3 | import program from 'commander' 4 | import Scheduler from '..' 5 | import { version } from '../../package.json' 6 | import { getAbsolutePath, parseJobs } from '../util' 7 | 8 | program 9 | .version(version) 10 | .option('-C, --config ', 'config file path, default: `jobs.json`') 11 | .option('-B, --basedir ', 'directory used as a base for relative config and job paths') 12 | .parse(process.argv) 13 | 14 | program.config = program.config || 'jobs.json' 15 | program.basedir = program.basedir || process.cwd() 16 | 17 | try { 18 | const config = require(getAbsolutePath(resolve(program.basedir, program.config))) 19 | config.jobs = parseJobs(config.jobs, program.basedir) 20 | const scheduler = new Scheduler(config) 21 | scheduler.init() 22 | } catch (e) { 23 | console.log('errored', e) 24 | throw e 25 | } 26 | -------------------------------------------------------------------------------- /src/client.js: -------------------------------------------------------------------------------- 1 | var zmq = require('zeromq') 2 | var requester = zmq.socket('req') 3 | 4 | const data = { 5 | name: 'randomSingle', 6 | delay: 5000, 7 | meta: { userId: '57bc570c6d505c72cc4d505f' } 8 | } 9 | 10 | const onMessage = function (reply) { 11 | console.log(reply.toString()) 12 | } 13 | 14 | requester.on('message', onMessage) 15 | requester.connect('tcp://localhost:5555') 16 | 17 | setInterval(() => { 18 | requester.send(JSON.stringify(data)) 19 | }, 1e3) 20 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import kue from 'kue' 2 | import later from 'later' 3 | import assert from 'assert' 4 | import zmq from 'zeromq' 5 | import { promisify } from 'bluebird' 6 | import { merge, omit } from 'lodash' 7 | import Logger from './logger' 8 | import Mailer from './mailer' 9 | import { parseBackoff } from './util' 10 | 11 | const { NODE_ENV } = process.env 12 | 13 | /** 14 | * Default scheduler config 15 | */ 16 | const defaultConfig = { 17 | jobs: [], 18 | options: { 19 | port: 5555, 20 | logging: { path: 'log' }, 21 | mailer: null 22 | } 23 | } 24 | 25 | /** 26 | * Promisified version of `kue.Job.rangeByType` 27 | * @type {Function} 28 | */ 29 | const getJobsByType = promisify(kue.Job.rangeByType) 30 | 31 | /** 32 | * Promisified version of `kue.Job.get` 33 | * 34 | * @type {Function} 35 | */ 36 | export const getJob = promisify(kue.Job.get) 37 | 38 | /** 39 | * Dispo Scheduler 40 | */ 41 | export default class Dispo { 42 | 43 | /** 44 | * Creates an instance of Dispo. 45 | * 46 | * @memberOf Dispo 47 | * @param {Object} [config={}] 48 | */ 49 | constructor (config = {}) { 50 | this.config = merge({}, defaultConfig, config) 51 | } 52 | 53 | /** 54 | * Initializes logging, socket bindings and the queue mechanism 55 | * 56 | * @memberOf Dispo 57 | * @return {Promise} 58 | */ 59 | async init () { 60 | this._logger = new Logger(this.config.options.logging) 61 | this._logger.init() 62 | 63 | if (this.config.options.mailer) { 64 | this._mailer = new Mailer(this.config.options.mailer) 65 | this._mailer.init() 66 | } 67 | 68 | this._initSocket() 69 | this._initQueue(this.config.options.queue) 70 | 71 | for (let job of this.config.jobs) { 72 | await this.defineJob(job) 73 | } 74 | } 75 | 76 | /** 77 | * @typedef {Object} DefineJobOptions 78 | * @property {String} name - Job name 79 | * @property {Function} fn - Job method that is executed when the job is run 80 | * @property {Number} attempts - Number of attempts a job is retried until marked as failure 81 | * @property {String} cron - Interval-based scheduling written in cron syntax, ignored when delay is given 82 | */ 83 | /** 84 | * Defines a job 85 | * 86 | * @memberOf Dispo 87 | * @param {DefineJobOptions} options - Job options 88 | * @return {Promise} 89 | */ 90 | async defineJob ({ attempts, cron, notifyOnError, fn, name, backoff }) { 91 | assert(name, 'Job must have a name') 92 | 93 | const options = { attempts, backoff } 94 | this._queue.process(name, (job, done) => fn(job).then(done, done)) 95 | 96 | if (notifyOnError) { 97 | options.notifyOnError = notifyOnError 98 | } 99 | 100 | if (cron) { 101 | options.cron = cron 102 | await this._queueJob(name, options) 103 | } 104 | } 105 | 106 | /** 107 | * Initializes the queue mechanism 108 | * 109 | * This is mostly done to set up queue level logging and to be able to automatically 110 | * queue the next runs of cronjobs after their previous runs have completed. 111 | * 112 | * @memberOf Dispo 113 | * @param {Object} [options={}] 114 | */ 115 | _initQueue (options = {}) { 116 | this._queue = kue.createQueue(options) 117 | this._queue.watchStuckJobs(5e3) 118 | 119 | if (NODE_ENV !== 'test') { 120 | this._queue.on('job start', async (id) => await this._handleStart(id)) 121 | this._queue.on('job failed attempt', async (id, msg) => await this._handleFailedAttempt(id, msg)) 122 | this._queue.on('job failed', async (id, msg) => await this._handleFailed(id, msg)) 123 | } 124 | 125 | this._queue.on('job complete', async (id) => await this._handleComplete(id)) 126 | } 127 | 128 | /** 129 | * Logs job starts 130 | * 131 | * @memberOf Dispo 132 | * @param {Number} id - Job id 133 | */ 134 | async _handleStart (id) { 135 | await this._logger.logStart(id) 136 | } 137 | 138 | /** 139 | * Logs failed attempts 140 | * 141 | * @memberOf Dispo 142 | * @param {Number} id - Job id 143 | * @param {String} msg - Error message 144 | */ 145 | async _handleFailedAttempt (id, msg) { 146 | await this._logger.logFailedAttempt(id, msg) 147 | } 148 | 149 | /** 150 | * Logs failed jobs and sends notification emails if configured to do so 151 | * 152 | * @memberOf Dispo 153 | * @param {Number} id - Job id 154 | * @param {String} msg - Error message 155 | */ 156 | async _handleFailed (id, msg) { 157 | await this._logger.logFailure(id, msg) 158 | const job = await getJob(id) 159 | if (this._mailer) { 160 | await this._mailer.sendMail(id, job.error()) 161 | } 162 | } 163 | 164 | /** 165 | * Logs completed jobs and re-queues them when defined as a cron 166 | * 167 | * @memberOf Dispo 168 | * @param {Number} id - Job id 169 | */ 170 | async _handleComplete (id) { 171 | if (NODE_ENV !== 'test') { 172 | await this._logger.logComplete(id) 173 | } 174 | 175 | const job = await getJob(id) 176 | if (job.data.cron) { 177 | await this._queueJob(job.data.name, job.data) 178 | } 179 | } 180 | 181 | /** 182 | * Initialize ØMQ reply socket 183 | * 184 | * Received messages add new jobs to the queue when the given job is defined in 185 | * the job configuration 186 | * 187 | * @memberOf Dispo 188 | * @param {Number|String} [port=this.config.options.port] 189 | */ 190 | _initSocket (port = this.config.options.port) { 191 | const responder = zmq.socket('rep') 192 | 193 | responder.on('message', async (message) => { 194 | const payload = JSON.parse(message.toString()) 195 | const job = this.config.jobs.filter((job) => job.name === payload.name).shift() 196 | 197 | if (job) { 198 | const data = omit(Object.assign(payload, job), 'fn', 'name') 199 | await this._queueJob(job.name, data) 200 | responder.send('ok') 201 | } 202 | }) 203 | 204 | responder.bind(`tcp://*:${port}`, (err) => { 205 | if (err) { 206 | throw new Error(`Port binding: ${err.message}`) 207 | } else if (NODE_ENV !== 'test') { 208 | this._logger.verbose(`ZeroMQ rep socket listening on port ${port}`) 209 | } 210 | }) 211 | } 212 | 213 | /** 214 | * Checks if a cronjob of the given `name` is already scheduled. 215 | * 216 | * @memberOf Dispo 217 | * @param {String} name - The jobs name 218 | * @return {Promise} 219 | */ 220 | async _isCronScheduled (name) { 221 | const jobsByType = await getJobsByType(name, 'delayed', 0, 10000, 'desc') 222 | const cronjobsByType = jobsByType.filter((job) => !!job.data.cron) 223 | return cronjobsByType.length > 0 224 | } 225 | 226 | /** 227 | * @typedef {Object} QueueJobOptions 228 | * @property {Number} attempts - Number of attempts a job is retried until marked as failure 229 | * @property {Number} delay - Delay job run by the given amount of miliseconds 230 | * @property {String} cron - Interval-based scheduling written in cron syntax, ignored when delay is given 231 | * @property {Boolean|{type:String,delay:Number}} backoff - Interval-based scheduling written in cron syntax, ignored when delay is given 232 | */ 233 | /** 234 | * Queues a job. 235 | * 236 | * @memberOf Dispo 237 | * @param {String} name - Job name 238 | * @param {QueueJobOptions} options - Job options 239 | * @return {Promise} 240 | */ 241 | async _queueJob (name, options) { 242 | const { attempts, cron, delay, backoff } = options 243 | assert(!!cron || !!delay, 'To queue a job, either `cron` or `delay` needs to be defined') 244 | 245 | const isScheduled = await this._isCronScheduled(name) 246 | 247 | if (!cron || !isScheduled) { 248 | const job = this._queue.create(name, Object.assign(options, { name })) 249 | .delay(delay || this._calculateDelay(cron)) 250 | .attempts(attempts) 251 | 252 | if (backoff) { 253 | console.log(name, backoff) 254 | job.backoff(parseBackoff(backoff)) 255 | } 256 | 257 | job.save((err) => { 258 | if (err) { 259 | throw new Error(`Job save: ${err.message}`) 260 | } else if (NODE_ENV !== 'test') { 261 | this._logger.logQueued(job) 262 | } 263 | }) 264 | } 265 | } 266 | 267 | /** 268 | * Calculates the delay until a cronjobs next run is due 269 | * 270 | * @memberOf Dispo 271 | * @param {String} cron - Interval-based scheduling written in cron syntax 272 | * @return {Number} Number of miliseconds until next cron run 273 | */ 274 | _calculateDelay (cron) { 275 | return later.schedule(later.parse.cron(cron)).next(2) 276 | .map((date) => date.getTime() - Date.now()) 277 | .filter((msec) => msec > 500) 278 | .shift() 279 | } 280 | } 281 | -------------------------------------------------------------------------------- /src/logger.js: -------------------------------------------------------------------------------- 1 | import path from 'path' 2 | import chalk from 'chalk' 3 | import winston from 'winston' 4 | import prettyMs from 'pretty-ms' 5 | import formatDate from 'dateformat' 6 | import { isEmpty } from 'lodash' 7 | import { existsSync, mkdirSync } from 'fs' 8 | import { getJob } from '.' 9 | 10 | /** 11 | * Maps log levels to terminal colors 12 | */ 13 | const mapLevelToColor = { 14 | error: 'red', 15 | info: 'white', 16 | verbose: 'cyan', 17 | warn: 'yellow' 18 | } 19 | 20 | /** 21 | * Default logger options 22 | */ 23 | const defaults = { 24 | winstonConfig: { 25 | level: 'verbose', 26 | transports: [ 27 | new winston.transports.Console({ 28 | timestamp: true, 29 | formatter: (options) => { 30 | const color = mapLevelToColor[options.level] 31 | let result = chalk[color](`[${dateTime()}] ${options.message}`) 32 | 33 | if (!isEmpty(options.meta)) { 34 | result += chalk.dim(chalk[color](`\n ${JSON.stringify(options.meta)}`)) 35 | } 36 | 37 | return result 38 | } 39 | }), 40 | new winston.transports.File({ filename: 'log/scheduler.log' }) 41 | ] 42 | } 43 | } 44 | 45 | /** 46 | * Returns human readable time of the next job run based on the jobs creation date and delay 47 | * 48 | * @param {String} createdAt - The jobs createdAt timestamp 49 | * @param {String} delay - The jobs delay 50 | * @return {String} Human readable time of the next job run 51 | */ 52 | const runsIn = (createdAt, delay) => { 53 | return prettyMs(Number(createdAt) + Number(delay) - Date.now()) 54 | } 55 | 56 | /** 57 | * Returns formatted date 58 | * 59 | * @param {Date} [date=Date.now()] - Date to be formatted 60 | * @param {String} [format=HH:MM:ss] - Date format 61 | * @return {String} Formatted date 62 | */ 63 | const dateTime = (date = Date.now(), format = 'HH:MM:ss') => { 64 | return formatDate(date, format) 65 | } 66 | 67 | /** 68 | * Logger 69 | */ 70 | export default class Logger { 71 | constructor (options) { 72 | this.config = Object.assign({}, defaults, options) 73 | } 74 | 75 | /** 76 | * @memberOf Logger 77 | */ 78 | init () { 79 | const logpath = path.resolve(this.config.path) 80 | if (!existsSync(logpath)) { 81 | mkdirSync(logpath) 82 | } 83 | this.winston = new winston.Logger(this.config.winstonConfig) 84 | } 85 | 86 | /** 87 | * @param {Array} params 88 | * @memberOf Logger 89 | */ 90 | error (...params) { this.winston.error(...params) } 91 | /** 92 | * @param {Array} params 93 | * @memberOf Logger 94 | */ 95 | info (...params) { this.winston.info(...params) } 96 | /** 97 | * @param {Array} params 98 | * @memberOf Logger 99 | */ 100 | verbose (...params) { this.winston.verbose(...params) } 101 | /** 102 | * @param {Array} params 103 | * @memberOf Logger 104 | */ 105 | warn (...params) { this.winston.warn(...params) } 106 | 107 | /** 108 | * @param {{data:{name:String},id:String}} job 109 | * @memberOf Logger 110 | */ 111 | async logStart (job) { 112 | const { data, id } = await getJob(job) 113 | this.winston.info(`Starting ${data.name}`, Object.assign({ id }, data)) 114 | } 115 | 116 | /** 117 | * @param {{_attempts:String,data:{name:String},duration:String,id:String}} job 118 | * @memberOf Logger 119 | */ 120 | async logComplete (job) { 121 | const { _attempts, data, duration, id } = await getJob(job) 122 | const message = `Finished ${data.name} after ${prettyMs(Number(duration))}` 123 | this.winston.info(message, Object.assign({ id, duration, tries: _attempts }, data)) 124 | } 125 | 126 | /** 127 | * @param {{data:{attempts:String,name:String},id:String}} job 128 | * @param {String} msg 129 | * @memberOf Logger 130 | */ 131 | async logFailure (job, msg) { 132 | const { data, id } = await getJob(job) 133 | const message = `Failed ${data.name} with message "${msg}" after ${data.attempts} tries` 134 | this.winston.error(message, Object.assign({ id, message: msg }, data)) 135 | } 136 | 137 | /** 138 | * @param {{data:{name:String},id:String}} job 139 | * @param {String} msg 140 | * @memberOf Logger 141 | */ 142 | async logFailedAttempt (job, msg) { 143 | const { data, id } = await getJob(job) 144 | this.winston.warn(`Error on ${data.name}`, Object.assign({ id, message: msg }, data)) 145 | } 146 | 147 | /** 148 | * @param {{data:{name:String},_delay:String,created_at:String,id:String}} job 149 | * @memberOf Logger 150 | */ 151 | async logQueued ({ data, id, created_at: createdAt, _delay: delay }) { 152 | const message = `Queued ${data.name} to run in ${runsIn(createdAt, delay)}` 153 | this.winston.info(message, Object.assign({ id }, data)) 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /src/mailer.js: -------------------------------------------------------------------------------- 1 | import nodemailer from 'nodemailer' 2 | import sendmailTransport from 'nodemailer-sendmail-transport' 3 | import { getJob } from '.' 4 | 5 | /** 6 | * Default logger options 7 | * @type {Object} 8 | */ 9 | const defaults = { 10 | transport: sendmailTransport(), 11 | mail: { 12 | from: 'Dispo ' 13 | } 14 | } 15 | 16 | /** 17 | * Mailer 18 | */ 19 | export default class Mailer { 20 | 21 | /** 22 | * Creates an instance of Mailer. 23 | * 24 | * @memberOf Mailer 25 | * @param {Object} [config={}] 26 | */ 27 | constructor (config) { 28 | this.config = Object.assign({}, defaults, config) 29 | } 30 | 31 | /** 32 | * Initializes a nodemailer transport 33 | * 34 | * @memberOf Mailer 35 | * @return {Promise} 36 | */ 37 | init () { 38 | this._mailer = nodemailer.createTransport(this.config.transport) 39 | } 40 | 41 | async sendMail (job, message) { 42 | const { data: { notifyOnError, name }, id } = await getJob(job) 43 | 44 | if (!notifyOnError) return 45 | 46 | this._mailer.sendMail(Object.assign({}, this.config.mail, { 47 | to: notifyOnError, 48 | subject: `Job "${name}" (id ${id}) failed`, 49 | text: `Job "${name}" (id ${id}) failed\n\n${message}` 50 | })) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/util.js: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import { resolve } from 'path' 3 | 4 | /** 5 | * @param {String} path 6 | * @param {Number} [mode=fs.R_OK] 7 | * @return {Boolean|String} 8 | */ 9 | export const getAbsolutePath = (path, mode = fs.R_OK) => { 10 | const dir = resolve(path) 11 | return fs.accessSync(dir, mode) || dir 12 | } 13 | 14 | /** 15 | * @param {Array} jobs 16 | * @param {String} basedir 17 | * @return {Array} 18 | */ 19 | export const parseJobs = (jobs, basedir) => { 20 | return Object 21 | .keys(jobs) 22 | .reduce((res, name) => { 23 | const { file, cron, attempts, backoff, notifyOnError } = jobs[name] 24 | 25 | if (!file) { 26 | throw new Error(`no file defined for job "${name}"`) 27 | } 28 | 29 | const job = require(getAbsolutePath(resolve(basedir, file))) 30 | 31 | const options = { 32 | attempts: attempts || 3, 33 | fn: job.default || job, 34 | name 35 | } 36 | 37 | if (cron) options.cron = cron 38 | if (backoff) options.backoff = backoff 39 | if (notifyOnError) options.notifyOnError = notifyOnError 40 | 41 | res.push(options) 42 | return res 43 | }, []) 44 | } 45 | 46 | /** 47 | * @typedef {Object} BackoffOptions 48 | * @property {String} type 49 | * @property {Number} [exponent=2] 50 | */ 51 | /** 52 | * Parses the given backoff options to iron out some inconveniences in kue.js, like e.g. type === 'exponential' only 53 | * doubling the delay instead of letting it grow exponentially. 54 | * 55 | * @param {Boolean|BackoffOptions} [backoff] Backoff algorithm configuration 56 | * @return {Boolean|BackoffOptions|Function} A backoff configuration that kue.js understands 57 | */ 58 | export const parseBackoff = (backoff) => { 59 | if (!backoff) { 60 | return 61 | } 62 | 63 | if (backoff.type === 'incremental') { 64 | return Object.assign({}, backoff, { type: 'exponential' }) 65 | } else if (backoff.type === 'exponential') { 66 | return function (attempt, delay) { 67 | let range = [] 68 | for (let n = 1; n < attempt; n++) { 69 | range.push(n) 70 | } 71 | return range.reduce((result, attempt) => { 72 | return Math.pow(result, 2) 73 | }, delay) 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import { spy } from 'sinon' 3 | import sinonChai from 'sinon-chai' 4 | import chai, { expect } from 'chai' 5 | import path, { resolve } from 'path' 6 | import Scheduler from '../src' 7 | import Logger from '../src/logger' 8 | import Mailer from '../src/mailer' 9 | 10 | import { 11 | getAbsolutePath, 12 | parseJobs, 13 | parseBackoff 14 | } from '../src/util' 15 | 16 | chai.use(sinonChai) 17 | 18 | describe('Utility methods', () => { 19 | describe('getAbsolutePath', () => { 20 | const existingPath = 'src/bin/dispo.js' 21 | 22 | it('throws when file doesnt exist', () => { 23 | expect(() => getAbsolutePath(existingPath)).to.not.throw(Error) 24 | expect(() => getAbsolutePath('nonexisting.json')).to.throw(Error) 25 | }) 26 | 27 | it('returns absolute path when file exists', () => { 28 | expect(getAbsolutePath(existingPath)).to.equal(resolve(existingPath)) 29 | }) 30 | 31 | it('uses native fs and path methods', () => { 32 | const accessSync = spy(fs, 'accessSync') 33 | const resolve = spy(path, 'resolve') 34 | getAbsolutePath(existingPath) 35 | expect(resolve).to.have.been.calledOnce 36 | expect(resolve).to.have.been.calledWith(existingPath) 37 | expect(accessSync).to.have.been.calledOnce 38 | expect(accessSync).to.have.been.calledWith(resolve(existingPath), fs.R_OK) 39 | accessSync.restore() 40 | resolve.restore() 41 | }) 42 | 43 | it('pass 2nd parameter to fs.accessSync', () => { 44 | const accessSync = spy(fs, 'accessSync') 45 | getAbsolutePath(existingPath, fs.W_OK) 46 | expect(accessSync).to.have.been.calledWith(resolve(existingPath), fs.W_OK) 47 | accessSync.restore() 48 | }) 49 | }) 50 | 51 | describe('parseJobs', () => { 52 | const base = process.cwd() 53 | 54 | it('throws for jobs without a file', () => { 55 | expect(() => parseJobs({ withoutFile: {} }, base)).to.throw(Error) 56 | expect(() => parseJobs({ nonExistingFile: { file: '404.js' } }, base)).to.throw(Error) 57 | expect(() => parseJobs({ existingFile: { file: 'example/jobs/random.js' } }, base)).to.not.throw(Error) 58 | }) 59 | 60 | it('returns an array of jobs objects', () => { 61 | const file = 'example/jobs/random.js' 62 | const fn = require(`../${file}`) 63 | 64 | const result = parseJobs({ 65 | random: { file, attempts: 2, notifyOnError: 'example@email.com' }, 66 | alsoRandom: { file, cron: '*/2 * * * *' }, 67 | backoffFixed: { file, attempts: 2, backoff: { delay: 3000, type: 'fixed' } }, 68 | backoffExponentially: { file, attempts: 2, backoff: { delay: 3000, type: 'exponential' } } 69 | }, base) 70 | 71 | expect(result).to.deep.equal([ 72 | { name: 'random', attempts: 2, fn, notifyOnError: 'example@email.com' }, 73 | { name: 'alsoRandom', attempts: 3, fn, cron: '*/2 * * * *' }, 74 | { name: 'backoffFixed', attempts: 2, fn, backoff: { delay: 3000, type: 'fixed' } }, 75 | { name: 'backoffExponentially', attempts: 2, fn, backoff: { delay: 3000, type: 'exponential' } } 76 | ]) 77 | }) 78 | }) 79 | 80 | describe('parseBackoff', () => { 81 | it('returns undefined when no backoff property is given', () => { 82 | const actualResult = parseBackoff() 83 | expect(actualResult).to.equal(undefined) 84 | }) 85 | 86 | it(`returns a an object with type 'exponential' when given the type 'incremental'`, () => { 87 | const actualOptions = parseBackoff({ type: 'incremental' }) 88 | expect(actualOptions).to.deep.equal({ type: 'exponential' }) 89 | }) 90 | 91 | it(`returns a exponential growth function when given the type 'exponential'`, () => { 92 | const delay = 3 93 | const fn = parseBackoff({ type: 'exponential' }) 94 | const actualResult2ndAttempt = fn(2, delay) 95 | const actualResult3rdAttempt = fn(3, delay) 96 | const actualResult4thAttempt = fn(4, delay) 97 | expect(actualResult2ndAttempt).to.equal(9) 98 | expect(actualResult3rdAttempt).to.equal(81) 99 | expect(actualResult4thAttempt).to.equal(6561) 100 | }) 101 | }) 102 | }) 103 | 104 | describe('Scheduler', () => { 105 | describe('initialization', () => { 106 | it('initializes a logger', () => { 107 | const scheduler = new Scheduler() 108 | scheduler.init() 109 | expect(scheduler._logger).to.be.instanceof(Logger) 110 | }) 111 | 112 | it('initializes a mailer when switch is on', () => { 113 | const scheduler = new Scheduler({options: {mailer: {enabled: true}}}) 114 | scheduler.init() 115 | expect(scheduler._mailer).to.be.instanceof(Mailer) 116 | }) 117 | 118 | it('doesn\'t initializes a mailer by default', () => { 119 | const scheduler = new Scheduler() 120 | scheduler.init() 121 | expect(scheduler._mailer).to.be.undefined 122 | }) 123 | }) 124 | }) 125 | --------------------------------------------------------------------------------