├── .npmrc ├── test ├── scripts │ ├── invalid.js │ ├── timeout.js │ ├── script.js │ ├── error.js │ ├── callbackError.js │ ├── gc.js │ ├── callback.js │ ├── callbackAfterEnd.js │ ├── okWithErrorProperty.js │ ├── unexpectedError.js │ ├── useDate.js │ ├── parallelCallbackCalls.js │ └── useBuffer.js └── test.js ├── .travis.yml ├── .eslintrc ├── lib ├── messageHandler.js ├── in-process.js ├── getRandomPort.js ├── worker-processes.js ├── manager-processes.js ├── manager-servers.js └── worker-servers.js ├── .gitignore ├── LICENSE ├── index.js ├── package.json └── README.md /.npmrc: -------------------------------------------------------------------------------- 1 | save-exact=true -------------------------------------------------------------------------------- /test/scripts/invalid.js: -------------------------------------------------------------------------------- 1 | console.log('executed') 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "8.0" 4 | -------------------------------------------------------------------------------- /test/scripts/timeout.js: -------------------------------------------------------------------------------- 1 | module.exports = function (inputs, callerCallback, done) { 2 | 3 | } 4 | -------------------------------------------------------------------------------- /test/scripts/script.js: -------------------------------------------------------------------------------- 1 | module.exports = function (inputs, callback, done) { 2 | done(null, inputs) 3 | } 4 | -------------------------------------------------------------------------------- /test/scripts/error.js: -------------------------------------------------------------------------------- 1 | module.exports = function (inputs, callerCallback, done) { 2 | done(new Error('foo')) 3 | } 4 | -------------------------------------------------------------------------------- /test/scripts/callbackError.js: -------------------------------------------------------------------------------- 1 | module.exports = function (inputs, callerCallback, done) { 2 | callerCallback('test', function () { 3 | done(new Error('foo')) 4 | }) 5 | } 6 | -------------------------------------------------------------------------------- /test/scripts/gc.js: -------------------------------------------------------------------------------- 1 | module.exports = function (inputs, callerCallback, done) { 2 | process.nextTick(function () { 3 | global.gc() 4 | done(null, inputs) 5 | }) 6 | } 7 | -------------------------------------------------------------------------------- /test/scripts/callback.js: -------------------------------------------------------------------------------- 1 | module.exports = function (inputs, callerCallback, done) { 2 | callerCallback('test', function (err, resp) { 3 | done(err, { test: resp }) 4 | }) 5 | } 6 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["standard"], 3 | "rules": { 4 | "arrow-parens": 0 5 | }, 6 | "env": { 7 | "node": true, 8 | "mocha": true, 9 | "browser": true 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test/scripts/callbackAfterEnd.js: -------------------------------------------------------------------------------- 1 | module.exports = function (inputs, callerCallback, done) { 2 | setTimeout(() => { 3 | callerCallback('test', () => {}) 4 | }, 100) 5 | 6 | done(null, { ok: true }) 7 | } 8 | -------------------------------------------------------------------------------- /test/scripts/okWithErrorProperty.js: -------------------------------------------------------------------------------- 1 | module.exports = function (inputs, callerCallback, done) { 2 | done(null, { 3 | error: { 4 | message: 'custom', 5 | stack: 'custom stack' 6 | } 7 | }) 8 | } 9 | -------------------------------------------------------------------------------- /test/scripts/unexpectedError.js: -------------------------------------------------------------------------------- 1 | module.exports = function (inputs, callerCallback, done) { 2 | setTimeout(function () { 3 | // eslint-disable-next-line no-undef, no-unused-vars 4 | var i = j + 1 5 | }) 6 | } 7 | -------------------------------------------------------------------------------- /lib/messageHandler.js: -------------------------------------------------------------------------------- 1 | var serializator = require('serializator') 2 | 3 | module.exports.serialize = function (data) { 4 | return serializator.serialize(data) 5 | } 6 | 7 | module.exports.parse = function (dataStr) { 8 | return serializator.parse(dataStr) 9 | } 10 | -------------------------------------------------------------------------------- /test/scripts/useDate.js: -------------------------------------------------------------------------------- 1 | module.exports = function (inputs, callbackResponse, done) { 2 | inputs.dateInTime = inputs.date.getTime() 3 | 4 | if (inputs.useCallback) { 5 | callbackResponse({ 6 | internalDate: new Date('2018-09-02') 7 | }, function (err, resp) { 8 | done(err, Object.assign({}, inputs, resp)) 9 | }) 10 | } else { 11 | done(null, inputs) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /test/scripts/parallelCallbackCalls.js: -------------------------------------------------------------------------------- 1 | var util = require('util') 2 | 3 | module.exports = function (inputs, callback, done) { 4 | var promises = [] 5 | 6 | var callbackAsync = util.promisify(callback) 7 | 8 | promises.push(callbackAsync(`${inputs.name} Matos`)) 9 | promises.push(callbackAsync(`${inputs.name} Morillo`)) 10 | 11 | Promise.all(promises).then(function (result) { 12 | done(null, result) 13 | }).catch(done) 14 | } 15 | -------------------------------------------------------------------------------- /test/scripts/useBuffer.js: -------------------------------------------------------------------------------- 1 | module.exports = function (inputs, callbackResponse, done) { 2 | inputs.bufInText = inputs.buf.toString() 3 | inputs.responseBuf = Buffer.from(inputs.bufInText + ' world') 4 | 5 | if (inputs.useCallback) { 6 | callbackResponse({ 7 | receivedBuf: Buffer.from('secret message') 8 | }, function (err, resp) { 9 | done(err, Object.assign({}, inputs, resp)) 10 | }) 11 | } else { 12 | done(null, inputs) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | # Logs 3 | logs 4 | *.log 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 18 | .grunt 19 | 20 | # Compiled binary addons (http://nodejs.org/api/addons.html) 21 | build/Release 22 | 23 | # Dependency directory 24 | # Commenting this out is preferred by some people, see 25 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 26 | node_modules 27 | 28 | # Users Environment Variables 29 | .lock-wscript 30 | 31 | test/temp/* 32 | 33 | .vscode -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Jan Blaha 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | function updateProcessArgs () { 2 | // fix freeze during debugging 3 | process.execArgv = process.execArgv.filter(a => a == null || (!a.startsWith('--debug') && !a.startsWith('--inspect'))) 4 | } 5 | 6 | module.exports = function (_options) { 7 | var options = Object.assign({}, _options) 8 | 9 | options.timeout = options.timeout || 10000 10 | options.strategy = options.strategy || 'http-server' 11 | 12 | if (options.strategy === 'http-server') { 13 | updateProcessArgs() 14 | return new (require('./lib/manager-servers.js'))(options) 15 | } 16 | 17 | if (options.strategy === 'dedicated-process') { 18 | updateProcessArgs() 19 | return new (require('./lib/manager-processes.js'))(options) 20 | } 21 | 22 | if (options.strategy === 'in-process') { 23 | return new (require('./lib/in-process.js'))(options) 24 | } 25 | 26 | throw new Error('Unsupported scripts manager strategy: ' + options.strategy) 27 | } 28 | 29 | module.exports.ScriptManager = require('./lib/manager-servers.js') 30 | module.exports.ScriptManagerOnHttpServers = module.exports.ScriptManager 31 | 32 | module.exports.ScriptManagerOnProcesses = require('./lib/manager-processes.js') 33 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "script-manager", 3 | "version": "0.10.2", 4 | "author": { 5 | "name": "Jan Blaha", 6 | "email": "jan.blaha@hotmail.com" 7 | }, 8 | "contributors": [], 9 | "description": "Manager for running foreign and potentionally dangerous scripts in the cluster", 10 | "keywords": [ 11 | "custom", 12 | "script", 13 | "manager" 14 | ], 15 | "homepage": "https://github.com/pofider/node-script-manager", 16 | "repository": { 17 | "type": "git", 18 | "url": "git@github.com:pofider/node-script-manager.git" 19 | }, 20 | "standard": { 21 | "env": { 22 | "node": true, 23 | "mocha": true, 24 | "browser": true 25 | } 26 | }, 27 | "dependencies": { 28 | "axios": "0.19.2", 29 | "serializator": "1.0.2", 30 | "uuid": "3.3.2" 31 | }, 32 | "devDependencies": { 33 | "eslint": "5.9.0", 34 | "in-publish": "2.0.1", 35 | "mocha": "5.2.0", 36 | "should": "13.2.3", 37 | "standard": "12.0.1" 38 | }, 39 | "scripts": { 40 | "test": "mocha test/test.js --timeout 20000 --exit && standard", 41 | "prepublish": "in-publish && standard || not-in-publish" 42 | }, 43 | "main": "index.js", 44 | "license": "MIT" 45 | } 46 | -------------------------------------------------------------------------------- /lib/in-process.js: -------------------------------------------------------------------------------- 1 | var ScriptsManager = module.exports = function (options) { 2 | this.options = options 3 | this.options.timeout = this.options.timeout || 10000 4 | } 5 | 6 | ScriptsManager.prototype.start = function (cb) { 7 | cb() 8 | } 9 | 10 | ScriptsManager.prototype.ensureStarted = function (cb) { 11 | cb() 12 | } 13 | 14 | ScriptsManager.prototype.execute = function (inputs, options, cb) { 15 | var self 16 | var resolved = false 17 | var timeoutValue = options.timeout || this.options.timeout 18 | var timeout 19 | 20 | if (timeoutValue !== -1) { 21 | timeout = setTimeout(function () { 22 | resolved = true 23 | cb(new Error(options.timeoutErrorMessage || 'Timeout error during executing script')) 24 | }, timeoutValue) 25 | } 26 | 27 | if (timeout) { 28 | timeout.unref() 29 | } 30 | 31 | require(options.execModulePath)(inputs, function () { 32 | if (resolved) { 33 | return 34 | } 35 | 36 | var params = Array.prototype.slice.call(arguments) 37 | var originalCbRespond = params.pop() 38 | 39 | params.push(function () { 40 | if (resolved) { 41 | return 42 | } 43 | 44 | var args = Array.prototype.slice.call(arguments) 45 | originalCbRespond.apply(undefined, args) 46 | }) 47 | 48 | options.callback.apply(self, params) 49 | }, function (err, res) { 50 | if (resolved) { 51 | return 52 | } 53 | 54 | resolved = true 55 | 56 | if (timeout) { 57 | clearTimeout(timeout) 58 | } 59 | 60 | cb(err, res) 61 | }) 62 | } 63 | 64 | ScriptsManager.prototype.kill = function () { 65 | } 66 | -------------------------------------------------------------------------------- /lib/getRandomPort.js: -------------------------------------------------------------------------------- 1 | var net = require('net') 2 | 3 | var DEFAULT_MIN = 1025 4 | var DEFAULT_MAX = 65535 5 | var DEFAULT_MAX_ATTEMPTS = 50 6 | 7 | function getRandomPort (opts, cb) { 8 | var min = DEFAULT_MIN 9 | var max = DEFAULT_MAX 10 | var maxAttempts = DEFAULT_MAX_ATTEMPTS 11 | var options = opts || {} 12 | var host 13 | var port 14 | var server 15 | 16 | if (options.min != null) { 17 | min = options.min 18 | } 19 | 20 | if (options.max != null) { 21 | max = options.max 22 | } 23 | 24 | if (options.maxAttempts != null) { 25 | maxAttempts = options.maxAttempts 26 | } 27 | 28 | if (options.host != null) { 29 | host = options.host 30 | } 31 | 32 | port = getRandomNumber(min, max) 33 | 34 | server = net.createServer() 35 | 36 | server.listen(port, host, function () { 37 | server.once('close', function () { cb(null, port) }) 38 | server.close() 39 | }) 40 | 41 | server.on('error', function () { 42 | if (--maxAttempts) { 43 | return getRandomPort({ 44 | min: min, 45 | max: max, 46 | maxAttempts: maxAttempts 47 | }, cb) 48 | } 49 | 50 | cb(new Error('Could not find an available port')) 51 | }) 52 | } 53 | 54 | function getRandomNumber (minimum, maximum) { 55 | if (maximum === undefined) { 56 | maximum = minimum 57 | minimum = 0 58 | } 59 | 60 | if (typeof minimum !== 'number' || typeof maximum !== 'number') { 61 | throw new TypeError('Expected all arguments to be numbers') 62 | } 63 | 64 | return Math.floor( 65 | (Math.random() * (maximum - minimum + 1)) + minimum 66 | ) 67 | } 68 | 69 | module.exports = getRandomPort 70 | -------------------------------------------------------------------------------- /lib/worker-processes.js: -------------------------------------------------------------------------------- 1 | var uuid = require('uuid').v4 2 | var messageHandler = require('./messageHandler') 3 | 4 | process.on('uncaughtException', function (err) { 5 | process.send(messageHandler.serialize({ 6 | error: err.message, 7 | errorStack: err.stack 8 | }), () => {}) 9 | 10 | process.exit() 11 | }) 12 | 13 | var cbs = {} 14 | 15 | function callback () { 16 | var cid = uuid() 17 | 18 | cbs[cid] = arguments[arguments.length - 1] 19 | 20 | var args = Array.prototype.slice.call(arguments) 21 | args.pop() 22 | 23 | process.send(messageHandler.serialize({ 24 | action: 'callback', 25 | cid: cid, 26 | pid: process.pid, 27 | params: args.sort() 28 | }), () => {}) 29 | } 30 | 31 | function sendAndExit (m) { 32 | // we check for the amount of arguments that `process.send` supports 33 | // to provide support for older versions (<=4.x.x) of node that doesn't support a callback 34 | if (process.send.length <= 2) { 35 | process.send(messageHandler.serialize(m), () => {}) 36 | 37 | setTimeout(function () { 38 | process.exit() 39 | }, 5000) 40 | } else { 41 | // since other arguments in `process.send` are optional a call with two arguments 42 | // works in the rest of versions 43 | process.send(messageHandler.serialize(m), function () { 44 | process.exit() 45 | }) 46 | } 47 | } 48 | 49 | process.on('message', function (rawM) { 50 | var m = messageHandler.parse(rawM) 51 | 52 | if (m.action === 'callback-response') { 53 | if (m.params.length) { 54 | if (m.params[0]) { 55 | m.params[0] = new Error(m.params[0]) 56 | } 57 | } 58 | 59 | var cb = cbs[m.cid] 60 | 61 | delete cbs[m.cid] 62 | 63 | return cb.apply(this, m.params) 64 | } 65 | 66 | require(m.options.execModulePath)(m.inputs, callback, function (err, val) { 67 | if (err) { 68 | sendAndExit({ 69 | error: err.message, 70 | errorStack: err.stack 71 | }) 72 | } else { 73 | sendAndExit({ 74 | action: 'process-response', 75 | value: val 76 | }) 77 | } 78 | }) 79 | }) 80 | -------------------------------------------------------------------------------- /lib/manager-processes.js: -------------------------------------------------------------------------------- 1 | var childProcess = require('child_process') 2 | var path = require('path') 3 | var messageHandler = require('serializator') 4 | 5 | var ScriptsManager = module.exports = function (options) { 6 | this.options = options 7 | this.options.timeout = this.options.timeout || 10000 8 | } 9 | 10 | ScriptsManager.prototype.start = function (cb) { 11 | cb() 12 | } 13 | 14 | ScriptsManager.prototype.ensureStarted = function (cb) { 15 | cb() 16 | } 17 | 18 | ScriptsManager.prototype.execute = function (inputs, options, cb) { 19 | var self = this 20 | var worker = childProcess.fork(path.join(__dirname, 'worker-processes.js'), self.options.forkOptions || {}) 21 | var killed = false 22 | var timeoutValue = options.timeout || this.options.timeout 23 | var timeout 24 | 25 | if (timeoutValue !== -1) { 26 | timeout = setTimeout(function () { 27 | worker.kill() 28 | killed = true 29 | cb(new Error(options.timeoutErrorMessage || 'Timeout error during executing script')) 30 | }, timeoutValue) 31 | } 32 | 33 | if (timeout) { 34 | timeout.unref() 35 | } 36 | 37 | worker.on('message', function (rawM) { 38 | var m = messageHandler.parse(rawM) 39 | 40 | if (killed) { 41 | return 42 | } 43 | 44 | if (m.error) { 45 | if (timeout) { 46 | clearTimeout(timeout) 47 | } 48 | 49 | var error = new Error(m.error) 50 | error.stack = m.errorStack 51 | return cb(error) 52 | } 53 | 54 | if (m.action === 'process-response') { 55 | if (timeout) { 56 | clearTimeout(timeout) 57 | } 58 | 59 | return cb(null, m.value) 60 | } 61 | 62 | if (m.action === 'callback') { 63 | m.params.push(function () { 64 | if (killed) { 65 | return 66 | } 67 | 68 | var args = Array.prototype.slice.call(arguments) 69 | 70 | if (args.length && args[0]) { 71 | args[0] = args[0].message 72 | } 73 | 74 | worker.send(messageHandler.serialize({ 75 | action: 'callback-response', 76 | cid: m.cid, 77 | params: args 78 | }), () => {}) 79 | }) 80 | 81 | options.callback.apply(self, m.params) 82 | } 83 | }) 84 | 85 | worker.send(messageHandler.serialize({ 86 | inputs: inputs, 87 | options: options 88 | }), () => {}) 89 | } 90 | 91 | ScriptsManager.prototype.kill = function () { 92 | } 93 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # script-manager 2 | [![NPM Version](http://img.shields.io/npm/v/script-manager.svg?style=flat-square)](https://npmjs.com/package/script-manager) 3 | [![License](http://img.shields.io/npm/l/script-manager.svg?style=flat-square)](http://opensource.org/licenses/MIT) 4 | [![Build Status](https://travis-ci.org/pofider/node-script-manager.png?branch=master)](https://travis-ci.org/pofider/node-script-manager) 5 | 6 | **node.js manager for running foreign and potentially dangerous scripts in the cluster** 7 | 8 | 9 | ## Basics 10 | 11 | You can use node.js vm module for running a custom javascript code, but when the code is bad it can quickly get your node.js process into an endless loop. For this reason it is better to run users's custom code in a separate node process which you can recycle when the script reaches timeout. This can be achieved using node child_process module, but a simple implementation has limitations in performance and scale because running each script in a new node child process can quickly spawn whole system with node processes. This package solves the problem of running user's custom javascript code in a load balanced cluster of node processes which are reused over the requests and recycled when needed. 12 | 13 | ```js 14 | var path = require('path') 15 | var scriptManager = require("script-manager")({ numberOfWorkers: 2 }); 16 | 17 | scriptManager.ensureStarted(function(err) { 18 | 19 | /*send user's script including some other specific options into 20 | wrapper specified by execModulePath*/ 21 | scriptManager.execute({ 22 | script: "return 'Jan';" 23 | }, { 24 | execModulePath: path.join(__dirname, "script.js"), 25 | timeout: 10 26 | }, function(err, res) { 27 | if (err) { 28 | return console.error('Error:', err) 29 | } 30 | 31 | console.log(res); 32 | }); 33 | 34 | }); 35 | ``` 36 | 37 | ```js 38 | /*script.js 39 | wrapper usually does some fancy thing and then runs the custom script using node.js vm module*/ 40 | module.exports = function(inputs, callback, done) { 41 | var result = require('vm').runInNewContext(inputs.script, { 42 | require: function() { throw new Error("Not supported"); } 43 | }); 44 | done(null result); 45 | }; 46 | ``` 47 | 48 | ## Callbacks 49 | The executing script can also callback to the caller process. The callback is provided using `node.js` cross process messages so it has some limitations, but should work when transferring just common objects in parameters. 50 | 51 | To provide caller callback you can add the `callback` property to the `execute` options: 52 | 53 | ```js 54 | scriptManager.execute({ 55 | script: "return 'Jan';" 56 | }, { 57 | execModulePath: path.join(__dirname, "script.js"), 58 | callback: function(argA, argB, cb) { 59 | cb(null, "foo"); 60 | } 61 | }, function(err, res) { 62 | console.log(res); 63 | }); 64 | ``` 65 | 66 | Then in the wrapper you can for example offer a function `funcA` to the users script which uses callback parameter to contact the original caller. 67 | 68 | ```js 69 | module.exports = function(inputs, callback, done) { 70 | var result = require('vm').runInNewContext(inputs.script, { 71 | require: function() { throw new Error("Not supported"); }, 72 | funcA: function(argA, cb) { 73 | callback(argA, cb); 74 | } 75 | }); 76 | done(result); 77 | }); 78 | ``` 79 | 80 | ## Options 81 | 82 | ```js 83 | var scriptManager = require("script-manager")({ 84 | /* number of worker node.js processes */ 85 | numberOfWorkers: 2, 86 | /* set a custom hostname on which script execution server is started, useful is cloud environments where you need to set specific IP */ 87 | host: '127.0.0.1', 88 | /* set a specific port range for script execution server */ 89 | portLeftBoundary: 1000, 90 | portRightBoundary: 2000, 91 | /* maximum size of message sent/received from/to worker in http-server strategy, pass -1 to have no limit*/ 92 | inputRequestLimit: 200e6, 93 | /* switch to use dedicated process for script evalution, this can help with 94 | some issues caused by corporate proxies */ 95 | strategy: "http-server | dedicated-process | in-process", 96 | /* options passed to forked node worker process: { execArgv: ['�-max-old-space-size=128'] } */ 97 | forkOptions: {} 98 | }); 99 | ``` 100 | 101 | 102 | ## License 103 | See [license](https://github.com/pofider/node-script-manager/blob/master/LICENSE) 104 | -------------------------------------------------------------------------------- /lib/manager-servers.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright(c) 2014 Jan Blaha 3 | * 4 | * TaskManager responsible for running async tasks. 5 | * It's using cluster on http server to load balance work and also provides 6 | * timout handling 7 | */ 8 | 9 | var childProcess = require('child_process') 10 | var path = require('path') 11 | var uuid = require('uuid').v4 12 | var axios = require('axios') 13 | var getRandomPort = require('./getRandomPort') 14 | var messageHandler = require('./messageHandler') 15 | 16 | var findFreePort = function (host, cb) { 17 | getRandomPort({ 18 | host: host 19 | }, cb) 20 | } 21 | 22 | var findFreePortInRange = function (host, portLeftBoundary, portRightBoundary, cb) { 23 | getRandomPort({ 24 | min: portLeftBoundary, 25 | max: portRightBoundary, 26 | host: host 27 | }, cb) 28 | } 29 | 30 | var ScriptsManager = module.exports = function (options) { 31 | this.options = options 32 | this.options.timeout = this.options.timeout || 10000 33 | this.options.numberOfWorkers = this.options.numberOfWorkers || 1 34 | this.options.host = this.options.host || '127.0.0.1' 35 | this._runningRequests = [] 36 | 37 | var self = this 38 | 39 | process.once('exit', function () { 40 | self.kill() 41 | }) 42 | 43 | if (this.options.portLeftBoundary && this.options.portRightBoundary) { 44 | this.findFreePort = function (cb) { 45 | findFreePortInRange(self.options.host, self.options.portLeftBoundary, self.options.portRightBoundary, cb) 46 | } 47 | } else { 48 | this.findFreePort = function (cb) { 49 | findFreePort(self.options.host, cb) 50 | } 51 | } 52 | } 53 | 54 | ScriptsManager.prototype.start = function (cb) { 55 | var self = this 56 | 57 | this.findFreePort(function (err, port) { 58 | if (err) { 59 | return cb(err) 60 | } 61 | 62 | self.options.port = port 63 | 64 | var forkOptions = self.options.forkOptions || {} 65 | forkOptions.env = Object.assign({}, process.env, forkOptions.env || {}) 66 | 67 | self.workersClusterId = uuid() 68 | 69 | forkOptions.env['SCRIPT_MANAGER_WORKERS_CLUSTER_ID'] = self.workersClusterId 70 | 71 | self.workersCluster = childProcess.fork(path.join(__dirname, 'worker-servers.js'), forkOptions) 72 | 73 | self.workersCluster.on('exit', function () { 74 | // manual kill 75 | if (!self.isStarted) { 76 | return 77 | } 78 | 79 | self.start(function () { 80 | 81 | }) 82 | }) 83 | 84 | self.workersCluster.on('message', function (rawM) { 85 | var m = messageHandler.parse(rawM) 86 | 87 | if (m.action === 'running') { 88 | self.isStarted = true 89 | cb() 90 | } 91 | }) 92 | 93 | self.workersCluster.on('message', function (rawM) { 94 | var m = messageHandler.parse(rawM) 95 | var reqOptions 96 | 97 | if (m.action === 'callback') { 98 | reqOptions = self._runningRequests.find(r => r.rid === m.rid) 99 | 100 | if (!reqOptions || reqOptions.isDone) { 101 | return 102 | } 103 | 104 | m.params.push(function () { 105 | if (reqOptions.isDone) { 106 | return 107 | } 108 | 109 | var args = Array.prototype.slice.call(arguments) 110 | 111 | if (args.length && args[0]) { 112 | args[0] = args[0].message 113 | } 114 | 115 | self.workersCluster.send(messageHandler.serialize({ 116 | action: 'callback-response', 117 | cid: m.cid, 118 | rid: m.rid, 119 | params: args 120 | })) 121 | }) 122 | 123 | reqOptions.callback.apply(self, m.params) 124 | } 125 | 126 | if (m.action === 'register') { 127 | reqOptions = self._runningRequests.find(r => r.rid === m.rid) 128 | 129 | if (!reqOptions) { 130 | return 131 | } 132 | 133 | var timeoutValue = reqOptions.timeout || self.options.timeout 134 | 135 | if (timeoutValue !== -1) { 136 | // TODO we should actually kill only the script that caused timeout and resend other requests from the same worker... some more complicated logic is required here 137 | reqOptions.timeoutRef = setTimeout(function () { 138 | if (reqOptions.isDone) { 139 | return 140 | } 141 | 142 | reqOptions.isDone = true 143 | 144 | self.workersCluster.send(messageHandler.serialize({ 145 | action: 'kill', 146 | rid: reqOptions.rid 147 | })) 148 | 149 | var error = new Error() 150 | error.weak = true 151 | error.message = reqOptions.timeoutErrorMessage || 'Timeout error during executing script' 152 | 153 | self._runningRequests = self._runningRequests.filter(r => r.rid !== reqOptions.rid) 154 | 155 | reqOptions.cb(error) 156 | }, timeoutValue) 157 | } 158 | 159 | if (reqOptions.timeoutRef) { 160 | reqOptions.timeoutRef.unref() 161 | } 162 | } 163 | }) 164 | 165 | self.workersCluster.send(messageHandler.serialize({ 166 | action: 'start', 167 | port: self.options.port, 168 | host: self.options.host, 169 | inputRequestLimit: self.options.inputRequestLimit || 200e6, 170 | numberOfWorkers: self.options.numberOfWorkers 171 | })) 172 | }) 173 | } 174 | 175 | ScriptsManager.prototype.ensureStarted = function (cb) { 176 | if (this.isStarted) { 177 | return cb() 178 | } 179 | 180 | // TODO we should probably make lock here to avoid multiple node.exe processes in parallel init 181 | this.start(cb) 182 | } 183 | 184 | ScriptsManager.prototype.execute = function (inputs, options, cb) { 185 | var self = this 186 | 187 | options.wcid = self.workersClusterId 188 | options.rid = options.rid = uuid() 189 | options.isDone = false 190 | options.cb = cb 191 | 192 | var body = { 193 | inputs: inputs, 194 | options: options 195 | } 196 | 197 | this._runningRequests.push(options) 198 | 199 | function handleResponse (err, response) { 200 | if (options.timeoutRef) { 201 | clearTimeout(options.timeoutRef) 202 | } 203 | 204 | if (options.isDone) { 205 | return 206 | } 207 | 208 | options.isDone = true 209 | 210 | self._runningRequests = self._runningRequests.filter(r => r.rid !== options.rid) 211 | 212 | if (err) { 213 | return cb(err) 214 | } 215 | 216 | var body = messageHandler.parse(response.data) 217 | 218 | if (!body) { 219 | return cb(new Error('Something went wrong in communication with internal scripting server. You may try to change scripting strategy from `http-server` to `dedicated-process`.')) 220 | } 221 | 222 | if (body.error && response.status !== 200) { 223 | var e = new Error() 224 | e.message = body.error.message 225 | e.stack = body.error.stack 226 | e.weak = true 227 | return cb(e) 228 | } 229 | 230 | cb(null, body) 231 | } 232 | 233 | axios({ 234 | method: 'post', 235 | url: 'http://' + this.options.host + ':' + this.options.port, 236 | headers: { 237 | 'Content-Type': 'application/json' 238 | }, 239 | // disable request/response body limit, don't throw on large payload response, or when the post body is large 240 | // https://github.com/axios/axios/issues/2696 241 | maxContentLength: Infinity, 242 | data: messageHandler.serialize(body), 243 | // we don't want any parsing in the response data, we want the raw form (string) 244 | transformResponse: [] 245 | }).then((response) => { 246 | handleResponse(null, response) 247 | }).catch(err => { 248 | if (err.response) { 249 | handleResponse(null, err.response) 250 | } else { 251 | handleResponse(err) 252 | } 253 | }) 254 | } 255 | 256 | ScriptsManager.prototype.kill = function () { 257 | if (this.workersCluster) { 258 | this.isStarted = false 259 | this.workersCluster.kill() 260 | } 261 | } 262 | -------------------------------------------------------------------------------- /lib/worker-servers.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright(c) 2014 Jan Blaha 3 | * 4 | * http server cluster listening on dedicated work and executing specified tasks 5 | */ 6 | 7 | var cluster = require('cluster') 8 | // eslint-disable-next-line 9 | var domain = require('domain') 10 | var uuid = require('uuid').v4 11 | var messageHandler = require('./messageHandler') 12 | 13 | var workers = [] 14 | var currentRequests = {} 15 | var port 16 | var host 17 | var inputRequestLimit 18 | var callbackRequests = {} 19 | 20 | var workersClusterId = process.env.SCRIPT_MANAGER_WORKERS_CLUSTER_ID 21 | delete process.env.SCRIPT_MANAGER_WORKERS_CLUSTER_ID 22 | 23 | if (cluster.isMaster) { 24 | var isRunning = false 25 | 26 | cluster.on('fork', function (worker) { 27 | worker.pid = worker.process.pid 28 | worker.isRunning = false 29 | workers.push(worker) 30 | 31 | worker.process.on('message', function (rawM) { 32 | var m = messageHandler.parse(rawM) 33 | 34 | if (m.action === 'register') { 35 | var worker = workers.find(w => w.pid === m.pid) 36 | 37 | // maybe worker was recycled before it started? unlikely, but to be sure we don't crash the master 38 | if (worker) { 39 | currentRequests[m.rid] = worker 40 | 41 | process.send(messageHandler.serialize({ 42 | action: 'register', 43 | rid: m.rid 44 | })) 45 | } 46 | } 47 | 48 | if (m.action === 'completed') { 49 | delete currentRequests[m.rid] 50 | } 51 | 52 | if (m.action === 'callback') { 53 | process.send(messageHandler.serialize(m)) 54 | } 55 | }) 56 | 57 | worker.on('exit', function (w, code, signal) { 58 | workers = workers.filter(w => w.pid !== worker.pid) 59 | 60 | var keysToDelete = [] 61 | 62 | for (var key in currentRequests) { 63 | if (currentRequests[key].pid === worker.pid) { 64 | keysToDelete.push(key) 65 | } 66 | } 67 | 68 | keysToDelete.forEach(function (k) { 69 | delete currentRequests[k] 70 | }) 71 | 72 | cluster.fork(Object.assign({}, process.env, { 73 | SCRIPT_MANAGER_WORKERS_CLUSTER_ID: workersClusterId 74 | })) 75 | }) 76 | 77 | worker.send(messageHandler.serialize({ 78 | action: 'start', 79 | port: port, 80 | host: host, 81 | inputRequestLimit: inputRequestLimit 82 | })) 83 | }) 84 | 85 | cluster.on('listening', function (worker) { 86 | if (isRunning) { 87 | return 88 | } 89 | 90 | worker.isRunning = true 91 | 92 | if (!workers.find(w => w.isRunning === false)) { 93 | isRunning = true 94 | 95 | process.send(messageHandler.serialize({ 96 | action: 'running' 97 | })) 98 | } 99 | }) 100 | 101 | process.on('message', function (rawM) { 102 | var m = messageHandler.parse(rawM) 103 | var worker 104 | 105 | if (m.action === 'kill') { 106 | worker = currentRequests[m.rid] 107 | 108 | if (worker) { 109 | worker.process.kill('SIGKILL') 110 | } 111 | } 112 | 113 | if (m.action === 'start') { 114 | port = m.port 115 | host = m.host 116 | inputRequestLimit = m.inputRequestLimit 117 | 118 | for (var i = 0; i < m.numberOfWorkers; i++) { 119 | cluster.fork(Object.assign({}, process.env, { 120 | SCRIPT_MANAGER_WORKERS_CLUSTER_ID: workersClusterId 121 | })) 122 | } 123 | } 124 | 125 | if (m.action === 'callback-response') { 126 | worker = currentRequests[m.rid] 127 | 128 | // worker could be recycled in the meantime 129 | if (worker) { 130 | worker.send(messageHandler.serialize(m)) 131 | } 132 | } 133 | }) 134 | } 135 | 136 | if (!cluster.isMaster) { 137 | var startListening = function (port, host) { 138 | var server = require('http').createServer(function (req, res) { 139 | // NOTE: we're still using domains here intentionally, 140 | // we have tried to avoid its usage but unfortunately there is no other way to 141 | // ensure that we are handling all kind of errors that can occur in an external script, 142 | // but everything is ok because node.js will only remove domains when they found an alternative 143 | // and when that time comes, we just need to migrate to that alternative. 144 | var d = domain.create() 145 | 146 | d.on('error', function (er) { 147 | try { 148 | // make sure we close down within 30 seconds 149 | var killtimer = setTimeout(function () { 150 | process.exit(1) 151 | }, 30000) 152 | 153 | // But don't keep the process open just for that! 154 | killtimer.unref() 155 | 156 | // stop taking new requests. 157 | server.close() 158 | 159 | // Let the master know we're dead. This will trigger a 160 | // 'disconnect' in the cluster master, and then it will fork 161 | // a new worker. 162 | if (cluster) { 163 | cluster.worker.disconnect() 164 | } 165 | 166 | error(res, er) 167 | } catch (er2) { 168 | // oh well, not much we can do at this point. 169 | console.error('Error sending 500!', er2.stack) 170 | } 171 | }) 172 | 173 | d.add(req) 174 | d.add(res) 175 | d.req = req 176 | 177 | d.run(function () { 178 | processRequest(workersClusterId, req, res) 179 | }) 180 | }) 181 | 182 | server.timeout = 0 183 | server.listen(port, host) 184 | } 185 | 186 | process.on('message', function (rawM) { 187 | var m = messageHandler.parse(rawM) 188 | inputRequestLimit = m.inputRequestLimit 189 | 190 | if (m.action === 'start') { 191 | startListening(m.port, m.host) 192 | } 193 | 194 | if (m.action === 'callback-response') { 195 | callbackRequests[m.rid](m) 196 | } 197 | }) 198 | } 199 | 200 | function error (res, err) { 201 | res.writeHead(500) 202 | 203 | res.end(messageHandler.serialize({ 204 | error: { 205 | message: err.message, 206 | stack: err.stack 207 | } 208 | })) 209 | } 210 | 211 | function processRequest (workersClusterId, req, res) { 212 | var body = [] 213 | var length = 0 214 | 215 | req.on('data', function (data) { 216 | body.push(data) 217 | length += data.length 218 | 219 | if (inputRequestLimit !== -1 && length > inputRequestLimit) { 220 | error(res, new Error('Input request exceeded inputRequestLimit')) 221 | res.destroy() 222 | } 223 | }) 224 | 225 | req.on('end', function () { 226 | req.body = messageHandler.parse(Buffer.concat(body).toString()) 227 | 228 | if (!req.body.options.wcid || req.body.options.wcid !== workersClusterId) { 229 | return error(res, new Error('Bad request')) 230 | } 231 | 232 | process.send(messageHandler.serialize({ 233 | action: 'register', 234 | rid: req.body.options.rid, 235 | pid: process.pid 236 | })) 237 | 238 | try { 239 | var cbs = {} 240 | 241 | var callback = function () { 242 | var cid = uuid() 243 | 244 | cbs[cid] = arguments[arguments.length - 1] 245 | 246 | if (!callbackRequests[req.body.options.rid]) { 247 | callbackRequests[req.body.options.rid] = function (m) { 248 | if (m.params.length) { 249 | if (m.params[0]) { 250 | m.params[0] = new Error(m.params[0]) 251 | } 252 | } 253 | 254 | var cb = cbs[m.cid] 255 | 256 | delete cbs[m.cid] 257 | 258 | cb.apply(this, m.params) 259 | 260 | if (Object.keys(cbs).length === 0) { 261 | delete callbackRequests[req.body.options.rid] 262 | } 263 | } 264 | } 265 | 266 | var args = Array.prototype.slice.call(arguments) 267 | 268 | args.pop() 269 | 270 | process.send(messageHandler.serialize({ 271 | action: 'callback', 272 | cid: cid, 273 | rid: req.body.options.rid, 274 | pid: process.pid, 275 | params: args.sort() 276 | })) 277 | } 278 | 279 | require(req.body.options.execModulePath)(req.body.inputs, callback, function (err, val) { 280 | if (err) { 281 | return error(res, err) 282 | } 283 | 284 | try { 285 | res.end(messageHandler.serialize(val)) 286 | } catch (eSerialize) { 287 | error(res, eSerialize) 288 | } 289 | }) 290 | } catch (e) { 291 | error(res, e) 292 | } 293 | }) 294 | } 295 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | var should = require('should') 2 | var path = require('path') 3 | var axios = require('axios') 4 | var ScriptsManager = require('../lib/manager-servers.js') 5 | var ScriptsManagerWithProcesses = require('../lib/manager-processes.js') 6 | var ScriptManagerInProcess = require('../lib/in-process.js') 7 | 8 | describe('scripts manager', function () { 9 | describe('servers', function () { 10 | var scriptsManager = new ScriptsManager({ numberOfWorkers: 2 }) 11 | 12 | beforeEach(function (done) { 13 | scriptsManager.ensureStarted(done) 14 | }) 15 | 16 | afterEach(function () { 17 | scriptsManager.kill() 18 | }) 19 | 20 | common(scriptsManager) 21 | commonForSafeExecution(scriptsManager) 22 | 23 | it('should not be able to process request directly to worker', function (done) { 24 | axios({ 25 | method: 'post', 26 | url: 'http://localhost:' + scriptsManager.options.port, 27 | data: { 28 | options: { 29 | rid: 12, 30 | wcid: 'invalid', 31 | execModulePath: path.join(__dirname, 'scripts', 'script.js') 32 | } 33 | } 34 | }).then((response) => { 35 | done(new Error('Request should not be able to end successfully')) 36 | }).catch((err) => { 37 | if (err.response) { 38 | err.response.data.error.message.should.be.eql('Bad request') 39 | done() 40 | } else { 41 | done(err) 42 | } 43 | }) 44 | }) 45 | 46 | it('should work after process recycles', function (done) { 47 | var scriptsManager2 = new ScriptsManager({ numberOfWorkers: 1 }) 48 | 49 | scriptsManager2.ensureStarted(function () { 50 | scriptsManager2.execute({}, { execModulePath: path.join(__dirname, 'scripts', 'unexpectedError.js') }, function (err, res) { 51 | if (!err) { 52 | scriptsManager2.kill() 53 | done(new Error('should have failed')) 54 | } 55 | 56 | // seems we need to wait a bit until it is restarted fully? 57 | setTimeout(function () { 58 | scriptsManager2.execute({}, { execModulePath: path.join(__dirname, 'scripts', 'script.js') }, function (err, res) { 59 | if (err) { 60 | scriptsManager2.kill() 61 | return done(err) 62 | } 63 | 64 | scriptsManager2.kill() 65 | done() 66 | }) 67 | }, 100) 68 | }) 69 | }) 70 | }) 71 | 72 | it('should be able to set up on custom port', function (done) { 73 | var scriptsManager2 = new ScriptsManager({ numberOfWorkers: 1, portLeftBoundary: 10000, portRightBoundary: 11000 }) 74 | 75 | scriptsManager2.start(function (err) { 76 | if (err) { 77 | return done(err) 78 | } 79 | 80 | scriptsManager2.execute({ foo: 'foo' }, { execModulePath: path.join(__dirname, 'scripts', 'script.js') }, function (err, res) { 81 | scriptsManager2.kill() 82 | 83 | if (err) { 84 | return done(err) 85 | } 86 | 87 | scriptsManager2.options.port.should.be.within(10000, 11000) 88 | res.foo.should.be.eql('foo') 89 | done() 90 | }) 91 | }) 92 | }) 93 | 94 | it('should be able to process high data volumes', function (done) { 95 | var data = { foo: 'foo', people: [] } 96 | 97 | for (var i = 0; i < 2000000; i++) { 98 | data.people.push(i) 99 | } 100 | 101 | scriptsManager.execute(data, { execModulePath: path.join(__dirname, 'scripts', 'script.js') }, function (err, res) { 102 | if (err) { 103 | return done(err) 104 | } 105 | 106 | res.foo.should.be.eql('foo') 107 | done() 108 | }) 109 | }) 110 | }) 111 | 112 | describe('servers with custom settings', function () { 113 | it('should fail when input exceeds the inputRequestLimit', function (done) { 114 | var scriptsManager = new ScriptsManager({ numberOfWorkers: 2, inputRequestLimit: 5 }) 115 | 116 | scriptsManager.ensureStarted(function (err) { 117 | if (err) { 118 | return done(err) 119 | } 120 | 121 | scriptsManager.execute('foooooo', { execModulePath: path.join(__dirname, 'scripts', 'script.js') }, function (err, res) { 122 | scriptsManager.kill() 123 | 124 | if (err) { 125 | return done() 126 | } 127 | 128 | done(new Error('It should have dailed')) 129 | }) 130 | }) 131 | }) 132 | 133 | it('should not fail when input is shorter the inputRequestLimit', function (done) { 134 | var scriptsManager = new ScriptsManager({ numberOfWorkers: 2, inputRequestLimit: 500 }) 135 | 136 | scriptsManager.ensureStarted(function (err) { 137 | if (err) { 138 | return done(err) 139 | } 140 | 141 | scriptsManager.execute('foooooo', { execModulePath: path.join(__dirname, 'scripts', 'script.js') }, function (err, res) { 142 | scriptsManager.kill() 143 | 144 | if (err) { 145 | return done(err) 146 | } 147 | 148 | done() 149 | }) 150 | }) 151 | }) 152 | 153 | it('should be able to expose gc through args to dedicated process', function (done) { 154 | var scriptsManager = new ScriptsManager({ numberOfWorkers: 2, strategy: 'dedicated-process', inputRequestLimit: 500, forkOptions: { execArgv: ['--expose-gc'] } }) 155 | 156 | scriptsManager.ensureStarted(function (err) { 157 | if (err) { 158 | return done(err) 159 | } 160 | 161 | scriptsManager.execute({ foo: 'foo' }, { execModulePath: path.join(__dirname, 'scripts', 'gc.js') }, function (err, res) { 162 | scriptsManager.kill() 163 | 164 | if (err) { 165 | return done(err) 166 | } 167 | 168 | res.foo.should.be.eql('foo') 169 | done() 170 | }) 171 | }) 172 | }) 173 | 174 | it('should be able to expose gc through args to http server', function (done) { 175 | var scriptsManager = new ScriptsManager({ numberOfWorkers: 2, strategy: 'http-server', inputRequestLimit: 500, forkOptions: { execArgv: ['--expose-gc'] } }) 176 | 177 | scriptsManager.ensureStarted(function (err) { 178 | if (err) { 179 | return done(err) 180 | } 181 | 182 | scriptsManager.execute({ foo: 'foo' }, { execModulePath: path.join(__dirname, 'scripts', 'gc.js') }, function (err, res) { 183 | scriptsManager.kill() 184 | 185 | if (err) { 186 | return done(err) 187 | } 188 | 189 | res.foo.should.be.eql('foo') 190 | done() 191 | }) 192 | }) 193 | }) 194 | }) 195 | 196 | describe('processes', function () { 197 | var scriptsManager = new ScriptsManagerWithProcesses({ numberOfWorkers: 2 }) 198 | 199 | beforeEach(function (done) { 200 | scriptsManager.ensureStarted(done) 201 | }) 202 | 203 | afterEach(function () { 204 | scriptsManager.kill() 205 | }) 206 | 207 | common(scriptsManager) 208 | commonForSafeExecution(scriptsManager) 209 | }) 210 | 211 | describe('in process', function () { 212 | var scriptsManager = new ScriptManagerInProcess({}) 213 | 214 | beforeEach(function (done) { 215 | scriptsManager.ensureStarted(done) 216 | }) 217 | 218 | afterEach(function () { 219 | scriptsManager.kill() 220 | }) 221 | 222 | common(scriptsManager) 223 | 224 | it('should handle timeouts', function (done) { 225 | var timeouted = false 226 | 227 | scriptsManager.execute({ foo: 'foo' }, 228 | { 229 | execModulePath: path.join(__dirname, 'scripts', 'timeout.js'), 230 | timeout: 10 231 | }, function (err) { 232 | if (err) { 233 | timeouted = true 234 | done() 235 | } 236 | }) 237 | 238 | setTimeout(function () { 239 | if (!timeouted) { 240 | done(new Error('It should timeout')) 241 | } 242 | }, 500) 243 | }) 244 | }) 245 | 246 | function commonForSafeExecution (scriptsManager) { 247 | it('should handle timeouts', function (done) { 248 | var timeouted = false 249 | 250 | scriptsManager.execute({ foo: 'foo' }, 251 | { 252 | execModulePath: path.join(__dirname, 'scripts', 'timeout.js'), 253 | timeout: 10 254 | }, function () { 255 | timeouted = true 256 | done() 257 | }) 258 | 259 | setTimeout(function () { 260 | if (!timeouted) { 261 | done(new Error('It should timeout')) 262 | } 263 | }, 500) 264 | }) 265 | 266 | it('should handle unexpected error', function (done) { 267 | scriptsManager.execute({ foo: 'foo' }, { execModulePath: path.join(__dirname, 'scripts', 'unexpectedError.js') }, function (err, res) { 268 | if (err) { 269 | return done() 270 | } 271 | 272 | done(new Error('There should be an error')) 273 | }) 274 | }) 275 | } 276 | 277 | function common (scriptsManager) { 278 | it('should be able to execute simple script', function (done) { 279 | scriptsManager.execute({ foo: 'foo' }, { execModulePath: path.join(__dirname, 'scripts', 'script.js') }, function (err, res) { 280 | if (err) { 281 | return done(err) 282 | } 283 | 284 | res.foo.should.be.eql('foo') 285 | done() 286 | }) 287 | }) 288 | 289 | it('should handle script error', function (done) { 290 | scriptsManager.execute({ foo: 'foo' }, { execModulePath: path.join(__dirname, 'scripts', 'error.js') }, function (err, res) { 291 | if (!err) { 292 | return done(new Error('It should have failed.')) 293 | } 294 | 295 | err.stack.should.containEql('error.js') 296 | done() 297 | }) 298 | }) 299 | 300 | it('should be able to handle date values', function (done) { 301 | scriptsManager.execute({ 302 | date: new Date('2018-09-01') 303 | }, { 304 | execModulePath: path.join(__dirname, 'scripts', 'useDate.js') 305 | }, function (err, res) { 306 | if (err) { 307 | return done(err) 308 | } 309 | 310 | res.date.getTime().should.be.eql(res.dateInTime) 311 | done() 312 | }) 313 | }) 314 | 315 | it('should be able to handle date values (callback)', function (done) { 316 | var callback = function (newData, cb) { 317 | cb(null, Object.assign({}, newData, { 318 | internalDateInTime: newData.internalDate.getTime() 319 | })) 320 | } 321 | 322 | scriptsManager.execute({ 323 | useCallback: true, 324 | date: new Date('2018-09-01') 325 | }, { 326 | execModulePath: path.join(__dirname, 'scripts', 'useDate.js'), 327 | callback: callback 328 | }, function (err, res) { 329 | if (err) { 330 | return done(err) 331 | } 332 | 333 | res.date.should.be.Date() 334 | res.date.getTime().should.be.eql(res.dateInTime) 335 | res.internalDate.should.be.Date() 336 | res.internalDate.getTime().should.be.eql(res.internalDateInTime) 337 | done() 338 | }) 339 | }) 340 | 341 | it('should be able to handle buffer values', function (done) { 342 | scriptsManager.execute({ 343 | buf: Buffer.from('hello') 344 | }, { 345 | execModulePath: path.join(__dirname, 'scripts', 'useBuffer.js') 346 | }, function (err, res) { 347 | if (err) { 348 | return done(err) 349 | } 350 | 351 | should(Buffer.isBuffer(res.buf)).be.true() 352 | res.bufInText.should.be.eql('hello') 353 | should(Buffer.isBuffer(res.responseBuf)).be.true() 354 | res.responseBuf.toString().should.be.eql('hello world') 355 | done() 356 | }) 357 | }) 358 | 359 | it('should be able to handle buffer values (callback)', function (done) { 360 | var callback = function (newData, cb) { 361 | cb(null, Object.assign({}, newData, { 362 | receivedBufInText: newData.receivedBuf.toString() 363 | })) 364 | } 365 | 366 | scriptsManager.execute({ 367 | useCallback: true, 368 | buf: Buffer.from('hello') 369 | }, { 370 | execModulePath: path.join(__dirname, 'scripts', 'useBuffer.js'), 371 | callback: callback 372 | }, function (err, res) { 373 | if (err) { 374 | return done(err) 375 | } 376 | 377 | should(Buffer.isBuffer(res.buf)).be.true() 378 | res.bufInText.should.be.eql('hello') 379 | should(Buffer.isBuffer(res.responseBuf)).be.true() 380 | res.responseBuf.toString().should.be.eql('hello world') 381 | should(Buffer.isBuffer(res.receivedBuf)).be.true() 382 | res.receivedBufInText.should.be.eql('secret message') 383 | done() 384 | }) 385 | }) 386 | 387 | it('should be able to callback to the caller', function (done) { 388 | function callback (str, cb) { 389 | cb(null, str + 'aaa') 390 | } 391 | 392 | scriptsManager.execute({}, { 393 | execModulePath: path.join(__dirname, 'scripts', 'callback.js'), 394 | callback: callback 395 | }, function (err, res) { 396 | if (err) { 397 | return done(err) 398 | } 399 | 400 | res.test.should.be.eql('testaaa') 401 | 402 | done() 403 | }) 404 | }) 405 | 406 | it('should be able to callback error to the caller', function (done) { 407 | function callback (str, cb) { 408 | cb(null, str + 'aaa') 409 | } 410 | 411 | scriptsManager.execute({}, { 412 | execModulePath: path.join(__dirname, 'scripts', 'callbackError.js'), 413 | callback: callback 414 | }, function (err, res) { 415 | if (err) { 416 | return done() 417 | } 418 | 419 | done(new Error('It should have failed')) 420 | }) 421 | }) 422 | 423 | it('should be able to handle parallel callback calls', function (done) { 424 | var callback = function (name, cb) { 425 | cb(null, 'hi ' + name) 426 | } 427 | 428 | scriptsManager.execute({ 429 | name: 'Boris' 430 | }, { 431 | execModulePath: path.join(__dirname, 'scripts', 'parallelCallbackCalls.js'), 432 | callback: callback 433 | }, function (err, res) { 434 | if (err) { 435 | return done(err) 436 | } 437 | 438 | res[0].should.be.eql('hi Boris Matos') 439 | res[1].should.be.eql('hi Boris Morillo') 440 | 441 | done() 442 | }) 443 | }) 444 | 445 | it('should be able to customize message when timeout error', function (done) { 446 | scriptsManager.execute({ foo: 'foo' }, { 447 | execModulePath: path.join(__dirname, 'scripts', 'timeout.js'), 448 | timeout: 10, 449 | timeoutErrorMessage: 'Timeout testing case' 450 | }, function (err) { 451 | err.message.should.be.eql('Timeout testing case') 452 | done() 453 | }) 454 | }) 455 | 456 | it('should not call callback after timeout error', function (done) { 457 | var resolved = 0 458 | 459 | function callback (str, cb) { 460 | setTimeout(() => { 461 | cb(null, str + '(callback executed)') 462 | }, 400) 463 | } 464 | 465 | scriptsManager.execute({}, { 466 | execModulePath: path.join(__dirname, 'scripts', 'callback.js'), 467 | timeout: 200, 468 | callback: callback 469 | }, function (err, res) { 470 | if (resolved > 0) { 471 | resolved++ 472 | return 473 | } 474 | 475 | if (resolved === 0) { 476 | err.message.should.containEql('Timeout') 477 | } 478 | 479 | resolved++ 480 | 481 | setTimeout(() => { 482 | resolved.should.be.eql(1) 483 | done() 484 | }, 1000) 485 | }) 486 | }) 487 | 488 | it('should not break when callback is called after script ends execution', function (done) { 489 | const callback = (str, cb) => { 490 | cb() 491 | } 492 | 493 | scriptsManager.execute({}, { 494 | execModulePath: path.join(__dirname, 'scripts', 'callbackAfterEnd.js'), 495 | callback: callback 496 | }, (err, res) => { 497 | if (err) { 498 | return done(err) 499 | } 500 | 501 | res.ok.should.be.True() 502 | setTimeout(() => { done() }, 500) 503 | }) 504 | }) 505 | 506 | it('should be able to differenciate between error and data with error property', function (done) { 507 | scriptsManager.execute({ foo: 'foo' }, { execModulePath: path.join(__dirname, 'scripts', 'okWithErrorProperty.js') }, function (err, res) { 508 | if (err) { 509 | return done(new Error('script should not fail with error')) 510 | } 511 | 512 | res.error.message.should.be.eql('custom') 513 | res.error.stack.should.be.eql('custom stack') 514 | 515 | done() 516 | }) 517 | }) 518 | 519 | it('should be able to process parallel requests', function (done) { 520 | function callback (str, cb) { 521 | setTimeout(function () { 522 | cb(null, str + 'aaa') 523 | }, 10) 524 | } 525 | 526 | var doneCounter = [] 527 | 528 | for (var i = 0; i < 20; i++) { 529 | scriptsManager.execute({}, { 530 | execModulePath: path.join(__dirname, 'scripts', 'callback.js'), 531 | callback: callback 532 | }, function (err, res) { 533 | if (err) { 534 | return done(err) 535 | } 536 | 537 | res.test.should.be.eql('testaaa') 538 | doneCounter++ 539 | 540 | if (doneCounter === 20) { 541 | done() 542 | } 543 | }) 544 | } 545 | }) 546 | 547 | it('should be able to execute script with giant input data', function (done) { 548 | var foo = 'xxx' 549 | 550 | for (var i = 0; i < 1000000; i++) { 551 | foo += 'yyyyyyyyyyyyy' 552 | } 553 | 554 | scriptsManager.execute({ 555 | foo: foo 556 | }, { 557 | execModulePath: path.join(__dirname, 'scripts', 'script.js'), 558 | timeout: 20000 559 | }, function (err, res) { 560 | if (err) { 561 | return done(err) 562 | } 563 | 564 | res.foo.should.be.eql(foo) 565 | done() 566 | }) 567 | }) 568 | } 569 | }) 570 | --------------------------------------------------------------------------------