├── .gitignore ├── examples ├── basic │ ├── child.js │ └── index.js └── pi │ ├── calc.js │ └── index.js ├── .travis.yml ├── tests ├── debug.js ├── child.js └── index.js ├── .editorconfig ├── lib ├── index.js ├── fork.js ├── child │ └── index.js └── farm.js ├── package.json ├── LICENSE.md ├── index.d.ts └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /examples/basic/child.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | module.exports = function (inp, callback) { 4 | callback(null, inp + ' BAR (' + process.pid + ')') 5 | } 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 6 4 | - 8 5 | - 10 6 | - 12 7 | branches: 8 | only: 9 | - master 10 | notifications: 11 | email: 12 | - rod@vagg.org 13 | -------------------------------------------------------------------------------- /tests/debug.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const workerFarm = require('../') 4 | , workers = workerFarm(require.resolve('./child'), ['args']) 5 | 6 | 7 | workers.args(function(err, result) { 8 | console.log(result); 9 | workerFarm.end(workers) 10 | console.log('FINISHED') 11 | process.exit(0) 12 | }) 13 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # This file is for unifying the coding style for different editors and IDEs 2 | # editorconfig.org 3 | 4 | root = true 5 | 6 | [*] 7 | end_of_line = lf 8 | charset = utf-8 9 | insert_final_newline = true 10 | trim_trailing_whitespace = true 11 | indent_style = space 12 | indent_size = 2 13 | 14 | [*.js] 15 | max_line_length = 80 16 | View 17 | -------------------------------------------------------------------------------- /examples/basic/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | let workerFarm = require('../../') 4 | , workers = workerFarm(require.resolve('./child')) 5 | , ret = 0 6 | 7 | for (let i = 0; i < 10; i++) { 8 | workers('#' + i + ' FOO', function (err, outp) { 9 | console.log(outp) 10 | if (++ret == 10) 11 | workerFarm.end(workers) 12 | }) 13 | } 14 | -------------------------------------------------------------------------------- /examples/pi/calc.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | /* A simple π estimation function using a Monte Carlo method 4 | * For 0 to `points`, take 2 random numbers < 1, square and add them to 5 | * find the area under that point in a 1x1 square. If that area is <= 1 6 | * then it's *within* a quarter-circle, otherwise it's outside. 7 | * Take the number of points <= 1 and multiply it by 4 and you have an 8 | * estimate! 9 | * Do this across multiple processes and average the results to 10 | * increase accuracy. 11 | */ 12 | 13 | module.exports = function (points, callback) { 14 | let inside = 0 15 | , i = points 16 | 17 | while (i--) 18 | if (Math.pow(Math.random(), 2) + Math.pow(Math.random(), 2) <= 1) 19 | inside++ 20 | 21 | callback(null, (inside / points) * 4) 22 | } 23 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const Farm = require('./farm') 4 | 5 | let farms = [] // keep record of farms so we can end() them if required 6 | 7 | 8 | function farm (options, path, methods) { 9 | if (typeof options == 'string') { 10 | methods = path 11 | path = options 12 | options = {} 13 | } 14 | 15 | let f = new Farm(options, path) 16 | , api = f.setup(methods) 17 | 18 | farms.push({ farm: f, api: api }) 19 | 20 | // return the public API 21 | return api 22 | } 23 | 24 | 25 | function end (api, callback) { 26 | for (let i = 0; i < farms.length; i++) 27 | if (farms[i] && farms[i].api === api) 28 | return farms[i].farm.end(callback) 29 | process.nextTick(callback.bind(null, new Error('Worker farm not found!'))) 30 | } 31 | 32 | 33 | module.exports = farm 34 | module.exports.end = end 35 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "worker-farm", 3 | "description": "Distribute processing tasks to child processes with an über-simple API and baked-in durability & custom concurrency options.", 4 | "version": "1.7.0", 5 | "homepage": "https://github.com/rvagg/node-worker-farm", 6 | "authors": [ 7 | "Rod Vagg @rvagg (https://github.com/rvagg)" 8 | ], 9 | "keywords": [ 10 | "worker", 11 | "child", 12 | "processing", 13 | "farm" 14 | ], 15 | "main": "./lib/index.js", 16 | "repository": { 17 | "type": "git", 18 | "url": "https://github.com/rvagg/node-worker-farm.git" 19 | }, 20 | "dependencies": { 21 | "errno": "~0.1.7" 22 | }, 23 | "devDependencies": { 24 | "tape": "~4.10.1" 25 | }, 26 | "scripts": { 27 | "test": "node ./tests/" 28 | }, 29 | "types": "./index.d.ts", 30 | "license": "MIT" 31 | } 32 | -------------------------------------------------------------------------------- /lib/fork.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const childProcess = require('child_process') 4 | , childModule = require.resolve('./child/index') 5 | 6 | 7 | function fork (forkModule, workerOptions) { 8 | // suppress --debug / --inspect flags while preserving others (like --harmony) 9 | let filteredArgs = process.execArgv.filter(function (v) { 10 | return !(/^--(debug|inspect)/).test(v) 11 | }) 12 | , options = Object.assign({ 13 | execArgv : filteredArgs 14 | , env : process.env 15 | , cwd : process.cwd() 16 | }, workerOptions) 17 | , child = childProcess.fork(childModule, process.argv, options) 18 | 19 | child.on('error', function() { 20 | // this *should* be picked up by onExit and the operation requeued 21 | }) 22 | 23 | child.send({ owner: 'farm', module: forkModule }) 24 | 25 | // return a send() function for this child 26 | return { 27 | send : child.send.bind(child) 28 | , child : child 29 | } 30 | } 31 | 32 | 33 | module.exports = fork 34 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | ===================== 3 | 4 | Copyright (c) 2014 LevelUP contributors 5 | --------------------------------------- 6 | 7 | *LevelUP contributors listed at * 8 | 9 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 14 | -------------------------------------------------------------------------------- /examples/pi/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const CHILDREN = 500 4 | , POINTS_PER_CHILD = 1000000 5 | , FARM_OPTIONS = { 6 | maxConcurrentWorkers : require('os').cpus().length 7 | , maxCallsPerWorker : Infinity 8 | , maxConcurrentCallsPerWorker : 1 9 | } 10 | 11 | let workerFarm = require('../../') 12 | , calcDirect = require('./calc') 13 | , calcWorker = workerFarm(FARM_OPTIONS, require.resolve('./calc')) 14 | 15 | , ret 16 | , start 17 | 18 | , tally = function (finish, err, avg) { 19 | ret.push(avg) 20 | if (ret.length == CHILDREN) { 21 | let pi = ret.reduce(function (a, b) { return a + b }) / ret.length 22 | , end = +new Date() 23 | console.log('π ≈', pi, '\t(' + Math.abs(pi - Math.PI), 'away from actual!)') 24 | console.log('took', end - start, 'milliseconds') 25 | if (finish) 26 | finish() 27 | } 28 | } 29 | 30 | , calc = function (method, callback) { 31 | ret = [] 32 | start = +new Date() 33 | for (let i = 0; i < CHILDREN; i++) 34 | method(POINTS_PER_CHILD, tally.bind(null, callback)) 35 | } 36 | 37 | console.log('Doing it the slow (single-process) way...') 38 | calc(calcDirect, function () { 39 | console.log('Doing it the fast (multi-process) way...') 40 | calc(calcWorker, process.exit) 41 | }) 42 | -------------------------------------------------------------------------------- /lib/child/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | let $module 4 | 5 | /* 6 | let contextProto = this.context; 7 | while (contextProto = Object.getPrototypeOf(contextProto)) { 8 | completionGroups.push(Object.getOwnPropertyNames(contextProto)); 9 | } 10 | */ 11 | 12 | 13 | function handle (data) { 14 | let idx = data.idx 15 | , child = data.child 16 | , method = data.method 17 | , args = data.args 18 | , callback = function () { 19 | let _args = Array.prototype.slice.call(arguments) 20 | if (_args[0] instanceof Error) { 21 | let e = _args[0] 22 | _args[0] = { 23 | '$error' : '$error' 24 | , 'type' : e.constructor.name 25 | , 'message' : e.message 26 | , 'stack' : e.stack 27 | } 28 | Object.keys(e).forEach(function(key) { 29 | _args[0][key] = e[key] 30 | }) 31 | } 32 | process.send({ owner: 'farm', idx: idx, child: child, args: _args }) 33 | } 34 | , exec 35 | 36 | if (method == null && typeof $module == 'function') 37 | exec = $module 38 | else if (typeof $module[method] == 'function') 39 | exec = $module[method] 40 | 41 | if (!exec) 42 | return console.error('NO SUCH METHOD:', method) 43 | 44 | exec.apply(null, args.concat([ callback ])) 45 | } 46 | 47 | 48 | process.on('message', function (data) { 49 | if (data.owner !== 'farm') { 50 | return; 51 | } 52 | 53 | if (!$module) return $module = require(data.module) 54 | if (data.event == 'die') return process.exit(0) 55 | handle(data) 56 | }) 57 | -------------------------------------------------------------------------------- /index.d.ts: -------------------------------------------------------------------------------- 1 | import { ForkOptions } from "child_process"; 2 | 3 | export = Farm; 4 | 5 | declare function Farm(name: string): Farm.Workers; 6 | declare function Farm(name: string, exportedMethods: string[]): Farm.Workers; 7 | declare function Farm(options: Farm.FarmOptions, name: string): Farm.Workers; 8 | declare function Farm( 9 | options: Farm.FarmOptions, 10 | name: string, 11 | exportedMethods: string[], 12 | ): Farm.Workers; 13 | 14 | type WorkerCallback0 = () => void; 15 | type WorkerCallback1 = (arg1: any) => void; 16 | type WorkerCallback2 = (arg1: any, arg2: any) => void; 17 | type WorkerCallback3 = (arg1: any, arg2: any, arg3: any) => void; 18 | type WorkerCallback4 = (arg1: any, arg2: any, arg3: any, arg4: any) => void; 19 | 20 | declare namespace Farm { 21 | export function end(workers: Workers, callback?: Function): void; 22 | 23 | export interface Workers { 24 | [x: string]: Workers, 25 | (callback: WorkerCallback): void; 26 | (arg1: any, callback: WorkerCallback): void; 27 | (arg1: any, arg2: any, callback: WorkerCallback): void; 28 | (arg1: any, arg2: any, arg3: any, callback: WorkerCallback): void; 29 | ( 30 | arg1: any, 31 | arg2: any, 32 | arg3: any, 33 | arg4: any, 34 | callback: WorkerCallback, 35 | ): void; 36 | } 37 | 38 | export interface FarmOptions { 39 | maxCallsPerWorker?: number; 40 | maxConcurrentWorkers?: number; 41 | maxConcurrentCallsPerWorker?: number; 42 | maxConcurrentCalls?: number; 43 | maxCallTime?: number; 44 | maxRetries?: number; 45 | autoStart?: boolean; 46 | workerOptions?: ForkOptions; 47 | } 48 | 49 | export type WorkerCallback = 50 | | WorkerCallback0 51 | | WorkerCallback1 52 | | WorkerCallback2 53 | | WorkerCallback3 54 | | WorkerCallback4; 55 | } 56 | -------------------------------------------------------------------------------- /tests/child.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const fs = require('fs') 4 | const started = Date.now() 5 | 6 | 7 | module.exports = function (timeout, callback) { 8 | callback = callback.bind(null, null, process.pid, Math.random(), timeout) 9 | if (timeout) 10 | return setTimeout(callback, timeout) 11 | callback() 12 | } 13 | 14 | 15 | module.exports.args = function (callback) { 16 | callback(null, { 17 | argv : process.argv 18 | , cwd : process.cwd() 19 | , execArgv : process.execArgv 20 | }) 21 | } 22 | 23 | 24 | module.exports.run0 = function (callback) { 25 | module.exports(0, callback) 26 | } 27 | 28 | 29 | module.exports.killable = function (id, callback) { 30 | if (Math.random() < 0.5) 31 | return process.exit(-1) 32 | callback(null, id, process.pid) 33 | } 34 | 35 | 36 | module.exports.err = function (type, message, data, callback) { 37 | if (typeof data == 'function') { 38 | callback = data 39 | data = null 40 | } else { 41 | let err = new Error(message) 42 | Object.keys(data).forEach(function(key) { 43 | err[key] = data[key] 44 | }) 45 | callback(err) 46 | return 47 | } 48 | 49 | if (type == 'TypeError') 50 | return callback(new TypeError(message)) 51 | callback(new Error(message)) 52 | } 53 | 54 | 55 | module.exports.block = function () { 56 | while (true); 57 | } 58 | 59 | 60 | // use provided file path to save retries count among terminated workers 61 | module.exports.stubborn = function (path, callback) { 62 | function isOutdated(path) { 63 | return ((new Date).getTime() - fs.statSync(path).mtime.getTime()) > 2000 64 | } 65 | 66 | // file may not be properly deleted, check if modified no earler than two seconds ago 67 | if (!fs.existsSync(path) || isOutdated(path)) { 68 | fs.writeFileSync(path, '1') 69 | process.exit(-1) 70 | } 71 | 72 | let retry = parseInt(fs.readFileSync(path, 'utf8')) 73 | if (Number.isNaN(retry)) 74 | return callback(new Error('file contents is not a number')) 75 | 76 | if (retry > 4) { 77 | callback(null, 12) 78 | } else { 79 | fs.writeFileSync(path, String(retry + 1)) 80 | process.exit(-1) 81 | } 82 | } 83 | 84 | 85 | module.exports.uptime = function (callback) { 86 | callback(null, Date.now() - started) 87 | } 88 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Worker Farm [![Build Status](https://secure.travis-ci.org/rvagg/node-worker-farm.svg)](http://travis-ci.org/rvagg/node-worker-farm) 2 | 3 | [![NPM](https://nodei.co/npm/worker-farm.png?downloads=true&downloadRank=true&stars=true)](https://nodei.co/npm/worker-farm/) 4 | 5 | 6 | Distribute processing tasks to child processes with an über-simple API and baked-in durability & custom concurrency options. *Available in npm as worker-farm*. 7 | 8 | ## Example 9 | 10 | Given a file, *child.js*: 11 | 12 | ```js 13 | module.exports = function (inp, callback) { 14 | callback(null, inp + ' BAR (' + process.pid + ')') 15 | } 16 | ``` 17 | 18 | And a main file: 19 | 20 | ```js 21 | var workerFarm = require('worker-farm') 22 | , workers = workerFarm(require.resolve('./child')) 23 | , ret = 0 24 | 25 | for (var i = 0; i < 10; i++) { 26 | workers('#' + i + ' FOO', function (err, outp) { 27 | console.log(outp) 28 | if (++ret == 10) 29 | workerFarm.end(workers) 30 | }) 31 | } 32 | ``` 33 | 34 | We'll get an output something like the following: 35 | 36 | ``` 37 | #1 FOO BAR (8546) 38 | #0 FOO BAR (8545) 39 | #8 FOO BAR (8545) 40 | #9 FOO BAR (8546) 41 | #2 FOO BAR (8548) 42 | #4 FOO BAR (8551) 43 | #3 FOO BAR (8549) 44 | #6 FOO BAR (8555) 45 | #5 FOO BAR (8553) 46 | #7 FOO BAR (8557) 47 | ``` 48 | 49 | This example is contained in the *[examples/basic](https://github.com/rvagg/node-worker-farm/tree/master/examples/basic/)* directory. 50 | 51 | ### Example #1: Estimating π using child workers 52 | 53 | You will also find a more complex example in *[examples/pi](https://github.com/rvagg/node-worker-farm/tree/master/examples/pi/)* that estimates the value of **π** by using a Monte Carlo *area-under-the-curve* method and compares the speed of doing it all in-process vs using child workers to complete separate portions. 54 | 55 | Running `node examples/pi` will give you something like: 56 | 57 | ``` 58 | Doing it the slow (single-process) way... 59 | π ≈ 3.1416269360000006 (0.0000342824102075312 away from actual!) 60 | took 8341 milliseconds 61 | Doing it the fast (multi-process) way... 62 | π ≈ 3.1416233600000036 (0.00003070641021052367 away from actual!) 63 | took 1985 milliseconds 64 | ``` 65 | 66 | ## Durability 67 | 68 | An important feature of Worker Farm is **call durability**. If a child process dies for any reason during the execution of call(s), those calls will be re-queued and taken care of by other child processes. In this way, when you ask for something to be done, unless there is something *seriously* wrong with what you're doing, you should get a result on your callback function. 69 | 70 | ## My use-case 71 | 72 | There are other libraries for managing worker processes available but my use-case was fairly specific: I need to make heavy use of the [node-java](https://github.com/nearinfinity/node-java) library to interact with JVM code. Unfortunately, because the JVM garbage collector is so difficult to interact with, it's prone to killing your Node process when the GC kicks under heavy load. For safety I needed a durable way to make calls so that (1) it wouldn't kill my main process and (2) any calls that weren't successful would be resubmitted for processing. 73 | 74 | Worker Farm allows me to spin up multiple JVMs to be controlled by Node, and have a single, uncomplicated API that acts the same way as an in-process API and the calls will be taken care of by a child process even if an error kills a child process while it is working as the call will simply be passed to a new child process. 75 | 76 | **But**, don't think that Worker Farm is specific to that use-case, it's designed to be very generic and simple to adapt to anything requiring the use of child Node processes. 77 | 78 | ## API 79 | 80 | Worker Farm exports a main function and an `end()` method. The main function sets up a "farm" of coordinated child-process workers and it can be used to instantiate multiple farms, all operating independently. 81 | 82 | ### workerFarm([options, ]pathToModule[, exportedMethods]) 83 | 84 | In its most basic form, you call `workerFarm()` with the path to a module file to be invoked by the child process. You should use an **absolute path** to the module file, the best way to obtain the path is with `require.resolve('./path/to/module')`, this function can be used in exactly the same way as `require('./path/to/module')` but it returns an absolute path. 85 | 86 | #### `exportedMethods` 87 | 88 | If your module exports a single function on `module.exports` then you should omit the final parameter. However, if you are exporting multiple functions on `module.exports` then you should list them in an Array of Strings: 89 | 90 | ```js 91 | var workers = workerFarm(require.resolve('./mod'), [ 'doSomething', 'doSomethingElse' ]) 92 | workers.doSomething(function () {}) 93 | workers.doSomethingElse(function () {}) 94 | ``` 95 | 96 | Listing the available methods will instruct Worker Farm what API to provide you with on the returned object. If you don't list a `exportedMethods` Array then you'll get a single callable function to use; but if you list the available methods then you'll get an object with callable functions by those names. 97 | 98 | **It is assumed that each function you call on your child module will take a `callback` function as the last argument.** 99 | 100 | #### `options` 101 | 102 | If you don't provide an `options` object then the following defaults will be used: 103 | 104 | ```js 105 | { 106 | workerOptions : {} 107 | , maxCallsPerWorker : Infinity 108 | , maxConcurrentWorkers : require('os').cpus().length 109 | , maxConcurrentCallsPerWorker : 10 110 | , maxConcurrentCalls : Infinity 111 | , maxCallTime : Infinity 112 | , maxRetries : Infinity 113 | , autoStart : false 114 | , onChild : function() {} 115 | } 116 | ``` 117 | 118 | * **workerOptions** allows you to customize all the parameters passed to child nodes. This object supports [all possible options of `child_process.fork`](https://nodejs.org/api/child_process.html#child_process_child_process_fork_modulepath_args_options). The default options passed are the parent `execArgv`, `cwd` and `env`. Any (or all) of them can be overridden, and others can be added as well. 119 | 120 | * **maxCallsPerWorker** allows you to control the lifespan of your child processes. A positive number will indicate that you only want each child to accept that many calls before it is terminated. This may be useful if you need to control memory leaks or similar in child processes. 121 | 122 | * **maxConcurrentWorkers** will set the number of child processes to maintain concurrently. By default it is set to the number of CPUs available on the current system, but it can be any reasonable number, including `1`. 123 | 124 | * **maxConcurrentCallsPerWorker** allows you to control the *concurrency* of individual child processes. Calls are placed into a queue and farmed out to child processes according to the number of calls they are allowed to handle concurrently. It is arbitrarily set to 10 by default so that calls are shared relatively evenly across workers, however if your calls predictably take a similar amount of time then you could set it to `Infinity` and Worker Farm won't queue any calls but spread them evenly across child processes and let them go at it. If your calls aren't I/O bound then it won't matter what value you use here as the individual workers won't be able to execute more than a single call at a time. 125 | 126 | * **maxConcurrentCalls** allows you to control the maximum number of calls in the queue—either actively being processed or waiting for a worker to be processed. `Infinity` indicates no limit but if you have conditions that may endlessly queue jobs and you need to set a limit then provide a `>0` value and any calls that push the limit will return on their callback with a `MaxConcurrentCallsError` error (check `err.type == 'MaxConcurrentCallsError'`). 127 | 128 | * **maxCallTime** *(use with caution, understand what this does before you use it!)* when `!== Infinity`, will cap a time, in milliseconds, that *any single call* can take to execute in a worker. If this time limit is exceeded by just a single call then the worker running that call will be killed and any calls running on that worker will have their callbacks returned with a `TimeoutError` (check `err.type == 'TimeoutError'`). If you are running with `maxConcurrentCallsPerWorker` value greater than `1` then **all calls currently executing** will fail and will be automatically resubmitted unless you've changed the `maxRetries` option. Use this if you have jobs that may potentially end in infinite loops that you can't programatically end with your child code. Preferably run this with a `maxConcurrentCallsPerWorker` so you don't interrupt other calls when you have a timeout. This timeout operates on a per-call basis but will interrupt a whole worker. 129 | 130 | * **maxRetries** allows you to control the max number of call requeues after worker termination (unexpected or timeout). By default this option is set to `Infinity` which means that each call of each terminated worker will always be auto requeued. When the number of retries exceeds `maxRetries` value, the job callback will be executed with a `ProcessTerminatedError`. Note that if you are running with finite `maxCallTime` and `maxConcurrentCallsPerWorkers` greater than `1` then any `TimeoutError` will increase the retries counter *for each* concurrent call of the terminated worker. 131 | 132 | * **autoStart** when set to `true` will start the workers as early as possible. Use this when your workers have to do expensive initialization. That way they'll be ready when the first request comes through. 133 | 134 | * **onChild** when new child process starts this callback will be called with subprocess object as an argument. Use this when you need to add some custom communication with child processes. 135 | 136 | ### workerFarm.end(farm) 137 | 138 | Child processes stay alive waiting for jobs indefinitely and your farm manager will stay alive managing its workers, so if you need it to stop then you have to do so explicitly. If you send your farm API to `workerFarm.end()` then it'll cleanly end your worker processes. Note though that it's a *soft* ending so it'll wait for child processes to finish what they are working on before asking them to die. 139 | 140 | Any calls that are queued and not yet being handled by a child process will be discarded. `end()` only waits for those currently in progress. 141 | 142 | Once you end a farm, it won't handle any more calls, so don't even try! 143 | 144 | ## Related 145 | 146 | * [farm-cli](https://github.com/Kikobeats/farm-cli) – Launch a farm of workers from CLI. 147 | 148 | ## License 149 | 150 | Worker Farm is Copyright (c) Rod Vagg and licensed under the MIT license. All rights not explicitly granted in the MIT license are reserved. See the included LICENSE.md file for more details. 151 | -------------------------------------------------------------------------------- /lib/farm.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const DEFAULT_OPTIONS = { 4 | workerOptions : {} 5 | , maxCallsPerWorker : Infinity 6 | , maxConcurrentWorkers : (require('os').cpus() || { length: 1 }).length 7 | , maxConcurrentCallsPerWorker : 10 8 | , maxConcurrentCalls : Infinity 9 | , maxCallTime : Infinity // exceed this and the whole worker is terminated 10 | , maxRetries : Infinity 11 | , forcedKillTime : 100 12 | , autoStart : false 13 | , onChild : function() {} 14 | } 15 | 16 | const fork = require('./fork') 17 | , TimeoutError = require('errno').create('TimeoutError') 18 | , ProcessTerminatedError = require('errno').create('ProcessTerminatedError') 19 | , MaxConcurrentCallsError = require('errno').create('MaxConcurrentCallsError') 20 | 21 | 22 | function Farm (options, path) { 23 | this.options = Object.assign({}, DEFAULT_OPTIONS, options) 24 | this.path = path 25 | this.activeCalls = 0 26 | } 27 | 28 | 29 | // make a handle to pass back in the form of an external API 30 | Farm.prototype.mkhandle = function (method) { 31 | return function () { 32 | let args = Array.prototype.slice.call(arguments) 33 | if (this.activeCalls + this.callQueue.length >= this.options.maxConcurrentCalls) { 34 | let err = new MaxConcurrentCallsError('Too many concurrent calls (active: ' + this.activeCalls + ', queued: ' + this.callQueue.length + ')') 35 | if (typeof args[args.length - 1] == 'function') 36 | return process.nextTick(args[args.length - 1].bind(null, err)) 37 | throw err 38 | } 39 | this.addCall({ 40 | method : method 41 | , callback : args.pop() 42 | , args : args 43 | , retries : 0 44 | }) 45 | }.bind(this) 46 | } 47 | 48 | 49 | // a constructor of sorts 50 | Farm.prototype.setup = function (methods) { 51 | let iface 52 | if (!methods) { // single-function export 53 | iface = this.mkhandle() 54 | } else { // multiple functions on the export 55 | iface = {} 56 | methods.forEach(function (m) { 57 | iface[m] = this.mkhandle(m) 58 | }.bind(this)) 59 | } 60 | 61 | this.searchStart = -1 62 | this.childId = -1 63 | this.children = {} 64 | this.activeChildren = 0 65 | this.callQueue = [] 66 | 67 | if (this.options.autoStart) { 68 | while (this.activeChildren < this.options.maxConcurrentWorkers) 69 | this.startChild() 70 | } 71 | 72 | return iface 73 | } 74 | 75 | 76 | // when a child exits, check if there are any outstanding jobs and requeue them 77 | Farm.prototype.onExit = function (childId) { 78 | // delay this to give any sends a chance to finish 79 | setTimeout(function () { 80 | let doQueue = false 81 | if (this.children[childId] && this.children[childId].activeCalls) { 82 | this.children[childId].calls.forEach(function (call, i) { 83 | if (!call) return 84 | else if (call.retries >= this.options.maxRetries) { 85 | this.receive({ 86 | idx : i 87 | , child : childId 88 | , args : [ new ProcessTerminatedError('cancel after ' + call.retries + ' retries!') ] 89 | }) 90 | } else { 91 | call.retries++ 92 | this.callQueue.unshift(call) 93 | doQueue = true 94 | } 95 | }.bind(this)) 96 | } 97 | this.stopChild(childId) 98 | doQueue && this.processQueue() 99 | }.bind(this), 10) 100 | } 101 | 102 | 103 | // start a new worker 104 | Farm.prototype.startChild = function () { 105 | this.childId++ 106 | 107 | let forked = fork(this.path, this.options.workerOptions) 108 | , id = this.childId 109 | , c = { 110 | send : forked.send 111 | , child : forked.child 112 | , calls : [] 113 | , activeCalls : 0 114 | , exitCode : null 115 | } 116 | 117 | this.options.onChild(forked.child); 118 | 119 | forked.child.on('message', function(data) { 120 | if (data.owner !== 'farm') { 121 | return; 122 | } 123 | this.receive(data); 124 | }.bind(this)) 125 | forked.child.once('exit', function (code) { 126 | c.exitCode = code 127 | this.onExit(id) 128 | }.bind(this)) 129 | 130 | this.activeChildren++ 131 | this.children[id] = c 132 | } 133 | 134 | 135 | // stop a worker, identified by id 136 | Farm.prototype.stopChild = function (childId) { 137 | let child = this.children[childId] 138 | if (child) { 139 | child.send({owner: 'farm', event: 'die'}) 140 | setTimeout(function () { 141 | if (child.exitCode === null) 142 | child.child.kill('SIGKILL') 143 | }, this.options.forcedKillTime).unref() 144 | ;delete this.children[childId] 145 | this.activeChildren-- 146 | } 147 | } 148 | 149 | 150 | // called from a child process, the data contains information needed to 151 | // look up the child and the original call so we can invoke the callback 152 | Farm.prototype.receive = function (data) { 153 | let idx = data.idx 154 | , childId = data.child 155 | , args = data.args 156 | , child = this.children[childId] 157 | , call 158 | 159 | if (!child) { 160 | return console.error( 161 | 'Worker Farm: Received message for unknown child. ' 162 | + 'This is likely as a result of premature child death, ' 163 | + 'the operation will have been re-queued.' 164 | ) 165 | } 166 | 167 | call = child.calls[idx] 168 | if (!call) { 169 | return console.error( 170 | 'Worker Farm: Received message for unknown index for existing child. ' 171 | + 'This should not happen!' 172 | ) 173 | } 174 | 175 | if (this.options.maxCallTime !== Infinity) 176 | clearTimeout(call.timer) 177 | 178 | if (args[0] && args[0].$error == '$error') { 179 | let e = args[0] 180 | switch (e.type) { 181 | case 'TypeError': args[0] = new TypeError(e.message); break 182 | case 'RangeError': args[0] = new RangeError(e.message); break 183 | case 'EvalError': args[0] = new EvalError(e.message); break 184 | case 'ReferenceError': args[0] = new ReferenceError(e.message); break 185 | case 'SyntaxError': args[0] = new SyntaxError(e.message); break 186 | case 'URIError': args[0] = new URIError(e.message); break 187 | default: args[0] = new Error(e.message) 188 | } 189 | args[0].type = e.type 190 | args[0].stack = e.stack 191 | 192 | // Copy any custom properties to pass it on. 193 | Object.keys(e).forEach(function(key) { 194 | args[0][key] = e[key]; 195 | }); 196 | } 197 | 198 | process.nextTick(function () { 199 | call.callback.apply(null, args) 200 | }) 201 | 202 | ;delete child.calls[idx] 203 | child.activeCalls-- 204 | this.activeCalls-- 205 | 206 | if (child.calls.length >= this.options.maxCallsPerWorker 207 | && !Object.keys(child.calls).length) { 208 | // this child has finished its run, kill it 209 | this.stopChild(childId) 210 | } 211 | 212 | // allow any outstanding calls to be processed 213 | this.processQueue() 214 | } 215 | 216 | 217 | Farm.prototype.childTimeout = function (childId) { 218 | let child = this.children[childId] 219 | , i 220 | 221 | if (!child) 222 | return 223 | 224 | for (i in child.calls) { 225 | this.receive({ 226 | idx : i 227 | , child : childId 228 | , args : [ new TimeoutError('worker call timed out!') ] 229 | }) 230 | } 231 | this.stopChild(childId) 232 | } 233 | 234 | 235 | // send a call to a worker, identified by id 236 | Farm.prototype.send = function (childId, call) { 237 | let child = this.children[childId] 238 | , idx = child.calls.length 239 | 240 | child.calls.push(call) 241 | child.activeCalls++ 242 | this.activeCalls++ 243 | 244 | child.send({ 245 | owner : 'farm' 246 | , idx : idx 247 | , child : childId 248 | , method : call.method 249 | , args : call.args 250 | }) 251 | 252 | if (this.options.maxCallTime !== Infinity) { 253 | call.timer = 254 | setTimeout(this.childTimeout.bind(this, childId), this.options.maxCallTime) 255 | } 256 | } 257 | 258 | 259 | // a list of active worker ids, in order, but the starting offset is 260 | // shifted each time this method is called, so we work our way through 261 | // all workers when handing out jobs 262 | Farm.prototype.childKeys = function () { 263 | let cka = Object.keys(this.children) 264 | , cks 265 | 266 | if (this.searchStart >= cka.length - 1) 267 | this.searchStart = 0 268 | else 269 | this.searchStart++ 270 | 271 | cks = cka.splice(0, this.searchStart) 272 | 273 | return cka.concat(cks) 274 | } 275 | 276 | 277 | // Calls are added to a queue, this processes the queue and is called 278 | // whenever there might be a chance to send more calls to the workers. 279 | // The various options all impact on when we're able to send calls, 280 | // they may need to be kept in a queue until a worker is ready. 281 | Farm.prototype.processQueue = function () { 282 | let cka, i = 0, childId 283 | 284 | if (!this.callQueue.length) 285 | return this.ending && this.end() 286 | 287 | if (this.activeChildren < this.options.maxConcurrentWorkers) 288 | this.startChild() 289 | 290 | for (cka = this.childKeys(); i < cka.length; i++) { 291 | childId = +cka[i] 292 | if (this.children[childId].activeCalls < this.options.maxConcurrentCallsPerWorker 293 | && this.children[childId].calls.length < this.options.maxCallsPerWorker) { 294 | 295 | this.send(childId, this.callQueue.shift()) 296 | if (!this.callQueue.length) 297 | return this.ending && this.end() 298 | } /*else { 299 | console.log( 300 | , this.children[childId].activeCalls < this.options.maxConcurrentCallsPerWorker 301 | , this.children[childId].calls.length < this.options.maxCallsPerWorker 302 | , this.children[childId].calls.length , this.options.maxCallsPerWorker) 303 | }*/ 304 | } 305 | 306 | if (this.ending) 307 | this.end() 308 | } 309 | 310 | 311 | // add a new call to the call queue, then trigger a process of the queue 312 | Farm.prototype.addCall = function (call) { 313 | if (this.ending) 314 | return this.end() // don't add anything new to the queue 315 | this.callQueue.push(call) 316 | this.processQueue() 317 | } 318 | 319 | 320 | // kills child workers when they're all done 321 | Farm.prototype.end = function (callback) { 322 | let complete = true 323 | if (this.ending === false) 324 | return 325 | if (callback) 326 | this.ending = callback 327 | else if (this.ending == null) 328 | this.ending = true 329 | Object.keys(this.children).forEach(function (child) { 330 | if (!this.children[child]) 331 | return 332 | if (!this.children[child].activeCalls) 333 | this.stopChild(child) 334 | else 335 | complete = false 336 | }.bind(this)) 337 | 338 | if (complete && typeof this.ending == 'function') { 339 | process.nextTick(function () { 340 | this.ending() 341 | this.ending = false 342 | }.bind(this)) 343 | } 344 | } 345 | 346 | 347 | module.exports = Farm 348 | module.exports.TimeoutError = TimeoutError 349 | -------------------------------------------------------------------------------- /tests/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const tape = require('tape') 4 | , child_process = require('child_process') 5 | , workerFarm = require('../') 6 | , childPath = require.resolve('./child') 7 | , fs = require('fs') 8 | , os = require('os') 9 | 10 | function uniq (ar) { 11 | let a = [], i, j 12 | o: for (i = 0; i < ar.length; ++i) { 13 | for (j = 0; j < a.length; ++j) if (a[j] == ar[i]) continue o 14 | a[a.length] = ar[i] 15 | } 16 | return a 17 | } 18 | 19 | 20 | // a child where module.exports = function ... 21 | tape('simple, exports=function test', function (t) { 22 | t.plan(4) 23 | 24 | let child = workerFarm(childPath) 25 | child(0, function (err, pid, rnd) { 26 | t.ok(pid > process.pid, 'pid makes sense') 27 | t.ok(pid < process.pid + 750, 'pid makes sense') 28 | t.ok(rnd >= 0 && rnd < 1, 'rnd result makes sense') 29 | }) 30 | 31 | workerFarm.end(child, function () { 32 | t.ok(true, 'workerFarm ended') 33 | }) 34 | }) 35 | 36 | 37 | // a child where we have module.exports.fn = function ... 38 | tape('simple, exports.fn test', function (t) { 39 | t.plan(4) 40 | 41 | let child = workerFarm(childPath, [ 'run0' ]) 42 | child.run0(function (err, pid, rnd) { 43 | t.ok(pid > process.pid, 'pid makes sense') 44 | t.ok(pid < process.pid + 750, 'pid makes sense') 45 | t.ok(rnd >= 0 && rnd < 1, 'rnd result makes sense') 46 | }) 47 | 48 | workerFarm.end(child, function () { 49 | t.ok(true, 'workerFarm ended') 50 | }) 51 | }) 52 | 53 | 54 | tape('on child', function (t) { 55 | t.plan(2) 56 | 57 | let child = workerFarm({ onChild: function(subprocess) { childPid = subprocess.pid } }, childPath) 58 | , childPid = null; 59 | 60 | child(0, function(err, pid) { 61 | t.equal(childPid, pid) 62 | }) 63 | 64 | workerFarm.end(child, function () { 65 | t.ok(true, 'workerFarm ended') 66 | }) 67 | }) 68 | 69 | 70 | // use the returned pids to check that we're using a single child process 71 | // when maxConcurrentWorkers = 1 72 | tape('single worker', function (t) { 73 | t.plan(2) 74 | 75 | let child = workerFarm({ maxConcurrentWorkers: 1 }, childPath) 76 | , pids = [] 77 | , i = 10 78 | 79 | while (i--) { 80 | child(0, function (err, pid) { 81 | pids.push(pid) 82 | if (pids.length == 10) { 83 | t.equal(1, uniq(pids).length, 'only a single process (by pid)') 84 | } else if (pids.length > 10) 85 | t.fail('too many callbacks!') 86 | }) 87 | } 88 | 89 | workerFarm.end(child, function () { 90 | t.ok(true, 'workerFarm ended') 91 | }) 92 | }) 93 | 94 | 95 | // use the returned pids to check that we're using two child processes 96 | // when maxConcurrentWorkers = 2 97 | tape('two workers', function (t) { 98 | t.plan(2) 99 | 100 | let child = workerFarm({ maxConcurrentWorkers: 2 }, childPath) 101 | , pids = [] 102 | , i = 10 103 | 104 | while (i--) { 105 | child(0, function (err, pid) { 106 | pids.push(pid) 107 | if (pids.length == 10) { 108 | t.equal(2, uniq(pids).length, 'only two child processes (by pid)') 109 | } else if (pids.length > 10) 110 | t.fail('too many callbacks!') 111 | }) 112 | } 113 | 114 | workerFarm.end(child, function () { 115 | t.ok(true, 'workerFarm ended') 116 | }) 117 | }) 118 | 119 | 120 | // use the returned pids to check that we're using a child process per 121 | // call when maxConcurrentWorkers = 10 122 | tape('many workers', function (t) { 123 | t.plan(2) 124 | 125 | let child = workerFarm({ maxConcurrentWorkers: 10 }, childPath) 126 | , pids = [] 127 | , i = 10 128 | 129 | while (i--) { 130 | child(1, function (err, pid) { 131 | pids.push(pid) 132 | if (pids.length == 10) { 133 | t.equal(10, uniq(pids).length, 'pids are all the same (by pid)') 134 | } else if (pids.length > 10) 135 | t.fail('too many callbacks!') 136 | }) 137 | } 138 | 139 | workerFarm.end(child, function () { 140 | t.ok(true, 'workerFarm ended') 141 | }) 142 | }) 143 | 144 | 145 | tape('auto start workers', function (t) { 146 | let child = workerFarm({ maxConcurrentWorkers: 3, autoStart: true }, childPath, ['uptime']) 147 | , pids = [] 148 | , count = 5 149 | , i = count 150 | , delay = 250 151 | 152 | t.plan(count + 1) 153 | 154 | setTimeout(function() { 155 | while (i--) 156 | child.uptime(function (err, uptime) { 157 | t.ok(uptime > 10, 'child has been up before the request (' + uptime + 'ms)') 158 | }) 159 | 160 | workerFarm.end(child, function () { 161 | t.ok(true, 'workerFarm ended') 162 | }) 163 | }, delay) 164 | }) 165 | 166 | 167 | // use the returned pids to check that we're using a child process per 168 | // call when we set maxCallsPerWorker = 1 even when we have maxConcurrentWorkers = 1 169 | tape('single call per worker', function (t) { 170 | t.plan(2) 171 | 172 | let child = workerFarm({ 173 | maxConcurrentWorkers: 1 174 | , maxConcurrentCallsPerWorker: Infinity 175 | , maxCallsPerWorker: 1 176 | , autoStart: true 177 | }, childPath) 178 | , pids = [] 179 | , count = 25 180 | , i = count 181 | 182 | while (i--) { 183 | child(0, function (err, pid) { 184 | pids.push(pid) 185 | if (pids.length == count) { 186 | t.equal(count, uniq(pids).length, 'one process for each call (by pid)') 187 | workerFarm.end(child, function () { 188 | t.ok(true, 'workerFarm ended') 189 | }) 190 | } else if (pids.length > count) 191 | t.fail('too many callbacks!') 192 | }) 193 | } 194 | }) 195 | 196 | 197 | // use the returned pids to check that we're using a child process per 198 | // two-calls when we set maxCallsPerWorker = 2 even when we have maxConcurrentWorkers = 1 199 | tape('two calls per worker', function (t) { 200 | t.plan(2) 201 | 202 | let child = workerFarm({ 203 | maxConcurrentWorkers: 1 204 | , maxConcurrentCallsPerWorker: Infinity 205 | , maxCallsPerWorker: 2 206 | , autoStart: true 207 | }, childPath) 208 | , pids = [] 209 | , count = 20 210 | , i = count 211 | 212 | while (i--) { 213 | child(0, function (err, pid) { 214 | pids.push(pid) 215 | if (pids.length == count) { 216 | t.equal(count / 2, uniq(pids).length, 'one process for each call (by pid)') 217 | workerFarm.end(child, function () { 218 | t.ok(true, 'workerFarm ended') 219 | }) 220 | } else if (pids.length > count) 221 | t.fail('too many callbacks!') 222 | }) 223 | } 224 | }) 225 | 226 | 227 | // use timing to confirm that one worker will process calls sequentially 228 | tape('many concurrent calls', function (t) { 229 | t.plan(2) 230 | 231 | let child = workerFarm({ 232 | maxConcurrentWorkers: 1 233 | , maxConcurrentCallsPerWorker: Infinity 234 | , maxCallsPerWorker: Infinity 235 | , autoStart: true 236 | }, childPath) 237 | , defer = 200 238 | , count = 200 239 | , i = count 240 | , cbc = 0 241 | 242 | setTimeout(function () { 243 | let start = Date.now() 244 | 245 | while (i--) { 246 | child(defer, function () { 247 | if (++cbc == count) { 248 | let time = Date.now() - start 249 | // upper-limit not tied to `count` at all 250 | t.ok(time > defer && time < (defer * 2.5), 'processed tasks concurrently (' + time + 'ms)') 251 | workerFarm.end(child, function () { 252 | t.ok(true, 'workerFarm ended') 253 | }) 254 | } else if (cbc > count) 255 | t.fail('too many callbacks!') 256 | }) 257 | } 258 | }, 250) 259 | }) 260 | 261 | 262 | // use timing to confirm that one child processes calls sequentially with 263 | // maxConcurrentCallsPerWorker = 1 264 | tape('single concurrent call', function (t) { 265 | t.plan(2) 266 | 267 | let child = workerFarm({ 268 | maxConcurrentWorkers: 1 269 | , maxConcurrentCallsPerWorker: 1 270 | , maxCallsPerWorker: Infinity 271 | , autoStart: true 272 | }, childPath) 273 | , defer = 20 274 | , count = 100 275 | , i = count 276 | , cbc = 0 277 | 278 | setTimeout(function () { 279 | let start = Date.now() 280 | 281 | while (i--) { 282 | child(defer, function () { 283 | if (++cbc == count) { 284 | let time = Date.now() - start 285 | // upper-limit tied closely to `count`, 1.3 is generous but accounts for all the timers 286 | // coming back at the same time and the IPC overhead 287 | t.ok(time > (defer * count) && time < (defer * count * 1.3), 'processed tasks sequentially (' + time + ')') 288 | workerFarm.end(child, function () { 289 | t.ok(true, 'workerFarm ended') 290 | }) 291 | } else if (cbc > count) 292 | t.fail('too many callbacks!') 293 | }) 294 | } 295 | }, 250) 296 | }) 297 | 298 | 299 | // use timing to confirm that one child processes *only* 5 calls concurrently 300 | tape('multiple concurrent calls', function (t) { 301 | t.plan(2) 302 | 303 | let callsPerWorker = 5 304 | , child = workerFarm({ 305 | maxConcurrentWorkers: 1 306 | , maxConcurrentCallsPerWorker: callsPerWorker 307 | , maxCallsPerWorker: Infinity 308 | , autoStart: true 309 | }, childPath) 310 | , defer = 100 311 | , count = 100 312 | , i = count 313 | , cbc = 0 314 | 315 | setTimeout(function () { 316 | let start = Date.now() 317 | 318 | while (i--) { 319 | child(defer, function () { 320 | if (++cbc == count) { 321 | let time = Date.now() - start 322 | let min = defer * 1.5 323 | // (defer * (count / callsPerWorker + 2)) - if precise it'd be count/callsPerWorker 324 | // but accounting for IPC and other overhead, we need to give it a bit of extra time, 325 | // hence the +2 326 | let max = defer * (count / callsPerWorker + 2) 327 | t.ok(time > min && time < max, 'processed tasks concurrently (' + time + ' > ' + min + ' && ' + time + ' < ' + max + ')') 328 | workerFarm.end(child, function () { 329 | t.ok(true, 'workerFarm ended') 330 | }) 331 | } else if (cbc > count) 332 | t.fail('too many callbacks!') 333 | }) 334 | } 335 | }, 250) 336 | }) 337 | 338 | 339 | // call a method that will die with a probability of 0.5 but expect that 340 | // we'll get results for each of our calls anyway 341 | tape('durability', function (t) { 342 | t.plan(3) 343 | 344 | let child = workerFarm({ maxConcurrentWorkers: 2 }, childPath, [ 'killable' ]) 345 | , ids = [] 346 | , pids = [] 347 | , count = 20 348 | , i = count 349 | 350 | while (i--) { 351 | child.killable(i, function (err, id, pid) { 352 | ids.push(id) 353 | pids.push(pid) 354 | if (ids.length == count) { 355 | t.ok(uniq(pids).length > 2, 'processed by many (' + uniq(pids).length + ') workers, but got there in the end!') 356 | t.ok(uniq(ids).length == count, 'received a single result for each unique call') 357 | workerFarm.end(child, function () { 358 | t.ok(true, 'workerFarm ended') 359 | }) 360 | } else if (ids.length > count) 361 | t.fail('too many callbacks!') 362 | }) 363 | } 364 | }) 365 | 366 | 367 | // a callback provided to .end() can and will be called (uses "simple, exports=function test" to create a child) 368 | tape('simple, end callback', function (t) { 369 | t.plan(4) 370 | 371 | let child = workerFarm(childPath) 372 | child(0, function (err, pid, rnd) { 373 | t.ok(pid > process.pid, 'pid makes sense ' + pid + ' vs ' + process.pid) 374 | t.ok(pid < process.pid + 750, 'pid makes sense ' + pid + ' vs ' + process.pid) 375 | t.ok(rnd >= 0 && rnd < 1, 'rnd result makes sense') 376 | }) 377 | 378 | workerFarm.end(child, function() { 379 | t.pass('an .end() callback was successfully called') 380 | }) 381 | }) 382 | 383 | 384 | tape('call timeout test', function (t) { 385 | t.plan(3 + 3 + 4 + 4 + 4 + 3 + 1) 386 | 387 | let child = workerFarm({ maxCallTime: 250, maxConcurrentWorkers: 1 }, childPath) 388 | 389 | // should come back ok 390 | child(50, function (err, pid, rnd) { 391 | t.ok(pid > process.pid, 'pid makes sense ' + pid + ' vs ' + process.pid) 392 | t.ok(pid < process.pid + 750, 'pid makes sense ' + pid + ' vs ' + process.pid) 393 | t.ok(rnd > 0 && rnd < 1, 'rnd result makes sense ' + rnd) 394 | }) 395 | 396 | // should come back ok 397 | child(50, function (err, pid, rnd) { 398 | t.ok(pid > process.pid, 'pid makes sense ' + pid + ' vs ' + process.pid) 399 | t.ok(pid < process.pid + 750, 'pid makes sense ' + pid + ' vs ' + process.pid) 400 | t.ok(rnd > 0 && rnd < 1, 'rnd result makes sense ' + rnd) 401 | }) 402 | 403 | // should die 404 | child(500, function (err, pid, rnd) { 405 | t.ok(err, 'got an error') 406 | t.equal(err.type, 'TimeoutError', 'correct error type') 407 | t.ok(pid === undefined, 'no pid') 408 | t.ok(rnd === undefined, 'no rnd') 409 | }) 410 | 411 | // should die 412 | child(1000, function (err, pid, rnd) { 413 | t.ok(err, 'got an error') 414 | t.equal(err.type, 'TimeoutError', 'correct error type') 415 | t.ok(pid === undefined, 'no pid') 416 | t.ok(rnd === undefined, 'no rnd') 417 | }) 418 | 419 | // should die even though it is only a 100ms task, it'll get caught up 420 | // in a dying worker 421 | setTimeout(function () { 422 | child(100, function (err, pid, rnd) { 423 | t.ok(err, 'got an error') 424 | t.equal(err.type, 'TimeoutError', 'correct error type') 425 | t.ok(pid === undefined, 'no pid') 426 | t.ok(rnd === undefined, 'no rnd') 427 | }) 428 | }, 200) 429 | 430 | // should be ok, new worker 431 | setTimeout(function () { 432 | child(50, function (err, pid, rnd) { 433 | t.ok(pid > process.pid, 'pid makes sense ' + pid + ' vs ' + process.pid) 434 | t.ok(pid < process.pid + 750, 'pid makes sense ' + pid + ' vs ' + process.pid) 435 | t.ok(rnd > 0 && rnd < 1, 'rnd result makes sense ' + rnd) 436 | }) 437 | workerFarm.end(child, function () { 438 | t.ok(true, 'workerFarm ended') 439 | }) 440 | }, 400) 441 | }) 442 | 443 | 444 | tape('test error passing', function (t) { 445 | t.plan(10) 446 | 447 | let child = workerFarm(childPath, [ 'err' ]) 448 | child.err('Error', 'this is an Error', function (err) { 449 | t.ok(err instanceof Error, 'is an Error object') 450 | t.equal('Error', err.type, 'correct type') 451 | t.equal('this is an Error', err.message, 'correct message') 452 | }) 453 | child.err('TypeError', 'this is a TypeError', function (err) { 454 | t.ok(err instanceof Error, 'is a TypeError object') 455 | t.equal('TypeError', err.type, 'correct type') 456 | t.equal('this is a TypeError', err.message, 'correct message') 457 | }) 458 | child.err('Error', 'this is an Error with custom props', {foo: 'bar', 'baz': 1}, function (err) { 459 | t.ok(err instanceof Error, 'is an Error object') 460 | t.equal(err.foo, 'bar', 'passes data') 461 | t.equal(err.baz, 1, 'passes data') 462 | }) 463 | 464 | workerFarm.end(child, function () { 465 | t.ok(true, 'workerFarm ended') 466 | }) 467 | }) 468 | 469 | 470 | tape('test maxConcurrentCalls', function (t) { 471 | t.plan(10) 472 | 473 | let child = workerFarm({ maxConcurrentCalls: 5 }, childPath) 474 | 475 | child(50, function (err) { t.notOk(err, 'no error') }) 476 | child(50, function (err) { t.notOk(err, 'no error') }) 477 | child(50, function (err) { t.notOk(err, 'no error') }) 478 | child(50, function (err) { t.notOk(err, 'no error') }) 479 | child(50, function (err) { t.notOk(err, 'no error') }) 480 | child(50, function (err) { 481 | t.ok(err) 482 | t.equal(err.type, 'MaxConcurrentCallsError', 'correct error type') 483 | }) 484 | child(50, function (err) { 485 | t.ok(err) 486 | t.equal(err.type, 'MaxConcurrentCallsError', 'correct error type') 487 | }) 488 | 489 | workerFarm.end(child, function () { 490 | t.ok(true, 'workerFarm ended') 491 | }) 492 | }) 493 | 494 | 495 | tape('test maxConcurrentCalls + queue', function (t) { 496 | t.plan(13) 497 | 498 | let child = workerFarm({ maxConcurrentCalls: 4, maxConcurrentWorkers: 2, maxConcurrentCallsPerWorker: 1 }, childPath) 499 | 500 | child(20, function (err) { console.log('ended short1'); t.notOk(err, 'no error, short call 1') }) 501 | child(20, function (err) { console.log('ended short2'); t.notOk(err, 'no error, short call 2') }) 502 | child(300, function (err) { t.notOk(err, 'no error, long call 1') }) 503 | child(300, function (err) { t.notOk(err, 'no error, long call 2') }) 504 | child(20, function (err) { 505 | t.ok(err, 'short call 3 should error') 506 | t.equal(err.type, 'MaxConcurrentCallsError', 'correct error type') 507 | }) 508 | child(20, function (err) { 509 | t.ok(err, 'short call 4 should error') 510 | t.equal(err.type, 'MaxConcurrentCallsError', 'correct error type') 511 | }) 512 | 513 | // cross fingers and hope the two short jobs have ended 514 | setTimeout(function () { 515 | child(20, function (err) { t.notOk(err, 'no error, delayed short call 1') }) 516 | child(20, function (err) { t.notOk(err, 'no error, delayed short call 2') }) 517 | child(20, function (err) { 518 | t.ok(err, 'delayed short call 3 should error') 519 | t.equal(err.type, 'MaxConcurrentCallsError', 'correct error type') 520 | }) 521 | 522 | workerFarm.end(child, function () { 523 | t.ok(true, 'workerFarm ended') 524 | }) 525 | }, 250) 526 | }) 527 | 528 | 529 | // this test should not keep the process running! if the test process 530 | // doesn't die then the problem is here 531 | tape('test timeout kill', function (t) { 532 | t.plan(3) 533 | 534 | let child = workerFarm({ maxCallTime: 250, maxConcurrentWorkers: 1 }, childPath, [ 'block' ]) 535 | child.block(function (err) { 536 | t.ok(err, 'got an error') 537 | t.equal(err.type, 'TimeoutError', 'correct error type') 538 | }) 539 | 540 | workerFarm.end(child, function () { 541 | t.ok(true, 'workerFarm ended') 542 | }) 543 | }) 544 | 545 | 546 | tape('test max retries after process terminate', function (t) { 547 | t.plan(7) 548 | 549 | // temporary file is used to store the number of retries among terminating workers 550 | let filepath1 = '.retries1' 551 | let child1 = workerFarm({ maxConcurrentWorkers: 1, maxRetries: 5}, childPath, [ 'stubborn' ]) 552 | child1.stubborn(filepath1, function (err, result) { 553 | t.notOk(err, 'no error') 554 | t.equal(result, 12, 'correct result') 555 | }) 556 | 557 | workerFarm.end(child1, function () { 558 | fs.unlinkSync(filepath1) 559 | t.ok(true, 'workerFarm ended') 560 | }) 561 | 562 | let filepath2 = '.retries2' 563 | let child2 = workerFarm({ maxConcurrentWorkers: 1, maxRetries: 3}, childPath, [ 'stubborn' ]) 564 | child2.stubborn(filepath2, function (err, result) { 565 | t.ok(err, 'got an error') 566 | t.equal(err.type, 'ProcessTerminatedError', 'correct error type') 567 | t.equal(err.message, 'cancel after 3 retries!', 'correct message and number of retries') 568 | }) 569 | 570 | workerFarm.end(child2, function () { 571 | fs.unlinkSync(filepath2) 572 | t.ok(true, 'workerFarm ended') 573 | }) 574 | }) 575 | 576 | 577 | tape('custom arguments can be passed to "fork"', function (t) { 578 | t.plan(3) 579 | 580 | // allocate a real, valid path, in any OS 581 | let cwd = fs.realpathSync(os.tmpdir()) 582 | , workerOptions = { 583 | cwd : cwd 584 | , execArgv : ['--expose-gc'] 585 | } 586 | , child = workerFarm({ maxConcurrentWorkers: 1, maxRetries: 5, workerOptions: workerOptions}, childPath, ['args']) 587 | 588 | child.args(function (err, result) { 589 | t.equal(result.execArgv[0], '--expose-gc', 'flags passed (overridden default)') 590 | t.equal(result.cwd, cwd, 'correct cwd folder') 591 | }) 592 | 593 | workerFarm.end(child, function () { 594 | t.ok(true, 'workerFarm ended') 595 | }) 596 | }) 597 | 598 | 599 | tape('ensure --debug/--inspect not propagated to children', function (t) { 600 | t.plan(3) 601 | 602 | let script = __dirname + '/debug.js' 603 | , debugArg = process.version.replace(/^v(\d+)\..*$/, '$1') >= 8 ? '--inspect' : '--debug=8881' 604 | , child = child_process.spawn(process.execPath, [ debugArg, script ]) 605 | , stdout = '' 606 | 607 | child.stdout.on('data', function (data) { 608 | stdout += data.toString() 609 | }) 610 | 611 | child.on('close', function (code) { 612 | t.equal(code, 0, 'exited without error (' + code + ')') 613 | t.ok(stdout.indexOf('FINISHED') > -1, 'process finished') 614 | t.ok(stdout.indexOf('--debug') === -1, 'child does not receive debug flag') 615 | }) 616 | }) 617 | --------------------------------------------------------------------------------