├── .eslintrc ├── .babelrc ├── .jscsrc ├── .editorconfig ├── .jshintrc ├── .gitignore ├── package.json ├── webpack.config.js ├── README.md ├── src ├── ProcCluster.js ├── NetCluster.js └── index.js ├── test └── test.js └── dist └── rxjs-cluster.js /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "babel-eslint", 3 | "rules": { 4 | "strict": 0 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | "es2015" 4 | ], 5 | "plugins": [ 6 | "transform-function-bind" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /.jscsrc: -------------------------------------------------------------------------------- 1 | { 2 | "esnext": true, 3 | "preset": "airbnb", 4 | "validateIndentation": 2, 5 | "excludeFiles": ["node_modules/**"] 6 | } 7 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | 11 | [*.md] 12 | trim_trailing_whitespace = false 13 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "node": true, 3 | "esnext": true, 4 | "bitwise": true, 5 | "camelcase": true, 6 | "curly": true, 7 | "eqeqeq": true, 8 | "immed": true, 9 | "indent": 2, 10 | "latedef": true, 11 | "newcap": true, 12 | "noarg": true, 13 | "quotmark": "single", 14 | "undef": true, 15 | "unused": true, 16 | "strict": true 17 | } 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | *.map 10 | *.tmp 11 | 12 | # Directory for instrumented libs generated by jscoverage/JSCover 13 | lib-cov 14 | 15 | # Coverage directory used by tools like istanbul 16 | coverage 17 | 18 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 19 | .grunt 20 | 21 | # node-waf configuration 22 | .lock-wscript 23 | 24 | # Compiled binary addons (http://nodejs.org/api/addons.html) 25 | build/Release 26 | 27 | # Dependency directory 28 | # https://docs.npmjs.com/misc/faq#should-i-check-my-node-modules-folder-into-git 29 | node_modules 30 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rxjs-cluster", 3 | "version": "0.3.15", 4 | "description": "Use Node.js cluster support to parallize Rx", 5 | "scripts": { 6 | "build": "webpack --config webpack.config.js --progress --colors --watch", 7 | "test": "mocha --compilers js:babel-core/register test/test.js", 8 | "test:host": "env test=net port=8090 client=8091 mocha --compilers js:babel-core/register test/test.js", 9 | "test:client": "env test=net port=8091 client=8091 mocha --compilers js:babel-core/register test/test.js", 10 | "test:elect": "http localhost:8090/be/master & http localhost:8091/be/slave" 11 | }, 12 | "main": "dist/rxjs-cluster.js", 13 | "author": "Jonathan Dunlap ", 14 | "repository": "jadbox/rxjs-cluster", 15 | "license": "ISC", 16 | "dependencies": { 17 | "body-parser": "^1.15.0", 18 | "connect-timeout": "^1.7.0", 19 | "express": "^4.13.4", 20 | "lodash": "^4.9.0", 21 | "request-json": "^0.5.5", 22 | "rx": "^4.0.8", 23 | "string-hash": "^1.1.0", 24 | "webpack": "^1.12.13" 25 | }, 26 | "devDependencies": { 27 | "babel-core": "^6.5.2", 28 | "babel-loader": "^6.2.2", 29 | "babel-plugin-transform-function-bind": "^6.5.2", 30 | "babel-preset-es2015": "^6.5.0", 31 | "babelify": "^7.2.0", 32 | "mocha": "^2.4.5", 33 | "path": "^0.12.7" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | var webpack = require('webpack'); 2 | var path = require('path'); 3 | var fs = require('fs'); 4 | 5 | var nodeModules = { classnames: 'commonjs classnames', react: 'commonjs react' }; 6 | fs.readdirSync('node_modules') 7 | .filter(function(x) { 8 | return ['.bin'].indexOf(x) === -1; 9 | }) 10 | .forEach(function(mod) { 11 | nodeModules[mod] = 'commonjs ' + mod; 12 | }); 13 | 14 | module.exports = { 15 | context: __dirname, 16 | entry: { 17 | javascript: path.join(__dirname, 'src', 'index.js') 18 | }, 19 | target: 'node', 20 | resolveLoader: { 21 | modulesDirectories: [ 22 | path.join(__dirname, "node_modules") 23 | ] 24 | }, 25 | module: { 26 | loaders: [{ 27 | test: /\.js$/, 28 | exclude: /(node_modules|test|views)/, 29 | loader: 'babel', 30 | query: { 31 | cacheDirectory: '/tmp', 32 | presets: ['es2015'], 33 | plugins: ["transform-function-bind"] 34 | } 35 | }] 36 | }, 37 | /*resolve: { 38 | extensions: ['', '*.js'] 39 | },*/ 40 | output: { 41 | filename: "rxjs-cluster.js", 42 | libraryTarget: "commonjs", 43 | library: "", 44 | path: path.resolve(__dirname, 'dist') 45 | }, 46 | plugins: [ 47 | ], 48 | externals: nodeModules, 49 | devtool: 'source-map', 50 | debug: true 51 | } 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rxjs-cluster 2 | 3 | _[Moore's law ceiling](http://www.agner.org/optimize/blog/read.php?i=417): "The biggest potential for improved performance is now, as I see it, on the software side [...] with parallelism which has so far not been [broadly] implemented."_ 4 | 5 | _(WIP: Working Beta)_ 6 | 7 | Using Rx, maximize CPU usage in Node by using the new clusterMap that uses cluster/forked processes 8 | 9 | ``` 10 | var Rx = require('rx'); 11 | var Cluster = require('rxjs-cluster'); // import 12 | var options = {}; 13 | var rc = new Cluster( options ); // instance 14 | 15 | var Observable = Rx.Observable; 16 | 17 | // Child function that returns raw value 18 | function childTest(x) { 19 | return "hello " + x + " from " + process.pid; 20 | } 21 | 22 | // Child function that returns an Observable 23 | function childTest$(x) { 24 | return Rx.Observable.range(0,3).map("hello " + x).toArray(); 25 | } 26 | 27 | function master() { 28 | Observable.from(['Jonathan', 'James', 'Edwin']) 29 | .clusterMap('childTest') 30 | .subscribe( 31 | function(x) { console.log(x); }, 32 | function(x) { console.log('Err ' + x); }, 33 | function() { console.log('Completed'); } 34 | ); 35 | 36 | Observable.from(['Jonathan', 'James', 'Edwin']) 37 | .clusterMap('childTest$') 38 | .subscribe( 39 | function(x) { console.log(x); }, 40 | function(x) { console.log('Err ' + x); }, 41 | function() { 42 | console.log('Completed'); 43 | rc.killall(); // kill all workers, clusterMap will no longer work 44 | } 45 | ); 46 | } 47 | 48 | // Define number of workers, master entry point, worker functions 49 | rc.entry(master, { 'childTest': childTest, 50 | 'childTest$': childTest$ }); 51 | 52 | // Or define leave the default number of workers to # of cpu cores 53 | // rc.entry(master, { 'childTest': childTest, 'childTest$': childTest$ }); 54 | 55 | ``` 56 | -------------------------------------------------------------------------------- /src/ProcCluster.js: -------------------------------------------------------------------------------- 1 | import cluster from 'cluster'; 2 | import _ from 'lodash'; 3 | 4 | export default function ProcCluster(options) { 5 | console.log('cluster: using ProcCluster'); 6 | if(!options.workers) options.workers = require('os').cpus().length; 7 | 8 | this.options = Object.assign({ 9 | 10 | }, options); 11 | 12 | 13 | this.clusterMapObs = _clusterMapObs.bind(this); 14 | this.setupChild = _setupChild.bind(this); 15 | this.startWorkers = _startWorkers.bind(this); 16 | this.killall = _killall.bind(this); 17 | this.isMasterCheck = (self, cb) => { 18 | console.log('cluster.isMaster', cluster.isMaster); 19 | cb(cluster.isMaster); 20 | } 21 | } 22 | 23 | function _killall(self) { 24 | _.forEach(self.workers, x => x.kill()); 25 | } 26 | 27 | function _setupChild(self, work) { 28 | work.concatMap(self.childWork, (y, x) => ({ 29 | data: x, 30 | id: y.id 31 | })) 32 | .subscribe( 33 | ({ 34 | data, id 35 | }) => process.send({ 36 | rdata: data, 37 | id 38 | }), (x) => console.log('Child ' + process.pid + ' err', x) 39 | ) 40 | 41 | process.on('message', function onChildMessage(x) { 42 | work.onNext(x); 43 | }); // push work unto task stream 44 | } 45 | 46 | function _startWorkers(self, workers, onReady) { 47 | const numWorkers = this.options.workers; 48 | // cluster manager 49 | var n = 0; 50 | //const workers = self.workers; 51 | 52 | if(cluster.setupMaster) cluster.setupMaster({ 53 | silent:false 54 | }); 55 | 56 | cluster.on('listening', (worker, address) => { 57 | console.log(`A worker is now connected to ${address.address}:${address.port}`); 58 | }); 59 | 60 | cluster.on('online', function(worker) { 61 | if (n === numWorkers) { 62 | console.log('cluster: All workers online'); 63 | onReady(); 64 | return; 65 | } 66 | console.log('cluster: Worker ' + worker.process.pid + ' is online'); 67 | if (worker.setMaxListeners) worker.setMaxListeners(0); 68 | workers.push(worker); 69 | n++; 70 | //worker.on('message', x => console.log('worker: ', x)); 71 | }); 72 | 73 | cluster.on('error', function(x) { 74 | throw new Error(x) 75 | }); 76 | 77 | /*cluster.on('disconnect', function(x) { 78 | console.log('disconnect'); 79 | throw new Error(x) 80 | }); 81 | 82 | cluster.on('exit', function(x) { 83 | console.log('exit'); 84 | throw new Error(x) 85 | });*/ 86 | 87 | for (var i = 0; i <= numWorkers; i++) { 88 | const f = cluster.fork(); 89 | //console.log('f', f.process.pid) 90 | if(f.process.stdout) f.process.stdout.on('data', function(data) { 91 | // output from the child process 92 | console.log('>>> '+ data); 93 | }); 94 | } 95 | } 96 | 97 | function _clusterMapObs(self, obs, data, funcName, jobIndex, worker) { 98 | worker.on('message', function handler({ 99 | rdata, id 100 | }) { 101 | if (id !== jobIndex) return; // ignore 102 | obs.onNext(rdata); 103 | obs.onCompleted(); 104 | worker.removeListener('message', handler); 105 | }); 106 | worker.send({ 107 | data, id: jobIndex, 108 | func: funcName 109 | }); 110 | } 111 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | import Rx from 'rx'; 2 | //import RC from '../src'; 3 | import assert from 'assert'; 4 | import cluster from 'cluster'; 5 | 6 | import RC, {NetSystem} from '../src'; 7 | const params = {}; 8 | 9 | if(process.env.test==='net') { 10 | params.system = new NetSystem( { 11 | port: process.env.port, 12 | clients: ['http://localhost:'+process.env.client + '/'] 13 | }) 14 | } 15 | const pc = new RC( params ); 16 | const clusterMap = pc.clusterMap; 17 | 18 | var Observable = Rx.Observable; 19 | 20 | // Child function that returns raw value 21 | function childTest(x) { 22 | return "hello " + x + " from " + process.pid 23 | } 24 | 25 | // Child function that returns an Observable 26 | function childTest$(x) { 27 | return Rx.Observable.range(0,3).map("hello " + x + " from " + process.pid).toArray(); 28 | } 29 | 30 | function master() { 31 | console.log('master'); 32 | return Observable.from(['Jonathan', 'James', 'Edwin']) 33 | ::clusterMap('childTest') 34 | .do( x => console.log('1', x)) 35 | /*.subscribe( 36 | function(x) { console.log(x); }, 37 | function(x) { console.log('Err ' + x); }, 38 | function() { } 39 | );*/ 40 | 41 | 42 | } 43 | 44 | function master2() { 45 | console.log('master2'); 46 | return Observable.from(['Jonathan', 'James', 'Edwin']) 47 | ::clusterMap('childTest$') 48 | .do( x => console.log('2', x)) 49 | /*.subscribe( 50 | function(x) { console.log(x); }, 51 | function(x) { console.log('Err ' + x); }, 52 | function() { 53 | //master3(); 54 | } 55 | );*/ 56 | } 57 | 58 | function master3() { 59 | console.log('master3'); 60 | return Observable.from(['Jonathan', 'James', 'Edwin']) 61 | ::clusterMap('childTest', 199) // use node index of hash id 199 62 | .do( x => console.log('3', x)) 63 | /*.subscribe( 64 | function(x) { console.log(x); }, 65 | function(x) { console.log('Err ' + x); }, 66 | function() { 67 | console.log('Completed 4'); 68 | //master4(); 69 | } 70 | );*/ 71 | 72 | 73 | } 74 | 75 | function master4() { 76 | console.log('master4'); 77 | return Observable.from(['Jonathan', 'James', 'Edwin', 'Edwin', 'Edwin', 'Flipper']) 78 | ::clusterMap('childTest$', x=>x) // use the name as the node index hash 79 | .do( x => console.log('4', x)) 80 | /*.subscribe( 81 | function(x) { console.log(x); }, 82 | function(x) { console.log('Err ' + x); }, 83 | function() { 84 | console.log('Completed'); 85 | //rc.killall(); // kill all workers, clusterMap will no longer work 86 | //_done(); 87 | //done(); 88 | } 89 | );*/ 90 | } 91 | 92 | function start() { 93 | _done(); 94 | console.log('--', cluster.isMaster) 95 | } 96 | 97 | if(cluster.isMaster) { 98 | describe('## rx master', function() { 99 | this.timeout(99000); 100 | before(function(done) { 101 | // setTimeout(done, 1000); 102 | _done = done; 103 | }); 104 | 105 | it('master entry', function (done) { 106 | console.log('STARTING') 107 | master().concatMap(master2).toArray() 108 | .concatMap(master3).toArray() 109 | .concatMap(master4).toArray() 110 | .subscribe( 111 | x=>x 112 | ,x=> { throw new Error(x) } 113 | ,x => { 114 | console.log('completed') 115 | pc.killall(); // kill all workers, clusterMap will no longer work 116 | done(); 117 | }) 118 | //_done = done; 119 | }) 120 | }) 121 | } 122 | else { 123 | describe('## rx client', function() { 124 | this.timeout(99000); 125 | before(function(done) { 126 | // setTimeout(done, 1000); 127 | 128 | }); 129 | it('client entry', function (done) { 130 | }); 131 | }); 132 | } 133 | 134 | pc.entry(start, { childTest: childTest, childTest$: childTest$ }); 135 | 136 | // Define number of workers, master entry point, worker functions 137 | var _done; 138 | 139 | // Or define leave the default number of workers to # of cpu cores 140 | // rc.entry(master, { childTest: childTest, childTest$: childTest$ }); 141 | -------------------------------------------------------------------------------- /src/NetCluster.js: -------------------------------------------------------------------------------- 1 | import cluster from 'cluster'; 2 | import _ from 'lodash'; 3 | import express from 'express'; 4 | import bodyParser from 'body-parser'; 5 | import request from 'request-json'; 6 | import timeout from 'connect-timeout'; 7 | 8 | export default function ProcCluster(options) { 9 | console.log('--WIP--', options); 10 | 11 | this.options = Object.assign({ 12 | clients: ['http://localhost:8090/'], 13 | port: 8090 14 | }, options || {}); 15 | if(!Array.isArray(this.options.clients)) this.options.clients = [ this.options.clients ]; 16 | this.options.port = parseInt(this.options.port); 17 | 18 | this.clusterMapObs = _clusterMapObs.bind(this); 19 | this.setupChild = _setupChild.bind(this); 20 | this.startWorkers = _startWorkers.bind(this); 21 | this.killall = _killall.bind(this); 22 | this.isMasterCheck = _isMasterCheck.bind(this); 23 | 24 | const app = this.app = express(); 25 | app.use(timeout('600s')); 26 | app.use(bodyParser.json()) 27 | this.appServer = app.listen(this.options.port); 28 | } 29 | 30 | function _isMasterCheck(self, cb) { 31 | const options = self._options; 32 | console.log('cluster: listening port:', this.options.port); 33 | let picked = false; 34 | 35 | this.app.get('/be/master/', function(req, res) { 36 | if(picked) { 37 | console.log('cluster: already picked as master'); 38 | return; 39 | } 40 | else picked = true; 41 | options.isMaster = true; 42 | options.isSlave = false; 43 | 44 | if(res && res.body && Array.isArray(res.body.clients)) { 45 | console.log('Using clients from master election'); 46 | this.options.clients = res.body.clients; 47 | } 48 | 49 | res.send('master elected'); 50 | cb(true); 51 | }); 52 | 53 | this.app.get('/be/slave/', function(req, res) { 54 | if(picked) { 55 | console.log('cluster: already picked as master'); 56 | return; 57 | } 58 | else picked = true; 59 | options.isMaster = false; 60 | options.isSlave = true; 61 | 62 | res.send('slave elected'); 63 | cb(false); 64 | }); 65 | } 66 | 67 | function _killall(self) { 68 | //_.forEach(self.workers, x => x.kill()); 69 | } 70 | 71 | function _setupChild(self, work) { 72 | const requests = {}; 73 | work.concatMap(self.childWork, (y, x) => ({ 74 | data: x, 75 | id: y.id 76 | })) 77 | .subscribe( 78 | ({ 79 | data, id 80 | }) => { 81 | if(requests[id] === undefined) throw new Error('request id not issued '+id); 82 | console.log('cluster: client: responding'); 83 | requests[id].send({data, id}); 84 | }, (x) => console.log('Net Child ' + process.pid + ' err', x) 85 | ) 86 | 87 | console.log('cluster: listening for /work/:', this.options.port); 88 | this.app.get('/ping/', function(req, res) { 89 | res.send('ping pong: client'); 90 | }); 91 | this.app.post('/work/', function(req, res) { 92 | if(!req.body || !req.body.func) { 93 | res.send('ping work'); 94 | return; 95 | } 96 | const {func, data, id} = req.body; 97 | const workParams = req.body; 98 | console.log('cluster: work recieved', workParams); 99 | requests[id] = res; 100 | work.onNext(workParams); 101 | //res.send('slave elacted'); // TODO 102 | }); 103 | } 104 | 105 | function _startWorkers(self, workers, onReady) { 106 | _.forEach(this.options.clients, c => { 107 | const worker = { url: c }; 108 | worker.client = request.createClient( worker.url ); 109 | workers.push( worker ); 110 | }); 111 | 112 | this.app.get('/ping/', function(req, res) { 113 | res.send('ping pong: server'); 114 | }); 115 | 116 | setTimeout(onReady, 3000); 117 | } 118 | 119 | function _clusterMapObs(self, obs, data, func, id, worker) { 120 | console.log('cluster: master sending post:'+worker.url+' rte:work func:' + func + ' id:' + id); 121 | worker.client.post('work', {func, data, id}, (err, res, body) => { 122 | if(res && parseInt(res.statusCode)!==200) { 123 | console.log(res.statusCode+' response from client.') 124 | return obs.onError(res.statusCode+' response from client.'); 125 | } 126 | if(err) { 127 | console.log('err', err); 128 | return obs.onError(err); 129 | } 130 | //if(self._options) 131 | console.log('cluster: master recieved:', err, res ? res.statusCode : res, func, id); 132 | obs.onNext(body.data); 133 | obs.onCompleted(); 134 | }) 135 | } 136 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import Rx from 'rx'; 2 | import hash from 'string-hash'; 3 | import _ from 'lodash'; 4 | import ProcCluster from './ProcCluster' 5 | import NetCluster from './NetCluster' 6 | 7 | const Observable = Rx.Observable; 8 | const observableProto = Observable.prototype; 9 | 10 | export var ProcSystem = ProcCluster; 11 | export var NetSystem = NetCluster; 12 | export default function Cluster(options) { 13 | this.workers = []; 14 | this.childEntries = {}; 15 | this._options = options = Object.assign({ 16 | debug: false 17 | }, options || {}); 18 | 19 | if(!options.system) options.system = new ProcCluster( { workers:options.workers } ); 20 | 21 | const sys = this.sys = options.system; 22 | 23 | this.n = 0; // round-robin scheduling 24 | this.work = new Rx.Subject(); // Children work 25 | 26 | const that = this; 27 | this.startWorkers = sys.startWorkers; 28 | this.clusterMap = function(x, y, z) { 29 | const ___clusterMap = _clusterMap.bind(this); 30 | return ___clusterMap(that, x, y, z) 31 | }; 32 | 33 | this.setupChild = sys.setupChild; 34 | this.childWork = _childWork.bind(this); 35 | this.entry = _entry.bind(this); 36 | this.getWorkers = _getWorkers.bind(this); 37 | this.killall = x => sys.killall(this); 38 | } 39 | 40 | function _childWork({ 41 | data, id, func 42 | }) { 43 | const funcRef = this.childEntries[func]; 44 | 45 | if (!funcRef) { 46 | console.log('Function not found in childMethod lookup:', func) 47 | throw new Error('Function not found in childMethod lookup: '+ func); 48 | return; 49 | } 50 | 51 | const exec = funcRef(data); 52 | 53 | if (!exec.subscribe) { 54 | return Rx.Observable.just(exec); 55 | } else return exec.first(); 56 | } 57 | 58 | /* 59 | @param numWorkers number of cpus 60 | @param entryFun the master entry function 61 | @param options options object 62 | */ 63 | function _entry(entryFun, childMethods) { 64 | console.log('cluster: slave/master check'); 65 | const options = this._options; 66 | 67 | const childEntries = this.childEntries; 68 | _.forEach(childMethods, (v, k) => { 69 | if (v && (v.subscribe || typeof v === 'function')) childEntries[k] = v; 70 | }); 71 | 72 | const isMasterEnv = process.env.isMaster === 'true' || this._options.isMaster === true; 73 | const isSlaveEnv = process.env.isSlave === 'true' || this._options.isSlave === true; 74 | const isMasterCheck = isMasterEnv ? (y,x) => setTimeout(x, 6000, true) : isSlaveEnv ? (y,x) => x(false) : this.sys.isMasterCheck; 75 | 76 | //const isMaster = this._options.isMaster || this.sys.isMaster; 77 | isMasterCheck(this, isMaster => { 78 | 79 | // Child entry point 80 | if (!isMaster) { 81 | console.log('cluster: slave elected'); 82 | this.setupChild(this, this.work); 83 | return; 84 | } else { 85 | // Master entry point 86 | console.log('cluster: master elected'); 87 | this.startWorkers(this, this.workers, entryFun); 88 | } 89 | }); 90 | } 91 | 92 | function _getWorkers() { 93 | return this.workers; 94 | } 95 | 96 | /* 97 | @param funcName function to invoke 98 | @param nodeSelector (optional) (function | string | int) used to pick node. If function, the value is the stream object and the return is (string | int). 99 | */ 100 | function _clusterMap(that, funcName, nodeSelector) { 101 | let key = null; 102 | if(nodeSelector !== undefined && nodeSelector !== null && typeof nodeSelector !== 'function') { 103 | key = Number.isInteger(nodeSelector) ? nodeSelector : hash(nodeSelector.toString()); 104 | nodeSelector = null; 105 | } 106 | const workers = that.workers; 107 | //const that = this; 108 | return this.flatMap(data => Rx.Observable.create(obs => { 109 | if (nodeSelector) { 110 | const nodeKey = nodeSelector(data); 111 | key = Number.isInteger(nodeKey) ? nodeKey : hash(nodeKey.toString()); 112 | } 113 | 114 | const workerIndex = key ? (key % workers.length) : (that.n++ % workers.length); 115 | //console.log(workerIndex, n, workers.length); 116 | //n++; 117 | //if( n === Number.MAX_SAFE_INTEGER) x = Number.MIN_SAFE_INTEGER; // should be safe 118 | //console.log(workers.length, workerIndex, x); 119 | const worker = workers[workerIndex]; 120 | 121 | worker.jobIndex = worker.jobIndex || 0; 122 | const jobIndex = worker.jobIndex; 123 | worker.jobIndex++; 124 | 125 | that.sys.clusterMapObs(that, obs, data, funcName, jobIndex, worker); 126 | } 127 | ).retryWhen(e => e.delay(3000)) 128 | ) 129 | }; 130 | -------------------------------------------------------------------------------- /dist/rxjs-cluster.js: -------------------------------------------------------------------------------- 1 | (function(e, a) { for(var i in a) e[i] = a[i]; }(exports, /******/ (function(modules) { // webpackBootstrap 2 | /******/ // The module cache 3 | /******/ var installedModules = {}; 4 | /******/ 5 | /******/ // The require function 6 | /******/ function __webpack_require__(moduleId) { 7 | /******/ 8 | /******/ // Check if module is in cache 9 | /******/ if(installedModules[moduleId]) 10 | /******/ return installedModules[moduleId].exports; 11 | /******/ 12 | /******/ // Create a new module (and put it into the cache) 13 | /******/ var module = installedModules[moduleId] = { 14 | /******/ exports: {}, 15 | /******/ id: moduleId, 16 | /******/ loaded: false 17 | /******/ }; 18 | /******/ 19 | /******/ // Execute the module function 20 | /******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__); 21 | /******/ 22 | /******/ // Flag the module as loaded 23 | /******/ module.loaded = true; 24 | /******/ 25 | /******/ // Return the exports of the module 26 | /******/ return module.exports; 27 | /******/ } 28 | /******/ 29 | /******/ 30 | /******/ // expose the modules object (__webpack_modules__) 31 | /******/ __webpack_require__.m = modules; 32 | /******/ 33 | /******/ // expose the module cache 34 | /******/ __webpack_require__.c = installedModules; 35 | /******/ 36 | /******/ // __webpack_public_path__ 37 | /******/ __webpack_require__.p = ""; 38 | /******/ 39 | /******/ // Load entry module and return exports 40 | /******/ return __webpack_require__(0); 41 | /******/ }) 42 | /************************************************************************/ 43 | /******/ ([ 44 | /* 0 */ 45 | /***/ function(module, exports, __webpack_require__) { 46 | 47 | 'use strict'; 48 | 49 | Object.defineProperty(exports, "__esModule", { 50 | value: true 51 | }); 52 | exports.NetSystem = exports.ProcSystem = undefined; 53 | exports.default = Cluster; 54 | 55 | var _rx = __webpack_require__(1); 56 | 57 | var _rx2 = _interopRequireDefault(_rx); 58 | 59 | var _stringHash = __webpack_require__(2); 60 | 61 | var _stringHash2 = _interopRequireDefault(_stringHash); 62 | 63 | var _lodash = __webpack_require__(3); 64 | 65 | var _lodash2 = _interopRequireDefault(_lodash); 66 | 67 | var _ProcCluster = __webpack_require__(4); 68 | 69 | var _ProcCluster2 = _interopRequireDefault(_ProcCluster); 70 | 71 | var _NetCluster = __webpack_require__(7); 72 | 73 | var _NetCluster2 = _interopRequireDefault(_NetCluster); 74 | 75 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 76 | 77 | var Observable = _rx2.default.Observable; 78 | var observableProto = Observable.prototype; 79 | 80 | var ProcSystem = exports.ProcSystem = _ProcCluster2.default; 81 | var NetSystem = exports.NetSystem = _NetCluster2.default; 82 | function Cluster(options) { 83 | var _this = this; 84 | 85 | this.workers = []; 86 | this.childEntries = {}; 87 | this._options = options = Object.assign({ 88 | debug: false 89 | }, options || {}); 90 | 91 | if (!options.system) options.system = new _ProcCluster2.default({ workers: options.workers }); 92 | 93 | var sys = this.sys = options.system; 94 | 95 | this.n = 0; // round-robin scheduling 96 | this.work = new _rx2.default.Subject(); // Children work 97 | 98 | var that = this; 99 | this.startWorkers = sys.startWorkers; 100 | this.clusterMap = function (x, y, z) { 101 | var ___clusterMap = _clusterMap.bind(this); 102 | return ___clusterMap(that, x, y, z); 103 | }; 104 | 105 | this.setupChild = sys.setupChild; 106 | this.childWork = _childWork.bind(this); 107 | this.entry = _entry.bind(this); 108 | this.getWorkers = _getWorkers.bind(this); 109 | this.killall = function (x) { 110 | return sys.killall(_this); 111 | }; 112 | } 113 | 114 | function _childWork(_ref) { 115 | var data = _ref.data; 116 | var id = _ref.id; 117 | var func = _ref.func; 118 | 119 | var funcRef = this.childEntries[func]; 120 | 121 | if (!funcRef) { 122 | console.log('Function not found in childMethod lookup:', func); 123 | throw new Error('Function not found in childMethod lookup: ' + func); 124 | return; 125 | } 126 | 127 | var exec = funcRef(data); 128 | 129 | if (!exec.subscribe) { 130 | return _rx2.default.Observable.just(exec); 131 | } else return exec.first(); 132 | } 133 | 134 | /* 135 | @param numWorkers number of cpus 136 | @param entryFun the master entry function 137 | @param options options object 138 | */ 139 | function _entry(entryFun, childMethods) { 140 | var _this2 = this; 141 | 142 | console.log('cluster: slave/master check'); 143 | var options = this._options; 144 | 145 | var childEntries = this.childEntries; 146 | _lodash2.default.forEach(childMethods, function (v, k) { 147 | if (v && (v.subscribe || typeof v === 'function')) childEntries[k] = v; 148 | }); 149 | 150 | var isMasterEnv = process.env.isMaster === 'true' || this._options.isMaster === true; 151 | var isSlaveEnv = process.env.isSlave === 'true' || this._options.isSlave === true; 152 | var isMasterCheck = isMasterEnv ? function (y, x) { 153 | return setTimeout(x, 6000, true); 154 | } : isSlaveEnv ? function (y, x) { 155 | return x(false); 156 | } : this.sys.isMasterCheck; 157 | 158 | //const isMaster = this._options.isMaster || this.sys.isMaster; 159 | isMasterCheck(this, function (isMaster) { 160 | 161 | // Child entry point 162 | if (!isMaster) { 163 | console.log('cluster: slave elected'); 164 | _this2.setupChild(_this2, _this2.work); 165 | return; 166 | } else { 167 | // Master entry point 168 | console.log('cluster: master elected'); 169 | _this2.startWorkers(_this2, _this2.workers, entryFun); 170 | } 171 | }); 172 | } 173 | 174 | function _getWorkers() { 175 | return this.workers; 176 | } 177 | 178 | /* 179 | @param funcName function to invoke 180 | @param nodeSelector (optional) (function | string | int) used to pick node. If function, the value is the stream object and the return is (string | int). 181 | */ 182 | function _clusterMap(that, funcName, nodeSelector) { 183 | var key = null; 184 | if (nodeSelector !== undefined && nodeSelector !== null && typeof nodeSelector !== 'function') { 185 | key = Number.isInteger(nodeSelector) ? nodeSelector : (0, _stringHash2.default)(nodeSelector.toString()); 186 | nodeSelector = null; 187 | } 188 | var workers = that.workers; 189 | //const that = this; 190 | return this.flatMap(function (data) { 191 | return _rx2.default.Observable.create(function (obs) { 192 | if (nodeSelector) { 193 | var nodeKey = nodeSelector(data); 194 | key = Number.isInteger(nodeKey) ? nodeKey : (0, _stringHash2.default)(nodeKey.toString()); 195 | } 196 | 197 | var workerIndex = key ? key % workers.length : that.n++ % workers.length; 198 | //console.log(workerIndex, n, workers.length); 199 | //n++; 200 | //if( n === Number.MAX_SAFE_INTEGER) x = Number.MIN_SAFE_INTEGER; // should be safe 201 | //console.log(workers.length, workerIndex, x); 202 | var worker = workers[workerIndex]; 203 | 204 | worker.jobIndex = worker.jobIndex || 0; 205 | var jobIndex = worker.jobIndex; 206 | worker.jobIndex++; 207 | 208 | that.sys.clusterMapObs(that, obs, data, funcName, jobIndex, worker); 209 | }).retryWhen(function (e) { 210 | return e.delay(3000); 211 | }); 212 | }); 213 | }; 214 | 215 | /***/ }, 216 | /* 1 */ 217 | /***/ function(module, exports) { 218 | 219 | module.exports = require("rx"); 220 | 221 | /***/ }, 222 | /* 2 */ 223 | /***/ function(module, exports) { 224 | 225 | module.exports = require("string-hash"); 226 | 227 | /***/ }, 228 | /* 3 */ 229 | /***/ function(module, exports) { 230 | 231 | module.exports = require("lodash"); 232 | 233 | /***/ }, 234 | /* 4 */ 235 | /***/ function(module, exports, __webpack_require__) { 236 | 237 | 'use strict'; 238 | 239 | Object.defineProperty(exports, "__esModule", { 240 | value: true 241 | }); 242 | exports.default = ProcCluster; 243 | 244 | var _cluster = __webpack_require__(5); 245 | 246 | var _cluster2 = _interopRequireDefault(_cluster); 247 | 248 | var _lodash = __webpack_require__(3); 249 | 250 | var _lodash2 = _interopRequireDefault(_lodash); 251 | 252 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 253 | 254 | function ProcCluster(options) { 255 | console.log('cluster: using ProcCluster'); 256 | if (!options.workers) options.workers = __webpack_require__(6).cpus().length; 257 | 258 | this.options = Object.assign({}, options); 259 | 260 | this.clusterMapObs = _clusterMapObs.bind(this); 261 | this.setupChild = _setupChild.bind(this); 262 | this.startWorkers = _startWorkers.bind(this); 263 | this.killall = _killall.bind(this); 264 | this.isMasterCheck = function (self, cb) { 265 | console.log('cluster.isMaster', _cluster2.default.isMaster); 266 | cb(_cluster2.default.isMaster); 267 | }; 268 | } 269 | 270 | function _killall(self) { 271 | _lodash2.default.forEach(self.workers, function (x) { 272 | return x.kill(); 273 | }); 274 | } 275 | 276 | function _setupChild(self, work) { 277 | work.concatMap(self.childWork, function (y, x) { 278 | return { 279 | data: x, 280 | id: y.id 281 | }; 282 | }).subscribe(function (_ref) { 283 | var data = _ref.data; 284 | var id = _ref.id; 285 | return process.send({ 286 | rdata: data, 287 | id: id 288 | }); 289 | }, function (x) { 290 | return console.log('Child ' + process.pid + ' err', x); 291 | }); 292 | 293 | process.on('message', function onChildMessage(x) { 294 | work.onNext(x); 295 | }); // push work unto task stream 296 | } 297 | 298 | function _startWorkers(self, workers, onReady) { 299 | var numWorkers = this.options.workers; 300 | // cluster manager 301 | var n = 0; 302 | //const workers = self.workers; 303 | 304 | if (_cluster2.default.setupMaster) _cluster2.default.setupMaster({ 305 | silent: false 306 | }); 307 | 308 | _cluster2.default.on('listening', function (worker, address) { 309 | console.log('A worker is now connected to ' + address.address + ':' + address.port); 310 | }); 311 | 312 | _cluster2.default.on('online', function (worker) { 313 | if (n === numWorkers) { 314 | console.log('cluster: All workers online'); 315 | onReady(); 316 | return; 317 | } 318 | console.log('cluster: Worker ' + worker.process.pid + ' is online'); 319 | if (worker.setMaxListeners) worker.setMaxListeners(0); 320 | workers.push(worker); 321 | n++; 322 | //worker.on('message', x => console.log('worker: ', x)); 323 | }); 324 | 325 | _cluster2.default.on('error', function (x) { 326 | throw new Error(x); 327 | }); 328 | 329 | /*cluster.on('disconnect', function(x) { 330 | console.log('disconnect'); 331 | throw new Error(x) 332 | }); 333 | cluster.on('exit', function(x) { 334 | console.log('exit'); 335 | throw new Error(x) 336 | });*/ 337 | 338 | for (var i = 0; i <= numWorkers; i++) { 339 | var f = _cluster2.default.fork(); 340 | //console.log('f', f.process.pid) 341 | if (f.process.stdout) f.process.stdout.on('data', function (data) { 342 | // output from the child process 343 | console.log('>>> ' + data); 344 | }); 345 | } 346 | } 347 | 348 | function _clusterMapObs(self, obs, data, funcName, jobIndex, worker) { 349 | worker.on('message', function handler(_ref2) { 350 | var rdata = _ref2.rdata; 351 | var id = _ref2.id; 352 | 353 | if (id !== jobIndex) return; // ignore 354 | obs.onNext(rdata); 355 | obs.onCompleted(); 356 | worker.removeListener('message', handler); 357 | }); 358 | worker.send({ 359 | data: data, id: jobIndex, 360 | func: funcName 361 | }); 362 | } 363 | 364 | /***/ }, 365 | /* 5 */ 366 | /***/ function(module, exports) { 367 | 368 | module.exports = require("cluster"); 369 | 370 | /***/ }, 371 | /* 6 */ 372 | /***/ function(module, exports) { 373 | 374 | module.exports = require("os"); 375 | 376 | /***/ }, 377 | /* 7 */ 378 | /***/ function(module, exports, __webpack_require__) { 379 | 380 | 'use strict'; 381 | 382 | Object.defineProperty(exports, "__esModule", { 383 | value: true 384 | }); 385 | exports.default = ProcCluster; 386 | 387 | var _cluster = __webpack_require__(5); 388 | 389 | var _cluster2 = _interopRequireDefault(_cluster); 390 | 391 | var _lodash = __webpack_require__(3); 392 | 393 | var _lodash2 = _interopRequireDefault(_lodash); 394 | 395 | var _express = __webpack_require__(8); 396 | 397 | var _express2 = _interopRequireDefault(_express); 398 | 399 | var _bodyParser = __webpack_require__(9); 400 | 401 | var _bodyParser2 = _interopRequireDefault(_bodyParser); 402 | 403 | var _requestJson = __webpack_require__(10); 404 | 405 | var _requestJson2 = _interopRequireDefault(_requestJson); 406 | 407 | var _connectTimeout = __webpack_require__(11); 408 | 409 | var _connectTimeout2 = _interopRequireDefault(_connectTimeout); 410 | 411 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 412 | 413 | function ProcCluster(options) { 414 | console.log('--WIP--', options); 415 | 416 | this.options = Object.assign({ 417 | clients: ['http://localhost:8090/'], 418 | port: 8090 419 | }, options || {}); 420 | if (!Array.isArray(this.options.clients)) this.options.clients = [this.options.clients]; 421 | this.options.port = parseInt(this.options.port); 422 | 423 | this.clusterMapObs = _clusterMapObs.bind(this); 424 | this.setupChild = _setupChild.bind(this); 425 | this.startWorkers = _startWorkers.bind(this); 426 | this.killall = _killall.bind(this); 427 | this.isMasterCheck = _isMasterCheck.bind(this); 428 | 429 | var app = this.app = (0, _express2.default)(); 430 | app.use((0, _connectTimeout2.default)('600s')); 431 | app.use(_bodyParser2.default.json()); 432 | this.appServer = app.listen(this.options.port); 433 | } 434 | 435 | function _isMasterCheck(self, cb) { 436 | var options = self._options; 437 | console.log('cluster: listening port:', this.options.port); 438 | var picked = false; 439 | 440 | this.app.get('/be/master/', function (req, res) { 441 | if (picked) { 442 | console.log('cluster: already picked as master'); 443 | return; 444 | } else picked = true; 445 | options.isMaster = true; 446 | options.isSlave = false; 447 | 448 | if (res && res.body && Array.isArray(res.body.clients)) { 449 | console.log('Using clients from master election'); 450 | this.options.clients = res.body.clients; 451 | } 452 | 453 | res.send('master elected'); 454 | cb(true); 455 | }); 456 | 457 | this.app.get('/be/slave/', function (req, res) { 458 | if (picked) { 459 | console.log('cluster: already picked as master'); 460 | return; 461 | } else picked = true; 462 | options.isMaster = false; 463 | options.isSlave = true; 464 | 465 | res.send('slave elected'); 466 | cb(false); 467 | }); 468 | } 469 | 470 | function _killall(self) { 471 | //_.forEach(self.workers, x => x.kill()); 472 | } 473 | 474 | function _setupChild(self, work) { 475 | var requests = {}; 476 | work.concatMap(self.childWork, function (y, x) { 477 | return { 478 | data: x, 479 | id: y.id 480 | }; 481 | }).subscribe(function (_ref) { 482 | var data = _ref.data; 483 | var id = _ref.id; 484 | 485 | if (requests[id] === undefined) throw new Error('request id not issued ' + id); 486 | console.log('cluster: client: responding'); 487 | requests[id].send({ data: data, id: id }); 488 | }, function (x) { 489 | return console.log('Net Child ' + process.pid + ' err', x); 490 | }); 491 | 492 | console.log('cluster: listening for /work/:', this.options.port); 493 | this.app.get('/ping/', function (req, res) { 494 | res.send('ping pong: client'); 495 | }); 496 | this.app.post('/work/', function (req, res) { 497 | if (!req.body || !req.body.func) { 498 | res.send('ping work'); 499 | return; 500 | } 501 | var _req$body = req.body; 502 | var func = _req$body.func; 503 | var data = _req$body.data; 504 | var id = _req$body.id; 505 | 506 | var workParams = req.body; 507 | console.log('cluster: work recieved', workParams); 508 | requests[id] = res; 509 | work.onNext(workParams); 510 | //res.send('slave elacted'); // TODO 511 | }); 512 | } 513 | 514 | function _startWorkers(self, workers, onReady) { 515 | _lodash2.default.forEach(this.options.clients, function (c) { 516 | var worker = { url: c }; 517 | worker.client = _requestJson2.default.createClient(worker.url); 518 | workers.push(worker); 519 | }); 520 | 521 | this.app.get('/ping/', function (req, res) { 522 | res.send('ping pong: server'); 523 | }); 524 | 525 | setTimeout(onReady, 3000); 526 | } 527 | 528 | function _clusterMapObs(self, obs, data, func, id, worker) { 529 | console.log('cluster: master sending post:' + worker.url + ' rte:work func:' + func + ' id:' + id); 530 | worker.client.post('work', { func: func, data: data, id: id }, function (err, res, body) { 531 | if (res && parseInt(res.statusCode) !== 200) { 532 | console.log(res.statusCode + ' response from client.'); 533 | return obs.onError(res.statusCode + ' response from client.'); 534 | } 535 | if (err) { 536 | console.log('err', err); 537 | return obs.onError(err); 538 | } 539 | //if(self._options) 540 | console.log('cluster: master recieved:', err, res ? res.statusCode : res, func, id); 541 | obs.onNext(body.data); 542 | obs.onCompleted(); 543 | }); 544 | } 545 | 546 | /***/ }, 547 | /* 8 */ 548 | /***/ function(module, exports) { 549 | 550 | module.exports = require("express"); 551 | 552 | /***/ }, 553 | /* 9 */ 554 | /***/ function(module, exports) { 555 | 556 | module.exports = require("body-parser"); 557 | 558 | /***/ }, 559 | /* 10 */ 560 | /***/ function(module, exports) { 561 | 562 | module.exports = require("request-json"); 563 | 564 | /***/ }, 565 | /* 11 */ 566 | /***/ function(module, exports) { 567 | 568 | module.exports = require("connect-timeout"); 569 | 570 | /***/ } 571 | /******/ ]))); 572 | //# sourceMappingURL=rxjs-cluster.js.map --------------------------------------------------------------------------------