├── .npmrc ├── scripts └── start.js ├── test ├── workers │ ├── master.js │ ├── basic.js │ ├── basic2.js │ ├── basic.json │ ├── legacyReady.js │ ├── basic2.json │ ├── restart_all.js │ ├── longInit.js │ └── inlineReady.js ├── commands │ └── custom.js ├── proxy │ └── proxy.json ├── commands2 │ └── custom-name.js ├── cmd-version.js ├── util.js ├── bin.js ├── cmd-health.js ├── master.js ├── cmd-exit.js ├── commands.js ├── cmd-workers.js ├── cmd-help.js ├── workers.js ├── start-restart.js ├── server-rest.js ├── control.js └── start-stop.js ├── bin ├── cservice └── cserviced ├── .travis.yml ├── .gitignore ├── examples ├── server.js ├── slow.js ├── certs │ ├── test1.txt │ ├── test1-pubkey.pem │ ├── test1-cert.pem │ └── test1-key.pem ├── proxy │ ├── v1 │ │ └── worker.js │ ├── v2 │ │ └── worker.js │ ├── v3 │ │ └── worker.js │ └── proxy.json └── lazy.js ├── lib ├── commands │ ├── version.js │ ├── health.js │ ├── locals.js │ ├── options.js │ ├── workerStart.js │ ├── workerExit.js │ ├── exit.js │ ├── help.js │ ├── info.js │ ├── proxy.js │ ├── workers.js │ ├── shutdown.js │ ├── start.js │ ├── restart.js │ └── upgrade.js ├── legacy.js ├── stop.js ├── trigger.js ├── worker-ready.js ├── cli.js ├── defaults.js ├── run.js ├── net-stats.js ├── commands.js ├── start.js ├── http-client.js ├── message-bus.js ├── workers.js ├── control.js ├── new-worker.js ├── net-servers.js ├── proxy-worker.js ├── http-server.js ├── util.js ├── master.js └── proxy.js ├── .jshintrc ├── .test-jshintrc ├── vs ├── node-cluster-service.sln └── node-cluster-service.njsproj ├── LICENSE.txt ├── package.json ├── .gitattributes ├── cluster-service.js ├── CHANGELOG.md └── README.md /.npmrc: -------------------------------------------------------------------------------- 1 | registry=http://registry.npmjs.org/ 2 | -------------------------------------------------------------------------------- /scripts/start.js: -------------------------------------------------------------------------------- 1 | require("../cluster-service").start(); 2 | -------------------------------------------------------------------------------- /test/workers/master.js: -------------------------------------------------------------------------------- 1 | 2 | 3 | exports = function() { 4 | }; 5 | -------------------------------------------------------------------------------- /bin/cservice: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | require('../cluster-service').start(); 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "6" 4 | - "7" 5 | - "8" 6 | 7 | -------------------------------------------------------------------------------- /test/commands/custom.js: -------------------------------------------------------------------------------- 1 | module.exports = function(evt, cb) { 2 | cb(null, true); 3 | }; 4 | -------------------------------------------------------------------------------- /test/workers/basic.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../../cluster-service"); 2 | 3 | cservice.workerReady(); -------------------------------------------------------------------------------- /test/proxy/proxy.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { "port": 3000, "workerCount": 2 } 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /test/commands2/custom-name.js: -------------------------------------------------------------------------------- 1 | module.exports = function(evt, cb) { 2 | cb(null, true); 3 | }; 4 | 5 | module.exports.id = "customName"; 6 | -------------------------------------------------------------------------------- /test/workers/basic2.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../../cluster-service"); 2 | 3 | setTimeout(function() { 4 | cservice.workerReady(); 5 | }, 100); 6 | -------------------------------------------------------------------------------- /test/workers/basic.json: -------------------------------------------------------------------------------- 1 | { 2 | "workers": { 3 | "basic": {"worker": "./test/workers/basic.js", "count": 1} 4 | }, "accessKey": "123", "cli": false 5 | } -------------------------------------------------------------------------------- /test/workers/legacyReady.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../../cluster-service"); 2 | 3 | setTimeout(function() { 4 | cservice.workerReady(); 5 | }, 1000); 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | .nyc_output 4 | 5 | *.suo 6 | 7 | npm-debug.log 8 | 9 | # Netbeans 10 | /nbproject/ 11 | 12 | .idea/ 13 | -------------------------------------------------------------------------------- /test/workers/basic2.json: -------------------------------------------------------------------------------- 1 | { 2 | "workers": { 3 | "basic2": {"worker": "./test/workers/basic2.js", "ready": false} 4 | }, "accessKey": "123", "cli": false 5 | } -------------------------------------------------------------------------------- /test/workers/restart_all.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../../cluster-service"); 2 | 3 | setTimeout(function() { 4 | cservice.trigger("restart", "all"); 5 | }, 5000); 6 | -------------------------------------------------------------------------------- /test/workers/longInit.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../../cluster-service"); 2 | 3 | cservice.workerReady(false); 4 | 5 | setTimeout(function() { 6 | cservice.workerReady(); 7 | }, 10000); 8 | -------------------------------------------------------------------------------- /test/workers/inlineReady.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../../cluster-service"); 2 | 3 | cservice.workerReady(false); 4 | 5 | setTimeout(function() { 6 | cservice.workerReady(); 7 | }, 1000); 8 | -------------------------------------------------------------------------------- /examples/server.js: -------------------------------------------------------------------------------- 1 | var http = require('http'); 2 | http.createServer(function (req, res) { 3 | res.writeHead(200, {'Content-Type': 'text/plain'}); 4 | res.end('Hello World\n'); 5 | }).listen(1337, '127.0.0.1'); 6 | console.log('Server running at http://127.0.0.1:1337/'); 7 | -------------------------------------------------------------------------------- /examples/slow.js: -------------------------------------------------------------------------------- 1 | var http = require('http'); 2 | var app = http.createServer(function (req, res) { 3 | setTimeout(function() { 4 | res.writeHead(200, {'Content-Type': 'text/plain'}); 5 | res.end('Hello World\n'); 6 | }, 2000).unref(); 7 | }).listen(1337, '127.0.0.1'); 8 | console.log('Server running at http://127.0.0.1:1337/'); 9 | -------------------------------------------------------------------------------- /lib/commands/version.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../../cluster-service"); 2 | 3 | module.exports = function(evt, cb) { 4 | var pkg = require("../../package.json"); 5 | cb(null, pkg.version); 6 | }; 7 | 8 | module.exports.more = function(cb) { 9 | cb(null, { 10 | info: "Get version of cluster-service.", 11 | command: "version" 12 | }); 13 | }; 14 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "bitwise":false, 3 | "camelcase":true, 4 | "curly":false, 5 | "eqeqeq":true, 6 | "freeze":true, 7 | "immed":true, 8 | "indent":2, 9 | "latedef":"nofunc", 10 | "laxbreak":true, 11 | "laxcomma":true, 12 | "maxlen":80, 13 | "newcap":true, 14 | "noarg":true, 15 | "node":true, 16 | "trailing":true, 17 | "undef":true 18 | } 19 | -------------------------------------------------------------------------------- /lib/commands/health.js: -------------------------------------------------------------------------------- 1 | var cluster = require("cluster"); 2 | 3 | module.exports = function(evt, cb) { 4 | cb(null, "OK"); 5 | }; 6 | 7 | module.exports.more = function(cb) { 8 | cb(null, { 9 | command: "health", 10 | info: [ 11 | "Returns health of service.", 12 | "May be overidden by service to expose app-specific data." 13 | ].join(' ') 14 | }); 15 | }; 16 | -------------------------------------------------------------------------------- /examples/certs/test1.txt: -------------------------------------------------------------------------------- 1 | Step 1: Create CSR request 2 | 3 | openssl req -x509 -newkey rsa:2048 -keyout test1-key.pem -out test1-cert.pem 4 | 5 | pass phrase: test1 6 | 7 | 8 | 9 | Step 2: Remove passphrase 10 | 11 | openssl rsa -in test1-key.pem -out test1-key.pem 12 | 13 | 14 | 15 | Step 3: Get Public Key 16 | 17 | openssl rsa -in test1-key.pem -pubout > test1-pubkey.pem 18 | 19 | -------------------------------------------------------------------------------- /examples/proxy/v1/worker.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../../../cluster-service"); 2 | var path = require("path"); 3 | 4 | cservice.workerReady(false); 5 | 6 | require("http").createServer(function(req, res) { 7 | res.writeHead(200, { 'Content-Type': 'text/plain' }); 8 | res.end("Hello from " + path.basename(__dirname)); 9 | }).listen(process.env.PROXY_PORT || 3000, cservice.workerReady); 10 | -------------------------------------------------------------------------------- /examples/proxy/v2/worker.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../../../cluster-service"); 2 | var path = require("path"); 3 | 4 | cservice.workerReady(false); 5 | 6 | require("http").createServer(function(req, res) { 7 | res.writeHead(200, { 'Content-Type': 'text/plain' }); 8 | res.end("Hello from " + path.basename(__dirname)); 9 | }).listen(process.env.PROXY_PORT || 3000, cservice.workerReady); 10 | -------------------------------------------------------------------------------- /examples/proxy/v3/worker.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../../../cluster-service"); 2 | var path = require("path"); 3 | 4 | cservice.workerReady(false); 5 | 6 | require("http").createServer(function(req, res) { 7 | res.writeHead(200, { 'Content-Type': 'text/plain' }); 8 | res.end("Hello from " + path.basename(__dirname)); 9 | }).listen(process.env.PROXY_PORT || 3000, cservice.workerReady); 10 | -------------------------------------------------------------------------------- /lib/legacy.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | 3 | module.exports = exports = legacySupport; 4 | 5 | function legacySupport(options) { 6 | if (options.worker) { 7 | cservice.log( 8 | "Option `worker` has been deprecated. Use `workers` instead.".warn 9 | ); 10 | options.workers = options.worker; 11 | delete options.worker; 12 | } 13 | } 14 | 15 | legacySupport(cservice.options); 16 | -------------------------------------------------------------------------------- /test/cmd-version.js: -------------------------------------------------------------------------------- 1 | var cmd = require('../lib/commands/version'); 2 | var assert = require("assert"); 3 | 4 | describe('[Version cmd]', function() { 5 | it('Get version', function(done) { 6 | var pkg = require("../package.json"); 7 | var evt = {}; 8 | var cb = function(err, data) { 9 | assert.ifError(err); 10 | assert.equal(data, pkg.version); 11 | done(); 12 | }; 13 | cmd(evt, cb); 14 | }); 15 | }); -------------------------------------------------------------------------------- /lib/commands/locals.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../../cluster-service"); 2 | 3 | module.exports = function(evt, cb) { 4 | cb(null, cservice.locals); 5 | }; 6 | 7 | module.exports.more = function(cb) { 8 | cb(null, { 9 | command: "locals", 10 | info: "Returns locals state object for debug purposes." 11 | }); 12 | }; 13 | 14 | module.exports.control = function() { 15 | return "local"; 16 | }; 17 | 18 | module.exports.visible = false; 19 | -------------------------------------------------------------------------------- /lib/commands/options.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../../cluster-service"); 2 | 3 | module.exports = function(evt, cb) { 4 | cb(null, cservice.locals); 5 | }; 6 | 7 | module.exports.more = function(cb) { 8 | cb(null, { 9 | command: "options", 10 | info: "Returns current options for debug purposes." 11 | }); 12 | }; 13 | 14 | module.exports.control = function() { 15 | return "local"; 16 | }; 17 | 18 | module.exports.visible = false; 19 | -------------------------------------------------------------------------------- /examples/proxy/proxy.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaultVersion": "v1", 3 | "bindings": [ 4 | { 5 | "port": 1000, 6 | "workerCount": 2 7 | }, 8 | { 9 | "port": 1001, 10 | "workerCount": 2, 11 | "tlsOptions": { 12 | "key": "./examples/certs/test1-key.pem", 13 | "cert": "./examples/certs/test1-cert.pem" 14 | } 15 | } 16 | ], 17 | "nonDefaultWorkerCount": 1, 18 | "nonDefaultWorkerIdleTime": 20 19 | } -------------------------------------------------------------------------------- /examples/certs/test1-pubkey.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApEyXSJ43EqU1Xz4UrLzc 3 | 84AfwX6WPNvldGtucagk0fTWjq0dSt65kJwkx3pN+JOA3MjuPP6BUiO4DF02tzAl 4 | P/ILia0yANvndZ1C6BsWsZxooFSOYhXEDn3tAcYRLa5Y4APomrOLc2oIwCPJrZqf 5 | nO3tbCbmumOEuI56ib9ADjvcR0DI7VFQUEJyQLZXXF8HUy97UgyRBhUX/6VO5qpj 6 | EmTXV6ZjILcpMMNPDTuzPmC49J/6InVaDQihR+orRpVYaG2pC+Y4jj5FpcUsCHd/ 7 | RR6aA7ZPsQHfQfzfjn1YkEIzl3isXZscda9oGUEtMwAhOMlwB3N4fSw43850G1ul 8 | cQIDAQAB 9 | -----END PUBLIC KEY----- 10 | -------------------------------------------------------------------------------- /test/util.js: -------------------------------------------------------------------------------- 1 | var assert = require("assert"); 2 | var util = require("../lib/util"); 3 | 4 | describe('Util funcs', function() { 5 | describe('getArgsFromQuestion', function() { 6 | it('Strings', function(done) { 7 | var args = util.getArgsFromQuestion( 8 | "health { \"check\": true, \"nested\": { } } \"arg #3\" [\"arg #4\"]", 9 | " " 10 | ); 11 | assert.equal(args.length, 4); 12 | assert.equal(args[1].check, true); 13 | done(); 14 | }); 15 | }); 16 | }); -------------------------------------------------------------------------------- /.test-jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "bitwise":false, 3 | "camelcase":true, 4 | "curly":false, 5 | "eqeqeq":true, 6 | "freeze":true, 7 | "immed":true, 8 | "indent":2, 9 | "latedef":"nofunc", 10 | "laxbreak":true, 11 | "laxcomma":true, 12 | "maxlen":80, 13 | "newcap":true, 14 | "noarg":true, 15 | "node":true, 16 | "trailing":true, 17 | "undef":true, 18 | "globals":{ 19 | "after":false, 20 | "afterEach":false, 21 | "before":false, 22 | "beforeEach":false, 23 | "describe":false, 24 | "it":false 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /lib/commands/workerStart.js: -------------------------------------------------------------------------------- 1 | var path = require("path"), 2 | cservice = require("../../cluster-service"); 3 | 4 | module.exports = function(evt, cb, worker, reason) { 5 | cservice.log( 6 | ("Worker " 7 | + path.basename(worker.cservice.worker) 8 | + "(" 9 | + worker.process.pid 10 | + ") start, reason: " 11 | + (reason || cservice.locals.reason || "Unknown")).success 12 | ); 13 | cb(); 14 | }; 15 | 16 | module.exports.control = function() { 17 | return "inproc"; 18 | }; 19 | 20 | module.exports.visible = false; 21 | -------------------------------------------------------------------------------- /lib/commands/workerExit.js: -------------------------------------------------------------------------------- 1 | var path = require("path"), 2 | cservice = require("../../cluster-service"); 3 | 4 | module.exports = function(evt, cb, worker, reason) { 5 | cservice.log( 6 | ("Worker " 7 | + path.basename(worker.cservice.worker) 8 | + "(" 9 | + worker.process.pid 10 | + ") exited, reason: " 11 | + (reason || worker.cservice.reason || 12 | cservice.locals.reason || "Unknown")).warn 13 | ); 14 | cb(); 15 | }; 16 | 17 | module.exports.control = function() { 18 | return "inproc"; 19 | }; 20 | 21 | module.exports.visible = false; 22 | -------------------------------------------------------------------------------- /bin/cserviced: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var path = require("path"); 4 | var spawn = require("child_process").spawn; 5 | var args = process.argv.slice(2); 6 | var cservicePath = path.resolve(__dirname, '..', 'bin', 'cservice'); 7 | var cservice; 8 | 9 | args.splice(0, 0, cservicePath); 10 | args.push("--cli"); 11 | args.push("false"); 12 | 13 | cservice = spawn(process.execPath, args, { 14 | stdio: "ignore", 15 | detached: true 16 | }); 17 | 18 | cservice.on('exit', function (code) { 19 | console.error('cluster-service exited: ' + code); 20 | }); 21 | 22 | cservice.unref(); // unreference so we may exit 23 | 24 | -------------------------------------------------------------------------------- /examples/lazy.js: -------------------------------------------------------------------------------- 1 | var http = require('http'); 2 | var cservice = require("../cluster-service"); 3 | 4 | cservice.workerReady(false); // inform cservice we're not ready yet 5 | 6 | // to emulate a slow startup where-in other 7 | // tasks must be completed prior to server listen 8 | setTimeout(function() { 9 | http.createServer(function (req, res) { 10 | res.writeHead(200, {'Content-Type': 'text/plain'}); 11 | res.end('Hello World\n'); 12 | }).listen(1337, '127.0.0.1', onReady); 13 | }, 2000); 14 | 15 | function onReady() { 16 | console.log('Server running at http://127.0.0.1:1337/'); 17 | cservice.workerReady(); // NOW we're ready 18 | } 19 | -------------------------------------------------------------------------------- /test/bin.js: -------------------------------------------------------------------------------- 1 | var assert = require("assert"); 2 | var fs = require("fs"); 3 | 4 | describe('[Bin]', function(){ 5 | it("bin/cservice", function(done) { 6 | fs.readFile("./bin/cservice", { encoding: "utf8" }, function(err, data) { 7 | assert.ifError(err); 8 | assert.equal(/\r/.test(data), false, "\r not permitted in bin files"); 9 | done(); 10 | }); 11 | }); 12 | 13 | it("bin/cserviced", function(done) { 14 | fs.readFile("./bin/cserviced", { encoding: "utf8" }, function(err, data) { 15 | assert.ifError(err); 16 | assert.equal(/\r/.test(data), false, "\r not permitted in bin files"); 17 | done(); 18 | }); 19 | }); 20 | }); -------------------------------------------------------------------------------- /lib/stop.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | 3 | module.exports = exports = stop; 4 | 5 | function stop(timeout, cb) { 6 | if (cservice.locals.state === 0) { 7 | if (cb) cb(null, "Not running"); 8 | return; 9 | } 10 | 11 | if (cservice.workers.length > 0) { // issue shutdown 12 | cservice.trigger("shutdown", function() { 13 | handleWorkersExited(cb); 14 | }, "all", timeout); 15 | } else { // gracefully shutdown 16 | handleWorkersExited(cb); 17 | } 18 | } 19 | 20 | function handleWorkersExited(cb) { 21 | if (cb) cb(null, "Shutting down..."); 22 | require("./http-server").close(); 23 | cservice.locals.state = 0; 24 | if (cservice.options.cli === true) { 25 | process.exit(1); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /lib/commands/exit.js: -------------------------------------------------------------------------------- 1 | var util = require("util"), 2 | cservice = require("../../cluster-service"); 3 | 4 | module.exports = function(evt, cb, cmd) { 5 | if (cmd !== "now") { 6 | cb("Invalid request, 'now' required. Try help exit"); 7 | return; 8 | } 9 | 10 | cservice.log("*** FORCEFUL TERMINATION REQUESTED ***".warn); 11 | cservice.log("Exiting now.".warn); 12 | cb(null, "Exiting now."); 13 | setTimeout(function() { 14 | process.exit(0); // exit master 15 | }, 100); 16 | }; 17 | 18 | module.exports.more = function(cb) { 19 | cb(null, { 20 | info: "Forcefully exits the service.", 21 | command: "exit now", 22 | "now": "Required. 'now' to force exit." 23 | }); 24 | }; 25 | 26 | module.exports.control = function() { 27 | return "local"; 28 | }; 29 | -------------------------------------------------------------------------------- /test/cmd-health.js: -------------------------------------------------------------------------------- 1 | var cmd = require('../lib/commands/health'); 2 | var assert = require("assert"); 3 | 4 | describe('[Health cmd]', function() { 5 | it('Issue command', function(done) { 6 | cmd({}, function(nullObj, data) { 7 | assert.equal(nullObj, null); 8 | assert.equal(data, "OK"); 9 | done(); 10 | }); 11 | }); 12 | 13 | it('more', function(done) { 14 | var callback = function(nullObj, data) { 15 | assert.equal(nullObj, null); 16 | assert.equal( 17 | data.info, 18 | [ 19 | "Returns health of service. May be overidden by service to expose", 20 | "app-specific data." 21 | ].join(' ') 22 | ); 23 | assert.equal(data.command, "health"); 24 | done(); 25 | }; 26 | 27 | cmd.more(callback); 28 | }); 29 | }); -------------------------------------------------------------------------------- /vs/node-cluster-service.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 2013 4 | VisualStudioVersion = 12.0.21005.1 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9092AA53-FB77-4645-B42D-1CCCA6BD08BD}") = "node-cluster-service", "node-cluster-service.njsproj", "{D1EF2FBF-74DC-4B13-B9A6-A422EF27216E}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {D1EF2FBF-74DC-4B13-B9A6-A422EF27216E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {D1EF2FBF-74DC-4B13-B9A6-A422EF27216E}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {D1EF2FBF-74DC-4B13-B9A6-A422EF27216E}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {D1EF2FBF-74DC-4B13-B9A6-A422EF27216E}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | EndGlobal 23 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 GoDaddy 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/commands/help.js: -------------------------------------------------------------------------------- 1 | var util = require("util"); 2 | 3 | module.exports = function(evt, cb, cmdName) { 4 | var evtName, cmd, ret = {}; 5 | if (typeof cmdName === "string") { 6 | ret.command = cmdName; 7 | cmd = evt.locals.events[cmdName]; 8 | if (!cmd) { 9 | ret.err = "Command not found"; 10 | } else { 11 | if (typeof cmd.cb.more === "function") { 12 | cmd.cb.more(function(err, result) { 13 | cb(null, result); 14 | }); 15 | return; 16 | } else { 17 | ret.more = "No additional details found."; 18 | } 19 | } 20 | } else { // full listing 21 | ret.more = "Commands (Use 'help [command_name]' for more details)"; 22 | ret.commands = []; 23 | for (evtName in evt.locals.events) { 24 | cmd = evt.locals.events[evtName]; 25 | if (cmd.cb.visible === false) 26 | continue; 27 | ret.commands.push(evtName); 28 | } 29 | } 30 | 31 | cb(null, ret); 32 | }; 33 | 34 | module.exports.more = function(cb) { 35 | cb(null, { 36 | "command": "help [command_name]", 37 | "command_name": "Optional if you want extended help" 38 | }); 39 | }; 40 | 41 | module.exports.control = function() { 42 | return "local"; 43 | }; 44 | -------------------------------------------------------------------------------- /lib/trigger.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | 3 | module.exports = exports = trigger; 4 | 5 | function trigger(eventName, cb) { 6 | var args = Array.prototype.slice.call(arguments); 7 | if (cservice.isWorker === true) { 8 | args.splice(1, 1); // remove cb from args if it exists 9 | cservice.msgBus.sendMessage("trigger", { args: args, cb: true }, 10 | null, function(err, result) { 11 | // wait for response from master 12 | if (typeof cb === "function") { 13 | cb(err, result); 14 | } 15 | }); 16 | return; 17 | } 18 | var evt = cservice.locals.events[eventName]; 19 | var i; 20 | if (!evt) { 21 | // invoke callback if provided instead of throwing 22 | if (typeof cb === "function") { 23 | cb("Event " + eventName + " not found"); 24 | } else { 25 | throw new Error("Event " + eventName + " not found"); 26 | } 27 | } 28 | 29 | args.splice(0, 1, evt); 30 | 31 | if (typeof cb !== "function") { 32 | // auto-inject dummy callback if not provided 33 | args.splice(1, 0, function DummyCallback(err, results) { 34 | // do nothing 35 | }); 36 | } 37 | 38 | // invoke event callback 39 | return evt.cb.apply(null, args); 40 | } 41 | -------------------------------------------------------------------------------- /examples/certs/test1-cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDXTCCAkWgAwIBAgIJAPWc702VeGKyMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV 3 | BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX 4 | aWRnaXRzIFB0eSBMdGQwHhcNMTMxMjE3MjExMzIwWhcNMTQwMTE2MjExMzIwWjBF 5 | MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 6 | ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB 7 | CgKCAQEApEyXSJ43EqU1Xz4UrLzc84AfwX6WPNvldGtucagk0fTWjq0dSt65kJwk 8 | x3pN+JOA3MjuPP6BUiO4DF02tzAlP/ILia0yANvndZ1C6BsWsZxooFSOYhXEDn3t 9 | AcYRLa5Y4APomrOLc2oIwCPJrZqfnO3tbCbmumOEuI56ib9ADjvcR0DI7VFQUEJy 10 | QLZXXF8HUy97UgyRBhUX/6VO5qpjEmTXV6ZjILcpMMNPDTuzPmC49J/6InVaDQih 11 | R+orRpVYaG2pC+Y4jj5FpcUsCHd/RR6aA7ZPsQHfQfzfjn1YkEIzl3isXZscda9o 12 | GUEtMwAhOMlwB3N4fSw43850G1ulcQIDAQABo1AwTjAdBgNVHQ4EFgQUo61sHkQA 13 | /AVAPvGP8XpXzaMgmWAwHwYDVR0jBBgwFoAUo61sHkQA/AVAPvGP8XpXzaMgmWAw 14 | DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOCAQEAWEKBAkRkBzS8AiW01SG+ 15 | bC24XgVAsLMbckNlYHr2uIGeCfBIBSQGSmIw18zM5FxEwqIV/U3ps19oGBE+M8PR 16 | nflsvOMYJcE8if815UAjN3Qo17wtYqT1X48717S/gLBXP2z3XvmIDr+HtjwKgaxf 17 | WkImDb5qG6H96bwv7mFW7B4r4DIZ8ovVkectaWpvXuK+0pfkYYDJk4ooFkns3dnn 18 | TZUxtz0LAfOOcAhIrTokOsR7+NgFy+DWtlw/YwRF24Cj+WpKb+/f9UncRM67/wT/ 19 | gzWSoMjWRbA68La0FGyuxPTTYV2PBe+S5TTeSoqdRXub9W+7Wu0Sz656Dc+QSnTN 20 | RQ== 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /test/master.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | var assert = require("assert"); 3 | var path = require("path"); 4 | 5 | if(cservice.isWorker){ 6 | it("WORKER", function(done) {}); 7 | } else { 8 | describe('[master option]', function(){ 9 | var masterFn = path.resolve("./test/workers/master.js"); 10 | 11 | // clear cache, just in case 12 | delete require.cache[masterFn]; 13 | 14 | it("Run", function(done) { 15 | cservice.start( 16 | { 17 | workers: "./test/workers/basic.js", 18 | workerCount: 1, 19 | accessKey: "123", 20 | cli: false, 21 | master: "./test/workers/master.js" 22 | }, 23 | function() { 24 | assert.equal( 25 | cservice.workers.length, 26 | 1, 27 | "1 worker expected, but " + cservice.workers.length + " found" 28 | ); 29 | assert.equal( 30 | masterFn in require.cache, 31 | true, 32 | "Master was not run by cservice.start" 33 | ); 34 | done(); 35 | } 36 | ); 37 | }); 38 | 39 | it('Stop workers', function(done) { 40 | cservice.stop(30000, function() { 41 | assert.equal( 42 | cservice.workers.length, 43 | 0, 44 | "0 workers expected, but " + cservice.workers.length + " found" 45 | ); 46 | done(); 47 | }); 48 | }); 49 | }); 50 | } 51 | -------------------------------------------------------------------------------- /test/cmd-exit.js: -------------------------------------------------------------------------------- 1 | var exit = require('../lib/commands/exit'); 2 | var assert = require("assert"); 3 | 4 | describe('[Exit cmd]', function() { 5 | it('Invalid request if cmd not equal to now', function(done) { 6 | var evt = {}; 7 | var cmd = "Not now"; 8 | var cb = function(message) { 9 | assert.equal(message, "Invalid request, 'now' required. Try help exit"); 10 | done(); 11 | }; 12 | exit(evt, cb, cmd); 13 | }); 14 | 15 | it('Calls process exit', function(done) { 16 | var evt = {}; 17 | var cmd = "now"; 18 | process.exit = function(input) { 19 | assert.equal(input, 0); 20 | }; 21 | var realLog = console.log; 22 | console.log = function(msg1, msg2) { 23 | assert.equal(msg1, "*** FORCEFUL TERMINATION REQUESTED ***"); 24 | assert.equal(msg2, "Exiting now."); 25 | }; 26 | var cb = function(nullObj, message) { 27 | assert.equal(nullObj, null); 28 | assert.equal(message, "Exiting now."); 29 | console.log = realLog; 30 | done(); 31 | }; 32 | exit(evt, cb, cmd); 33 | }); 34 | 35 | it('more', function(done) { 36 | var callback = function(nullObj, obj) { 37 | assert.equal(nullObj, null); 38 | assert.equal(obj.info, "Forcefully exits the service."); 39 | assert.equal(obj.command, "exit now"); 40 | assert.equal(obj.now, "Required. 'now' to force exit."); 41 | done(); 42 | }; 43 | 44 | exit.more(callback); 45 | }); 46 | }); 47 | -------------------------------------------------------------------------------- /lib/commands/info.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../../cluster-service"); 2 | 3 | module.exports = function(evt, cb, cmd) { 4 | cservice.trigger("workers", function(err, results) { 5 | if (err) { 6 | cb(err); 7 | return; 8 | } 9 | 10 | var workers = results.workers; 11 | var summary = { 12 | workers: { active: workers.length }, 13 | memory: { rss: 0, heapTotal: 0, heapUsed: 0 }, 14 | net: { 15 | connections: 0, 16 | connectionsOpen: 0, 17 | requests: 0, 18 | avgRequests: 0, 19 | avgConnections: 0 20 | } 21 | }; 22 | 23 | for (var i = 0; i < workers.length; i++) { 24 | var w = workers[i]; 25 | var p = w.process; 26 | summary.memory.rss += p.memory.rss; 27 | summary.memory.heapTotal += p.memory.heapTotal; 28 | summary.memory.heapUsed += p.memory.heapUsed; 29 | if (p.net) { 30 | summary.net.connections += p.net.connections; 31 | summary.net.connectionsOpen += p.net.connectionsOpen; 32 | summary.net.requests += p.net.requests; 33 | summary.net.avgRequests += p.net.avgRequests; 34 | summary.net.avgConnections += p.net.avgConnections; 35 | } 36 | } 37 | 38 | cb(null, summary); 39 | }, "simple"); 40 | }; 41 | 42 | module.exports.more = function(cb) { 43 | cb(null, { 44 | command: "info", 45 | info: "Returns summary of process & workers." 46 | }); 47 | }; 48 | 49 | module.exports.control = function() { 50 | return "remote"; // consistent with "workers" command, 51 | // but may be locked down in the future 52 | }; 53 | -------------------------------------------------------------------------------- /test/commands.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | var assert = require("assert"); 3 | 4 | cservice.log = function() { 5 | }; 6 | if(cservice.isWorker){ 7 | it("WORKER", function(done) {}); 8 | } else { 9 | describe('[Commands]', function() { 10 | it('Start worker', function(done) { 11 | assert.equal( 12 | cservice.workers.length, 13 | 0, 14 | "0 workers expected, but " + cservice.workers.length + " found" 15 | ); 16 | cservice.start({ workers: null, 17 | commands: "./test/commands,./test/commands2"}, function() { 18 | assert.equal( 19 | cservice.workers.length, 20 | 0, 21 | "0 worker expected, but " + cservice.workers.length + " found" 22 | ); 23 | done(); 24 | }); 25 | }); 26 | 27 | it('Command "custom"', function(done) { 28 | cservice.trigger("custom", function(err, result) { 29 | assert.ifError(err); 30 | assert.equal(result, true, "Expect result of true, but got " + result); 31 | done(); 32 | }); 33 | }); 34 | 35 | it('Command "customName"', function(done) { 36 | cservice.trigger("customName", function(err, result) { 37 | assert.ifError(err); 38 | assert.equal(result, true, "Expect result of true, but got " + result); 39 | done(); 40 | }); 41 | }); 42 | 43 | it('Stop workers', function(done) { 44 | cservice.stop(30000, function() { 45 | assert.equal( 46 | cservice.workers.length, 47 | 0, 48 | "0 workers expected, but " + cservice.workers.length + " found" 49 | ); 50 | done(); 51 | }); 52 | }); 53 | }); 54 | } -------------------------------------------------------------------------------- /test/cmd-workers.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | var workers = require('../lib/commands/workers'); 3 | var assert = require("assert"); 4 | 5 | cservice.log = function() {}; 6 | if(cservice.isWorker){ 7 | it("WORKER", function(done) {}); 8 | } else { 9 | describe('[Workers cmd]', function() { 10 | it('Start', function(done) { 11 | cservice.start( 12 | { 13 | workers: null, 14 | workerCount: 1, 15 | accessKey: "123", 16 | cli: false 17 | }, 18 | function() { 19 | assert.equal( 20 | cservice.workers.length, 21 | 0, 22 | "0 worker expected, but " + cservice.workers.length + " found" 23 | ); 24 | done(); 25 | } 26 | ); 27 | }); 28 | 29 | it('Test workers command', function(done) { 30 | cservice.trigger("workers", function(err, data) { 31 | assert.equal(err, null); 32 | assert.equal(data.workers.length, 0); 33 | done(); 34 | }); 35 | }); 36 | 37 | it('more', function(done) { 38 | var callback = function(nullObj, obj) { 39 | assert.equal(nullObj, null); 40 | assert.equal(obj.info, "Returns list of active worker processes."); 41 | assert.equal(obj.command, "workers [simple|details]"); 42 | done(); 43 | }; 44 | 45 | workers.more(callback); 46 | }); 47 | 48 | it('Stop', function(done) { 49 | cservice.stop(30000, function() { 50 | assert.equal( 51 | cservice.workers.length, 52 | 0, 53 | "0 workers expected, but " + cservice.workers.length + " found" 54 | ); 55 | done(); 56 | }); 57 | }); 58 | }); 59 | } -------------------------------------------------------------------------------- /examples/certs/test1-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEApEyXSJ43EqU1Xz4UrLzc84AfwX6WPNvldGtucagk0fTWjq0d 3 | St65kJwkx3pN+JOA3MjuPP6BUiO4DF02tzAlP/ILia0yANvndZ1C6BsWsZxooFSO 4 | YhXEDn3tAcYRLa5Y4APomrOLc2oIwCPJrZqfnO3tbCbmumOEuI56ib9ADjvcR0DI 5 | 7VFQUEJyQLZXXF8HUy97UgyRBhUX/6VO5qpjEmTXV6ZjILcpMMNPDTuzPmC49J/6 6 | InVaDQihR+orRpVYaG2pC+Y4jj5FpcUsCHd/RR6aA7ZPsQHfQfzfjn1YkEIzl3is 7 | XZscda9oGUEtMwAhOMlwB3N4fSw43850G1ulcQIDAQABAoIBAFdJoa4g8F1ljD93 8 | egBzrmdnkHd6S1M1+Gerk9eqXzV0gHD4o/Fc9vVPH3MjFT2VEAc8cOXSyN3cwDFB 9 | bIpSd9fLPjn82+385rFjxWIO0jW2RRe5FJQjwC9602n30rSURf9t1Cwsa0/76345 10 | BTLITThQZ6zn1fj8Wky61XtNMjjc1o1W+ZCE8QIU5WAoU4zf4DeNuRd6jxPEChUM 11 | DvBlbTmCKV+rLOBhurYr0ejggKtdcjmVhZ89jGZS/ostF0LfRnArHJkrb9JYmgff 12 | lGcv6Z/Ys5iKiTajaLpbEbw5PtDaJi7a3Yy4lSZt7GoZDnEwXgpGCUPq/scNaVbL 13 | 4jv/s0kCgYEA1Q90bavDdkYuQtOnFuPlIni44GjXVdBUrbDfcPzN5Voqhh2tOZz/ 14 | 4/uiWljSdUh21WdRj3KlGx0wEw6Qg3YxC916oGRMx9EU0mpHl1Fkv0SYj7ZET89z 15 | I6XIUX2h5wTmjTTc8taMVra0NH9CHzuz9bCDXT7t/y3RLeGPzqfljtcCgYEAxWlf 16 | V2RyCIO0dZ7ABfNPwYk7rWLozhxkVVwYOtiUhwVjv+XXe86iNGaBHIzdwN2mQ2+a 17 | n8QvoBgRk0gDiiO4pvQUN6okoYglqwyZW1DTqNBgfNh7ADXuM96k5phqbXws4VwW 18 | FRWMbY1unX4LwXf17kBwZdo1qh2bazElAgFhTPcCgYEAwEytqhrAVWzsbhZ4Ffnl 19 | IqLRQoJ98I8TDp24XlNeRqaGAPyiD4D7mLrSgzbt5TtdPil9fLpd+MX0UQ7xMiYo 20 | CGyDNGaywhqc73lLWnD1PIjeJb+9kkdLxZ3o2lxJF6jdqg9PaMJqcg1/Qm6lsGkD 21 | eToypqOYzZt91Cpk0IHLeIsCgYBIehhL4I/ROnF9oWwEg0Dr4DNtw9uPGHNpt2cZ 22 | 67wUGlF1+a90P/fjXyLV1Y3wqi/JoGbXc1K85zlEpnLOO7EmcoQdr7TFLVQPCZAg 23 | K3uaBe72xw/ZkvNCTeKi2qBwU9+yWXmuAfxNmFhdMBKm1CEReM0LR+Ld8wLFhwR8 24 | SP9tHwKBgET/wCASds1wvEat2BjjccSyTVwBRaqugwtI+EBN6wJSfpqWUIkcBVtC 25 | hI71RuP/HzFDHjFP+xCbnWbvHOSUzCBnW8I6KO0L5DOeYKHecwrvQ2CEkr3pA6RQ 26 | LIJlSft7D73FbrwbhNqQRZHSW0WO3M0wR+h26tDgR1UE9/j5hXdr 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /lib/worker-ready.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"), 2 | cluster = require("cluster"), 3 | util = require("util"), 4 | messageBus = require("./message-bus"), 5 | onWorkerStop = null; 6 | 7 | module.exports = exports = workerReady; 8 | 9 | function workerReady(options, forceIsWorker) { 10 | if (cluster.isMaster === true && forceIsWorker !== true) { 11 | return; // ignore if coming from master 12 | } 13 | 14 | if (cservice.locals.workerReady === true) { 15 | return; // ignore dup calls 16 | } 17 | 18 | if (options === false) { 19 | cservice.locals.workerReady = false; 20 | 21 | return; // do not continue 22 | } 23 | 24 | cservice.locals.workerReady = true; 25 | 26 | options = options || {}; 27 | 28 | if (options.servers) { 29 | require("./net-servers").add(options.servers); 30 | } 31 | 32 | onWorkerStop = options.onWorkerStop; 33 | 34 | process.on("message", onMessageFromMaster); 35 | 36 | // allow worker to inform the master when ready to speed up initialization 37 | cservice.processSafeSend(process, 38 | messageBus.createMessage("workerReady", { 39 | onStop: (typeof options.onWorkerStop === "function") 40 | }) 41 | ); 42 | } 43 | 44 | function onMessageFromMaster(msg) { 45 | if (!messageBus.isValidMessage(msg) || 46 | cservice.msgBus.processMessage(msg)) { 47 | return; 48 | } 49 | 50 | switch (msg.cservice.cmd) { 51 | case "onWorkerStop": 52 | cservice.netServers.close(function() { 53 | if (typeof onWorkerStop === "function") { 54 | // if custom handler is provided rely on that to 55 | // cleanup and exit process 56 | onWorkerStop(); 57 | } else { 58 | // otherwise we can exit now that net servers have exited gracefully 59 | process.exit(); 60 | } 61 | }); 62 | break; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /lib/commands/proxy.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../../cluster-service"); 2 | 3 | module.exports = function(evt, cb, cmd) { 4 | var versionStr, options; 5 | switch (cmd) { 6 | case "start": 7 | cservice.proxy.start({ configPath: arguments[3] }, cb); 8 | break; 9 | case "stop": 10 | cservice.proxy.stop(cb); 11 | break; 12 | case "version": 13 | versionStr = arguments[3]; 14 | options = {}; 15 | if (arguments[4]) { 16 | options.workerCount = parseInt(arguments[4]); 17 | } 18 | cservice.proxy.version(versionStr, options, cb); 19 | break; 20 | case "promote": 21 | versionStr = arguments[3]; 22 | options = {}; 23 | if (arguments[4]) { 24 | options.workerCount = parseInt(arguments[4]); 25 | } 26 | cservice.proxy.promote(versionStr, options, cb); 27 | break; 28 | case "info": 29 | cservice.proxy.info(cb); 30 | break; 31 | default: 32 | cb("Proxy command " + cmd + 33 | " not recognized. Try 'help proxy' for more info."); 34 | break; 35 | } 36 | }; 37 | 38 | module.exports.more = function(cb) { 39 | cb(null, { 40 | command: "proxy {cmd} {options}", 41 | info: "Perform a proxy operation.", 42 | cmd: "Available commands:", 43 | "* start {configPath}": "Start proxy", 44 | "* stop": "Stop proxy", 45 | "* version {version} {workerCount}": 46 | "Set a given version to desired worker count", 47 | "* promote {version} [workerCount]": "Promote a worker version", 48 | "* info": "Return proxy info, including a list of active versions", 49 | "options": "Available options:", 50 | "* configPath": "Path of proxy config file", 51 | "* version": "Worker version (path)", 52 | "* workerCount": "Desired number of workers" 53 | }); 54 | }; 55 | 56 | module.exports.control = function() { 57 | return "local"; 58 | }; 59 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cluster-service", 3 | "version": "2.1.4", 4 | "author": { 5 | "name": "Aaron Silvas", 6 | "email": "asilvas@godaddy.com" 7 | }, 8 | "description": "Turns your single process code into a fault-resilient multi-process service with built-in REST & CLI support", 9 | "main": "./cluster-service.js", 10 | "scripts": { 11 | "start": "node scripts/start.js", 12 | "lint": "npm run-script lint-src && npm run-script lint-test", 13 | "lint-src": "jshint bin lib cluster-service.js", 14 | "lint-test": "jshint --config .test-jshintrc test", 15 | "cover": "nyc mocha --ui bdd -R spec -t 5000", 16 | "test-devel": "mocha bdd -R spec -t 5000 test/*.js test/workers/*.js", 17 | "test": "npm run-script lint && npm run-script cover" 18 | }, 19 | "dependencies": { 20 | "async": "^3.2.2", 21 | "colors": "^1.4.0", 22 | "extend": "^3.0.2", 23 | "minimist": "^1.2.6" 24 | }, 25 | "devDependencies": { 26 | "jshint": "^2.12.0", 27 | "mocha": "^5.2.0", 28 | "nyc": "^15.1.0", 29 | "request": "^2.88.2", 30 | "sinon": "^7.5.0" 31 | }, 32 | "repository": { 33 | "type": "git", 34 | "url": "https://github.com/godaddy/node-cluster-service.git" 35 | }, 36 | "license": "MIT", 37 | "homepage": "https://github.com/godaddy/node-cluster-service", 38 | "bugs": { 39 | "url": "https://github.com/godaddy/node-cluster-service/issues" 40 | }, 41 | "bin": { 42 | "cluster-service": "./bin/cservice", 43 | "cservice": "./bin/cservice", 44 | "cserviced": "./bin/cserviced" 45 | }, 46 | "engines": { 47 | "node": ">=6" 48 | }, 49 | "keywords": [ 50 | "cluster", 51 | "service", 52 | "ha", 53 | "high availability", 54 | "cli", 55 | "remote access", 56 | "multi process", 57 | "master", 58 | "child", 59 | "process", 60 | "monitor", 61 | "monitoring", 62 | "continous integration", 63 | "healthcheck", 64 | "heartbeat", 65 | "health check", 66 | "heart beat", 67 | "REST", 68 | "resilient" 69 | ] 70 | } 71 | -------------------------------------------------------------------------------- /lib/cli.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"), 2 | util = require("util"), 3 | options = null; 4 | 5 | exports.init = function(o) { 6 | options = o; 7 | 8 | util.inspect.styles.name = "grey"; 9 | 10 | cservice.log( 11 | "CLI is now available. Enter 'help [enter]' for instructions.".info 12 | ); 13 | process.stdin.resume(); 14 | process.stdin.setEncoding('utf8'); 15 | 16 | process.stdin.on("data", onCommand); 17 | 18 | // wait momentarily before attaching CLI. allows workers a little time 19 | // to output as needed 20 | setTimeout(function() { 21 | process.stdout.write("cservice> ".cservice); 22 | }, 1000); 23 | }; 24 | 25 | exports.close = function() { 26 | try { 27 | process.stdin.pause(); 28 | } catch (ex) { 29 | } 30 | }; 31 | 32 | function onCommand(question) { 33 | if (cservice.locals.isBusy === true) { 34 | cservice.error("Busy... Try again after previous command returns."); 35 | return; 36 | } 37 | 38 | cservice.locals.isBusy = true; 39 | 40 | var args; 41 | question = question.replace(/[\r\n]/g, ""); 42 | args = require("./util").getArgsFromQuestion(question, " "); 43 | args = [args[0], onCallback].concat(args.slice(1)); 44 | 45 | if (!cservice.locals.events[args[0]]) { 46 | onCallback("Command " + args[0] + " not found. Try 'help'."); 47 | 48 | return; 49 | } 50 | 51 | try { 52 | cservice.trigger.apply(null, args); 53 | } catch (ex) { 54 | cservice.error( 55 | "Command Error " + args[0], 56 | util.inspect(ex, {depth: null}), ex.stack || new Error().stack 57 | ); 58 | onCallback(); 59 | } 60 | } 61 | 62 | function onCallback(err, result) { 63 | delete cservice.locals.reason; 64 | 65 | cservice.locals.isBusy = false; 66 | 67 | if (err) { 68 | cservice.error( 69 | "Error: ", 70 | err, 71 | err.stack 72 | ? util.inspect(err.stack, {depth: null, colors: true}) 73 | : "" 74 | ); 75 | } else if (result) { 76 | cservice.log(util.inspect(result, {depth: null, colors: true})); 77 | } 78 | 79 | //cservice.log("");//newline 80 | 81 | process.stdout.write("cservice> ".cservice); 82 | } 83 | -------------------------------------------------------------------------------- /lib/defaults.js: -------------------------------------------------------------------------------- 1 | var os = require("os"); 2 | 3 | module.exports = exports = { 4 | firstTime: true, 5 | events: {}, 6 | workers: {}, 7 | workerProcesses: {}, 8 | state: 0, // 0-not running, 1-starting, 2-running 9 | isBusy: false, 10 | isAttached: false, // attached to CLI over REST 11 | workerReady: undefined, 12 | restartOnFailure: true, 13 | net: { servers: {} }, 14 | proxy: { 15 | configPath: undefined, 16 | versionPath: undefined, 17 | options: { 18 | versionPath: undefined, 19 | versionHeader: "x-version", 20 | workerFilename: "worker.js", 21 | versionPorts: "11000-12000", 22 | nonDefaultWorkerCount: 1, 23 | nonDefaultWorkerIdleTime: 3600, 24 | bindings: [ 25 | /* 26 | { 27 | port: 80, 28 | workerCount: 2, 29 | redirect: 443 30 | }, 31 | { 32 | port: 443, 33 | workerCount: 2, 34 | tlsOptions: { 35 | key: '/my/cert.key', 36 | cert: '/my/cert.crt' 37 | } 38 | } 39 | */ 40 | ] 41 | }, 42 | versions: { 43 | /* 44 | 'versionStr': { 45 | port: 7112, 46 | isDefault: false, 47 | online: false 48 | } 49 | */ 50 | } 51 | }, 52 | options: { 53 | host: "localhost", 54 | port: 11987, 55 | accessKey: undefined, 56 | workers: undefined, 57 | workerCount: os.cpus().length, 58 | restartDelayMs: 100, 59 | restartConcurrencyRatio: 0.33, 60 | allowHttpGet: false, // useful for testing -- not safe for production use 61 | restartsPerMinute: 10, // not yet supported 62 | cli: false, 63 | silent: false, 64 | log: console.log, 65 | error: console.error, 66 | debug: console.debug, 67 | json: false, // output as JSON 68 | restartOnMemUsage: undefined, 69 | restartOnUpTime: undefined, 70 | commands: undefined, 71 | proxy: undefined, 72 | workerGid: undefined, 73 | workerUid: undefined, 74 | colors: { 75 | cservice: "grey", 76 | success: "green", 77 | error: "red", 78 | data: "cyan", 79 | warn: "yellow", 80 | info: "magenta", 81 | debug: "grey" 82 | } 83 | } 84 | }; 85 | -------------------------------------------------------------------------------- /lib/run.js: -------------------------------------------------------------------------------- 1 | var util = require("util"), 2 | http = require("http"), 3 | querystring = require("querystring"), 4 | options = null, 5 | cservice = require("../cluster-service"); 6 | 7 | exports.start = function(o, cb) { 8 | var cmd; 9 | options = o; 10 | 11 | cmd = options.run; 12 | delete options.run; 13 | 14 | run(cmd, cb); 15 | }; 16 | 17 | function run(question, cb) { 18 | var qs = querystring.stringify({ 19 | cmd: question, 20 | accessKey: options.accessKey 21 | }); 22 | var body = "", err; 23 | var url = "http://" 24 | + (options.host || "localhost") 25 | + ":" 26 | + (options.port || 11987) 27 | + "/cli" 28 | + "?" + qs 29 | ; 30 | cservice.log( 31 | "Running remote command: ".warn 32 | + url.replace(/accessKey=.*/i, "accessKey={ACCESS_KEY}").data 33 | ); 34 | http.request({ 35 | host: options.host || "localhost", 36 | port: options.port || 11987, 37 | path: "/cli?" + qs, 38 | method: "POST" 39 | } 40 | , function(res) { 41 | res.setEncoding('utf8'); 42 | res.on("data", function(chunk) { 43 | body += chunk; 44 | }); 45 | res.on("end", function() { 46 | if (res.statusCode !== 200 && body) { 47 | err = body; 48 | } 49 | if (err) { 50 | cservice.error("Error: ", err); 51 | body = {statusCode: res ? res.statusCode : "no response", error: err}; 52 | } else if ( 53 | typeof body === "string" 54 | && ( 55 | body.indexOf("{") === 0 56 | || body.indexOf("[") === 0 57 | ) 58 | ) { 59 | body = JSON.parse(body); // deserialize 60 | } 61 | if (options.json === true) { 62 | cservice.results(JSON.stringify(body)); 63 | } else { 64 | cservice.results(util.inspect(body, {depth: null, colors: true})); 65 | } 66 | 67 | if (cb) { 68 | cb(err, body); 69 | } 70 | }); 71 | } 72 | ).on( 73 | "error" 74 | , function(err) { 75 | body = err; 76 | 77 | if (options.json === true) { 78 | cservice.results(JSON.stringify(body)); 79 | } else { 80 | cservice.results(util.inspect(body, {depth: null, colors: true})); 81 | } 82 | 83 | if (cb) { 84 | cb(err, body); 85 | } 86 | } 87 | ) 88 | .end(); 89 | } 90 | -------------------------------------------------------------------------------- /lib/net-stats.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | 3 | module.exports = exports = netStats; 4 | 5 | var monitoring = false, statTimer; 6 | var net = { 7 | requests: 0, 8 | connections: 0, 9 | connectionsOpen: 0, 10 | avgRequests: 0, 11 | avgConnections: 0 12 | }; 13 | var stats = { 14 | lastCheck: new Date().getTime(), 15 | connections: 0, 16 | requests: 0 17 | }; 18 | 19 | function netStats(server) { 20 | if (monitoring === false) { 21 | monitoring = true; 22 | monitor(); // init monitor 23 | } 24 | 25 | server.on("connection", function(connection) { 26 | net.connections++; 27 | net.connectionsOpen++; 28 | connection.on("close", function() { 29 | net.connectionsOpen--; 30 | }); 31 | }); 32 | 33 | server.on("request", function() { 34 | net.requests++; 35 | }); 36 | } 37 | 38 | var STAT_FREQUENCY = 1000; 39 | var STAT_FACTOR = 3; 40 | 41 | // Why in its own monitor? Worker may reference different version of cservice 42 | // than the master, which is where these stats are tracked (to avoid blasting 43 | // messages when the data is not needed). 44 | // TODO: Move worker stats to its own class so this technique can be leveraged 45 | // for any type of stats in the future. 46 | function monitor() { 47 | statTimer = setInterval(statTracker, STAT_FREQUENCY); 48 | statTimer.unref(); 49 | 50 | process.on("message", function(msg) { 51 | if (!cservice.msgBus.isValidMessage(msg)) { 52 | return; // ignore 53 | } 54 | 55 | switch (msg.cservice.cmd) { 56 | case "netStats": 57 | cservice.processSafeSend(process, 58 | cservice.msgBus.createMessage("netStats", { 59 | netStats: net 60 | })); 61 | break; 62 | } 63 | }); 64 | } 65 | 66 | function statTracker() { 67 | var now = new Date().getTime(); 68 | var timeDiff = now - stats.lastCheck; 69 | var reqDiff = net.requests - stats.requests; 70 | var conDiff = net.connections - stats.connections; 71 | var reqPsec = (reqDiff / timeDiff) * 1000; 72 | var conPsec = (conDiff / timeDiff) * 1000; 73 | 74 | net.avgRequests = (reqPsec + (net.avgRequests * (STAT_FACTOR - 1))) 75 | / STAT_FACTOR 76 | ; 77 | net.avgConnections = (conPsec + (net.avgConnections * (STAT_FACTOR - 1))) 78 | / STAT_FACTOR 79 | ; 80 | 81 | // reset 82 | stats.lastCheck = now; 83 | stats.requests = net.requests; 84 | stats.connections = net.connections; 85 | } 86 | -------------------------------------------------------------------------------- /lib/commands.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"), 2 | path = require("path"), 3 | fs = require("fs"), 4 | cluster = require("cluster"); 5 | 6 | exports.on = on; 7 | exports.register = register; 8 | 9 | function on(eventName, cb, overwriteExisting) { 10 | var evt; 11 | var controls; 12 | if (cluster.isMaster === false) { 13 | // no action to take on workers -- convenience feature as to not 14 | // pollute master code 15 | return; 16 | } 17 | 18 | overwriteExisting = overwriteExisting || true; 19 | if (!overwriteExisting && eventName in cservice.locals.events) { 20 | return; // do not overwrite existing 21 | } 22 | 23 | evt = { 24 | name: eventName, 25 | service: cservice, 26 | locals: cservice.locals, 27 | cb: cb 28 | }; 29 | 30 | // Adding control for this eventName 31 | if (typeof cb.control === "function") { 32 | controls = {}; 33 | controls[eventName] = cb.control(); 34 | require("./control").addControls(controls); 35 | } 36 | 37 | // overwrite existing, if any 38 | cservice.locals.events[eventName] = evt; 39 | } 40 | 41 | function register(commands, overwriteExisting) { 42 | if (typeof commands === "string") { 43 | commands = commands.split(","); 44 | } else if (typeof commands !== "object") { 45 | throw new Error("Option 'commands' must be a comma-delimited " + 46 | "string or an array of strings."); 47 | } 48 | 49 | for (var i = 0; i < commands.length; i++) { 50 | var dir = commands[i]; 51 | if (dir.indexOf(":\\") !== 1 && dir.indexOf("/") !== 0) { 52 | // if not absolute, resolve path from cwd 53 | dir = path.resolve(process.cwd(), dir); 54 | } 55 | 56 | var files = fs.readdirSync(dir); 57 | for (var f = 0; f < files.length; f++) { 58 | var fn = path.resolve(dir, files[f]); 59 | var ext = path.extname(fn); 60 | if (ext !== ".js") 61 | continue; // only js files permitted 62 | var basename = path.basename(fn, ".js"); 63 | var mod = require(fn); // load command module 64 | if (typeof mod.id === "string") { 65 | basename = mod.id; // use module id if available 66 | } 67 | on(basename, mod, overwriteExisting); 68 | } 69 | } 70 | } 71 | 72 | if (cluster.isMaster === true && cservice.locals.firstTime === true) { 73 | cservice.locals.firstTime = false; 74 | 75 | var dir = path.dirname(module.filename); 76 | register(path.resolve(dir, "./commands"), false); 77 | } -------------------------------------------------------------------------------- /test/cmd-help.js: -------------------------------------------------------------------------------- 1 | /* jshint camelcase:false */ 2 | var cmd = require('../lib/commands/help'); 3 | var assert = require("assert"); 4 | var cservice = require("../cluster-service"); 5 | 6 | cservice.log = function() {}; 7 | if(cservice.isWorker){ 8 | it("WORKER", function(done) {}); 9 | } else { 10 | describe('[Help cmd]', function() { 11 | it('Start', function(done) { 12 | cservice.start( 13 | { 14 | workers: null, 15 | workerCount: 1, 16 | accessKey: "123", 17 | cli: false 18 | }, 19 | function() { 20 | assert.equal( 21 | cservice.workers.length, 22 | 0, 23 | "0 worker expected, but " + cservice.workers.length + " found" 24 | ); 25 | done(); 26 | } 27 | ); 28 | }); 29 | 30 | it('Help all', function(done) { 31 | cservice.trigger("help", function(err, data) { 32 | assert.equal(err, null); 33 | assert.equal( 34 | data.more, 35 | "Commands (Use 'help [command_name]' for more details)" 36 | ); 37 | assert.ok(data.commands); 38 | var invisible_commands = data.commands.filter( 39 | function(el) { 40 | return (el === "workerStart" || el === "workerExit"); 41 | } 42 | ); 43 | assert.equal(invisible_commands.length, 0, 44 | "Expected to find 0 invisible commands, but found " + 45 | invisible_commands.length 46 | ); 47 | done(); 48 | }); 49 | }); 50 | 51 | it('Help health', function(done) { 52 | cservice.trigger("help", function(err, data) { 53 | assert.equal(err, null); 54 | assert.equal( 55 | data.info, 56 | [ 57 | "Returns health of service. May be overidden by service to expose", 58 | "app-specific data." 59 | ].join(' ') 60 | ); 61 | assert.equal(data.command, "health"); 62 | done(); 63 | }, "health"); 64 | }); 65 | 66 | it('more', function(done) { 67 | var callback = function(nullObj, obj) { 68 | assert.equal(nullObj, null); 69 | assert.equal(obj.command_name, "Optional if you want extended help"); 70 | assert.equal(obj.command, "help [command_name]"); 71 | done(); 72 | }; 73 | 74 | cmd.more(callback); 75 | }); 76 | 77 | it('Stop', function(done) { 78 | cservice.stop(30000, function() { 79 | assert.equal( 80 | cservice.workers.length, 81 | 0, 82 | "0 workers expected, but " + cservice.workers.length + " found" 83 | ); 84 | done(); 85 | }); 86 | }); 87 | }); 88 | } -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text eol=lf 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /lib/commands/workers.js: -------------------------------------------------------------------------------- 1 | var async = require("async"); 2 | var cservice = require("../../cluster-service"); 3 | 4 | module.exports = function(evt, cb, cmd) { 5 | processDetails(evt.service.workers, function(err, workers) { 6 | var ret = {}; 7 | cmd = cmd || "simple"; 8 | switch (cmd) { 9 | case "details": 10 | ret.workers = workers; 11 | break; 12 | default: 13 | ret.workers = workerSummary(workers); 14 | break; 15 | } 16 | 17 | cb(err, ret); 18 | }); 19 | }; 20 | 21 | module.exports.more = function(cb) { 22 | cb(null, { 23 | command: "workers [simple|details]", 24 | info: "Returns list of active worker processes.", 25 | "simple|details": "Defaults to 'simple'.", 26 | "* simple": "Simple overview of running workers.", 27 | "* details": "Full details of running workers." 28 | }); 29 | }; 30 | 31 | function processDetails(workers, cb) { 32 | var tasks = [], i, w; 33 | 34 | for (i = 0; i < workers.length; i++) { 35 | w = workers[i]; 36 | tasks.push(getProcessDetails(w)); 37 | } 38 | async.parallel(tasks, function(err, results) { 39 | cb(err, workers); 40 | }); 41 | } 42 | 43 | function getProcessDetails(worker) { 44 | return function(cb) { 45 | var timer, msgCb, processDetails, netStats; 46 | msgCb = function (msg) { 47 | if (msg && msg.cservice.processDetails) { 48 | processDetails = msg.cservice.processDetails; 49 | } 50 | if (msg && msg.cservice.netStats) { 51 | netStats = msg.cservice.netStats; 52 | } 53 | if (processDetails && netStats) { 54 | clearTimeout(timer); 55 | worker.removeListener("message", msgCb); 56 | worker.processDetails = processDetails; 57 | processDetails.net = netStats; 58 | cb(null, worker); 59 | } 60 | }; 61 | timer = setTimeout(function() { 62 | worker.removeListener("message", msgCb); 63 | if (processDetails) { // net stats not required for success 64 | worker.processDetails = processDetails; 65 | cb(null, worker); 66 | } else { 67 | cb("getProcessDetails TIMEOUT"); 68 | } 69 | }, 1000); 70 | 71 | worker.on("message", msgCb); 72 | cservice.processSafeSend(worker.process, 73 | cservice.msgBus.createMessage("processDetails") 74 | ); 75 | cservice.processSafeSend(worker.process, 76 | cservice.msgBus.createMessage("netStats") 77 | ); 78 | }; 79 | } 80 | 81 | function workerSummary(workers) { 82 | var ret = [], i, w; 83 | 84 | for (i = 0; i < workers.length; i++) { 85 | w = workers[i]; 86 | ret.push({ 87 | id: w.id, 88 | pid: w.pid, 89 | state: w.state, 90 | worker: w.cservice.worker, 91 | cwd: w.cservice.cwd, 92 | process: w.processDetails 93 | }); 94 | } 95 | 96 | return ret; 97 | } 98 | -------------------------------------------------------------------------------- /lib/start.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"), 2 | cluster = require("cluster"), 3 | fs = require("fs"), 4 | path = require("path"), 5 | colors = require("colors"), 6 | util = require("util"); 7 | 8 | module.exports = exports = start; 9 | 10 | start.prepArgs = prepArgs; 11 | 12 | function start(options, masterCb) { 13 | var argv; 14 | if (cluster.isWorker === true) { 15 | // ignore starts if not master. do NOT invoke masterCb, as that is 16 | // reserved for master callback 17 | 18 | return; 19 | } 20 | 21 | if (arguments.length === 0) { 22 | argv = require("minimist")(process.argv.slice(2)); 23 | 24 | options = argv; // use command-line arguments instead 25 | if (!("cli" in options)) { 26 | options.cli = true; // auto-enable cli if run from command-line 27 | } 28 | prepArgs(options); 29 | masterCb = masterCallback; 30 | } 31 | 32 | options = options || {}; 33 | if ("config" in options) { 34 | // only extend with config, do not overwrite command-line options 35 | var fileOptions = JSON.parse(fs.readFileSync(options.config)); 36 | options = util._extend(fileOptions, options); 37 | } 38 | cservice.locals.options = util._extend(cservice.locals.options, options); 39 | if ("workers" in options) { // overwrite workers if provided 40 | cservice.locals.options.workers = options.workers; 41 | } 42 | options = cservice.locals.options; 43 | if (typeof options.workers === "string") { 44 | options.workers = { 45 | main: { 46 | worker: options.workers 47 | } 48 | }; 49 | } 50 | if (options.commands) { 51 | cservice.registerCommands(options.commands); 52 | } 53 | 54 | colors.setTheme(options.colors); 55 | 56 | require("./legacy"); 57 | 58 | if (options.run) { 59 | require("./run").start(options, function(err, result) { 60 | if (masterCb && masterCb(err, result) === false) { 61 | return; // do not exit if cb returns false 62 | } 63 | process.exit(0); // graceful exit 64 | }); 65 | } else { 66 | require("./master").start(options, masterCb); 67 | } 68 | } 69 | 70 | function masterCallback(err) { 71 | if (err) { 72 | cservice.error(err); 73 | cservice.log("Startup failed, exiting...".warn); 74 | process.exit(0); // graceful exit 75 | } 76 | } 77 | 78 | function prepArgs(options) { 79 | var ext; 80 | if (options._ && options._.length > 0) { 81 | ext = path.extname(options._[0]).toLowerCase(); 82 | if (ext === ".js") { // if js file, use as worker 83 | options.workers = options._[0]; 84 | } else if (ext === ".json") { // if json file, use as config 85 | options.config = options._[0]; 86 | } else { // otherwise assume it is a command to execute 87 | options.run = options._[0]; 88 | if (options.json === true) { 89 | options.cli = false; 90 | } 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /lib/http-client.js: -------------------------------------------------------------------------------- 1 | var util = require("util"), 2 | http = require("http"), 3 | querystring = require("querystring"), 4 | options = null, 5 | cservice = require("../cluster-service"); 6 | 7 | exports.init = function(o) { 8 | options = o; 9 | 10 | cservice.log([ 11 | "Service already running. Attached CLI to master service.", 12 | "Enter 'help [enter]' for help." 13 | ] 14 | .join(' ') 15 | .info 16 | ); 17 | 18 | if (!options || options.silentMode !== true) { 19 | process.stdin.resume(); 20 | process.stdin.setEncoding('utf8'); 21 | process.stdin.on("data", onCommand); 22 | process.stdout.write("cservice> ".cservice); 23 | } 24 | }; 25 | 26 | exports.execute = onCommand; 27 | 28 | function onCommand(question, cb) { 29 | var split, qs, url, body, err; 30 | question = question.replace(/[\r\n]/g, ""); 31 | split = question.split(" "); 32 | if (split[0] === "exit") { 33 | cservice.log("Exiting CLI ONLY.".yellow); 34 | process.kill(process.pid, "SIGKILL"); // exit by force 35 | return; 36 | } 37 | qs = querystring.stringify({ 38 | cmd: question, 39 | accessKey: options.accessKey 40 | }); 41 | url = "http://" 42 | + (options.host || "localhost") 43 | + ":" 44 | + (options.port || 11987) 45 | + "/cli" 46 | + "?" 47 | + qs; 48 | cservice.log( 49 | "Running remote command: ".warn 50 | + url.replace(/accessKey=.*/i, "accessKey={ACCESS_KEY}").data 51 | ); 52 | body = ""; 53 | http.request( 54 | { 55 | host: options.host || "localhost", 56 | port: options.port || 11987, 57 | path: "/cli?" + qs, 58 | method: "POST" 59 | } 60 | , function(res) { 61 | res.setEncoding('utf8'); 62 | res.on("data", function(chunk) { 63 | body += chunk; 64 | }); 65 | res.on("end", function() { 66 | if (res.statusCode !== 200 && body) { 67 | err = body; 68 | } 69 | onCallback(err, body, res, cb); 70 | }); 71 | } 72 | ).on("error", function(err) { 73 | body = err; 74 | onCallback(err, body, null, cb); 75 | } 76 | ).end(); 77 | } 78 | 79 | function onCallback(err, result, res, cb) { 80 | if (err) { 81 | cservice.error("Error: ", err); 82 | result = {statusCode: res ? res.statusCode : "unknown", error: err}; 83 | } else if (result) { 84 | if ( 85 | typeof result === "string" 86 | && (result.indexOf("{") === 0 || result.indexOf("[") === 0) 87 | ) { 88 | result = JSON.parse(result); // deserialize 89 | } 90 | } 91 | cservice.log(util.inspect(result, {depth: null, colors: true})); 92 | 93 | if (!options || options.silentMode !== true) { 94 | //cservice.log("");//newline 95 | process.stdout.write("cservice> ".cservice); 96 | } 97 | 98 | if(cb){ 99 | cb(err, result); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /cluster-service.js: -------------------------------------------------------------------------------- 1 | var cluster = require("cluster"); 2 | var colors = require("colors"); 3 | if (!('cservice' in global)) { 4 | global.cservice = { locals: require("./lib/defaults") }; 5 | } 6 | 7 | module.exports = exports; 8 | 9 | exports.debug = require("./lib/util").debug; 10 | exports.log = require("./lib/util").log; 11 | exports.error = require("./lib/util").error; 12 | exports.results = require("./lib/util").results; 13 | exports.processSafeSend = require("./lib/util").processSafeSend; 14 | exports.msgBus = require("./lib/message-bus"); 15 | 16 | exports.workerReady = require("./lib/worker-ready"); 17 | 18 | Object.defineProperty(exports, "workers", { 19 | get: require("./lib/workers").get 20 | }); 21 | 22 | Object.defineProperty(exports, "isMaster", { 23 | get: function() { 24 | return cluster.isMaster; 25 | } 26 | }); 27 | 28 | Object.defineProperty(exports, "isWorker", { 29 | get: function() { 30 | return cluster.isWorker; 31 | } 32 | }); 33 | 34 | Object.defineProperty(exports, "options", { 35 | get: function() { 36 | return global.cservice.locals.options; 37 | } 38 | }); 39 | 40 | Object.defineProperty(exports, "locals", { 41 | get: function() { 42 | return global.cservice.locals; 43 | } 44 | }); 45 | 46 | if (cluster.isMaster === true) { 47 | exports.control = require("./lib/control").addControls; 48 | exports.stop = require("./lib/stop"); 49 | exports.newWorker = require("./lib/new-worker"); 50 | exports.on = require("./lib/commands").on; 51 | exports.registerCommands = require("./lib/commands").register; 52 | } else { 53 | exports.on = function() { }; 54 | exports.registerCommands = function() { }; 55 | } 56 | 57 | exports.trigger = require("./lib/trigger"); 58 | exports.start = require("./lib/start"); 59 | exports.netServers = require("./lib/net-servers"); 60 | exports.netStats = require("./lib/net-stats"); 61 | exports.proxy = require('./lib/proxy'); 62 | 63 | if ( 64 | cluster.isWorker === true 65 | && typeof (cluster.worker.module) === "undefined" 66 | ){ 67 | // intermediate state to prevent 2nd call while async in progress 68 | cluster.worker.module = {}; 69 | cluster.worker.env = process.env; 70 | 71 | var workers = require("./lib/workers"); 72 | 73 | workers.demote(); 74 | 75 | // load the worker if not already loaded 76 | // async, in case worker loads cluster-service, we need to return before 77 | // it's avail 78 | setImmediate(function() { 79 | cluster.worker.module = require(process.env.worker); 80 | if (global.cservice.locals.workerReady === undefined 81 | && process.env.ready.toString() === "false") { 82 | // if workerReady not invoked explicitly, we'll track it automatically 83 | exports.workerReady(false); 84 | exports.netServers.waitForReady(function() { 85 | exports.workerReady(); // NOW we're ready 86 | }); 87 | } 88 | }); 89 | 90 | // start worker monitor to establish two-way relationship with master 91 | workers.monitor(); 92 | } 93 | -------------------------------------------------------------------------------- /lib/commands/shutdown.js: -------------------------------------------------------------------------------- 1 | /* jshint loopfunc:true */ 2 | 3 | var util = require("util"), 4 | cservice = require("../../cluster-service"); 5 | 6 | module.exports = function(evt, cb, cmd, options) { 7 | var pid = parseInt(cmd); 8 | var workersToKill; 9 | var exiting; 10 | 11 | options = options || {}; 12 | options.timeout = parseInt(options.timeout) || 60000; 13 | if (cmd !== "all" && !pid) { 14 | cb("Invalid request. Try help shutdown"); 15 | return; 16 | } 17 | 18 | evt.locals.reason = "shutdown"; 19 | 20 | workersToKill = 0; 21 | 22 | exiting = false; 23 | evt.service.workers.forEach(function(worker){ 24 | if (pid && worker.process.pid !== pid) { 25 | return; // cannot kill external processes 26 | } 27 | 28 | exiting = true; 29 | workersToKill++; 30 | 31 | var killTimeout = options.timeout > 0 32 | ? setTimeout(getKiller(worker), options.timeout) 33 | : null; 34 | worker.on("exit", getExitHandler(evt, worker, killTimeout, function() { 35 | workersToKill--; 36 | if (workersToKill === 0) { 37 | // no workers remain 38 | if (evt.service.workers.length === 0) { 39 | evt.locals.reason = "kill"; 40 | cservice.log("All workers shutdown. Exiting...".warn); 41 | evt.service.stop(options.timeout, cb); 42 | } else { 43 | cb(null, "Worker shutdown"); // DONE 44 | } 45 | } 46 | })); 47 | 48 | require("../workers").exitGracefully(worker); 49 | }); 50 | 51 | if (exiting === false) { 52 | if (evt.service.workers.length === 0) { 53 | evt.locals.reason = "kill"; 54 | cservice.log("All workers shutdown. Exiting..."); 55 | evt.service.stop(options.timeout, cb); 56 | } else { 57 | cb("No workers were shutdown"); 58 | } 59 | } else { 60 | cservice.log( 61 | "Killing workers... timeout: ".warn + 62 | (options.timeout || 0).toString().info 63 | ); 64 | } 65 | }; 66 | 67 | module.exports.more = function(cb) { 68 | cb(null, { 69 | info: [ 70 | "Gracefully shutdown service, waiting up to timeout before terminating", 71 | "workers." 72 | ].join(' '), 73 | command: "shutdown all|pid { \"option1\": \"value\" }", 74 | "all|pid": [ 75 | "Required. 'all' to force shutdown of all workers, otherwise the pid of", 76 | "the specific worker to shutdown" 77 | ].join(' '), 78 | "options": "An object of options.", 79 | "* timeout": [ 80 | "Timeout, in milliseconds, before terminating workers. 0 for infinite", 81 | "wait." 82 | ].join(' ') 83 | }); 84 | }; 85 | 86 | module.exports.control = function() { 87 | return "local"; 88 | }; 89 | 90 | function getKiller(worker) { 91 | return function() { 92 | worker.kill("SIGKILL"); // go get'em, killer 93 | }; 94 | } 95 | 96 | function getExitHandler(evt, worker, killer, cb) { 97 | return function() { 98 | if (killer) { 99 | clearTimeout(killer); 100 | } 101 | 102 | cb(); 103 | }; 104 | } -------------------------------------------------------------------------------- /lib/message-bus.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | var util = require("util"); 3 | var cluster = require("cluster"); 4 | var async = require("async"); 5 | 6 | module.exports = { 7 | createMessage: createMessage, 8 | sendMessage: sendMessage, 9 | isValidMessage: isValidMessage, 10 | respondToMessage: respondToMessage, 11 | processMessage: processMessage 12 | }; 13 | 14 | var waiters = {}; 15 | var waiterId = 0; 16 | 17 | function isValidMessage(msg) { 18 | if (!msg || !msg.cservice || !msg.cservice.cmd) { 19 | return false; // ignore invalid cluster-service messages 20 | } 21 | 22 | return true; 23 | } 24 | 25 | function createMessage(cmd, options) { 26 | var msg = { 27 | cservice: util._extend({}, options || {}) 28 | }; 29 | msg.cservice.cmd = cmd; 30 | return msg; 31 | } 32 | 33 | function sendMessage(cmd, options, filter, cb) { 34 | var msg = createMessage(cmd, options); 35 | 36 | if (cluster.isWorker === true) { 37 | createWaiter(msg, process, cb); 38 | // send to master and wait for response 39 | return cservice.processSafeSend(process, msg); 40 | } 41 | 42 | var workers = cluster.workers; 43 | if (typeof filter === "function") { 44 | // filter as directed 45 | workers = workers.filter(filter); 46 | } 47 | 48 | var tasks = []; 49 | 50 | workers.forEach(function(worker) { 51 | tasks.push(createWaiterTask(msg, worker.process, cb)); 52 | }); 53 | 54 | // process worker messages, but callback only once 55 | async.parallel(tasks, function (err, data) { 56 | if (typeof cb === "function") { 57 | cb(err, data); 58 | } 59 | }); 60 | } 61 | 62 | function createWaiter(msg, process, cb) { 63 | if (typeof cb !== "function") { 64 | return; 65 | } 66 | 67 | msg.waiterId = "id" + (++waiterId); 68 | var waiter = waiters[msg.waiterId] = { 69 | msg: msg, 70 | process: process, 71 | cb: cb 72 | }; 73 | 74 | var cleanupCheck = setTimeout(function cleanup() { 75 | if (waiters[msg.waiterId]) { 76 | delete waiters[msg.waiterId]; 77 | cb(new Error('Timed out waiting in message bus')); 78 | } 79 | }, 5000); 80 | // If process exits, no need to clean up anymore 81 | cleanupCheck.unref(); 82 | 83 | return waiter; 84 | } 85 | 86 | function createWaiterTask(msg, process) { 87 | return function(asyncCb) { 88 | var waiter = createWaiter(msg, process, asyncCb); 89 | cservice.processSafeSend(process, msg); 90 | if (!waiter) { 91 | // if no waiter, cb immediately 92 | asyncCb(); 93 | } 94 | }; 95 | } 96 | 97 | function respondToMessage(msg, process, error, response) { 98 | msg.error = error; 99 | msg.response = response; 100 | cservice.processSafeSend(process, msg); 101 | } 102 | 103 | function processMessage(msg) { 104 | var waiter = waiters[msg.waiterId]; 105 | if (!waiter) { 106 | return false; 107 | } 108 | 109 | delete waiters[msg.waiterId]; 110 | 111 | waiter.cb(msg.error, msg.response); 112 | 113 | return true; 114 | } 115 | -------------------------------------------------------------------------------- /lib/commands/start.js: -------------------------------------------------------------------------------- 1 | var async = require("async"), 2 | util = require("util"), 3 | cservice = require("../../cluster-service"); 4 | 5 | module.exports = function(evt, cb, workerPath, options) { 6 | var tasks; 7 | var i; 8 | 9 | options = options || {}; 10 | options.cwd = options.cwd || process.cwd(); 11 | options.count = parseInt(options.count) || 1; 12 | options.timeout = parseInt(options.timeout) || 60000; 13 | options.worker = workerPath; 14 | if (typeof workerPath !== "string" || options.count < 1) { 15 | cb("Invalid request. Try help start"); 16 | return; 17 | } 18 | 19 | evt.locals.reason = "start"; 20 | var originalAutoRestart = evt.locals.restartOnFailure; 21 | evt.locals.restartOnFailure = false; 22 | 23 | tasks = []; 24 | 25 | cservice.log("Starting workers... timeout: " + (options.timeout || 0)); 26 | 27 | for (i = 0; i < options.count; i++) { 28 | tasks.push(getTask(evt, options)); 29 | } 30 | 31 | async.series(tasks, function(err) { 32 | evt.locals.restartOnFailure = originalAutoRestart; // restore 33 | 34 | if (err) { 35 | cb(err); 36 | } else { 37 | cb(null, tasks.length + " workers started successfully"); 38 | } 39 | }); 40 | }; 41 | 42 | module.exports.more = function(cb) { 43 | cb(null, { 44 | info: "Gracefully start service, one worker at a time.", 45 | command: "start workerPath { \"option1\": \"value\" }", 46 | "workerPath": [ 47 | "Path of worker file (i.e. /workers/worker) to start, absolute path, or", 48 | "relative to cwd." 49 | ].join(' '), 50 | "options": "An object of options.", 51 | "* cwd": [ 52 | "Path to set as the current working directory. If not provided, existing", 53 | "cwd will be used." 54 | ].join(' '), 55 | "* count": "The number of workers to start, or 1 if not specified.", 56 | "* timeout": [ 57 | "Timeout, in milliseconds, before terminating replaced workers. 0 for", 58 | "infinite wait." 59 | ].join(' '), 60 | "* ready": 61 | "If false, will wait for workerReady event before assuming success." 62 | }); 63 | }; 64 | 65 | function getTask(evt, options) { 66 | return function(cb) { 67 | var pendingWorker; 68 | 69 | // kill new worker if takes too long 70 | var startTimeout = null; 71 | var isWorkerTerminated = false; 72 | if (options.timeout > 0) { // start timeout if specified 73 | startTimeout = setTimeout(function() { 74 | if (!pendingWorker) 75 | return; 76 | isWorkerTerminated = true; 77 | pendingWorker.on('exit', function () { 78 | cb("timed out"); 79 | }); 80 | pendingWorker.kill("SIGKILL"); // go get'em, killer 81 | }, options.timeout); 82 | startTimeout.unref(); 83 | } 84 | 85 | // lets start new worker 86 | pendingWorker = evt.service.newWorker(options, function(err) { 87 | pendingWorker = null; 88 | 89 | if (startTimeout) { // timeout no longer needed 90 | clearTimeout(startTimeout); 91 | } 92 | 93 | if (!isWorkerTerminated) { 94 | cb(err); 95 | } 96 | }); 97 | }; 98 | } 99 | -------------------------------------------------------------------------------- /test/workers.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | var assert = require("assert"); 3 | var sinon = require("sinon"); 4 | var stats = { 5 | onWorkerReady: 0, 6 | onWorkerStop: 0 7 | }; 8 | var workerStopped = 0; 9 | var workerReady = require("../lib/worker-ready"); 10 | 11 | cservice.log = function() {}; 12 | if(cservice.isWorker){ 13 | it("WORKER", function(done) {}); 14 | } else { 15 | describe('[Works]', function() { 16 | before(function(done) { 17 | process.send = fakeSend; 18 | 19 | done(); 20 | }); 21 | 22 | after(function() { 23 | delete process.send; 24 | }); 25 | 26 | it('workerReady.isWorker', function(done) { 27 | workerReady({onWorkerStop: onWorkerStop}, true); 28 | assert.equal( 29 | stats.onWorkerReady, 30 | 1, 31 | "1 onWorkerReady expected, but " + stats.onWorkerReady + " detected" 32 | ); 33 | workerReady({onWorkerStop: onWorkerStop}, true); 34 | assert.equal( 35 | stats.onWorkerReady, 36 | 1, 37 | "1 onWorkerReady expected, but " + stats.onWorkerReady + " detected" 38 | ); 39 | done(); 40 | }); 41 | 42 | it('workerReady.ifMaster', function(done) { 43 | workerReady({onWorkerStop: onWorkerStop}); 44 | assert.equal( 45 | stats.onWorkerReady, 46 | 1, 47 | "1 onWorkerReady expected, but " + stats.onWorkerReady + " detected" 48 | ); 49 | done(); 50 | }); 51 | 52 | it('onWorkerStop', function(done) { 53 | process.emit("message", {cservice: {cmd: 'onWorkerStop'}}); 54 | assert.equal( 55 | stats.onWorkerStop, 56 | 1, 57 | "1 onWorkerStop expected, but " + stats.onWorkerStop + " detected" 58 | ); 59 | done(); 60 | }); 61 | 62 | it('BAD onWorkerStop', function(done) { 63 | process.emit("message", {cservice: {}}); 64 | assert.equal( 65 | stats.onWorkerStop, 66 | 1, 67 | "1 onWorkerStop expected, but " + stats.onWorkerStop + " detected" 68 | ); 69 | done(); 70 | }); 71 | 72 | describe("#send", function(){ 73 | var original; 74 | 75 | beforeEach(function(){ 76 | original = []; 77 | cservice.workers.map(function(worker){ 78 | var stub = sinon.stub(); 79 | original.push(worker); 80 | stub.pid = worker.pid; 81 | return stub; 82 | }); 83 | }); 84 | 85 | afterEach(function(){ 86 | cservice.workers.map(original.shift); 87 | }); 88 | 89 | it("sends the message to all workers", function() { 90 | var workersCalled = 0; 91 | //act 92 | cservice.workers.send({boo:true}); 93 | //assert 94 | cservice.workers.map(function(stub){ 95 | sinon.calledWith(stub, sinon.match({boo:true})); 96 | workersCalled+=1; 97 | }); 98 | 99 | assert.equal(workersCalled, original.length); 100 | }); 101 | }); 102 | }); 103 | } 104 | 105 | function fakeSend(o) { 106 | stats.onWorkerReady++; 107 | } 108 | 109 | function onWorkerStop() { 110 | stats.onWorkerStop++; 111 | } -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## v2.1.4 - 9/21/2020 2 | 3 | ### Fixes: 4 | 5 | - Dependency updates. 6 | 7 | ## v2.1.1 - 5/30/2018 8 | 9 | ### Fixes: 10 | 11 | - Memory leak in message bus (thanks @jakepusateri !)- #78 12 | 13 | ## v2.1.0 - 9/1/2016 14 | 15 | ### Breaking: 16 | 17 | - Updated to require Node v6 or later. 18 | 19 | ## v2.0.0 - 9/1/2016 20 | 21 | ### Features: 22 | 23 | - High worker restart concurrency to speed up upgrades/restarts via `restartConcurrencyRatio` 24 | - Proxy support with dynamic app versioning 25 | - Central proxy config file, promotions persisted to disk to survive service restarts 26 | - Isolated worker processes per version 27 | - On demand versioning via `x-version` header 28 | - Version promotion brings up worker count for desired version and brings down worker count of old version 29 | - Worker process downgrade from root via `workerGid` & `workerUid` 30 | - Backward compatibility with `v1.x`! Who does that with a major release?! 31 | 32 | ### Enhancements: 33 | 34 | - Refactored communication between processes 35 | - Worker processes may now trigger commands (same) and wait for response (new). 36 | The new message bus enables RPC and other flows like this between processes 37 | 38 | ### Fixes: 39 | 40 | - Worker process.send crash 41 | 42 | ## v1.0.0 - 5/8/2014 43 | 44 | ### Features: 45 | 46 | - #56. Ability to restart web server gracefully under load with no code 47 | - No code required to enable net statistics 48 | 49 | ### Fixes: 50 | 51 | - Prevent additional commands in CLI while in progress to resolve state issues 52 | - Handle failed `start` command gracefully 53 | - Handle failed `upgrade` command gracefully 54 | - Fixed `start` command from preventing failed workers to auto-restart 55 | - Fixed `restart` command from preventing failed workers to auto-restart 56 | - Further streamlined management of state across multiple cservice instances 57 | - Allow numeric accessKey's 58 | - Fix crash if no command provided in REST call 59 | - #57. Cannot use '--run' and '--config' together 60 | - #58. Custom events from master & workers 61 | 62 | ## v0.10.0 - 5/2/2014 63 | 64 | ### Legacy: 65 | 66 | - Dropped support for 'ready' flag, was causing support/complexity issues 67 | 68 | ### Features: 69 | 70 | - Ability to start without worker 71 | - Ability to support multiple versions of cservice through shared state 72 | - Added example workers to `examples/` 73 | 74 | ## v0.9.0 - 2/17/2014 75 | 76 | ### Features: 77 | 78 | - #51. Lockdown "help" command to "local" by default 79 | - #49. Support for multiple keys within 'accessKey' option. (Undocumented,experimental) 80 | - #54. Allow REST command to originate from body 81 | 82 | ### Fixes: 83 | 84 | - #46. Expect API response even on "shutdown" or "exit" 85 | - #52. processDetails not returned during message floods 86 | - #53. Net Statistics doesn't work if using multiple instances of cservice 87 | 88 | ## v0.8.0 - 1/13/2014 89 | 90 | ### Features: 91 | 92 | - #40. Net statistics support 93 | 94 | ## v0.7.0 - 1/3/2014 95 | 96 | ### Features: 97 | 98 | - #39. Support for hidden(internal) commands 99 | - #38. Option 'commands' support 100 | - #37. Disable cli by default, unless run from command-line 101 | - #30. Smarter async workerReady logic 102 | -------------------------------------------------------------------------------- /test/start-restart.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | var assert = require("assert"); 3 | 4 | cservice.log = function() {}; 5 | 6 | if(cservice.isWorker){ 7 | it("WORKER", function(done) {}); 8 | } else { 9 | describe('[Restart]', function() { 10 | it('Start workers', function(done) { 11 | assert.equal( 12 | cservice.workers.length, 13 | 0, 14 | "0 workers expected, but " + cservice.workers.length + " found" 15 | ); 16 | cservice.start( 17 | { 18 | workers: { 19 | basic2: { 20 | worker: "./test/workers/basic2", 21 | count: 2, 22 | ready: false 23 | } 24 | }, 25 | accessKey: "123", 26 | cli: false 27 | }, 28 | function() { 29 | assert.equal( 30 | cservice.workers.length, 31 | 2, 32 | "2 workers expected, but " + cservice.workers.length + " found" 33 | ); 34 | done(); 35 | } 36 | ); 37 | }); 38 | 39 | it('Bad input #1', function(done) { 40 | cservice.trigger("restart", function(err) { 41 | assert.equal(err, "Invalid request. Try help restart"); 42 | done(); 43 | }, "GG" 44 | ); 45 | }); 46 | 47 | it('Bad input #2', function(done) { 48 | cservice.trigger("restart", function(err) { 49 | assert.equal(err, "Invalid request. Try help restart"); 50 | done(); 51 | }, 0 52 | ); 53 | }); 54 | 55 | it('Restart without timeout', function(done) { 56 | cservice.trigger("restart", function() { 57 | assert.equal( 58 | cservice.workers.length, 59 | 2, 60 | "2 workers expected, but " + cservice.workers.length + " found" 61 | ); 62 | done(); 63 | }, "all", {timeout: 30000} // with timeout 64 | ); 65 | }); 66 | 67 | it('Restart with timeout', function(done) { 68 | cservice.trigger("restart", function(err) { 69 | assert.equal(err, "timed out"); 70 | setTimeout(function() { 71 | assert.equal( 72 | cservice.workers.length, 73 | 2, 74 | "2 workers expected, but " + cservice.workers.length + " found" 75 | ); 76 | done(); 77 | }, 1000); 78 | }, "all", {timeout: 1} // with timeout 79 | ); 80 | }); 81 | 82 | it('Stop workers', function(done) { 83 | cservice.stop(30000, function(err, msg) { 84 | assert.ok(!err, 'Error: ' + err); 85 | assert.equal( 86 | cservice.workers.length, 87 | 0, 88 | "0 workers expected, but " 89 | + cservice.workers.length 90 | + " found. Message: " + msg 91 | ); 92 | done(); 93 | }); 94 | }); 95 | 96 | it('Restart with no workers', function(done) { 97 | cservice.trigger("restart", function(err) { 98 | assert.equal(err, "No workers to restart"); 99 | assert.equal( 100 | cservice.workers.length, 101 | 0, 102 | "0 workers expected, but " + cservice.workers.length + " found" 103 | ); 104 | done(); 105 | }, "all"); 106 | }); 107 | }); 108 | } -------------------------------------------------------------------------------- /lib/workers.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"), 2 | cluster = require("cluster"); 3 | 4 | exports.get = get; 5 | exports.monitor = monitor; 6 | exports.getByPID = getByPID; 7 | exports.getByPIDFromCache = getByPIDFromCache; 8 | exports.exitGracefully = exitGracefully; 9 | exports.demote = demote; 10 | 11 | function get() { 12 | var workers = []; 13 | var cworkers = cluster.workers; 14 | var k; 15 | var worker; 16 | for (k in cworkers) { 17 | worker = cworkers[k]; 18 | if ((!worker.isDead || !worker.isDead()) 19 | && worker.exitedAfterDisconnect !== true 20 | && worker.state !== "none") { 21 | worker.pid = worker.process.pid; 22 | workers.push(worker); 23 | } 24 | } 25 | 26 | workers.send = send; 27 | 28 | return workers; 29 | } 30 | 31 | // i hate O(N) lookups, but not hit hard enough to worry about optimizing at 32 | // this point. freshness is more important 33 | function getByPID(pid) { 34 | var workers = get(); 35 | var i; 36 | var worker; 37 | 38 | for (i = 0; i < workers.length; i++) { 39 | worker = workers[i]; 40 | if (worker.pid === pid) { 41 | return worker; 42 | } 43 | } 44 | // else return undefined 45 | } 46 | 47 | function getByPIDFromCache(pid) { 48 | return cservice.locals.workers[pid]; 49 | } 50 | 51 | function monitor() { 52 | process.on("message", function(msg) { 53 | if (!cservice.msgBus.isValidMessage(msg)) { 54 | return; // end 55 | } 56 | 57 | switch (msg.cservice.cmd) { 58 | case "processDetails": 59 | cservice.processSafeSend(process, 60 | cservice.msgBus.createMessage("processDetails", { 61 | processDetails: { 62 | memory: process.memoryUsage(), 63 | title: process.title, 64 | uptime: process.uptime(), 65 | hrtime: process.hrtime() 66 | } 67 | })); 68 | break; 69 | } 70 | }); 71 | } 72 | 73 | function demote() { 74 | // only demote if: 75 | // 1. process.getgid is defined (not Windows) 76 | // 2. Running as root 77 | // 3. workerGid is string and not a proxy worker 78 | var gid = cservice.options.workerGid || 'nobody'; 79 | var uid = cservice.options.workerUid || 'nobody'; 80 | if (process.getgid && process.getgid() === 0) { 81 | if ( // but do not auto-demote proxy 82 | // workers as they require priveledged port access 83 | cluster.worker.env.type !== "proxy" && 84 | typeof cservice.options.workerGid === 'string' 85 | ) { 86 | process.setgid(gid); 87 | process.setuid(uid); 88 | } else { 89 | cservice.log( 90 | "Worker running as root. Not advised for Production." + 91 | " Consider workerGid & workerUid options.".warn 92 | ); 93 | } 94 | 95 | } 96 | 97 | } 98 | 99 | /** 100 | * This is shorthand for: 101 | * 
102 |  *  module.workers.forEach(function(worker){...});
103 |  *
104 | */ 105 | function send(){ 106 | this.forEach(function(worker){ 107 | worker.send.apply(worker, [].slice.apply(arguments)); 108 | }); 109 | } 110 | 111 | function exitGracefully(worker) { 112 | // inform the worker to exit gracefully 113 | worker.send(cservice.msgBus.createMessage("onWorkerStop")); 114 | } 115 | -------------------------------------------------------------------------------- /lib/control.js: -------------------------------------------------------------------------------- 1 | var _controls = {}; 2 | var _keys = {}; 3 | 4 | var levels = { 5 | "remote": 10, // anyone with credentials can access 6 | "local": 20, // anyone locally with credentials can access 7 | "inproc": 30, // CLI of the master process 8 | "disabled": 99 // disabled 9 | }; 10 | 11 | function setControls(controls) { 12 | _controls = {}; 13 | return addControls(controls); 14 | } 15 | 16 | function addControls(controls) { 17 | var control; 18 | for (control in controls) { 19 | if (levels[controls[control]] === undefined) { 20 | throw(controls[control] + " is not a valid control level."); 21 | } 22 | _controls[control] = levels[controls[control]]; 23 | } 24 | return _controls; 25 | } 26 | 27 | function setAccessKey(keys) { 28 | _keys = {}; 29 | 30 | var keyArr = keys.split(";"); 31 | for (var i = 0; i < keyArr.length; i++) { 32 | var fullKey = keyArr[i]; 33 | var keyName = /([a-zA-Z0-9]*)?/.exec(fullKey)[0]; 34 | var keyDisabled = /[a-zA-Z0-9]*\:disabled/.test(fullKey); 35 | if (keyDisabled === true) { 36 | _keys[keyName] = false; 37 | continue; 38 | } 39 | 40 | var key = { }; 41 | var cmdList = /\[(.*)?\]/.exec(fullKey); 42 | if (cmdList && cmdList.length > 0) { 43 | var cmds = cmdList[1].split(","); 44 | for (var i2 = 0; i2 < cmds.length; i2++ ){ 45 | var fullCmd = cmds[i2]; 46 | var cmdName = /([a-zA-Z0-9]*)?/.exec(fullCmd)[0]; 47 | var cmdValStr = /\:(.*)?/.exec(fullCmd); 48 | var cmdVal = true; 49 | if (cmdValStr && cmdValStr.length > 0) { 50 | switch (cmdValStr[1]) { 51 | case "false": 52 | case "disabled": 53 | cmdVal = "disabled"; 54 | break; 55 | case "remote": 56 | cmdVal = "remote"; 57 | break; 58 | case "local": 59 | cmdVal = "local"; 60 | break; 61 | case "inproc": 62 | cmdVal = "inproc"; 63 | break; 64 | } 65 | 66 | key[cmdName] = cmdVal; 67 | } 68 | } 69 | } 70 | 71 | _keys[keyName] = key; 72 | } 73 | } 74 | 75 | function authorize(name, currentControl, accessKey) { 76 | if (typeof accessKey === "string" && accessKey in _keys) { 77 | // if access key available, check rights 78 | var rights = _keys[accessKey]; 79 | if (rights === false) { 80 | return false; // DENIED 81 | } else if (typeof rights === "object" && name in rights) { 82 | // custom rights detected 83 | var commandRight = rights[name]; 84 | if (typeof commandRight === "boolean") { 85 | return commandRight; // return as is 86 | } else if (typeof commandRight === "string" && commandRight in levels) { 87 | return currentControl >= levels[commandRight]; 88 | } 89 | } 90 | } 91 | 92 | if (_controls[name]) { 93 | return currentControl >= _controls[name]; 94 | } 95 | // We default to "remote" which is full access 96 | return currentControl >= levels.remote; 97 | } 98 | 99 | exports.setControls = setControls; 100 | exports.addControls = addControls; 101 | exports.setAccessKey = setAccessKey; 102 | exports.authorize = authorize; 103 | exports.levels = levels; -------------------------------------------------------------------------------- /lib/new-worker.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"), 2 | cluster = require("cluster"), 3 | path = require("path"), 4 | fs = require("fs"), 5 | util = require("util"); 6 | 7 | module.exports = exports = newWorker; 8 | 9 | function newWorker(options, cb) { 10 | var worker; 11 | options = util._extend(util._extend({}, { 12 | worker: "./worker.js", 13 | count: undefined, 14 | restart: true, 15 | type: 'user', 16 | version: undefined, 17 | cwd: undefined, 18 | onStop: false 19 | }), options); 20 | options.ready = false; 21 | if ( 22 | options.worker.indexOf(".") === 0 23 | || (options.worker.indexOf("//") !== 0 24 | && options.worker.indexOf(":\\") < 0) 25 | ) { 26 | // resolve if not absolute 27 | options.worker = path.resolve(options.worker); 28 | } 29 | if ( 30 | fs.existsSync(options.worker) === false 31 | && fs.existsSync(options.worker + ".js") === false 32 | ) { 33 | if(cb){ 34 | cb( 35 | "Worker not found: '" 36 | + options.worker 37 | + "'. Set 'workers' option to proper path." 38 | ); 39 | } 40 | return null; 41 | } 42 | options.cwd = options.cwd || process.cwd(); 43 | options.onReady = cb; 44 | 45 | var version; 46 | if (options.version) { 47 | // track workers with version 48 | version = cservice.locals.proxy.versions[options.version]; 49 | if (!version) { 50 | version = { 51 | name: options.version, 52 | port: options.PROXY_PORT, 53 | lastAccess: Date.now(), 54 | online: false 55 | }; 56 | cservice.locals.proxy.versions[options.version] = version; 57 | } 58 | } 59 | 60 | worker = cluster.fork(options); 61 | worker.cservice = options; 62 | worker.on("message", onMessageFromWorker); 63 | 64 | // track every worker by pid 65 | cservice.locals.workerProcesses[worker.process.pid] = worker; 66 | 67 | return worker; 68 | } 69 | 70 | function onMessageFromWorker(msg) { 71 | var worker = this; 72 | if (!cservice.msgBus.isValidMessage(msg)) { 73 | return; // ignore invalid cluster-service messages 74 | } 75 | 76 | var args, version; 77 | 78 | switch (msg.cservice.cmd) { 79 | case "workerReady": 80 | version = cservice.locals.proxy.versions[worker.cservice.version]; 81 | if (version) { 82 | // if version detected within worker, flag as online 83 | version.online = true; 84 | 85 | // notify proxy workers of version update 86 | cservice.proxy.updateProxyWorkers(); 87 | } 88 | if (worker.cservice.ready === false) { 89 | // preserve preference between restarts, etc 90 | worker.cservice.ready = true; 91 | worker.cservice.onStop = (msg.cservice.onStop === true); 92 | if(typeof worker.cservice.onReady === "function"){ 93 | worker.cservice.onReady(null, worker); 94 | } 95 | } 96 | break; 97 | case "trigger": 98 | args = msg.cservice.args; 99 | if (args && args.length > 0) { 100 | if (msg.cservice.cb === true) { 101 | args.splice(1, 0, function(err, result) { 102 | // forward response to worker that requested the trigger 103 | cservice.msgBus.respondToMessage(msg, worker.process, err, result); 104 | }); 105 | } else { 106 | args.splice(1, 0, null); // no callback necessary 107 | } 108 | cservice.trigger.apply(cservice, args); 109 | } 110 | break; 111 | case "versionUpdateLastAccess": 112 | version = cservice.locals.proxy.versions[msg.cservice.version]; 113 | if (version) { // update freshness 114 | version.lastAccess = Date.now(); 115 | } 116 | break; 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /lib/commands/restart.js: -------------------------------------------------------------------------------- 1 | var async = require("async"), 2 | util = require("util"), 3 | cservice = require("../../cluster-service"); 4 | 5 | module.exports = function(evt, cb, cmd, options) { 6 | var pid = parseInt(cmd), 7 | originalAutoRestart, 8 | tasks; 9 | options = options || {}; 10 | options.timeout = parseInt(options.timeout) || 60000; 11 | if (cmd !== "all" && !pid) { 12 | cb("Invalid request. Try help restart"); 13 | return; 14 | } 15 | 16 | evt.locals.reason = "restart"; 17 | originalAutoRestart = evt.locals.restartOnFailure; 18 | evt.locals.restartOnFailure = false; 19 | 20 | tasks = []; 21 | 22 | evt.service.workers.forEach(function(worker){ 23 | if (pid && worker.process.pid !== pid) { 24 | return; // cannot kill external processes 25 | } 26 | 27 | tasks.push(getTask(evt, worker, options, (pid ? true : false))); 28 | }); 29 | 30 | if (tasks.length === 0) { 31 | cb("No workers to restart"); 32 | } else { 33 | cservice.log( 34 | "Restarting workers... timeout: ".warn + options.timeout.toString().info 35 | ); 36 | 37 | var limit = 1 + 38 | Math.floor( 39 | tasks.length * cservice.options.restartConcurrencyRatio 40 | ); 41 | async.parallelLimit(tasks, limit, function(err) { 42 | evt.locals.restartOnFailure = originalAutoRestart; // restore 43 | 44 | if (err) { 45 | cb(err); 46 | } else { 47 | cb(null, tasks.length + " workers restarted successfully"); 48 | } 49 | }); 50 | } 51 | }; 52 | 53 | module.exports.more = function(cb) { 54 | cb(null, { 55 | info: [ 56 | "Gracefully restart service, waiting up to timeout before terminating", 57 | "workers." 58 | ].join(' '), 59 | command: "restart all|pid { \"option1\": \"value\" }", 60 | "all|pid": [ 61 | "Required. 'all' to force shutdown of all workers, otherwise the pid of", 62 | "the specific worker to restart" 63 | ].join(' '), 64 | "options": "An object of options.", 65 | "* timeout": [ 66 | "Timeout, in milliseconds, before terminating workers.", 67 | "0 for infinite wait." 68 | ].join(' ') 69 | }); 70 | }; 71 | 72 | function getTask(evt, worker, options, explicitRestart) { 73 | return function(cb) { 74 | var pendingWorker = null; 75 | 76 | if (worker.cservice.restart === false && explicitRestart === false) { 77 | cservice.log( 78 | "Worker process " + worker.process.pid + " immune to restarts" 79 | ); 80 | cb(); 81 | return; 82 | } 83 | 84 | // kill new worker if takes too long 85 | var newWorkerTimeout = null; 86 | var isNewWorkerTerminated = false; 87 | if (options.timeout > 0) { // start timeout if specified 88 | newWorkerTimeout = setTimeout(function() { 89 | if (pendingWorker) { 90 | isNewWorkerTerminated = true; 91 | pendingWorker.on('exit', function () { 92 | cb("timed out"); 93 | }); 94 | pendingWorker.kill("SIGKILL"); // go get'em, killer 95 | } 96 | }, options.timeout); 97 | } 98 | 99 | // lets start new worker 100 | pendingWorker = evt.service.newWorker(worker.cservice, function(err) { 101 | pendingWorker = null; 102 | if (newWorkerTimeout) { // timeout no longer needed 103 | clearTimeout(newWorkerTimeout); 104 | } 105 | if (isNewWorkerTerminated) return; 106 | 107 | // ok, lets stop old worker 108 | var oldWorkerTimeout = null; 109 | if (options.timeout > 0) { // start timeout if specified 110 | oldWorkerTimeout = setTimeout(function() { 111 | worker.kill("SIGKILL"); // go get'em, killer 112 | }, options.timeout); 113 | } 114 | 115 | worker.on("exit", function() { 116 | if (oldWorkerTimeout) { 117 | clearTimeout(oldWorkerTimeout); 118 | } 119 | 120 | // exit complete, fire callback 121 | setImmediate(cb); // slight delay in case other events are piled up 122 | }); 123 | 124 | require("../workers").exitGracefully(worker); 125 | }); 126 | }; 127 | } 128 | -------------------------------------------------------------------------------- /test/server-rest.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | var assert = require("assert"); 3 | var httpclient = require("../lib/http-client"); 4 | var util = require("util"); 5 | var request = require("request"); 6 | 7 | cservice.log = function() {}; 8 | cservice.results = function() {}; 9 | 10 | if(cservice.isWorker){ 11 | it("WORKER", function(done) {}); 12 | } else { 13 | describe('[REST Server]', function() { 14 | it('Start worker', function(done) { 15 | cservice.start( 16 | { 17 | workers: null, 18 | workerCount: 0, 19 | accessKey: "123", 20 | cli: false 21 | }, 22 | function() { 23 | assert.equal( 24 | cservice.workers.length, 25 | 0, 26 | "0 worker expected, but " + cservice.workers.length + " found" 27 | ); 28 | done(); 29 | } 30 | ); 31 | }); 32 | 33 | httpclient.init( 34 | util._extend(cservice.options, {accessKey: "123", silentMode: true}) 35 | ); 36 | it('Health check', function(done) { 37 | httpclient.execute("health", function(err, result) { 38 | assert.equal( 39 | result, "\"OK\"", "Expected OK. result=" + result + ", err=" + err 40 | ); 41 | done(); 42 | } 43 | ); 44 | }); 45 | 46 | it('Bad command', function(done) { 47 | httpclient.execute("x98s7df987sdf", function(err, result) { 48 | assert.equal( 49 | err, 50 | "Not found. Try /help", "Expected 'Not found. Try /help'. result=" 51 | + result 52 | + ", err=" + err 53 | ); 54 | done(); 55 | } 56 | ); 57 | }); 58 | 59 | var disabledCmd = function(evt, cb) { 60 | cb(null, "You shouldn't be able to see my data"); 61 | }; 62 | disabledCmd.control = function() { 63 | return "disabled"; 64 | }; 65 | 66 | it('Run cmd', function(done) { 67 | cservice.start({run: "health"}, function(err, result) { 68 | assert.ifError(err); 69 | assert.equal(result, "\"OK\"", "Expected OK, but received: " + result); 70 | done(); 71 | return false; 72 | }); 73 | }); 74 | 75 | it('Command authorization', function(done) { 76 | cservice.on("disabledCmd", disabledCmd); 77 | var url = "http://localhost:11987/cli?cmd=disabledCmd&accessKey=123"; 78 | request.post(url, function(err, res, result) { 79 | assert.equal( 80 | result, 81 | "Not authorized to execute 'disabledCmd' remotely" 82 | ); 83 | done(); 84 | }); 85 | }); 86 | 87 | it('Request authorization', function(done) { 88 | cservice.on("disabledCmd", disabledCmd); 89 | var url = "http://localhost:11987/cli?cmd=disabledCmd&accessKey=BAD"; 90 | request.post(url, function(err, res, result) { 91 | assert.equal(result, "Not authorized"); 92 | done(); 93 | }); 94 | }); 95 | 96 | it('Method Not Allowed', function(done) { 97 | var url = "http://localhost:11987/cli?cmd=health&accessKey=123"; 98 | request.get(url, function(err, res, result) { 99 | assert.equal( 100 | result, 101 | "Method Not Allowed", "Expected 'Method Not Allowed'. result=" 102 | + result 103 | + ", err=" + err 104 | ); 105 | done(); 106 | }); 107 | }); 108 | 109 | it('Page Not Found', function(done) { 110 | var url = "http://localhost:11987/BADCLI?cmd=health&accessKey=123"; 111 | request.post(url, function(err, res, result) { 112 | assert.equal( 113 | result, 114 | "Page Not Found", "Expected 'Page Not Found'. result=" 115 | + result 116 | + ", err=" + err 117 | ); 118 | done(); 119 | }); 120 | }); 121 | 122 | it('Stop workers', function(done) { 123 | cservice.stop(30000, function() { 124 | assert.equal( 125 | cservice.workers.length, 126 | 0, 127 | "0 workers expected, but " + cservice.workers.length + " found" 128 | ); 129 | done(); 130 | }); 131 | }); 132 | }); 133 | } -------------------------------------------------------------------------------- /lib/net-servers.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | var util = require("util"); 3 | var net = require("net"); 4 | var netStats = require("./net-stats"); 5 | var async = require("async"); 6 | 7 | var netServers = { 8 | add: netServersAdd, 9 | remove: netServersRemove, 10 | waitForReady: netServersWaitForReady, 11 | close: netServersClose 12 | }; 13 | 14 | module.exports = exports = netServers; 15 | 16 | wireupNetServerProto(); // init 17 | 18 | function netServersAdd(servers) { 19 | if (util.isArray(servers) === false) { 20 | servers = [servers]; 21 | } 22 | 23 | for (var i = 0; i < servers.length; i++) { 24 | var server = servers[i]; 25 | if (server.cservice) 26 | continue; // ignore if already added 27 | 28 | server.cservice = { 29 | id: Math.random().toString(), // track by id 30 | isReady: true // assume true unless known to be otherwise 31 | }; 32 | cservice.locals.net.servers[server.cservice.id] = server; 33 | 34 | listenToNetServer(server); 35 | netStats(server); 36 | } 37 | } 38 | 39 | function netServersRemove(servers) { 40 | if (util.isArray(servers) === false) { 41 | servers = [servers]; 42 | } 43 | 44 | for (var i = 0; i < servers.length; i++) { 45 | var server = servers[i]; 46 | if (!server.cservice) 47 | continue; // ignore if not tracked by cservice 48 | // stop tracking 49 | delete cservice.locals.net.servers[server.cservice.id]; 50 | // unreference from cservice 51 | delete server.cservice; 52 | 53 | stopListeningToNetServer(server); 54 | } 55 | } 56 | 57 | function netServersWaitForReady(cb) { 58 | var tasks = []; 59 | 60 | for (var id in cservice.locals.net.servers) { 61 | var server = cservice.locals.net.servers[id]; 62 | if (!server.cservice || server.cservice.isReady === true) 63 | continue; 64 | 65 | tasks.push(createWaitForReadyTask(server)); 66 | } 67 | 68 | if (tasks.length === 0) { 69 | return cb(); 70 | } 71 | 72 | async.parallel(tasks, cb); 73 | } 74 | 75 | function createWaitForReadyTask(server) { 76 | return function(cb) { 77 | var ticks = 0; 78 | var timer = setInterval(function() { 79 | if (!server.cservice // unregistered 80 | || server.cservice.isReady === true // now ready 81 | || ticks++ > 1000 // timeout (~10sec) 82 | ) { 83 | clearInterval(timer); 84 | cb(null, true); 85 | } 86 | }, 10); // aggressive polling loop since it is uncommon task but a priority 87 | timer.unref(); 88 | }; 89 | } 90 | 91 | function netServersClose(cb) { 92 | var tasks = []; 93 | 94 | for (var id in cservice.locals.net.servers) { 95 | var server = cservice.locals.net.servers[id]; 96 | if (!server.cservice || server.cservice.isReady === false) 97 | continue; 98 | 99 | tasks.push(createWaitForCloseTask(server)); 100 | } 101 | 102 | if (tasks.length === 0) { 103 | return cb(); 104 | } 105 | 106 | async.parallel(tasks, cb); 107 | } 108 | 109 | function createWaitForCloseTask(server) { 110 | return function(cb) { 111 | server.once("close", function() { cb(null, true); }); 112 | server.close(); 113 | }; 114 | } 115 | 116 | var serverListenOld; 117 | 118 | function wireupNetServerProto() { 119 | serverListenOld = net.Server.prototype.listen; 120 | 121 | net.Server.prototype.listen = serverListenNew; 122 | } 123 | 124 | function listenToNetServer(server) { 125 | server.on("close", serverOnClose); 126 | } 127 | 128 | function stopListeningToNetServer(server) { 129 | server.removeListener("listening", serverOnListening); 130 | server.removeListener("close", serverOnClose); 131 | } 132 | 133 | function serverListenNew() { 134 | netServersAdd(this); // track net server 135 | 136 | this.cservice.isReady = false; // not ready 137 | this.on("listening", serverOnListening); // ready on event 138 | 139 | return serverListenOld.apply(this, arguments); // call original listen 140 | } 141 | 142 | function serverOnListening() { 143 | if (!this.cservice) 144 | return; // ignore 145 | 146 | this.cservice.isReady = true; 147 | } 148 | 149 | function serverOnClose() { 150 | if (!this.cservice) 151 | return; // ignore 152 | 153 | this.cservice.isReady = false; 154 | 155 | // stop monitoring closed connections 156 | netServersRemove(this); 157 | } 158 | -------------------------------------------------------------------------------- /lib/commands/upgrade.js: -------------------------------------------------------------------------------- 1 | var async = require("async"), 2 | util = require("util"), 3 | cservice = require("../../cluster-service"); 4 | 5 | module.exports = function(evt, cb, cmd, workerPath, options) { 6 | var pid = parseInt(cmd); 7 | var originalAutoRestart; 8 | var tasks; 9 | var workerOptions; 10 | 11 | options = options || {}; 12 | options.timeout = parseInt(options.timeout) || 60000; 13 | options.worker = workerPath; 14 | if (typeof workerPath !== "string" || (cmd !== "all" && !pid)) { 15 | cb("Invalid request. Try help upgrade"); 16 | return; 17 | } 18 | 19 | evt.locals.reason = "upgrade"; 20 | originalAutoRestart = evt.locals.restartOnFailure; 21 | evt.locals.restartOnFailure = false; 22 | 23 | tasks = []; 24 | 25 | evt.service.workers.forEach(function(worker){ 26 | if (pid && worker.process.pid !== pid) { 27 | return; // cannot kill external processes 28 | } 29 | 30 | // use original worker options as default, by overwrite using new options 31 | workerOptions = util._extend(util._extend({}, worker.cservice), options); 32 | 33 | tasks.push(getTask(evt, worker, workerOptions)); 34 | }); 35 | 36 | if (tasks.length === 0) { 37 | cb("No workers to upgrade"); 38 | } else { 39 | cservice.log("Upgrading workers... timeout: " + (options.timeout || 0)); 40 | 41 | var limit = 1 + 42 | Math.floor( 43 | tasks.length * cservice.options.restartConcurrencyRatio 44 | ); 45 | async.parallelLimit(tasks, limit, function(err) { 46 | evt.locals.restartOnFailure = originalAutoRestart; 47 | 48 | if (err) { 49 | cb(err); 50 | } else { 51 | cb(null, tasks.length + " workers upgraded successfully"); 52 | } 53 | }); 54 | } 55 | }; 56 | 57 | module.exports.more = function(cb) { 58 | cb(null, { 59 | info: "Gracefully upgrade service, one worker at a time.", 60 | command: "upgrade all|pid workerPath { \"option1\": \"value\" }", 61 | "all|pid": [ 62 | "Required. 'all' to force shutdown of all workers, otherwise the pid of", 63 | "the specific worker to upgrade" 64 | ].join(' '), 65 | "workerPath":[ 66 | "Path of worker file (i.e. /workers/worker) to start, absolute path, or", 67 | "relative to cwd." 68 | ].join(' '), 69 | "options": "An object of options.", 70 | "* cwd":[ 71 | "Path to set as the current working directory. If not provided, existing", 72 | "cwd will be used." 73 | ].join(' '), 74 | "* timeout": [ 75 | "Timeout, in milliseconds, before terminating replaced workers. 0 for", 76 | "infinite wait." 77 | ].join(' ') 78 | }); 79 | }; 80 | 81 | function getTask(evt, worker, options) { 82 | return function(cb) { 83 | 84 | var pendingWorker; 85 | 86 | // kill new worker if takes too long 87 | var newWorkerTimeout = null; 88 | var isNewWorkerTerminated = false; 89 | if (options.timeout > 0) { // start timeout if specified 90 | newWorkerTimeout = setTimeout(function() { 91 | if (!pendingWorker) return; 92 | 93 | isNewWorkerTerminated = true; 94 | pendingWorker.on('exit', function () { 95 | cb("timed out"); 96 | }); 97 | pendingWorker.kill("SIGKILL"); // go get'em, killer 98 | }, options.timeout); 99 | } 100 | 101 | // lets start new worker 102 | pendingWorker = evt.service.newWorker(options, function (err) { 103 | pendingWorker = null; 104 | if (newWorkerTimeout) { // timeout no longer needed 105 | clearTimeout(newWorkerTimeout); 106 | } 107 | 108 | if (err) { 109 | cb(err); 110 | return; 111 | } 112 | 113 | // ok, lets stop old worker 114 | var oldWorkerTimeout = null; 115 | if (options.timeout > 0) { // start timeout if specified 116 | oldWorkerTimeout = setTimeout(function() { 117 | worker.kill("SIGKILL"); // go get'em, killer 118 | }, options.timeout); 119 | } 120 | 121 | worker.on("exit", function() { 122 | if (oldWorkerTimeout) { 123 | clearTimeout(oldWorkerTimeout); 124 | } 125 | 126 | // exit complete, fire callback 127 | setImmediate(cb); // slight delay in case other events are piled up 128 | }); 129 | 130 | require("../workers").exitGracefully(worker); 131 | }); 132 | }; 133 | } 134 | -------------------------------------------------------------------------------- /lib/proxy-worker.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | var cluster = require("cluster"); 3 | var fs = require("fs"); 4 | var path = require("path"); 5 | var httpProxy = require('http-proxy'); 6 | var msgBus = require('./message-bus'); 7 | 8 | cservice.workerReady(false); 9 | 10 | var proxy = httpProxy.createProxyServer({}); 11 | var versionHeader = cluster.worker.env.versionHeader; 12 | var versionPath = cluster.worker.env.versionPath; 13 | var bindingInfo = JSON.parse(cluster.worker.env.bindingInfo); 14 | var workerFilename = cluster.worker.env.workerFilename; 15 | var waiters = {}; 16 | cservice.locals.proxy.defaultVersion = null; // default not set til online 17 | 18 | // set initial versions based on state provided by master 19 | cservice.locals.proxy.versions = JSON.parse(cluster.worker.env.versions); 20 | 21 | var proxyServer; 22 | 23 | if (bindingInfo.tlsOptions) { 24 | // https 25 | 26 | if (typeof bindingInfo.tlsOptions.key === "string") { 27 | bindingInfo.tlsOptions.key = fs.readFileSync(bindingInfo.tlsOptions.key); 28 | } 29 | if (typeof bindingInfo.tlsOptions.cert === "string") { 30 | bindingInfo.tlsOptions.cert = fs.readFileSync(bindingInfo.tlsOptions.cert); 31 | } 32 | if (typeof bindingInfo.tlsOptions.pem === "string") { 33 | bindingInfo.tlsOptions.pem = fs.readFileSync(bindingInfo.tlsOptions.pem); 34 | } 35 | 36 | proxyServer = require("https").createServer( 37 | bindingInfo.tlsOptions, proxyServerRequest 38 | ); 39 | } else { 40 | // http 41 | 42 | proxyServer = require("http").createServer(proxyServerRequest); 43 | } 44 | 45 | process.on("message", onMessageFromMaster); 46 | 47 | function proxyServerRequest(req, res) { 48 | var versionStr = req.headers[versionHeader] || 49 | cservice.locals.proxy.options.defaultVersion 50 | ; 51 | 52 | getProxyVersion(versionStr, function (err, version) { 53 | if (err) { 54 | cservice.log("Failed to load proxy version " + versionStr, err); 55 | 56 | // todo: add option to set 404 content 57 | // via options.customResponses[404] file 58 | res.writeHead(404); 59 | return void res.end("Not found"); 60 | } 61 | proxy.web(req, res, { target: "http://127.0.0.1:" + version.port }); 62 | }); 63 | } 64 | 65 | proxyServer.listen(bindingInfo.port, cservice.workerReady); 66 | 67 | function getProxyVersion(versionStr, cb) { 68 | var version = cservice.locals.proxy.versions[versionStr]; 69 | if (version) { 70 | updateVersionLastAccess(version); 71 | return waitForVersionToComeOnline(versionStr, cb); 72 | } 73 | 74 | // version not found, lets start it 75 | cservice.trigger("proxy", function(err, result) { 76 | if (err) { 77 | return cb(err); 78 | } 79 | 80 | // return newly started version 81 | waitForVersionToComeOnline(versionStr, cb); 82 | }, "version", versionStr); 83 | } 84 | 85 | function waitForVersionToComeOnline(versionStr, cb) { 86 | var version = cservice.locals.proxy.versions[versionStr]; 87 | if (version && version.online === true) { 88 | return cb(null, version); // ready! 89 | } 90 | 91 | // try waiting 92 | var timer, attempts = 0; 93 | timer = setInterval(function() { 94 | version = cservice.locals.proxy.versions[versionStr]; 95 | if (version && version.online === true) { 96 | clearInterval(timer); 97 | return cb(null, version); 98 | } 99 | attempts++; 100 | if (attempts > 240) { 101 | clearInterval(timer); 102 | return cb("Timed out waiting for version to come online!"); 103 | } 104 | }, 250); 105 | timer.unref(); 106 | } 107 | 108 | function onMessageFromMaster(msg) { 109 | if (!msgBus.isValidMessage(msg)) return; 110 | 111 | switch (msg.cservice.cmd) { 112 | case "proxyVersions": 113 | // update versions state 114 | cservice.locals.proxy.versions = msg.cservice.versions; 115 | cservice.locals.proxy.options.defaultVersion = 116 | msg.cservice.defaultVersion 117 | ; 118 | 119 | // update version for default port mapping 120 | cservice.locals.proxy.defaultVersion = 121 | cservice.locals.proxy.versions[msg.cservice.defaultVersion] 122 | ; 123 | 124 | break; 125 | } 126 | } 127 | 128 | function updateVersionLastAccess(version) { 129 | var now = Date.now(); 130 | var diff = now - version.lastAccess; 131 | if (diff >= 0 && diff < 5000) { 132 | return; // if less than 5 seconds since last access, 133 | // don't bother updating master 134 | } 135 | version.lastAccess = now; 136 | var msg = cservice.msgBus.createMessage("versionUpdateLastAccess", { 137 | version: version.name 138 | }); 139 | cservice.processSafeSend(process, msg); 140 | } 141 | -------------------------------------------------------------------------------- /lib/http-server.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"), 2 | util = require("util"), 3 | http = require("http"), 4 | https = require("https"), 5 | querystring = require("querystring"), 6 | control = require("./control"), 7 | options = null, 8 | server = null; 9 | 10 | exports.init = function(o, cb) { 11 | options = o; 12 | 13 | if (options.ssl) { // HTTPS 14 | server = cservice.locals.http = 15 | https.createServer(options.ssl, processRequest) 16 | ; 17 | } else { // HTTP 18 | server = cservice.locals.http = 19 | http.createServer(processRequest) 20 | ; 21 | } 22 | 23 | server.on("error", cb); 24 | server.listen(options.port, options.host, cb); 25 | }; 26 | 27 | exports.close = function() { 28 | try { 29 | server.close(); 30 | } catch (ex) { 31 | } 32 | }; 33 | 34 | function processRequest(req, res) { 35 | var qsIdx, qs, question; 36 | try { 37 | cservice.log( 38 | "API: " 39 | + req.url.replace(/accessKey=.*/i, "accessKey={ACCESS_KEY}").data 40 | ); 41 | 42 | if (req.url.indexOf("/cli?") !== 0) { 43 | res.writeHead(404); 44 | res.end("Page Not Found"); 45 | return; 46 | } 47 | 48 | if (req.method !== "POST" && cservice.options.allowHttpGet !== true) { 49 | res.writeHead(405); 50 | res.end("Method Not Allowed"); 51 | return; 52 | } 53 | 54 | qsIdx = req.url.indexOf("?"); 55 | 56 | qs = querystring.parse(req.url.substr(qsIdx + 1)); 57 | if (!qs.accessKey || qs.accessKey !== options.accessKey) { 58 | res.writeHead(401); 59 | res.end("Not authorized"); 60 | return; 61 | } 62 | 63 | question = qs.cmd || ""; 64 | 65 | req.on('data', function (chunk) { 66 | question += chunk; 67 | }); 68 | 69 | req.on('end', function () { 70 | onCommand(req, res, question, qs.accessKey); 71 | }); 72 | } catch (ex) { 73 | cservice.error( 74 | "Woops, an ERROR!".error, 75 | util.inspect(ex, {depth: null}), 76 | util.inspect(ex.stack || new Error().stack, {depth: null}) 77 | ); 78 | } 79 | } 80 | 81 | function onCommand(req, res, question, accessKey) { 82 | var args = require("./util").getArgsFromQuestion(question, " "); 83 | var controlLevel; 84 | var isAuthorized; 85 | 86 | args = [args[0], function(err, result) { 87 | onCallback(req, res, err, result); 88 | }].concat(args.slice(1)); 89 | 90 | if (!cservice.locals.events[args[0]]) { 91 | cservice.error("Command " + (args[0] + "").cyan + " not found".error); 92 | res.writeHead(404); 93 | res.end("Not found. Try /help"); 94 | return; 95 | } 96 | 97 | controlLevel = control.levels.remote; 98 | if (req.connection.remoteAddress === "127.0.0.1") { 99 | controlLevel = control.levels.local; 100 | } 101 | 102 | isAuthorized = control.authorize(args[0], controlLevel, accessKey); 103 | 104 | if (!isAuthorized) { 105 | res.writeHead(401); 106 | res.end("Not authorized to execute '" + args[0] + "' remotely"); 107 | return; 108 | } 109 | 110 | try { 111 | cservice.trigger.apply(null, args); 112 | } catch (ex) { 113 | res.writeHead(400); 114 | res.end(JSON.stringify( 115 | { 116 | ex: ex, 117 | stack: ex.stack || new Error().stack, 118 | more: "Error. Try /help" 119 | } 120 | )); 121 | } 122 | } 123 | 124 | function onCallback(req, res, err, result) { 125 | var body; 126 | try { 127 | delete cservice.locals.reason; 128 | 129 | if (err) { // should do nothing if response already sent 130 | res.writeHead(400); 131 | res.end(err); 132 | } else { 133 | if (result) { 134 | try 135 | { 136 | body = JSON.stringify(result, function(key, val) { 137 | if (key[0] === "_") { 138 | return undefined; 139 | } else { 140 | return val; 141 | } 142 | }); 143 | res.writeHead( 144 | 200, 145 | { 146 | "Content-Type": "text/json; charset=UTF-8", 147 | "Content-Length": Buffer.byteLength(body) 148 | } 149 | ); 150 | res.end(body); 151 | } catch (ex) { 152 | err = util.inspect(ex, {depth: null}); 153 | res.writeHead(400); 154 | res.end(JSON.stringify({error: err})); 155 | cservice.error( 156 | "Woops, an ERROR!".error, 157 | err, 158 | util.inspect(ex.stack || new Error().stack, {depth: null}) 159 | ); 160 | } 161 | } else { 162 | res.writeHead(200); 163 | res.end("No data"); 164 | } 165 | } 166 | } catch (ex) { 167 | cservice.error( 168 | "Woops, an ERROR!".error, 169 | util.inspect(ex, {depth: null}), 170 | util.inspect(ex.stack || new Error().stack, {depth: null}) 171 | ); 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /test/control.js: -------------------------------------------------------------------------------- 1 | var control = require("../lib/control"); 2 | var assert = require("assert"); 3 | 4 | describe('Control', function() { 5 | describe('levels', function() { 6 | it('should have inproc, local and remote', function(done) { 7 | assert.notEqual( 8 | control.levels.inproc, 9 | undefined, 10 | "control.levels.inproc should exist" 11 | ); 12 | assert.notEqual( 13 | control.levels.local, 14 | undefined, 15 | "control.levels.local should exist" 16 | ); 17 | assert.notEqual( 18 | control.levels.remote, 19 | undefined, 20 | "control.levels.remote should exist" 21 | ); 22 | done(); 23 | }); 24 | }); 25 | 26 | describe('levels', function() { 27 | it('should have hierarchy', function(done) { 28 | assert.equal( 29 | control.levels.inproc > control.levels.local, 30 | true, 31 | "control.levels.inproc should be greater than control.levels.local" 32 | ); 33 | assert.equal( 34 | control.levels.inproc > control.levels.remote, 35 | true, 36 | "control.levels.inproc should be greater than control.levels.remote" 37 | ); 38 | assert.equal( 39 | control.levels.local > control.levels.remote, 40 | true, 41 | "control.levels.local should be greater than control.levels.remote" 42 | ); 43 | done(); 44 | }); 45 | }); 46 | 47 | describe('setControls', function() { 48 | it('returns controls', function(done) { 49 | var controls = control.setControls({"test": "inproc"}); 50 | assert.equal(controls.test, control.levels.inproc); 51 | done(); 52 | }); 53 | }); 54 | 55 | describe('setControls', function() { 56 | it('should throw if level does not exist', function(done) { 57 | assert.throws(function() { 58 | control.setControls({"test": "does not exist"}); 59 | }); 60 | done(); 61 | }); 62 | }); 63 | 64 | describe('addControls', function() { 65 | it('should add to controls', function(done) { 66 | control.setControls({"test": "inproc"}); 67 | var controls = control.addControls({"test2": "local"}); 68 | assert.equal(controls.test, control.levels.inproc); 69 | assert.equal(controls.test2, control.levels.local); 70 | done(); 71 | }); 72 | }); 73 | 74 | describe('addControls', function() { 75 | it('should override existing controls', function(done) { 76 | control.setControls({"test": "inproc"}); 77 | var controls = control.addControls({"test": "local"}); 78 | assert.equal(controls.test, control.levels.local); 79 | done(); 80 | }); 81 | }); 82 | 83 | describe('authorize', function() { 84 | it('should authorize for exact match', function(done) { 85 | control.setControls({"test": "inproc"}); 86 | var isAuthorized = control.authorize("test", control.levels.inproc); 87 | assert.equal(isAuthorized, true, "isAuthorized should be true."); 88 | done(); 89 | }); 90 | }); 91 | 92 | describe('authorize', function() { 93 | it('should authorize inproc if allowed control is local', function(done) { 94 | control.setControls({"test": "local"}); 95 | var isAuthorized = control.authorize("test", control.levels.inproc); 96 | assert.equal(isAuthorized, true, "isAuthorized should be true."); 97 | done(); 98 | }); 99 | }); 100 | 101 | describe('authorize', function() { 102 | it('default is remote and should authorize', function(done) { 103 | control.setControls({}); 104 | var isAuthorized = control.authorize("test", control.levels.local); 105 | assert.equal(isAuthorized, true, "isAuthorized should be true."); 106 | done(); 107 | }); 108 | }); 109 | 110 | describe('authorize keys', function() { 111 | it('key not found, access NOT granted', function(done) { 112 | control.setControls({ "test": "local"}); 113 | control.setAccessKey("abc"); 114 | var isAuthorized = control.authorize("test", control.levels.remote, "X"); 115 | assert.equal(isAuthorized, false, "isAuthorized should be false."); 116 | done(); 117 | }); 118 | it('key not found, access IS granted', function(done) { 119 | control.setControls({ "test": "local"}); 120 | control.setAccessKey("abc"); 121 | var isAuthorized = control.authorize("test", control.levels.local, "X"); 122 | assert.equal(isAuthorized, true, "isAuthorized should be true."); 123 | done(); 124 | }); 125 | it('key found, access NOT granted', function(done) { 126 | control.setControls({ "test": "local"}); 127 | control.setAccessKey("abc:disabled"); 128 | var isAuthorized = control.authorize("test", control.levels.local, "abc"); 129 | assert.equal(isAuthorized, false, "isAuthorized should be false."); 130 | done(); 131 | }); 132 | it('key found, access IS granted', function(done) { 133 | control.setControls({ "test": "local"}); 134 | control.setAccessKey("abc"); 135 | var isAuthorized = control.authorize("test", control.levels.local, "abc"); 136 | assert.equal(isAuthorized, true, "isAuthorized should be true."); 137 | done(); 138 | }); 139 | it('key found, specials rights PREVENT access', function(done) { 140 | control.setControls({ "test": "local"}); 141 | control.setAccessKey("abc[test:inproc]"); 142 | var isAuthorized = control.authorize("test", control.levels.local, "abc"); 143 | assert.equal(isAuthorized, false, "isAuthorized should be false."); 144 | done(); 145 | }); 146 | it('key found, specials rights ALLOW access', function(done) { 147 | control.setControls({ "test": "local"}); 148 | control.setAccessKey("abc[test:remote]"); 149 | var isAuthorized = 150 | control.authorize("test", control.levels.remote, "abc"); 151 | assert.equal(isAuthorized, true, "isAuthorized should be true."); 152 | done(); 153 | }); 154 | }); 155 | }); -------------------------------------------------------------------------------- /test/start-stop.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | var assert = require("assert"); 3 | var start = require("../lib/start"); 4 | 5 | cservice.log = function() { 6 | }; 7 | if(cservice.isWorker){ 8 | it("WORKER", function(done) {}); 9 | } else { 10 | describe('[Start & Stop]', function() { 11 | it('Start worker', function(done) { 12 | assert.equal( 13 | cservice.workers.length, 14 | 0, 15 | "0 workers expected, but " + cservice.workers.length + " found" 16 | ); 17 | cservice.start({config: "./test/workers/basic.json"}, function() { 18 | assert.equal( 19 | cservice.workers.length, 20 | 1, 21 | "1 worker expected, but " + cservice.workers.length + " found" 22 | ); 23 | done(); 24 | }); 25 | }); 26 | 27 | it("start.prepArgs.js", function(done) { 28 | var o = {_: ["Server.JS"]}; 29 | start.prepArgs(o); 30 | assert.equal( 31 | o.workers, 32 | "Server.JS", "Expected 'Server.JS', got " + o.workers 33 | ); 34 | done(); 35 | }); 36 | 37 | it("start.prepArgs.json", function(done) { 38 | var o = {_: ["Config.JSon"]}; 39 | start.prepArgs(o); 40 | assert.equal( 41 | o.config, 42 | "Config.JSon", "Expected 'Config.JSon', got " + o.config 43 | ); 44 | done(); 45 | }); 46 | 47 | it("start.prepArgs.run", function(done) { 48 | var o = {_: ["some command"]}; 49 | start.prepArgs(o); 50 | assert.equal( 51 | o.run, 52 | "some command", "Expected 'some command', got "+o.run 53 | ); 54 | assert.ifError(o.json); 55 | done(); 56 | }); 57 | 58 | it("start.prepArgs.run.json", function(done) { 59 | var o = {_: ["some command"], json: true}; 60 | start.prepArgs(o); 61 | assert.equal( 62 | o.run, 63 | "some command", "Expected 'some command', got "+o.run 64 | ); 65 | assert.equal(o.cli, false, "CLI should be disabled"); 66 | done(); 67 | }); 68 | 69 | it('Add 2nd worker', function(done) { 70 | cservice.trigger("start", function(err, result) { 71 | assert.equal( 72 | cservice.workers.length, 73 | 2, 74 | "2 workers expected, but " + cservice.workers.length + " found" 75 | ); 76 | done(); 77 | }, "./test/workers/basic", {count: 1, timeout: 10000}); 78 | }); 79 | 80 | it('Timeout on new worker', function(done) { 81 | cservice.trigger("start", function(err, result) { 82 | assert.equal(err, "timed out"); 83 | done(); 84 | }, "./test/workers/longInit", {count: 1, timeout: 100}); 85 | }); 86 | 87 | it('Start help', function(done) { 88 | cservice.trigger("help", function(err, result) { 89 | assert.equal( 90 | result.info, 91 | "Gracefully start service, one worker at a time." 92 | ); 93 | done(); 94 | }, "start"); 95 | }); 96 | 97 | it('Bad worker start', function(done) { 98 | cservice.trigger("start", function(err, result) { 99 | assert.equal(err, "Invalid request. Try help start"); 100 | done(); 101 | }, null, {count: 1, timeout: 1000}); 102 | }); 103 | 104 | it('Restart workers', function(done) { 105 | cservice.trigger("restart", function() { 106 | assert.equal( 107 | cservice.workers.length, 108 | 2, 109 | "2 workers expected, but " + cservice.workers.length + " found" 110 | ); 111 | done(); 112 | }, "all" 113 | ); 114 | }); 115 | 116 | it('Upgrade workers', function(done) { 117 | cservice.trigger("upgrade", function() { 118 | assert.equal( 119 | cservice.workers.length, 120 | 2, 121 | "2 workers expected, but " + cservice.workers.length + " found" 122 | ); 123 | done(); 124 | }, "all", "./test/workers/basic2" 125 | ); 126 | }); 127 | 128 | it('Stop workers after upgrade', function(done) { 129 | cservice.stop(30000, function(err, msg) { 130 | assert.ok(!err, 'Received error ' + err); 131 | assert.equal( 132 | cservice.workers.length, 133 | 0, 134 | "0 workers expected, but " 135 | + cservice.workers.length 136 | + " found. Message: " + msg 137 | ); 138 | done(); 139 | }); 140 | }); 141 | 142 | it('Stop an already stopped service', function(done) { 143 | cservice.stop(30000, function() { 144 | assert.equal( 145 | cservice.workers.length, 146 | 0, 147 | "0 workers expected, but " + cservice.workers.length + " found" 148 | ); 149 | done(); 150 | }); 151 | }); 152 | 153 | it('Start inline async worker', function(done) { 154 | var startTime = new Date().getTime(); 155 | cservice.start( 156 | { 157 | workers: 158 | { 159 | inlineReady: { 160 | worker: "./test/workers/inlineReady.js", 161 | count: 1 162 | } 163 | } 164 | }, 165 | function() { 166 | assert.equal( 167 | cservice.workers.length, 168 | 1, 169 | "1 workers expected, but " + cservice.workers.length + " found" 170 | ); 171 | var diffTime = (new Date().getTime() - startTime); 172 | assert.ok(diffTime >= 1000, 173 | "Inline workerReady logic should have taken >= 1000ms, " + 174 | "but returned in " + diffTime + "ms" 175 | ); 176 | done(); 177 | } 178 | ); 179 | }); 180 | 181 | it('Stop workers', function(done) { 182 | cservice.stop(30000, function(err, msg) { 183 | assert.ok(!err, 'Received error ' + err); 184 | assert.equal( 185 | cservice.workers.length, 186 | 0, 187 | "0 workers expected, but " 188 | + cservice.workers.length 189 | + " found. Message: " + msg 190 | ); 191 | done(); 192 | }); 193 | }); 194 | }); 195 | } 196 | -------------------------------------------------------------------------------- /vs/node-cluster-service.njsproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | 2.0 6 | {d1ef2fbf-74dc-4b13-b9a6-a422ef27216e} 7 | ..\ 8 | ShowAllFiles 9 | cluster-service.js 10 | . 11 | . 12 | {3AF33F2E-1136-4D97-BBB7-1795711AC8B8};{349c5851-65df-11da-9384-00065b846f21};{9092AA53-FB77-4645-B42D-1CCCA6BD08BD} 13 | 11.0 14 | $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | False 99 | True 100 | 0 101 | / 102 | http://localhost:48022/ 103 | False 104 | True 105 | http://localhost:1337 106 | False 107 | 108 | 109 | 110 | 111 | 112 | 113 | CurrentPage 114 | True 115 | False 116 | False 117 | False 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | False 127 | False 128 | 129 | 130 | 131 | 132 | -------------------------------------------------------------------------------- /lib/util.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | 3 | /* 4 | * question - Question to split apart. 5 | Ex: prop1 "prop #2" { "prop": 3 } [ "prop #4" ] 5 6 | * delimiter - What splits the properties? Can be one or more characters. 7 | * return - An array of arguments. 8 | Ex: [ "prop1", "prop #2", { "prop": 3 }, [ "prop #4" ], 5 ] 9 | */ 10 | exports.getArgsFromQuestion = getArgsFromQuestion; 11 | exports.debug = debug; 12 | exports.log = log; 13 | exports.error = error; 14 | exports.results = results; 15 | exports.processSafeSend = processSafeSend; 16 | 17 | function debug() { 18 | var args; 19 | var i; 20 | if (cservice.options.cli === true && cservice.options.debug) { 21 | if(process.stdout.clearLine){ 22 | process.stdout.clearLine(); 23 | } 24 | if(process.stdout.cursorTo){ 25 | process.stdout.cursorTo(0); 26 | } 27 | 28 | args = Array.prototype.slice.call(arguments); 29 | for (i = 0; i < args.length; i++) { 30 | if (typeof args[i] === "string") { 31 | args[i] = args[i].debug; 32 | } 33 | } 34 | if (args.length > 0 && typeof args[0] === "string" && args[0][0] === "{") { 35 | cservice.options.debug("cservice:".cservice); 36 | } else { 37 | args = ["cservice: ".cservice].concat(args); 38 | } 39 | cservice.options.debug.apply(this, args); 40 | } 41 | } 42 | 43 | function log() { 44 | var args; 45 | if (cservice.options.cli === true && cservice.options.log) { 46 | if(process.stdout.clearLine){ 47 | process.stdout.clearLine(); 48 | } 49 | if(process.stdout.cursorTo){ 50 | process.stdout.cursorTo(0); 51 | } 52 | 53 | args = Array.prototype.slice.call(arguments); 54 | if (args.length > 0 && typeof args[0] === "string" && args[0][0] === "{") { 55 | cservice.options.log("cservice:".cservice); 56 | } else { 57 | args = ["cservice: ".cservice].concat(args); 58 | } 59 | cservice.options.log.apply(this, args); 60 | } 61 | } 62 | 63 | function error() { 64 | var args; 65 | var i; 66 | if (cservice.options.cli === true && cservice.options.error) { 67 | if(process.stdout.clearLine){ 68 | process.stdout.clearLine(); 69 | } 70 | if(process.stdout.cursorTo){ 71 | process.stdout.cursorTo(0); 72 | } 73 | 74 | args = Array.prototype.slice.call(arguments); 75 | for (i = 0; i < args.length; i++) { 76 | if (typeof args[i] === "string") { 77 | args[i] = args[i].error; 78 | } 79 | } 80 | if (args.length > 0 && typeof args[0] === "string" && args[0][0] === "{") { 81 | cservice.options.error("cservice:".cservice); 82 | } else { 83 | args = ["cservice: ".cservice].concat(args); 84 | } 85 | cservice.options.error.apply(this, args); 86 | } 87 | } 88 | 89 | function results() { 90 | if(cservice.options.log){ 91 | cservice.options.log.apply(this, arguments); 92 | } 93 | } 94 | 95 | function getArgsFromQuestion(question, delimiter) { 96 | 97 | // OLD WAY - simply breaks args by delimiter 98 | //var split = question.split(" "); 99 | //var args = [split[0], onCallback].concat(split.slice(1)); 100 | 101 | // parser needs to be smarter, to account for various data types: 102 | // single word strings: hello 103 | // phrases: "hello world" 104 | // numbers: 1 or 1.3 105 | // JSON: [] or { "a": { "b": "hello \"world\"" } } 106 | var arg = [] 107 | , args = [] 108 | , stringOpen = false 109 | , jsonLevel = 0 110 | , arrayLevel = 0 111 | , i 112 | , isDelim 113 | , c 114 | , cprev 115 | , cnext; 116 | 117 | for (i = 0; i < question.length; i++) { 118 | cprev = i > 0 ? question[i - 1] : ""; 119 | c = question[i]; 120 | cnext = (i < question.length - 1) ? question[i + 1] : ""; 121 | isDelim = (c === delimiter); 122 | if (stringOpen === true) { // processing quotted string 123 | if (c === "\"" && cprev !== "\\") { // closer 124 | // close string 125 | stringOpen = false; 126 | // add string arg, even if empty 127 | args.push(getArgFromValue(arg.join(""))); 128 | // reset arg 129 | arg = []; 130 | } else { // just another char 131 | arg.push(c); 132 | } 133 | } else if (jsonLevel > 0) { // processing JSON object 134 | if (c === "}" && cprev !== "\\") { // closer 135 | jsonLevel--; 136 | } else if (c === "{" && cprev !== "\\") { // opener 137 | jsonLevel++; 138 | } 139 | 140 | arg.push(c); 141 | 142 | if (jsonLevel === 0) { // closed 143 | args.push(getArgFromValue(arg.join(""))); 144 | // reset arg 145 | arg = []; 146 | } 147 | } else if (arrayLevel > 0) { // processing JSON object 148 | if (c === "]" && cprev !== "\\") { // closer 149 | arrayLevel--; 150 | } else if (c === "[" && cprev !== "\\") { // opener 151 | arrayLevel++; 152 | } 153 | 154 | arg.push(c); 155 | 156 | if (arrayLevel === 0) { // closed 157 | args.push(getArgFromValue(arg.join(""))); 158 | // reset arg 159 | arg = []; 160 | } 161 | } else { // processing basic arg 162 | if (c === delimiter) { // delimiter 163 | if (arg.length > 0) { // if arg, add it 164 | args.push(getArgFromValue(arg.join(""))); 165 | // reset arg 166 | arg = []; 167 | } 168 | } else if (c === "{" && arg.length === 0) { // JSON opener 169 | jsonLevel++; 170 | arg.push(c); 171 | } else if (c === "[" && arg.length === 0) { // Array opener 172 | arrayLevel++; 173 | arg.push(c); 174 | } else if (c === "\"" && arg.length === 0) { // string opener 175 | stringOpen = true; 176 | } else { // add it 177 | arg.push(c); 178 | } 179 | } 180 | } 181 | 182 | if (arg.length > 0) { // if arg remains, add it too 183 | args.push(getArgFromValue(arg.join(""))); 184 | } 185 | 186 | return args; 187 | } 188 | 189 | function getArgFromValue(val) { 190 | try { 191 | // \" tags should be standard quotes after parsed 192 | val = val.replace(/\\\"/g, '"'); 193 | 194 | // try to process as JSON first 195 | // Typical use cases: 196 | // 1 - number 197 | // 1.3 - number 198 | // [] - array 199 | // { "a": { } } - object 200 | return JSON.parse(val); 201 | } catch (ex) { 202 | return val; // use as-is 203 | } 204 | } 205 | 206 | function processSafeSend(process, msg) { 207 | try { 208 | process.send(msg); 209 | } catch (ex) { 210 | return ex; 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /lib/master.js: -------------------------------------------------------------------------------- 1 | /* jshint loopfunc:true */ 2 | var cservice = require("../cluster-service"), 3 | cluster = require("cluster"), 4 | httpserver = require("./http-server"), 5 | async = require("async"), 6 | path = require("path"), 7 | startRequests = []; // queued start requests 8 | 9 | exports.start = startMaster; 10 | 11 | function startMaster(options, cb) { 12 | var workersRemaining; 13 | var workersForked; 14 | var workers; 15 | var i; 16 | var workerName; 17 | var worker; 18 | var workerCount; 19 | 20 | options = options || {}; 21 | options.workerCount = options.workerCount || 1; 22 | 23 | if (cservice.locals.state === 0) { // one-time initializers 24 | cservice.locals.state = 1; // starting 25 | 26 | require("./commands/version")({}, function(err, ver) { 27 | cservice.log("cluster-service v".info + ver.data + " starting...".info); 28 | }); 29 | 30 | /*process.on("uncaughtException", function(err) { 31 | cservice.log("uncaughtException", util.inspect(err)); 32 | });*/ 33 | 34 | // queue up our request 35 | startRequests.push(function() { 36 | startMaster(options, cb); 37 | }); 38 | 39 | startListener(options, function(err) { 40 | var i; 41 | if (err) { 42 | cservice.locals.isAttached = true; 43 | 44 | // start the http client 45 | require("./http-client").init(options); 46 | } else { // we're the single-master 47 | cservice.locals.isAttached = false; 48 | 49 | cluster.setupMaster({silent: (options.silent === true)}); 50 | 51 | cluster.on("online", function(worker) { 52 | cservice.trigger("workerStart", worker); 53 | }); 54 | cluster.on("exit", function(worker, code, signal) { 55 | // stop tracking 56 | var version = cservice.locals.proxy.versions[worker.cservice.version]; 57 | if (version) { 58 | // get all proxy workers for a specific version 59 | var versionWorkers = 60 | cservice.proxy.getVersionWorkers(worker.cservice.version); 61 | // exclude our exiting worker process in case it's still returned 62 | versionWorkers = versionWorkers.filter(function(versionWorker) { 63 | return worker.process.pid !== versionWorker.process.pid; 64 | }); 65 | 66 | if (versionWorkers.length === 0) { 67 | // if no workers remain for a given version, drop the version 68 | delete cservice.locals.proxy.versions[worker.cservice.version]; 69 | 70 | // inform proxy workers of version change 71 | cservice.proxy.updateProxyWorkers(); 72 | } 73 | } 74 | delete cservice.locals.workerProcesses[worker.process.pid]; 75 | 76 | cservice.trigger("workerExit", worker); 77 | // do not restart if there is a reason, or disabled 78 | if ( 79 | !(cservice.locals.reason || worker.cservice.reason) 80 | && worker.exitedAfterDisconnect !== true 81 | && cservice.locals.restartOnFailure === true 82 | ) { 83 | setTimeout(function() { 84 | // lets replace lost worker. 85 | cservice.newWorker(worker.cservice); 86 | }, options.restartDelayMs); 87 | } 88 | }); 89 | 90 | // start monitor 91 | monitorWorkers(); 92 | 93 | if (options.cli === true) { 94 | // wire-up CLI 95 | require("./cli").init(options); 96 | } 97 | } 98 | 99 | cservice.proxy.start({}, function() { 100 | cservice.locals.state = 2; // running 101 | 102 | // now that listener is ready, process queued start requests 103 | for (i = 0; i < startRequests.length; i++) { 104 | startRequests[i](); // execute 105 | } 106 | startRequests = []; 107 | }); 108 | }); 109 | } else if (cservice.locals.state === 1) { // if still starting, queue requests 110 | startRequests.push(function() { 111 | startMaster(options, cb); 112 | }); 113 | // if we're NOT attached, we can spawn the workers now 114 | } else if (cservice.locals.isAttached === false) { 115 | // fork it, i'm out of here 116 | workersRemaining = 0; 117 | workersForked = 0; 118 | 119 | if (options.workers !== null) { 120 | workers = typeof options.workers === "string" 121 | ? {main: {worker: options.workers}} 122 | : options.workers; 123 | for (workerName in workers) { 124 | worker = workers[workerName]; 125 | workerCount = worker.count || options.workerCount; 126 | workersRemaining += workerCount; 127 | workersForked += workerCount; 128 | for (i = 0; i < workerCount; i++) { 129 | cservice.newWorker(worker, function(err) { 130 | workersRemaining--; 131 | if (err) { 132 | workersRemaining = 0; // callback now 133 | } 134 | if (workersRemaining === 0) { 135 | if (typeof options.master === "string") { 136 | require(path.resolve(options.master)); 137 | } 138 | if(cb){ 139 | cb(err); 140 | } 141 | } 142 | }); 143 | } 144 | } 145 | } 146 | 147 | // if no forking took place, make sure cb is invoked 148 | if (workersForked === 0) { 149 | cservice.log("No workers running. Try 'start server.js'.".info); 150 | if(cb){ 151 | cb(); 152 | } 153 | } 154 | } else { // nothing else to do 155 | if(cb){ 156 | cb(); 157 | } 158 | } 159 | } 160 | 161 | function startListener(options, cb) { 162 | if (typeof options.accessKey === "undefined") { // in-proc mode only 163 | cservice.log( 164 | [ 165 | "LOCAL ONLY MODE. Run with 'accessKey' option to enable communication", 166 | "channel." 167 | ] 168 | .join(' ') 169 | .info 170 | ); 171 | cb(); 172 | return; 173 | } else { 174 | options.accessKey = options.accessKey.toString(); 175 | require("./control").setAccessKey(options.accessKey); 176 | } 177 | 178 | httpserver.init(options, function(err) { 179 | if (!err) { 180 | cservice.log( 181 | ("Listening at " 182 | + ( 183 | (options.ssl ? "https://" : "http://") 184 | + options.host 185 | + ":" 186 | + options.port 187 | + "/cli" 188 | ) 189 | .data 190 | ) 191 | .info 192 | ); 193 | } 194 | 195 | cb(err); 196 | }); 197 | } 198 | 199 | function monitorWorkers() { 200 | if (cservice.options.restartOnMemUsage || cservice.options.restartOnUpTime) { 201 | setTimeout(onMonitorWorkers, 20000).unref(); // do not hold server open 202 | } 203 | } 204 | 205 | function onMonitorWorkers() { 206 | cservice.trigger("workers", function(err, results) { 207 | var workers; 208 | var restarts; 209 | var memUsage; 210 | var upTime; 211 | var i; 212 | var w; 213 | 214 | if (err || !results || !results.workers) { 215 | // nothing we can do about it at this time 216 | setTimeout(onMonitorWorkers, 60000).unref(); // do not hold server open 217 | return; 218 | } 219 | workers = results.workers; 220 | restarts = []; 221 | memUsage = cservice.options.restartOnMemUsage; 222 | upTime = cservice.options.restartOnUpTime; 223 | for (i = 0; i < workers.length; i++) { 224 | w = workers[i]; 225 | if ( 226 | (memUsage && w.process.memory.rss > memUsage) 227 | || 228 | (upTime && w.process.uptime > upTime) 229 | ) { 230 | restarts.push(getWorkerToRestart(w)); 231 | } 232 | } 233 | if (restarts.length > 0) { 234 | async.series(restarts, function(err, results) { 235 | setTimeout(onMonitorWorkers, 20000).unref(); // do not hold server open 236 | }); 237 | } else { 238 | setTimeout(onMonitorWorkers, 30000).unref(); // do not hold server open 239 | } 240 | }, "simple"); 241 | } 242 | 243 | function getWorkerToRestart(worker) { 244 | return function(cb) { 245 | cservice.trigger("restart", cb, worker.pid); 246 | }; 247 | } 248 | -------------------------------------------------------------------------------- /lib/proxy.js: -------------------------------------------------------------------------------- 1 | var cservice = require("../cluster-service"); 2 | var cluster = require("cluster"); 3 | var workersHelper = require("./workers"); 4 | var path = require("path"); 5 | var async = require("async"); 6 | var fs = require("fs"); 7 | 8 | module.exports = { 9 | start: start, 10 | stop: stop, 11 | version: version, 12 | promote: promote, 13 | updateProxyWorkers: updateProxyWorkers, 14 | getProxyWorkers: getProxyWorkers, 15 | getVersionWorkers: getVersionWorkers, 16 | info: info 17 | }; 18 | 19 | function start(options, cb) { 20 | if (cluster.isWorker === true) { 21 | cservice.log("Proxy cannot be started from worker".warn); 22 | return cb && cb("Proxy cannot be started from worker"); 23 | } 24 | 25 | var configPath = options.configPath || cservice.options.proxy; 26 | 27 | if (typeof configPath !== "string") { 28 | // disabled 29 | return cb && cb(); 30 | } 31 | 32 | if (cservice.locals.proxy.enabled === true) { 33 | cservice.log("Proxy already running".warn); 34 | return cb && cb("Proxy already running"); 35 | } 36 | 37 | cservice.locals.proxy.configPath = path.resolve(configPath); 38 | cservice.locals.proxy.options = 39 | JSON.parse(fs.readFileSync(cservice.locals.proxy.configPath)) 40 | ; 41 | 42 | options = cservice.locals.proxy.options; 43 | 44 | if (Array.isArray(options.bindings) === false || 45 | options.bindings.length === 0) { 46 | options.bindings = [{ port: 80, workerCount: 2 }]; // default 47 | } 48 | 49 | options.nonDefaultWorkerCount = options.nonDefaultWorkerCount || 1; 50 | options.nonDefaultWorkerIdleTime = options.nonDefaultWorkerIdleTime || 3600; 51 | 52 | cservice.locals.proxy.versionPath = 53 | options.versionPath || path.dirname(cservice.locals.proxy.configPath) 54 | ; 55 | cservice.locals.proxy.versionPath = 56 | path.resolve(cservice.locals.proxy.versionPath) 57 | ; 58 | cservice.locals.proxy.workerFilename = options.workerFilename || "worker.js"; 59 | cservice.locals.proxy.versionHeader = options.versionHeader || "x-version"; 60 | 61 | var portRange = (options.versionPorts || "11000-12000").split("-"); 62 | cservice.locals.proxy.portRange = { 63 | min: parseInt(portRange[0]), 64 | max: parseInt(portRange[1]) 65 | }; 66 | cservice.locals.proxy.nextAvailablePortIndex = 0; 67 | 68 | var proxyWorkerTasks = options.bindings.map(function(b) { 69 | return function(cb) { 70 | var workerOptions = { 71 | type: "proxy", // proxy worker type 72 | worker: path.resolve(__dirname, "proxy-worker.js"), 73 | bindingInfo: JSON.stringify(b), 74 | versionPath: cservice.locals.proxy.versionPath, 75 | versionHeader: cservice.locals.proxy.versionHeader, 76 | workerFilename: cservice.locals.proxy.workerFilename, 77 | versions: JSON.stringify(cservice.locals.proxy.versions), 78 | count: b.workerCount || 2 79 | }; 80 | cservice.newWorker(workerOptions, cb); 81 | }; 82 | }); 83 | 84 | cservice.locals.proxy.refreshnessTimer = 85 | setInterval(checkVersionsForFreshness, 10000) 86 | ; 87 | cservice.locals.proxy.refreshnessTimer.unref(); 88 | 89 | cservice.locals.proxy.enabled = true; 90 | 91 | async.parallel(proxyWorkerTasks, function (err) { 92 | var portArr = options.bindings.map(function(b) { 93 | return b.port.toString().data; 94 | }); 95 | 96 | if (err) { 97 | cservice.error("Proxy failed to run on ports ".error + 98 | portArr.join(",".info) + " with error ".error + err.toString().data); 99 | return cb && cb(err); 100 | } 101 | 102 | cservice.log("Proxy running on ports ".info + portArr.join(",".info)); 103 | 104 | if (!cservice.locals.proxy.options.defaultVersion) { 105 | // no current version 106 | 107 | return cb && cb(); 108 | } 109 | 110 | version(cservice.locals.proxy.options.defaultVersion, 111 | { workerCount: cservice.locals.options.workerCount }, 112 | function (err, version) { 113 | if (err) { 114 | cservice.error("Proxy failed to run on ports ".error + 115 | portArr.join(",".info) + " with error ".error + err.toString().data 116 | ); 117 | return cb && cb(err); 118 | } 119 | 120 | return cb && cb(); 121 | }); 122 | }); 123 | } 124 | 125 | function stop(cb) { 126 | if (cluster.isWorker === true) { 127 | cservice.log("Proxy cannot be stopped from worker".warn); 128 | return cb && cb("Proxy cannot be stopped from worker"); 129 | } 130 | 131 | if (typeof cservice.locals.proxy.configPath !== "string") { 132 | // disabled 133 | return cb && cb(); 134 | } 135 | 136 | if (cservice.locals.proxy.enabled === false) { 137 | cservice.log("Proxy not running".warn); 138 | return cb && cb("Proxy not running"); 139 | } 140 | 141 | clearInterval(cservice.locals.proxy.refreshnessTimer); 142 | cservice.locals.proxy.refreshnessTimer = null; 143 | 144 | // now lets trigger a shutdown 145 | cservice.trigger("shutdown", function(err, result) { 146 | cservice.locals.proxy.enabled = false; 147 | return cb && cb(); 148 | }, "all"); 149 | } 150 | 151 | function version(versionStr, options, cb) { 152 | if (cluster.isWorker === true) { 153 | cservice.log("Proxy cannot invoke 'version' from worker".warn); 154 | return cb && cb("Proxy cannot invoke 'version' from worker"); 155 | } 156 | 157 | options = options || {}; 158 | if (isNaN(options.workerCount) === true) { 159 | options.workerCount = 160 | (versionStr === cservice.locals.proxy.options.defaultVersion) 161 | ? cservice.locals.options.workerCount 162 | : cservice.locals.proxy.options.nonDefaultWorkerCount 163 | ; 164 | } 165 | 166 | // detect current version worker count 167 | var currentVersionWorkers = getVersionWorkers(versionStr); 168 | 169 | // determine worker delta from desired count and actual count 170 | var workerCountDelta = options.workerCount - currentVersionWorkers.length; 171 | 172 | // get existing version listing 173 | var v = cservice.locals.proxy.versions[versionStr]; 174 | 175 | if (v) { 176 | // update version lastAccess 177 | v.lastAccess = Date.now(); 178 | } 179 | 180 | // if version worker count is already current, nothing more to do 181 | if (workerCountDelta === 0) { 182 | return cb && cb(); 183 | } 184 | 185 | if (workerCountDelta < 0) { 186 | // if desired version count is less than current, reduce worker count 187 | var workersToShutdown = currentVersionWorkers.length - options.workerCount; 188 | var shutdownTasks = []; 189 | for (var workerToShutdown = 0; workerToShutdown < workersToShutdown; 190 | workerToShutdown++) { 191 | shutdownTasks.push( 192 | getWorkerShutdownTask( 193 | currentVersionWorkers[workerToShutdown], 194 | options.reason || "proxy version" 195 | ) 196 | ); 197 | } 198 | 199 | // shutdown all at once 200 | async.parallel(shutdownTasks, function (err, results) { 201 | return cb && cb(err); 202 | }); 203 | 204 | return; 205 | } 206 | 207 | // if desired version count is more than current, spin up new workers 208 | 209 | var workerPath = path.resolve(cservice.locals.proxy.versionPath, 210 | versionStr, cservice.locals.proxy.workerFilename 211 | ); 212 | 213 | // use existing port if available, otherwise allocate a new one 214 | var versionPort = (v && v.port) || getNextAvailablePort(); 215 | 216 | cservice.trigger("start", function (err, result) { 217 | return cb && cb(err, result); 218 | }, workerPath, { 219 | count: workerCountDelta, 220 | version: versionStr, 221 | PROXY_PORT: versionPort 222 | }); 223 | } 224 | 225 | function getWorkerShutdownTask(worker, reason) { 226 | return function(cb) { 227 | worker.cservice.reason = reason; 228 | cservice.trigger("shutdown", cb, worker.process.pid); 229 | }; 230 | } 231 | 232 | function getNextAvailablePort() { 233 | // always continue where we left off from the last time 234 | // we fetched an available port. 235 | // this generally will allow us to return in O(1), 236 | // unless there are tons of active versions. 237 | var totalPorts = cservice.locals.proxy.portRange.max 238 | - cservice.locals.proxy.portRange.min 239 | ; 240 | for (var i = 0; i < totalPorts; i++) { 241 | var port = cservice.locals.proxy.portRange.min 242 | + ((cservice.locals.proxy.nextAvailablePortIndex + i) % totalPorts); 243 | if (isPortInUse(port) === false) { 244 | cservice.locals.proxy.nextAvailablePortIndex = ((i + 1) % totalPorts); 245 | return port; 246 | } 247 | } 248 | 249 | throw new Error( 250 | "All proxy ports have been used up! Try increasing range of " + 251 | "`proxy.versionPorts` or reducing `proxy.nonDefaultWorkerIdleTime`."); 252 | } 253 | 254 | function isPortInUse(port) { 255 | for (var k in cservice.locals.proxy.versions) { 256 | if (cservice.locals.proxy.versions.hasOwnProperty(k) === false) { 257 | continue; // ignore 258 | } 259 | if (cservice.locals.proxy.versions[k].port === port) { 260 | return true; // NOT available 261 | } 262 | } 263 | 264 | // if we get this far, we're OK to use port 265 | return false; 266 | } 267 | 268 | function promote(versionStr, options, cb) { 269 | if (cluster.isWorker === true) { 270 | cservice.log("Proxy cannot invoke 'promote' from worker".warn); 271 | return cb && cb("Proxy cannot invoke 'promote' from worker"); 272 | } 273 | 274 | options = options || {}; 275 | options.workerCount = options.workerCount || 276 | cservice.locals.options.workerCount 277 | ; 278 | 279 | var oldVersion = cservice.locals.proxy.versions[ 280 | cservice.locals.proxy.options.defaultVersion 281 | ]; 282 | 283 | // set to-be-promoted version to desired worker count 284 | version(versionStr, options, function (err) { 285 | if (err) { 286 | // pass failure on 287 | return cb && cb(err); 288 | } 289 | 290 | // persist to-be-promoted version 291 | cservice.locals.proxy.options.defaultVersion = versionStr; 292 | fs.writeFile(cservice.locals.proxy.configPath, 293 | JSON.stringify(cservice.locals.proxy.options, null, " "), function(err) { 294 | if (err) { 295 | // pass failure on 296 | cservice.error("Failed to proxy promote version".error + 297 | versionStr.info 298 | ); 299 | return cb && cb(err); 300 | } 301 | 302 | // notify proxy-workers of promoted version 303 | updateProxyWorkers(); 304 | 305 | cservice.log("Proxy promoted version ".success + 306 | versionStr.info + 307 | " successfully".success 308 | ); 309 | 310 | // bring previously promoted version down to 311 | // `nonDefaultWorkerCount` workers, but no need to wait for callback 312 | if (oldVersion && oldVersion.name !== versionStr) { 313 | version(oldVersion.name, 314 | { 315 | workerCount: cservice.locals.proxy.options.nonDefaultWorkerCount, 316 | reason: "proxy demote" 317 | }, function(err) { 318 | if (err) { 319 | return cservice.error("Failed to proxy demote version".error + 320 | oldVersion.name.info 321 | ); 322 | } 323 | 324 | cservice.log("Proxy demoted old version ".success + 325 | oldVersion.name.info + 326 | " successfully".success 327 | ); 328 | } 329 | ); 330 | } 331 | 332 | if (cb) { 333 | setImmediate(cb); 334 | } 335 | }); 336 | }); 337 | } 338 | 339 | function updateProxyWorkers() { 340 | var msg = cservice.msgBus.createMessage("proxyVersions", { 341 | versions: cservice.locals.proxy.versions, 342 | defaultVersion: cservice.locals.proxy.options.defaultVersion 343 | }); 344 | getProxyWorkers().forEach(function(worker) { 345 | worker.send(msg); 346 | }); 347 | } 348 | 349 | function info(cb) { 350 | if (cluster.isWorker === true) { 351 | cservice.log("Proxy cannot invoke 'info' from worker".warn); 352 | return cb && cb("Proxy cannot invoke 'info' from worker"); 353 | } 354 | 355 | var now = Date.now(); 356 | var proxyWorkers = getProxyWorkers().map(function(worker) { 357 | var bindingInfo = JSON.parse(worker.cservice.bindingInfo); 358 | return { 359 | port: bindingInfo.port, 360 | ssl: typeof bindingInfo.tlsOptions === "object" 361 | }; 362 | }); 363 | var versionWorkers = getVersionWorkers().map(function(worker) { 364 | var versionInfo = cservice.locals.proxy.versions[worker.cservice.version]; 365 | return { 366 | worker: worker.cservice.worker, 367 | version: worker.cservice.version, 368 | lastAccess: versionInfo ? 369 | Math.round((now - versionInfo.lastAccess) / 1000) : "?" 370 | }; 371 | }); 372 | 373 | cb(null, { 374 | versionPath: cservice.locals.proxy.versionPath, 375 | workerFilename: cservice.locals.proxy.workerFilename, 376 | portRange: cservice.locals.proxy.portRange, 377 | options: cservice.locals.proxy.options, 378 | proxyWorkers: proxyWorkers, 379 | versionWorkers: versionWorkers 380 | }); 381 | 382 | } 383 | 384 | function getProxyWorkers() { 385 | return cservice.workers.filter(function(worker) { 386 | return worker.cservice.type === "proxy"; 387 | }); 388 | } 389 | 390 | function getVersionWorkers(explicitVersion) { 391 | return cservice.workers.filter(function(worker) { 392 | var result = 393 | typeof worker.cservice.version === "string" && 394 | (!explicitVersion || worker.cservice.version === explicitVersion) 395 | ; 396 | return result; 397 | }); 398 | } 399 | 400 | function isVersionRunning(versionStr, cb) { 401 | if (!(versionStr in cservice.locals.proxy.versions)) { 402 | return false; // version not available 403 | } 404 | 405 | // are any worker processes running desired version? 406 | var workers = cservice.locals.workerProcesses; 407 | 408 | for (var i = 0; i < workers.length; i++) { 409 | var worker = workers[i]; 410 | var pid = worker.process.pid; 411 | } 412 | 413 | return false; // no workers running desired version 414 | } 415 | 416 | function checkVersionsForFreshness() { 417 | var now = Date.now(); 418 | for (var k in cservice.locals.proxy.versions) { 419 | if ( 420 | // live version is exempt 421 | k === cservice.locals.proxy.options.defaultVersion || 422 | // verify a valid version 423 | !cservice.locals.proxy.versions.hasOwnProperty(k)) { 424 | continue; // skip 425 | } 426 | var v = cservice.locals.proxy.versions[k]; 427 | var diff = (now - v.lastAccess) / 1000; // seconds 428 | if (diff < cservice.locals.proxy.options.nonDefaultWorkerIdleTime) { 429 | continue; // all OK 430 | } 431 | 432 | cservice.log("Proxy version ".warn + k.info + 433 | " shutting down due to inactivity".warn 434 | ); 435 | 436 | // kill all the things 437 | version(k, { workerCount: 0 }); 438 | } 439 | } 440 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cluster-service 2 | 3 | [![Build Status](https://travis-ci.org/godaddy/node-cluster-service.png)](https://travis-ci.org/godaddy/node-cluster-service) [![NPM version](https://badge.fury.io/js/cluster-service.png)](http://badge.fury.io/js/cluster-service) [![Dependency Status](https://gemnasium.com/godaddy/node-cluster-service.png)](https://gemnasium.com/godaddy/node-cluster-service) [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/godaddy/node-cluster-service/trend.png)](https://bitdeli.com/free "Bitdeli Badge") 4 | 5 | [![NPM](https://nodei.co/npm/cluster-service.png?downloads=true&stars=true&downloadRank=true)](https://www.npmjs.org/package/cluster-service) [![NPM](https://nodei.co/npm-dl/cluster-service.png?height=2)](https://nodei.co/npm/cluster-service/) 6 | 7 | ## Install 8 | 9 | npm install cluster-service 10 | 11 | https://npmjs.org/package/cluster-service 12 | 13 | 14 | 15 | ## About 16 | 17 | Turn your single process code into a fault-resilient, multi-process service with 18 | built-in REST & CLI support. Restart or hot upgrade your web servers with zero 19 | downtime or impact to clients. 20 | 21 | Presentation: 22 | 23 | http://x.co/bpnode 24 | 25 | Video: 26 | 27 | http://x.co/bpnodevid 28 | 29 | 30 | ## Getting Started 31 | 32 | Your existing application, be it console app or service of some kind: 33 | 34 | // server.js 35 | console.log("Hello World"); 36 | 37 | Leveraging ```cluster-service``` without adding a line of code: 38 | 39 | npm install -g cluster-service 40 | cservice "server.js" --accessKey "lksjdf982734" 41 | // cserviced "server.js" --accessKey "lksjdf982734" // daemon 42 | 43 | This can be done without a global install as well, by updating your ```package.json```: 44 | 45 | "scripts": { 46 | "start": "cservice server.js --accessKey lksjdf982734" 47 | }, 48 | "dependencies": { 49 | "cluster-service": ">=0.5.0" 50 | } 51 | 52 | Now we can leverage ```npm``` to find our local install of ```cluster-service```: 53 | 54 | npm start 55 | 56 | Or, if you prefer to control ```cluster-service``` within your code, we've got you covered: 57 | 58 | // server.js 59 | require("cluster-service").start({ workers: "./worker.js", accessKey: "lksjdf982734" }); 60 | 61 | // worker.js 62 | console.log("Hello World"); // notice we moved our original app logic to the worker 63 | 64 | 65 | 66 | ## Talk to it 67 | 68 | Now that your service is resilient to worker failure, and utilizing all cores of your machine, lets talk to it. 69 | With your service running, type into the command-line: 70 | 71 | restart all 72 | 73 | or for a full list of commands... 74 | 75 | help 76 | 77 | or for help on a specific command: 78 | 79 | help {command} 80 | 81 | We can also issue commands from a seperate process, or even a remote machine (assuming proper access): 82 | 83 | npm install -g cluster-service 84 | cservice "restart all" --accessKey "my_access_key" 85 | 86 | You can even pipe raw JSON for processing: 87 | 88 | cservice "restart all" --accessKey "my_access_key" --json 89 | 90 | Check out ***Cluster Commands*** for more. 91 | 92 | 93 | 94 | ## Start Options 95 | 96 | When initializing your service, you have a number of options available: 97 | 98 | cservice "server.js" --accessKey "123" 99 | 100 | Or via JSON config: 101 | 102 | cservice "config.json" 103 | 104 | Or within your node app: 105 | 106 | // server.js 107 | // inline options 108 | require("cluster-service").start({ workers: "worker.js", accessKey: "123" }); 109 | // or via config 110 | require("cluster-service").start({ config: "config.json" }); 111 | 112 | ### Options: 113 | 114 | * `workers` - Path of worker to start. A string indicates a single worker, 115 | forked based on value of ```workerCount```. An object indicates one or more worker objects: 116 | ```{ "worker1": { worker: "worker1.js", cwd: process.cwd(), count: 2, restart: true } }```. 117 | This option is automatically set if run via command-line ```cservice "worker.js"``` if 118 | the ```.js``` extension is detected. 119 | * `accessKey` - A secret key that must be specified if you wish to invoke commands from outside 120 | your process. Allows CLI & REST interfaces. 121 | * `config` - A filename to the configuration to load. Useful to keep options from having to be inline. 122 | This option is automatically set if run via command-line ```cservice "config.json"``` if 123 | the ```.json``` extension is detected. 124 | * `host` (default: "localhost") - Host to bind to for REST interface. (Will only bind if `accessKey` 125 | is provided) 126 | * `port` (default: 11987) - Port to bind to. If you leverage more than one cluster-service on a 127 | machine, you'll want to assign unique ports. (Will only bind if accessKey is provided) 128 | * `workerCount` (default: os.cpus().length) - Gives you control over the number of processes to 129 | run the same worker concurrently. Recommended to be 2 or more to improve availability. But some 130 | workers do not impact availability, such as task queues, and can be run as a single instance. 131 | * `cli` (default: true) - Enable the command line interface. Can be disabled for background 132 | services, or test cases. Running `cserviced` will automatically disable the CLI. 133 | * `ssl` - If provided, will bind using HTTPS by passing this object as the 134 | [TLS options](http://nodejs.org/api/tls.html#tls_tls_createserver_options_secureconnectionlistener). 135 | * `run` - Ability to run a command, output result, and exit. This option is automatically 136 | set if run via command-line ```cservice "restart all"``` and no extension is detected. 137 | * `json` (default: false) - If specified in conjunction with ```run```, 138 | will *only* output the result in JSON for 139 | consumption from other tasks/services. No other data will be output. 140 | * `silent` (default: false) - If true, forked workers will not send their output to parent's stdio. 141 | * `allowHttpGet` (default: false) - For development purposes, can be enabled for testing, but is 142 | not recommended otherwise. 143 | * `restartOnMemUsage` (default: disabled) - If a worker process exceeds the specified memory threshold 144 | (in bytes), the process will be restarted gracefully. Only one worker will be restarted at a time. 145 | * `restartOnUpTime` (default: disabled) - If a worker process exceeds the specified uptime threshold 146 | (in seconds), the process will be restarted gracefully. Only one worker will be restarted at a time. 147 | * `restartConcurrencyRatio` (default `0.33`) - The ratio of workers that can be restarted concurrently 148 | during a restart or upgrade process. This can greatly improve the speed of restarts for applications 149 | with many concurrent workers and/or slow initializing workers. 150 | * `commands` - A single directory, an array of directories, or a comma-delimited list of directories 151 | may be provided to auto-register commands found in the provided folders that match the ".js" 152 | extension. If the module exposes the "id" property, that will be the name of the command, 153 | otherwise the filename (minus the extension) will be used as the name of the command. If relative 154 | paths are provided, they will be resolved from process.cwd(). 155 | * `master` - An optional module to execute for the master process only, once ```start``` has been completed. 156 | * `proxy` - Optional path to a JSON config file to enable Proxy Support. 157 | * `workerGid` - Group ID to assign to child worker processes (recommended `nobody`). 158 | * `workerUid` - User ID to assign to child worker processes (recommended `nobody`). 159 | 160 | 161 | ## Console & REST API 162 | 163 | A DPS Cluster Service has two interfaces, the console (stdio), and an HTTP REST API. The two 164 | interfaces are treated identical, as console input/output is piped over the REST API. The 165 | reason for the piping is that a DPS Cluster Service is intentionally designed to only 166 | support one instance of the given service running at any one time, and the port binding 167 | is the resource constraint. This allows secondary services to act as console-only 168 | interfaces as they pipe all input/output over HTTP to the already running service 169 | that owns the port. This flow enables the CLI to background processes. 170 | The REST API is locked to a "accessKey" expected in the query string. The console 171 | automatically passes this key to the REST API, but for external REST API access, 172 | the key will need to be known. 173 | 174 | { host: "localhost", port: 11987, accessKey: "lksjdf982734" } 175 | 176 | Invoking the REST interface directly would look something like: 177 | 178 | curl -d "" "http://localhost:11987/cli?cmd=help&accessKey=lksjdf982734" 179 | 180 | Or better yet, use the ```run``` option to do the work for you: 181 | 182 | cservice "help" --accessKey "lksjdf982734" 183 | // same as 184 | cservice --run "help" --accessKey "lksjdf982734" 185 | 186 | 187 | 188 | ## Cluster Commands 189 | 190 | While a Cluster Service may provide its own custom commands, below are provided out-of-the-box. 191 | Commands may be disabled by overriding them. 192 | 193 | * `start workerPath [cwd] { [timeout:60000] }` - Gracefully start service, one worker at a time. 194 | * `restart all|pid { [timeout:60000] }` - Gracefully restart service, waiting up to timeout before terminating workers. 195 | * `shutdown all|pid { [timeout:60000] }` - Gracefully shutdown service, waiting up to timeout before terminating workers. 196 | * `exit now` - Forcefully exits the service. 197 | * `help [cmd]` - Get help. 198 | * `upgrade all|pid workerPath { [cwd] [timeout:60000] }` - Gracefully upgrade service, one worker at a time. (continuous deployment support). 199 | * `workers` - Returns list of active worker processes. 200 | * `health` - Returns health of service. Can be overidden by service to expose app-specific data. 201 | * `info` - Returns summary of process & workers. 202 | 203 | 204 | 205 | ## Commands & Events 206 | 207 | Creating custom, or overriding commands and events is as simple as: 208 | 209 | cservice "server.js" --commands "./commands,../some_more_commands" 210 | 211 | Or if you prefer to manually do so via code: 212 | 213 | var cservice = require("cluster-service"); 214 | cservice.on("custom", function(evt, cb, arg1, arg2) { // "custom" command 215 | // can also fire custom events 216 | cservice.trigger("on.custom.complete", 1, 2, 3); 217 | }; 218 | 219 | cservice.on("test", function(evt, cb, testScript, timeout) { // we're overriding the "test" command 220 | // arguments 221 | // do something, no callback required (events may optionally be triggered) 222 | }; 223 | 224 | // can also issue commands programatically 225 | cservice.trigger("custom", function(err) { /* my callback */ }, "arg1value", "arg2value"); 226 | 227 | 228 | ## Cluster Events 229 | 230 | Events are emitted to interested parties. 231 | 232 | * `workerStart (pid, reason)` - Upon exit of any worker process, the process id of the exited worker. Reasons include: "start", "restart", "failure", and "upgrade". 233 | * `workerExit (pid, reason)` - Upon start of any worker process. Reasons include: "start", "restart", "failure", and "upgrade". 234 | 235 | 236 | 237 | ## Async Support 238 | 239 | While web servers are automatically wired up and do not require async logic (as of v1.0), if 240 | your service requires any other asynchronous initialization code before being ready, this 241 | is how it can be done. 242 | 243 | Have the worker inform the master once it is actually ready: 244 | 245 | // worker.js 246 | require("cluster-service").workerReady(false); // we're NOT ready! 247 | setTimeout(funtion() { 248 | // dumb example of async support 249 | require("cluster-service").workerReady(); // we're ready! 250 | }, 1000); 251 | 252 | Additionally, a worker may optionally perform cleanup tasks prior to exit, via: 253 | 254 | // worker.js 255 | require("cluster-service").workerReady({ 256 | onWorkerStop: function() { 257 | // lets clean this place up 258 | process.exit(); // we're responsible for exiting if we register this cb 259 | } 260 | }); 261 | 262 | 263 | 264 | ## Access Control 265 | 266 | Commands may be granted "inproc" (high risk), "local" (low risk), or "remote" (no risk). Setting 267 | access control can be done within the command, like so: 268 | 269 | ```javascript 270 | // exit.js 271 | module.exports.control = function(){ 272 | return "local"; 273 | }; 274 | ``` 275 | 276 | Or may be overriden at runtime via: 277 | 278 | ```javascript 279 | // server.js 280 | require("cluster-service").control({ "exit": "local" }); 281 | ``` 282 | 283 | ## Proxy Support 284 | 285 | Proxy mode specifically caters to Web Servers that you want to enable automatic 286 | versioning of your service. Any version requested (via `versionHeader`) that is 287 | not yet loaded will automatically have a worker process spun up with the new 288 | version, and after ready, the proxy will route to that worker. 289 | 290 | Every version of your app *must* adhere to the `PROXY_PORT` environment 291 | variable like so: 292 | 293 | require("http").createServer(function(req, res) { 294 | res.writeHead(200); 295 | res.end("Hello world!"); 296 | }).listen(process.env.PROXY_PORT || 3000 /* port to use when not running in proxy mode */); 297 | 298 | ### Proxy Options 299 | 300 | * `versionPath` (default: same directory as proxy JSON config) - Can override 301 | to point to a new version folder. 302 | * `defaultVersion` - The version (folder name) that is currently active/live. 303 | If you do not initially set this option, making a request to the Proxy without 304 | a `versionHeader` will result in a 404 (Not Found) since there is no active/live 305 | version. 306 | Upgrades will automatically update this option to the latest upgraded version. 307 | * `versionHeader` (default: `x-version`) - HTTP Header to use when determining 308 | non-default version to route to. 309 | * `workerFilename` (default: `worker.js`) - Filename of worker file. 310 | * `bindings` (default: `[{ port: 80, workerCount: 2 }]`) - An array of `Proxy Bindings`. 311 | * `versionPorts` (default: `11000-12000`) - Reserved port range that can be used to 312 | assign ports to different App versions via `PROXY_PORT`. 313 | * `nonDefaultWorkerCount` (default: 1) - If a version is requested that is not 314 | a default version, this will be the number of worker processes dedicated to 315 | that version. 316 | * `nonDefaultWorkerIdleTime` (default: 3600) - The number of seconds of inactivity 317 | before a non-default version will have its workers shut down. 318 | 319 | ### Proxy Bindings 320 | 321 | Binding options: 322 | 323 | * `port` - Proxy port to bind to. 324 | * `workerCount` (default: 2) - Number of worker processes to use for this 325 | binding. Typically more than 2 is unnecessary for a proxy, and less than 2 326 | is a potential failure point if a proxy worker ever goes down. 327 | * `tlsOptions` - TLS Options if binding for HTTPS. 328 | * `key` - Filename that contains the Certificate Key. 329 | * `cert` - Filename that contains the Certificate. 330 | * `pem` - Filename that contains the Certificate PEM if applicable. 331 | 332 | A full list of TLS Options: https://nodejs.org/api/tls.html#tls_tls_createserver_options_secureconnectionlistener 333 | 334 | ### Proxy Commands 335 | 336 | Work like any other `Cluster Commands`. 337 | 338 | * `proxy start configPath` - Start the proxy using the provided JSON config file. 339 | * `proxy stop` - Shutdown the proxy service. 340 | * `proxy version workerVersion workerCount` - Set a given App version to the 341 | desired number of worker processes. If the version is not already running, 342 | it will be started. If 2 workers for the version are already running, and you 343 | request 4, 2 more will be started. If 4 workers for the version are already 344 | running, and you request 2, 2 will be stopped. 345 | * `proxy promote workerVersion workerCount` - Works identical to the 346 | `proxy version` command, except this will flag the version as active/live, 347 | resulting in the Proxy Config file being updated with the new `defaultVersion`. 348 | * `proxy info` - Fetch information about the proxy service. 349 | 350 | 351 | 352 | ## Tests & Code Coverage 353 | 354 | Download and install: 355 | 356 | git clone https://github.com/godaddy/node-cluster-service.git 357 | cd node-cluster-service 358 | npm install 359 | 360 | Now test: 361 | 362 | npm test 363 | 364 | View code coverage in any browser: 365 | 366 | coverage/lcov-report/index.html 367 | 368 | 369 | ## Change Log 370 | 371 | [Change Log](https://github.com/godaddy/node-cluster-service/blob/master/CHANGELOG.md) 372 | 373 | 374 | 375 | ## License 376 | 377 | [MIT](https://github.com/godaddy/node-cluster-service/blob/master/LICENSE.txt) 378 | --------------------------------------------------------------------------------