├── data └── .gitkeep ├── doc └── .gitkeep ├── coverage └── .gitkeep ├── examples ├── .gitkeep ├── usage │ ├── ping │ │ ├── index.js │ │ ├── example.js │ │ ├── ping.js │ │ └── test.js │ ├── dlm │ │ ├── index.js │ │ ├── example.js │ │ └── test.js │ └── dsm │ │ ├── index.js │ │ ├── example.js │ │ └── test.js └── gen_server │ └── dlm │ ├── README.md │ ├── test.js │ ├── multi_test.js │ └── dlm.js ├── test ├── unit │ ├── .gitkeep │ ├── dlm │ │ ├── index.js │ │ └── lock.js │ ├── dsem │ │ ├── index.js │ │ └── semaphore.js │ ├── index.js │ ├── queue.js │ ├── node.js │ ├── mtable.js │ ├── utils.js │ ├── cluster_node.js │ ├── conn.js │ ├── vclock.js │ └── gen_server.js ├── integration │ ├── .gitkeep │ ├── index.js │ ├── gen_server.js │ ├── dlm.js │ └── dsm.js ├── mocks │ ├── index.js │ └── ipc.js └── index.js ├── lib ├── dlm │ ├── index.js │ └── lock.js ├── dsem │ ├── index.js │ └── semaphore.js ├── index.js ├── queue.js ├── consts.js ├── node.js ├── utils.js ├── cluster_node.js ├── conn.js ├── mtable.js ├── command_server.js └── vclock.js ├── .gitignore ├── .jsdoc.conf.json ├── LICENSE.md ├── CONTRIBUTING.md ├── .travis.yml ├── Gruntfile.js ├── package.json ├── CHANGELOG.md └── bin └── cli.js /data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /coverage/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/unit/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/integration/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/usage/ping/index.js: -------------------------------------------------------------------------------- 1 | module.exports = require("./ping"); 2 | -------------------------------------------------------------------------------- /examples/usage/dlm/index.js: -------------------------------------------------------------------------------- 1 | module.exports = require("../../../lib/dlm/dlm"); 2 | -------------------------------------------------------------------------------- /examples/usage/dsm/index.js: -------------------------------------------------------------------------------- 1 | module.exports = require("../../../lib/dsem/dsm"); 2 | -------------------------------------------------------------------------------- /lib/dlm/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | DLMServer: require("./dlm"), 3 | Lock: require("./lock") 4 | }; 5 | -------------------------------------------------------------------------------- /lib/dsem/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | DSMServer: require("./dsm"), 3 | Semaphore: require("./semaphore") 4 | }; 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | npm-debug.log 3 | 4 | .idea/ 5 | doc/* 6 | !doc/.gitkeep 7 | coverage/* 8 | !coverage/.gitkeep 9 | 10 | -------------------------------------------------------------------------------- /test/mocks/index.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | path = require("path"); 3 | 4 | module.exports = { 5 | ipc: require("./ipc") 6 | }; 7 | -------------------------------------------------------------------------------- /test/unit/dlm/index.js: -------------------------------------------------------------------------------- 1 | module.exports = function (mocks, lib) { 2 | describe("DLM unit tests", function () { 3 | require("./lock")(mocks, lib); 4 | require("./dlm")(mocks, lib); 5 | }); 6 | }; 7 | -------------------------------------------------------------------------------- /test/unit/dsem/index.js: -------------------------------------------------------------------------------- 1 | module.exports = function (mocks, lib) { 2 | describe("DSem unit tests", function () { 3 | require("./semaphore")(mocks, lib); 4 | require("./dsm")(mocks, lib); 5 | }); 6 | }; 7 | -------------------------------------------------------------------------------- /test/integration/index.js: -------------------------------------------------------------------------------- 1 | module.exports = function (mocks, lib) { 2 | describe("Integration tests", function () { 3 | require("./gen_server")(mocks, lib); 4 | require("./dlm")(mocks, lib); 5 | require("./dsm")(mocks, lib); 6 | }); 7 | }; 8 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | path = require("path"); 3 | 4 | describe("Clusterluck tests", function () { 5 | var mocks = require("./mocks"); 6 | var lib = require("../lib"); 7 | require(path.join(__dirname, "unit"))(mocks, lib); 8 | require(path.join(__dirname, "integration"))(mocks, lib); 9 | }); 10 | -------------------------------------------------------------------------------- /examples/usage/ping/example.js: -------------------------------------------------------------------------------- 1 | const cl = require("../../../index"), 2 | PingServer = require("./ping"); 3 | 4 | const node = cl.createCluster("node_id"); 5 | const pingServe = new PingServer(node.kernel()); 6 | 7 | pingServe.start("ping_server"); 8 | pingServe.on("ping", (data, from) => { 9 | console.log("Received ping request from:", from); 10 | }); 11 | 12 | node.start("cookie", "ring", () => { 13 | console.log("Node %s listening!", node.kernel().self().id()); 14 | }); 15 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | var fs = require("fs"), 2 | path = require("path"), 3 | _ = require("lodash"); 4 | 5 | var hidden = /^\./; 6 | var files = fs.readdirSync(__dirname); 7 | // filter out hidden files 8 | files = _.filter(files, function (val) { 9 | return !hidden.test(val) && val !== "index.js"; 10 | }); 11 | var lib = _.reduce(files, function(memo, file){ 12 | memo[path.basename(file, ".js")] = require(path.join(__dirname, file)); 13 | return memo; 14 | }, {}); 15 | 16 | module.exports = lib; 17 | -------------------------------------------------------------------------------- /.jsdoc.conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "tags": { 3 | "allowUnknownTags": true 4 | }, 5 | "plugins": ["plugins/markdown"], 6 | "templates": { 7 | "cleverLinks": false, 8 | "monospaceLinks": false, 9 | "dateFormat": "ddd MMM Do YYYY", 10 | "outputSourceFiles": true, 11 | "outputSourcePath": true, 12 | "systemName": "Clusterluck", 13 | "footer": "", 14 | "copyright": "Azuqua © 2017", 15 | "navType": "vertical", 16 | "linenums": true, 17 | "collapseSymbols": false, 18 | "inverseNav": true, 19 | "highlightTutorialCode": true, 20 | "protocol": "html://" 21 | }, 22 | "markdown": { 23 | "parser": "gfm", 24 | "hardwrap": true 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /examples/usage/ping/ping.js: -------------------------------------------------------------------------------- 1 | const cl = require("../../../index"), 2 | _ = require("lodash"); 3 | 4 | class PingServer extends cl.GenServer { 5 | constructor(kernel) { 6 | super(kernel); 7 | } 8 | 9 | start(name) { 10 | super.start(name); 11 | 12 | const handler = this._doPing.bind(this); 13 | this.on("ping", handler); 14 | this.once("stop", _.partial(this.removeListener, "ping", handler).bind(this)); 15 | 16 | return this; 17 | } 18 | 19 | ping(node, cb) { 20 | this.call({node: node, id: this._id}, "ping", null, cb); 21 | } 22 | 23 | _doPing(data, from) { 24 | return this.reply(from, "pong"); 25 | } 26 | } 27 | 28 | module.exports = PingServer; 29 | -------------------------------------------------------------------------------- /test/unit/index.js: -------------------------------------------------------------------------------- 1 | module.exports = function (mocks, lib) { 2 | describe("Unit tests", function () { 3 | require("./node")(mocks, lib); 4 | require("./queue")(mocks, lib); 5 | require("./conn")(mocks, lib); 6 | require("./chash")(mocks, lib); 7 | require("./vclock")(mocks, lib); 8 | require("./kernel")(mocks, lib); 9 | require("./gen_server")(mocks, lib); 10 | require("./gossip")(mocks, lib); 11 | require("./command_server")(mocks, lib); 12 | require("./utils")(mocks, lib); 13 | require("./cluster_node")(mocks, lib); 14 | require("./dtable")(mocks, lib); 15 | require("./mtable")(mocks, lib); 16 | require("./dlm")(mocks, lib); 17 | require("./dsem")(mocks, lib); 18 | }); 19 | }; 20 | -------------------------------------------------------------------------------- /examples/usage/dlm/example.js: -------------------------------------------------------------------------------- 1 | const cl = require("../../../index"), 2 | DLMServer = require("./index"); 3 | 4 | const node = cl.createCluster("node_id"); 5 | const server = new DLMServer(node.gossip(), node.kernel()); 6 | 7 | server.start("dlm_server"); 8 | server.load((err) => { 9 | if (err) process.exit(1); 10 | server.on("rlock", (data, from) => { 11 | console.log("Received rlock request on %s with holder %s", data.id, data.holder); 12 | }); 13 | server.on("wlock", (data, from) => { 14 | console.log("Received rlock request on %s with holder %s", data.id, data.holder); 15 | }); 16 | 17 | node.start("cookie", "ring", () => { 18 | console.log("Node %s listening!", node.kernel().self().id()); 19 | }); 20 | }); 21 | -------------------------------------------------------------------------------- /lib/queue.js: -------------------------------------------------------------------------------- 1 | class Queue { 2 | constructor() { 3 | this._front = []; 4 | this._back = []; 5 | } 6 | 7 | enqueue(val) { 8 | this._back.push(val); 9 | return this; 10 | } 11 | 12 | dequeue() { 13 | if (this._front.length > 0) { 14 | return this._front.pop(); 15 | } 16 | this._front = this._back.reverse(); 17 | this._back = []; 18 | return this._front.pop(); 19 | } 20 | 21 | flush() { 22 | var ret = this._front.reverse().concat(this._back); 23 | this._front = []; 24 | this._back = []; 25 | return ret; 26 | } 27 | 28 | size() { 29 | return this._front.length + this._back.length; 30 | } 31 | 32 | peek() { 33 | if (this._front.length > 0) return this._front[this._front.length-1]; 34 | return this._back[0]; 35 | } 36 | } 37 | 38 | module.exports = Queue; 39 | -------------------------------------------------------------------------------- /examples/usage/dsm/example.js: -------------------------------------------------------------------------------- 1 | const cl = require("../../../index"), 2 | DSMServer = require("./index"); 3 | 4 | const node = cl.createCluster("node_id"); 5 | const server = new DSMServer(node.gossip(), node.kernel()); 6 | 7 | server.start("dsm_server"); 8 | server.load((err) => { 9 | if (err) process.exit(1); 10 | server.on("create", (data, from) => { 11 | console.log("Received create request on semaphore %s with concurrency limit %i", data.id, data.n); 12 | }); 13 | server.on("post", (data, from) => { 14 | console.log("Received post request on %s with holder %s", data.id, data.holder); 15 | }); 16 | server.on("close", (data, from) => { 17 | console.log("Received close request on %s with holder %s", data.id, data.holder); 18 | }); 19 | 20 | node.start("cookie", "ring", () => { 21 | console.log("Node %s listening!", node.kernel().self().id()); 22 | }); 23 | }); 24 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Azuqua 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | # Forth Eorlingas! 3 | 4 | ## Pull Requests 5 | 6 | For most PR's (read: additive feature requests), please submit against the `develop` branch; 7 | this is to ensure that we may quickly merge the changes in and allow the community to critique/modify 8 | the proposed changes from a common source. 9 | 10 | ## Code Format 11 | 12 | Follow the same coding format seen in the source code; the one hard requirement is that code indentation 13 | **must** be two hard spaces (no soft tabs), this is to ensure that diff views of code submission remains legible. 14 | 15 | ## Tests 16 | 17 | There is an exhaustive unit test suite under `/test`, which can be run using both `mocha test` or `grunt test`. 18 | 19 | PR's that provide additional functionality should also provide corresponding unit test cases. 20 | 21 | ## Documentation 22 | 23 | Anybody and everybody can help with documentation on this project, and there's a lot to be done. Specifically with documenting events and when they're fired/listened to, and also with examples of how to use different modules in this library. So if you find some topic confusing or lacking in documentation, please make a PR or an issue (and thanks in advance). 24 | -------------------------------------------------------------------------------- /test/unit/dlm/lock.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | assert = require("chai").assert; 4 | 5 | module.exports = function (mocks, lib) { 6 | describe("Lock unit tests", function () { 7 | var Lock = lib.dlm.Lock; 8 | var lock, 9 | type = "read", 10 | id = "id", 11 | timeout = new Map(); 12 | beforeEach(function () { 13 | lock = new Lock(type, id, timeout); 14 | }); 15 | 16 | it("Should construct a lock", function () { 17 | assert.equal(lock._id, id); 18 | assert.equal(lock._type, type); 19 | assert.ok(_.isEqual(lock._timeout, timeout)); 20 | 21 | lock = new Lock("write", "id2", timeout); 22 | assert.equal(lock._id, "id2"); 23 | assert.equal(lock._type, "write"); 24 | assert.ok(_.isEqual(lock._timeout, timeout)); 25 | }); 26 | 27 | it("Should get/set the id of a lock", function () { 28 | assert.equal(lock.id(), id); 29 | lock.id("id2"); 30 | assert.equal(lock.id(), "id2"); 31 | }); 32 | 33 | it("Should get/set the type of a lock", function () { 34 | assert.equal(lock.type(), type); 35 | lock.type("write"); 36 | assert.equal(lock.type(), "write"); 37 | }); 38 | 39 | it("Should get/set the timeout of a lock", function () { 40 | assert.ok(_.isEqual(lock.timeout(), timeout)); 41 | lock.timeout(8001); 42 | assert.equal(lock.timeout(), 8001); 43 | }); 44 | }); 45 | }; 46 | -------------------------------------------------------------------------------- /test/unit/dsem/semaphore.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | assert = require("chai").assert; 4 | 5 | module.exports = function (mocks, lib) { 6 | describe("Semaphore unit tests", function () { 7 | var Semaphore = lib.dsem.Semaphore; 8 | var sem, 9 | id = "id", 10 | size = 3, 11 | timeouts = new Map(); 12 | beforeEach(function () { 13 | sem = new Semaphore(id, size, timeouts); 14 | }); 15 | 16 | it("Should construct a semaphore", function () { 17 | assert.equal(sem._id, id); 18 | assert.equal(sem._size, size); 19 | assert.ok(_.isEqual(sem._timeouts, timeouts)); 20 | 21 | sem = new Semaphore("id2", 5, timeouts); 22 | assert.equal(sem._id, "id2"); 23 | assert.equal(sem._size, 5); 24 | assert.ok(_.isEqual(sem._timeouts, timeouts)); 25 | }); 26 | 27 | it("Should get/set the id of a semaphore", function () { 28 | assert.equal(sem.id(), id); 29 | sem.id("id2"); 30 | assert.equal(sem.id(), "id2"); 31 | }); 32 | 33 | it("Should get/set the type of a semaphore", function () { 34 | assert.equal(sem.size(), size); 35 | sem.size(5); 36 | assert.equal(sem.size(), 5); 37 | }); 38 | 39 | it("Should get/set the timeouts of a semaphore", function () { 40 | assert.ok(_.isEqual(sem.timeouts(), timeouts)); 41 | sem.timeouts(new Map([["foo", "bar"]])); 42 | assert.ok(_.isEqual(sem.timeouts(), new Map([["foo", "bar"]]))); 43 | }); 44 | }); 45 | }; 46 | -------------------------------------------------------------------------------- /examples/gen_server/dlm/README.md: -------------------------------------------------------------------------------- 1 | DLM example 2 | =========== 3 | 4 | A distributed lock manager `gen_server` implementation for reference, using the [Redlock algorithm](https://redis.io/topics/distlock). May be added into notp as an optional feature if there's community interest. 5 | 6 | Example Usage 7 | ------------- 8 | 9 | A simple example usage of this `gen_server` follows. For a more detailed integration test on a single node, refer to `test.js`. For a multi-node integration test, refer to `multi_test.js`. In this test, we try to lock the same resource with two different requesters. 10 | 11 | ``` javascript 12 | const cl = require("../../index"), 13 | os = require("os"), 14 | assert = require("chai").assert; 15 | 16 | const nodeID = "name", 17 | port = 7022, 18 | host = os.hostname(); 19 | 20 | const node = cl.createCluster(nodeID, host, port), 21 | gossip = node.gossip(), 22 | kernel = node.kernel(); 23 | 24 | const DLMServer = require("./dlm"); 25 | const dlm = new DLMServer(gossip, kernel, { 26 | rquorum: 0.51, 27 | wquorum: 0.51 28 | }); 29 | 30 | // load node state 31 | node.load(() => { 32 | // first start dlm, then start network kernel 33 | dlm.start("locker"); 34 | node.start("cookie", "ring", () => { 35 | console.log("Node %s listening on hostname %s, port %s", nodeID, host, port); 36 | dlm.wlock("id", "holder", 30000, (err, nodes) => { 37 | assert.notOk(err); 38 | assert.isArray(nodes); 39 | dlm.wlock("id", "holder", 30000, (err, wNodes) => { 40 | assert.ok(err); 41 | }); 42 | }); 43 | }); 44 | }); 45 | ``` 46 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | os: 2 | - linux 3 | language: node_js 4 | notifications: 5 | email: 6 | on_success: never 7 | on_failure: never 8 | sudo: true 9 | branches: 10 | only: 11 | - master 12 | - develop 13 | node_js: 14 | - '7' 15 | - '6' 16 | cache: 17 | apt: true 18 | directories: 19 | - node_modules 20 | before_install: npm install -g grunt-cli 21 | script: npm run-script test-travis 22 | after_script: 23 | - cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js 24 | - | 25 | [ $TRAVIS_BRANCH = master ] && 26 | [ $TRAVIS_PULL_REQUEST = false ] && 27 | echo "Building docs for GH pages..." && 28 | grunt docs && 29 | sudo pip install ghp-import && 30 | ghp-import -n doc && 31 | git push -fq https://${GH_TOKEN}@github.com/${TRAVIS_REPO_SLUG}.git gh-pages && 32 | echo "Finished building docs for GH pages." 33 | env: 34 | global: 35 | - CXX=g++-4.8 36 | - secure: S2ogBC/CkkTZ3bvYxRPq1cNdZfg045OERatJDSW4U6yUoHnRxU9Mau5meeCovRvquGrH37mWahO/h3++iDyn9Mefb8oUDXCAlCjyzkks8wpF4dTDPo6J2AcCNHspK8X4jFVa0azIoF1k44gbqX5FG42F5meeL70FEPurO4mHT19lFHzwp/FdJ8D/uLc3nnQZ0ifjlltv16Olbth4rw3lHvxK5V2Q8pU8fzZfAgfI5zfjfwVt94G1uhsnc9BcojENyLfjmpfXYicTnA48PbpYtzIe8O206xuTgdxKTy89GrL1zVUw47Tor1Lb0wae1JQWYy+ZWipE1/PKoTqoJ9vO5RxwuG/TngkLLGSRsiMuQZRa7kPxfjKux5HCIeRXwdVa99y5XBVdUkFEHWJ9Bc762cIuqQE/7yJz29tyLakUrj/Ok7ELu+kgX6Nh9ukbGOVhbyXRIj60psOSrCqBdLH/unwweJvk1pqObPQqBfy+ffym0mHtdfBt8eUbRNQ1ahzoJ94RW8VUmM3c1YUvZZB2RiMvMKgsIo49uePl4fesfs/XadAEv3To5P+C1IooYVmLjz1eKp1PuVGb4S06WXtAbjnY4AeH/brGlR03oAkRCOSNB6/TwHTBdRAPgwQjG7zhebujbtTJDgXnQ2cOHSLaTHChV/BjB+hib7vEE9cmke8= 37 | addons: 38 | apt: 39 | sources: 40 | - ubuntu-toolchain-r-test 41 | packages: 42 | - g++-4.8 43 | -------------------------------------------------------------------------------- /examples/usage/ping/test.js: -------------------------------------------------------------------------------- 1 | const cl = require("../../../index"), 2 | _ = require("lodash"), 3 | os = require("os"), 4 | assert = require("chai").assert, 5 | debug = require("debug")("examples:usage:ping:test"), 6 | async = require("async"), 7 | PingServer = require("./ping"); 8 | 9 | const nodeID = process.argv[2], 10 | port = parseInt(process.argv[3]), 11 | nodeID2 = process.argv[4], 12 | port2 = parseInt(process.argv[5]), 13 | host = os.hostname(); 14 | 15 | const nodes = []; 16 | const servers = []; 17 | async.each([[nodeID, port], [nodeID2, port2]], (config, next) => { 18 | const node = cl.createCluster(config[0], host, config[1]), 19 | kernel = node.kernel(); 20 | 21 | const ping = new PingServer(kernel); 22 | nodes.push(node); 23 | servers.push(ping); 24 | node.load(() => { 25 | ping.start("ping_server"); 26 | node.start("cookie", "ring", () => { 27 | debug("Node %s listening on hostname %s, port %s!", config[0], host, config[1]); 28 | next(); 29 | }); 30 | }); 31 | }, () => { 32 | setTimeout(() => { 33 | // make sure nodes know about each other 34 | assert.ok(_.find(nodes[0].gossip().ring().nodes(), (node) => { 35 | return node.equals(nodes[1].kernel().self()); 36 | })); 37 | const server = servers[0]; 38 | server.ping(nodes[1].kernel().self(), (err, res) => { 39 | assert.notOk(err); 40 | assert.equal(res, "pong"); 41 | debug("Successfully received pong!"); 42 | process.exit(0); 43 | }); 44 | }, 1000); 45 | // make nodes meet each other first 46 | nodes[0].gossip().meet(nodes[1].kernel().self()); 47 | debug("Waiting for nodes to meet each other, swear I had something for this..."); 48 | }); 49 | -------------------------------------------------------------------------------- /Gruntfile.js: -------------------------------------------------------------------------------- 1 | var path = require("path"), 2 | _ = require("lodash"); 3 | 4 | module.exports = function (grunt) { 5 | var files = [ 6 | "index.js", 7 | "lib/chash.js", 8 | "lib/cluster_node.js", 9 | "lib/conn.js", 10 | "lib/gen_server.js", 11 | "lib/gossip.js", 12 | "lib/kernel.js", 13 | "lib/node.js", 14 | "lib/vclock.js", 15 | "lib/dtable.js", 16 | "lib/mtable.js", 17 | "lib/dlm/dlm.js", 18 | "lib/dlm/lock.js", 19 | "lib/dsem/semaphore.js", 20 | "lib/dsem/dsm.js" 21 | ]; 22 | 23 | grunt.initConfig({ 24 | pkg: grunt.file.readJSON("package.json"), 25 | 26 | jsdoc: { 27 | dist: { 28 | src: files, 29 | options: { 30 | readme: "README.md", 31 | destination: "doc", 32 | configure: ".jsdoc.conf.json" 33 | } 34 | } 35 | }, 36 | 37 | jsdoc2md: { 38 | separateOutputFilePerInput: { 39 | files: files.map((file) => { 40 | var path = file.split("/"); 41 | var last = _.last(path); 42 | last = last.slice(0, last.length-3) + ".md"; 43 | path[path.length-1] = last; 44 | return {src: file, dest: "doc/" + _.drop(path).join("/")}; 45 | }) 46 | } 47 | }, 48 | 49 | mochaTest: { 50 | run: { 51 | options: {reporter: "spec", checkLeaks: true}, 52 | src: ["test/**/*.js"] 53 | } 54 | } 55 | }); 56 | 57 | grunt.loadNpmTasks("grunt-jsdoc"); 58 | grunt.loadNpmTasks("grunt-jsdoc-to-markdown"); 59 | // create documentation 60 | grunt.registerTask("docs", ["jsdoc"]); 61 | grunt.registerTask("md", ["jsdoc2md"]); 62 | 63 | grunt.loadNpmTasks("grunt-contrib-jshint"); 64 | grunt.loadNpmTasks("grunt-mocha-test"); 65 | 66 | grunt.registerTask("test", ["mochaTest:run"]); 67 | 68 | grunt.registerTask("default", ["test"]); 69 | }; 70 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "notp", 3 | "version": "2.0.1", 4 | "description": "Distributed systems library for gossip protocols, consistent hash rings, and vector clocks.", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "grunt", 8 | "test-travis": "./node_modules/istanbul/lib/cli.js cover ./node_modules/mocha/bin/_mocha -- -R spec ./test/*.js" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "https://github.com/azuqua/notp.git" 13 | }, 14 | "engines": { 15 | "node": ">=6.9.1" 16 | }, 17 | "author": "Kevin Wilson ", 18 | "license": "MIT", 19 | "bugs": { 20 | "url": "https://github.com/azuqua/notp/issues" 21 | }, 22 | "keywords": [ 23 | "consistent hashing", 24 | "vector clocks", 25 | "gossip protocol", 26 | "distributed", 27 | "decentralized", 28 | "communication", 29 | "IPC", 30 | "cluster", 31 | "full-mesh topology", 32 | "actor model" 33 | ], 34 | "bin": { 35 | "ncl": "./bin/cli.js" 36 | }, 37 | "homepage": "https://github.com/azuqua/notp", 38 | "dependencies": { 39 | "async": "^1.5.2", 40 | "debug": "^2.2.0", 41 | "functional-red-black-tree": "^1.0.1", 42 | "lodash": "^4.17.15", 43 | "lru-cache": "^4.1.1", 44 | "microtime": "^2.1.2", 45 | "node-ipc": "^8.9.3", 46 | "shortid": "^2.2.8", 47 | "uuid": "^3.0.1", 48 | "vorpal": "^1.11.4", 49 | "yargs": "^7.0.2" 50 | }, 51 | "devDependencies": { 52 | "chai": "^3.5.0", 53 | "coveralls": "^2.12.0", 54 | "docdash": "^0.4.0", 55 | "grunt": "^1.0.1", 56 | "grunt-contrib-jshint": "^1.0.0", 57 | "grunt-jsdoc": "^2.1.0", 58 | "grunt-jsdoc-to-markdown": "^3.0.0", 59 | "grunt-mocha-test": "^0.12.7", 60 | "ink-docstrap": "^1.3.0", 61 | "istanbul": "^0.4.5", 62 | "mocha": "^3.0.2", 63 | "node-yaml-config": "*", 64 | "sinon": "^1.17.5", 65 | "supertest": "^2.0.0" 66 | }, 67 | "directories": { 68 | "test": "test" 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /test/unit/queue.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | assert = require("chai").assert; 4 | 5 | module.exports = function (mocks, lib) { 6 | describe("Queue unit tests", function () { 7 | var Queue = lib.queue, 8 | queue; 9 | 10 | beforeEach(function () { 11 | queue = new Queue(); 12 | }); 13 | 14 | it("Should construct a queue", function () { 15 | assert.lengthOf(queue._front, 0); 16 | assert.lengthOf(queue._back, 0); 17 | }); 18 | 19 | it("Should enqueue value", function () { 20 | queue.enqueue("value"); 21 | assert.deepEqual(queue._back, ["value"]); 22 | queue.enqueue("value2"); 23 | assert.deepEqual(queue._back, ["value", "value2"]); 24 | }); 25 | 26 | it("Should dequeue value", function () { 27 | queue.enqueue("value"); 28 | queue.enqueue("value2"); 29 | var val = queue.dequeue(); 30 | assert.equal(val, "value"); 31 | queue.enqueue("value3"); 32 | val = queue.dequeue(); 33 | assert.equal(val, "value2"); 34 | assert.lengthOf(queue._front, 0); 35 | assert.lengthOf(queue._back, 1); 36 | }); 37 | 38 | it("Should flush queue", function () { 39 | var vals = ["foo", "bar", "baz"]; 40 | vals.forEach((val) => { 41 | queue.enqueue(val); 42 | }); 43 | var out = queue.flush(); 44 | assert.deepEqual(out, vals); 45 | assert.lengthOf(queue._front, 0); 46 | assert.lengthOf(queue._back, 0); 47 | }); 48 | 49 | it("Should return queue size", function () { 50 | assert.equal(queue.size(), 0); 51 | queue.enqueue("foo"); 52 | assert.equal(queue.size(), 1); 53 | }); 54 | 55 | it("Should return first entry when peeking", function () { 56 | queue.enqueue("foo"); 57 | queue.enqueue("bar"); 58 | assert.equal(queue.peek(), "foo"); 59 | queue.dequeue(); 60 | assert.equal(queue.peek(), "bar"); 61 | queue.dequeue(); 62 | assert.equal(queue.peek(), undefined); 63 | }); 64 | }); 65 | }; 66 | -------------------------------------------------------------------------------- /lib/dlm/lock.js: -------------------------------------------------------------------------------- 1 | class Lock { 2 | /** 3 | * 4 | * Lock class to hold the type, id, and timeout information for a lock. Additional metadata about the holder(s) of a lock, such as start time and initial TTL, are stored in the DLM server's underlying table. 5 | * 6 | * @class Lock 7 | * @memberof Clusterluck 8 | * 9 | * @param {String} type - Type of lock. Can either be 'read' be 'write'. 10 | * @param {String} id - ID of lock. 11 | * @param {Timeout|Map} timeout - Timeout object for the lock. If a read lock, this value will be a map from holders to timeout objects. 12 | * 13 | */ 14 | constructor(type, id, timeout) { 15 | this._type = type; 16 | this._id = id; 17 | this._timeout = timeout; 18 | } 19 | 20 | /** 21 | * 22 | * Acts as a getter/setter for the type of this lock. 23 | * 24 | * @method type 25 | * @memberof Clusterluck.Lock 26 | * @instance 27 | * 28 | * @param {String} [type] - Type to set on this lock. 29 | * 30 | * @return {String} This lock's type. 31 | * 32 | */ 33 | type(type) { 34 | if (type !== undefined) { 35 | this._type = type; 36 | } 37 | return this._type; 38 | } 39 | 40 | /** 41 | * 42 | * Acts as a getter/setter for the ID of this lock. 43 | * 44 | * @method id 45 | * @memberof Clusterluck.Lock 46 | * @instance 47 | * 48 | * @param {String} [id] - ID to set on this lock. 49 | * 50 | * @return {String} This lock's ID. 51 | * 52 | */ 53 | id(id) { 54 | if (id !== undefined) { 55 | this._id = id; 56 | } 57 | return this._id; 58 | } 59 | 60 | /** 61 | * 62 | * Acts as a getter/setter for the timeout of this lock. 63 | * 64 | * @method timeout 65 | * @memberof Clusterluck.Lock 66 | * @instance 67 | * 68 | * @param {Timeout|Map} [timeout] - Timeout to set on this lock. 69 | * 70 | * @return {Timeout|Map} This lock's timeout object(s). 71 | * 72 | */ 73 | timeout(timeout) { 74 | if (timeout !== undefined) { 75 | this._timeout = timeout; 76 | } 77 | return this._timeout; 78 | } 79 | } 80 | 81 | module.exports = Lock; 82 | -------------------------------------------------------------------------------- /lib/dsem/semaphore.js: -------------------------------------------------------------------------------- 1 | class Semaphore { 2 | /** 3 | * 4 | * Semaphore class to hold the id, concurrency limit, and timeouts information for a semaphore. Additional metadata about the holders of a semaphore, such as start time and initial TTL, are stored in the DSem server's underlying table. 5 | * 6 | * @class Semaphore 7 | * @memberof Clusterluck 8 | * 9 | * @param {String} id - ID of semaphore. 10 | * @param {Number} size - Concurrency limit of semaphore. 11 | * @param {Map} timeouts - Map from holders to timeout objects. 12 | * 13 | */ 14 | constructor(id, size, timeouts) { 15 | this._id = id; 16 | this._size = size; 17 | this._timeouts = timeouts; 18 | } 19 | 20 | /** 21 | * 22 | * Acts as a getter/setter for the ID of this semaphore. 23 | * 24 | * @method id 25 | * @memberof Clusterluck.Semaphore 26 | * @instance 27 | * 28 | * @param {String} [id] - ID to set on this semaphore. 29 | * 30 | * @return {String} This semaphore's ID. 31 | * 32 | */ 33 | id(id) { 34 | if (id !== undefined) { 35 | this._id = id; 36 | } 37 | return this._id; 38 | } 39 | 40 | /** 41 | * 42 | * Acts as a getter/setter for the concurrency limit of this semaphore. 43 | * 44 | * @method size 45 | * @memberof Clusterluck.Semaphore 46 | * @instance 47 | * 48 | * @param {Number} [size] - Concurrency limit to set on this semaphore. 49 | * 50 | * @return {Number} This semaphore's concurrency limit. 51 | * 52 | */ 53 | size(size) { 54 | if (size !== undefined) { 55 | this._size = size; 56 | } 57 | return this._size; 58 | } 59 | 60 | /** 61 | * 62 | * Acts as a getter/setter for the timeout map of this semaphore. 63 | * 64 | * @method timeouts 65 | * @memberof Clusterluck.Semaphore 66 | * @instance 67 | * 68 | * @param {Map} [timeouts] - Timeout map to set on this semaphore. 69 | * 70 | * @return {Map} Timeout map of this semaphore. 71 | * 72 | */ 73 | timeouts(timeouts) { 74 | if (timeouts !== undefined) { 75 | this._timeouts = timeouts; 76 | } 77 | return this._timeouts; 78 | } 79 | } 80 | 81 | module.exports = Semaphore; 82 | -------------------------------------------------------------------------------- /test/unit/node.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | assert = require("chai").assert; 4 | 5 | module.exports = function (mocks, lib) { 6 | describe("Node unit tests", function () { 7 | var Node = lib.node; 8 | var node, 9 | id = "id", 10 | host = "localhost", 11 | port = 8000; 12 | beforeEach(function () { 13 | node = new Node(id, host, port); 14 | }); 15 | 16 | it("Should construct a node", function () { 17 | assert.equal(node._id, id); 18 | assert.equal(node._host, host); 19 | assert.equal(node._port, port); 20 | 21 | node = new Node("id2", host, port); 22 | assert.equal(node._id, "id2"); 23 | assert.equal(node._host, host); 24 | assert.equal(node._port, port); 25 | }); 26 | 27 | it("Should get/set the id of a node", function () { 28 | assert.equal(node.id(), id); 29 | node.id("id2"); 30 | assert.equal(node.id(), "id2"); 31 | node.id(5); 32 | assert.equal(node.id(), "id2"); 33 | }); 34 | 35 | it("Should get/set the host of a node", function () { 36 | assert.equal(node.host(), host); 37 | node.host("localhost2"); 38 | assert.equal(node.host(), "localhost2"); 39 | node.host(5); 40 | assert.equal(node.host(), "localhost2"); 41 | }); 42 | 43 | it("Should get/set the port of a node", function () { 44 | assert.equal(node.port(), port); 45 | node.port(8001); 46 | assert.equal(node.port(), 8001); 47 | node.port("foo"); 48 | assert.equal(node.port(), 8001); 49 | }); 50 | 51 | it("Should compare two different nodes", function () { 52 | assert(node.equals(node)); 53 | var node2 = new Node("id2", "localhost2", 8001); 54 | assert.notOk(node.equals(node2)); 55 | }); 56 | 57 | it("Should turn node into JSON", function () { 58 | var out = node.toJSON(); 59 | assert.deepEqual(out, { 60 | id: node.id(), 61 | host: node.host(), 62 | port: node.port() 63 | }); 64 | out = node.toJSON(true); 65 | assert.deepEqual(out, { 66 | id: node.id(), 67 | host: node.host(), 68 | port: node.port() 69 | }); 70 | }); 71 | 72 | it("Should create node from JSON", function () { 73 | var out = node.toJSON(true); 74 | var node2 = Node.from(out); 75 | assert(node.equals(node2)); 76 | }); 77 | }); 78 | }; 79 | -------------------------------------------------------------------------------- /examples/gen_server/dlm/test.js: -------------------------------------------------------------------------------- 1 | const cl = require("../../../index"), 2 | os = require("os"), 3 | assert = require("chai").assert, 4 | debug = require("debug")("examples:gen_server:dlm:test"), 5 | async = require("async"); 6 | 7 | const nodeID = process.argv[2], 8 | port = parseInt(process.argv[3]), 9 | host = os.hostname(); 10 | 11 | const node = cl.createCluster(nodeID, host, port), 12 | gossip = node.gossip(), 13 | kernel = node.kernel(); 14 | 15 | const DLMServer = require("./dlm"); 16 | const dlm = new DLMServer(gossip, kernel, { 17 | rquorum: 0.51, 18 | wquorum: 0.51 19 | }); 20 | 21 | // load node state 22 | node.load(() => { 23 | // first start dlm, then start network kernel 24 | dlm.start("locker"); 25 | node.start("cookie", "ring", () => { 26 | debug("Node %s listening on hostname %s, port %s!", nodeID, host, port); 27 | var holdingNodes; 28 | async.series([ 29 | (next) => { 30 | dlm.wlock("id", "holder", 30000, (err, nodes) => { 31 | assert.notOk(err); 32 | debug("Successfully grabbed write lock on id '%s' with holder '%s'", "id", "holder"); 33 | assert.isArray(nodes); 34 | holdingNodes = nodes; 35 | next(); 36 | }); 37 | }, 38 | (next) => { 39 | dlm.wlock("id", "holder2", 30000, (err, wNodes) => { 40 | assert.ok(err); 41 | debug("Failed to grab write lock id '%s' with holder '%s'", "id", "holder2"); 42 | next(); 43 | }); 44 | }, 45 | (next) => { 46 | dlm.wunlock(holdingNodes, "id", "holder", (err) => { 47 | assert.notOk(err); 48 | next(); 49 | }); 50 | }, 51 | (next) => { 52 | dlm.rlock("id", "holder", 30000, (err, nodes) => { 53 | assert.notOk(err); 54 | debug("Successfully grabbed read lock on id '%s' with holder '%s'", "id", "holder"); 55 | assert.isArray(nodes); 56 | holdingNodes = nodes; 57 | next(); 58 | }); 59 | }, 60 | (next) => { 61 | dlm.rlock("id", "holder2", 30000, (err, nodes) => { 62 | assert.notOk(err); 63 | debug("Successfully grabbed read lock on id '%s' with holder '%s'", "id", "holder2"); 64 | next(); 65 | }); 66 | }, 67 | (next) => { 68 | dlm.wlock("id", "holder3", 30000, (err, wNodes) => { 69 | assert.ok(err); 70 | debug("Failed to grab write lock id '%s' with holder '%s'", "id", "holder2"); 71 | next(); 72 | }); 73 | }, 74 | (next) => { 75 | dlm.runlockAsync(holdingNodes, "id", "holder"); 76 | dlm.runlockAsync(holdingNodes, "id", "holder2"); 77 | next(); 78 | } 79 | ], () => { 80 | dlm.stop(true); 81 | debug("Done!"); 82 | process.exit(0); 83 | }); 84 | }); 85 | }); 86 | -------------------------------------------------------------------------------- /lib/consts.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | os = require("os"); 3 | 4 | // vector clock trim defaults 5 | var vclockOpts = { 6 | // number of elements that need to exist for trimming to occur 7 | lowerBound: 10, 8 | // how old the youngest member needs to be before considering trimming 9 | youngBound: 20000000, 10 | // when trimming, trim to at least this number of elements 11 | upperBound: 50, 12 | // when trimming, trim any members at least this old 13 | oldBound: 86400000000 14 | }; 15 | 16 | var dtableOpts = { 17 | writeThreshold: 100, 18 | autoSave: 180000, 19 | fsyncInterval: 1000 20 | }; 21 | 22 | var networkHost = os.hostname(); 23 | var networkPort = 7022; 24 | 25 | var connOpts = { 26 | maxLen: 1024 27 | }; 28 | 29 | module.exports = { 30 | networkHost: networkHost, 31 | networkPort: networkPort, 32 | // defaults for network kernel 33 | kernelOpts: Object.freeze({ 34 | networkHost: networkHost, 35 | networkPort: networkPort, 36 | // how long to wait before retrying a connection attempt between 37 | // two nodes 38 | retry: 5000, 39 | // TLS options; see https://riaevangelist.github.io/node-ipc/ for accepted options 40 | tls: null, 41 | // number of attempts to reconnect to a node; currently Infinity is supported since 42 | // the Connection class only listens on the 'connect' and 'disconnect' events 43 | maxRetries: Infinity, 44 | // silence all node-ipc logs 45 | silent: true 46 | }), 47 | // defaults for IPC connection 48 | connDefaults: Object.freeze(connOpts), 49 | // defaults for gossip processor 50 | gossipOpts: Object.freeze({ 51 | // replication factor for the consistent hash ring 52 | rfactor: 3, 53 | // persistence factor for the consistent hash ring 54 | pfactor: 2, 55 | // interval to communicate with a random member of the cluster 56 | interval: 1000, 57 | // interval to flush the ring to disk 58 | flushInterval: 1000, 59 | // path to flush state to; by default, state is not flushed 60 | flushPath: null, 61 | // vector clock options for managing the internal vector clock corresponding 62 | // to the hash ring 63 | vclockOpts: _.cloneDeep(vclockOpts), 64 | connOpts: _.clone(connOpts) 65 | }), 66 | // vector clock defaults, in the event we want direct creation/manipulation of vector 67 | // clocks 68 | vclockOpts: Object.freeze(_.cloneDeep(vclockOpts)), 69 | chashOpts: Object.freeze({ 70 | maxCacheSize: 5000 71 | }), 72 | dtableOpts: Object.freeze(dtableOpts), 73 | dlmOpts: Object.freeze(_.extend({ 74 | rquorum: 0.51, 75 | wquorum: 0.51, 76 | rfactor: 3, 77 | minWaitTimeout: 10, 78 | maxWaitTimeout: 100, 79 | disk: false 80 | }, dtableOpts)), 81 | dsemOpts: Object.freeze(_.extend({ 82 | minWaitTimeout: 10, 83 | maxWaitTimeout: 100, 84 | disk: false 85 | }, dtableOpts)) 86 | }; 87 | -------------------------------------------------------------------------------- /test/mocks/ipc.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | uuid = require("uuid"), 3 | async = require("async"), 4 | EventEmitter = require("events").EventEmitter, 5 | sinon = require("sinon"); 6 | 7 | var network = new Map(); 8 | 9 | class Socket extends EventEmitter { 10 | constructor(id) { 11 | super(); 12 | this.id = id; 13 | } 14 | } 15 | 16 | class MockIPC { 17 | constructor(opts) { 18 | this.config = opts || {}; 19 | this._id = null; 20 | this._host = null; 21 | this._port = null; 22 | this.of = {}; 23 | this.server = null; 24 | } 25 | 26 | serveNet(host, port, cb) { 27 | this._id = this.config.id; 28 | this._host = host; 29 | this._port = port; 30 | if (network.has(this._id)) { 31 | return async.nextTick(cb); 32 | } 33 | this.server = new Server(this._id, this._host, this._port); 34 | return async.nextTick(cb); 35 | } 36 | 37 | connectToNet(id, host, port, cb) { 38 | var server = network.get(id); 39 | var conn = new Conn(server); 40 | this.of[id] = conn; 41 | server.on(conn.id(), (event, data) => { 42 | var socket = server.getSocket(conn._id); 43 | server.emit(event, data, socket); 44 | }); 45 | async.nextTick(() => { 46 | conn.emit("connect"); 47 | if (typeof cb === "function") return cb(); 48 | }); 49 | } 50 | 51 | disconnect(id) { 52 | var conn = this.of[id]; 53 | var server = network.get(id); 54 | server.removeSocket(conn.id()); 55 | delete this.of[id]; 56 | async.nextTick(() => { 57 | conn.emit("disconnect"); 58 | conn.removeAllListeners(); 59 | }); 60 | } 61 | 62 | network() { 63 | return network; 64 | } 65 | } 66 | 67 | class Server extends EventEmitter { 68 | constructor(id, host, port) { 69 | super(); 70 | this._id = id; 71 | this._host = host; 72 | this._port = port; 73 | this._sockets = new Map(); 74 | } 75 | 76 | start() { 77 | network.set(this._id, this); 78 | return this; 79 | } 80 | 81 | stop() { 82 | this._sockets = new Map(); 83 | this.removeAllListeners(); 84 | network.delete(this._id); 85 | return this; 86 | } 87 | 88 | addSocket(id) { 89 | this._sockets.set(id, new Socket(id)); 90 | return this; 91 | } 92 | 93 | getSocket(id) { 94 | return this._sockets.get(id); 95 | } 96 | 97 | removeSocket(id) { 98 | var socket = this._sockets.get(id); 99 | this._sockets.delete(id); 100 | this.emit("socket.disconnected", socket, id); 101 | return this; 102 | } 103 | } 104 | 105 | class Conn extends EventEmitter { 106 | constructor(server) { 107 | super(); 108 | this._id = uuid.v4(); 109 | this._server = server; 110 | this._server.addSocket(this._id); 111 | this.socket = new Socket(); 112 | } 113 | 114 | id() { 115 | return this._id; 116 | } 117 | 118 | emit(event) { 119 | var args = _.drop(Array.from(arguments)); 120 | this._server.emit.apply(this._server, [this._id, event].concat(args)); 121 | return super.emit.apply(this, [event].concat(args)); 122 | } 123 | } 124 | 125 | module.exports = MockIPC; 126 | -------------------------------------------------------------------------------- /examples/usage/dlm/test.js: -------------------------------------------------------------------------------- 1 | const cl = require("../../../index"), 2 | _ = require("lodash"), 3 | os = require("os"), 4 | assert = require("chai").assert, 5 | debug = require("debug")("examples:usage:dlm:test"), 6 | async = require("async"), 7 | DLMServer = cl.DLMServer; 8 | 9 | const nodeID = process.argv[2], 10 | port = parseInt(process.argv[3]), 11 | nodeID2 = process.argv[4], 12 | port2 = parseInt(process.argv[5]), 13 | host = os.hostname(); 14 | 15 | var nodes = []; 16 | var dlms = []; 17 | async.each([[nodeID, port], [nodeID2, port2]], (config, next) => { 18 | const node = cl.createCluster(config[0], host, config[1]), 19 | gossip = node.gossip(), 20 | kernel = node.kernel(); 21 | 22 | const dlm = new DLMServer(gossip, kernel, { 23 | rquorum: 0.51, 24 | wquorum: 0.51 25 | }); 26 | nodes.push(node); 27 | dlms.push(dlm); 28 | node.load(() => { 29 | dlm.start("locker"); 30 | node.start("cookie", "ring", () => { 31 | debug("Node %s listening on hostname %s, port %s!", config[0], host, config[1]); 32 | next(); 33 | }); 34 | }); 35 | }, () => { 36 | setTimeout(() => { 37 | // make sure nodes know about each other 38 | assert.ok(_.find(nodes[0].gossip().ring().nodes(), (node) => { 39 | return node.equals(nodes[1].kernel().self()); 40 | })); 41 | var dlm = dlms[0]; 42 | async.series([ 43 | (next) => { 44 | dlm.wlock("id", "holder", 30000, (err, nodes) => { 45 | assert.notOk(err); 46 | debug("Successfully grabbed write lock on id '%s' with holder '%s'", "id", "holder"); 47 | assert.lengthOf(nodes, 2); 48 | next(); 49 | }, 1000, 1); 50 | }, 51 | (next) => { 52 | dlm.wlock("id", "holder2", 30000, (err, wNodes) => { 53 | assert.ok(err); 54 | debug("Failed to grab write lock id '%s' with holder '%s'", "id", "holder2"); 55 | next(); 56 | }, 1000, 1); 57 | }, 58 | (next) => { 59 | dlm.wunlock("id", "holder", (err) => { 60 | assert.notOk(err); 61 | next(); 62 | }); 63 | }, 64 | (next) => { 65 | dlm.rlock("id", "holder", 30000, (err, nodes) => { 66 | assert.notOk(err); 67 | debug("Successfully grabbed read lock on id '%s' with holder '%s'", "id", "holder"); 68 | assert.lengthOf(nodes, 2); 69 | next(); 70 | }, 1000, 1); 71 | }, 72 | (next) => { 73 | dlm.rlock("id", "holder2", 30000, (err, nodes) => { 74 | assert.notOk(err); 75 | debug("Successfully grabbed read lock on id '%s' with holder '%s'", "id", "holder2"); 76 | next(); 77 | }, 1000, 1); 78 | }, 79 | (next) => { 80 | dlm.wlock("id", "holder3", 30000, (err, wNodes) => { 81 | assert.ok(err); 82 | debug("Failed to grab write lock id '%s' with holder '%s'", "id", "holder2"); 83 | next(); 84 | }, 1000, 1); 85 | }, 86 | (next) => { 87 | dlm.runlockAsync("id", "holder"); 88 | dlm.runlockAsync("id", "holder2"); 89 | next(); 90 | } 91 | ], () => { 92 | dlms.forEach((dlm) => { 93 | dlm.stop(); 94 | }); 95 | debug("Done!"); 96 | process.exit(0); 97 | }); 98 | }, 1000); 99 | // make nodes meet each other first 100 | nodes[0].gossip().meet(nodes[1].kernel().self()); 101 | debug("Waiting for nodes to meet each other, swear I had something for this..."); 102 | }); 103 | -------------------------------------------------------------------------------- /lib/node.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | debug = require("debug")("notp:lib:node"); 3 | 4 | class Node { 5 | /** 6 | * 7 | * Node abstraction class. 8 | * 9 | * @class Node Node 10 | * @memberof Clusterluck 11 | * 12 | * @param {String} id - ID of this node. 13 | * @param {String} host - Hostname of this node. 14 | * @param {Number} port - Port number of this node. 15 | * 16 | */ 17 | constructor(id, host, port) { 18 | this._id = id; 19 | this._host = host; 20 | this._port = port; 21 | } 22 | 23 | /** 24 | * 25 | * Acts as a getter/setter for the ID of this node. 26 | * 27 | * @method id 28 | * @memberof NetKernel.Node 29 | * @instance 30 | * 31 | * @param {String} [id] - ID to set. 32 | * 33 | * @return {String} ID of this instance. 34 | * 35 | */ 36 | id(id) { 37 | if (typeof id === "string") { 38 | this._id = id; 39 | } 40 | return this._id; 41 | } 42 | 43 | /** 44 | * 45 | * Acts as a getter/setter for the hostname of this node. 46 | * 47 | * @method host 48 | * @memberof NetKernel.Node 49 | * @instance 50 | * 51 | * @param {String} [host] - Hostname to set. 52 | * 53 | * @return {String} Hostname of this instance. 54 | * 55 | */ 56 | host(host) { 57 | if (typeof host === "string") { 58 | this._host = host; 59 | } 60 | return this._host; 61 | } 62 | 63 | /** 64 | * 65 | * Acts as a getter/setter for the port of this node. 66 | * 67 | * @method port 68 | * @memberof NetKernel.Node 69 | * @instance 70 | * 71 | * @param {Number} [port] - Port to set. 72 | * 73 | * @return {Number} Port of this instance. 74 | * 75 | */ 76 | port(port) { 77 | if (typeof port === "number") { 78 | this._port = port; 79 | } 80 | return this._port; 81 | } 82 | 83 | /** 84 | * 85 | * Returns whether this instance equals another node. Existence means the id, host, and port match on both instances. 86 | * 87 | * @method equals 88 | * @memberof NetKernel.Node 89 | * @instance 90 | * 91 | * @param {Clusterluck.Node} node - Node to check equality against. 92 | * 93 | * @return {Boolean} Whether this instance equals `node`. 94 | * 95 | */ 96 | equals(node) { 97 | return this._id === node.id() && 98 | this._host === node.host() && 99 | this._port === node.port(); 100 | } 101 | 102 | /** 103 | * 104 | * Computes the JSON serialization of this instance. 105 | * 106 | * @method toJSON 107 | * @memberof NetKernel.Node 108 | * @instance 109 | * 110 | * @param {Boolean} [fast] - Whether to create a copy of internal state when exporting to JSON or not. 111 | * 112 | * @return {Object} JSON serialization of this instance. 113 | * 114 | */ 115 | toJSON(fast) { 116 | var out = { 117 | id: this._id, 118 | host: this._host, 119 | port: this._port 120 | }; 121 | return fast === true ? out : _.clone(out); 122 | } 123 | 124 | /** 125 | * 126 | * Create an instance of the Node class from JSON data. 127 | * 128 | * @method from 129 | * @memberof NetKernel.Node 130 | * @static 131 | * 132 | * @param {Object} data - Data to initialize node state from. 133 | * @param {String} data.id - ID of node. 134 | * @param {String} data.host - Hostname of node. 135 | * @param {Number} data.port - Port of node. 136 | * 137 | * @return {Clusterluck.Node} New instance of Node class. 138 | * 139 | */ 140 | static from(data) { 141 | return new Node(data.id, data.host, data.port); 142 | } 143 | } 144 | 145 | module.exports = Node; 146 | -------------------------------------------------------------------------------- /examples/gen_server/dlm/multi_test.js: -------------------------------------------------------------------------------- 1 | const cl = require("../../../index"), 2 | _ = require("lodash"), 3 | os = require("os"), 4 | assert = require("chai").assert, 5 | debug = require("debug")("examples:gen_server:dlm:multi_test"), 6 | async = require("async"), 7 | DLMServer = require("./dlm"); 8 | 9 | const nodeID = process.argv[2], 10 | port = parseInt(process.argv[3]), 11 | nodeID2 = process.argv[4], 12 | port2 = parseInt(process.argv[5]), 13 | host = os.hostname(); 14 | 15 | var nodes = []; 16 | var dlms = []; 17 | async.each([[nodeID, port], [nodeID2, port2]], (config, next) => { 18 | const node = cl.createCluster(config[0], host, config[1]), 19 | gossip = node.gossip(), 20 | kernel = node.kernel(); 21 | 22 | const dlm = new DLMServer(gossip, kernel, { 23 | rquorum: 0.51, 24 | wquorum: 0.51 25 | }); 26 | nodes.push(node); 27 | dlms.push(dlm); 28 | node.load(() => { 29 | dlm.start("locker"); 30 | node.start("cookie", "ring", () => { 31 | debug("Node %s listening on hostname %s, port %s!", config[0], host, config[1]); 32 | next(); 33 | }); 34 | }); 35 | }, () => { 36 | setTimeout(() => { 37 | // make sure nodes know about each other 38 | assert.ok(_.find(nodes[0].gossip().ring().nodes(), (node) => { 39 | return node.equals(nodes[1].kernel().self()); 40 | })); 41 | var dlm = dlms[0]; 42 | var holdingNodes; 43 | async.series([ 44 | (next) => { 45 | dlm.wlock("id", "holder", 30000, (err, nodes) => { 46 | assert.notOk(err); 47 | debug("Successfully grabbed write lock on id '%s' with holder '%s'", "id", "holder"); 48 | assert.lengthOf(nodes, 2); 49 | holdingNodes = nodes; 50 | next(); 51 | }); 52 | }, 53 | (next) => { 54 | dlm.wlock("id", "holder2", 30000, (err, wNodes) => { 55 | assert.ok(err); 56 | debug("Failed to grab write lock id '%s' with holder '%s'", "id", "holder2"); 57 | next(); 58 | }); 59 | }, 60 | (next) => { 61 | dlm.wunlock(holdingNodes, "id", "holder", (err) => { 62 | assert.notOk(err); 63 | next(); 64 | }); 65 | }, 66 | (next) => { 67 | dlm.rlock("id", "holder", 30000, (err, nodes) => { 68 | assert.notOk(err); 69 | debug("Successfully grabbed read lock on id '%s' with holder '%s'", "id", "holder"); 70 | assert.lengthOf(nodes, 2); 71 | holdingNodes = nodes; 72 | next(); 73 | }); 74 | }, 75 | (next) => { 76 | dlm.rlock("id", "holder2", 30000, (err, nodes) => { 77 | assert.notOk(err); 78 | debug("Successfully grabbed read lock on id '%s' with holder '%s'", "id", "holder2"); 79 | next(); 80 | }); 81 | }, 82 | (next) => { 83 | dlm.wlock("id", "holder3", 30000, (err, wNodes) => { 84 | assert.ok(err); 85 | debug("Failed to grab write lock id '%s' with holder '%s'", "id", "holder2"); 86 | next(); 87 | }); 88 | }, 89 | (next) => { 90 | dlm.runlockAsync(holdingNodes, "id", "holder"); 91 | dlm.runlockAsync(holdingNodes, "id", "holder2"); 92 | next(); 93 | } 94 | ], () => { 95 | dlms.forEach((dlm) => { 96 | dlm.stop(true); 97 | }); 98 | debug("Done!"); 99 | process.exit(0); 100 | }); 101 | }, 1000); 102 | // make nodes meet each other first 103 | nodes[0].gossip().meet(nodes[1].kernel().self()); 104 | debug("Waiting for nodes to meet each other, swear I had something for this..."); 105 | }); 106 | -------------------------------------------------------------------------------- /examples/usage/dsm/test.js: -------------------------------------------------------------------------------- 1 | const cl = require("../../../index"), 2 | _ = require("lodash"), 3 | os = require("os"), 4 | assert = require("chai").assert, 5 | debug = require("debug")("examples:usage:dsm:test"), 6 | async = require("async"), 7 | DSMServer = cl.DSMServer; 8 | 9 | const nodeID = process.argv[2], 10 | port = parseInt(process.argv[3]), 11 | nodeID2 = process.argv[4], 12 | port2 = parseInt(process.argv[5]), 13 | host = os.hostname(); 14 | 15 | var nodes = []; 16 | var dsms = []; 17 | async.each([[nodeID, port], [nodeID2, port2]], (config, next) => { 18 | const node = cl.createCluster(config[0], host, config[1]), 19 | gossip = node.gossip(), 20 | kernel = node.kernel(); 21 | 22 | const dsm = new DSMServer(gossip, kernel, { 23 | rquorum: 0.51, 24 | wquorum: 0.51 25 | }); 26 | nodes.push(node); 27 | dsms.push(dsm); 28 | node.load(() => { 29 | dsm.start("sem_server"); 30 | node.start("cookie", "ring", () => { 31 | debug("Node %s listening on hostname %s, port %s!", config[0], host, config[1]); 32 | next(); 33 | }); 34 | }); 35 | }, () => { 36 | setTimeout(() => { 37 | // make sure nodes know about each other 38 | assert.ok(_.find(nodes[0].gossip().ring().nodes(), (node) => { 39 | return node.equals(nodes[1].kernel().self()); 40 | })); 41 | var dsm = dsms[0]; 42 | async.series([ 43 | (next) => { 44 | dsm.create("id", 2, (err) => { 45 | assert.notOk(err); 46 | debug("Successfully created semaphore 'id' with concurrency limit of 3"); 47 | next(); 48 | }); 49 | }, 50 | (next) => { 51 | dsm.read("id", (err, out) => { 52 | assert.notOk(err); 53 | assert.deepEqual(out, { 54 | n: 2, 55 | active: 0 56 | }); 57 | next(); 58 | }); 59 | }, 60 | (next) => { 61 | dsm.post("id", "holder", 10000, (err) => { 62 | assert.notOk(err); 63 | debug("Successfully grabbed semaphore 'id' with holder 'holder'"); 64 | next(); 65 | }); 66 | }, 67 | (next) => { 68 | dsm.post("id", "holder2", 10000, (err) => { 69 | assert.notOk(err); 70 | debug("Successfully grabbed semaphore 'id' with holder 'holder2'"); 71 | next(); 72 | }); 73 | }, 74 | (next) => { 75 | dsm.post("id", "holder3", 10000, (err) => { 76 | assert.ok(err); 77 | debug("Failed to grab semaphore 'id' with holder 'holder3', limit reached"); 78 | next(); 79 | }, 1000, 0); 80 | }, 81 | (next) => { 82 | dsm.read("id", (err, out) => { 83 | assert.notOk(err); 84 | assert.deepEqual(out, { 85 | n: 2, 86 | active: 2 87 | }); 88 | next(); 89 | }); 90 | }, 91 | (next) => { 92 | async.each(["holder", "holder2"], (holder, done) => { 93 | dsm.close("id", holder, done); 94 | }, next); 95 | }, 96 | (next) => { 97 | dsm.read("id", (err, out) => { 98 | assert.notOk(err); 99 | assert.deepEqual(out, { 100 | n: 2, 101 | active: 0 102 | }); 103 | next(); 104 | }); 105 | }, 106 | (next) => { 107 | dsm.destroy("id", next); 108 | } 109 | ], () => { 110 | dsms.forEach((dsm) => { 111 | dsm.stop(); 112 | }); 113 | debug("Done!"); 114 | process.exit(0); 115 | }); 116 | }, 1000); 117 | // make nodes meet each other first 118 | nodes[0].gossip().meet(nodes[1].kernel().self()); 119 | debug("Waiting for nodes to meet each other, swear I had something for this..."); 120 | }); 121 | -------------------------------------------------------------------------------- /test/integration/gen_server.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | os = require("os"), 4 | ipc = require("node-ipc"), 5 | assert = require("chai").assert; 6 | 7 | var host = os.hostname(); 8 | 9 | module.exports = function (mocks, lib) { 10 | var GenServer = lib.gen_server; 11 | var Gossip = lib.gossip; 12 | var NetKernel = lib.kernel; 13 | var CHash = lib.chash; 14 | var VClock = lib.vclock; 15 | var consts = lib.consts; 16 | 17 | var kernelOpts = consts.kernelOpts; 18 | 19 | describe("GenServer integration tests", function () { 20 | var nodes = []; 21 | var serves = []; 22 | var origin, target; 23 | var node1 = "foo", node2 = "bar"; 24 | var port1 = 8000, port2 = 8001; 25 | 26 | before(function (done) { 27 | async.each([[node1, port1], [node2, port2]], (config, next) => { 28 | var inst = new ipc.IPC(); 29 | inst.config.networkHost = host; 30 | inst.config.networkPort = config[1]; 31 | inst.config.retry = kernelOpts.retry; 32 | inst.config.maxRetries = kernelOpts.maxRetries; 33 | inst.config.tls = kernelOpts.tls; 34 | inst.config.silent = kernelOpts.silent; 35 | const kernel = new NetKernel(inst, config[0], host, config[1]); 36 | var chash = (new CHash(3, 2)).insert(kernel.self()); 37 | var vclock = new VClock(); 38 | const gossip = new Gossip(kernel, chash, vclock, consts.gossipOpts); 39 | 40 | const server = new GenServer(kernel, { 41 | rquorum: 0.51, 42 | wquorum: 0.51 43 | }); 44 | server.on("echo", (data, from) => { 45 | server.reply(from, data); 46 | }); 47 | server.on("one-way", (data, from) => { 48 | server.emit("have a bone"); 49 | }); 50 | 51 | nodes.push({gossip: gossip, kernel: kernel}); 52 | serves.push(server); 53 | server.start("server_here"); 54 | gossip.start("ring"); 55 | kernel.start({cookie: "cookie"}); 56 | kernel.once("_ready", next); 57 | }, function () { 58 | origin = serves[0]; 59 | target = serves[1]; 60 | nodes[0].gossip.meet(nodes[1].kernel.self()); 61 | nodes[0].gossip.once("process", _.ary(done, 0)); 62 | }); 63 | }); 64 | 65 | after(function () { 66 | serves.forEach(function (server) { 67 | server.stop(true); 68 | }); 69 | nodes.forEach(function (node) { 70 | node.gossip.stop(true); 71 | node.kernel.sinks().forEach(function (sink) { 72 | node.kernel.disconnect(sink, true); 73 | }); 74 | node.kernel.stop(); 75 | }); 76 | }); 77 | 78 | it("Should send a call internally", function (done) { 79 | origin.call("server_here", "echo", "bar", (err,res) => { 80 | assert.notOk(err); 81 | assert.equal(res, "bar"); 82 | done(); 83 | }); 84 | }); 85 | 86 | it("Should send a call externally", function (done) { 87 | origin.call({id: "server_here", node: target.kernel().self()}, "echo", "bar", (err,res) => { 88 | assert.notOk(err); 89 | assert.equal(res, "bar"); 90 | done(); 91 | }); 92 | }); 93 | 94 | it("Should send a cast internally", function (done) { 95 | origin.once("have a bone", _.ary(done, 0)); 96 | origin.cast("server_here", "one-way", "bar"); 97 | }); 98 | 99 | it("Should send a cast externally", function (done) { 100 | origin.cast({id: "server_here", node: target.kernel().self()}, "one-way", "bar"); 101 | target.once("have a bone", () => { 102 | target.cast({id: "server_here", node: origin.kernel().self()}, "one-way", "baz"); 103 | }); 104 | origin.once("have a bone", _.ary(done, 0)); 105 | }); 106 | }); 107 | }; 108 | -------------------------------------------------------------------------------- /lib/utils.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | util = require("util"); 3 | 4 | var Node = require("./node"); 5 | var types = [ 6 | {key: "id", valid: [util.isString], str: "Node 'id' must be a JSON string."}, 7 | {key: "host", valid: [util.isString], str: "Node 'host' must be a JSON string."}, 8 | {key: "port", valid: [util.isNumber], str: "Node 'port' must be a JSON number."} 9 | ]; 10 | 11 | function parseNode(node) { 12 | if (!util.isObject(node)) { 13 | return new Error("foo"); 14 | } 15 | var misconfig = types.reduce((memo, check) => { 16 | var valid = _.some(check.valid, (fn) => { 17 | return fn(node[check.key]); 18 | }); 19 | if (!valid) memo[check.key] = check.str; 20 | return memo; 21 | }, {}); 22 | if (Object.keys(misconfig).length > 0) { 23 | return _.extend(new Error("Invalid node parameter types."), { 24 | fails: misconfig 25 | }); 26 | } 27 | return new Node(node.id, node.host, node.port); 28 | } 29 | 30 | function parseNodeListMemo(nodes, memo) { 31 | if (nodes.length === 0) return memo; 32 | var first = nodes.pop(); 33 | var out = parseNode(first); 34 | if (out instanceof Error) return out; 35 | memo.push(out); 36 | return parseNodeListMemo(nodes, memo); 37 | } 38 | 39 | var utils = { 40 | scanIterator: (iterator, num, memo = []) => { 41 | var value = iterator.next(); 42 | if (value.done === true) return {iterator: iterator, done: true, values: memo}; 43 | memo.push(value.value); 44 | num--; 45 | if (num === 0) return {iterator: iterator, values: memo}; 46 | return utils.scanIterator(iterator, num, memo); 47 | }, 48 | 49 | errorToObject: (err) => { 50 | err._error = err._error ? err._error : true; 51 | if (err instanceof Error) { 52 | return _.transform(err, (out, v, k) => { 53 | out[k] = v; 54 | }, _.extend(process.env.NODE_ENV === "production" ? {} : {stack: err.stack}, { 55 | message: err.message 56 | })); 57 | } 58 | return err; 59 | }, 60 | 61 | mapValues: (map) => { 62 | var memo = []; 63 | map.forEach((val) => { 64 | memo.push(val); 65 | }); 66 | return memo; 67 | }, 68 | 69 | safeParse: (val, reviver) => { 70 | try { 71 | return JSON.parse(val, reviver); 72 | } catch (e) { 73 | return e; 74 | } 75 | }, 76 | 77 | isPlainObject: (val) => { 78 | return util.isObject(val) && !Array.isArray(val); 79 | }, 80 | 81 | hasID: (data) => { 82 | if (!data || (!util.isString(data.id) || !data.id)) { 83 | return new Error("Missing required 'id' parameter."); 84 | } 85 | return data; 86 | }, 87 | 88 | parseNode: (data) => { 89 | if (!data) return new Error("Input data is not an object."); 90 | var out = parseNode(data.node); 91 | if (out instanceof Error) return out; 92 | data.node = out; 93 | return data; 94 | }, 95 | 96 | parseNodeList: (data) => { 97 | if (!data || !Array.isArray(data.nodes)) { 98 | return new Error("Invalid node list format, should be: [{id: , host: , port: }]."); 99 | } 100 | var out = parseNodeListMemo(data.nodes, []); 101 | if (out instanceof Error) return out; 102 | data.nodes = out; 103 | return data; 104 | }, 105 | 106 | mapToObject: (data) => { 107 | var obj = {}; 108 | data.forEach((val, key) => { 109 | obj[key] = val; 110 | }); 111 | return obj; 112 | }, 113 | 114 | setToList: (data) => { 115 | var memo = []; 116 | data.forEach((val) => { 117 | memo.push(val); 118 | }); 119 | return memo; 120 | }, 121 | 122 | mapToList: (data) => { 123 | var memo = []; 124 | data.forEach((val, key) => { 125 | memo.push([key, val]); 126 | }); 127 | return memo; 128 | }, 129 | 130 | isPositiveNumber: (val) => { 131 | return util.isNumber(val) && val > 0; 132 | } 133 | }; 134 | 135 | module.exports = utils; 136 | -------------------------------------------------------------------------------- /test/unit/mtable.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | assert = require("chai").assert; 4 | 5 | module.exports = function (mocks, lib) { 6 | var MTable = lib.mtable; 7 | 8 | describe("MTable unit tests", function () { 9 | var mtable; 10 | beforeEach(function (done) { 11 | mtable = new MTable({path: "./data"}); 12 | async.nextTick(done); 13 | }); 14 | 15 | it("Should construct a MTable", function () { 16 | assert.ok(_.isEqual(mtable._table, new Map())); 17 | }); 18 | 19 | it("Should start mtable instance", function () { 20 | mtable.start("foo"); 21 | assert.isString(mtable._id); 22 | }); 23 | 24 | it("Should stop mtable instance", function (done) { 25 | mtable.start("foo"); 26 | mtable.stop(() => { 27 | assert.equal(mtable._id, null); 28 | done(); 29 | }); 30 | }); 31 | 32 | it("Should check if mtable instance is idle", function () { 33 | assert.equal(mtable.idle(), true); 34 | }); 35 | 36 | it("Should get value in table", function () { 37 | mtable._table.set("key", "val"); 38 | assert.equal(mtable.get("key"), "val"); 39 | }); 40 | 41 | it("Should smember value in table", function () { 42 | mtable.sset("key", "val"); 43 | assert.equal(mtable.smember("key", "val"), true); 44 | 45 | mtable.set("key", "val"); 46 | assert.throws(_.partial(mtable.smember, "key", "val").bind(mtable)); 47 | }); 48 | 49 | it("Should hget value in table", function () { 50 | mtable.hset("key", "hkey", "val"); 51 | assert.equal(mtable.hget("key", "hkey"), "val"); 52 | 53 | mtable.set("key", "val"); 54 | assert.throws(_.partial(mtable.hget, "key", "hkey").bind(mtable)); 55 | }); 56 | 57 | it("Should set value in table", function () { 58 | mtable.set("key", "val"); 59 | assert.equal(mtable.get("key"), "val"); 60 | }); 61 | 62 | it("Should sset value in table", function () { 63 | mtable.sset("key", "val"); 64 | assert.equal(mtable.smember("key", "val"), true); 65 | 66 | mtable.set("key", "val"); 67 | assert.throws(_.partial(mtable.sset, "key", "val").bind(mtable)); 68 | }); 69 | 70 | it("Should hset value in table", function () { 71 | mtable.hset("key", "hkey", "val"); 72 | assert.equal(mtable.hget("key", "hkey"), "val"); 73 | 74 | mtable.set("key", "val"); 75 | assert.throws(_.partial(mtable.hset, "key", "hkey", "val").bind(mtable)); 76 | }); 77 | 78 | it("Should del value in table", function () { 79 | mtable.set("key", "val"); 80 | mtable.del("key"); 81 | assert.notOk(mtable.get("key")); 82 | }); 83 | 84 | it("Should sdel value in table", function () { 85 | mtable.sset("key", "val"); 86 | mtable.sset("key", "val2"); 87 | mtable.sdel("key", "val"); 88 | assert.equal(mtable.smember("key", "val"), false); 89 | assert.equal(mtable.smember("key", "val2"), true); 90 | mtable.sdel("key", "val2"); 91 | assert.equal(mtable.smember("key", "val2"), false); 92 | 93 | mtable.set("key", "val"); 94 | assert.throws(_.partial(mtable.sdel, "key", "val").bind(mtable)); 95 | }); 96 | 97 | it("Should hdel value in table", function () { 98 | mtable.hset("key", "hkey", "val"); 99 | mtable.hset("key", "hkey2", "val2"); 100 | mtable.hdel("key", "hkey"); 101 | assert.notOk(mtable.hget("key", "hkey")); 102 | assert.equal(mtable.hget("key", "hkey2"), "val2"); 103 | mtable.hdel("key", "hkey2"); 104 | assert.notOk(mtable.hget("key", "hkey2")); 105 | 106 | mtable.set("key", "val"); 107 | assert.throws(_.partial(mtable.hdel, "key", "hkey").bind(mtable)); 108 | }); 109 | 110 | it("Should run an async forEach over table", function (done) { 111 | mtable.set("key", "val"); 112 | mtable.set("key2", "val2"); 113 | const memo = {}; 114 | mtable.forEach((key, val, next) => { 115 | memo[key] = val; 116 | next(); 117 | }, (err) => { 118 | assert.notOk(err); 119 | assert.deepEqual(memo, { 120 | key: "val", 121 | key2: "val2" 122 | }); 123 | done(); 124 | }); 125 | }); 126 | 127 | it("Should run sync forEach over table", function () { 128 | mtable.set("key", "val"); 129 | mtable.set("key2", "val2"); 130 | const memo = {}; 131 | mtable.forEachSync((key, val) => { 132 | memo[key] = val; 133 | }); 134 | assert.deepEqual(memo, { 135 | key: "val", 136 | key2: "val2" 137 | }); 138 | }); 139 | }); 140 | 141 | describe("MTable static unit tests", function () { 142 | it("Should return invalid type error", function () { 143 | var error = MTable.invalidTypeError("command", "key", "type"); 144 | assert.ok(error instanceof Error); 145 | assert.equal(error.type, "INVALID_TYPE"); 146 | }); 147 | }); 148 | }; 149 | -------------------------------------------------------------------------------- /test/unit/utils.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | assert = require("chai").assert; 4 | 5 | module.exports = function (mocks, lib) { 6 | describe("Utils unit tests", function () { 7 | var utils = lib.utils; 8 | var Node = lib.node; 9 | 10 | it("Should return map values", function () { 11 | var m = new Map([["foo", "bar"]]); 12 | assert.deepEqual(utils.mapValues(m), ["bar"]); 13 | }); 14 | 15 | it("Should make an object out of an error", function () { 16 | var err = _.extend(new Error("foo"), {hello: "world"}); 17 | var old = process.env.NODE_ENV; 18 | process.env.NODE_ENV = "production"; 19 | assert.deepEqual(utils.errorToObject(err), { 20 | message: "foo", 21 | hello: "world", 22 | _error: true 23 | }); 24 | 25 | process.env.NODE_ENV = old; 26 | assert.deepEqual(_.omit(utils.errorToObject(err), "stack"), { 27 | message: "foo", 28 | hello: "world", 29 | _error: true 30 | }); 31 | 32 | assert.deepEqual(utils.errorToObject({message: "foo"}), { 33 | message: "foo", 34 | _error: true 35 | }); 36 | }); 37 | 38 | it("Should scan iterator, consuming all values", function () { 39 | var m = new Map([["foo", "bar"]]); 40 | var out = utils.scanIterator(m.entries(), 2); 41 | assert.deepEqual(out.values, [["foo", "bar"]]); 42 | assert.equal(out.iterator.next().done, true); 43 | }); 44 | 45 | it("Should scan iterator, consuming exact amount of values", function () { 46 | var m = new Map([["foo", "bar"]]); 47 | var out = utils.scanIterator(m.entries(), 1); 48 | assert.deepEqual(out.values, [["foo", "bar"]]); 49 | assert.equal(out.iterator.next().done, true); 50 | }); 51 | 52 | it("Should safely parse JSON", function () { 53 | var val = JSON.stringify("foo"); 54 | var out = utils.safeParse(val); 55 | assert.equal(out, "foo"); 56 | 57 | val = "foo"; 58 | out = utils.safeParse(val); 59 | assert.ok(out instanceof Error); 60 | }); 61 | 62 | it("Should return if value is a plain object", function () { 63 | var val = "foo"; 64 | assert.notOk(utils.isPlainObject(val)); 65 | 66 | val = {}; 67 | assert.ok(utils.isPlainObject(val)); 68 | }); 69 | 70 | it("Should check if data payload has id param", function () { 71 | var data = {id: ""}; 72 | var out = utils.hasID(data); 73 | assert.ok(out instanceof Error); 74 | 75 | data = {id: 5}; 76 | out = utils.hasID(data); 77 | assert.ok(out instanceof Error); 78 | 79 | data = {id: "foo"}; 80 | out = utils.hasID(data); 81 | assert.deepEqual(out, data); 82 | }); 83 | 84 | it("Should fail to parse node, bad format", function () { 85 | var out = utils.parseNode({node: ""}); 86 | assert.ok(out instanceof Error); 87 | }); 88 | 89 | it("Should fail to parse node, bad entry", function () { 90 | var bads = [ 91 | {id: 1, host: 1, port: "foo"}, 92 | {id: 1, host: "localhost", port: 8000}, 93 | {id: 1, host: 1, port: 8000}, 94 | {id: 1, host: "localhost", port: "foo"}, 95 | {id: "foo", host: 1, port: 8000}, 96 | {id: "foo", host: 1, port: "foo"}, 97 | {id: "foo", host: "localhost", port: "foo"} 98 | ]; 99 | bads.forEach((bad) => { 100 | var out = utils.parseNode({node: bad}); 101 | assert.ok(out instanceof Error); 102 | }); 103 | }); 104 | 105 | it("Should successfully parse node", function () { 106 | var out = utils.parseNode({node: {id: "foo", host: "localhost", port: 8000}}); 107 | assert.ok(out.node.equals(new Node("foo", "localhost", 8000))); 108 | }); 109 | 110 | it("Should fail to parse list of nodes with memo, bad format", function () { 111 | // first entry not an object 112 | var out = utils.parseNodeList({nodes: [""]}, []); 113 | assert.ok(out instanceof Error); 114 | }); 115 | 116 | it("Should fail to parse list of nodes with memo, bad entry", function () { 117 | // id should be a string 118 | var out = utils.parseNodeList({nodes: [{id: 5}]}, []); 119 | assert.ok(out instanceof Error); 120 | }); 121 | 122 | it("Should successfully parse list of nodes with memo", function () { 123 | var out = utils.parseNodeList({ 124 | nodes: [{id: "foo", host: "localhost", port: 8000}] 125 | }, []); 126 | assert.ok(Array.isArray(out.nodes)); 127 | assert.lengthOf(out.nodes, 1); 128 | assert.ok(out.nodes[0].equals(new Node("foo", "localhost", 8000))); 129 | }); 130 | 131 | it("Should fail to parse list of nodes, bad format", function () { 132 | var out = utils.parseNodeList({nodes: ""}); 133 | assert.ok(out instanceof Error); 134 | }); 135 | 136 | it("Should transform map to object", function () { 137 | var list = [["key", "val"], ["key2", "val2"]]; 138 | var map = new Map(list); 139 | var out = utils.mapToObject(map); 140 | assert.deepEqual(out, { 141 | key: "val", 142 | key2: "val2" 143 | }); 144 | }); 145 | 146 | it("Should transform set to list", function () { 147 | var list = ["val", "val2"]; 148 | var set = new Set(list); 149 | var out = utils.setToList(set); 150 | assert.deepEqual(out, list); 151 | }); 152 | 153 | it("Should transform map to list", function () { 154 | var list = [["key", "val"], ["key2", "val2"]]; 155 | var map = new Map(list); 156 | var out = utils.mapToList(map); 157 | assert.deepEqual(out, list); 158 | }); 159 | }); 160 | }; 161 | -------------------------------------------------------------------------------- /test/unit/cluster_node.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | uuid = require("uuid"), 4 | sinon = require("sinon"), 5 | fs = require("fs"), 6 | assert = require("chai").assert; 7 | 8 | module.exports = function (mocks, lib) { 9 | describe("ClusterNode unit tests", function () { 10 | var VectorClock = lib.vclock, 11 | CHash = lib.chash, 12 | NetKernel = lib.kernel, 13 | GossipRing = lib.gossip, 14 | CommsServer = lib.command_server, 15 | ClusterNode = lib.cluster_node, 16 | Node = lib.node, 17 | MockIPC = mocks.ipc; 18 | 19 | describe("ClusterNode state tests", function () { 20 | var kernel, 21 | nkernel, 22 | gossip, 23 | comms, 24 | vclock, 25 | chash, 26 | cluster, 27 | opts, 28 | id = "id", 29 | host = "localhost", 30 | port = 8000; 31 | 32 | before(function () { 33 | kernel = new NetKernel(new MockIPC(), id, host, port); 34 | nkernel = new NetKernel(new MockIPC(), "id2", host, port+1); 35 | chash = new CHash(3, 3); 36 | chash.insert(new Node(id, host, port)); 37 | vclock = new VectorClock(); 38 | gossip = new GossipRing(kernel, chash, vclock, { 39 | interval: 100, 40 | flushInterval: 100, 41 | flushPath: "/foo/bar", 42 | vclockOpts: {} 43 | }); 44 | comms = new CommsServer(gossip, kernel); 45 | }); 46 | 47 | beforeEach(function () { 48 | cluster = new ClusterNode(kernel, gossip, comms); 49 | }); 50 | 51 | it("Should construct a cluster", function () { 52 | assert.deepEqual(cluster._kernel, kernel); 53 | assert.deepEqual(cluster._gossip, gossip); 54 | }); 55 | 56 | it("Should grab kernel", function () { 57 | cluster._kernel = "foo"; 58 | assert.equal(cluster.kernel(), cluster._kernel); 59 | }); 60 | 61 | it("Should set kernel", function () { 62 | cluster.kernel("foo"); 63 | assert.equal(cluster.kernel(), "foo"); 64 | }); 65 | 66 | it("Should grab gossip", function () { 67 | cluster._gossip = "foo"; 68 | assert.equal(cluster.gossip(), cluster._gossip); 69 | }); 70 | 71 | it("Should set gossip", function () { 72 | cluster.gossip("foo"); 73 | assert.equal(cluster.gossip(), "foo"); 74 | }); 75 | 76 | it("Should grab command server", function () { 77 | cluster._comms = "foo"; 78 | assert.equal(cluster.commandServer(), "foo"); 79 | }); 80 | 81 | it("Should set command server", function () { 82 | cluster.commandServer("foo"); 83 | assert.equal(cluster.commandServer(), "foo"); 84 | }); 85 | 86 | it("Should load data from disk", function (done) { 87 | sinon.stub(cluster._gossip, "load", (cb) => { 88 | cluster._gossip.ring().insert(new Node("id2", host, port+1)); 89 | async.nextTick(cb); 90 | }); 91 | var adds = []; 92 | sinon.stub(cluster._kernel, "connect", function (node) { 93 | adds.push(node); 94 | }); 95 | cluster.load(() => { 96 | assert.deepEqual(adds, [nkernel.self()]); 97 | cluster._gossip.load.restore(); 98 | cluster._kernel.connect.restore(); 99 | done(); 100 | }); 101 | }); 102 | 103 | it("Should error out loading data from disk if error occurs in gossip load", function (done) { 104 | sinon.stub(cluster._gossip, "load", (cb) => { 105 | async.nextTick(_.partial(cb, new Error("error"))); 106 | }); 107 | cluster.load((err) => { 108 | assert.ok(err); 109 | cluster._gossip.load.restore(); 110 | done(); 111 | }); 112 | }); 113 | 114 | it("Should fail to start node due to mismatched ring IDs", function (done) { 115 | cluster.gossip()._ringID = "ring2"; 116 | cluster.start("cookie", "ring", (err) => { 117 | assert.ok(err); 118 | cluster.gossip()._ringID = null; 119 | done(); 120 | }); 121 | }); 122 | 123 | it("Should start node successfully", function (done) { 124 | cluster.start("cookie", "ring"); 125 | cluster.once("ready", () => { 126 | assert.equal(cluster.gossip()._ringID, "ring"); 127 | assert.equal(cluster.kernel()._cookie, "cookie"); 128 | cluster.stop(); 129 | done(); 130 | }); 131 | }); 132 | 133 | it("Should start node successfully, use listener callback", function (done) { 134 | cluster.start("cookie", "ring", () => { 135 | assert.equal(cluster.gossip()._ringID, "ring"); 136 | assert.equal(cluster.kernel()._cookie, "cookie"); 137 | cluster.stop(); 138 | done(); 139 | }); 140 | }); 141 | 142 | it("Should stop node non-forcefully", function (done) { 143 | cluster.start("cookie", "ring", () => { 144 | cluster.once("stop", () => { 145 | assert.notOk(cluster.gossip()._ringID); 146 | assert.equal(cluster.kernel().sources().size, 0); 147 | done(); 148 | }); 149 | cluster.stop(false); 150 | }); 151 | }); 152 | 153 | it("Should stop node forcefully", function (done) { 154 | cluster.start("cookie", "ring", () => { 155 | cluster.once("stop", () => { 156 | assert.notOk(cluster.gossip()._ringID); 157 | assert.equal(cluster.kernel().sources().size, 0); 158 | done(); 159 | }); 160 | cluster.stop(true); 161 | }); 162 | }); 163 | }); 164 | }); 165 | }; 166 | -------------------------------------------------------------------------------- /lib/cluster_node.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | EventEmitter = require("events").EventEmitter, 4 | utils = require("./utils"); 5 | 6 | /** @namespace Clusterluck */ 7 | 8 | class ClusterNode extends EventEmitter { 9 | /** 10 | * 11 | * Cluster wrapper class. Used to start/load/stop the cluster, both network kernel and gossip ring. 12 | * 13 | * @class ClusterNode ClusterNode 14 | * @memberof Clusterluck 15 | * 16 | * @param {Clusterluck.NetKernel} kernel - Network kernel to use to communicate with other nodes. 17 | * @param {Clusterluck.GossipRing} gossip - Gossip ring to use to route messages to across a cluster. 18 | * 19 | */ 20 | constructor(kernel, gossip, comms) { 21 | super(); 22 | this._kernel = kernel; 23 | this._gossip = gossip; 24 | this._comms = comms; 25 | } 26 | 27 | /** 28 | * 29 | * Acts as a getter/setter for the netkernel of this instance. 30 | * 31 | * @method kernel 32 | * @memberof ClusterLuck.ClusterNode 33 | * @instance 34 | * 35 | * @param {Clusterluck.NetKernel} [kernel] - Network kernel to set on this instance. 36 | * 37 | * @return {Clusterluck.NetKernel} Network kernel of this instance. 38 | * 39 | */ 40 | kernel(kernel) { 41 | if (kernel !== undefined) { 42 | this._kernel = kernel; 43 | } 44 | return this._kernel; 45 | } 46 | 47 | /** 48 | * 49 | * Acts as a getter/setter for the gossip ring of this instance. 50 | * 51 | * @method gossip 52 | * @memberof Clusterluck.ClusterNode 53 | * @instance 54 | * 55 | * @param {Clusterluck.GossipRing} [gossip] - Gossip ring to set on this instance. 56 | * 57 | * @return {Clusterluck.GossipRing} Gossip ring of this instance. 58 | * 59 | */ 60 | gossip(gossip) { 61 | if (gossip !== undefined) { 62 | this._gossip = gossip; 63 | } 64 | return this._gossip; 65 | } 66 | 67 | /** 68 | * 69 | * Acts as a getter/setter for the command handler of this instance. 70 | * 71 | * @method commandServer 72 | * @memberof Clusterluck.ClusterNode 73 | * @instance 74 | * 75 | * @param {Clusterluck.CommandServer} [commandServer] - Command handler to set on this instance. 76 | * 77 | * @return {Clusterluck.CommandServer} Command handler of this instance. 78 | * 79 | */ 80 | commandServer(comm) { 81 | if (comm !== undefined) { 82 | this._comms = comm; 83 | } 84 | return this._comms; 85 | } 86 | 87 | /** 88 | * 89 | * Loads gossip state from disk and establishes all node connections derived from 90 | * the newly loaded hash ring. 91 | * 92 | * @method load 93 | * @memberof Clusterluck.ClusterNode 94 | * @instance 95 | * 96 | * @param {Function} cb - Function to call once state has been loaded. 97 | * 98 | */ 99 | load(cb) { 100 | this._gossip.load((err) => { 101 | if (err) return cb(err); 102 | this._gossip.ring().nodes().forEach((node) => { 103 | if (node.id() === this._kernel.self().id()) return; 104 | this._kernel.connect(node); 105 | }); 106 | return cb(); 107 | }); 108 | } 109 | 110 | /** 111 | * 112 | * Starts a network kernel and gossip ring on this node. 113 | * 114 | * @method start 115 | * @memberof Clusterluck.ClusterNode 116 | * @instance 117 | * 118 | * @param {String} cookie - Distributed cookie to use when communicating with other nodes and signing payloads. 119 | * @param {String} ringID - Ring ID to start gossip ring on. 120 | * @param {Function} [cb] - Optional callback; called when network kernel has been fully started and listening for IPC messages. 121 | * @param {Object} [opts] - Network kernel options. 122 | * @param {Number} [opts.tokenGenInterval] - Interval in which to generate a new reply token for synchronous cluster calls. Defaults to 1 hour. 123 | * 124 | * @return {Clusterluck.ClusterNode} This instance. 125 | * 126 | */ 127 | start(cookie, ringID, cb, kernelOpts = {}) { 128 | // if ring was successfully read from disk and ring ID different than input, error out 129 | if (typeof this._gossip._ringID === "string" && 130 | this._gossip._ringID !== ringID) { 131 | return cb(new Error("Loaded ring ID '" + this._gossip._ringID + "' does not match '" + ringID + "'")); 132 | } 133 | this._comms.start("command"); 134 | // clear ring ID so that we can start the gossip server (we know they're the same, so it's a noop) 135 | this._gossip.ringID(null); 136 | this._gossip.start(ringID); 137 | this._kernel.start(_.extend(_.clone(kernelOpts), {cookie: cookie})); 138 | 139 | /** 140 | * 141 | * Emitted when the command line server, gossip ring server, and network kernel have all started and are ready to start processing messages. 142 | * 143 | * @event Clusterluck.ClusterNode#ClusterNode:ready 144 | * @memberof Clusterluck.ClusterNode 145 | * 146 | */ 147 | this._kernel.once("_ready", () => {this.emit("ready");}); 148 | if (typeof cb === "function") { 149 | this.on("ready", cb); 150 | } 151 | return this; 152 | } 153 | 154 | /** 155 | * 156 | * Stops the gossip ring and network kernel, as well as closing all network connections with any external nodes. 157 | * 158 | * @method stop 159 | * @memberof Clusterluck.ClusterNode 160 | * @instance 161 | * 162 | * @fires Clusterluck.ClusterNode#ClusterNode:stop 163 | * @listens Clusterluck.CommandServer#CommandServer:stop 164 | * @listens Clusterluck.GossipRing#GossipRing:stop 165 | * 166 | * @param {Boolean} [force] - Whether to forcibly stop this node or not. 167 | * 168 | * @return {Clusterluck.ClusterNode} This instance. 169 | * 170 | */ 171 | stop(force) { 172 | async.series([ 173 | (next) => { 174 | this._comms.once("stop", next); 175 | this._comms.stop(force); 176 | }, 177 | (next) => { 178 | this._gossip.once("stop", next); 179 | this._gossip.stop(force); 180 | }, 181 | (next) => { 182 | this._kernel.stop(); 183 | this._kernel.sinks().forEach((val) => { 184 | this._kernel.disconnect(val.node(), true); 185 | }); 186 | 187 | /** 188 | * 189 | * Emitted when the command line server, gossip ring server, and network kernel have all stopped processing messages. 190 | * 191 | * @event Clusterluck.ClusterNode#ClusterNode:stop 192 | * @memberof Clusterluck.ClusterNode 193 | * 194 | */ 195 | this.emit("stop"); 196 | } 197 | ]); 198 | } 199 | } 200 | 201 | module.exports = ClusterNode; 202 | -------------------------------------------------------------------------------- /test/integration/dlm.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | os = require("os"), 4 | ipc = require("node-ipc"), 5 | assert = require("chai").assert; 6 | 7 | var host = os.hostname(); 8 | 9 | module.exports = function (mocks, lib) { 10 | var DLMServer = lib.dlm.DLMServer; 11 | var Gossip = lib.gossip; 12 | var NetKernel = lib.kernel; 13 | var CHash = lib.chash; 14 | var VClock = lib.vclock; 15 | var consts = lib.consts; 16 | 17 | var kernelOpts = consts.kernelOpts; 18 | var gossipOpts = consts.gossipOpts; 19 | 20 | describe("DLM integration tests", function () { 21 | var nodes = []; 22 | var dlms = []; 23 | var origin, target; 24 | var node1 = "foo", node2 = "bar"; 25 | var port1 = 8000, port2 = 8001; 26 | 27 | before(function (done) { 28 | async.each([[node1, port1], [node2, port2]], (config, next) => { 29 | var inst = new ipc.IPC(); 30 | inst.config.networkHost = host; 31 | inst.config.networkPort = config[1]; 32 | inst.config.retry = kernelOpts.retry; 33 | inst.config.maxRetries = kernelOpts.maxRetries; 34 | inst.config.tls = kernelOpts.tls; 35 | inst.config.silent = kernelOpts.silent; 36 | const kernel = new NetKernel(inst, config[0], host, config[1]); 37 | var chash = (new CHash(3, 2)).insert(kernel.self()); 38 | var vclock = new VClock(); 39 | const gossip = new Gossip(kernel, chash, vclock, consts.gossipOpts); 40 | 41 | const dlm = new DLMServer(gossip, kernel, { 42 | rquorum: 0.51, 43 | wquorum: 0.51 44 | }); 45 | nodes.push({gossip: gossip, kernel: kernel}); 46 | dlms.push(dlm); 47 | dlm.start("locker"); 48 | gossip.start("ring"); 49 | kernel.start({cookie: "cookie"}); 50 | kernel.once("_ready", next); 51 | }, function () { 52 | origin = dlms[0]; 53 | target = dlms[1]; 54 | nodes[0].gossip.meet(nodes[1].kernel.self()); 55 | nodes[0].gossip.once("process", _.ary(done, 0)); 56 | }); 57 | }); 58 | 59 | after(function () { 60 | dlms.forEach(function (dlm) { 61 | dlm.stop(); 62 | }); 63 | nodes.forEach(function (node) { 64 | node.gossip.stop(true); 65 | node.kernel.sinks().forEach(function (sink) { 66 | node.kernel.disconnect(sink, true); 67 | }); 68 | node.kernel.stop(); 69 | }); 70 | }); 71 | 72 | it("Should create write lock", function (done) { 73 | origin.wlock("id", "holder", 30000, (err, nodes) => { 74 | assert.notOk(err); 75 | assert.ok(origin._locks.has("id")); 76 | assert.ok(target._locks.has("id")); 77 | origin.wunlock("id", "holder", done); 78 | }, 1000, 0); 79 | }); 80 | 81 | it("Should create write lock, and then fail subsequent write lock", function (done) { 82 | async.series([ 83 | function (next) { 84 | origin.wlock("id", "holder", 30000, (err) => { 85 | assert.notOk(err); 86 | assert.ok(origin._locks.has("id")); 87 | assert.ok(target._locks.has("id")); 88 | next(); 89 | }, 1000, 0); 90 | }, 91 | function (next) { 92 | origin.wlock("id", "holder2", 30000, (err) => { 93 | assert.ok(err); 94 | next(); 95 | }, 1000, 0); 96 | }, 97 | function (next) { 98 | origin.wunlock("id", "holder", (err) => { 99 | assert.notOk(err); 100 | assert.notOk(origin._locks.has("id")); 101 | assert.notOk(target._locks.has("id")); 102 | next(); 103 | }); 104 | } 105 | ], done); 106 | }); 107 | 108 | it("Should create write lock, and then fail subsequent read lock", function (done) { 109 | async.series([ 110 | function (next) { 111 | origin.wlock("id", "holder", 30000, (err) => { 112 | assert.notOk(err); 113 | assert.ok(origin._locks.has("id")); 114 | assert.ok(target._locks.has("id")); 115 | next(); 116 | }, 1000, 0); 117 | }, 118 | function (next) { 119 | origin.rlock("id", "holder2", 30000, (err) => { 120 | assert.ok(err); 121 | next(); 122 | }, 1000, 0); 123 | }, 124 | function (next) { 125 | origin.wunlock("id", "holder", (err) => { 126 | assert.notOk(err); 127 | assert.notOk(origin._locks.has("id")); 128 | assert.notOk(target._locks.has("id")); 129 | next(); 130 | }); 131 | } 132 | ], done); 133 | }); 134 | 135 | it("Should create read lock", function (done) { 136 | origin.rlock("id", "holder", 30000, (err) => { 137 | assert.notOk(err); 138 | assert.ok(origin._locks.has("id")); 139 | assert.ok(target._locks.has("id")); 140 | origin.runlock("id", "holder", done); 141 | }, 1000, 0); 142 | }); 143 | 144 | it("Should create read lock, and then fail subsequent write lock", function (done) { 145 | async.series([ 146 | function (next) { 147 | origin.rlock("id", "holder", 30000, (err) => { 148 | assert.notOk(err); 149 | assert.ok(origin._locks.has("id")); 150 | assert.ok(target._locks.has("id")); 151 | next(); 152 | }, 1000, 0); 153 | }, 154 | function (next) { 155 | origin.wlock("id", "holder2", 30000, (err) => { 156 | assert.ok(err); 157 | next(); 158 | }, 1000, 0); 159 | }, 160 | function (next) { 161 | origin.runlock("id", "holder", (err) => { 162 | assert.notOk(err); 163 | assert.notOk(origin._locks.has("id")); 164 | assert.notOk(target._locks.has("id")); 165 | next(); 166 | }); 167 | } 168 | ], done); 169 | }); 170 | 171 | it("Should create read lock, and then succeed subsequent read lock", function (done) { 172 | async.series([ 173 | function (next) { 174 | origin.rlock("id", "holder", 30000, (err) => { 175 | assert.notOk(err); 176 | assert.ok(origin._locks.has("id")); 177 | assert.ok(target._locks.has("id")); 178 | next(); 179 | }, 1000, 0); 180 | }, 181 | function (next) { 182 | origin.rlock("id", "holder2", 30000, (err) => { 183 | assert.notOk(err); 184 | assert.equal(origin._locks.get("id").timeout().size, 2); 185 | assert.equal(target._locks.get("id").timeout().size, 2); 186 | next(); 187 | }, 1000, 0); 188 | }, 189 | function (next) { 190 | origin.runlock("id", "holder", (err) => { 191 | assert.notOk(err); 192 | assert.equal(origin._locks.get("id").timeout().size, 1); 193 | assert.equal(target._locks.get("id").timeout().size, 1); 194 | next(); 195 | }); 196 | }, 197 | function (next) { 198 | origin.runlock("id", "holder2", (err) => { 199 | assert.notOk(err); 200 | assert.notOk(origin._locks.has("id")); 201 | assert.notOk(target._locks.has("id")); 202 | next(); 203 | }); 204 | } 205 | ], done); 206 | }); 207 | }); 208 | }; 209 | -------------------------------------------------------------------------------- /test/integration/dsm.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | os = require("os"), 4 | ipc = require("node-ipc"), 5 | assert = require("chai").assert; 6 | 7 | var host = os.hostname(); 8 | 9 | function findKeyForOther(node, ring) { 10 | var node1; 11 | var total = ring.weights().get(node.id()); 12 | for (var i = 1; i <= total; ++i) { 13 | var inner = ring.find(node.id() + "_" + i); 14 | if (inner.id() !== node.id()) node1 = (node.id() + "_" + i); 15 | break; 16 | } 17 | return node1; 18 | } 19 | 20 | module.exports = function (mocks, lib) { 21 | var DSMServer = lib.dsem.DSMServer; 22 | var Gossip = lib.gossip; 23 | var NetKernel = lib.kernel; 24 | var CHash = lib.chash; 25 | var VClock = lib.vclock; 26 | var consts = lib.consts; 27 | 28 | var kernelOpts = consts.kernelOpts; 29 | 30 | describe("DSM integration tests", function () { 31 | var nodes = []; 32 | var dsms = []; 33 | var origin, target; 34 | var key1, key2; 35 | var node1 = "foo", node2 = "bar"; 36 | var port1 = 8000, port2 = 8001; 37 | 38 | before(function (done) { 39 | async.each([[node1, port1], [node2, port2]], (config, next) => { 40 | var inst = new ipc.IPC(); 41 | inst.config.networkHost = host; 42 | inst.config.networkPort = config[1]; 43 | inst.config.retry = kernelOpts.retry; 44 | inst.config.maxRetries = kernelOpts.maxRetries; 45 | inst.config.tls = kernelOpts.tls; 46 | inst.config.silent = kernelOpts.silent; 47 | const kernel = new NetKernel(inst, config[0], host, config[1]); 48 | var chash = (new CHash(3, 2)).insert(kernel.self()); 49 | var vclock = new VClock(); 50 | const gossip = new Gossip(kernel, chash, vclock, consts.gossipOpts); 51 | 52 | const dsm = new DSMServer(gossip, kernel, { 53 | rquorum: 0.51, 54 | wquorum: 0.51 55 | }); 56 | nodes.push({gossip: gossip, kernel: kernel}); 57 | dsms.push(dsm); 58 | dsm.start("locker"); 59 | gossip.start("ring"); 60 | kernel.start({cookie: "cookie"}); 61 | kernel.once("_ready", next); 62 | }, function () { 63 | origin = dsms[0]; 64 | target = dsms[1]; 65 | nodes[0].gossip.meet(nodes[1].kernel.self()); 66 | nodes[0].gossip.once("process", () => { 67 | key1 = findKeyForOther(nodes[1].kernel.self(), nodes[0].gossip.ring()); 68 | key2 = findKeyForOther(nodes[0].kernel.self(), nodes[0].gossip.ring()); 69 | origin.create(key1, 2, () => { 70 | target.create(key2, 2, done); 71 | }); 72 | }); 73 | }); 74 | }); 75 | 76 | after(function () { 77 | dsms.forEach(function (dsm) { 78 | dsm.stop(); 79 | }); 80 | nodes.forEach(function (node) { 81 | node.gossip.stop(true); 82 | node.kernel.sinks().forEach(function (sink) { 83 | node.kernel.disconnect(sink, true); 84 | }); 85 | node.kernel.stop(); 86 | }); 87 | }); 88 | 89 | it("Should post to a semaphore", function (done) { 90 | origin.post(key1, "holder", 30000, (err, nodes) => { 91 | assert.notOk(err); 92 | assert.ok(origin._semaphores.has(key1)); 93 | assert.notOk(target._semaphores.has(key1)); 94 | origin.close(key1, "holder", done); 95 | }, 1000, 0); 96 | }); 97 | 98 | it("Should post to an external semaphore", function (done) { 99 | origin.post(key2, "holder", 30000, (err, nodes) => { 100 | assert.notOk(err); 101 | assert.ok(target._semaphores.has(key2)); 102 | assert.notOk(origin._semaphores.has(key2)); 103 | target.close(key2, "holder", done); 104 | }, 1000, 0); 105 | }); 106 | 107 | it("Should exhaust semaphore count on a node", function (done) { 108 | async.series([ 109 | function (next) { 110 | origin.post(key1, "holder", 30000, (err) => { 111 | assert.notOk(err); 112 | assert.equal(origin._semaphores.get(key1).timeouts().size, 1); 113 | assert.notOk(target._semaphores.has(key1)); 114 | next(); 115 | }, 1000, 0); 116 | }, 117 | function (next) { 118 | origin.post(key1, "holder2", 30000, (err) => { 119 | assert.notOk(err); 120 | assert.equal(origin._semaphores.get(key1).timeouts().size, 2); 121 | assert.notOk(target._semaphores.has(key1)); 122 | next(); 123 | }, 1000, 0); 124 | }, 125 | function (next) { 126 | origin.close(key1, "holder", (err) => { 127 | assert.notOk(err); 128 | assert.equal(origin._semaphores.get(key1).timeouts().size, 1); 129 | assert.notOk(target._semaphores.has(key1)); 130 | next(); 131 | }); 132 | }, 133 | function (next) { 134 | origin.close(key1, "holder2", (err) => { 135 | assert.notOk(err); 136 | assert.equal(origin._semaphores.get(key1).timeouts().size, 0); 137 | assert.notOk(target._semaphores.has(key1)); 138 | next(); 139 | }); 140 | } 141 | ], done); 142 | }); 143 | 144 | it("Should exhaust semaphore count on an external node", function (done) { 145 | async.series([ 146 | function (next) { 147 | origin.post(key2, "holder", 30000, (err) => { 148 | assert.notOk(err); 149 | assert.equal(target._semaphores.get(key2).timeouts().size, 1); 150 | assert.notOk(origin._semaphores.has(key2)); 151 | next(); 152 | }, 1000, 0); 153 | }, 154 | function (next) { 155 | origin.post(key2, "holder2", 30000, (err) => { 156 | assert.notOk(err); 157 | assert.equal(target._semaphores.get(key2).timeouts().size, 2); 158 | assert.notOk(origin._semaphores.has(key2)); 159 | next(); 160 | }, 1000, 0); 161 | }, 162 | function (next) { 163 | target.close(key2, "holder", (err) => { 164 | assert.notOk(err); 165 | assert.equal(target._semaphores.get(key2).timeouts().size, 1); 166 | assert.notOk(origin._semaphores.has(key2)); 167 | next(); 168 | }); 169 | }, 170 | function (next) { 171 | target.close(key2, "holder2", (err) => { 172 | assert.notOk(err); 173 | assert.equal(target._semaphores.get(key2).timeouts().size, 0); 174 | assert.notOk(origin._semaphores.has(key2)); 175 | next(); 176 | }); 177 | } 178 | ], done); 179 | }); 180 | 181 | it("Should fail to write to semaphore that doesn't exist", function (done) { 182 | async.series([ 183 | function (next) { 184 | origin.post("id", "holder", 30000, (err) => { 185 | assert.ok(err); 186 | next(); 187 | }, 1000, 0); 188 | }, 189 | function (next) { 190 | target.post("id", "holder2", 30000, (err) => { 191 | assert.ok(err); 192 | next(); 193 | }, 1000, 0); 194 | } 195 | ], done); 196 | }); 197 | 198 | it("Should read semaphores", function (done) { 199 | async.series([ 200 | function (next) { 201 | origin.read(key1, (err, out) => { 202 | assert.notOk(err); 203 | assert.equal(out.n, 2); 204 | assert.equal(out.active, 0); 205 | next(); 206 | }, 1000); 207 | }, 208 | function (next) { 209 | origin.read(key2, (err, out) => { 210 | assert.notOk(err); 211 | assert.equal(out.n, 2); 212 | assert.equal(out.active, 0); 213 | next(); 214 | }, 1000, 0); 215 | } 216 | ], done); 217 | }); 218 | }); 219 | }; 220 | -------------------------------------------------------------------------------- /lib/conn.js: -------------------------------------------------------------------------------- 1 | const _ = require("lodash"), 2 | consts = require("./consts"), 3 | debug = require("debug")("notp:lib:conn"), 4 | EventEmitter = require("events").EventEmitter, 5 | Queue = require("./queue"); 6 | 7 | const connDefaults = consts.connDefaults; 8 | 9 | class Connection extends EventEmitter { 10 | /** 11 | * 12 | * Connection abstraction class. Handles reconnection logic when the client IPC socket disconnects, internal message buffering during reconnection, and state management for safe connection closure. 13 | * 14 | * @class Connection 15 | * @memberof Clusterluck 16 | * 17 | * @param {IPC} ipc - IPC module to create connection over. 18 | * @param {Clusterluck.Node} node - Node this connection communicates with. 19 | * @param {Object} [opts] - Options object for connection. 20 | * @param {Number} [opts.maxLen] - Maximum length of messages that can buffered while IPC socket is down. Defaults to 1024. Once breached, the oldest messages will be dropped until the queue is of this size. For unbounded buffering, set this to `Infinity`. 21 | * 22 | */ 23 | constructor(ipc, node, opts=connDefaults) { 24 | super(); 25 | opts = _.defaults(opts, connDefaults); 26 | this._ipc = ipc; 27 | this._node = node; 28 | this._queue = new Queue(); 29 | this._connecting = false; 30 | this._active = false; 31 | this._streams = new Map(); 32 | this._maxLen = opts.maxLen; 33 | } 34 | 35 | /** 36 | * 37 | * Initializes IPC client socket to `node`, along with listeners for socket disconnects. 38 | * 39 | * @method start 40 | * @memberof Clusterluck.Connection 41 | * @instance 42 | * 43 | */ 44 | start() { 45 | // maybe add routine for removing old messages still in queue to avoid backup 46 | // on catastrophic neighbor failures 47 | var node = this._node; 48 | this._active = true; 49 | this._connecting = true; 50 | this._ipc.connectToNet(node.id(), node.host(), node.port()); 51 | this._ipc.of[node.id()].on("connect", this._handleConnect.bind(this)); 52 | this._ipc.of[node.id()].on("disconnect", this._handleDisconnect.bind(this)); 53 | } 54 | 55 | /** 56 | * 57 | * Closes IPC client socket to `node`. Can be done synchronously using the force option, or asynchronously by waiting for an idle/connected state to occur. 58 | * 59 | * @method stop 60 | * @memberof Clusterluck.Connection 61 | * @instance 62 | * 63 | * @param {Boolean} [force] - Whether to forcibly close this connection or not. If true, will bypass waiting for an 'idle' state, immediately flushing the internal message buffer and clobeering state about which streams are still active over this connection. Otherwise, this will asynchronously close, waiting for all messages and streams to finish first. 64 | * 65 | * @return {Clusterluck.Connection} This instance. 66 | * 67 | */ 68 | stop(force = false) { 69 | debug("Stopping connection to node " + this._node.id() + (force ? " forcefully" : " gracefully")); 70 | if (!this.idle() && force !== true) { 71 | this.once("idle", this.stop.bind(this)); 72 | return this; 73 | } 74 | if (this._connecting === true && force !== true) { 75 | this.once("connect", this.stop.bind(this)); 76 | return this; 77 | } 78 | this._connecting = false; 79 | this._active = false; 80 | this._queue.flush(); 81 | this._streams = new Map(); 82 | this._ipc.disconnect(this._node.id()); 83 | return this; 84 | } 85 | 86 | /** 87 | * 88 | * Acts as a getter for the node this connection communicates with. 89 | * 90 | * @method node 91 | * @memberof Clusterluck.Connection 92 | * @instance 93 | * 94 | * @return {Clusterluck.Node} Node this instance communicates with. 95 | * 96 | */ 97 | node() { 98 | return this._node; 99 | } 100 | 101 | /** 102 | * 103 | * Acts as a getter for the internal message buffer. 104 | * 105 | * @method queue 106 | * @memberof Clusterluck.Connection 107 | * @instance 108 | * 109 | * @return {Queue} Internal message buffer of this instance. 110 | * 111 | */ 112 | queue() { 113 | return this._queue; 114 | } 115 | 116 | /** 117 | * 118 | * Returns whether this connection has been stopped or not. 119 | * 120 | * @method active 121 | * @memberof Clusterluck.Connection 122 | * @instance 123 | * 124 | * @return {Boolean} Whether this connection is active or not. 125 | * 126 | */ 127 | active() { 128 | return this._active; 129 | } 130 | 131 | /** 132 | * 133 | * Returns whether this connection is in a reconnection state or not. 134 | * 135 | * @method connecting 136 | * @memberof Clusterluck.Connection 137 | * @instance 138 | * 139 | * @return {Boolean} Whether this connection is in the middle of reconnection logic. 140 | * 141 | */ 142 | connecting() { 143 | return this._connecting; 144 | } 145 | 146 | /** 147 | * 148 | * Returns whether this connection is in an idle state. 149 | * 150 | * @method idle 151 | * @memberof Clusterluck.Connection 152 | * @instance 153 | * 154 | * @return {Boolean} Whether this connection is currently idle. 155 | * 156 | */ 157 | idle() { 158 | return this._streams.size === 0 && this._queue.size() === 0; 159 | } 160 | 161 | /** 162 | * 163 | * Acts as a getter/setter for the max length of the internal message queue 164 | * for this IPC socket connection. 165 | * 166 | * @method maxLen 167 | * @memberof Clusterluck.Connection 168 | * @instance 169 | * 170 | * @param {Number} [len] - Number to set maximum message queue length to. 171 | * 172 | * @return {Number} The maximum message queue length of this IPC socket. 173 | * 174 | */ 175 | maxLen(len) { 176 | if (typeof len === "number" && len >= 0) { 177 | this._maxLen = len; 178 | while (this._queue.size() > this._maxLen) { 179 | this._queue.dequeue(); 180 | } 181 | } 182 | return this._maxLen; 183 | } 184 | 185 | /** 186 | * 187 | * Sends message `data` under event `event` through this IPC socket. 188 | * 189 | * @method send 190 | * @memberof Clusterluck.Connection 191 | * @instance 192 | * 193 | * @param {String} event - Event to identify IPC message with. 194 | * @param {Object} data - Data to send with this IPC message. 195 | * 196 | * @return {Clusterluck.Connection} This instance. 197 | * 198 | */ 199 | send(event, data) { 200 | if (this._active === false) { 201 | return new Error("Cannot write to inactive connection."); 202 | } 203 | if (this._connecting === true) { 204 | if (this._queue.size() >= this._maxLen) { 205 | this._queue.dequeue(); 206 | } 207 | this._queue.enqueue({ 208 | event: event, 209 | data: data 210 | }); 211 | return this; 212 | } 213 | this._ipc.of[this._node.id()].emit(event, data); 214 | this.emit("send", event, data); 215 | this._updateStream(data.stream); 216 | return this; 217 | } 218 | 219 | /** 220 | * 221 | * Marks message stream `stream` in order to indicate to this connection beforehand that it is not 222 | * in an idle state. 223 | * 224 | * @method initiateStream 225 | * @memberof Clusterluck.Connection 226 | * @instance 227 | * 228 | * @param {Object} stream - Message stream to mark. 229 | * @param {Object} stream.stream - Unique ID of mesage stream. 230 | * 231 | * @return {Clusterluck.Connection} This instance. 232 | * 233 | */ 234 | initiateStream(stream) { 235 | this._streams.set(stream.stream, true); 236 | return this; 237 | } 238 | 239 | /** 240 | * 241 | * Handler for when this connection has finished reconnection logic. 242 | * 243 | * @method _handleConnect 244 | * @memberof Clusterluck.Connection 245 | * @private 246 | * @instance 247 | * 248 | */ 249 | _handleConnect() { 250 | debug("Connected to TCP connection to node " + this._node.id()); 251 | this._connecting = false; 252 | this.emit("connect"); 253 | // flush queue after emitting "connect" 254 | var out = this._queue.flush(); 255 | out.forEach((msg) => { 256 | this.send(msg.event, msg.data); 257 | }); 258 | } 259 | 260 | /** 261 | * 262 | * Handler for when this connection has entered reconnection logic. 263 | * 264 | * @method _handleDisconnect 265 | * @memberof Clusterluck.Connection 266 | * @private 267 | * @instance 268 | * 269 | */ 270 | _handleDisconnect() { 271 | debug("Disconnected from TCP connection to node " + this._node.id()); 272 | if (this._active) { 273 | this._connecting = true; 274 | } else { 275 | this._connecting = false; 276 | } 277 | this.emit("disconnect"); 278 | } 279 | 280 | /** 281 | * 282 | * Updates the stream state of this instance. If the stream is finished, removes the stream ID. If no stream IDs are left, then an idle event is emitted. 283 | * 284 | * @method _updateStream 285 | * @memberof Clusterluck.Connection 286 | * @private 287 | * @instance 288 | * 289 | * @param {Object} stream - Stream to update internal state about. 290 | * 291 | */ 292 | _updateStream(stream) { 293 | if (stream.done && stream.stream) { 294 | this._streams.delete(stream.stream); 295 | if (this._streams.size === 0) { 296 | this.emit("idle"); 297 | } 298 | } else if (stream.stream) { 299 | this._streams.set(stream.stream, true); 300 | } 301 | } 302 | } 303 | 304 | module.exports = Connection; 305 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | ## [v2.0.1](https://github.com/azuqua/clusterluck/tree/v2.0.1) (2017-10-13) 4 | 5 | [Full Changelog](https://github.com/azuqua/clusterluck/compare/v2.0.0...v2.0.1) 6 | 7 | **Merged pull requests:** 8 | 9 | - Develop [\#53](https://github.com/azuqua/clusterluck/pull/53) ([kevinwilson541](https://github.com/kevinwilson541)) 10 | - Kevin/develop [\#52](https://github.com/azuqua/clusterluck/pull/52) ([kevinwilson541](https://github.com/kevinwilson541)) 11 | 12 | ## [v2.0.0](https://github.com/azuqua/clusterluck/tree/v2.0.0) (2017-10-08) 13 | [Full Changelog](https://github.com/azuqua/clusterluck/compare/v1.3.0...v2.0.0) 14 | 15 | **Implemented enhancements:** 16 | 17 | - Add ability to explicitly set rfactor on node insertion/update, grab explicit range length of nodes [\#45](https://github.com/azuqua/notp/issues/45) 18 | - Documentation or interface goodies for setting configurations on external IPC connections [\#42](https://github.com/azuqua/notp/issues/42) 19 | - Add convenience commands to CLI tool for reading nodes from a ring, pinging external nodes [\#41](https://github.com/azuqua/notp/issues/41) 20 | 21 | **Merged pull requests:** 22 | 23 | - Develop [\#51](https://github.com/azuqua/notp/pull/51) ([kevinwilson541](https://github.com/kevinwilson541)) 24 | - update missing documentation for parameters on insert/minsert/range f… [\#50](https://github.com/azuqua/notp/pull/50) ([kevinwilson541](https://github.com/kevinwilson541)) 25 | - 2.0.0 alpha [\#49](https://github.com/azuqua/notp/pull/49) ([kevinwilson541](https://github.com/kevinwilson541)) 26 | - update CLI output format to be parseable by jq and other JSON parsing… [\#48](https://github.com/azuqua/notp/pull/48) ([kevinwilson541](https://github.com/kevinwilson541)) 27 | - integration tests for dlm/dsm/gen\_server [\#47](https://github.com/azuqua/notp/pull/47) ([kevinwilson541](https://github.com/kevinwilson541)) 28 | - Kevin/experiment [\#46](https://github.com/azuqua/notp/pull/46) ([kevinwilson541](https://github.com/kevinwilson541)) 29 | - update README with new CLI commands \(nodes, ping\) [\#44](https://github.com/azuqua/notp/pull/44) ([kevinwilson541](https://github.com/kevinwilson541)) 30 | - Kevin/experiment [\#43](https://github.com/azuqua/notp/pull/43) ([kevinwilson541](https://github.com/kevinwilson541)) 31 | - Kevin/experiment [\#40](https://github.com/azuqua/notp/pull/40) ([kevinwilson541](https://github.com/kevinwilson541)) 32 | - Develop [\#39](https://github.com/azuqua/notp/pull/39) ([kevinwilson541](https://github.com/kevinwilson541)) 33 | - updated .travis.yml file [\#38](https://github.com/azuqua/notp/pull/38) ([kevinwilson541](https://github.com/kevinwilson541)) 34 | 35 | ## [v1.3.0](https://github.com/azuqua/notp/tree/v1.3.0) (2017-07-12) 36 | [Full Changelog](https://github.com/azuqua/notp/compare/v1.2.3...v1.3.0) 37 | 38 | **Merged pull requests:** 39 | 40 | - DLM/DSM gen\_server functionality, more examples [\#37](https://github.com/azuqua/notp/pull/37) ([kevinwilson541](https://github.com/kevinwilson541)) 41 | - DLM/DSM gen\_servers, more examples, MTable class for non-persistent DTable [\#36](https://github.com/azuqua/notp/pull/36) ([kevinwilson541](https://github.com/kevinwilson541)) 42 | 43 | ## [v1.2.3](https://github.com/azuqua/notp/tree/v1.2.3) (2017-06-30) 44 | [Full Changelog](https://github.com/azuqua/notp/compare/v1.2.2...v1.2.3) 45 | 46 | **Merged pull requests:** 47 | 48 | - Add sessionless command execution on CLI, CHANGELOG.md file generated [\#35](https://github.com/azuqua/notp/pull/35) ([kevinwilson541](https://github.com/kevinwilson541)) 49 | - Kevin/develop [\#33](https://github.com/azuqua/notp/pull/33) ([kevinwilson541](https://github.com/kevinwilson541)) 50 | 51 | ## [v1.2.2](https://github.com/azuqua/notp/tree/v1.2.2) (2017-06-16) 52 | [Full Changelog](https://github.com/azuqua/notp/compare/v1.2.1...v1.2.2) 53 | 54 | **Merged pull requests:** 55 | 56 | - bug fixes for AOF load, updated tests to reflect changes [\#32](https://github.com/azuqua/notp/pull/32) ([kevinwilson541](https://github.com/kevinwilson541)) 57 | - Kevin/develop [\#31](https://github.com/azuqua/notp/pull/31) ([kevinwilson541](https://github.com/kevinwilson541)) 58 | 59 | ## [v1.2.1](https://github.com/azuqua/notp/tree/v1.2.1) (2017-06-09) 60 | [Full Changelog](https://github.com/azuqua/notp/compare/v1.2.0...v1.2.1) 61 | 62 | **Merged pull requests:** 63 | 64 | - Develop [\#30](https://github.com/azuqua/notp/pull/30) ([kevinwilson541](https://github.com/kevinwilson541)) 65 | - add dtable as available class/creator function in index.js [\#29](https://github.com/azuqua/notp/pull/29) ([kevinwilson541](https://github.com/kevinwilson541)) 66 | 67 | ## [v1.2.0](https://github.com/azuqua/notp/tree/v1.2.0) (2017-06-09) 68 | [Full Changelog](https://github.com/azuqua/notp/compare/v1.1.4...v1.2.0) 69 | 70 | **Merged pull requests:** 71 | 72 | - Added examples for gen\_servers, dtable module for persistent in-memory key/value storage [\#28](https://github.com/azuqua/notp/pull/28) ([kevinwilson541](https://github.com/kevinwilson541)) 73 | - keep data directory for tests [\#27](https://github.com/azuqua/notp/pull/27) ([kevinwilson541](https://github.com/kevinwilson541)) 74 | - add more documentation to dtable class [\#25](https://github.com/azuqua/notp/pull/25) ([kevinwilson541](https://github.com/kevinwilson541)) 75 | - add disk-based table using AOF+snapshot persistence model \(for future… [\#24](https://github.com/azuqua/notp/pull/24) ([kevinwilson541](https://github.com/kevinwilson541)) 76 | - fix dlm example to reference data.holder correctly on lock timeout [\#23](https://github.com/azuqua/notp/pull/23) ([kevinwilson541](https://github.com/kevinwilson541)) 77 | - Kevin/develop [\#22](https://github.com/azuqua/notp/pull/22) ([kevinwilson541](https://github.com/kevinwilson541)) 78 | 79 | ## [v1.1.4](https://github.com/azuqua/notp/tree/v1.1.4) (2017-04-26) 80 | [Full Changelog](https://github.com/azuqua/notp/compare/v1.1.3...v1.1.4) 81 | 82 | **Merged pull requests:** 83 | 84 | - Develop [\#21](https://github.com/azuqua/notp/pull/21) ([kevinwilson541](https://github.com/kevinwilson541)) 85 | - Kevin/develop [\#20](https://github.com/azuqua/notp/pull/20) ([kevinwilson541](https://github.com/kevinwilson541)) 86 | - update vector clock constants to fix milli/micro error [\#19](https://github.com/azuqua/notp/pull/19) ([kevinwilson541](https://github.com/kevinwilson541)) 87 | - Update Gruntfile.js [\#18](https://github.com/azuqua/notp/pull/18) ([aembke](https://github.com/aembke)) 88 | 89 | ## [v1.1.3](https://github.com/azuqua/notp/tree/v1.1.3) (2017-04-14) 90 | [Full Changelog](https://github.com/azuqua/notp/compare/v1.1.2...v1.1.3) 91 | 92 | **Merged pull requests:** 93 | 94 | - Develop [\#17](https://github.com/azuqua/notp/pull/17) ([kevinwilson541](https://github.com/kevinwilson541)) 95 | - Kevin/develop [\#16](https://github.com/azuqua/notp/pull/16) ([kevinwilson541](https://github.com/kevinwilson541)) 96 | 97 | ## [v1.1.2](https://github.com/azuqua/notp/tree/v1.1.2) (2017-03-30) 98 | [Full Changelog](https://github.com/azuqua/notp/compare/v1.1.1...v1.1.2) 99 | 100 | **Merged pull requests:** 101 | 102 | - Develop [\#15](https://github.com/azuqua/notp/pull/15) ([kevinwilson541](https://github.com/kevinwilson541)) 103 | - remove TODOs from CLI and update README with patch [\#14](https://github.com/azuqua/notp/pull/14) ([kevinwilson541](https://github.com/kevinwilson541)) 104 | 105 | ## [v1.1.1](https://github.com/azuqua/notp/tree/v1.1.1) (2017-03-29) 106 | [Full Changelog](https://github.com/azuqua/notp/compare/v1.1.0...v1.1.1) 107 | 108 | **Merged pull requests:** 109 | 110 | - Develop [\#13](https://github.com/azuqua/notp/pull/13) ([kevinwilson541](https://github.com/kevinwilson541)) 111 | - Kevin/develop [\#12](https://github.com/azuqua/notp/pull/12) ([kevinwilson541](https://github.com/kevinwilson541)) 112 | 113 | ## [v1.1.0](https://github.com/azuqua/notp/tree/v1.1.0) (2017-03-29) 114 | [Full Changelog](https://github.com/azuqua/notp/compare/v1.0.0...v1.1.0) 115 | 116 | **Merged pull requests:** 117 | 118 | - v1.1.0 commits for documentation and GenServer request timeouts [\#11](https://github.com/azuqua/notp/pull/11) ([kevinwilson541](https://github.com/kevinwilson541)) 119 | - Enhanced documentation, added request timeouts GenServer side [\#10](https://github.com/azuqua/notp/pull/10) ([kevinwilson541](https://github.com/kevinwilson541)) 120 | - don't need to overwrite path to index in docs [\#9](https://github.com/azuqua/notp/pull/9) ([aembke](https://github.com/aembke)) 121 | - back to sudo [\#8](https://github.com/azuqua/notp/pull/8) ([aembke](https://github.com/aembke)) 122 | - no sudo [\#7](https://github.com/azuqua/notp/pull/7) ([aembke](https://github.com/aembke)) 123 | - docs, travis, coveralls [\#6](https://github.com/azuqua/notp/pull/6) ([aembke](https://github.com/aembke)) 124 | 125 | ## [v1.0.0](https://github.com/azuqua/notp/tree/v1.0.0) (2017-03-24) 126 | **Merged pull requests:** 127 | 128 | - v0.0.1 [\#5](https://github.com/azuqua/notp/pull/5) ([kevinwilson541](https://github.com/kevinwilson541)) 129 | - Kevin/develop [\#4](https://github.com/azuqua/notp/pull/4) ([kevinwilson541](https://github.com/kevinwilson541)) 130 | - changes to command server to preserve message tag, cli under bin dire… [\#3](https://github.com/azuqua/notp/pull/3) ([kevinwilson541](https://github.com/kevinwilson541)) 131 | - command server for impending cli tool for cluster management, tests [\#2](https://github.com/azuqua/notp/pull/2) ([kevinwilson541](https://github.com/kevinwilson541)) 132 | - Kevin/develop [\#1](https://github.com/azuqua/notp/pull/1) ([kevinwilson541](https://github.com/kevinwilson541)) 133 | 134 | 135 | 136 | \* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* 137 | -------------------------------------------------------------------------------- /test/unit/conn.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | uuid = require("uuid"), 4 | assert = require("chai").assert; 5 | 6 | module.exports = function (mocks, lib) { 7 | describe("Connection unit tests", function () { 8 | var Connection = lib.conn, 9 | Node = lib.node, 10 | server, 11 | ipc, 12 | node, 13 | conn; 14 | 15 | before(function (done) { 16 | ipc = new mocks.ipc(); 17 | ipc.config.id = "id"; 18 | ipc.serveNet("localhost", 8000, done); 19 | ipc.server.start(); 20 | }); 21 | 22 | beforeEach(function () { 23 | node = new Node("id", "localhost", 8000); 24 | conn = new Connection(ipc, node); 25 | }); 26 | 27 | after(function () { 28 | ipc.server.stop(); 29 | }); 30 | 31 | it("Should construct a connection", function () { 32 | assert.deepEqual(conn._ipc, ipc); 33 | assert.ok(conn._node.equals(node)); 34 | assert.equal(conn._queue.size(), 0); 35 | assert.equal(conn._connecting, false); 36 | assert.equal(conn._active, false); 37 | assert.equal(conn._streams.size, 0); 38 | }); 39 | 40 | it("Should start connection", function (done) { 41 | conn.start(); 42 | assert.ok(conn._connecting); 43 | assert.ok(conn._active); 44 | conn.on("connect", () => { 45 | assert.notOk(conn._connecting); 46 | assert.lengthOf(conn._ipc.of[node.id()].listeners("connect"), 1); 47 | assert.lengthOf(conn._ipc.of[node.id()].listeners("disconnect"), 1); 48 | conn._ipc.disconnect(conn._node.id()); 49 | done(); 50 | }); 51 | }); 52 | 53 | it("Should stop connection", function (done) { 54 | conn.start(); 55 | conn.once("connect", () => { 56 | conn.stop(); 57 | conn.once("disconnect", () => { 58 | assert.notOk(conn._connecting); 59 | assert.notOk(conn._active); 60 | assert.equal(conn._queue.size(), 0); 61 | done(); 62 | }); 63 | }); 64 | }); 65 | 66 | it("Should wait for idle state via stream check before stopping connection", function (done) { 67 | conn.start(); 68 | conn.once("connect", () => { 69 | conn._streams.set("foo", true); 70 | conn.once("disconnect", () => { 71 | assert.notOk(conn._connecting); 72 | assert.notOk(conn._active); 73 | assert.equal(conn._queue.size(), 0); 74 | assert.equal(conn._streams.size, 0); 75 | done(); 76 | }); 77 | conn.stop(); 78 | conn._streams.delete("foo"); 79 | conn.emit("idle"); 80 | }); 81 | }); 82 | 83 | it("Should wait for idle state via queue check before stopping connection", function (done) { 84 | conn.start(); 85 | conn.once("connect", () => { 86 | conn._queue.enqueue({ 87 | event: "foo", 88 | data: { 89 | stream: {stream: "id", done: true}, 90 | data: "bar" 91 | } 92 | }); 93 | conn.once("disconnect", () => { 94 | assert.notOk(conn._connecting); 95 | assert.notOk(conn._active); 96 | assert.equal(conn._queue.size(), 0); 97 | assert.equal(conn._streams.size, 0); 98 | done(); 99 | }); 100 | conn.stop(); 101 | conn._queue.dequeue(); 102 | conn.emit("idle"); 103 | }); 104 | }); 105 | 106 | it("Should forcibly stop connection", function (done) { 107 | conn.start(); 108 | conn.once("connect", () => { 109 | conn.stop(true); 110 | assert.notOk(conn._connecting); 111 | assert.notOk(conn._active); 112 | assert.equal(conn._queue.size(), 0); 113 | assert.equal(conn._streams.size, 0); 114 | done(); 115 | }); 116 | }); 117 | 118 | it("Should return node at connection", function () { 119 | assert(conn.node().equals(node)); 120 | }); 121 | 122 | it("Should return queue at connection", function () { 123 | assert.equal(conn.queue().size(), 0); 124 | conn.queue().enqueue("val"); 125 | assert.equal(conn.queue().size(), 1); 126 | }); 127 | 128 | it("Should return active status of connection", function () { 129 | assert.equal(conn.active(), false); 130 | }); 131 | 132 | it("Should return connecting status of connection", function () { 133 | assert.equal(conn.connecting(), false); 134 | }); 135 | 136 | it("Should return idle state of connection", function () { 137 | assert.equal(conn.idle(), true); 138 | conn.queue().enqueue("val"); 139 | assert.equal(conn.idle(), false); 140 | }); 141 | 142 | it("Should return max queue length", function () { 143 | assert.equal(conn.maxLen(), 1024); 144 | }); 145 | 146 | it("Should set new max queue length", function () { 147 | conn.maxLen(1023); 148 | assert.equal(conn.maxLen(), 1023); 149 | conn.maxLen(-1); 150 | assert.equal(conn.maxLen(), 1023); 151 | 152 | conn._queue.enqueue("foo"); 153 | conn._queue.enqueue("bar"); 154 | conn.maxLen(1); 155 | assert.equal(conn.maxLen(), 1); 156 | assert.equal(conn._queue.size(), 1); 157 | }); 158 | 159 | it("Should throw if connection inactive and data is sent", function () { 160 | var data = { 161 | data: "bar", 162 | stream: {stream: uuid.v4(), done: false} 163 | }; 164 | var out = conn.send("foo", data); 165 | assert(out instanceof Error); 166 | }); 167 | 168 | it("Should queue data if connection reconnecting", function () { 169 | conn._active = true; 170 | conn._connecting = true; 171 | var data = { 172 | data: "bar", 173 | stream: {stream: uuid.v4(), done: false} 174 | }; 175 | conn.send("foo", data); 176 | assert.equal(conn.queue().size(), 1); 177 | assert.deepEqual(conn.queue().dequeue(), { 178 | event: "foo", 179 | data: data 180 | }); 181 | }); 182 | 183 | it("Should queue data and drop data if queue has reached max size", function () { 184 | conn._active = true; 185 | conn._connecting = true; 186 | conn._maxLen = 1; 187 | conn._queue.enqueue("data"); 188 | var data = { 189 | data: "bar", 190 | stream: {stream: uuid.v4(), done: false} 191 | }; 192 | conn.send("foo", data); 193 | assert.equal(conn.queue().size(), 1); 194 | assert.notEqual(conn._queue.peek(), "data"); 195 | assert.deepEqual(conn.queue().dequeue(), { 196 | event: "foo", 197 | data: data 198 | }); 199 | }); 200 | 201 | it("Should write data if connection active and not reconnecting", function (done) { 202 | conn.start(); 203 | conn.once("connect", () => { 204 | var val = { 205 | data: "bar", 206 | stream: {stream: uuid.v4(), done: false} 207 | }; 208 | conn.once("send", (event, data) => { 209 | assert.equal(event, "foo"); 210 | assert.equal(data, val); 211 | conn.stop(); 212 | done(); 213 | }); 214 | ipc.of[node.id()].on("foo", (data) => { 215 | assert.deepEqual(data, val); 216 | }); 217 | conn.send("foo", val); 218 | }); 219 | }); 220 | 221 | it("Should handle connection logic", function (done) { 222 | var vals = [ 223 | { 224 | event: "foo", 225 | data: { 226 | data: 0, 227 | stream: {stream: uuid.v4(), done: false} 228 | } 229 | }, 230 | { 231 | event: "bar", 232 | data: { 233 | data: 1, 234 | stream: {stream: uuid.v4(), done: false} 235 | } 236 | }, 237 | { 238 | event: "baz", 239 | data: { 240 | data: 2, 241 | stream: {stream: uuid.v4(), done: false} 242 | } 243 | } 244 | ]; 245 | conn.start(); 246 | conn.once("connect", () => { 247 | var called = 0; 248 | conn.on("send", (event, data) => { 249 | assert.deepEqual(vals[called], { 250 | event: event, 251 | data: data 252 | }); 253 | called++; 254 | if (called === 3) { 255 | assert.equal(conn.connecting(), false); 256 | conn.removeAllListeners("send"); 257 | conn.stop(); 258 | return done(); 259 | } 260 | }); 261 | vals.forEach((val) => { 262 | conn._queue.enqueue(val); 263 | }); 264 | conn._handleConnect(); 265 | }); 266 | }); 267 | 268 | it("Should handle disconnection logic when active", function (done) { 269 | conn.start(); 270 | conn.once("connect", () => { 271 | conn.once("disconnect", () => { 272 | assert.equal(conn.connecting(), true); 273 | conn.stop(); 274 | done(); 275 | }); 276 | conn._handleDisconnect(); 277 | }); 278 | }); 279 | 280 | it("Should handle disconnection logic after stop", function (done) { 281 | conn.start(); 282 | conn.once("connect", () => { 283 | conn.once("disconnect", () => { 284 | assert.equal(conn.connecting(), false); 285 | conn.stop(); 286 | done(); 287 | }); 288 | conn._active = false; 289 | conn._handleDisconnect(); 290 | }); 291 | }); 292 | 293 | it("Should update stream map", function (done) { 294 | var id = uuid.v4(); 295 | conn._updateStream({stream: id, done: false}); 296 | assert.equal(conn._streams.size, 1); 297 | conn.on("idle", () => { 298 | assert.equal(conn._streams.size, 0); 299 | done(); 300 | }); 301 | conn._updateStream({stream: id, done: true}); 302 | }); 303 | }); 304 | }; 305 | -------------------------------------------------------------------------------- /test/unit/vclock.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | rbt = require("functional-red-black-tree"), 4 | assert = require("chai").assert; 5 | 6 | module.exports = function (mocks, lib) { 7 | describe("VectorClock unit tests", function () { 8 | var VectorClock = lib.vclock; 9 | var vclock; 10 | beforeEach(function () { 11 | vclock = new VectorClock(); 12 | }); 13 | 14 | it("Should construct a vclock", function () { 15 | assert.equal(vclock._size, 0); 16 | 17 | vclock = new VectorClock("id", 1); 18 | assert.equal(vclock.size(), 1); 19 | assert.deepEqual(vclock.get("id").count, 1); 20 | }); 21 | 22 | it("Should insert nodes", function () { 23 | var node = "key"; 24 | vclock.insert(node); 25 | assert.equal(vclock.size(), 1); 26 | assert.equal(vclock.get("key").count, 0); 27 | 28 | vclock.increment(node); 29 | vclock.insert(node); 30 | assert.equal(vclock.size(), 1); 31 | assert.equal(vclock.getCount(node), 1); 32 | }); 33 | 34 | it("Should update node in clock", function () { 35 | var node = "key"; 36 | vclock.update(node, 1); 37 | assert.equal(vclock.getCount(node), 1); 38 | assert.equal(vclock.size(), 1); 39 | 40 | var date = Date.now(); 41 | vclock.update(node, 2, date); 42 | assert.equal(vclock.getCount(node), 2); 43 | assert.equal(vclock.getTimestamp(node), date); 44 | assert.equal(vclock.size(), 1); 45 | }); 46 | 47 | it("Should remove node in clock", function () { 48 | var node = "key"; 49 | vclock.remove(node); 50 | assert.equal(vclock.get(node), undefined); 51 | assert.equal(vclock.size(), 0); 52 | 53 | vclock.insert(node); 54 | vclock.remove(node); 55 | assert.equal(vclock.get(node), undefined); 56 | assert.equal(vclock.size(), 0); 57 | }); 58 | 59 | it("Should increment node in clock", function () { 60 | var node = "key"; 61 | vclock.increment(node); 62 | assert.equal(vclock.size(), 1); 63 | assert.equal(vclock.getCount(node), 1); 64 | 65 | vclock.increment(node); 66 | assert.equal(vclock.size(), 1); 67 | assert.equal(vclock.getCount(node), 2); 68 | }); 69 | 70 | it("Should get value stored at a node", function () { 71 | var node = "key"; 72 | vclock.increment(node); 73 | assert.equal(vclock.get(node).count, 1); 74 | }); 75 | 76 | it("Should get count stored at a node", function () { 77 | var node = "key"; 78 | vclock.increment(node); 79 | assert.equal(vclock.getCount(node), 1); 80 | }); 81 | 82 | it("Should get insert timestamp stored at a node", function () { 83 | var node = "key"; 84 | var date = Date.now(); 85 | vclock._vector[node] = {count: 1, insert: date}; 86 | assert.equal(vclock.getInsert(node), date); 87 | }); 88 | 89 | it("Should get timestamp stored at a node", function () { 90 | var node = "key"; 91 | var date = Date.now(); 92 | vclock._vector[node] = {count: 1, time: date}; 93 | assert.equal(vclock.getTimestamp(node), date); 94 | }); 95 | 96 | it("Should return if clock has node or not", function () { 97 | var node = "key"; 98 | assert.equal(vclock.has(node), false); 99 | vclock.insert(node); 100 | assert.equal(vclock.has(node), true); 101 | }); 102 | 103 | it("Should not merge clocks if input is empty", function () { 104 | var node = "key"; 105 | vclock.insert(node); 106 | var v2 = new VectorClock(); 107 | vclock.merge(v2); 108 | assert.equal(vclock.size(), 1); 109 | }); 110 | 111 | it("Should clone input clock if local vector is empty", function () { 112 | var node = "key"; 113 | var v2 = new VectorClock(); 114 | v2.insert(node); 115 | vclock.merge(v2); 116 | assert.equal(vclock.size(), 1); 117 | assert.deepEqual(vclock._vector, v2._vector); 118 | }); 119 | 120 | it("Should merge two clocks, extending with nonexistant values", function () { 121 | var node = "key", 122 | node2 = "key2"; 123 | var v2 = new VectorClock(); 124 | vclock.insert(node); 125 | v2.insert(node2); 126 | vclock.merge(v2); 127 | assert.equal(vclock.size(), 2); 128 | assert.deepEqual(vclock.get(node2), v2.get(node2)); 129 | }); 130 | 131 | it("Should merge two clocks, prioritizing higher counts", function () { 132 | var node = "key"; 133 | var v2 = new VectorClock(); 134 | vclock.update(node, 1); 135 | v2.update(node, 2); 136 | vclock.merge(v2); 137 | assert.equal(vclock.size(), 1); 138 | assert.deepEqual(vclock.get(node), v2.get(node)); 139 | 140 | vclock.update(node, 3, Date.now()); 141 | v2.update(node, 1, Date.now()-5); 142 | vclock.merge(v2); 143 | assert.equal(vclock.size(), 1); 144 | assert.equal(vclock.getCount(node), 3); 145 | assert.notEqual(vclock.getTimestamp(node), v2.getTimestamp(node)); 146 | }); 147 | 148 | it("Should merge two clocks, prioritizing timestamp after count", function () { 149 | var node = "key"; 150 | var v2 = new VectorClock(); 151 | vclock.update(node, 1, Date.now()-5); 152 | v2.update(node, 1, Date.now()); 153 | vclock.merge(v2); 154 | assert.equal(vclock.size(), 1); 155 | assert.deepEqual(vclock.get(node), v2.get(node)); 156 | 157 | vclock.update(node, 1, Date.now()); 158 | v2.update(node, 1, Date.now()-5); 159 | vclock.merge(v2); 160 | assert.equal(vclock.size(), 1); 161 | assert.equal(vclock.getCount(node), 1); 162 | assert.notEqual(vclock.getTimestamp(node), v2.getTimestamp(node)); 163 | }); 164 | 165 | it("Should descend from trivial clock", function () { 166 | var node = "key"; 167 | var v2 = new VectorClock(); 168 | vclock.insert(node); 169 | assert.equal(vclock.descends(v2), true); 170 | }); 171 | 172 | it("Should not descend from input clock unless input is trivial", function () { 173 | var node = "key"; 174 | var v2 = new VectorClock(); 175 | v2.insert(node); 176 | assert.equal(vclock.descends(v2), false); 177 | }); 178 | 179 | it("Should not descend from clock if one element is newer in input", function () { 180 | var node = "key"; 181 | var v2 = new VectorClock(); 182 | vclock.update(node, 1); 183 | v2.update(node, 2); 184 | assert.equal(vclock.descends(v2), false); 185 | }); 186 | 187 | it("Should not descend from clock if one element missing in clock", function () { 188 | var node = "key", 189 | node2 = "key2"; 190 | var v2 = new VectorClock(); 191 | vclock.update(node, 1); 192 | v2.update(node2, 1); 193 | assert.equal(vclock.descends(v2), false); 194 | }); 195 | 196 | it("Should descend from clock if all elements older in input", function () { 197 | var node = "key"; 198 | var v2 = new VectorClock(); 199 | vclock.update(node, 2); 200 | v2.update(node, 1); 201 | assert.equal(vclock.descends(v2), true); 202 | }); 203 | 204 | it("Should return false if two vector clocks don't have same size", function () { 205 | var v2 = new VectorClock("foo", 1); 206 | assert.notOk(vclock.equals(v2)); 207 | }); 208 | 209 | it("Should return false if two vector clocks have different nodes", function () { 210 | var v2 = new VectorClock("foo", 1); 211 | vclock.increment("bar"); 212 | assert.notOk(vclock.equals(v2)); 213 | }); 214 | 215 | it("Should return true if two vector clocks equal each other", function () { 216 | vclock.increment("foo"); 217 | var v2 = (new VectorClock()).fromJSON(vclock.toJSON(true)); 218 | assert.ok(vclock.equals(v2)); 219 | }); 220 | 221 | it("Should not trim clock if too small", function () { 222 | var opts = { 223 | lowerBound: 2 224 | }; 225 | var threshold = Date.now(); 226 | var node = "key"; 227 | vclock.update(node, 1); 228 | vclock.trim(threshold, opts); 229 | assert.equal(vclock.size(), 1); 230 | }); 231 | 232 | it("Should not trim clock if oldest element too young", function () { 233 | var opts = { 234 | lowerBound: 1, 235 | youngBound: 10 236 | }; 237 | var threshold = Date.now(); 238 | var node = "key", 239 | node2 = "key2"; 240 | vclock.update(node, 1, threshold-5); 241 | vclock.update(node2, 1, threshold-5); 242 | vclock.trim(threshold, opts); 243 | assert.equal(vclock.size(), 2); 244 | }); 245 | 246 | it("Should trim clock if too large", function () { 247 | var opts = { 248 | lowerBound: 1, 249 | youngBound: 10, 250 | upperBound: 1, 251 | oldBound: 15 252 | }; 253 | var threshold = Date.now(); 254 | var node = "key", 255 | node2 = "key2"; 256 | vclock.update(node, 1, threshold-11); 257 | vclock.update(node2, 1, threshold-12); 258 | vclock.trim(threshold, opts); 259 | assert.equal(vclock.size(), 1); 260 | assert.ok(vclock.get(node)); 261 | }); 262 | 263 | it("Should trim clock if too old", function () { 264 | var opts = { 265 | lowerBound: 1, 266 | youngBound: 10, 267 | upperBound: 2, 268 | oldBound: 15 269 | }; 270 | var threshold = Date.now(); 271 | var node = "key", 272 | node2 = "key2"; 273 | vclock.update(node, 1, threshold-11); 274 | vclock.update(node2, 1, threshold-16); 275 | vclock.trim(threshold, opts); 276 | assert.equal(vclock.size(), 1); 277 | assert.ok(vclock.get(node)); 278 | }); 279 | 280 | it("Should return nodes in clock", function () { 281 | vclock.insert("node"); 282 | vclock.insert("node2"); 283 | assert.lengthOf(_.xor(vclock.nodes(), ["node", "node2"]), 0); 284 | }); 285 | 286 | it("Should convert clock into json", function () { 287 | vclock.insert("node"); 288 | assert.deepEqual(vclock.toJSON(), vclock._vector); 289 | assert.deepEqual(vclock.toJSON(true), vclock._vector); 290 | }); 291 | 292 | it("Should convert json to vclock", function () { 293 | var vector = { 294 | foo: { 295 | count: 1, 296 | time: 0 297 | } 298 | }; 299 | vclock.fromJSON(vector); 300 | assert.equal(vclock.size(), 1); 301 | assert.deepEqual(vclock._vector, vector); 302 | }); 303 | 304 | it("Should check for valid JSON", function () { 305 | var val = "foo"; 306 | assert.notOk(VectorClock.validJSON(val)); 307 | 308 | val = {}; 309 | assert.ok(VectorClock.validJSON(val)); 310 | }); 311 | }); 312 | }; 313 | -------------------------------------------------------------------------------- /lib/mtable.js: -------------------------------------------------------------------------------- 1 | const _ = require("lodash"), 2 | debug = require("debug")("notp:lib:mtable"), 3 | async = require("async"), 4 | shortid = require("shortid"), 5 | EventEmitter = require("events").EventEmitter; 6 | 7 | class MTable extends EventEmitter { 8 | /** 9 | * 10 | * In-memory key/value storage with the same data structure API as the DTable class. Does not include functionality to persist to disk. 11 | * 12 | * @class MTable 13 | * @memberof Clusterluck 14 | * 15 | * @param {Object} opts 16 | * @param {String} opts.path 17 | * @param {Number} [opts.writeThreshold] 18 | * @param {Number} [opts.autoSave] 19 | * @param {Number} [opts.fsyncInterval] 20 | * 21 | */ 22 | constructor() { 23 | super(); 24 | this._table = new Map(); 25 | } 26 | 27 | /** 28 | * 29 | * Starts dtable instance, which triggers an fopen call to LATEST.LOG, the fsync interval for this log file, as well as other internal intervals to check for storage snapshot flush conditions. 30 | * 31 | * @method start 32 | * @memberof Clusterluck.MTable 33 | * @instance 34 | * 35 | * @param {String} name - Name of table, meant for debugging purposes. 36 | * 37 | * @return {Clusterluck.MTable} This instance. 38 | * 39 | */ 40 | start(name) { 41 | this._name = name; 42 | this._id = shortid.generate(); 43 | return this; 44 | } 45 | 46 | /** 47 | * 48 | * Stops the table, including all internal disk-based logic. If the table is idle and has an open file descriptor against LATEST.LOG, it will close immediately. If it's idle but the file descriptor against LATEST.LOG has been closed, this call will wait for a file descriptor to open again before continuing. Otherwise, the table isn't idle and therefore we wait for this condition. 49 | * 50 | * @method stop 51 | * @memberof Clusterluck.MTable 52 | * @instance 53 | * 54 | * @param {Function} cb - Callback called once the table has been stopped. 55 | * 56 | */ 57 | stop(cb) { 58 | this.emit("stop"); 59 | this._id = null; 60 | async.nextTick(cb); 61 | } 62 | 63 | /** 64 | * 65 | * Trivial call to load state. 66 | * 67 | * @method load 68 | * @memberof Clusterluck.MTable 69 | * @instance 70 | * 71 | * @param {Function} cb - Function of the form `function (err) {...}`, where `err` will be passed if an error occurs loading state from disk. 72 | * 73 | */ 74 | load(cb) { 75 | async.nextTick(cb); 76 | } 77 | 78 | /** 79 | * 80 | * Returns whether this table is in an idle state or not. 81 | * 82 | * @method idle 83 | * @memberof Clusterluck.MTable 84 | * @instance 85 | * 86 | * @return {Boolean} Whether this table is idle or not. 87 | * 88 | */ 89 | idle() { 90 | return true; 91 | } 92 | 93 | /** 94 | * 95 | * Retrieves value stored at `key`, returning `undefined` if no such data exists. 96 | * 97 | * @method get 98 | * @memberof Clusterluck.MTable 99 | * @instance 100 | * 101 | * @param {String} key - Key to retrieve data from. 102 | * 103 | * @return {Map|Set|JSON} Value stored at `key`. 104 | * 105 | */ 106 | get(key) { 107 | return this._table.get(key); 108 | } 109 | 110 | /** 111 | * 112 | * Returns whether `val` is a member of the set stored at `key`. 113 | * 114 | * @method smember 115 | * @memberof Clusterluck.MTable 116 | * @instance 117 | * 118 | * @param {String} key - Key to retrieve set from. 119 | * @param {String} val - Value to check existence of in the set. 120 | * 121 | * @return {Boolean} Whether `val` is a member of the set stored at `key`. 122 | * 123 | */ 124 | smember(key, val) { 125 | const out = this._table.get(key) || new Set(); 126 | if (!(out instanceof Set)) { 127 | throw MTable.invalidTypeError("smember", key, typeof out); 128 | } 129 | return out.has(val); 130 | } 131 | 132 | /** 133 | * 134 | * Retrieves value stored at hash key `hkey` under storage key `key`, returning `undefined` if no such data exists. 135 | * 136 | * @method hget 137 | * @memberof Clusterluck.MTable 138 | * @instance 139 | * 140 | * @param {String} key - Key to retrieve hash map from. 141 | * @param {String} hkey - Hash key to retrieve data from. 142 | * 143 | * @return {JSON} - Value stored under hash key `hkey` at the hash map stored under `key`. 144 | * 145 | */ 146 | hget(key, hkey) { 147 | const out = this._table.get(key) || new Map(); 148 | if (!(out instanceof Map)) { 149 | throw new MTable.invalidTypeError("hget", key, typeof out); 150 | } 151 | return out.get(hkey); 152 | } 153 | 154 | /** 155 | * 156 | * Sets value `value` under key `key`. 157 | * 158 | * @method set 159 | * @memberof Clusterluck.MTable 160 | * @instance 161 | * 162 | * @param {String} key 163 | * @param {Map|Set|JSON} val 164 | * 165 | * @return {Map|Set|JSON} 166 | * 167 | */ 168 | set(key, val) { 169 | this._table.set(key, val); 170 | return val; 171 | } 172 | 173 | /** 174 | * 175 | * Inserts `val` into the set stored at key `key`. 176 | * 177 | * @method sset 178 | * @memberof Clusterluck.MTable 179 | * @instance 180 | * 181 | * @param {String} key - Key which holds the set to insert `val` under. 182 | * @param {String} val - Value to insert into the set. 183 | * 184 | * @return {String} The set stored at `key`. 185 | * 186 | */ 187 | sset(key, val) { 188 | const out = this._table.get(key) || new Set(); 189 | if (!(out instanceof Set)) { 190 | throw MTable.invalidTypeError("sset", key, typeof out); 191 | } 192 | out.add(val); 193 | this._table.set(key, out); 194 | return out; 195 | } 196 | 197 | /** 198 | * 199 | * Sets `value` under the hash key `hkey` in the hash map stored at `key`. 200 | * 201 | * @method hset 202 | * @memberof Clusterluck.MTable 203 | * @instance 204 | * 205 | * @param {String} key - Key which holds the hash map. 206 | * @param {String} hkey - Hash key to insert `val` under. 207 | * @param {JSON} val - Value to set under `hkey` in the hash map. 208 | * 209 | * @return {Map} The map stored at `key`. 210 | * 211 | */ 212 | hset(key, hkey, val) { 213 | const out = this._table.get(key) || new Map(); 214 | if (!(out instanceof Map)) { 215 | throw MTable.invalidTypeError("hset", key, typeof out); 216 | } 217 | out.set(hkey, val); 218 | this._table.set(key, out); 219 | return out; 220 | } 221 | 222 | /** 223 | * 224 | * Removes key `key` from this table. 225 | * 226 | * @method del 227 | * @memberof Clusterluck.MTable 228 | * @instance 229 | * 230 | * @param {String} key - Key to remove from this table. 231 | * 232 | * @return {Clusterluck.MTable} This instance. 233 | * 234 | */ 235 | del(key) { 236 | this._table.delete(key); 237 | return this; 238 | } 239 | 240 | /** 241 | * 242 | * Deletes `val` from the set stored under `key`. 243 | * 244 | * @method sdel 245 | * @memberof Clusterluck.MTable 246 | * @instance 247 | * 248 | * @param {String} key - Key which olds the set to remove `val` from. 249 | * @param {String} val - Value to remove from the set. 250 | * 251 | * @return {Clusterluck.MTable} This instance. 252 | * 253 | */ 254 | sdel(key, val) { 255 | const out = this._table.get(key) || new Set(); 256 | if (!(out instanceof Set)) { 257 | throw MTable.invalidTypeError("sdel", key, typeof out); 258 | } 259 | out.delete(val); 260 | if (out.size === 0) this._table.delete(key); 261 | return this; 262 | } 263 | 264 | /** 265 | * 266 | * Removes the hash key `hkey` from the hash map stored under `key`. 267 | * 268 | * @method hdel 269 | * @memberof Clusterluck.MTable 270 | * @instance 271 | * 272 | * @param {String} key - Key which holds the hash map that `hkey` will be removed from. 273 | * @param {String} hkey - The hash key to remove from the hash map. 274 | * 275 | * @return {Clusterluck.MTable} This instance. 276 | * 277 | */ 278 | hdel(key, hkey) { 279 | const out = this._table.get(key) || new Map(); 280 | if (!(out instanceof Map)) { 281 | throw MTable.invalidTypeError("hdel", key, typeof out); 282 | } 283 | out.delete(hkey); 284 | if (out.size === 0) this._table.delete(key); 285 | return this; 286 | } 287 | 288 | /** 289 | * 290 | * Clears the contents of this table. 291 | * 292 | * @method clear 293 | * @memberof Clusterluck.MTable 294 | * @instance 295 | * 296 | * @return {Clusterluck.MTable} This instance. 297 | * 298 | */ 299 | clear() { 300 | this._table.clear(); 301 | return this; 302 | } 303 | 304 | /** 305 | * 306 | * Asynchronously iterates over each key/value pair stored in this table at the point of this call. 307 | * 308 | * @method forEach 309 | * @memberof Clusterluck.MTable 310 | * @instance 311 | * 312 | * @param {Function} cb - Function to call on each key/value pair. Has the signature `function (key, val, next) {...}`. 313 | * @param {Function} fin -Finishing callback to call once iteration has completed. Hash the signature `function (err) {...}`, where `err` is populated if passed into the `next` callback at any point of iteration.. 314 | * 315 | */ 316 | forEach(cb, fin) { 317 | const entries = this._table.entries(); 318 | let done = false; 319 | async.whilst(() => { 320 | return done !== true; 321 | }, (next) => { 322 | const val = entries.next(); 323 | if (val.done === true) { 324 | done = true; 325 | return next(); 326 | } 327 | cb(val.value[0], val.value[1], next); 328 | }, fin); 329 | } 330 | 331 | /** 332 | * 333 | * Synchronously iterates over each key/value pair stored in this table. 334 | * 335 | * @method forEachSync 336 | * @memberof Clusterluck.MTable 337 | * @instance 338 | * 339 | * @param {Function} cb - Function call on each key/value pair. Has the signature `function (key, val) {...}`. 340 | * 341 | * @return {Clusterluck.MTable} This instance. 342 | * 343 | */ 344 | forEachSync(cb) { 345 | this._table.forEach((val, key) => { 346 | cb(key, val); 347 | }); 348 | return this; 349 | } 350 | 351 | /** 352 | * 353 | * @method invalidTypeError 354 | * @memberof Clusterluck.MTable 355 | * @static 356 | * 357 | * @param {String} command 358 | * @param {String} key 359 | * @param {String} type 360 | * 361 | * @return {Error} 362 | * 363 | */ 364 | static invalidTypeError(command, key, type) { 365 | let msg = "Invalid command '" + command + "' against key '" + key + "'"; 366 | msg += " of type '" + type +"'"; 367 | return _.extend(new Error(msg), { 368 | type: "INVALID_TYPE" 369 | }); 370 | } 371 | } 372 | 373 | module.exports = MTable; 374 | -------------------------------------------------------------------------------- /bin/cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/node 2 | 3 | const path = require("path"), 4 | os = require("os"), 5 | ipc = require("node-ipc"), 6 | NetKernel = require("../lib/kernel"), 7 | Queue = require("../lib/queue"), 8 | EventEmitter = require("events").EventEmitter, 9 | uuid = require("uuid"), 10 | crypto = require("crypto"), 11 | _ = require("lodash"), 12 | util = require("util"); 13 | 14 | const vorpal = require("vorpal")(); 15 | const argv = require("yargs") 16 | .usage("Usage: $0 [OPTIONS] [cmd [arg [arg ...]]]") 17 | .demand([]) 18 | .help("help") 19 | .describe("I", "Unique instance identifier of the node being connected to.") 20 | .alias("I", "instance") 21 | .describe("H", "Server hostname of the node being connected to.") 22 | .alias("H", "hostname") 23 | .describe("p", "Server port of the node being connected to.") 24 | .alias("p", "port") 25 | .describe("a", "Distributed cookie to use for signing requests against the connecting node.") 26 | .alias("a", "key") 27 | .default({ 28 | I: os.hostname(), 29 | H: os.hostname(), 30 | p: 7022, 31 | a: "", 32 | }) 33 | .number(["port"]) 34 | .string(["instance", "hostname", "key"]) 35 | .check((args, opts) => { 36 | return util.isNumber(args.port) && !isNaN(args.port); 37 | }) 38 | .argv; 39 | 40 | var singular = argv._.length === 0; 41 | 42 | function log(...args) { 43 | if (singular) console.log.apply(null, args); 44 | } 45 | 46 | const from = {id: argv.I + "_" + process.pid}; 47 | let client; 48 | 49 | class Client extends EventEmitter { 50 | constructor(ipc, id, host, port, cookie) { 51 | super(); 52 | this._ipc = ipc; 53 | this._id = id; 54 | this._host = host; 55 | this._port = port; 56 | this._cookie = cookie; 57 | this._rcv = new Queue(); 58 | this._connected = true; 59 | this._disLog = false; 60 | } 61 | 62 | start() { 63 | this._ipc.of[this._id].on("message", (data) => { 64 | if (data && data.tag === this._rcv.peek().tag) { 65 | this._rcv.dequeue().cb(data); 66 | } 67 | }); 68 | this._ipc.of[this._id].on("connect", _.partial(this._handleConnect).bind(this)); 69 | this._ipc.of[this._id].on("disconnect", _.partial(this._handleDisconnect).bind(this)); 70 | } 71 | 72 | stop() { 73 | this._ipc.disconnect(this._id); 74 | return this; 75 | } 76 | 77 | send(comm, message, cb) { 78 | const data = Buffer.from(JSON.stringify({event: comm, data: message})); 79 | const msg = { 80 | id: "command", 81 | tag: uuid.v4(), 82 | from: from, 83 | stream: {stream: uuid.v4(), done: true}, 84 | data: data 85 | }; 86 | if (this._connected) { 87 | ipc.of[argv.I].emit("message", NetKernel._encodeMsg(this._cookie, msg)); 88 | this._rcv.enqueue({ 89 | tag: msg.tag, 90 | cb: (data) => { 91 | data = _.omit(NetKernel._decodeMsg(this._cookie, data), "tag"); 92 | console.log(JSON.stringify(data, null, 2)); 93 | cb(); 94 | } 95 | }); 96 | return this; 97 | } 98 | cb(new Error("Disconnected from server.")); 99 | return this; 100 | } 101 | 102 | _handleConnect() { 103 | if (singular) { 104 | log("Connected to %s", this._id); 105 | } 106 | this._connected = true; 107 | this._disLog = false; 108 | this.emit("connect"); 109 | return this; 110 | } 111 | 112 | _handleDisconnect() { 113 | if (!this._disLog) { 114 | this._disLog = true; 115 | log("Disconnected from %s", this._id); 116 | } 117 | this._connected = false; 118 | this._rcv.flush().forEach((el) => { 119 | el.cb(new Error("Disconnected from server.")); 120 | }); 121 | this.emit("disconnect"); 122 | return this; 123 | } 124 | } 125 | 126 | let forceStr = "Execute this command before any existing " + 127 | "message streams on this node's gossip processor. Defaults to " + 128 | "false."; 129 | 130 | function parseNodeList(nodeList) { 131 | return _.chunk(nodeList, 3).reduce((memo, node) => { 132 | if (node.length !== 3) return memo; 133 | memo.push({ 134 | id: node[0], 135 | host: node[1], 136 | port: parseInt(node[2]) 137 | }); 138 | return memo; 139 | }, []); 140 | } 141 | 142 | vorpal 143 | .command("inspect") 144 | .description("Prints the ring of this node to the console.") 145 | .action(function (args, cb) { 146 | client.send("inspect", null, cb); 147 | }); 148 | 149 | vorpal 150 | .command("nodes") 151 | .description("Prints the nodes of the ring of this node to the console.") 152 | .action(function (args, cb) { 153 | client.send("nodes", null, cb); 154 | }); 155 | 156 | vorpal 157 | .command("get ") 158 | .description("Returns information about a node's hostname and port in this node's cluster.") 159 | .types({ 160 | string: ["id"] 161 | }) 162 | .action(function (args, cb) { 163 | client.send("get", {id: args.id}, cb); 164 | }); 165 | 166 | vorpal 167 | .command("has ") 168 | .description("Returns whether the targeted node exists in this node's cluster.") 169 | .types({ 170 | string: ["id"] 171 | }) 172 | .action(function (args, cb) { 173 | client.send("has", {id: args.id}, cb); 174 | }); 175 | 176 | vorpal 177 | .command("join ") 178 | .description("Joins a new cluster if not already present or not a member of any cluster.") 179 | .types({ 180 | string: ["id"] 181 | }) 182 | .action(function (args, cb) { 183 | client.send("join", {id: args.id}, cb); 184 | }); 185 | 186 | vorpal 187 | .command("meet ") 188 | .description("Meets a node in this node's cluster. This is the only way to do transitive additions to the cluster.") 189 | .types({ 190 | string: ["id", "host", "port"], 191 | }) 192 | .action(function (args, cb) { 193 | client.send("meet", { 194 | node: {id: args.id, host: args.host, port: parseInt(args.port)} 195 | }, cb); 196 | }); 197 | 198 | vorpal 199 | .command("weight ") 200 | .types({ 201 | string: ["id"] 202 | }) 203 | .action(function (args, cb) { 204 | client.send("weight", { 205 | id: args.id 206 | }, cb); 207 | }); 208 | 209 | vorpal 210 | .command("weights") 211 | .action(function (args, cb) { 212 | client.send("weights", null, cb); 213 | }); 214 | 215 | vorpal 216 | .command("leave") 217 | .description("Leaves the cluster this node is a part of. " + 218 | "If force is passed, the gossip processor on this node " + 219 | "won't wait for current message streams to be processed " + 220 | "before executing this command.") 221 | .option("-f, --force", forceStr, [false]) 222 | .types({ 223 | boolean: ["force"] 224 | }) 225 | .action(function (args, cb) { 226 | client.send("leave", {force: args.options.force}, cb); 227 | }); 228 | 229 | vorpal 230 | .command("insert ") 231 | .description("Inserts a node into this node's cluster. " + 232 | "If force is passed, the gossip processor on this node " + 233 | "won't wait for current message streams to be processed " + 234 | "before executing this command.") 235 | .option("-f, --force", forceStr) 236 | .option("-w, --weight ", "Number of virtual nodes to assign to the node being inserted. Defaults to the `rfactor` of the session node.") 237 | .types({ 238 | string: ["id", "host", "port", "weight"] 239 | }) 240 | .action(function (args, cb) { 241 | client.send("insert", { 242 | force: args.options.force, 243 | weight: parseInt(args.options.weight), 244 | node: {id: args.id, host: args.host, port: parseInt(args.port)} 245 | }, cb); 246 | }); 247 | 248 | vorpal 249 | .command("minsert [nodes...]") 250 | .description("Inserts multiple nodes into this node's cluster. " + 251 | "If force is passed, the gossip processor on this node " + 252 | "won't wait for current message streams to be processed " + 253 | "before executing this command.") 254 | .option("-f, --force", forceStr) 255 | .option("-w, --weight ", "Number of virtual nodes to assign to the nodes being inserted. Defaults to the `rfactor` of the session node.") 256 | .action(function (args, cb) { 257 | const nodes = parseNodeList(args.nodes); 258 | client.send("minsert", { 259 | force: args.options.force, 260 | weight: parseInt(args.options.weight), 261 | nodes: nodes 262 | }, cb); 263 | }); 264 | 265 | vorpal 266 | .command("update [weight]") 267 | .description("Inserts a node into this node's cluster. " + 268 | "If force is passed, the gossip processor on this node " + 269 | "won't wait for current message streams to be processed " + 270 | "before executing this command.") 271 | .option("-f, --force", forceStr) 272 | .types({ 273 | string: ["id", "host", "port", "weight"] 274 | }) 275 | .action(function (args, cb) { 276 | client.send("update", { 277 | force: args.options.force, 278 | weight: parseInt(args.weight), 279 | node: {id: args.id, host: args.host, port: parseInt(args.port)} 280 | }, cb); 281 | }); 282 | 283 | vorpal 284 | .command("remove ") 285 | .description("Removes a node from this node's cluster. " + 286 | "If force is passed, the gossip processor on this node " + 287 | "won't wait for current message streams to be processed " + 288 | "before executing this command.") 289 | .option("-f, --force", forceStr) 290 | .types({ 291 | string: ["id", "host", "port"] 292 | }) 293 | .action(function (args, cb) { 294 | client.send("remove", { 295 | force: args.options.force, 296 | node: {id: args.id, host: args.host, port: parseInt(args.port)} 297 | }, cb); 298 | }); 299 | 300 | vorpal 301 | .command("mremove [nodes...]") 302 | .description("Removes multiple nodes from this node's cluster. " + 303 | "If force is passed, the gossip processor on this node " + 304 | "won't wait for current message streams to be processed " + 305 | "before executing this command.") 306 | .option("-f, --force", forceStr) 307 | .action(function (args, cb) { 308 | const nodes = parseNodeList(args.nodes); 309 | client.send("mremove", { 310 | force: args.options.force, 311 | nodes: nodes 312 | }, cb); 313 | }); 314 | 315 | vorpal 316 | .command("ping") 317 | .description("Ping a node in the cluster.") 318 | .action(function (args, cb) { 319 | client.send("ping", null, cb); 320 | }); 321 | 322 | if (singular) { 323 | log("Connecting to IPC server on node: %s, host: %s, port: %s", argv.I, argv.H, argv.p); 324 | } 325 | ipc.config.silent = true; 326 | ipc.config.sync = true; 327 | ipc.connectToNet(argv.I, argv.H, argv.p, () => { 328 | client = new Client(ipc, argv.I, argv.H, argv.p, argv.a); 329 | client.start(); 330 | client.once("connect", () => { 331 | if (singular) { 332 | vorpal 333 | .delimiter("> ") 334 | .show(); 335 | } else { 336 | vorpal.exec(argv._.join(" "), function (err, res) { 337 | client.stop(); 338 | if (err) { 339 | process.exit(1); 340 | } else { 341 | process.exit(0); 342 | } 343 | }); 344 | } 345 | }); 346 | }); 347 | -------------------------------------------------------------------------------- /examples/gen_server/dlm/dlm.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | microtime = require("microtime"), 3 | cl = require("../../../index"), 4 | util = require("util"), 5 | debug = require("debug")("notp:examples:dlm"); 6 | 7 | var GenServer = cl.GenServer; 8 | const mcsToMs = 1000; 9 | 10 | class Lock { 11 | constructor(type, id, holder, timeout) { 12 | this._type = type; 13 | this._id = id; 14 | this._holder = holder; 15 | this._timeout = timeout; 16 | } 17 | 18 | type(type) { 19 | if (type !== undefined) { 20 | this._type = type; 21 | } 22 | return this._type; 23 | } 24 | 25 | id(id) { 26 | if (id !== undefined) { 27 | this._id = id; 28 | } 29 | return this._id; 30 | } 31 | 32 | holder(holder) { 33 | if (holder !== undefined) { 34 | this._holder = holder; 35 | } 36 | return this._holder; 37 | } 38 | 39 | timeout(timeout) { 40 | if (timeout !== undefined) { 41 | this._timeout = timeout; 42 | } 43 | return this._timeout; 44 | } 45 | } 46 | 47 | class DLMServer extends GenServer { 48 | /** 49 | * 50 | * @class DLMServer 51 | * @memberof Clusterluck 52 | * 53 | * @param {Clusterluck.GossipRing} gossip 54 | * @param {Clusterluck.NetKernel} kernel 55 | * @param {Object} [opts] 56 | * @param {Number} [opts.rquorum] 57 | * @param {Number} [opts.wquorum] 58 | * 59 | */ 60 | constructor(gossip, kernel, opts = {rquorum: 0.51, wquorum: 0.51}) { 61 | super(kernel); 62 | this._gossip = gossip; 63 | this._kernel = kernel; 64 | this._locks = new Map(); 65 | this._rquorum = opts.rquorum; 66 | this._wquorum = opts.wquorum; 67 | } 68 | 69 | /** 70 | * 71 | * @method start 72 | * @memberof Clusterluck.DLMServer 73 | * @instance 74 | * 75 | * @param {String} [name] 76 | * 77 | * @return {Clusterluck.DLMServer} 78 | * 79 | */ 80 | start(name) { 81 | super.start(name); 82 | 83 | var jobs = [ 84 | {event: "rlock", method: "doRLock"}, 85 | {event: "wlock", method: "doWLock"}, 86 | {event: "runlock", method: "doRUnlock"}, 87 | {event: "wunlock", method: "doWUnlock"} 88 | ]; 89 | jobs.forEach((job) => { 90 | var handler = this[job.method].bind(this); 91 | this.on(job.event, handler); 92 | this.once("stop", _.partial(this.removeListener, job.event, handler).bind(this)); 93 | }); 94 | return this; 95 | } 96 | 97 | /** 98 | * 99 | * @method stop 100 | * @memberof Clusterluck.DLMServer 101 | * @instance 102 | * 103 | * @param {Boolean} [force] 104 | * 105 | * @return {Clusterluck.DLMServer} 106 | * 107 | */ 108 | stop(force = false) { 109 | if (this.idle() || force === true) { 110 | this._locks.clear(); 111 | super.stop(); 112 | return this; 113 | } 114 | this.once("idle", _.partial(this.stop, force).bind(this)); 115 | return this; 116 | } 117 | 118 | /** 119 | * 120 | * @method rlock 121 | * @memberof Clusterluck.DLMServer 122 | * @instance 123 | * 124 | * @param {String} id 125 | * @param {String} holder 126 | * @param {Number} timeout 127 | * @param {Function} cb 128 | * @param {Number} [reqTimeout] 129 | * 130 | */ 131 | rlock(id, holder, timeout, cb, reqTimeout=Infinity) { 132 | var nodes = this._gossip.find(id); 133 | var time = microtime.now(); 134 | this.multicall(nodes, this._id, "rlock", { 135 | id: id, 136 | holder: holder, 137 | timeout: timeout 138 | }, (err, data) => { 139 | if (err) return cb(err); 140 | var delta = (microtime.now()-time)/mcsToMs; 141 | var nData = DLMServer.findLockPasses(nodes, data); 142 | if (nData.length/data.length >= this._rquorum && delta < timeout) { 143 | return cb(null, nData); 144 | } else { 145 | this.runlockAsync(nData, id, holder); 146 | return cb(new Error("Failed to achieve rlock quorum.")); 147 | } 148 | }, reqTimeout); 149 | } 150 | 151 | /** 152 | * 153 | * @method wlock 154 | * @memberof Clusterluck.DLMServer 155 | * @instance 156 | * 157 | * @param {String} id 158 | * @param {String} holderr 159 | * @param {Number} timeout 160 | * @param {Function} cb 161 | * @param {Number} [reqTimeout] 162 | * 163 | */ 164 | wlock(id, holder, timeout, cb, reqTimeout=Infinity) { 165 | var nodes = this._gossip.find(id); 166 | var time = microtime.now(); 167 | this.multicall(nodes, this._id, "wlock", { 168 | id: id, 169 | holder: holder, 170 | timeout: timeout 171 | }, (err, data) => { 172 | if (err) return cb(err); 173 | var delta = (microtime.now()-time)/mcsToMs; 174 | var nData = DLMServer.findLockPasses(nodes, data); 175 | if (nData.length/data.length >= this._wquorum && delta < timeout) { 176 | return cb(null, nData); 177 | } else { 178 | this.wunlockAsync(nData, id, holder); 179 | return cb(new Error("Failed to achieve wlock quorum.")); 180 | } 181 | }, reqTimeout); 182 | } 183 | 184 | /** 185 | * 186 | * @method runlock 187 | * @memberof Clusterluck.DLMServer 188 | * @instance 189 | * 190 | * @param {Array} nodes 191 | * @param {String} id 192 | * @param {String} holder 193 | * @param {Function} cb 194 | * @param {Number} [reqTimeout] 195 | * 196 | */ 197 | runlock(nodes, id, holder, cb, reqTimeout=Infinity) { 198 | this.multicall(nodes, this._id, "runlock", { 199 | id: id, 200 | holder: holder 201 | }, (err, res) => { 202 | if (err) return cb(err); 203 | return cb(); 204 | }, reqTimeout); 205 | } 206 | 207 | /** 208 | * 209 | * @method runlockAsync 210 | * @memberof Clusterluck.DLMServer 211 | * @instance 212 | * 213 | * @param {Array} nodes 214 | * @param {String} id 215 | * @param {String} holder 216 | * 217 | */ 218 | runlockAsync(nodes, id, holder) { 219 | this.abcast(nodes, this._id, "runlock", { 220 | id: id, 221 | holder: holder 222 | }); 223 | } 224 | 225 | /** 226 | * 227 | * @method wunlock 228 | * @memberof Clusterluck.DLMServer 229 | * @instance 230 | * 231 | * @param {Array} nodes 232 | * @param {String} id 233 | * @param {String} holder 234 | * @param {Function} cb 235 | * @param {Number} [reqTimeout] 236 | * 237 | */ 238 | wunlock(nodes, id, holder, cb, reqTimeout=Infinity) { 239 | this.multicall(nodes, this._id, "wunlock", { 240 | id: id, 241 | holder: holder 242 | }, (err, res) => { 243 | if (err) return cb(err); 244 | return cb(); 245 | }, reqTimeout); 246 | } 247 | 248 | /** 249 | * 250 | * @method wunlockAsync 251 | * @memberof Clusterluck.DLMServer 252 | * @instance 253 | * 254 | * @param {Array} nodes 255 | * @param {String} id 256 | * @param {String} holder 257 | * 258 | */ 259 | wunlockAsync(nodes, id, holder) { 260 | this.abcast(nodes, this._id, "wunlock", { 261 | id: id, 262 | holder: holder 263 | }); 264 | } 265 | 266 | /** 267 | * 268 | * @method decodeJob 269 | * @memberof Clusterluck.DLMServer 270 | * @instance 271 | * 272 | * @param {Buffer} buf 273 | * 274 | * @return {Object|Error} 275 | * 276 | */ 277 | decodeJob(buf) { 278 | var out = super.decodeJob(buf); 279 | if (out instanceof Error) return out; 280 | var data = out.data; 281 | if (out.event.endsWith("unlock")) { 282 | data = DLMServer.parseUnlockJob(data); 283 | } else { 284 | data = DLMServer.parseLockJob(data); 285 | } 286 | if (data instanceof Error) { 287 | return data; 288 | } 289 | out.data = data; 290 | return out; 291 | } 292 | 293 | /** 294 | * 295 | * @method doRLock 296 | * @memberof Clusterluck.DLMServer 297 | * @instance 298 | * @private 299 | * 300 | * @param {Object} data 301 | * @param {Object} from 302 | * 303 | */ 304 | doRLock(data, from) { 305 | var lock = this._locks.get(data.id); 306 | if (lock && lock.type() === "write") { 307 | return this.reply(from, DLMServer.encodeResp({ok: false})); 308 | } else if (lock && lock.holder().has(data.holder)) { 309 | return this.reply(from, DLMServer.encodeResp({ok: true})); 310 | } 311 | var timeout = setTimeout(() => { 312 | var lock = this._locks.get(data.id); 313 | if (!lock || lock.type() === "write") { 314 | return; 315 | } 316 | var holder = lock.holder(); 317 | holder.delete(data.holder); 318 | if (holder.size === 0) { 319 | this._locks.delete(data.id); 320 | } 321 | }, data.timeout); 322 | if (!lock) { 323 | lock = new Lock("read", data.id, new Set(), new Map()); 324 | } 325 | lock.holder().add(data.holder); 326 | lock.timeout().set(data.holder, timeout); 327 | this._locks.set(data.id, lock); 328 | this.reply(from, DLMServer.encodeResp({ok: true})); 329 | } 330 | 331 | /** 332 | * 333 | * @method doWLock 334 | * @memberof Clusterluck.DLMServer 335 | * @instance 336 | * @private 337 | * 338 | * @param {Object} data 339 | * @param {Object} from 340 | * 341 | */ 342 | doWLock(data, from) { 343 | if (this._locks.has(data.id)) { 344 | return this.reply(from, DLMServer.encodeResp({ok: false})); 345 | } 346 | var timeout = setTimeout(() => { 347 | var lock = this._locks.get(data.id); 348 | if (!lock || lock.holder() !== data.holder) { 349 | return; 350 | } 351 | this._locks.delete(data.id); 352 | }, data.timeout); 353 | this._locks.set(data.id, new Lock("write", data.id, data.holder, timeout)); 354 | this.reply(from, DLMServer.encodeResp({ok: true})); 355 | } 356 | 357 | /** 358 | * 359 | * @method doRUnlock 360 | * @memberof Clusterluck.DLMServer 361 | * @instance 362 | * @private 363 | * 364 | * @param {Object} data 365 | * @param {Object} from 366 | * 367 | */ 368 | doRUnlock(data, from) { 369 | var lock = this._locks.get(data.id); 370 | if (!lock || lock.type() !== "read") { 371 | return this._safeReply(from, DLMServer.encodeResp({ok: false})); 372 | } 373 | 374 | var holders = lock.holder(); 375 | var timeouts = lock.timeout(); 376 | holders.delete(data.holder); 377 | clearTimeout(timeouts.get(data.holder)); 378 | timeouts.delete(data.holder); 379 | 380 | if (holders.size === 0) { 381 | this._locks.delete(data.id); 382 | } 383 | this._safeReply(from, DLMServer.encodeResp({ok: true})); 384 | } 385 | 386 | /** 387 | * 388 | * @method doWUnlock 389 | * @memberof Clusterluck.DLMServer 390 | * @instance 391 | * @private 392 | * 393 | * @param {Object} data 394 | * @param {Object} from 395 | * 396 | */ 397 | doWUnlock(data, from) { 398 | var lock = this._locks.get(data.id); 399 | if (!lock || lock.type() !== "write") { 400 | return this._safeReply(from, DLMServer.encodeResp({ok: false})); 401 | } 402 | 403 | var holder = lock.holder(); 404 | if (holder === data.holder) { 405 | clearTimeout(lock.timeout()); 406 | this._locks.delete(data.id); 407 | } 408 | this._safeReply(from, DLMServer.encodeResp({ok: true})); 409 | } 410 | 411 | /** 412 | * 413 | * @method parseLockJob 414 | * @memberof Clusterluck.DLMServer 415 | * @static 416 | * 417 | * @param {Object} job 418 | * 419 | * @return {Object|Error} 420 | * 421 | */ 422 | static parseLockJob(job) { 423 | if (!(util.isObject(job) && 424 | util.isString(job.id) && 425 | util.isString(job.holder) && 426 | util.isNumber(job.timeout))) { 427 | return new Error("Malformed lock job."); 428 | } 429 | return job; 430 | } 431 | 432 | /** 433 | * 434 | * @method parseUnlockJob 435 | * @memberof Clusterluck.DLMServer 436 | * @static 437 | * 438 | * @param {Object} job 439 | * 440 | * @return {Object|Error} 441 | * 442 | */ 443 | static parseUnlockJob(job) { 444 | if (!(util.isObject(job) && 445 | util.isString(job.id) && 446 | util.isString(job.holder))) { 447 | return new Error("Malformed unlock job."); 448 | } 449 | return job; 450 | } 451 | 452 | /** 453 | * 454 | * @method encodeResp 455 | * @memberof Clusterluck.DLMServer 456 | * @static 457 | * 458 | * @param {Any} res 459 | * 460 | * @return {String} 461 | * 462 | */ 463 | static encodeResp(res) { 464 | return JSON.stringify(res); 465 | } 466 | 467 | /** 468 | * 469 | * @method findLockPasses 470 | * @memberof Clusterluck.DLMServer 471 | * @static 472 | * 473 | * @param {Array} nodes 474 | * @param {Array} data 475 | * 476 | * @return {Array} 477 | * 478 | */ 479 | static findLockPasses(nodes, data) { 480 | return data.reduce((memo, val, idx) => { 481 | val = JSON.parse(val); 482 | if (util.isObject(val) && val.ok === true) { 483 | memo.push(nodes[idx]); 484 | } 485 | return memo; 486 | }, []); 487 | } 488 | } 489 | 490 | module.exports = DLMServer; 491 | -------------------------------------------------------------------------------- /lib/command_server.js: -------------------------------------------------------------------------------- 1 | const _ = require("lodash"), 2 | debug = require("debug")("notp:lib:command_server"); 3 | 4 | const GenServer = require("./gen_server"), 5 | NetKernel = require("./kernel"), 6 | Node = require("./node"), 7 | utils = require("./utils"); 8 | 9 | const commands = { 10 | "join": utils.hasID, 11 | "leave": _.identity, 12 | "meet": utils.parseNode, 13 | "insert": utils.parseNode, 14 | "minsert": utils.parseNodeList, 15 | "update": utils.parseNode, 16 | "remove": utils.parseNode, 17 | "mremove": utils.parseNodeList, 18 | "inspect": _.identity, 19 | "nodes": _.identity, 20 | "has": utils.hasID, 21 | "get": utils.hasID, 22 | "weight": utils.hasID, 23 | "weights": _.identity, 24 | "ping": _.identity 25 | }; 26 | 27 | class CommandServer extends GenServer { 28 | /** 29 | * 30 | * @class CommandServer 31 | * @memberof Clusterluck 32 | * 33 | * @param {Clusterluck.GossipRing} gossip 34 | * @param {Clusterluck.NetKernel} kernel 35 | * 36 | */ 37 | constructor(gossip, kernel) { 38 | super(kernel); 39 | this._gossip = gossip; 40 | this._kernel = kernel; 41 | } 42 | 43 | /** 44 | * 45 | * @method start 46 | * @memberof Clusterluck.CommandServer 47 | * @instance 48 | * 49 | * @param {String} [name] 50 | * 51 | * @return {Clusterluck.CommandServer} 52 | * 53 | */ 54 | start(name) { 55 | super.start(name); 56 | Object.keys(commands).forEach((key) => { 57 | const handle = this[key].bind(this); 58 | this.on(key, handle); 59 | this.once("stop", _.partial(this.removeListener, key, handle).bind(this)); 60 | }); 61 | return this; 62 | } 63 | 64 | /** 65 | * 66 | * @method stop 67 | * @memberof Clusterluck.CommandServer 68 | * @instance 69 | * 70 | * @param {Boolean} [force] 71 | * 72 | * @return {Clusterluck.CommandServer} 73 | * 74 | */ 75 | stop(force = false) { 76 | if (this.idle() || force === true) { 77 | super.stop(); 78 | return this; 79 | } 80 | this.once("idle", _.partial(this.stop, force).bind(this)); 81 | return this; 82 | } 83 | 84 | /** 85 | * 86 | * @method decodeJob 87 | * @memberof Clusterluck.CommandServer 88 | * @instance 89 | * 90 | * @param {Buffer} buf 91 | * 92 | * @return {Object|Error} 93 | * 94 | */ 95 | decodeJob(buf) { 96 | const out = super.decodeJob(buf); 97 | if (out instanceof Error) return out; 98 | let data = out.data; 99 | if (commands[out.event] === undefined) { 100 | return new Error("Cannot run against unknown command '" + out.event + "'"); 101 | } 102 | const parser = commands[out.event]; 103 | data = parser(data); 104 | if (data instanceof Error) { 105 | return data; 106 | } 107 | out.data = data; 108 | return out; 109 | } 110 | 111 | /** 112 | * 113 | * @method decodeSingleton 114 | * @memberof Clusterluck.CommandServer 115 | * @instance 116 | * 117 | * @param {Object} data 118 | * 119 | * @return {Object|Error} 120 | * 121 | */ 122 | decodeSingleton(data) { 123 | const out = super.decodeSingleton(data); 124 | if (commands[out.event] === undefined) { 125 | return new Error("Cannot run against unknown command '" + data.event + "'"); 126 | } 127 | const parser = commands[out.event]; 128 | data = parser(out.data); 129 | if (data instanceof Error) return data; 130 | out.data = data; 131 | return out; 132 | } 133 | 134 | /** 135 | * 136 | * @method join 137 | * @memberof Clusterluck.CommandServer 138 | * @instance 139 | * 140 | * @param {Object} data 141 | * @param {String} data.id 142 | * @param {Object} from 143 | * 144 | * @return {Clusterluck.CommandServer} 145 | * 146 | */ 147 | join(data, from) { 148 | const out = this._gossip.join(data.id); 149 | if (out instanceof Error) { 150 | return this._encodedReply(from, { 151 | ok: false, 152 | error: utils.errorToObject(out) 153 | }); 154 | } 155 | return this._encodedReply(from, {ok: true}); 156 | } 157 | 158 | /** 159 | * 160 | * @method meet 161 | * @memberof Clusterluck.CommandServer 162 | * @instance 163 | * 164 | * @param {Object} data 165 | * @param {Clusterluck.Node} data.node 166 | * @param {Object} from 167 | * 168 | * @return {Clusterluck.CommandServer} 169 | * 170 | */ 171 | meet(data, from) { 172 | this._gossip.meet(data.node); 173 | return this._encodedReply(from, {ok: true}); 174 | } 175 | 176 | /** 177 | * 178 | * @method leave 179 | * @memberof Clusterluck.CommandServer 180 | * @instance 181 | * 182 | * @param {Object} data 183 | * @param {Boolean} data.force 184 | * @param {Object} from 185 | * 186 | * @return {Clusterluck.CommandServer} 187 | * 188 | */ 189 | leave(data, from) { 190 | data.force = data.force === true ? true : false; 191 | this._gossip.leave(data.force); 192 | return this._encodedReply(from, {ok: true}); 193 | } 194 | 195 | /** 196 | * 197 | * @method insert 198 | * @memberof Clusterluck.CommandServer 199 | * @instance 200 | * 201 | * @param {Object} data 202 | * @param {Clusterluck.Node} data.node 203 | * @param {Boolean} data.force 204 | * @param {Object} from 205 | * 206 | * @return {Clusterluck.CommandServer} 207 | * 208 | */ 209 | insert(data, from) { 210 | const force = data.force === true ? true : false; 211 | const weight = typeof data.weight === "number" && data.weight > 0 ? data.weight : this._gossip.ring().rfactor(); 212 | this._gossip.insert(data.node, weight, force); 213 | return this._encodedReply(from, {ok: true}); 214 | } 215 | 216 | /** 217 | * 218 | * @method minsert 219 | * @memberof Clusterluck.CommandServer 220 | * @instance 221 | * 222 | * @param {Object} data 223 | * @param {Array} data.nodes 224 | * @param {Boolean} data.force 225 | * @param {Object} from 226 | * 227 | * @return {Clusterluck.CommandServer} 228 | * 229 | */ 230 | minsert(data, from) { 231 | const force = data.force === true ? true : false; 232 | const weight = typeof data.weight === "number" && data.weight > 0 ? data.weight : this._gossip.ring().rfactor(); 233 | let nodes = data.nodes; 234 | nodes = nodes.filter((node) => {return node.id() !== this._kernel.self().id();}); 235 | this._gossip.minsert(nodes, weight, force); 236 | return this._encodedReply(from, {ok: true}); 237 | } 238 | 239 | /** 240 | * 241 | * @method update 242 | * @memberof Clusterluck.CommandServer 243 | * @instance 244 | * 245 | * @param {Object} data 246 | * @param {Clusterluck.Node} data.node 247 | * @param {Boolean} data.force 248 | * @param {Number} data.weight 249 | * @param {Object} from 250 | * 251 | * @return {Clusterluck.CommandServer} 252 | * 253 | */ 254 | update(data, from) { 255 | const force = data.force === true ? true : false; 256 | const weight = typeof data.weight === "number" && data.weight > 0 ? data.weight : this._gossip.ring().rfactor(); 257 | this._gossip.update(data.node, weight, force); 258 | return this._encodedReply(from, {ok: true}); 259 | } 260 | 261 | /** 262 | * 263 | * @method remove 264 | * @memberof Clusterluck.CommandServer 265 | * @instance 266 | * 267 | * @param {Object} data 268 | * @param {Clusterluck.Node} data.node 269 | * @param {Boolean} data.force 270 | * @param {Object} from 271 | * 272 | * @return {Clusterluck.CommandServer} 273 | * 274 | */ 275 | remove(data, from) { 276 | const force = data.force === true ? true : false; 277 | this._gossip.remove(data.node, force); 278 | return this._encodedReply(from, {ok: true}); 279 | } 280 | 281 | /** 282 | * 283 | * @method mremove 284 | * @memberof Clusterluck.CommandServer 285 | * @instance 286 | * 287 | * @param {Object} data 288 | * @param {Array} data.nodes 289 | * @param {Boolean} data.force 290 | * @param {Object} from 291 | * 292 | * @return {Clusterluck.CommandServer} 293 | * 294 | */ 295 | mremove(data, from) { 296 | const force = data.force === true ? true : false; 297 | let nodes = data.nodes; 298 | nodes = nodes.filter((node) => {return node.id() !== this._kernel.self().id();}); 299 | this._gossip.mremove(nodes, force); 300 | return this._encodedReply(from, {ok: true}); 301 | } 302 | 303 | /** 304 | * 305 | * @method inspect 306 | * @memberof Clusterluck.CommandServer 307 | * @instance 308 | * 309 | * @param {Any} data 310 | * @param {Object} from 311 | * 312 | * @return {Clusterluck.CommandServer} 313 | * 314 | */ 315 | inspect(data, from) { 316 | const ring = this._gossip.ring(); 317 | return this._encodedReply(from, { 318 | ok: true, 319 | data: ring.toJSON(true) 320 | }); 321 | } 322 | 323 | /** 324 | * 325 | * @method nodes 326 | * @memberof Clusterluck.CommandServer 327 | * @instance 328 | * 329 | * @param {Any} data 330 | * @param {Object} from 331 | * 332 | * @return {Clusterluck.CommandServer} 333 | * 334 | */ 335 | nodes(data, from) { 336 | const nodes = this._gossip.ring().nodes(); 337 | return this._encodedReply(from, { 338 | ok: true, 339 | data: nodes 340 | }); 341 | } 342 | 343 | /** 344 | * 345 | * @method has 346 | * @memberof Clusterluck.CommandServer 347 | * @instance 348 | * 349 | * @param {Object} data 350 | * @param {String} data.id 351 | * @param {Object} from 352 | * 353 | * @return {Clusterluck.CommandServer} 354 | * 355 | */ 356 | has(data, from) { 357 | const ring = this._gossip.ring(); 358 | const node = new Node(data.id); 359 | return this._encodedReply(from, { 360 | ok: true, 361 | data: ring.isDefined(node) 362 | }); 363 | } 364 | 365 | /** 366 | * 367 | * @method get 368 | * @memberof Clusterluck.CommandServer 369 | * @instance 370 | * 371 | * @param {Object} data 372 | * @param {String} data.id 373 | * @param {Object} from 374 | * 375 | * @return {Clusterluck.CommandServer} 376 | * 377 | */ 378 | get(data, from) { 379 | const ring = this._gossip.ring(); 380 | const node = ring.get(new Node(data.id)); 381 | if (node === undefined) { 382 | const msg = "'" + data.id + "' is not defined in this ring."; 383 | return this._encodedReply(from, { 384 | ok: false, 385 | error: utils.errorToObject(new Error(msg)) 386 | }); 387 | } 388 | return this._encodedReply(from, { 389 | ok: true, 390 | data: node.toJSON(true) 391 | }); 392 | } 393 | 394 | /** 395 | * 396 | * @method weight 397 | * @memberof Clusterluck.CommandServer 398 | * @instance 399 | * 400 | * @param {Object} data 401 | * @param {String} data.id 402 | * @param {Object} from 403 | * 404 | * @return {Clsuterluck.CommandServer} 405 | * 406 | */ 407 | weight(data, from) { 408 | const ring = this._gossip.ring(); 409 | const weight = ring.weights().get(data.id); 410 | if (weight === undefined) { 411 | const msg = "'" + data.id + "' is not defined in this ring."; 412 | return this._encodedReply(from, { 413 | ok: false, 414 | error: utils.errorToObject(new Error(msg)) 415 | }); 416 | } 417 | return this._encodedReply(from, { 418 | ok: true, 419 | data: weight 420 | }); 421 | } 422 | 423 | /** 424 | * 425 | * @method weights 426 | * @memberof Clusterluck.CommandServer 427 | * @instance 428 | * 429 | * @param {Any} data 430 | * @param {Object} from 431 | * 432 | * @return {Clsuterluck.CommandServer} 433 | * 434 | */ 435 | weights(data, from) { 436 | const ring = this._gossip.ring(); 437 | const weights = ring.weights(); 438 | return this._encodedReply(from, { 439 | ok: true, 440 | data: utils.mapToObject(weights) 441 | }); 442 | } 443 | 444 | /** 445 | * 446 | * @method ping 447 | * @memberof Clusterluck.CommandServer 448 | * @instance 449 | * 450 | * @param {Any} data 451 | * @param {Object} from 452 | * 453 | * @return {Clusterluck.CommandServer} 454 | * 455 | */ 456 | ping(data, from) { 457 | return this._encodedReply(from, { 458 | ok: true, 459 | data: "pong" 460 | }); 461 | } 462 | 463 | /** 464 | * 465 | * @method _encodedReply 466 | * @memberof Clusterluck.CommandServer 467 | * @private 468 | * @instance 469 | * 470 | * @param {Object} from 471 | * @param {Object} msg 472 | * 473 | * @return {Clusterluck.CommandServer} 474 | * 475 | */ 476 | _encodedReply(from, msg) { 477 | const sendMsg = _.extend(NetKernel._encodeMsg(this._kernel.cookie(), _.extend(msg, { 478 | tag: from.tag 479 | }))); 480 | this._kernel.ipc().server.emit(from.socket, "message", sendMsg); 481 | return this; 482 | } 483 | 484 | /** 485 | * 486 | * @method _parse 487 | * @memberof Clusterluck.CommandServer 488 | * @private 489 | * @instance 490 | * 491 | * @param {Object} data 492 | * @param {Object} stream 493 | * @param {Object} from 494 | * 495 | * @return {Clusterluck.CommandServer} 496 | * 497 | */ 498 | _parse(data, stream, from) { 499 | if (this._kernel.sinks().has(from.node.id()) || 500 | from.node.id() === this._kernel.self().id()) return this; 501 | return super._parse(data, stream, from); 502 | } 503 | } 504 | 505 | module.exports = CommandServer; 506 | -------------------------------------------------------------------------------- /lib/vclock.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | microtime = require("microtime"), 4 | assert = require("assert"), 5 | debug = require("debug")("notp:lib:vclock"); 6 | 7 | var utils = require("./utils"); 8 | 9 | class VectorClock { 10 | /** 11 | * 12 | * Vector clock implementation. Maintains a map from node IDs to clock information, containing an updated UNIX timestamp and an atomic counter (node-side). 13 | * 14 | * @class VectorClock VectorClock 15 | * @memberof Clusterluck 16 | * 17 | * @param {String} [node] - Node to start vector clock with. 18 | * @param {Number} [count] - Count of `node` to initialize with. 19 | * 20 | */ 21 | constructor(node, count) { 22 | this._vector = {}; 23 | this._size = 0; 24 | if (typeof node === "string" && typeof count === "number") { 25 | var time = microtime.now(); 26 | this._vector[node] = { 27 | count: count, 28 | insert: time, 29 | time: time 30 | }; 31 | this._size = 1; 32 | } 33 | } 34 | 35 | /** 36 | * 37 | * Inserts `node` into this vector clock. Initializes node state with a 0 counter and the current UNIX timestamp on this node. 38 | * 39 | * @method insert 40 | * @memberof Clusterluck.VectorClock 41 | * @instance 42 | * 43 | * @param {String} node - Node to insert into this instance. 44 | * 45 | * @return {Clusterluck.VectorClock} This instance. 46 | * 47 | */ 48 | insert(node) { 49 | if (this._vector[node]) return this; 50 | this._size++; 51 | var time = microtime.now(); 52 | this._vector[node] = { 53 | count: 0, 54 | insert: time, 55 | time: time 56 | }; 57 | return this; 58 | } 59 | 60 | /** 61 | * 62 | * Updates the state of `node` in this vector clock with a counter and an optional UNIX timestamp. If `time` isn't provided, uses current UNIX timestamp. 63 | * 64 | * @method update 65 | * @memberof Clusterluck.VectorClock 66 | * @instance 67 | * 68 | * @param {String} node - Node to insert into this vector clock. 69 | * @param {Number} count - Count to update node state with. 70 | * @param {Number} [time] - UNIX timestamp to update node state with. 71 | * 72 | * @return {Clusterluck.VectorClock} This instance. 73 | * 74 | */ 75 | update(node, count, time) { 76 | if (!this._vector[node]) this._size++; 77 | var inTime = microtime.now(); 78 | var res = this._vector[node] || {count: 0, insert: inTime}; 79 | res.count = count; 80 | res.time = time === undefined ? inTime : time; 81 | this._vector[node] = res; 82 | return this; 83 | } 84 | 85 | /** 86 | * 87 | * Removes `node` from this vector clock. 88 | * 89 | * @method remove 90 | * @memberof Clusterluck.VectorClock 91 | * @instance 92 | * 93 | * @param {String} node - Node to remove from this vector clock. 94 | * 95 | * @return {Clusterluck.VectorClock} This instance. 96 | * 97 | */ 98 | remove(node) { 99 | if (this._vector[node]) this._size--; 100 | delete this._vector[node]; 101 | return this; 102 | } 103 | 104 | /** 105 | * 106 | * Increments the counter stored at `node` in this vector clock. If `node` doesn't exist, it's inserted and incremented. The timestamp stored at `node` is updated to the current UNIX timestamp. 107 | * 108 | * @method increment 109 | * @memberof Clusterluck.VectorClock 110 | * @instance 111 | * 112 | * @param {String} node - Node to increment counter of. 113 | * 114 | * @return {Clusterluck.VectorClock} This instance. 115 | * 116 | */ 117 | increment(node) { 118 | if (!this._vector[node]) this._size++; 119 | var time = microtime.now(); 120 | var res = this._vector[node] || {count: 0, insert: time}; 121 | res.count++; 122 | res.time = time; 123 | this._vector[node] = res; 124 | return this; 125 | } 126 | 127 | /** 128 | * 129 | * Get state from this vector clock at `node`. 130 | * 131 | * @method get 132 | * @memberof Clusterluck.VectorClock 133 | * @instance 134 | * 135 | * @param {String} node - Node to return state of. 136 | * 137 | * @return {Object} State of `node`. 138 | * 139 | */ 140 | get(node) { 141 | return this._vector[node]; 142 | } 143 | 144 | /** 145 | * 146 | * Gets the count stored `node` in this vector clock. 147 | * 148 | * @method getCount 149 | * @memberof Clusterluck.VectorClock 150 | * @instance 151 | * 152 | * @param {String} node - Node to return count of. 153 | * 154 | * @return {Number} Count of `node`. 155 | * 156 | */ 157 | getCount(node) { 158 | var res = this._vector[node]; 159 | return res !== undefined ? res.count : undefined; 160 | } 161 | 162 | /** 163 | * 164 | * Gets the microsecond UNIX insert timestamp stored at `node` in this vector clock. 165 | * 166 | * @method getInsert 167 | * @memberof Clusterluck.VectorClock 168 | * @instance 169 | * 170 | * @param {String} node - Node to return insert time of. 171 | * 172 | * @return {Number} Microsecond UNIX timestamp of `node`. 173 | * 174 | */ 175 | getInsert(node) { 176 | var res = this._vector[node]; 177 | return res !== undefined ? res.insert : undefined; 178 | } 179 | 180 | /** 181 | * 182 | * Gets the UNIX timestamp stored at `node` in this vector clock. 183 | * 184 | * @method getTimestamp 185 | * @memberof Clusterluck.VectorClock 186 | * @instance 187 | * 188 | * @param {String} node - Node to return timestamp of. 189 | * 190 | * @return {Number} UNIX timestamp of `node`. 191 | * 192 | */ 193 | getTimestamp(node) { 194 | var res = this._vector[node]; 195 | return res !== undefined ? res.time : undefined; 196 | } 197 | 198 | /** 199 | * 200 | * Returns whether `node` exists in this vector clock. 201 | * 202 | * @method has 203 | * @memberof Clusterluck.VectorClock 204 | * @instance 205 | * 206 | * @param {String} node - Node to check existence of. 207 | * 208 | * @return {Boolean} Whether `node` has an entry in this vector clock. 209 | * 210 | */ 211 | has(node) { 212 | return !!this._vector[node]; 213 | } 214 | 215 | /** 216 | * 217 | * Merges two vector clocks, using the following policy: 218 | * - If `entry` exists in v1 but not in v2, entry in v1 doesn't change 219 | * - If `entry` exists in v2 but not in v1, entry is added into v1 as is 220 | * - If `entry` exists in both v1 and v2 but the count at v2 is smaller than v1, the state of v1 is kept 221 | * - If `entry` exists in both v1 and v2 but the count at v1 is smaller than v2, the state of v2 is used 222 | * - If `entry` exists in both v1 and v2 and the counts are the same, prioritize based on UNIX timestamp 223 | * 224 | * @method merge 225 | * @memberof Clusterluck.VectorClock 226 | * @instance 227 | * 228 | * @param {Clusterluck.VectorClock} vclock - Vector clock to merge this instance with. 229 | * 230 | * @return {Clusterluck.VectorClock} This instance. 231 | * 232 | */ 233 | merge(vclock) { 234 | if (vclock.size() === 0) return this; 235 | if (this.size() === 0) { 236 | this._vector = _.cloneDeep(vclock._vector); 237 | this._size = vclock.size(); 238 | return this; 239 | } 240 | _.extendWith(this._vector, vclock._vector, (val, other) => { 241 | if (val === undefined) return _.clone(other); 242 | if (other === undefined) return val; 243 | 244 | if (val.count < other.count) { 245 | return _.clone(other); 246 | } 247 | if (val.count > other.count) { 248 | return val; 249 | } 250 | var max = Math.max(val.time, other.time); 251 | return max === val.time ? val : _.clone(other); 252 | }); 253 | this._size = Object.keys(this._vector).length; 254 | return this; 255 | } 256 | 257 | /** 258 | * 259 | * Returns whether this vector clocks descends from `vclock`. 260 | * 261 | * @method descends 262 | * @memberof Clusterluck.VectorClock 263 | * @instance 264 | * 265 | * @param {Clusterluck.VectorClock} vclock - Vector clock to check if this instance descends from. 266 | * 267 | * @return {Boolean} Whether this vector clock descends from `vclock`. 268 | * 269 | */ 270 | descends(vclock) { 271 | // if input is trivial, this vector clock descends from it 272 | if (vclock.size() === 0) return true; 273 | // if this instance is trivial, it doesn't descend from input 274 | if (this.size() === 0) return false; 275 | 276 | return _.every(vclock._vector, (val, key) => { 277 | if (!this._vector[key]) return false; 278 | var node = this._vector[key]; 279 | return node.count >= val.count; 280 | }); 281 | } 282 | 283 | /** 284 | * 285 | * Returns whether this vector clock equals `vclock`. 286 | * 287 | * @method equals 288 | * @memberof Clusterluck.VectorClock 289 | * @instance 290 | * 291 | * @param {Clusterluck.VectorClock} vclock - Vector clock to check equality with. 292 | * 293 | * @return {Boolean} Whether this vector clock equals `vclock`. 294 | * 295 | */ 296 | equals(vclock) { 297 | if (this._size !== vclock.size()) return false; 298 | return _.every(this._vector, (val, key) => { 299 | if (!vclock._vector[key]) return false; 300 | var node = vclock._vector[key]; 301 | return node.count === val.count && 302 | node.insert === val.insert && 303 | node.time === val.time; 304 | }); 305 | } 306 | 307 | /** 308 | * 309 | * Trims this vector clock. Requires that the number of entries in this clock be greater than `lowerBound`, and the oldest entry be older than `threshold`-`youngBound`. Any entries older than `oldBound` will be trimmed, as well as any entries above the `upperBound` limit. 310 | * 311 | * @method trim 312 | * @memberof Clusterluck.VectorClock 313 | * @instance 314 | * 315 | * @param {Number} threshold - UNIX timestamp to check clock entries against. 316 | * @param {Object} opts - Object containing trim parameters. 317 | * @param {Number} opts.lowerBound - Number of entries required from trimming to occur. 318 | * @param {Number} opts.youngBound - Minimum difference between oldest entry and `threshold` for trimming to occur. 319 | * @param {Number} opts.upperBound - Maximum number of entries this vector clock can contain. 320 | * @param {Number} opts.oldBound - Maximum difference between any entry and `threshold`. 321 | * 322 | * @return {Clusterluck.VectorClock} This instance. 323 | * 324 | */ 325 | trim(threshold, opts) { 326 | opts = opts || {}; 327 | if (this.size() <= opts.lowerBound) { 328 | return this; 329 | } 330 | var keys = Object.keys(this._vector).sort((a, b) => { 331 | var objA = this._vector[a].time; 332 | var objB = this._vector[b].time; 333 | return objA > objB ? 1 : (objA === objB ? 0 : -1); 334 | }); 335 | var val = this._vector[keys[0]]; 336 | if (threshold - val.time > opts.youngBound) { 337 | this._trimClock(keys.reverse(), threshold, opts); 338 | this._size = Object.keys(this._vector).length; 339 | } 340 | return this; 341 | } 342 | 343 | /** 344 | * 345 | * Returns the nodes present in this vector clock. 346 | * 347 | * @method nodes 348 | * @memberof Clusterluck.VectorClock 349 | * @instance 350 | * 351 | * @return {Array} Node IDs in this vector clock. 352 | * 353 | */ 354 | nodes() { 355 | return Object.keys(this._vector); 356 | } 357 | 358 | /** 359 | * 360 | * Returns the number of entries in this vector clock. 361 | * 362 | * @method size 363 | * @memberof Clusterluck.VectorClock 364 | * @instance 365 | * 366 | * @return {Number} Number of entries in this vector clock. 367 | * 368 | */ 369 | size() { 370 | return this._size; 371 | } 372 | 373 | /** 374 | * 375 | * Serializes this instance into a JSON object. 376 | * 377 | * @method toJSON 378 | * @memberof Netkernel.VectorClock 379 | * @instance 380 | * 381 | * @param {Boolean} [fast] - Whether to compute the JSON serialization of this isntance quickly or safely. 382 | * 383 | * @return {Object} JSON serialization of this instance. 384 | * 385 | */ 386 | toJSON(fast) { 387 | return fast === true ? this._vector : _.cloneDeep(this._vector); 388 | } 389 | 390 | /** 391 | * 392 | * Populates data for this instance from JSON object `ent`. 393 | * 394 | * @method fromJSON 395 | * @memberof Netkernel.VectorClock 396 | * @instance 397 | * 398 | * @param {Object} ent - JSON object to instantiate state from. 399 | * 400 | * @return {Clusterluck.VectorClock} This instance. 401 | * 402 | */ 403 | fromJSON(ent) { 404 | assert.ok(VectorClock.validJSON(ent)); 405 | this._vector = ent; 406 | this._size = Object.keys(ent).length; 407 | return this; 408 | } 409 | 410 | /** 411 | * 412 | * @method _trimClock 413 | * @memberof Clusterluck.VectorClock 414 | * @private 415 | * @instance 416 | * 417 | */ 418 | _trimClock(keys, threshold, opts) { 419 | if (keys.length === 0) return; 420 | var val = this._vector[_.last(keys)]; 421 | if (keys.length > opts.upperBound || 422 | threshold - val.time > opts.oldBound) { 423 | delete this._vector[_.last(keys)]; 424 | keys.pop(); 425 | return this._trimClock(keys, threshold, opts); 426 | } 427 | else return; 428 | } 429 | 430 | static validJSON(data) { 431 | return utils.isPlainObject(data); 432 | } 433 | } 434 | 435 | module.exports = VectorClock; 436 | -------------------------------------------------------------------------------- /test/unit/gen_server.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"), 2 | async = require("async"), 3 | uuid = require("uuid"), 4 | stream = require("stream"), 5 | sinon = require("sinon"), 6 | fs = require("fs"), 7 | microtime = require("microtime"), 8 | assert = require("chai").assert; 9 | 10 | module.exports = function (mocks, lib) { 11 | describe("GenServer unit tests", function () { 12 | var GenServer = lib.gen_server, 13 | NetKernel = lib.kernel, 14 | Node = lib.node, 15 | MockIPC = mocks.ipc; 16 | 17 | describe("GenServer state tests", function () { 18 | var kernel, 19 | server, 20 | opts, 21 | id = "id", 22 | host = "localhost", 23 | port = 8000; 24 | 25 | before(function () { 26 | kernel = new NetKernel(new MockIPC(), id, host, port); 27 | }); 28 | 29 | beforeEach(function () { 30 | server = new GenServer(kernel); 31 | }); 32 | 33 | it("Should construct a generic server", function () { 34 | assert.deepEqual(server._kernel, kernel); 35 | assert(server._streams instanceof Map); 36 | assert.equal(server._streams.size, 0); 37 | assert.equal(server._streamTimeout, 30000); 38 | 39 | server = new GenServer(kernel, {streamTimeout: 5000}); 40 | assert.equal(server._streamTimeout, 5000); 41 | }); 42 | 43 | it("Should grab id of generic server", function () { 44 | var id = server._id; 45 | assert.deepEqual(server.id(), id); 46 | }); 47 | 48 | it("Should set id of generic server", function () { 49 | server.id("foo"); 50 | assert.deepEqual(server.id(), "foo"); 51 | }); 52 | 53 | it("Should grab kernel of gossip ring", function () { 54 | assert.deepEqual(server.kernel(), kernel); 55 | }); 56 | 57 | it("Should set kernel of gossip ring", function () { 58 | var kernel2 = new NetKernel(new MockIPC(), "id2", host, port+1); 59 | server.kernel(kernel2); 60 | assert.deepEqual(server._kernel, kernel2); 61 | }); 62 | 63 | it("Should grab active streams of gossip ring", function () { 64 | assert(server.streams() instanceof Map); 65 | assert.equal(server.streams().size, 0); 66 | }); 67 | 68 | it("Should set active streams of gossip ring", function () { 69 | var streams = new Map([["key", "value"]]); 70 | server.streams(streams); 71 | assert(server.streams() instanceof Map); 72 | assert.equal(server.streams().size, 1); 73 | }); 74 | 75 | it("Should return if gossip process is idle or not", function () { 76 | assert.ok(server.idle()); 77 | var streams = new Map([["key", "value"]]); 78 | server.streams(streams); 79 | assert.notOk(server.idle()); 80 | }); 81 | 82 | it("Should fail to start generic server with existent name", function () { 83 | var name = "foo"; 84 | var handler = _.identity; 85 | server.kernel().once(name, handler); 86 | var out; 87 | try { 88 | out = server.start(name); 89 | } catch (e) { 90 | out = e; 91 | } 92 | assert.ok(out instanceof Error); 93 | server.kernel().removeListener(name, handler); 94 | }); 95 | 96 | it("Should start generic server with name", function () { 97 | var name = "foo"; 98 | server.start(name); 99 | assert.equal(server._id, name); 100 | assert.lengthOf(server.kernel().listeners(name), 1); 101 | assert.lengthOf(server.listeners("pause"), 1); 102 | server.stop(); 103 | }); 104 | 105 | it("Should start generic server w/o name", function () { 106 | var id = server.id(); 107 | server.start(); 108 | assert.equal(server._id, id); 109 | assert.lengthOf(server.kernel().listeners(id), 1); 110 | assert.lengthOf(server.listeners("pause"), 1); 111 | server.stop(); 112 | }); 113 | 114 | it("Should stop generic server", function (done) { 115 | var name = "foo"; 116 | server.start(name); 117 | server.once("stop", () => { 118 | assert.equal(server._streams.size, 0); 119 | assert.lengthOf(server.listeners("pause"), 0); 120 | assert.lengthOf(server.kernel().listeners(name), 0); 121 | done(); 122 | }); 123 | server.stop(); 124 | }); 125 | 126 | it("Should forcefully stop generic server", function (done) { 127 | var name = "foo"; 128 | server.start(name); 129 | server.once("stop", () => { 130 | assert.equal(server._streams.size, 0); 131 | assert.lengthOf(server.listeners("pause"), 0); 132 | assert.lengthOf(server.kernel().listeners(name), 0); 133 | done(); 134 | }); 135 | server.stop(true); 136 | }); 137 | 138 | it("Should decode job", function () { 139 | var job = Buffer.from(JSON.stringify({event: "foo", data: "bar", hello: "world"})); 140 | var out = server.decodeJob(job); 141 | assert.deepEqual(out, {event: "foo", data: "bar"}); 142 | 143 | job = Buffer.from(JSON.stringify({event: "foo", data: Buffer.from("bar"), hello: "world"})); 144 | out = server.decodeJob(job); 145 | assert.deepEqual(out, {event: "foo", data: Buffer.from("bar")}); 146 | 147 | out = server.decodeJob(Buffer.from("foo")); 148 | assert.ok(out instanceof Error); 149 | }); 150 | 151 | it("Should decode singleton", function () { 152 | var job = "foo"; 153 | var out = server.decodeSingleton(job); 154 | assert.equal(job, out); 155 | 156 | job = {event: "foo", data: "bar"}; 157 | out = server.decodeSingleton(job); 158 | assert.deepEqual(job, out); 159 | 160 | job = {event: "foo", data: {}}; 161 | out = server.decodeSingleton(job); 162 | assert.deepEqual(job, out); 163 | 164 | job = {event: "foo", data: Buffer.from("bar").toJSON()}; 165 | out = server.decodeSingleton(job); 166 | assert.deepEqual(out, {event: "foo", data: Buffer.from("bar")}); 167 | }); 168 | }); 169 | 170 | describe("GenServer operation tests", function () { 171 | var kernel, 172 | nKernel, 173 | server, 174 | opts, 175 | id = "id", 176 | host = "localhost", 177 | port = 8000; 178 | 179 | before(function (done) { 180 | kernel = new NetKernel(new MockIPC(), id, host, port); 181 | nKernel = new NetKernel(new MockIPC(), "id2", host, port+1); 182 | async.each([kernel, nKernel], (k, next) => { 183 | k.start({retry: 500, maxRetries: false}); 184 | k.once("_ready", next); 185 | }, () => { 186 | kernel.connect(nKernel.self()); 187 | nKernel.connect(kernel.self()); 188 | done(); 189 | }); 190 | }); 191 | 192 | beforeEach(function () { 193 | var node = new Node(id, host, port); 194 | // var node2 = new Node("id2", host, port+1); 195 | server = new GenServer(kernel); 196 | server.start("foo"); 197 | }); 198 | 199 | afterEach(function (done) { 200 | server.stop(true); 201 | done(); 202 | }); 203 | 204 | after(function () { 205 | kernel.stop(); 206 | nKernel.stop(); 207 | }); 208 | 209 | it("Should skip parsing job stream if paused", function () { 210 | server._paused = true; 211 | sinon.spy(server, "decodeSingleton"); 212 | server._parse("", {done: true}, {}); 213 | assert.notOk(server.decodeSingleton.called); 214 | server.decodeSingleton.restore(); 215 | server._paused = false; 216 | }); 217 | 218 | it("Should parse incoming singleton", function (done) { 219 | server.once("foo", () => { 220 | async.nextTick(done); 221 | }); 222 | server._parse({event: "foo", data: "bar"}, {done: true}, {}); 223 | }); 224 | 225 | it("Should skip emitting message on failed singleton", function () { 226 | sinon.spy(server, "emit"); 227 | sinon.stub(server, "decodeSingleton", () => { 228 | return new Error("foo"); 229 | }); 230 | server._parse({event: "foo", data: "bar"}, {done: true}, {}); 231 | assert.notOk(server.emit.called); 232 | server.emit.restore(); 233 | server.decodeSingleton.restore(); 234 | }); 235 | 236 | it("Should skip emitting singleton when stream has error", function () { 237 | sinon.spy(server, "emit"); 238 | server._parse(null, {error: {}, done: true}, {}); 239 | assert.notOk(server.emit.called); 240 | server.emit.restore(); 241 | }); 242 | 243 | it("Should parse incoming job streams", function () { 244 | var data = Buffer.from(JSON.stringify({ok: true})); 245 | var stream = {stream: uuid.v4(), done: false}; 246 | assert.notOk(server.streams().has(stream.stream)); 247 | server._parse(data, stream, {}); 248 | assert.ok(server.streams().has(stream.stream)); 249 | assert.equal(Buffer.compare(data, server.streams().get(stream.stream).data), 0); 250 | }); 251 | 252 | it("Should parse incoming job streams, with existing stream data", function () { 253 | var data = Buffer.from(JSON.stringify({ok: true})); 254 | var stream = {stream: uuid.v4(), done: false}; 255 | var init = Buffer.from("foo"); 256 | server.streams().set(stream.stream, {data: init}); 257 | server._parse(data, stream, {}); 258 | assert.ok(server.streams().has(stream.stream)); 259 | var exp = Buffer.concat([init, data], init.length + data.length); 260 | assert.equal(Buffer.compare(exp, server.streams().get(stream.stream).data), 0); 261 | }); 262 | 263 | it("Should skip parsing full job if stream errors", function () { 264 | sinon.spy(server, "decodeJob"); 265 | var stream = {stream: uuid.v4(), error: {foo: "bar"}, done: true}; 266 | var init = Buffer.from("foo"); 267 | server.streams().set(stream.stream, {data: init}); 268 | server._parse(null, stream, {}); 269 | assert.notOk(server.streams().has(stream.stream)); 270 | assert.notOk(server.decodeJob.called); 271 | server.decodeJob.restore(); 272 | }); 273 | 274 | it("Should parse a full job", function () { 275 | sinon.stub(server, "emit", (event, val) => { 276 | if (event === "idle") return; 277 | assert.equal(event, "msg"); 278 | assert.equal(val, "foo"); 279 | }); 280 | var data = Buffer.from(JSON.stringify({ 281 | event: "msg", 282 | data: "foo" 283 | })); 284 | var stream = {stream: uuid.v4(), done: false}; 285 | server._parse(data, stream, {}); 286 | server._parse(null, {stream: stream.stream, done: true}, {}); 287 | assert.notOk(server._streams.has(stream.stream)); 288 | assert.ok(server.emit.called); 289 | server.emit.restore(); 290 | }); 291 | 292 | it("Should parse full job, not emitting idle event", function () { 293 | server._streams.set("foo", "bar"); 294 | sinon.stub(server, "emit"); 295 | var data = Buffer.from(JSON.stringify({ 296 | event: "msg", 297 | data: "foo" 298 | })); 299 | var stream = {stream: uuid.v4(), done: false}; 300 | server._parse(data, stream, {}); 301 | server._parse(null, {stream: stream.stream, done: true}, {}); 302 | assert.notOk(server._streams.has(stream.stream)); 303 | assert.notOk(server.emit.calledWith(["idle"])); 304 | server.emit.restore(); 305 | }); 306 | 307 | it("Should parse recipient as local node", function () { 308 | var id = "id"; 309 | var out = server._parseRecipient(id); 310 | assert.equal(out.id, id); 311 | assert.ok(out.node.equals(kernel.self())); 312 | }); 313 | 314 | it("Should parse recipient as another node", function () { 315 | var input = {id: "id", node: kernel.self()}; 316 | var out = server._parseRecipient(input); 317 | assert.equal(out.id, input.id); 318 | assert.ok(out.node.equals(input.node)); 319 | }); 320 | 321 | it("Should call another generic server", function (done) { 322 | var server2 = new GenServer(kernel); 323 | server2.start("bar"); 324 | server2.on("msg", (data, from) => { 325 | server2.reply(from, data); 326 | }); 327 | server.call("bar", "msg", Buffer.from("hello"), (err, res) => { 328 | assert.notOk(err); 329 | assert.equal(Buffer.compare(res, Buffer.from("hello")), 0); 330 | server2.stop(true); 331 | done(); 332 | }, 5000); 333 | }); 334 | 335 | it("Should multicall generic servers", function (done) { 336 | var server2 = new GenServer(kernel); 337 | server2.start("bar"); 338 | server2.on("msg", (data, from) => { 339 | server2.reply(from, data); 340 | }); 341 | var server3 = new GenServer(nKernel); 342 | server3.start("bar"); 343 | server3.on("msg", (data, from) => { 344 | server3.reply(from, data); 345 | }); 346 | var data = Buffer.from("hello"); 347 | server.multicall([kernel.self(), nKernel.self()], "bar", "msg", data, (err, res) => { 348 | assert.notOk(err); 349 | assert.isArray(res); 350 | assert.lengthOf(res, 2); 351 | assert.equal(Buffer.compare(res[0], Buffer.from("hello")), 0); 352 | assert.equal(Buffer.compare(res[1], Buffer.from("hello")), 0); 353 | server2.stop(true); 354 | server3.stop(true); 355 | done(); 356 | }, 5000); 357 | }); 358 | 359 | it("Should cast to another generic server", function (done) { 360 | var server2 = new GenServer(kernel); 361 | server2.start("bar"); 362 | server2.on("msg", (data, from) => { 363 | assert.ok(from.node.equals(kernel.self())); 364 | assert.equal(Buffer.compare(data, Buffer.from("hello")), 0); 365 | server2.stop(true); 366 | done(); 367 | }); 368 | server.cast("bar", "msg", Buffer.from("hello")); 369 | }); 370 | 371 | it("Should multicast to generic servers", function (done) { 372 | var server2 = new GenServer(kernel); 373 | server2.start("bar"); 374 | var count = 0; 375 | server2.on("msg", (data, from) => { 376 | count++; 377 | assert.ok(from.node.equals(kernel.self())); 378 | assert.equal(Buffer.compare(data, Buffer.from("hello")), 0); 379 | server2.stop(true); 380 | if (count === 2) done(); 381 | }); 382 | var server3 = new GenServer(nKernel); 383 | server3.start("bar"); 384 | server3.on("msg", (data, from) => { 385 | count++; 386 | assert.ok(from.node.equals(kernel.self())); 387 | assert.equal(Buffer.compare(data, Buffer.from("hello")), 0); 388 | server3.stop(true); 389 | if (count === 2) done(); 390 | }); 391 | server.abcast([kernel.self(), nKernel.self()], "bar", "msg", Buffer.from("hello")); 392 | }); 393 | 394 | it("Should safely reply to a request", function () { 395 | var out = server._safeReply({}, new stream.PassThrough()); 396 | assert.equal(out, false); 397 | 398 | sinon.stub(server._kernel, "reply"); 399 | out = server._safeReply({tag: "foo", node: server._kernel.self()}); 400 | assert.equal(out, true); 401 | server._kernel.reply.restore(); 402 | }); 403 | 404 | it("Should register timeout, but stream doesn't exist later", function (done) { 405 | server._streamTimeout = 0; 406 | sinon.spy(server, "_safeReply"); 407 | server._registerTimeout({stream: "foo"}, {}); 408 | async.nextTick(() => { 409 | assert.notOk(server._streams.has("foo")); 410 | assert.notOk(server._safeReply.called); 411 | server._safeReply.restore(); 412 | server._streamTimeout = 30000; 413 | done(); 414 | }); 415 | }); 416 | 417 | it("Shopuld register timeout, but not reply to async msg", function (done) { 418 | var called = false; 419 | server._streamTimeout = 0; 420 | sinon.stub(server, "_safeReply", (from, istream) => { 421 | istream.once("error", () => {called = true;}); 422 | return false; 423 | }); 424 | server._streams.set("foo", "bar"); 425 | server._registerTimeout({stream: "foo"}, {}); 426 | setTimeout(() => { 427 | assert.notOk(server._streams.has("foo")); 428 | assert.equal(called, false); 429 | server._safeReply.restore(); 430 | server._streamTimeout = 30000; 431 | done(); 432 | }, 0); 433 | }); 434 | 435 | it("Should register timeout, reply and clear stream", function (done) { 436 | var called = false; 437 | server._streamTimeout = 0; 438 | sinon.stub(server, "_safeReply", (from, istream) => { 439 | istream.once("error", () => {called = true;}); 440 | return true; 441 | }); 442 | server._streams.set("foo", "bar"); 443 | server._registerTimeout({stream: "foo"}, {tag: "baz"}); 444 | setTimeout(() => { 445 | assert.notOk(server._streams.has("foo")); 446 | assert.equal(called, true); 447 | server._safeReply.restore(); 448 | server._streamTimeout = 30000; 449 | done(); 450 | }, 0); 451 | }); 452 | 453 | it("Should register timeout, reply and clear stream, not emit 'idle'", function (done) { 454 | var called = false; 455 | var idle = false; 456 | server._streamTimeout = 0; 457 | sinon.stub(server, "_safeReply", (from, istream) => { 458 | istream.once("error", () => {called = true;}); 459 | return true; 460 | }); 461 | server._streams.set("foo", "bar"); 462 | server._streams.set("a", "b"); 463 | server._registerTimeout({stream: "foo"}, {tag: "baz"}); 464 | server.once("idle", () => {idle = true;}); 465 | setTimeout(() => { 466 | assert.notOk(server._streams.has("foo")); 467 | assert.equal(called, true); 468 | assert.equal(idle, false); 469 | server._safeReply.restore(); 470 | server._streamTimeout = 30000; 471 | done(); 472 | }, 0); 473 | }); 474 | }); 475 | }); 476 | }; 477 | --------------------------------------------------------------------------------