├── circle.yml ├── channels ├── index.js ├── unimplemented-channel.js ├── in-memory-channel.js ├── socket-io-channel.js ├── redis-channel.js └── channel-interface.js ├── helpers └── prettify-joi-error.js ├── .gitignore ├── test ├── channels │ ├── edge-coverage │ │ ├── socket-io.js │ │ └── redis.js │ ├── interface │ │ ├── unimplemented.js │ │ └── inconsistent.js │ └── integration │ │ └── shared.js ├── options-validation.js └── raft.js ├── lib ├── socket-io-server-enhancer.js ├── factory.js └── gaggle.js ├── package.json └── README.md /circle.yml: -------------------------------------------------------------------------------- 1 | test: 2 | override: 3 | - npm run test-ci 4 | -------------------------------------------------------------------------------- /channels/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | memory: require('./in-memory-channel') 3 | , 'socket.io': require('./socket-io-channel') 4 | , redis: require('./redis-channel') 5 | } 6 | -------------------------------------------------------------------------------- /helpers/prettify-joi-error.js: -------------------------------------------------------------------------------- 1 | module.exports = function prettifyJoiError (err) { 2 | return 'Invalid options: ' + err.details.map(function (e) { 3 | return e.message 4 | }).join(', ') 5 | } 6 | -------------------------------------------------------------------------------- /channels/unimplemented-channel.js: -------------------------------------------------------------------------------- 1 | var ChannelInterface = require('./channel-interface') 2 | , util = require('util') 3 | 4 | /** 5 | * Intentionally empty channel for testing channel implementation error detection 6 | */ 7 | 8 | function UnimplementedChannel () { 9 | ChannelInterface.apply(this, Array.prototype.slice.call(arguments)) 10 | } 11 | 12 | util.inherits(UnimplementedChannel, ChannelInterface) 13 | 14 | module.exports = UnimplementedChannel 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | .nyc_output 17 | 18 | # node-waf configuration 19 | .lock-wscript 20 | 21 | # Compiled binary addons (http://nodejs.org/api/addons.html) 22 | build/Release 23 | 24 | # Dependency directory 25 | node_modules 26 | -------------------------------------------------------------------------------- /test/channels/edge-coverage/socket-io.js: -------------------------------------------------------------------------------- 1 | var SocketIOChannel = require('../../../channels/socket-io-channel') 2 | , uuid = require('uuid') 3 | , t = require('tap') 4 | 5 | t.test('socket.io channel - fails options validation', function (t) { 6 | t.throws(function () { 7 | /*eslint-disable no-unused-vars*/ 8 | var c = new SocketIOChannel({ 9 | id: uuid.v4() 10 | }) 11 | /*eslint-enable no-unused-vars*/ 12 | }, /Invalid options: "channelOptions" is required/, 'Should throw if missing host') 13 | 14 | t.end() 15 | }) 16 | -------------------------------------------------------------------------------- /test/channels/interface/unimplemented.js: -------------------------------------------------------------------------------- 1 | var t = require('tap') 2 | , uuid = require('uuid') 3 | , Channel = require('../../../channels/unimplemented-channel') 4 | 5 | t.test('throws when stub methods are called', function (t) { 6 | var c = new Channel({id: uuid.v4()}) 7 | 8 | t.throws(function () { 9 | c.connect() 10 | }, 'throws when connect is called') 11 | 12 | t.throws(function () { 13 | c.broadcast() 14 | }, 'throws when broadcast is called') 15 | 16 | t.throws(function () { 17 | c.send() 18 | }, 'throws when send is called') 19 | 20 | t.throws(function () { 21 | c.disconnect() 22 | }, 'throws when disconnect is called') 23 | 24 | t.end() 25 | }) 26 | -------------------------------------------------------------------------------- /test/channels/interface/inconsistent.js: -------------------------------------------------------------------------------- 1 | var t = require('tap') 2 | , uuid = require('uuid') 3 | , Channel = require('../../../channels/unimplemented-channel') 4 | 5 | t.test('channels throw when inconsistent', function (t) { 6 | t.throws(function () { 7 | var c = new Channel({id: uuid.v4()}) 8 | c._recieved('bogus', 'data') 9 | }, 'throws when _recieved is called when channel is disconnected') 10 | 11 | t.throws(function () { 12 | var c = new Channel({id: uuid.v4()}) 13 | c._connected() 14 | c._connected() 15 | }, 'throws when _connected is called when channel is already connected') 16 | 17 | t.throws(function () { 18 | var c = new Channel({id: uuid.v4()}) 19 | c._disconnected() 20 | }, 'throws when _disconnected is called when channel is already disconnected') 21 | 22 | t.end() 23 | }) 24 | -------------------------------------------------------------------------------- /test/options-validation.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Test if leader election is working 3 | */ 4 | 5 | var t = require('tap') 6 | , gaggle = require('../') 7 | 8 | t.test('fails when required Gaggle options are missing', function (t) { 9 | t.throws(function () { 10 | gaggle({ 11 | id: 'i am required' 12 | , channel: { 13 | 'name': 'memory' 14 | } 15 | }) 16 | }, /Invalid options: "clusterSize" is required/, 'throws the expected error') 17 | 18 | t.end() 19 | }) 20 | 21 | t.test('fails when required factory options are missing', function (t) { 22 | t.throws(function () { 23 | gaggle({ 24 | id: 'i am required' 25 | }) 26 | }, /Invalid options: "channel" is required/, 'throws the expected error') 27 | 28 | t.end() 29 | }) 30 | 31 | t.test('fails properly when no options are given', function (t) { 32 | t.throws(function () { 33 | gaggle() 34 | }, /Invalid options: "id" is required/, 'throws the expected error') 35 | 36 | t.end() 37 | }) 38 | -------------------------------------------------------------------------------- /test/channels/edge-coverage/redis.js: -------------------------------------------------------------------------------- 1 | var RedisChannel = require('../../../channels/redis-channel') 2 | , uuid = require('uuid') 3 | , t = require('tap') 4 | 5 | t.test('redis channel - fails options validation', function (t) { 6 | t.throws(function () { 7 | /*eslint-disable no-unused-vars*/ 8 | var c = new RedisChannel({ 9 | id: uuid.v4() 10 | }) 11 | /*eslint-enable no-unused-vars*/ 12 | }, /Invalid options: "channelOptions" is required/, 'Should throw if missing redisChannel') 13 | 14 | t.end() 15 | }) 16 | 17 | t.test('redis channel - connects with custom connection string', function (t) { 18 | var c 19 | 20 | t.plan(3) 21 | 22 | t.doesNotThrow(function () { 23 | c = new RedisChannel({ 24 | id: uuid.v4() 25 | , logFunction: console.error 26 | , channelOptions: { 27 | connectionString: 'redis://127.0.0.1' 28 | , channelName: 'dummy;neverused' 29 | } 30 | }) 31 | }, 'Should not throw') 32 | 33 | c.once('disconnected', function () { 34 | t.pass('disconnected') 35 | }) 36 | 37 | c.once('connected', function () { 38 | t.pass('connected') 39 | 40 | c.disconnect() 41 | }) 42 | 43 | c.connect() 44 | }) 45 | -------------------------------------------------------------------------------- /lib/socket-io-server-enhancer.js: -------------------------------------------------------------------------------- 1 | var SocketIO = require('socket.io') 2 | , _ = require('lodash') 3 | 4 | function enhanceServerForGaggleSocketIOChannel (server) { 5 | var io = SocketIO(server) 6 | , sockets = [] 7 | , cleanupAfterSocketDisconnect 8 | 9 | cleanupAfterSocketDisconnect = function cleanupAfterSocketDisconnect (socket) { 10 | _.remove(sockets, {handle: socket}) 11 | socket.removeAllListeners() 12 | socket.disconnect() 13 | } 14 | 15 | io.sockets.on('connection', function (socket) { 16 | sockets.push({ 17 | handle: socket 18 | , channel: 'unknown' 19 | }) 20 | 21 | // broadcast everything to everyone in the same channel 22 | socket.on('msg', function (dat) { 23 | var sourceChannel = _.find(sockets, {handle: socket}).channel 24 | 25 | _.each(sockets, function (s) { 26 | if (s.channel === sourceChannel) { 27 | s.handle.emit('msg', dat) 28 | } 29 | }) 30 | }) 31 | 32 | socket.on('declareChannel', function (channel) { 33 | _.find(sockets, {handle: socket}).channel = channel 34 | }) 35 | 36 | socket.on('disconnect', _.bind(cleanupAfterSocketDisconnect, null, socket)) 37 | }) 38 | 39 | return function teardown (cb) { 40 | io.close() 41 | 42 | _.each(sockets, cleanupAfterSocketDisconnect) 43 | } 44 | } 45 | 46 | module.exports = enhanceServerForGaggleSocketIOChannel 47 | -------------------------------------------------------------------------------- /lib/factory.js: -------------------------------------------------------------------------------- 1 | var Joi = require('joi') 2 | , _ = require('lodash') 3 | , channels = require('../channels') 4 | , channelNames = _.keys(channels) 5 | , prettifyJoiError = require('../helpers/prettify-joi-error') 6 | , Gaggle = require('./gaggle') 7 | , schema 8 | 9 | /** 10 | * Validate the bare minimum, leave the rest up to the Channel and Gaggle 11 | * constructors to handle. 12 | */ 13 | schema = Joi.object().keys({ 14 | id: Joi.string() 15 | , channel: Joi.object().keys({ 16 | name: Joi.string().valid(channelNames) 17 | }) 18 | }).requiredKeys([ 19 | 'id' 20 | , 'channel' 21 | , 'channel.name' 22 | ]) 23 | 24 | 25 | module.exports = function GaggleFactory (opts) { 26 | var validatedOptions = Joi.validate(opts || {}, schema, {allowUnknown: true, stripUnknown: false}) 27 | , channelInst 28 | 29 | if (validatedOptions.error != null) { 30 | throw new Error(prettifyJoiError(validatedOptions.error)) 31 | } 32 | 33 | opts = validatedOptions.value 34 | 35 | channelInst = new channels[opts.channel.name]({ 36 | id: opts.id 37 | , channelOptions: _.omit(opts.channel, 'name') 38 | }) 39 | 40 | return new Gaggle(_.assign({}, opts, {channel: channelInst})) 41 | } 42 | 43 | module.exports.enhanceServerForSocketIOChannel = require('./socket-io-server-enhancer.js') 44 | 45 | module.exports._STATES = Gaggle._STATES 46 | module.exports._RPC_TYPE = Gaggle._RPC_TYPE 47 | -------------------------------------------------------------------------------- /channels/in-memory-channel.js: -------------------------------------------------------------------------------- 1 | var ChannelInterface = require('./channel-interface') 2 | , util = require('util') 3 | , _ = require('lodash') 4 | , instanceMap = {} 5 | 6 | /** 7 | * Intended for use in testing, this channel only works 8 | * within the same process, and uses timeouts to simulate 9 | * network delay 10 | */ 11 | 12 | function InMemoryChannel () { 13 | ChannelInterface.apply(this, Array.prototype.slice.call(arguments)) 14 | } 15 | 16 | util.inherits(InMemoryChannel, ChannelInterface) 17 | 18 | InMemoryChannel.prototype._connect = function _connect () { 19 | var self = this 20 | 21 | setTimeout(function () { 22 | instanceMap[self.id] = self 23 | self._connected() 24 | }, 0) 25 | } 26 | 27 | InMemoryChannel.prototype._disconnect = function _disconnect () { 28 | var self = this 29 | 30 | setTimeout(function () { 31 | instanceMap[self.id] = null 32 | self._disconnected() 33 | }, 0) 34 | } 35 | 36 | InMemoryChannel.prototype._broadcast = function _broadcast (data) { 37 | var self = this 38 | 39 | _.each(instanceMap, function (instance, key) { 40 | if (instance != null) { 41 | self._send(key, data) 42 | } 43 | }) 44 | } 45 | 46 | InMemoryChannel.prototype._send = function _send (nodeId, data) { 47 | var self = this 48 | 49 | setTimeout(function () { 50 | if (instanceMap[nodeId] != null) { 51 | instanceMap[nodeId]._recieved(self.id, data) 52 | } 53 | }, 0) 54 | } 55 | 56 | module.exports = InMemoryChannel 57 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gaggle", 3 | "version": "2.1.4", 4 | "description": "A distributed log", 5 | "main": "lib/factory", 6 | "directories": { 7 | "test": "test" 8 | }, 9 | "scripts": { 10 | "test": "tap test/**/*.js --reporter=spec --statements=100 --coverage-report=text-summary", 11 | "coverage": "tap test/**/*.js --reporter=spec --coverage --coverage-report=html && open -a \"Safari\" coverage/index.html", 12 | "test-ci": "tap test/**/*.js --reporter=spec --statements=100", 13 | "tdd": "nodemon -x npm test", 14 | "doctoc": "doctoc README.md", 15 | "prepublish": "in-publish && npm run doctoc || not-in-publish" 16 | }, 17 | "repository": { 18 | "type": "git", 19 | "url": "git+https://github.com/ben-ng/gaggle.git" 20 | }, 21 | "keywords": [ 22 | "raft", 23 | "paxos" 24 | ], 25 | "author": "Ben Ng ", 26 | "license": "MIT", 27 | "browser": { 28 | "./channels/redis-channel.js": false, 29 | "./lib/socket-io-server-enhancer.js": false 30 | }, 31 | "bugs": { 32 | "url": "https://github.com/ben-ng/gaggle/issues" 33 | }, 34 | "homepage": "https://github.com/ben-ng/gaggle#readme", 35 | "devDependencies": { 36 | "async": "^1.5.2", 37 | "doctoc": "^0.15.0", 38 | "nodemon": "^1.4.1", 39 | "tap": "^5.1.1" 40 | }, 41 | "dependencies": { 42 | "bluebird": "^3.1.1", 43 | "in-publish": "^2.0.0", 44 | "joi": "^6.10.0", 45 | "lodash": "^4.0.0", 46 | "once": "^1.3.3", 47 | "redis": "^2.4.2", 48 | "socket.io": "^1.4.5", 49 | "socket.io-client": "^1.4.5", 50 | "uuid": "^2.0.1" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /channels/socket-io-channel.js: -------------------------------------------------------------------------------- 1 | var ChannelInterface = require('./channel-interface') 2 | , Joi = require('joi') 3 | , prettifyJoiError = require('../helpers/prettify-joi-error') 4 | , SocketIOClient = require('socket.io-client') 5 | , util = require('util') 6 | , _ = require('lodash') 7 | 8 | /** 9 | * A simple channel that uses Redis's pub/sub 10 | */ 11 | 12 | function SocketIOChannel (opts) { 13 | var validatedOptions = Joi.validate(opts || {}, Joi.object().keys({ 14 | channelOptions: Joi.object().keys({ 15 | host: Joi.string() 16 | , channel: Joi.string() 17 | }) 18 | , logFunction: Joi.func() 19 | , id: Joi.string() 20 | }).requiredKeys('channelOptions', 'channelOptions.host', 'channelOptions.channel'), { 21 | convert: false 22 | }) 23 | 24 | ChannelInterface.apply(this, Array.prototype.slice.call(arguments)) 25 | 26 | if (validatedOptions.error != null) { 27 | throw new Error(prettifyJoiError(validatedOptions.error)) 28 | } 29 | 30 | this._host = _.get(validatedOptions, 'value.channelOptions.host') 31 | this._channel = _.get(validatedOptions, 'value.channelOptions.channel') 32 | } 33 | 34 | util.inherits(SocketIOChannel, ChannelInterface) 35 | 36 | SocketIOChannel.prototype._connect = function _connect () { 37 | var self = this 38 | , client = SocketIOClient(this._host) 39 | , onPossibilityOfServerLosingChannelInformation 40 | 41 | onPossibilityOfServerLosingChannelInformation = function declareChannel () { 42 | client.emit('declareChannel', self._channel) 43 | 44 | if (!self.state.connected) { 45 | self._connected() 46 | } 47 | } 48 | 49 | client.on('msg', function (msg) { 50 | if ((msg.to == null || msg.to === self.id) && self.state.connected) { 51 | 52 | self._recieved(msg.from, msg.data) 53 | } 54 | }) 55 | 56 | client.on('reconnect', onPossibilityOfServerLosingChannelInformation) 57 | client.on('unknownChannel', onPossibilityOfServerLosingChannelInformation) 58 | client.once('connect', onPossibilityOfServerLosingChannelInformation) 59 | 60 | this._client = client 61 | } 62 | 63 | SocketIOChannel.prototype._disconnect = function _disconnect () { 64 | var self = this 65 | 66 | this._client.removeAllListeners() 67 | 68 | this._client.once('disconnect', function () { 69 | self._disconnected() 70 | }) 71 | 72 | this._client.disconnect() 73 | } 74 | 75 | SocketIOChannel.prototype._broadcast = function _broadcast (data) { 76 | this._client.emit('msg', { 77 | from: this.id 78 | , data: data 79 | }) 80 | } 81 | 82 | SocketIOChannel.prototype._send = function _send (nodeId, data) { 83 | this._client.emit('msg', { 84 | from: this.id 85 | , to: nodeId 86 | , data: data 87 | }) 88 | } 89 | 90 | module.exports = SocketIOChannel 91 | -------------------------------------------------------------------------------- /channels/redis-channel.js: -------------------------------------------------------------------------------- 1 | var ChannelInterface = require('./channel-interface') 2 | , Joi = require('joi') 3 | , prettifyJoiError = require('../helpers/prettify-joi-error') 4 | , redis = require('redis') 5 | , util = require('util') 6 | , _ = require('lodash') 7 | 8 | /** 9 | * A simple channel that uses Redis's pub/sub 10 | */ 11 | 12 | function RedisChannel (opts) { 13 | var validatedOptions = Joi.validate(opts || {}, Joi.object().keys({ 14 | channelOptions: Joi.object().keys({ 15 | connectionString: Joi.string() 16 | , channelName: Joi.string() 17 | }) 18 | , logFunction: Joi.func() 19 | , id: Joi.string() 20 | }).requiredKeys('channelOptions', 'channelOptions.channelName'), { 21 | convert: false 22 | }) 23 | 24 | ChannelInterface.apply(this, Array.prototype.slice.call(arguments)) 25 | 26 | if (validatedOptions.error != null) { 27 | throw new Error(prettifyJoiError(validatedOptions.error)) 28 | } 29 | 30 | this._connString = _.get(validatedOptions, 'value.channelOptions.connectionString') 31 | this._redisChannel = _.get(validatedOptions, 'value.channelOptions.channelName') 32 | } 33 | 34 | util.inherits(RedisChannel, ChannelInterface) 35 | 36 | RedisChannel.prototype._connect = function _connect () { 37 | var self = this 38 | , redisChannel = this._redisChannel 39 | , connString = this._connString 40 | , subClient = connString != null ? redis.createClient(connString) : redis.createClient() 41 | , pubClient = connString != null ? redis.createClient(connString) : redis.createClient() 42 | , connectedAfterTwoCalls = _.after(2, function () { 43 | self._connected() 44 | }) 45 | 46 | subClient.on('error', this._logFunction) 47 | pubClient.on('error', this._logFunction) 48 | 49 | subClient.on('message', function (channel, message) { 50 | var parsed 51 | 52 | try { 53 | parsed = JSON.parse(message) 54 | } 55 | catch (e) {} 56 | 57 | if (channel === redisChannel && (parsed.to == null || parsed.to === self.id) && self.state.connected) { 58 | self._recieved(parsed.from, parsed.data) 59 | } 60 | }) 61 | 62 | subClient.on('subscribe', function (channel) { 63 | if (channel === redisChannel) { 64 | connectedAfterTwoCalls() 65 | } 66 | }) 67 | 68 | subClient.subscribe(redisChannel) 69 | 70 | pubClient.on('ready', function () { 71 | connectedAfterTwoCalls() 72 | }) 73 | 74 | this._sub = subClient 75 | this._pub = pubClient 76 | } 77 | 78 | RedisChannel.prototype._disconnect = function _disconnect () { 79 | var self = this 80 | , disconnectedAfterTwoCalls = _.after(2, function () { 81 | self._disconnected() 82 | }) 83 | 84 | this._sub.unsubscribe(this._redisChannel) 85 | this._sub.removeAllListeners() 86 | this._sub.once('end', function () { 87 | disconnectedAfterTwoCalls() 88 | }) 89 | this._sub.quit() 90 | 91 | this._pub.removeAllListeners() 92 | this._pub.once('end', function () { 93 | disconnectedAfterTwoCalls() 94 | }) 95 | this._pub.quit() 96 | } 97 | 98 | RedisChannel.prototype._broadcast = function _broadcast (data) { 99 | this._pub.publish(this._redisChannel, JSON.stringify({ 100 | from: this.id 101 | , data: data 102 | })) 103 | } 104 | 105 | RedisChannel.prototype._send = function _send (nodeId, data) { 106 | this._pub.publish(this._redisChannel, JSON.stringify({ 107 | from: this.id 108 | , to: nodeId 109 | , data: data 110 | })) 111 | } 112 | 113 | module.exports = RedisChannel 114 | -------------------------------------------------------------------------------- /channels/channel-interface.js: -------------------------------------------------------------------------------- 1 | var EventEmitter = require('events').EventEmitter 2 | , util = require('util') 3 | , Joi = require('joi') 4 | , _ = require('lodash') 5 | , prettifyJoiError = require('../helpers/prettify-joi-error') 6 | 7 | /** 8 | * Channels are how nodes on the network communicate and must be initialized 9 | * with the process ID. 10 | * 11 | * Events: 12 | * connected 13 | * disconnected 14 | * recieved(int originNodeId, data) 15 | * 16 | * Channel implementors should extend this class with the methods: 17 | * _connect 18 | * _disconnect 19 | * _broadcast 20 | * _send 21 | * 22 | * Implementors should use the following protected methods: 23 | * _connected 24 | * _disconnected 25 | * _recieved 26 | * 27 | * Channel consumers should use the public interface: 28 | * connect 29 | * disconnect 30 | * broadcast 31 | * send 32 | * 33 | */ 34 | 35 | function ChannelInterface (opts) { 36 | EventEmitter.call(this) 37 | 38 | var validatedOptions = Joi.validate(opts || {}, Joi.object().keys({ 39 | id: Joi.string() 40 | , logFunction: Joi.func().default(function noop () {}) 41 | , channelOptions: Joi.object() 42 | }).requiredKeys('id')) 43 | 44 | if (validatedOptions.error != null) { 45 | throw new Error(prettifyJoiError(validatedOptions.error)) 46 | } 47 | 48 | this.id = validatedOptions.value.id 49 | 50 | this.state = { 51 | connected: false 52 | , isReconnecting: false 53 | } 54 | 55 | this._logFunction = validatedOptions.value.logFunction 56 | 57 | // Avoid duplicate messages 58 | this._lastRecievedMap = {} 59 | this._sequence = 0 60 | } 61 | 62 | util.inherits(ChannelInterface, EventEmitter) 63 | 64 | /** 65 | * For channel implementors: 66 | * Call this when a message is recieved 67 | */ 68 | ChannelInterface.prototype._recieved = function _recieved (originNodeId, packet) { 69 | if (this.state.connected === false) { 70 | throw new Error('_recieved was called although the channel is in the disconnected state') 71 | } 72 | else { 73 | if (this._lastRecievedMap[originNodeId] == null || this._lastRecievedMap[originNodeId] < packet.sequence) { 74 | this._lastRecievedMap[originNodeId] = packet.sequence 75 | 76 | this._logFunction(this.id + ' recieved from ' + originNodeId + '\n' + JSON.stringify(packet.data, null, 2)) 77 | this.emit('recieved', originNodeId, packet.data) 78 | } 79 | // else, this is a duplicate, and we should ignore it 80 | } 81 | } 82 | 83 | /** 84 | * For channel implementors: 85 | * Call this when the channel has connected 86 | */ 87 | ChannelInterface.prototype._connected = function _connected () { 88 | if (this.state.connected === true) { 89 | throw new Error('_connected was called although the channel is already in the connected state') 90 | } 91 | else { 92 | this.state.connected = true 93 | this.emit('connected') 94 | } 95 | } 96 | 97 | /** 98 | * For channel implementors: 99 | * Call this when the channel is disconnected 100 | */ 101 | ChannelInterface.prototype._disconnected = function _disconnected () { 102 | if (this.state.connected === false) { 103 | throw new Error('_disconnected was called although the channel is already in the disconnected state') 104 | } 105 | else { 106 | this.state.connected = false 107 | this.emit('disconnected') 108 | } 109 | } 110 | 111 | /** 112 | * For channel consumers: 113 | * Connect to the network 114 | * 115 | * Call _connected once the connection is established, 116 | * and call _disconnected when the connection is lost. 117 | * In the event of disconnection, channels should 118 | * automatically attempt to reconnect. 119 | */ 120 | ChannelInterface.prototype.connect = function connect () { 121 | if (typeof this._connect === 'function') { 122 | return this._connect() 123 | } 124 | else { 125 | throw new Error('Not implemented') 126 | } 127 | } 128 | 129 | /** 130 | * For channel consumers: 131 | * Disconnect from the network 132 | * 133 | * Takes care to close all connections so that the process can 134 | * quickly and cleanly exit. 135 | */ 136 | ChannelInterface.prototype.disconnect = function disconnect () { 137 | this._broadcast = _.noop 138 | this._send = _.noop 139 | this._recieved = _.noop 140 | 141 | if (typeof this._disconnect === 'function') { 142 | return this._disconnect() 143 | } 144 | else { 145 | throw new Error('Not implemented') 146 | } 147 | } 148 | 149 | /** 150 | * For channel consumers: 151 | * Send a message to all nodes on the network 152 | */ 153 | ChannelInterface.prototype.broadcast = function broadcast (data) { 154 | if (typeof this._broadcast === 'function') { 155 | this._logFunction(this.id + ' broadcasted\n' + JSON.stringify(data, null, 2)) 156 | return this._broadcast(this._createPacket(data)) 157 | } 158 | else { 159 | throw new Error('Not implemented') 160 | } 161 | } 162 | 163 | /** 164 | * For channel consumers: 165 | * Send a message to a node on the network 166 | */ 167 | ChannelInterface.prototype.send = function send (nodeId, data) { 168 | if (typeof this._send === 'function') { 169 | this._logFunction(this.id + ' sent to ' + nodeId + '\n' + JSON.stringify(data, null, 2)) 170 | return this._send(nodeId, this._createPacket(data)) 171 | } 172 | else { 173 | throw new Error('Not implemented') 174 | } 175 | } 176 | 177 | /** 178 | * Private method that creates a data packet 179 | */ 180 | ChannelInterface.prototype._createPacket = function createPacket (data) { 181 | var seq = this._sequence 182 | 183 | this._sequence = this._sequence + 1 184 | 185 | return { 186 | data: data 187 | , sequence: seq 188 | } 189 | } 190 | 191 | module.exports = ChannelInterface 192 | -------------------------------------------------------------------------------- /test/channels/integration/shared.js: -------------------------------------------------------------------------------- 1 | var t = require('tap') 2 | , uuid = require('uuid') 3 | , _ = require('lodash') 4 | , http = require('http') 5 | , serverEnhancer = require('../../../lib/socket-io-server-enhancer') 6 | , RANDOM_HIGH_PORT = _.random(9000, 65535) 7 | , InMemoryChannel = require('../../../channels/in-memory-channel') 8 | , RedisChannel = require('../../../channels/redis-channel') 9 | , SocketIOChannel = require('../../../channels/socket-io-channel') 10 | , channelsToTest = { 11 | InMemory: { 12 | create: function createInMemoryChannel () { 13 | return new InMemoryChannel({id: uuid.v4()}) 14 | } 15 | , cls: InMemoryChannel 16 | } 17 | , Redis: { 18 | create: function createRedisChannel () { 19 | return new RedisChannel({ 20 | id: uuid.v4() 21 | , channelOptions: { 22 | channelName: 'channelIntegrationTestChannel' 23 | } 24 | }) 25 | } 26 | , cls: RedisChannel 27 | } 28 | , SocketIO: { 29 | setup: function setupSocketIOServer (cb) { 30 | var noop = function noop (req, resp) { 31 | resp.writeHead(200) 32 | resp.end() 33 | } 34 | , server = http.createServer(noop) 35 | , closeServer 36 | 37 | closeServer = serverEnhancer(server) 38 | 39 | server.listen(RANDOM_HIGH_PORT, function () { 40 | cb(function exposedTeardownCb (teardownCb) { 41 | server.once('close', teardownCb) 42 | closeServer() 43 | }) 44 | }) 45 | } 46 | , create: function createSocketIOChannel () { 47 | return new SocketIOChannel({ 48 | id: uuid.v4() 49 | , channelOptions: { 50 | host: 'http://127.0.0.1:' + RANDOM_HIGH_PORT 51 | , channel: 'gaggle' 52 | } 53 | }) 54 | } 55 | , cls: RedisChannel 56 | } 57 | } 58 | 59 | // Start outer EACH 60 | // Define these t.tests once for each channel 61 | _.each(channelsToTest, function (channelDetails, channelName) { 62 | 63 | var createChannel = channelDetails.create 64 | , customTestSetup = channelDetails.setup 65 | , Channel = channelDetails.cls 66 | 67 | if (customTestSetup == null) { 68 | customTestSetup = function noopSetup (cb) { 69 | cb(function noopCleanup (_cb) {_cb()}) 70 | } 71 | } 72 | 73 | function setupTest (t, requestedChannelCount, cb) { 74 | var channels = [] 75 | , i 76 | , tempChannel 77 | , connectedCounter = 0 78 | , connectedMap = {} 79 | , onConnect 80 | , cleanup 81 | , cleanedUp = false 82 | , executionsCounter = 0 83 | , customTestCleanup 84 | 85 | cleanup = function () { 86 | var onDisconnect 87 | 88 | if (cleanedUp) { 89 | throw new Error('Cannot clean up the same t.test twice') 90 | } 91 | else { 92 | cleanedUp = true 93 | } 94 | 95 | if (requestedChannelCount === 0) { 96 | customTestCleanup(t.end) 97 | return 98 | } 99 | 100 | onDisconnect = function (whichChannel) { 101 | connectedCounter -= 1 102 | 103 | if (connectedCounter === 0) { 104 | customTestCleanup(t.end) 105 | } 106 | } 107 | 108 | for (var channelId in connectedMap) { 109 | if (connectedMap.hasOwnProperty(channelId) && connectedMap[channelId] != null) { 110 | connectedMap[channelId].once('disconnected', onDisconnect.bind(this, tempChannel)) 111 | connectedMap[channelId].disconnect() 112 | connectedMap[channelId] = null 113 | } 114 | } 115 | } 116 | 117 | cleanup.after = function (executionsNeeded) { 118 | executionsCounter++ 119 | 120 | if (executionsCounter === executionsNeeded) { 121 | cleanup() 122 | } 123 | } 124 | 125 | onConnect = function (whichChannel) { 126 | if (connectedMap[whichChannel.id] == null) { 127 | connectedCounter += 1 128 | 129 | connectedMap[whichChannel.id] = whichChannel 130 | 131 | if (connectedCounter === requestedChannelCount) { 132 | cb.apply(this, channels.concat(cleanup)) 133 | } 134 | } 135 | } 136 | 137 | customTestSetup(function (teardownFunc) { 138 | customTestCleanup = teardownFunc 139 | 140 | if (requestedChannelCount === 0) { 141 | cb(cleanup) 142 | return 143 | } 144 | 145 | for (i=0; i 6 | 7 | **Contents** 8 | 9 | - [Quick Example](#quick-example) 10 | - [API](#api) 11 | - [Gaggle](#gaggle) 12 | - [Creating an instance](#creating-an-instance) 13 | - [Appending Messages](#appending-messages) 14 | - [Performing RPC calls on the leader](#performing-rpc-calls-on-the-leader) 15 | - [Checking for uncommitted entries in previous terms](#checking-for-uncommitted-entries-in-previous-terms) 16 | - [Deconstructing an instance](#deconstructing-an-instance) 17 | - [Getting the state of the node](#getting-the-state-of-the-node) 18 | - [Getting the log](#getting-the-log) 19 | - [Getting the commit index](#getting-the-commit-index) 20 | - [Event: appended](#event-appended) 21 | - [Event: committed](#event-committed) 22 | - [Event: leaderElected](#event-leaderelected) 23 | - [Channels](#channels) 24 | - [Socket.io](#socketio) 25 | - [Socket.io Channel Options](#socketio-channel-options) 26 | - [Redis](#redis) 27 | - [Redis Channel Options](#redis-channel-options) 28 | - [Redis Channel Example](#redis-channel-example) 29 | - [Memory](#memory) 30 | - [Memory Channel Options](#memory-channel-options) 31 | - [Memory Channel Example](#memory-channel-example) 32 | - [Running Tests](#running-tests) 33 | - [License](#license) 34 | 35 | 36 | 37 | ## Quick Example 38 | 39 | ```js 40 | var gaggle = require('gaggle') 41 | var uuid = require('uuid') 42 | var defaults = require('lodash/defaults') 43 | var opts = { 44 | channel: { 45 | name: 'redis' 46 | , redisChannel: 'foo' 47 | } 48 | , clusterSize: 3 49 | } 50 | 51 | var nodeA = gaggle(defaults({id: uuid.v4()}, opts)) 52 | var nodeB = gaggle(defaults({id: uuid.v4()}, opts)) 53 | var nodeC = gaggle(defaults({id: uuid.v4()}, opts)) 54 | 55 | // Nodes will emit "committed" events whenever the cluster 56 | // comes to consensus about an entry 57 | nodeC.on('committed', function (data) { 58 | console.log(data) 59 | }) 60 | 61 | // You can be notified when a specific message is committed 62 | // by providing a callback 63 | nodeC.append('mary', function () { 64 | console.log(',') 65 | }) 66 | 67 | // Or, you can use promises 68 | nodeA.append('had').then(function () { 69 | console.log('!') 70 | }) 71 | 72 | // Or, you can just cross your fingers and hope that an error 73 | // doesn't happen by neglecting the return result and callback 74 | nodeA.append('a') 75 | 76 | // Entry data can also be numbers, arrays, or objects 77 | // we were just using strings here for simplicity 78 | nodeB.append({foo: 'lamb'}) 79 | 80 | // You can specify a timeout as a second argument 81 | nodeA.append('little', 1000) 82 | 83 | // By default, gaggle will wait indefinitely for a message to commit 84 | nodeC.append('a', function () { 85 | // I may never be called! 86 | }) 87 | 88 | // This example prints the sentence: 89 | // "mary , had a little {foo: 'lamb'} !" 90 | // in SOME order; Raft only guarantees that all nodes will commit entries in 91 | // the same order, but nodes sent at different times may not be committed 92 | // in the order that they were sent. 93 | ``` 94 | 95 | ## API 96 | 97 | ### Gaggle 98 | 99 | #### Creating an instance 100 | 101 | ```js 102 | var gaggle = require('gaggle') 103 | // uuids are recommended, but you can use any string id 104 | , uuid = require('uuid') 105 | , g = gaggle({ 106 | /** 107 | * Required settings 108 | */ 109 | 110 | id: uuid.v4() 111 | , clusterSize: 5 112 | , channel: { 113 | name: 'redis' // or "memory", etc ... 114 | 115 | // ... additional keys are passed as options to the 116 | // "redis" channel. see channel docs for available 117 | // options. 118 | } 119 | 120 | /** 121 | * Optional settings 122 | */ 123 | 124 | // Can be called through dispatchOnLeader() 125 | , rpc: { 126 | foo: function foo (a, b, c, d) { 127 | // "this" inside here refers to the leader Gaggle instance 128 | // so you can do things like this... 129 | if (this.hasUncommittedEntriesFromPreviousTerms()) { 130 | this.append('noop') 131 | 132 | return new Error('I am not ready yet, try again in a few seconds') 133 | } 134 | else { 135 | return 'foo' 136 | } 137 | } 138 | } 139 | 140 | // How long to wait before declaring the leader dead? 141 | , electionTimeout: { 142 | min: 300 143 | , max: 500 144 | } 145 | 146 | // How often should the leader send heartbeats? 147 | , heartbeatInterval: 50 148 | 149 | // Should the leader send a heartbeat if it would speed 150 | // up the commit of a message? 151 | , accelerateHeartbeats: false 152 | }) 153 | ``` 154 | 155 | #### Appending Messages 156 | 157 | ```js 158 | g.append(Mixed data, [Number timeout], [function(Error) callback]) 159 | ``` 160 | 161 | Anything that can be serialized and deserialized as JSON is valid message data. If `callback` is not provided, a `Promise` will be returned. 162 | 163 | ```js 164 | g.append(data, function (err) {}) 165 | g.append(data, timeout, function (err) {}) 166 | 167 | g.append(data).then() 168 | g.append(data, timeout).then() 169 | ``` 170 | 171 | 172 | #### Performing RPC calls on the leader 173 | 174 | ```js 175 | g.dispatchOnLeader(String functionName, Array args, [Number timeout], [function(Error, Mixed result) callback]) 176 | ``` 177 | 178 | If you're building something on top of Gaggle, you'll probably have to use the leader as a coordinator. This is a helper function that simplifies that. While the `timeout` period is optional, omitting it means that the operation may never complete. You should *probably* always specify a timeout to handle lost messages and leader crashes. 179 | 180 | ```js 181 | // Calls the function at key "foo" on the "rpc" object that was passed in as 182 | // an option to the Gaggle constructor with the arguments "bar" and "baz". 183 | g.dispatchOnLeader('foo', ['bar', 'baz'], 5000, function (err, result) { 184 | }) 185 | 186 | g.dispatchOnLeader('foo', ['bar', 'baz'], 5000) 187 | .then(function (result) { 188 | 189 | }) 190 | .catch(function (err) { 191 | 192 | }) 193 | ``` 194 | 195 | #### Checking for uncommitted entries in previous terms 196 | 197 | ```js 198 | g.hasUncommittedEntriesInPreviousTerms() 199 | ``` 200 | 201 | You'll need to use this in your RPC functions in order to safely handle leadership changes. Since leaders do not commit entries in earlier terms, you might need to "nudge" the cluster into a consistent state by appending a no-op message. 202 | 203 | #### Deconstructing an instance 204 | 205 | ```js 206 | g.close([function(Error) callback]) 207 | ``` 208 | 209 | When you're done, call `close` to remove event listeners and disconnect the channel. 210 | 211 | ```js 212 | g.close(function (err) {}) 213 | 214 | g.close().then() 215 | ``` 216 | 217 | #### Getting the state of the node 218 | 219 | ```js 220 | g.isLeader() 221 | ``` 222 | 223 | Returns `true` if the current node is the leader state. Note that multiple nodes may return `true` at the same time because they can be leaders in different terms. 224 | 225 | #### Getting the log 226 | 227 | ```js 228 | g.getLog() 229 | ``` 230 | 231 | Returns the log, which is an array of entries. 232 | 233 | #### Getting the commit index 234 | 235 | ```js 236 | g.getCommitIndex() 237 | ``` 238 | 239 | Returns the commit index, which is the index of the last committed log entry. 240 | 241 | #### Event: appended 242 | 243 | Emitted by a leader whenever an entry is appended (but not committed) to its log. 244 | 245 | ```js 246 | g.on('appended', function (entry, index) { 247 | // entry => {id: 'some-uuid', term: 1, data: {foo: bar}} 248 | // index => 1 249 | }) 250 | ``` 251 | 252 | #### Event: committed 253 | 254 | Emitted whenever an entry is committed to the node's log. 255 | 256 | ```js 257 | g.on('committed', function (entry, index) { 258 | // entry => {id: 'some-uuid', term: 1, data: {foo: bar}} 259 | // index => 1 260 | }) 261 | ``` 262 | 263 | #### Event: leaderElected 264 | 265 | Emitted whenever a node discovers that a new leader has been elected. 266 | 267 | ```js 268 | g.on('leaderElected', function () { 269 | console.log('four! more! years!') 270 | }) 271 | ``` 272 | 273 | ### Channels 274 | 275 | #### Socket.io 276 | 277 | A pretty fast channel that works on either the server or the browser. You need to host your own Socket.io server. Gaggle exports a helper function to assist with this. 278 | 279 | ```js 280 | var serverEnhancer = require('gaggle').enhanceServerForSocketIOChannel 281 | 282 | var server = http.createServer(function (req, resp) { 283 | resp.writeHead(200) 284 | resp.end() 285 | }) 286 | 287 | var closeServer = serverEnhancer(server) 288 | 289 | server.listen(8000) 290 | 291 | // When you need to cleanly shut down `server`: 292 | closeServer() 293 | ``` 294 | 295 | ##### Socket.io Channel Options 296 | 297 | * *required* String `name` Set to 'socket.io' to use this channel 298 | * *required* String `host` Where your socket.io server is running, e.g. `http://localhost:9000` 299 | * *required* String `channel` What channel to use 300 | 301 | #### Redis 302 | 303 | Fast, but relies heavily on your Redis server. Only works server-side. 304 | 305 | ##### Redis Channel Options 306 | 307 | * *required* String `name` Set to 'redis' to use this channel 308 | * *required* String `channelName` What channel to pub/sub to 309 | * *optional* String `connectionString` The redis URL to connect to 310 | 311 | #### Redis Channel Example 312 | 313 | ```js 314 | gaggle({ 315 | id: uuid.v4() 316 | , clusterSize: 5 317 | , channel: { 318 | name: 'redis' 319 | 320 | // required, the channel to pub/sub to 321 | , channelName: 'foobar' 322 | // optional, defaults to redis's defaults 323 | , connectionString: 'redis://user:password@127.0.0.1:1234' 324 | } 325 | }) 326 | ``` 327 | 328 | #### Memory 329 | 330 | Useful for testing, only works in the same process. 331 | 332 | ##### Memory Channel Options 333 | 334 | * *required* String `name` Set to 'memory' to use this channel 335 | 336 | #### Memory Channel Example 337 | 338 | ```js 339 | gaggle({ 340 | id: uuid.v4() 341 | , clusterSize: 5 342 | , channel: { 343 | name: 'memory' 344 | } 345 | }) 346 | ``` 347 | 348 | ## Running Tests 349 | 350 | ```sh 351 | # You need to install Redis to run the tests! 352 | brew install redis 353 | 354 | # To have launchd start redis now and restart at login: 355 | brew services start redis 356 | # Or, if you don't want/need a background service you can just run: 357 | redis-server /usr/local/etc/redis.confV 358 | 359 | npm test 360 | ``` 361 | 362 | ## License 363 | 364 | Copyright (c) 2015 Ben Ng 365 | 366 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 367 | 368 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 369 | 370 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 371 | -------------------------------------------------------------------------------- /lib/gaggle.js: -------------------------------------------------------------------------------- 1 | var Joi = require('joi') 2 | , util = require('util') 3 | , Promise = require('bluebird') 4 | , _ = require('lodash') 5 | , EventEmitter = require('events').EventEmitter 6 | , uuid = require('uuid') 7 | , once = require('once') 8 | , prettifyJoiError = require('../helpers/prettify-joi-error') 9 | , STATES = { 10 | CANDIDATE: 'CANDIDATE' 11 | , LEADER: 'LEADER' 12 | , FOLLOWER: 'FOLLOWER' 13 | } 14 | , RPC_TYPE = { 15 | REQUEST_VOTE: 'REQUEST_VOTE' 16 | , REQUEST_VOTE_REPLY: 'REQUEST_VOTE_REPLY' 17 | , APPEND_ENTRIES: 'APPEND_ENTRIES' 18 | , APPEND_ENTRIES_REPLY: 'APPEND_ENTRIES_REPLY' 19 | , APPEND_ENTRY: 'APPEND_ENTRY' 20 | , DISPATCH: 'DISPATCH' 21 | , DISPATCH_SUCCESS: 'DISPATCH_SUCCESS' 22 | , DISPATCH_ERROR: 'DISPATCH_ERROR' 23 | } 24 | 25 | Promise.config({ 26 | warnings: {wForgottenReturn: false} 27 | }) 28 | 29 | function Gaggle (opts) { 30 | var self = this 31 | , validatedOptions = Joi.validate(opts, Joi.object().keys({ 32 | clusterSize: Joi.number().min(1) 33 | , channel: Joi.object() 34 | , id: Joi.string() 35 | , electionTimeout: Joi.object().keys({ 36 | min: Joi.number().min(0) 37 | , max: Joi.number().min(Joi.ref('min')) 38 | }).default({min: 300, max: 500}) 39 | , heartbeatInterval: Joi.number().min(0).default(50) 40 | , accelerateHeartbeats: Joi.boolean().default(false) 41 | , rpc: Joi.object().default() 42 | }).requiredKeys('id', 'channel', 'clusterSize'), { 43 | convert: false 44 | }) 45 | , electMin 46 | , electMax 47 | , heartbeatInterval 48 | 49 | if (validatedOptions.error != null) { 50 | throw new Error(prettifyJoiError(validatedOptions.error)) 51 | } 52 | 53 | // For convenience 54 | electMin = validatedOptions.value.electionTimeout.min 55 | electMax = validatedOptions.value.electionTimeout.max 56 | this._clusterSize = validatedOptions.value.clusterSize 57 | this._unlockTimeout = validatedOptions.value.unlockTimeout 58 | this._rpc = validatedOptions.value.rpc 59 | heartbeatInterval = validatedOptions.value.heartbeatInterval 60 | 61 | this.id = validatedOptions.value.id 62 | this._closed = false 63 | 64 | opts.channel.connect() 65 | this._channel = opts.channel 66 | 67 | // Used for internal communication, such as when an entry is committed 68 | this._emitter = new EventEmitter() 69 | this._emitter.setMaxListeners(100) 70 | 71 | // Volatile state on all servers 72 | 73 | // "When servers start up, they begin as followers" 74 | // p16 75 | this._state = STATES.FOLLOWER 76 | this._leader = null 77 | 78 | this._currentTerm = 0 79 | this._votedFor = null 80 | this._log = [] // [{term: 1, data: {}}, ...] 81 | this._commitIndex = -1 82 | this._lastApplied = -1 83 | 84 | // This is the "state machine" that log entries will be applied to 85 | // It's just a map in this format: 86 | // { 87 | // key_a: {nonce: 'foo', ttl: 1234}, 88 | // key_b: {nonce: 'foo', ttl: 1234}, 89 | // ... 90 | // } 91 | this._lockMap = {} 92 | 93 | // Proxy these internal events to the outside world 94 | this._emitter.on('appended', function () { 95 | self.emit.apply(self, ['appended'].concat(Array.prototype.slice.call(arguments))) 96 | }) 97 | 98 | this._emitter.on('committed', function () { 99 | self.emit.apply(self, ['committed'].concat(Array.prototype.slice.call(arguments))) 100 | }) 101 | 102 | this._emitter.on('leaderElected', function () { 103 | self.emit.apply(self, ['leaderElected'].concat(Array.prototype.slice.call(arguments))) 104 | }) 105 | 106 | // Volatile state on leaders 107 | this._nextIndex = {} 108 | this._matchIndex = {} 109 | 110 | // Volatile state on candidates 111 | this._votes = {} 112 | 113 | // "If a follower recieves no communication over a period of time called the election timeout 114 | // then it assumes there is no viable leader and begins an election to choose a new leader" 115 | // p16 116 | this._lastCommunicationTimestamp = Date.now() 117 | 118 | this._generateRandomElectionTimeout = function _generateRandomElectionTimeout () { 119 | return _.random(electMin, electMax, false) // no floating points 120 | } 121 | 122 | this._beginHeartbeat = function _beginHeartbeat () { 123 | var sendHeartbeat 124 | 125 | // Send initial blank entry. Don't broadcast this, as we 126 | // do NOT want to recieve our own append entry message... 127 | _.each(self._votes, function (v, nodeId) { 128 | self._channel.send(nodeId, { 129 | type: RPC_TYPE.APPEND_ENTRIES 130 | , term: self._currentTerm 131 | , leaderId: self.id 132 | , prevLogIndex: -1 133 | , prevLogTerm: -1 134 | , entries: [] 135 | , leaderCommit: self._commitIndex 136 | }) 137 | }) 138 | 139 | clearInterval(self._leaderHeartbeatInterval) 140 | 141 | sendHeartbeat = function sendHeartbeat () { 142 | _.each(self._votes, function (v, nodeId) { 143 | var entriesToSend = [] 144 | , prevLogIndex = -1 145 | , prevLogTerm = -1 146 | 147 | // Initialize to leader last log index + 1 if empty 148 | // (Reset after each election) 149 | if (self._nextIndex[nodeId] == null) { 150 | self._nextIndex[nodeId] = self._log.length 151 | } 152 | 153 | for (var i=self._nextIndex[nodeId], ii=self._log.length; i 0) { 158 | prevLogIndex = entriesToSend[0].index - 1 159 | 160 | if (prevLogIndex > -1) { 161 | prevLogTerm = self._log[prevLogIndex].term 162 | } 163 | } 164 | 165 | self._channel.send(nodeId, { 166 | type: RPC_TYPE.APPEND_ENTRIES 167 | , term: self._currentTerm 168 | , leaderId: self.id 169 | , prevLogIndex: prevLogIndex 170 | , prevLogTerm: prevLogTerm 171 | , entries: entriesToSend 172 | , leaderCommit: self._commitIndex 173 | }) 174 | }) 175 | } 176 | 177 | self._leaderHeartbeatInterval = setInterval(sendHeartbeat, heartbeatInterval) 178 | 179 | if (validatedOptions.value.accelerateHeartbeats) { 180 | self._forceHeartbeat = function _forceHeartbeat () { 181 | clearInterval(self._leaderHeartbeatInterval) 182 | sendHeartbeat() 183 | self._leaderHeartbeatInterval = setInterval(sendHeartbeat, heartbeatInterval) 184 | } 185 | } 186 | } 187 | 188 | this._onMessageRecieved = _.bind(this._onMessageRecieved, this) 189 | this._channel.on('recieved', this._onMessageRecieved) 190 | 191 | this._emitter.on('dirty', function () { 192 | if (typeof self._forceHeartbeat === 'function') { 193 | self._forceHeartbeat() 194 | } 195 | }) 196 | 197 | this._resetElectionTimeout() 198 | } 199 | 200 | util.inherits(Gaggle, EventEmitter) 201 | 202 | Gaggle.prototype._onMessageRecieved = function _onMessageRecieved (originNodeId, data) { 203 | var self = this 204 | , i 205 | , ii 206 | 207 | self._resetElectionTimeout() 208 | 209 | self._handleMessage(originNodeId, data) 210 | 211 | // If we are the leader, must try to increase our commitIndex here, so that 212 | // followers will find out about it and increment their own commitIndex 213 | if (self._state === STATES.LEADER) { 214 | // After handling any message, check to see if we can increment our commitIndex 215 | var highestPossibleCommitIndex = -1 216 | 217 | for (i=self._commitIndex + 1, ii=self._log.length; i= i 221 | _.filter(self._matchIndex, function (matchIndex) { 222 | return matchIndex >= i 223 | }).length > Math.ceil(self._clusterSize/2)) { 224 | highestPossibleCommitIndex = i 225 | } 226 | } 227 | 228 | if (highestPossibleCommitIndex > self._commitIndex) { 229 | self._commitIndex = highestPossibleCommitIndex 230 | 231 | self._emitter.emit('dirty') 232 | } 233 | } 234 | 235 | // All nodes should commit entries between lastApplied and commitIndex 236 | for (i=self._lastApplied + 1, ii=self._commitIndex; i<=ii; ++i) { 237 | self._emitter.emit('committed', JSON.parse(JSON.stringify(self._log[i])), i) 238 | self._lastApplied = i 239 | } 240 | } 241 | 242 | Gaggle.prototype.dispatchOnLeader = function dispatchOnLeader () { 243 | var args = Array.prototype.slice.call(arguments) 244 | return this._dispatchOnLeader.apply(this, [uuid.v4()].concat(args)) 245 | } 246 | 247 | Gaggle.prototype._dispatchOnLeader = function _dispatchOnLeader (rpcId, methodName, args, timeout, cb) { 248 | var self = this 249 | , performRequest 250 | , p 251 | 252 | if (typeof timeout === 'function') { 253 | cb = timeout 254 | timeout = -1 255 | } 256 | 257 | timeout = typeof timeout === 'number' ? timeout : -1 258 | 259 | /** 260 | * This odd pattern is because there is a possibility that 261 | * we were elected the leader after the leaderElected event 262 | * fires. So we wait until its time to perform the request 263 | * to decide if we need to delegate to the leader, or perform 264 | * the logic ourselves 265 | */ 266 | performRequest = once(function performRequest () { 267 | if (self._state === STATES.LEADER) { 268 | 269 | if (self._rpc[methodName] == null) { 270 | self._channel.broadcast({ 271 | type: RPC_TYPE.DISPATCH_ERROR 272 | , term: self._currentTerm 273 | , id: rpcId 274 | , message: 'The RPC method ' + methodName + ' does not exist' 275 | , stack: 'Gaggle.prototype._dispatchOnLeader' 276 | }) 277 | } 278 | else { 279 | var usedDeprecatedAPI = false 280 | , result = self._rpc[methodName].apply(self, args.concat([function () { 281 | usedDeprecatedAPI = true 282 | }])) 283 | 284 | if (usedDeprecatedAPI) { 285 | result = new Error('As of version 3, RPC calls are synchronous, and should return a value rather than use this callback. This warning will be removed in a future version.') 286 | } 287 | 288 | if (result instanceof Error) { 289 | self._channel.broadcast({ 290 | type: RPC_TYPE.DISPATCH_ERROR 291 | , term: self._currentTerm 292 | , id: rpcId 293 | , message: result.message 294 | , stack: result.stack 295 | }) 296 | } 297 | else { 298 | self._channel.broadcast({ 299 | type: RPC_TYPE.DISPATCH_SUCCESS 300 | , term: self._currentTerm 301 | , id: rpcId 302 | , returnValue: result 303 | }) 304 | } 305 | } 306 | } 307 | else { 308 | self._channel.send(self._leader, { 309 | type: RPC_TYPE.DISPATCH 310 | , term: self._currentTerm 311 | , id: rpcId 312 | , methodName: methodName 313 | , args: args 314 | }) 315 | } 316 | }) 317 | 318 | if (self._state === STATES.LEADER) { 319 | performRequest() 320 | } 321 | else if (self._state === STATES.FOLLOWER && self._leader != null) { 322 | performRequest() 323 | } 324 | else { 325 | self._emitter.once('leaderElected', performRequest) 326 | } 327 | 328 | p = new Promise(function (resolve, reject) { 329 | var resolveOnAck = function _resolveOnAck (id, ret) { 330 | if (id === rpcId) { 331 | resolve(ret) 332 | cleanup() 333 | } 334 | } 335 | , rejectOnError = function _rejectOnError(id, err) { 336 | if (id === rpcId) { 337 | reject(err) 338 | cleanup() 339 | } 340 | } 341 | , cleanup = function _cleanup () { 342 | self._emitter.removeListener('leaderElected', performRequest) 343 | self._emitter.removeListener('rpcSuccess', resolveOnAck) 344 | self._emitter.removeListener('rpcError', rejectOnError) 345 | clearTimeout(timeoutHandle) 346 | } 347 | , timeoutHandle 348 | 349 | // And wait for acknowledgement... 350 | self._emitter.on('rpcSuccess', resolveOnAck) 351 | self._emitter.on('rpcError', rejectOnError) 352 | 353 | // Or if we time out before the message is committed... 354 | if (timeout > -1) { 355 | timeoutHandle = setTimeout(function _failOnTimeout () { 356 | reject(new Error('Timed out before the rpc method returned')) 357 | cleanup() 358 | }, timeout) 359 | } 360 | }) 361 | 362 | if (cb != null) { 363 | p.then(function (args) { 364 | cb.apply(null, [null].concat(args)) 365 | }).catch(cb) 366 | } 367 | else { 368 | return p 369 | } 370 | } 371 | 372 | Gaggle.prototype.getLog = function () { 373 | return this._log 374 | } 375 | 376 | Gaggle.prototype.getCommitIndex = function () { 377 | return this._commitIndex 378 | } 379 | 380 | Gaggle.prototype.append = function append (data, timeout, cb) { 381 | var self = this 382 | , msgId = self.id + '_' + uuid.v4() 383 | , performRequest 384 | , p 385 | 386 | if (typeof timeout === 'function') { 387 | cb = timeout 388 | timeout = -1 389 | } 390 | 391 | timeout = typeof timeout === 'number' ? timeout : -1 392 | 393 | /** 394 | * This odd pattern is because there is a possibility that 395 | * we were elected the leader after the leaderElected event 396 | * fires. So we wait until its time to perform the request 397 | * to decide if we need to delegate to the leader, or perform 398 | * the logic ourselves 399 | */ 400 | performRequest = once(function performRequest () { 401 | var entry 402 | 403 | if (self._state === STATES.LEADER) { 404 | entry = { 405 | term: self._currentTerm 406 | , data: data 407 | , id: msgId 408 | } 409 | 410 | self._log.push(entry) 411 | self._emitter.emit('appended', entry, self._log.length - 1) 412 | } 413 | else { 414 | self._channel.send(self._leader, { 415 | type: RPC_TYPE.APPEND_ENTRY 416 | , data: data 417 | , id: msgId 418 | }) 419 | } 420 | }) 421 | 422 | if (self._state === STATES.LEADER) { 423 | performRequest() 424 | } 425 | else if (self._state === STATES.FOLLOWER && self._leader != null) { 426 | performRequest() 427 | } 428 | else { 429 | self._emitter.once('leaderElected', performRequest) 430 | } 431 | 432 | p = new Promise(function (resolve, reject) { 433 | var resolveOnCommitted = function _resolveOnCommitted (entry) { 434 | if (entry.id === msgId) { 435 | resolve() 436 | cleanup() 437 | } 438 | } 439 | , cleanup = function _cleanup () { 440 | self._emitter.removeListener('leaderElected', performRequest) 441 | self._emitter.removeListener('committed', resolveOnCommitted) 442 | clearTimeout(timeoutHandle) 443 | } 444 | , timeoutHandle 445 | 446 | // And wait for acknowledgement... 447 | self._emitter.on('committed', resolveOnCommitted) 448 | 449 | // Or if we time out before the message is committed... 450 | if (timeout > -1) { 451 | timeoutHandle = setTimeout(function _failOnTimeout () { 452 | reject(new Error('Timed out before the entry was committed')) 453 | cleanup() 454 | }, timeout) 455 | } 456 | }) 457 | 458 | if (cb != null) { 459 | p.then(_.bind(cb, null, null)).catch(cb) 460 | } 461 | else { 462 | return p 463 | } 464 | } 465 | 466 | Gaggle.prototype.isLeader = function isLeader () { 467 | return this._state === STATES.LEADER 468 | } 469 | 470 | Gaggle.prototype.hasUncommittedEntriesInPreviousTerms = function hasUncommittedEntriesInPreviousTerms () { 471 | var self = this 472 | 473 | return _.find(self._log, function (entry, idx) { 474 | return entry.term < self._currentTerm && idx > self._commitIndex 475 | }) != null 476 | } 477 | 478 | Gaggle.prototype._handleMessage = function _handleMessage (originNodeId, data) { 479 | 480 | var self = this 481 | , conflictedAt = -1 482 | , entry 483 | 484 | // data always has the following keys: 485 | // { 486 | // type: the RPC method call or response 487 | // term: some integer 488 | // } 489 | 490 | if (data.term > self._currentTerm) { 491 | self._currentTerm = data.term 492 | self._leader = null 493 | self._votedFor = null 494 | self._state = STATES.FOLLOWER 495 | clearInterval(self._leaderHeartbeatInterval) 496 | } 497 | 498 | switch (data.type) { 499 | case RPC_TYPE.REQUEST_VOTE: 500 | // Do not combine these conditions, its intentionally written this way so that the 501 | // code coverage tool can do a thorough analysis 502 | if (data.term >= self._currentTerm) { 503 | var lastLogEntry = _.last(self._log) 504 | , lastLogTerm = lastLogEntry != null ? lastLogEntry.term : -1 505 | , candidateIsAtLeastAsUpToDate = data.lastLogTerm > lastLogTerm || // Its either in a later term... 506 | // or same term, and at least at the same index 507 | data.lastLogTerm === lastLogTerm && data.lastLogIndex >= self._log.length - 1 508 | 509 | if ((self._votedFor == null || self._votedFor === data.candidateId) && candidateIsAtLeastAsUpToDate) { 510 | self._votedFor = data.candidateId 511 | self._channel.send(data.candidateId, { 512 | type: RPC_TYPE.REQUEST_VOTE_REPLY 513 | , term: self._currentTerm 514 | , voteGranted: true 515 | }) 516 | return 517 | } 518 | } 519 | 520 | self._channel.send(originNodeId, { 521 | type: RPC_TYPE.REQUEST_VOTE_REPLY 522 | , term: self._currentTerm 523 | , voteGranted: false 524 | }) 525 | break 526 | 527 | case RPC_TYPE.REQUEST_VOTE_REPLY: 528 | // broadcasts reach ourselves, so we'll actually vote for ourselves here 529 | if (data.term === self._currentTerm && 530 | data.voteGranted === true) { 531 | 532 | // Keep collecting votes because that's how we discover what process ids are 533 | // in the system 534 | self._votes[originNodeId] = true 535 | 536 | // Stops this from happening when extra votes come in 537 | if (self._state === STATES.CANDIDATE && 538 | _.values(self._votes).length > Math.ceil(this._clusterSize/2) // Wait for a majority 539 | ) { 540 | self._state = STATES.LEADER 541 | self._nextIndex = {} 542 | self._matchIndex = {} 543 | self._beginHeartbeat() 544 | self._emitter.emit('leaderElected') 545 | } 546 | } 547 | break 548 | 549 | case RPC_TYPE.APPEND_ENTRY: 550 | if (self._state === STATES.LEADER) { 551 | entry = { 552 | term: self._currentTerm 553 | , id: data.id 554 | , data: data.data 555 | } 556 | 557 | self._log.push(entry) 558 | self._emitter.emit('appended', entry, self._log.length - 1) 559 | 560 | self._emitter.emit('dirty') 561 | } 562 | break 563 | 564 | case RPC_TYPE.APPEND_ENTRIES: 565 | 566 | // This is how you lose an election 567 | if (data.term >= self._currentTerm && self._state !== STATES.LEADER) { 568 | self._state = STATES.FOLLOWER 569 | self._leader = originNodeId 570 | self._emitter.emit('leaderElected') 571 | } 572 | 573 | // Reciever Implementation 1 & 2 574 | // p13 575 | 576 | if (data.term < self._currentTerm) { // 1. reply false if term < currentTerm (section 3.3) 577 | self._channel.send(originNodeId, { 578 | type: RPC_TYPE.APPEND_ENTRIES_REPLY 579 | , term: self._currentTerm 580 | , success: false 581 | }) 582 | return 583 | } 584 | 585 | // This could be merged into the previous if statement, but then istanbul can't detect if tests have covered this branch 586 | if (data.prevLogIndex > -1 && // Don't do this if log is empty; it'll fail for the wrong reasons 587 | (self._log[data.prevLogIndex] == null || // 2. reply false if log doesn't contain an entry at prevLogIndex 588 | self._log[data.prevLogIndex].term !== data.prevLogTerm)) { // whose term matches prevLogTerm (section 3.5) 589 | 590 | self._channel.send(originNodeId, { 591 | type: RPC_TYPE.APPEND_ENTRIES_REPLY 592 | , term: self._currentTerm 593 | , success: false 594 | }) 595 | return 596 | } 597 | 598 | // 3. If an existing entry conflicts with a new one (same index but different terms), 599 | // delete the existing entry and all that follow it. (section 3.5) 600 | // p13 601 | _.each(data.entries, function (entry) { 602 | // entry is: 603 | // {index: 0, term: 0, data: {foo: bar}} 604 | var idx = entry.index 605 | 606 | if (self._log[idx] != null && self._log[idx].term !== entry.term) { 607 | conflictedAt = idx 608 | } 609 | }) 610 | 611 | if (conflictedAt > 0) { 612 | self._log = self._log.slice(0, conflictedAt) 613 | } 614 | 615 | // 4. Append any new entries not already in the log 616 | _.each(data.entries, function (entry) { 617 | var idx = entry.index 618 | 619 | if (self._log[idx] == null) { 620 | self._log[idx] = { 621 | term: entry.term 622 | , data: entry.data 623 | , id: entry.id 624 | } 625 | } 626 | }) 627 | 628 | // 5. If leaderCommit > commitIndex, set commitIndex = min(leaderCommit, index of last new entry) 629 | if (data.leaderCommit > self._commitIndex) { 630 | self._commitIndex = Math.min(data.leaderCommit, self._log.length - 1) 631 | } 632 | 633 | self._channel.send(originNodeId, { 634 | type: RPC_TYPE.APPEND_ENTRIES_REPLY 635 | , term: self._currentTerm 636 | , success: true 637 | , lastLogIndex: self._log.length - 1 638 | }) 639 | 640 | break 641 | 642 | case RPC_TYPE.APPEND_ENTRIES_REPLY: 643 | if (self._state === STATES.LEADER && self._currentTerm === data.term) { 644 | if (data.success === true && data.lastLogIndex > -1) { 645 | self._nextIndex[originNodeId] = data.lastLogIndex + 1 646 | self._matchIndex[originNodeId] = data.lastLogIndex 647 | 648 | // Find out what the highest commited entry is 649 | } 650 | else { 651 | if (self._nextIndex[originNodeId] == null) { 652 | self._nextIndex[originNodeId] = self._log.length 653 | } 654 | 655 | self._nextIndex[originNodeId] = Math.max(self._nextIndex[originNodeId] - 1, 0) 656 | } 657 | } 658 | break 659 | 660 | case RPC_TYPE.DISPATCH: 661 | if (self._state === STATES.LEADER && self._currentTerm === data.term) { 662 | self._dispatchOnLeader(data.id, data.methodName, data.args).catch(_.noop) 663 | } 664 | break 665 | 666 | case RPC_TYPE.DISPATCH_ERROR: 667 | var err = new Error(data.message) 668 | err.stack = data.stack 669 | self._emitter.emit.apply(self._emitter, ['rpcError', data.id, err]) 670 | break 671 | 672 | case RPC_TYPE.DISPATCH_SUCCESS: 673 | self._emitter.emit.apply(self._emitter, ['rpcSuccess', data.id, data.returnValue]) 674 | break 675 | } 676 | } 677 | 678 | Gaggle.prototype._resetElectionTimeout = function _resetElectionTimeout () { 679 | var self = this 680 | , timeout = self._generateRandomElectionTimeout() 681 | 682 | if (self._electionTimeout != null) { 683 | clearTimeout(self._electionTimeout) 684 | } 685 | 686 | self._electionTimeout = setTimeout(function () { 687 | self._resetElectionTimeout() 688 | self._beginElection() 689 | }, timeout) 690 | } 691 | 692 | Gaggle.prototype._beginElection = function _beginElection () { 693 | // To begin an election, a follower increments its current term and transitions to 694 | // candidate state. It then votes for itself and issues RequestVote RPCs in parallel 695 | // to each of the other servers in the cluster. 696 | // p16 697 | var lastLogIndex 698 | 699 | this._currentTerm = this._currentTerm + 1 700 | this._state = STATES.CANDIDATE 701 | this._leader = null 702 | this._votedFor = this.id 703 | this._votes = {} 704 | 705 | lastLogIndex = this._log.length - 1 706 | 707 | this._channel.broadcast({ 708 | type: RPC_TYPE.REQUEST_VOTE 709 | , term: this._currentTerm 710 | , candidateId: this.id 711 | , lastLogIndex: lastLogIndex 712 | , lastLogTerm: lastLogIndex > -1 ? this._log[lastLogIndex].term : -1 713 | }) 714 | } 715 | 716 | Gaggle.prototype.close = function close (cb) { 717 | var self = this 718 | , p 719 | 720 | this._channel.removeListener('recieved', this._onMessageRecieved) 721 | clearTimeout(this._electionTimeout) 722 | clearInterval(this._leaderHeartbeatInterval) 723 | 724 | this._emitter.removeAllListeners() 725 | 726 | p = new Promise(function (resolve, reject) { 727 | self._channel.once('disconnected', function () { 728 | resolve() 729 | }) 730 | self._channel.disconnect() 731 | }) 732 | 733 | if (cb != null) { 734 | p.then(_.bind(cb, null, null)).catch(cb) 735 | } 736 | else { 737 | return p 738 | } 739 | } 740 | 741 | module.exports = Gaggle 742 | module.exports._STATES = _.cloneDeep(STATES) 743 | module.exports._RPC_TYPE = _.cloneDeep(RPC_TYPE) 744 | 745 | -------------------------------------------------------------------------------- /test/raft.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Test leader election and log replication 3 | */ 4 | 5 | var t = require('tap') 6 | , async = require('async') 7 | , uuid = require('uuid') 8 | , _ = require('lodash') 9 | , Promise = require('bluebird') 10 | , gaggle = require('../') 11 | , createCluster 12 | , createClusterWithLeader 13 | 14 | createClusterWithLeader = function (opts, cb) { 15 | var POLLING_INTERVAL = 100 16 | , CONSENSUS_TIMEOUT = 10000 17 | , testStart = Date.now() 18 | , cluster 19 | , hasReachedLeaderConsensus 20 | 21 | opts = _.defaults(opts, { 22 | rpc: {} 23 | , accelerate: false 24 | }) 25 | 26 | cluster = createCluster(opts) 27 | 28 | hasReachedLeaderConsensus = function hasReachedLeaderConsensus () { 29 | var maxTerm = Math.max.apply(null, _.map(cluster, '_currentTerm')) 30 | , leaders = _(cluster).filter(function (node) { 31 | return node._currentTerm === maxTerm 32 | }).map('_leader').compact().valueOf() 33 | , followerCount = _.filter(cluster, function (node) { 34 | return node._currentTerm === maxTerm && node._state === gaggle._STATES.FOLLOWER 35 | }).length 36 | 37 | if (leaders.length === cluster.length - 1 && 38 | _.uniq(leaders).length === 1 && 39 | followerCount === cluster.length - 1) { 40 | return leaders[0] 41 | } 42 | else { 43 | return false 44 | } 45 | } 46 | 47 | async.whilst(function () { 48 | return !hasReachedLeaderConsensus() && Date.now() - testStart < CONSENSUS_TIMEOUT 49 | }, function (next) { 50 | setTimeout(next, POLLING_INTERVAL) 51 | }, function () { 52 | var leaderId = hasReachedLeaderConsensus() 53 | , leader = _.find(cluster, function (node) { 54 | return node.id === leaderId 55 | }) 56 | 57 | if (leader != null) { 58 | cb(null, cluster, leader, function cleanup () { 59 | return Promise.map(cluster, function (node) { 60 | return node.close() 61 | }) 62 | }) 63 | } 64 | else { 65 | cb(new Error('the cluster did not elect a leader in time')) 66 | } 67 | }) 68 | } 69 | 70 | createCluster = function createCluster (opts) { 71 | var cluster = [] 72 | 73 | for (var i=0; i