├── .gitignore ├── README.md ├── example.js ├── generate_commands.js ├── index.js ├── lib └── commands.js ├── package.json └── redisClusterSlot.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | test 3 | .c9revisions 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # node_redis_cluster 2 | ## A thin layer over node_redis to handle Redis Clusters 3 | 4 | 5 | Redis Cluster is coming out later this year, but I can't wait for it so I made this module. 6 | 7 | All it does is connect to the nodes of a Redis Cluster, and before sending any commands it checks in which slot the key is with `HASH_SLOT = CRC16(key) mod 16384` and then sends the command to the node that has that slot. 8 | 9 | # Installation 10 | 11 | npm install redis-cluster 12 | 13 | # Usage 14 | 15 | This module exports two objects. `clusterClient` is to be used with a regular Redis Cluster, you just need to supply a link (like `127.0.0.1:6379`) and the other members of the cluster will be found, after that you can use it pretty much like the original `node_redis` module: 16 | 17 | ```javascript 18 | var RedisCluster = require('redis-cluster').clusterClient; 19 | var redis = RedisCluster; 20 | var redisPubSub = RedisCluster; 21 | var assert = require('assert'); 22 | 23 | var firstLink = '127.0.0.1:6379'; // Used to discover the rest of the cluster 24 | new redis.clusterInstance(firstLink, function (err, r) { 25 | if (err) throw err; 26 | r.set('foo', 'bar', function (err, reply) { 27 | if (err) throw err; 28 | assert.equal(reply,'OK'); 29 | 30 | r.get('foo', function (err, reply) { 31 | if (err) throw err; 32 | assert.equal(reply, 'bar'); 33 | }); 34 | }); 35 | }); 36 | 37 | new redisPubSub.clusterInstance(firstLink, function (err, r) { 38 | r.subscribe('channel'); 39 | 40 | for( var link in redisPubSub.redisLinks ) 41 | { 42 | redisPubSub.redisLinks[link].link.on('message', function (channel, message) { 43 | // New message in a channel, necessarily 'channel' here because it's the only one we're subscribed to. 44 | }); 45 | } 46 | }); 47 | ``` 48 | 49 | Don't forget that despite being a thin wrapper above `node_redis`, you still can't use all the commands you would use against a normal Redis server. For instance, don't expect the `KEYS` command to work (in fact, in the [Redis Cluster spec](http://redis.io/topics/cluster-spec) it says that "all the operations where in theory keys are not available in the same node are not implemented"). 50 | 51 | # But Redis Cluster is unstable! 52 | 53 | If you really want to have a cluster of Redis nodes but don't want to run unstable software, you can always use the **Poor Man's Cluster Client**, also supplied by this module. 54 | 55 | This time you can't supply a link of one node of the cluster (maybe because it's not a real cluster), you have to supply the links to all the nodes, like this: 56 | 57 | ```javascript 58 | var RedisCluster = require('redis-cluster').poorMansClusterClient; 59 | var assert = require('assert'); 60 | 61 | var cluster = [ 62 | {name: 'redis01', link: '127.0.0.1:6379', slots: [ 0, 5462], options: {max_attempts: 5}}, 63 | {name: 'redis02', link: '127.0.0.1:7379', slots: [5463, 12742], options: {max_attempts: 5}}, 64 | {name: 'redis03', link: '127.0.0.1:8379', slots: [12743, 16384], options: {max_attempts: 5}} 65 | ]; 66 | 67 | var r = poorMansClusterClient(cluster); 68 | 69 | r.set('foo', 'bar', function (err, reply) { 70 | if (err) throw err; 71 | assert.equal(reply,'OK'); 72 | 73 | r.get('foo', function (err, reply) { 74 | if (err) throw err; 75 | assert.equal(reply, 'bar'); 76 | }); 77 | }); 78 | ``` 79 | As you noticed, you must specify the interval of slots allocated to each node. All 16384 slots must be covered, otherwise you will run in some nasty errors (some keys might have no where to go). 80 | 81 | Options are optional and may be added or left out. All valid options for the redis client may be found in the redis client documentation. 82 | 83 | If you decide to re-allocate the slots, add or remove a node, you must move all the affected keys yourself. The [MIGRATE](http://redis.io/commands/migrate) command might help you with that. 84 | 85 | # Notes on performance 86 | 87 | Before every operation, a CRC16 of the key gets computed, so we can know in which node of the cluster this key is. It turns out it's not such an expensive operation to run for every command after all. My laptop can hash 2793296.089 strings of 32 characters per second, that will in no way be a bottleneck to all database operations. 88 | 89 | Some quick test showed it achieved a very similar performance to the `node_redis` module, so I'll assume it's not so bad and as soon as I have time I'll publish some tests 90 | 91 | # Other notes 92 | 93 | This is of course not intended for production and has probably stupid (not bad, stupid) code inside, but I just needed something that works as there are no modules to work with Redis Clusters yet. 94 | 95 | # Credits 96 | 97 | This module shamelessly borrows some code from the mranney's `node_redis` module and alexgorbatchev's `node-crc`, although I didn't feel the need to include it as a dependency because only CRC16 is needed. -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | var rcluster = require('./index.js').clusterClient; 2 | 3 | new rcluster.clusterInstance('127.0.0.1:7001', function (err, r) { 4 | if (err) throw err; 5 | 6 | r.hmset('hset:1', {a:1,b:2,c:'hello'}, function(e,d){ 7 | console.log(e,d); 8 | }); 9 | 10 | function doIt() { 11 | r.set('foo', 'bar', function (err, reply) { 12 | if (err) throw err; 13 | 14 | r.get('foo', function (err, reply) { 15 | if (err) throw err; 16 | console.log(err, reply); 17 | }); 18 | }); 19 | 20 | r.hgetall('hset:1', function(e, d){ 21 | console.log(e,d); 22 | }); 23 | 24 | try { 25 | console.log('hmget'); 26 | r.hmget('hset:1', 'a', 'b', 'f', function(e, d){ 27 | console.log('hmget',e,d); 28 | }); 29 | } catch(e) { 30 | console.log('exception', e, e.stack) 31 | } 32 | 33 | setTimeout(doIt, 5000); 34 | } 35 | doIt(); 36 | }); -------------------------------------------------------------------------------- /generate_commands.js: -------------------------------------------------------------------------------- 1 | var http = require("http"), 2 | fs = require("fs"); 3 | 4 | function write_file(commands, path) { 5 | 6 | console.log("Writing " + Object.keys(commands).length + " commands to " + path); 7 | 8 | var file_contents = "// This file was generated by ./generate_commands.js on " + (new Date()).toLocaleString() + "\n"; 9 | 10 | var out_commands = Object.keys(commands).map(function (key) { 11 | return key.toLowerCase(); 12 | }); 13 | 14 | file_contents += "module.exports = " + JSON.stringify(out_commands, null, " ") + ";\n"; 15 | 16 | fs.writeFile(path, file_contents); 17 | } 18 | 19 | http.get({host: "redis.io", path: "/commands.json"}, function (res) { 20 | var body = ""; 21 | 22 | console.log("Response from redis.io/commands.json: " + res.statusCode); 23 | 24 | res.on('data', function (chunk) { 25 | body += chunk; 26 | }); 27 | 28 | res.on('end', function () { 29 | write_file(JSON.parse(body), "lib/commands.js"); 30 | }); 31 | }).on('error', function (e) { 32 | console.log("Error fetching command list from redis.io: " + e.message); 33 | }); -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var redis = require('redis'); 2 | var events = require('events'); 3 | var fastRedis = null; 4 | try { 5 | fastRedis = require('redis-fast-driver'); 6 | } catch(e) {} 7 | 8 | var redisClusterSlot = require('./redisClusterSlot'); 9 | var commands = require('./lib/commands'); 10 | 11 | var connectToLink = function(str, auth, options) { 12 | var spl = str.split(':'); 13 | options = options || {}; 14 | if (auth) { 15 | if(fastRedis) { 16 | var link =new fastRedis({ 17 | host: spl[0], 18 | port: spl[1], 19 | auth: auth 20 | }); 21 | link.on('error', function onErrorFromRedisDriver(err){ 22 | console.log('error from redis driver %s:', str, err); 23 | }); 24 | return link; 25 | } 26 | return (redis.createClient(spl[1], spl[0], options).auth(auth)); 27 | } else { 28 | if(fastRedis) { 29 | var link =new fastRedis({ 30 | host: spl[0], 31 | port: spl[1] 32 | }); 33 | link.on('error', function onErrorFromRedisDriver(err){ 34 | console.log('error from redis driver %s:', str, err); 35 | }); 36 | return link; 37 | } 38 | return (redis.createClient(spl[1], spl[0], options)); 39 | } 40 | }; 41 | 42 | /* 43 | Connect to a node of a Redis Cluster, discover the other nodes and 44 | respective slots with the "CLUSTER NODES" command, connect to them 45 | and return an array of the links to all the nodes in the cluster. 46 | */ 47 | function connectToNodesOfCluster (firstLink, callback) { 48 | var redisLinks = []; 49 | var fireStarter = connectToLink(firstLink); 50 | var clusterFn = fastRedis ? function(subcommand, cb) { 51 | fireStarter.rawCall(['cluster', subcommand], cb); 52 | } : fireStarter.cluster.bind(fireStarter); 53 | clusterFn('nodes', function(err, nodes) { 54 | if(err && err.indexOf('cluster support disabled') !== -1) { 55 | err = null; 56 | var port = firstLink.split(':').pop(); 57 | nodes = '0000000000000000000000000000000000000000 :'+port+' myself,master - 0 0 1 connected 0-16383\n'; 58 | } else if (err) { 59 | callback(err, null); 60 | return; 61 | } 62 | var lines = nodes.split('\n'); 63 | var n = lines.length -1; 64 | while (n--) { 65 | var items = lines[n].split(' '); 66 | var name = items[0]; 67 | var flags = items[2]; 68 | var link = ( flags === 'myself' || flags === 'myself,master' || flags === 'myself,slave') ? firstLink : items[1]; 69 | if(flags === 'slave' || flags === 'myself,slave') { 70 | if (n === 0) { 71 | callback(err, redisLinks); 72 | return; 73 | } 74 | continue; 75 | } 76 | //var lastPingSent = items[4]; 77 | //var lastPongReceived = items[5]; 78 | var linkState = items[7]; 79 | 80 | if (lines.length === 1 && lines[1] === '') { 81 | var slots = [0, 16383] 82 | } else { 83 | var slots = []; 84 | for(var i = 8; i-') !== -1) { 86 | //migrate in process... 87 | continue; 88 | } 89 | if(items[i].indexOf('-') === -1) { 90 | slots.push(items[i], items[i]); 91 | continue; 92 | } 93 | var t = items[i].split('-'); 94 | slots.push(t[0], t[1]); 95 | } 96 | } 97 | 98 | if (linkState === 'connected') { 99 | redisLinks.push({ 100 | name: name, 101 | connectStr: link, 102 | link: connectToLink(link), 103 | slots: slots 104 | }); 105 | } 106 | if (n === 0) { 107 | callback(err, redisLinks); 108 | } 109 | } 110 | }); 111 | } 112 | 113 | /* 114 | Connect to all the nodes that form a cluster. Takes an array in the form of 115 | [ 116 | {name: "node1", link: "127.0.0.1:6379", slots: [0, 8192], auth: foobared}, 117 | {name: "node2", link: "127.0.0.1:7379", slots: [8193, 16384], auth:foobared}, 118 | ] 119 | 120 | *auth is optional 121 | 122 | You decide the allocation of the 4096 slots, but they must be all covered, and 123 | if you decide to add/remove a node from the "cluster", don't forget to MIGRATE 124 | the keys accordingly to the new slots allocation. 125 | 126 | */ 127 | function connectToNodes (cluster) { 128 | var redisLinks = []; 129 | var n = cluster.length; 130 | while (n--) { 131 | var node = cluster[n]; 132 | var options = node.options || {}; 133 | redisLinks.push({ 134 | name: node.name, 135 | link: connectToLink(node.link, node.auth, options), 136 | slots: node.slots 137 | }); 138 | } 139 | return (redisLinks); 140 | } 141 | 142 | function bindCommands (nodes, oldClient) { 143 | var client = oldClient || new events.EventEmitter(); 144 | client.nodes = nodes; 145 | //catch on error from nodes 146 | function onError(err) { 147 | console.log('got error from ', this); 148 | client.emit('error', err); 149 | } 150 | for(var i=0;i 5) { 204 | if(o_callback) 205 | o_callback('Too much redirections'); 206 | return; 207 | } 208 | //console.log('ASK redirection') 209 | var connectStr = e.split(' ')[2]; 210 | var node = null; 211 | for(var i=0;i= slots[r]) && (slot <= slots[r+1])) { 266 | callNode(node); 267 | return; 268 | } 269 | } 270 | } 271 | 272 | throw new Error('slot '+slot+' found on no nodes'); 273 | 274 | function callNode(node, argumentsAlreadyFixed) { 275 | // console.log('callNode',node); 276 | lastusednode = node; 277 | if(fastRedis) { 278 | if(!argumentsAlreadyFixed) o_arguments.unshift(command); 279 | if(command === 'hgetall') { 280 | node.link.rawCall(o_arguments, function(e, d){ 281 | if(e) return callback(e); 282 | if(!Array.isArray(d) || d.length < 1) 283 | return callback(e, d); 284 | var obj = {}; 285 | for(var i=0;i", 5 | "contributors": ["Arseniy Pavlenko "], 6 | "main": "./index.js", 7 | "dependencies" : { 8 | "redis" : "0.8.x" 9 | }, 10 | "optionalDependencies": { 11 | "redis-fast-driver": "*" 12 | }, 13 | "repository": { 14 | "type": "git", 15 | "url": "git@github.com:joaojeronimo/node_redis_cluster.git" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /redisClusterSlot.js: -------------------------------------------------------------------------------- 1 | // Source: http://redis.io/topics/cluster-spec 2 | 3 | /* CRC16 implementation according to CCITT standards. 4 | * 5 | * Note by @antirez: this is actually the XMODEM CRC 16 algorithm, using the 6 | * following parameters: 7 | * 8 | * Name : "XMODEM", also known as "ZMODEM", "CRC-16/ACORN" 9 | * Width : 16 bit 10 | * Poly : 1021 (That is actually x^16 + x^12 + x^5 + 1) 11 | * Initialization : 0000 12 | * Reflect Input byte : False 13 | * Reflect Output CRC : False 14 | * Xor constant to output CRC : 0000 15 | * Output for "123456789" : 31C3 16 | */ 17 | 18 | var CRC16_TAB = [ 19 | 0x0000,0x1021,0x2042,0x3063,0x4084,0x50A5,0x60C6,0x70E7, 20 | 0x8108,0x9129,0xA14A,0xB16B,0xC18C,0xD1AD,0xE1CE,0xF1EF, 21 | 0x1231,0x0210,0x3273,0x2252,0x52B5,0x4294,0x72F7,0x62D6, 22 | 0x9339,0x8318,0xB37B,0xA35A,0xD3BD,0xC39C,0xF3FF,0xE3DE, 23 | 0x2462,0x3443,0x0420,0x1401,0x64E6,0x74C7,0x44A4,0x5485, 24 | 0xA56A,0xB54B,0x8528,0x9509,0xE5EE,0xF5CF,0xC5AC,0xD58D, 25 | 0x3653,0x2672,0x1611,0x0630,0x76D7,0x66F6,0x5695,0x46B4, 26 | 0xB75B,0xA77A,0x9719,0x8738,0xF7DF,0xE7FE,0xD79D,0xC7BC, 27 | 0x48C4,0x58E5,0x6886,0x78A7,0x0840,0x1861,0x2802,0x3823, 28 | 0xC9CC,0xD9ED,0xE98E,0xF9AF,0x8948,0x9969,0xA90A,0xB92B, 29 | 0x5AF5,0x4AD4,0x7AB7,0x6A96,0x1A71,0x0A50,0x3A33,0x2A12, 30 | 0xDBFD,0xCBDC,0xFBBF,0xEB9E,0x9B79,0x8B58,0xBB3B,0xAB1A, 31 | 0x6CA6,0x7C87,0x4CE4,0x5CC5,0x2C22,0x3C03,0x0C60,0x1C41, 32 | 0xEDAE,0xFD8F,0xCDEC,0xDDCD,0xAD2A,0xBD0B,0x8D68,0x9D49, 33 | 0x7E97,0x6EB6,0x5ED5,0x4EF4,0x3E13,0x2E32,0x1E51,0x0E70, 34 | 0xFF9F,0xEFBE,0xDFDD,0xCFFC,0xBF1B,0xAF3A,0x9F59,0x8F78, 35 | 0x9188,0x81A9,0xB1CA,0xA1EB,0xD10C,0xC12D,0xF14E,0xE16F, 36 | 0x1080,0x00A1,0x30C2,0x20E3,0x5004,0x4025,0x7046,0x6067, 37 | 0x83B9,0x9398,0xA3FB,0xB3DA,0xC33D,0xD31C,0xE37F,0xF35E, 38 | 0x02B1,0x1290,0x22F3,0x32D2,0x4235,0x5214,0x6277,0x7256, 39 | 0xB5EA,0xA5CB,0x95A8,0x8589,0xF56E,0xE54F,0xD52C,0xC50D, 40 | 0x34E2,0x24C3,0x14A0,0x0481,0x7466,0x6447,0x5424,0x4405, 41 | 0xA7DB,0xB7FA,0x8799,0x97B8,0xE75F,0xF77E,0xC71D,0xD73C, 42 | 0x26D3,0x36F2,0x0691,0x16B0,0x6657,0x7676,0x4615,0x5634, 43 | 0xD94C,0xC96D,0xF90E,0xE92F,0x99C8,0x89E9,0xB98A,0xA9AB, 44 | 0x5844,0x4865,0x7806,0x6827,0x18C0,0x08E1,0x3882,0x28A3, 45 | 0xCB7D,0xDB5C,0xEB3F,0xFB1E,0x8BF9,0x9BD8,0xABBB,0xBB9A, 46 | 0x4A75,0x5A54,0x6A37,0x7A16,0x0AF1,0x1AD0,0x2AB3,0x3A92, 47 | 0xFD2E,0xED0F,0xDD6C,0xCD4D,0xBDAA,0xAD8B,0x9DE8,0x8DC9, 48 | 0x7C26,0x6C07,0x5C64,0x4C45,0x3CA2,0x2C83,0x1CE0,0x0CC1, 49 | 0xEF1F,0xFF3E,0xCF5D,0xDF7C,0xAF9B,0xBFBA,0x8FD9,0x9FF8, 50 | 0x6E17,0x7E36,0x4E55,0x5E74,0x2E93,0x3EB2,0x0ED1,0x1EF0 51 | ]; 52 | 53 | function crc16(str) { 54 | var len = str.length; 55 | var crc = 0; 56 | for(var i = 0; i < len; i++) { 57 | crc = ((crc<<8)&0xFFFF) ^ CRC16_TAB[((crc>>8) ^ str.charCodeAt(i))&0x00FF]; 58 | } 59 | return crc; 60 | } 61 | 62 | module.exports = function (str) { 63 | // HASH_SLOT = CRC16(key) mod 16384 64 | // 14 out of 16 bits of the output of CRC16 are used 65 | // this is why there is a modulo 16384 operation in the formula above 66 | return(crc16(str) % 16384); 67 | }; 68 | --------------------------------------------------------------------------------