├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── Vagrantfile ├── example.js ├── index.js ├── package.json ├── test ├── 1-server-integration.js ├── 3-servers-integration.js ├── 5-servers-integration.js ├── containers-helper.js └── unit.js ├── util └── reset_vagrant_and_run_tests └── virt ├── apt.sh ├── docker.sh ├── make.sh ├── redis └── Dockerfile └── ttyfix.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # Compiled binary addons (http://nodejs.org/api/addons.html) 20 | build/Release 21 | 22 | # Dependency directory 23 | # Commenting this out is preferred by some people, see 24 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 25 | node_modules 26 | 27 | # Users Environment Variables 28 | .lock-wscript 29 | 30 | # I don't even... 31 | testamundo* 32 | 33 | # Vim shit 34 | *.swp 35 | *.swo 36 | 37 | # Other shit 38 | .vagrant 39 | *.rdb 40 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.10" 4 | 5 | script: "mocha test/unit.js" 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Lauri Kangassalo 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # multiredlock [![Build Status](https://travis-ci.org/lakka/redlock-nodejs.svg?branch=master)](https://travis-ci.org/lakka/redlock-nodejs) 2 | A distributed lock algorithm for redis, see http://redis.io/topics/distlock 3 | 4 | This is experimental software, use with your own risk. 5 | 6 | ## Install 7 | `npm install multiredlock` 8 | 9 | ## Usage 10 | Simple example, which can be found in `example.js`: 11 | 12 | ```js 13 | var Redlock = require('multiredlock'); 14 | 15 | var redlock = new Redlock([{host:'localhost', port:6379}]); 16 | 17 | redlock.on('connect', function() { 18 | // Let's lock resource 'console' for 10 seconds! 19 | redlock.lock('console', 10000, function(err, lock) { 20 | if(err) { 21 | console.log(err); 22 | return; 23 | } 24 | 25 | // Do some stuff with the resource... 26 | console.log(lock); 27 | 28 | // Release lock when you're done with the resource 29 | redlock.unlock('console', lock.value); 30 | }); 31 | }); 32 | ``` 33 | 34 | Will output something like: 35 | 36 | $ node example.js 37 | { validity: 9899, 38 | resource: 'console', 39 | value: 'yourhost_1421844817874_6f554e61' } 40 | 41 | Note: you must have redis-server running on localhost at port 6379. 42 | 43 | ## API 44 | 45 | ### Events 46 | - `connect`: emitted when multiredlock has connected to at least `N/2 + 1` redis-servers, where `N` is the total amount of servers supplied to multiredlock. In other words, `connect` will be emitted when multiredlock is able to acquire locks. 47 | - `disconnect`: emitted when multiredlock is connected to less than `N/2 + 1` redis-servers, meaning that it will not be able to acquire locks. However, unlocking currently locked resources is possible. 48 | 49 | ### Variables 50 | - `connected`: `true`, when connected to at least `N/2 + 1` redis-servers, `false` otherwise. 51 | 52 | ### Constructor 53 | The constructor takes two arguments, an array `servers` and an object `options`: 54 | - new Redlock() = new Redlock([{host:'localhost', port:6379}]); 55 | - new Redlock([{host:'192.168.0.100', port:6380}, {host:'example.com', port:6379}, {host:'localhost', port:6381, auth:'foobarded'}], options); will connect to three redis-servers residing at `192.168.0.100`, `example.com` and `localhost`. 56 | 57 | One may omit the `port`-property from the server object. Multiredlock will then will assume port number `6379`. 58 | 59 | An optional auth value may be passed in the server object to connect to a password protected Redis instance. e.g. `auth: 'foobared'` will connect with the password `foobarded` 60 | 61 | The object `options` may have the following properties: 62 | - `id`: an optional ID string to give for this instance of multiredlock. Is used in identifying locks. Defaults to the machine's hostname. 63 | - `drift`: set the maximum clock drift between servers in milliseconds. This will be subtracted from lock validity time. The default is 100ms. 64 | - `debug`: run multiredlock in debug mode. Boolean. 65 | 66 | ### setRetry(retries, retryWait) 67 | Sets the amount of retries and the maximum time to wait between retries when acquiring a lock. The actual time to retry will be chosen randomly between 0 and `retryWait` milliseconds before each retry. The default is 3 retries with 100ms maximum wait. This means that multiredlock will try to acquire a lock a total of 4 times before giving up. Setting `retries` to 0 will disable retrying. 68 | 69 | ### lock(resource, ttl, callback) 70 | Will try to acquire a lock for `resource`. The lock will be acquired if `N/2 + 1` redis-servers approve of the locking attempt (in other words, they don't think the resource is already locked). Will retry acquiring lock according to retry policies if it is not granted at first time, see `setRetry()`. 71 | 72 | - `resource`: the name of the resource to lock, arbitrary but must be a string. 73 | - `ttl`: Time To Live, the amount of milliseconds before the lock is released, should the resource not be unlocked with `unlock` before that. 74 | - `callback`: will be called with `callback(err, lock)`, where: 75 | - `err`: error message if the lock was not acquired. `null` if lock was acquired. 76 | - `lock`: an object detailing the acquired lock. Has the following properties: 77 | - `validity`: how long the lock is valid, in milliseconds. 78 | - `resource`: the name of the resource that was locked. 79 | - `value`: unique id for the acquired lock. Needed when unlocking a resource or renewing the lock. 80 | 81 | ### renew(resource, value, ttl, callback) 82 | Sets a new TTL for an existing lock. 83 | 84 | - `resource`: name of the locked resource 85 | - `value`: value of the lock to renew. Can be found in the `lock` object provided by `lock()`. 86 | - `ttl`: new Time To Live value for the lock in milliseconds. 87 | - `callback`: will be called with `callback(err, lock)`, where: 88 | - `err`: error message if the lock was not renewed. `null` if lock was renewed. 89 | - `lock`: an object detailing the renewed lock. Has the following properties: 90 | - `validity`: how long the lock is valid, in milliseconds. 91 | - `resource`: the name of the resource that had its lock renewed. 92 | - `value`: same as the value of the lock before renewing. 93 | 94 | Example: 95 | ```js 96 | redlock.lock('test', 200, function(err, lock) { 97 | // Do some stuff with resource 'test'... 98 | // Realize that this stuff is going to take longer than 200ms 99 | redlock.renew('test', lock.value, 200, function(err, lock) { 100 | // If no errors occured, lock.validity is now a little less than 200ms 101 | } 102 | } 103 | ``` 104 | ### unlock(resource, value) 105 | Unlocks a resource. 106 | 107 | - `resource`: the name of the resource to unlock. 108 | - `value`: the unique id of the lock to release. See `lock()`. 109 | 110 | Notice: unlock does not provide a callback, for unlocking attempts are best-effort, and there is no guarantee that the resource will be unlocked. 111 | 112 | ### close() 113 | Disconnect from all redis-servers. 114 | 115 | This maybe useful for releasing the event loop for letting your program exit. 116 | Note that this effectively makes this multiredlock instance unusable. 117 | 118 | ## Tests 119 | This project has integration and failover tests implemented with Vagrant and Docker. 120 | The tests simulate a production environment where redis servers may crash. 121 | 122 | To run these tests, your CPU needs to support VT-x, and you need at least 4GB memory. 123 | You also must have VirtualBox and Vagrant installed. 124 | 125 | $ vagrant up 126 | (be patient...) 127 | $ vagrant ssh 128 | 129 | redlock:~$ cd redlock 130 | redlock:~$ npm install 131 | redlock:~$ npm test 132 | 133 | If you wish to run only unit tests, you can do so by issuing `mocha test/unit.js` in the project root. 134 | 135 | ---- 136 | Special thanks to [amv](https://github.com/amv) for sharing his thoughts on this project, and ordinary thanks to [virta](https://github.com/virta). 137 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | 2 | # Vagrantfile for Gearsloth test environment 3 | # vi: set sw=2 ts=2 sts=2 ft=ruby : 4 | 5 | VAGRANTFILE_API_VERSION = "2" 6 | 7 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 8 | ### Machine settings 9 | # 10 | config.vm.hostname = "redlock" 11 | config.vm.box = "ubuntu/trusty64" 12 | config.vm.box_url = "https://cloud-images.ubuntu.com/vagrant/trusty/current/trusty-server-cloudimg-amd64-vagrant-disk1.box" 13 | 14 | ### Provisioning 15 | # 16 | config.vm.provision :shell, path: "virt/ttyfix.sh" 17 | config.vm.provision :shell, path: "virt/apt.sh" 18 | config.vm.provision :shell, path: "virt/docker.sh" 19 | config.vm.provision :shell, path: "virt/make.sh", :privileged => false 20 | 21 | ### Virtalbox configuration 22 | # 23 | config.vm.provider :virtualbox do |virtualbox| 24 | virtualbox.name = "redlock-node-testenv" 25 | virtualbox.memory = 2048 26 | virtualbox.cpus = 2 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | var Redlock = require('./index'); 2 | 3 | var redlock = new Redlock([{host:'localhost', port:6379}]) 4 | 5 | redlock.on('connect', function() { 6 | // Let's lock resource 'console' for 10 seconds! 7 | redlock.lock('console', 10000, function(err, lock) { 8 | if(err) { 9 | console.log(err); 10 | return; 11 | } 12 | 13 | // Do some stuff with the resource... 14 | console.log(lock); 15 | 16 | // Release lock when you're done with the resource 17 | redlock.unlock('console', lock.value); 18 | }); 19 | }); 20 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var redis = require('redis'), 2 | events = require('events'), 3 | util = require('util'), 4 | async = require('async'), 5 | os = require('os'); 6 | 7 | function Redlock(servers, options) { 8 | if(servers && !Array.isArray(servers)) { 9 | if(servers.host) { 10 | servers = [servers]; 11 | } else if(!options) { 12 | options = servers; 13 | servers = null; 14 | } 15 | } 16 | this.options = options || {}; 17 | this.servers = servers || [{host: 'localhost', port: 6379}]; 18 | this.id = this.options.id || os.hostname(); 19 | this.retries = 3; 20 | this.retryWait = 100; 21 | this.drift = this.options.drift || 100; 22 | this.unlockScript = ' \ 23 | if redis.call("get",KEYS[1]) == ARGV[1] then \ 24 | return redis.call("del",KEYS[1]) \ 25 | else \ 26 | return 0 \ 27 | end \ 28 | '; 29 | this.renewScript = ' \ 30 | if redis.call("get",KEYS[1]) == ARGV[1] then \ 31 | return redis.call("pexpire",KEYS[1],ARGV[2]) \ 32 | else \ 33 | return 0 \ 34 | end \ 35 | '; 36 | this.quorum = Math.floor(this.servers.length / 2) + 1; 37 | this.clients = []; 38 | this.connected = false; 39 | this._connectedClients = 0; 40 | this._connect(); 41 | this._registerListeners(); 42 | if(this.options.debug) { 43 | console.log("Initialized with quorum",this.quorum, 44 | ", total servers", servers.length); 45 | } 46 | } 47 | 48 | 49 | util.inherits(Redlock, events.EventEmitter); 50 | 51 | Redlock.prototype.setRetry = function(retries, retryWait) { 52 | this.retries = retries; 53 | this.retryWait = retryWait; 54 | }; 55 | 56 | Redlock.prototype._connect = function() { 57 | var onError = (this.options.debug) ? console.log : function() {}; 58 | this.clients = this.servers.map(function(server) { 59 | var port = server.port || 6379; 60 | var client = redis.createClient(port, server.host, 61 | {enable_offline_queue:false}); 62 | if (typeof server.auth !== 'undefined') { 63 | client.auth(server.auth); 64 | } 65 | client.on('error', onError); 66 | return client; 67 | }); 68 | }; 69 | 70 | Redlock.prototype._registerListeners = function() { 71 | var that = this; 72 | this.clients.forEach(function(client) { 73 | client.on('ready', function() { 74 | if(++that._connectedClients === that.quorum) { 75 | that.connected = true; 76 | that.emit('connect'); 77 | } 78 | }); 79 | client.on('end', function() { 80 | if(--that._connectedClients === (that.quorum - 1)) { 81 | that.connected = false; 82 | that.emit('disconnect'); 83 | } 84 | }); 85 | }); 86 | }; 87 | 88 | Redlock.prototype._lockInstance = function(client, resource, value, ttl, callback) { 89 | var that = this; 90 | client.set(resource, value, 'NX', 'PX', ttl, function(err, reply) { 91 | if(err || !reply) { 92 | err = err || new Error('resource locked'); 93 | if(that.options.debug) { 94 | console.log('Failed to lock instance:', err); 95 | } 96 | callback(err); 97 | } 98 | else 99 | callback(); 100 | }); 101 | }; 102 | 103 | Redlock.prototype._renewInstance = function(client, resource, value, ttl, callback) { 104 | var that = this; 105 | client.eval(this.renewScript, 1, resource, value, ttl, function(err, reply) { 106 | if(err || !reply) { 107 | err = err || new Error('resource does not exist'); 108 | if(that.options.debug) { 109 | console.log('Failed to renew instance:', err); 110 | } 111 | callback(err); 112 | return; 113 | } 114 | callback(); 115 | }); 116 | }; 117 | 118 | Redlock.prototype._unlockInstance = function(client, resource, value) { 119 | client.eval(this.unlockScript, 1, resource, value, function() {}); 120 | // Unlocking is best-effort, so we don't care about errors 121 | }; 122 | 123 | 124 | Redlock.prototype._getUniqueLockId = function(callback) { 125 | return this.id + "_" + Date.now() + "_" + Math.random().toString(16).slice(2); 126 | }; 127 | 128 | Redlock.prototype._acquireLock = function(resource, value, ttl, lockFunction, callback) { 129 | var that = this; 130 | var value = value || this._getUniqueLockId(); 131 | var n = 0; 132 | var startTime = Date.now(); 133 | 134 | async.series([ 135 | function(locksSet) { 136 | async.each(that.clients, function(client, done) { 137 | lockFunction.apply(that, [client, resource, value, ttl, function(err) { 138 | if(!err) 139 | n++; 140 | done(); 141 | }]); 142 | }, locksSet); 143 | }, 144 | function() { 145 | var timeSpent = Date.now() - startTime; 146 | if(that.options.debug) { 147 | console.log('Time spent locking:', timeSpent, 'ms'); 148 | console.log(n + "", 'servers approve our lock'); 149 | } 150 | var validityTime = ttl - timeSpent - that.drift; 151 | if(n >= that.quorum && validityTime > 0) { 152 | callback(null, { 153 | validity: validityTime, 154 | resource: resource, 155 | value: value 156 | }); 157 | } else { 158 | that.unlock(resource, value); 159 | callback(new Error('Could not lock resource ' + resource)); 160 | } 161 | } 162 | ]); 163 | }; 164 | 165 | Redlock.prototype.lock = function(resource, ttl, callback) { 166 | var that = this; 167 | var retries = this.retries; 168 | var retryCallback = function(err, lock) { 169 | if(err) { 170 | if(retries > 0) { 171 | retries--; 172 | var timeout = Math.floor(Math.random() * that.retryWait); 173 | if(that.options.debug) { 174 | console.log('Retrying locking in', timeout, 'ms'); 175 | } 176 | setTimeout( 177 | that._acquireLock.bind(that, resource, null, ttl, that._lockInstance, retryCallback), 178 | timeout); 179 | } else { 180 | callback(err); 181 | } 182 | return; 183 | } 184 | callback(null, lock); 185 | }; 186 | this._acquireLock(resource, null, ttl, this._lockInstance, retryCallback); 187 | }; 188 | 189 | Redlock.prototype.renew = function(resource, value, ttl, callback) { 190 | this._acquireLock(resource, value, ttl, this._renewInstance, callback); 191 | }; 192 | 193 | Redlock.prototype.unlock = function(resource, value) { 194 | var that = this; 195 | this.clients.forEach(function(client) { 196 | that._unlockInstance(client, resource, value); 197 | }); 198 | }; 199 | 200 | Redlock.prototype.close = function() { 201 | this.clients.forEach(function(client) { 202 | client.quit(); 203 | }); 204 | }; 205 | 206 | module.exports = Redlock; 207 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "multiredlock", 3 | "version": "0.2.0", 4 | "description": "A distributed lock algorithm for redis, see http://redis.io/topics/distlock", 5 | "author": "Lauri Kangassalo ", 6 | "homepage": "https://github.com/lakka/redlock-nodejs", 7 | "license": "MIT", 8 | "main": "index.js", 9 | "keywords": [ 10 | "redis", 11 | "distributed lock manager" 12 | ], 13 | "repository": { 14 | "type": "git", 15 | "url": "git://github.com/lakka/redlock-nodejs.git" 16 | }, 17 | "bugs": { 18 | "url": "https://github.com/lakka/redlock-nodejs/issues" 19 | }, 20 | "dependencies": { 21 | "async": "*", 22 | "redis": "^0.8.3" 23 | }, 24 | "devDependencies": { 25 | "chai": "*", 26 | "sinon": "*", 27 | "sinon-chai": "*", 28 | "redis": "*", 29 | "async": "*", 30 | "mocha": "*", 31 | "dockerode": "*" 32 | }, 33 | "scripts": { 34 | "test": "mocha ./test" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /test/1-server-integration.js: -------------------------------------------------------------------------------- 1 | var Docker = require('dockerode'), 2 | docker = new Docker(), 3 | async = require('async'), 4 | redis = require('redis'), 5 | dockerhelper = require('./containers-helper'), 6 | Redlock = require('../index'); 7 | 8 | /* 9 | * This test requires a redis-server docker container named: 10 | * redis-1 11 | */ 12 | 13 | describe('(integration) Redlock with five redis-servers', function() { 14 | var servers, containers, redlock, clients; 15 | 16 | beforeEach(function(done) { 17 | async.series([ 18 | function(next) { 19 | dockerhelper.startRedisServers(1, function(err, serv, cli, cont) { 20 | if(err) { 21 | next(err); 22 | return; 23 | } 24 | servers = serv; 25 | clients = cli; 26 | containers = cont; 27 | next(); 28 | }); 29 | }, function(done) { 30 | redlock = new Redlock(servers); 31 | redlock.on('connect', done); 32 | }], done); 33 | }); 34 | 35 | after(function(done) { 36 | dockerhelper.stopEverything(done); 37 | }); 38 | 39 | describe('lock()', function() { 40 | it('should acquire lock by retrying if previous lock expires', function(done) { 41 | redlock.setRetry(9,100); 42 | redlock.lock('test', 150, function(err, lock) { 43 | if(err) { 44 | done(new Error('Could not lock resource with clean database')); 45 | return; 46 | } 47 | redlock.lock('test', 150, function(err, lock) { 48 | done(err); 49 | }); 50 | }); 51 | }); 52 | it('should acquire lock by retrying if previous lock is unlocked', function(done) { 53 | redlock.setRetry(9,100); 54 | redlock.lock('test', 2000, function(err, lock) { 55 | if(err) { 56 | done(new Error('Could not lock resource with clean database')); 57 | return; 58 | } 59 | redlock.unlock('test', lock.value); 60 | redlock.lock('test', 150, function(err, lock) { 61 | done(err); 62 | }); 63 | }); 64 | }); 65 | it('should acquire lock by retrying when server comes back up', function(done) { 66 | redlock.setRetry(9,100); 67 | containers[0].pause(function() { // Restarting a container will change it's IP 68 | redlock.lock('test', 150, function(err, lock) { 69 | done(err); 70 | }); 71 | containers[0].unpause(function() {}); 72 | }); 73 | }); 74 | }); 75 | }); 76 | -------------------------------------------------------------------------------- /test/3-servers-integration.js: -------------------------------------------------------------------------------- 1 | var Docker = require('dockerode'), 2 | docker = new Docker(), 3 | async = require('async'), 4 | redis = require('redis'), 5 | dockerhelper = require('./containers-helper'), 6 | Redlock = require('../index'); 7 | 8 | /* 9 | * This test requires three redis-server docker containers named: 10 | * redis-1, redis-2 and redis-3 11 | */ 12 | 13 | describe('(integration) Redlock with three redis-servers', function() { 14 | var servers, containers, redlock, clients; 15 | var that = this; 16 | 17 | beforeEach(function(done) { 18 | async.series([ 19 | function(next) { 20 | dockerhelper.startRedisServers(3, function(err, serv, cli, cont) { 21 | if(err) { 22 | next(err); 23 | return; 24 | } 25 | servers = serv; 26 | clients = cli; 27 | containers = cont; 28 | next(); 29 | }); 30 | }, function(done) { 31 | redlock = new Redlock(servers); 32 | redlock.on('connect', done); 33 | }], done); 34 | }); 35 | 36 | after(function(done) { 37 | dockerhelper.stopEverything(done); 38 | }); 39 | 40 | describe('#lock()', function() { 41 | it('should acquire lock if all servers approve', function(done) { 42 | redlock.lock('test', 1000, done); 43 | }); 44 | }); 45 | describe('#renew()', function() { 46 | it('should extend existing lock\'s ttl', function(done) { 47 | redlock.lock('test', 200, function(err, lock) { 48 | redlock.renew('test', lock.value, 1500, function(err, lock) { 49 | setTimeout(function() { 50 | clients[0].get('test', function(err, reply) { 51 | if(!err && reply) { 52 | done(); 53 | } else { 54 | done(new Error('Lock was deleted before it was supposed to')); 55 | } 56 | }); 57 | }, 300); 58 | }); 59 | }); 60 | }); 61 | }); 62 | 63 | describe('failover', function() { 64 | var random = Math.floor(Math.random() * 10) % 3; 65 | it('should acquire lock if a server crashes', function(done) { 66 | redlock.setRetry(0,0); 67 | containers[random].kill(function(err, data) { 68 | redlock.lock('test', 2000, function(err) { 69 | done(err); 70 | }); 71 | }); 72 | }); 73 | it('should not acquire lock if res is locked and a server crashes', function(done) { 74 | redlock.setRetry(0,0); 75 | redlock.lock('test', 2000, function() { 76 | containers[random].kill(function(err, data) { 77 | redlock.lock('test', 2000, function(err) { 78 | if(err) done(); 79 | else done(new Error('lock acquired')); 80 | }); 81 | }); 82 | }); 83 | }); 84 | it('servers A, B, C; C down -> lock acquired -> C up, B down -> lock released ' + 85 | '-> B up, C down -> lock should be acquired', function(done) { 86 | var value; 87 | async.series([function(next) { 88 | containers[2].kill(next); 89 | }, function(next) { 90 | redlock.lock('test', 2000, function(err, data) { 91 | if(err) { 92 | next(err); 93 | return; 94 | } 95 | value = data.value; 96 | next(); 97 | }); 98 | }, function(next) { 99 | async.parallel([function(ready) { 100 | containers[2].start(ready); 101 | }, function(ready) { 102 | containers[1].kill(ready); 103 | }], next); 104 | }, function(next) { 105 | redlock.unlock('test', value); 106 | next(); 107 | }, function(next) { 108 | async.parallel([function(ready) { 109 | containers[1].start(ready); 110 | }, function(ready) { 111 | containers[2].kill(ready); 112 | }], next); 113 | }, function(next) { 114 | redlock.lock('test', 500, function(err) { 115 | if(err) { 116 | next(new Error('Last step failed, lock not acquired')); 117 | return; 118 | } 119 | next(); 120 | }); 121 | }], done); 122 | }); 123 | }); 124 | 125 | }); 126 | -------------------------------------------------------------------------------- /test/5-servers-integration.js: -------------------------------------------------------------------------------- 1 | var Docker = require('dockerode'), 2 | docker = new Docker(), 3 | async = require('async'), 4 | redis = require('redis'), 5 | dockerhelper = require('./containers-helper'), 6 | Redlock = require('../index'); 7 | 8 | /* 9 | * This test requires five redis-server docker containers named: 10 | * redis-1, redis-2, redis-3, redis-4 and redis-5 11 | */ 12 | 13 | describe('(integration) Redlock with five redis-servers', function() { 14 | var servers, containers, redlock, clients; 15 | 16 | beforeEach(function(done) { 17 | async.series([ 18 | function(next) { 19 | dockerhelper.startRedisServers(5, function(err, serv, cli, cont) { 20 | if(err) { 21 | next(err); 22 | return; 23 | } 24 | servers = serv; 25 | clients = cli; 26 | containers = cont; 27 | next(); 28 | }); 29 | }, function(done) { 30 | redlock = new Redlock(servers); 31 | redlock.on('connect', done); 32 | }], done); 33 | }); 34 | 35 | after(function(done) { 36 | dockerhelper.stopEverything(done); 37 | }); 38 | 39 | describe('failover', function() { 40 | it('should acquire lock if two servers crash', function(done) { 41 | async.times(2, function(n, next) { 42 | containers[n].kill(function(err, data) { 43 | next(err); 44 | }); 45 | }, function() { 46 | redlock.lock('test', 2000, function(err) { 47 | done(err); 48 | }); 49 | }); 50 | }); 51 | it('should not acquire lock if three servers crash', function(done) { 52 | async.times(3, function(n, next) { 53 | containers[n].kill(function(err, data) { 54 | next(); 55 | }); 56 | }, function() { 57 | redlock.lock('test', 2000, function(err) { 58 | if(err) done(); 59 | else done(new Error('Callback not called with error')); 60 | }); 61 | }); 62 | }); 63 | }); 64 | }); 65 | -------------------------------------------------------------------------------- /test/containers-helper.js: -------------------------------------------------------------------------------- 1 | var async = require('async'), 2 | Docker = require('dockerode'), 3 | docker = new Docker(), 4 | redis = require('redis'); 5 | 6 | exports.startRedisServers = function(n, callback) { 7 | var servers = []; 8 | var containers = []; 9 | var clients = []; 10 | // Start redis containers 11 | async.times(n, 12 | function(n, next) { 13 | var containerName = "redis-" + (n + 1); 14 | var container = docker.getContainer(containerName); 15 | if(!container) { 16 | next('Could not find container ' + containerName + '. Abort.'); 17 | return; 18 | } 19 | containers.push(container); 20 | container.start(function(err, data) { 21 | if(err && err.statusCode != 304) { 22 | next(err); 23 | } 24 | container.unpause(function() { }); 25 | container.inspect(function(err, containerInfo) { 26 | if(err) next(err); 27 | var server = { 28 | host: containerInfo.NetworkSettings.IPAddress, 29 | port: 6379 30 | }; 31 | servers.push(server); 32 | 33 | // Clear previous data 34 | var client = redis.createClient(server.port, server.host); 35 | client.on('error', function() {}); 36 | clients.push(client); 37 | 38 | // Retry flushall to Redis for a total of about 4 seconds 39 | // with timeouts of 16 ms, 32 ms, 64 ms .. 2048 ms 40 | var retryCount = 0; 41 | var flushallRetryHandler = function(err) { 42 | if (err) { 43 | retryCount++; 44 | if (retryCount > 8 ) { 45 | next(new Error('Unable to find started redis N=' + n + '. Last error: ' + err )); 46 | } 47 | else { 48 | setTimeout( 49 | function() { 50 | client.flushall(flushallRetryHandler); 51 | }, 52 | Math.pow(2, retryCount+3) 53 | ); 54 | } 55 | } 56 | else { 57 | next(); 58 | } 59 | }; 60 | client.flushall(flushallRetryHandler); 61 | }); 62 | }); 63 | }, function(err) { 64 | callback(err, servers, clients, containers); 65 | }); 66 | }; 67 | 68 | exports.stopEverything = function(callback) { 69 | docker.listContainers(function (err, containers) { 70 | async.each(containers, function (containerInfo, next) { 71 | var container = docker.getContainer(containerInfo.Id); 72 | container.stop(function() { 73 | next(); 74 | }); 75 | }, callback); 76 | }); 77 | }; 78 | -------------------------------------------------------------------------------- /test/unit.js: -------------------------------------------------------------------------------- 1 | var chai = require("chai"); 2 | var expect = chai.expect; 3 | var sinon = require('sinon'); 4 | chai.use(require('sinon-chai')); 5 | var redis = require('redis'); 6 | var async = require('async'); 7 | var Redlock = require('../index'); 8 | 9 | describe('(unit) Redlock with one server', function() { 10 | var sandbox = sinon.sandbox.create(); 11 | var redisStub, redlock, clientStub, setSpy; 12 | beforeEach(function() { 13 | clientStub = sandbox.stub(redis.createClient().on('error', function() {})); 14 | clientStub.set.onFirstCall().yields(null, 'OK'); 15 | clientStub.set.onSecondCall().yields(); 16 | clientStub.emit('ready'); 17 | redisStub = sandbox.stub(redis, 'createClient'); 18 | redisStub.returns(clientStub); 19 | redlock = new Redlock([{host:'localhost', port:6739}]); 20 | }); 21 | 22 | afterEach(function() { 23 | sandbox.restore() 24 | }); 25 | 26 | describe('constructor', function() { 27 | it('should call redis.createClient once', function() { 28 | expect(redisStub).to.have.been.calledOnce; 29 | }); 30 | }); 31 | 32 | describe('#lock()', function() { 33 | it('should call redisClient.set once', function() { 34 | redlock.lock('test', 1000, function() { 35 | expect(clientStub.set).to.have.been.calledOnce; 36 | }); 37 | }); 38 | it('should call back with error if no redis servers approve of the lock', function() { 39 | redlock.lock('test', 1000, function() { 40 | redlock.lock('test', 1000, function(err) { 41 | expect(err).to.be.not.null; 42 | }); 43 | }); 44 | }); 45 | it('should call back with no error and an object on success', function() { 46 | redlock.lock('test', 1000, function(err, res) { 47 | expect(err).to.be.null; 48 | expect(res).to.be.an.object; 49 | }); 50 | }); 51 | it('should retry on fail', function() { 52 | clientStub.set.onFirstCall().yields(); 53 | clientStub.set.onSecondCall().yields(null, 'OK'); 54 | redlock.lock('test', 1000, function(err, res) { 55 | expect(clientStub.set).to.have.been.calledTwice; 56 | }); 57 | }); 58 | it('should try to lock three times if retries: 2 set', function() { 59 | redlock = new Redlock([{host:'localhost', port:6739}], 60 | {retries:2}); 61 | clientStub.set.onFirstCall().yields(); 62 | clientStub.set.onSecondCall().yields(); 63 | clientStub.set.onThirdCall().yields(null, 'OK'); 64 | redlock.lock('test', 1000, function(err, res) { 65 | expect(clientStub.set).to.have.been.calledThrice; 66 | }); 67 | }); 68 | }); 69 | 70 | describe('#renew()', function() { 71 | it('should call clientStub.eval once', function() { 72 | redlock.lock('test', 1000, function(err, lock) { 73 | if(err) return; 74 | redlock.renew('test', lock.value, 100, function() { 75 | expect(clientStub.eval).to.have.been.calledOnce; 76 | }); 77 | }); 78 | }); 79 | it('should err if resource does not exist', function(done) { 80 | clientStub.eval.onFirstCall().yields(null, 0); 81 | redlock.renew('test', 'randomVal', 100, function(err) { 82 | if(err) done(); 83 | else done(new Error('callback was not called with error')); 84 | }); 85 | }); 86 | }); 87 | 88 | describe('#unlock()', function() { 89 | it('should call redisClient.eval once', function() { 90 | redlock.unlock('test', 'value', function() {}); 91 | expect(clientStub.eval).to.have.been.calledOnce; 92 | }); 93 | }); 94 | }); 95 | 96 | describe('Redlock with three servers', function() { 97 | var sandbox = sinon.sandbox.create(); 98 | var servers = [ 99 | {host:'localhost', port:6739}, 100 | {host:'jaakkomaa', port:6739}, 101 | {host:'localhost', port:6799} 102 | ]; 103 | var redisStub, redlock, clientStubError, clientStub, setSpy; 104 | beforeEach(function() { 105 | clientStub = sandbox.stub(redis.createClient().on('error', function() {})); 106 | clientStub.on.restore(); 107 | clientStub.emit.restore(); 108 | redisStub = sandbox.stub(redis, 'createClient'); 109 | redisStub.returns(clientStub); 110 | redlock = new Redlock(servers); 111 | }); 112 | 113 | afterEach(function() { 114 | sandbox.restore(); 115 | }); 116 | describe('constructor', function() { 117 | it('should call redis.createClient thrice', function() { 118 | expect(redisStub).to.have.been.calledThrice; 119 | }); 120 | it('should emit connect', function(done) { 121 | redlock.on('connect', done); 122 | clientStub.emit('ready'); 123 | }); 124 | it('should emit disconnect if all servers go down', function(done) { 125 | redlock.on('disconnect', done); 126 | clientStub.emit('ready'); 127 | clientStub.emit('end'); 128 | }); 129 | }); 130 | 131 | describe('#lock()', function() { 132 | it('should call redisClient.set thrice', function() { 133 | redlock.lock('test', 1000, function() { 134 | expect(clientStub.set).to.have.been.calledThrice; 135 | }); 136 | }); 137 | it('should err if less than two servers approve of lock', function() { 138 | clientStub.set.onFirstCall().yields(null, 'OK'); 139 | clientStub.set.onSecondCall().yields(); 140 | clientStub.set.onThirdCall().yields(); 141 | redlock.lock('test', 1000, function(err) { 142 | expect(err).to.be.not.null; 143 | }); 144 | }); 145 | it('should err if one server approves lock, one errs and one disapproves', function() { 146 | clientStub.set.onFirstCall().yields(null, 'OK'); 147 | clientStub.set.onSecondCall().yields(new Error('test error')); 148 | clientStub.set.onThirdCall().yields(); 149 | redlock.lock('test', 1000, function(err) { 150 | expect(err).to.be.not.null; 151 | }); 152 | }); 153 | it('should succeed if two servers approve of lock', function() { 154 | clientStub.set.onFirstCall().yields(null, 'OK'); 155 | clientStub.set.onSecondCall().yields(null, 'OK'); 156 | clientStub.set.onThirdCall().yields(); 157 | redlock.lock('test', 1000, function(err, res) { 158 | expect(err).to.be.null; 159 | expect(res).to.be.an.object; 160 | }); 161 | }); 162 | }); 163 | 164 | describe('#unlock()', function() { 165 | it('should call redisClient.eval thrice', function() { 166 | redlock.unlock('test', 'value', function() {}); 167 | expect(clientStub.eval).to.have.been.calledThrice; 168 | }); 169 | }); 170 | 171 | describe('#close()', function() { 172 | it('should call redisClient.quit thrice', function() { 173 | redlock.close(); 174 | expect(clientStub.quit).to.have.been.calledThrice; 175 | }); 176 | }); 177 | }); 178 | -------------------------------------------------------------------------------- /util/reset_vagrant_and_run_tests: -------------------------------------------------------------------------------- 1 | vagrant destroy -f 2 | vagrant up 3 | vagrant ssh -c 'cd redlock; npm install; npm test' 4 | -------------------------------------------------------------------------------- /virt/apt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cat >/etc/apt/sources.list </etc/apt/apt.conf.d/99translations < /etc/apt/sources.list.d/docker.list 18 | 19 | apt-get update 20 | 21 | apt-get remove --yes node 22 | 23 | apt-get install --yes --no-install-recommends \ 24 | lxc-docker \ 25 | git \ 26 | nodejs-legacy \ 27 | npm \ 28 | redis-server \ 29 | redis-tools 30 | -------------------------------------------------------------------------------- /virt/docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | gpasswd -a vagrant docker 3 | service docker restart 4 | sleep 3 5 | 6 | # Build redis docker images 7 | docker build -t redis /vagrant/virt/redis 8 | docker run -d --name redis-1 redis 9 | docker run -d --name redis-2 redis 10 | docker run -d --name redis-3 redis 11 | docker run -d --name redis-4 redis 12 | docker run -d --name redis-5 redis 13 | -------------------------------------------------------------------------------- /virt/make.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Take a clean snapshot of the git repository for testing. 3 | 4 | DIR=$HOME/redlock 5 | 6 | if [ ! -e $DIR ]; then 7 | git clone /vagrant $DIR 8 | fi 9 | 10 | cd $DIR 11 | -------------------------------------------------------------------------------- /virt/redis/Dockerfile: -------------------------------------------------------------------------------- 1 | # Dockerfile for redis 2 | 3 | FROM ubuntu:trusty 4 | 5 | RUN echo "deb http://archive.ubuntu.com/ubuntu trusty main universe" > /etc/apt/sources.list 6 | RUN echo "deb http://archive.ubuntu.com/ubuntu trusty-updates main universe" >> /etc/apt/sources.list 7 | RUN apt-get update 8 | RUN DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends redis-server 9 | 10 | EXPOSE 6379 11 | 12 | CMD ["redis-server"] 13 | -------------------------------------------------------------------------------- /virt/ttyfix.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | sed -i 's/^mesg n$/tty -s \&\& mesg n/g' /root/.profile 4 | --------------------------------------------------------------------------------