├── .jshintignore ├── test ├── event-and-callback.js ├── bit-ring.js ├── event-failures.js └── index.js ├── bit-ring.js ├── .jshintrc ├── benchmark.js ├── docs.mli ├── LICENCE ├── README.md ├── .gitignore ├── package.json └── index.js /.jshintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | -------------------------------------------------------------------------------- /test/event-and-callback.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var test = global.it; 3 | 4 | var Prober = require('../index'); 5 | 6 | test('Prober detecting failures by both callback and event', function(end) { 7 | var prober = new Prober({ 8 | detectFailuresBy: Prober.detectBy.BOTH, 9 | backend: { 10 | on: function() {} 11 | }, 12 | failureEvent: '', 13 | successEvent: '' 14 | }); 15 | 16 | var called = false; 17 | prober.probe(function(callback) { callback(); }, assert.fail, function () { 18 | called = true; 19 | }); 20 | assert.ok(called); 21 | end(); 22 | }); 23 | -------------------------------------------------------------------------------- /bit-ring.js: -------------------------------------------------------------------------------- 1 | function BitRing(capacity) { 2 | this.capacity = capacity; 3 | this.bits = new Uint8ClampedArray(capacity); 4 | this.pos = 0; 5 | this.length = 0; 6 | this._count = 0; 7 | } 8 | 9 | // Update the count and set or clear the next bit in the ring 10 | BitRing.prototype.push = function(bool) { 11 | var num = bool === true ? 1 : 0; 12 | this._count += num - this.bits[this.pos]; 13 | this.bits[this.pos] = num; 14 | this.pos = (this.pos + 1) % this.capacity; 15 | if (this.length < this.capacity) { 16 | this.length++; 17 | } 18 | }; 19 | 20 | // Return the number of bits set 21 | BitRing.prototype.count = function() { 22 | return this._count; 23 | }; 24 | 25 | module.exports = BitRing; 26 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "bitwise": false, 3 | "camelcase": true, 4 | "curly": false, 5 | "eqeqeq": true, 6 | "forin": true, 7 | "immed": true, 8 | "indent": 4, 9 | "latedef": true, 10 | "newcap": true, 11 | "noarg": true, 12 | "nonew": true, 13 | "plusplus": false, 14 | "quotmark": false, 15 | "regexp": false, 16 | "undef": true, 17 | "unused": true, 18 | "strict": false, 19 | "trailing": true, 20 | "node": true, 21 | "noempty": true, 22 | "maxdepth": 4, 23 | "maxparams": 4, 24 | "globals": { 25 | "console": true, 26 | "Buffer": true, 27 | "setTimeout": true, 28 | "clearTimeout": true, 29 | "setInterval": true, 30 | "clearInterval": true 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /benchmark.js: -------------------------------------------------------------------------------- 1 | var Benchmark = require('benchmark'); 2 | var BitRing = require('./bit-ring'); 3 | var Prober = require('./index'); 4 | 5 | var suite = new Benchmark.Suite(); 6 | var prober = new Prober(); 7 | var healthy = function(callback) { 8 | callback(); 9 | }; 10 | var error = new Error('unhealthy'); 11 | var unhealthy = function(callback) { 12 | callback(error); 13 | }; 14 | var bitRing = new BitRing(10); 15 | 16 | suite.add('probe healthy', function() { 17 | prober.probe(healthy); 18 | }).add('probe unhealthy', function() { 19 | prober.probe(unhealthy); 20 | }).add('bitRing', function() { 21 | bitRing.count(); 22 | bitRing.push(true); 23 | bitRing.count(); 24 | }).on('cycle', function(event) { 25 | console.log(String(event.target)); 26 | }).run(); 27 | -------------------------------------------------------------------------------- /docs.mli: -------------------------------------------------------------------------------- 1 | type Prober := { 2 | isHealthy: () => Boolean, 3 | isSick: () => Boolean, 4 | notok: () => void, 5 | ok: () => void, 6 | prober: ( 7 | request: (Callback) => void, 8 | bypass: (Error) => void, 9 | callback: (Callback) => void 10 | ) => void, 11 | setLogger: (WinstonLoggerClient) => void 12 | } 13 | 14 | rt-prober := ({ 15 | title: String, 16 | statsd?: { increment: (String) => void }, 17 | threshold?: Number, 18 | window?: Number, 19 | defaultWaitPeriod?: Number, 20 | maxWaitPeriod?: Number, 21 | enabled?: Boolean, 22 | detectFailuresBy?: 'event' | 'callback' | 'both', 23 | logger?: WinstonLoggerClient, 24 | backend?: EventEmitter, 25 | failureEvent?: String, 26 | successEvent?: String, 27 | now?: () => Number 28 | }) => Prober 29 | -------------------------------------------------------------------------------- /test/bit-ring.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var BitRing = require('../bit-ring'); 3 | var test = global.it; 4 | 5 | test('empty bit ring', function(end) { 6 | var bitRing = new BitRing(3); 7 | assert.equal(bitRing.length, 0); 8 | assert.equal(bitRing.count(), 0); 9 | end(); 10 | }); 11 | 12 | test('set bit', function(end) { 13 | var bitRing = new BitRing(3); 14 | bitRing.push(true); 15 | assert.equal(bitRing.length, 1); 16 | assert.equal(bitRing.count(), 1); 17 | end(); 18 | }); 19 | 20 | test('clear bit', function(end) { 21 | var bitRing = new BitRing(3); 22 | bitRing.push(true); 23 | bitRing.push(true); 24 | assert.equal(bitRing.length, 2); 25 | assert.equal(bitRing.count(), 2); 26 | bitRing.push(false); 27 | assert.equal(bitRing.length, 3); 28 | assert.equal(bitRing.count(), 2); 29 | bitRing.push(false); 30 | assert.equal(bitRing.length, 3); 31 | assert.equal(bitRing.count(), 1); 32 | end(); 33 | }); 34 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Uber. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # airlock 2 | 3 | A prober to probe HTTP, [tchannel](https://github.com/uber/tchannel), or potentially other protocols based backends for health 4 | 5 | ## Example 6 | 7 | ```js 8 | var Prober = require("airlock") 9 | 10 | var prober = new Prober({ 11 | title: 'probe interface', 12 | statsd: { increment: function (key) { 13 | // send increment command to a statsd server. 14 | } }, 15 | logger: { 16 | warn: function (message) { 17 | /* sink this message to your logging system */ 18 | } 19 | } 20 | }) 21 | 22 | var thunk = request.bind(null, { 23 | uri: 'http://www.example.com/foo', 24 | method: 'POST', 25 | json: { ... } 26 | }) 27 | prober.probe(thunk, function (err, res, body) { 28 | /* we probed the async task and have the result 29 | if the async task fails a lot then the prober 30 | automatically rate limits 31 | */ 32 | }) 33 | ``` 34 | 35 | ## Installation 36 | 37 | `npm install airlock` 38 | 39 | ## Contributors 40 | 41 | - Raynos 42 | - markyen 43 | - jwolski 44 | - zhijinli 45 | 46 | ## MIT Licenced 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source # 2 | ################### 3 | *.com 4 | *.class 5 | *.dll 6 | *.exe 7 | *.a 8 | *.o 9 | *.so 10 | *.node 11 | bin/* 12 | 13 | # Node Waf Byproducts # 14 | ####################### 15 | .lock-wscript 16 | build/ 17 | autom4te.cache/ 18 | 19 | # Node Modules # 20 | ################ 21 | # Better to let npm install these from the package.json defintion 22 | # rather than maintain this manually 23 | node_modules/ 24 | 25 | # Packages # 26 | ############ 27 | # it's better to unpack these files and commit the raw source 28 | # git has its own built in compression methods 29 | *.7z 30 | *.dmg 31 | *.gz 32 | *.iso 33 | *.jar 34 | *.rar 35 | *.tar 36 | *.zip 37 | 38 | # Logs and databases # 39 | ###################### 40 | *.log 41 | dump.rdb 42 | *.js.tap 43 | *.coffee.tap 44 | 45 | # OS generated files # 46 | ###################### 47 | .DS_Store? 48 | .DS_Store 49 | ehthumbs.db 50 | Icon? 51 | Thumbs.db 52 | coverage 53 | 54 | # Text Editor Byproducts # 55 | ########################## 56 | *.swp 57 | *.swo 58 | .idea/ 59 | 60 | *.pyc 61 | 62 | # All translation files # 63 | ######################### 64 | static/translations-s3/ 65 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "airlock", 3 | "version": "2.2.0", 4 | "description": "A prober to probe HTTP based backends for health", 5 | "keywords": [], 6 | "author": "Raynos ", 7 | "repository": "git://github.com/uber/airlock.git", 8 | "main": "index", 9 | "homepage": "https://github.com/uber/airlock", 10 | "bugs": { 11 | "url": "https://github.com/uber/airlock/issues", 12 | "email": "raynos2@gmail.com" 13 | }, 14 | "collaborators": [ 15 | { 16 | "name": "jeff wolski" 17 | }, 18 | { 19 | "name": "markyen" 20 | } 21 | ], 22 | "dependencies": {}, 23 | "devDependencies": { 24 | "benchmark": "^1.0.0", 25 | "istanbul": "~0.1.46", 26 | "lodash.times": "~2.4.1", 27 | "mocha": "~1.15.1", 28 | "time-mock": "~0.1.2" 29 | }, 30 | "scripts": { 31 | "test": "npm run jshint && mocha --reporter tap ./test 2>&1 | tee ./test/test.js.tap", 32 | "jshint": "jshint --verbose .", 33 | "cover": "istanbul cover --report none --print detail _mocha -- test --reporter tap", 34 | "view-cover": "istanbul report html && open ./coverage/index.html" 35 | }, 36 | "engine": { 37 | "node": ">= 0.8.x" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /test/event-failures.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var test = global.it; 3 | 4 | var Prober = require('../index'); 5 | 6 | test('Prober detecting failures by event', function(end) { 7 | var events = []; 8 | var mockEmitter = { 9 | on: function (eventName) { 10 | events.push(eventName); 11 | } 12 | }; 13 | var failureEvent = 'failureEvent'; 14 | var successEvent = 'successEvent'; 15 | 16 | var prober = new Prober({ 17 | detectFailuresBy: Prober.detectBy.EVENT, 18 | backend: mockEmitter, 19 | window: 1, 20 | failureEvent: failureEvent, 21 | successEvent: successEvent 22 | }); 23 | 24 | assert.equal(events.length, 2); 25 | assert.deepEqual(events, [failureEvent, successEvent]); 26 | 27 | // failures detected by events do not have a callback 28 | // argument so calling it will throw and thus not trigger 29 | // the actual callback to probe. 30 | // instead we just do a side effect and the emitter will 31 | // emit success & failure events which get probed 32 | try { 33 | prober.probe(function(callback) { 34 | callback(); 35 | }, assert.fail, assert.fail); 36 | } catch (err) { /*ignore*/ } 37 | 38 | assert.ok(!prober.isHealthy()); 39 | end(); 40 | }); 41 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var BitRing = require('./bit-ring'); 3 | 4 | var defaults = { 5 | title: 'general', 6 | threshold: 6, 7 | window: 10, 8 | defaultWaitPeriod: 1000, 9 | maxWaitPeriod: 60 * 1000, 10 | isUnhealthyFunc: function isUnhealthy(err, resp) { 11 | // default is for HTTP, tchannel/other protocal needs to pass in different function 12 | return err || resp && !isNaN(resp.statusCode) && resp.statusCode >= 500; 13 | } 14 | }; 15 | 16 | function Prober(options) { 17 | if (!(this instanceof Prober)) { 18 | return new Prober(options); 19 | } 20 | 21 | options = options || {}; 22 | 23 | this.title = options.title || defaults.title; 24 | this.threshold = options.threshold || defaults.threshold; 25 | this.window = options.window || defaults.window; 26 | this.now = options.now || Date.now; 27 | this.defaultWaitPeriod = options.defaultWaitPeriod || 28 | defaults.defaultWaitPeriod; 29 | this.maxWaitPeriod = options.maxWaitPeriod || defaults.maxWaitPeriod; 30 | this.enabled = 'enabled' in options ? options.enabled : true; 31 | var detectFailuresBy = options.detectFailuresBy || Prober.detectBy.CALLBACK; 32 | this.detectFailuresByCallback = 33 | (detectFailuresBy === Prober.detectBy.CALLBACK) || 34 | (detectFailuresBy === Prober.detectBy.BOTH); 35 | this.detectFailuresByEvent = 36 | (detectFailuresBy === Prober.detectBy.EVENT) || 37 | (detectFailuresBy === Prober.detectBy.BOTH); 38 | 39 | this.logger = options.logger || null; 40 | this.bitRing = new BitRing(this.window); 41 | this.waitPeriod = this.defaultWaitPeriod; 42 | this.lastBackendRequest = this.now(); 43 | this.statsd = options.statsd || null; 44 | 45 | this.isUnhealthyFunc = typeof options.isUnhealthyFunc === 'function' && 46 | options.isUnhealthyFunc || defaults.isUnhealthyFunc; 47 | 48 | if (this.detectFailuresByEvent) { 49 | if (!options.backend) { 50 | if (this.logger) { 51 | this.logger.warn('Prober missing backend from' + 52 | ' initialization options'); 53 | } 54 | return; 55 | } 56 | 57 | options.backend.on(options.failureEvent, this.notok.bind(this)); 58 | options.backend.on(options.successEvent, this.ok.bind(this)); 59 | } 60 | } 61 | 62 | Prober.detectBy = { 63 | CALLBACK: 'callback', 64 | EVENT: 'event', 65 | BOTH: 'both' 66 | }; 67 | 68 | Prober.prototype.isHealthy = function isHealthy() { 69 | return this.bitRing.length < this.window || 70 | this.bitRing.count() >= this.threshold; 71 | }; 72 | 73 | Prober.prototype.isSick = function isSick() { 74 | return !this.isHealthy(); 75 | }; 76 | 77 | Prober.prototype.notok = function notok() { 78 | if (!this.enabled) { 79 | return; 80 | } 81 | 82 | this._addProbe(false); 83 | if (this.statsd) { 84 | this.statsd.increment('prober.' + this.title + '.probe.notok'); 85 | } 86 | }; 87 | 88 | Prober.prototype.setEnabled = function setEnabled(enabled) { 89 | assert(typeof enabled === 'boolean', 'setEnabled() takes a boolean'); 90 | this.enabled = enabled; 91 | }; 92 | 93 | Prober.prototype.notOk = Prober.prototype.notok; 94 | 95 | Prober.prototype.ok = function ok() { 96 | if (!this.enabled) { 97 | return; 98 | } 99 | 100 | this._addProbe(true); 101 | if (this.statsd) { 102 | this.statsd.increment('prober.' + this.title + '.probe.ok'); 103 | } 104 | }; 105 | 106 | Prober.prototype.setLogger = function setLogger(logger) { 107 | this.logger = logger; 108 | }; 109 | 110 | Prober.prototype.probe = function probe(request, bypass, callback) { 111 | var self = this; 112 | 113 | if (!callback) { 114 | callback = bypass; 115 | } 116 | 117 | var wrappedCallback; 118 | if (this.detectFailuresByCallback) { 119 | wrappedCallback = function(err, resp) { 120 | if (self.isUnhealthyFunc(err, resp)) { 121 | self.notok(); 122 | } else { 123 | self.ok(); 124 | } 125 | 126 | if (callback && typeof callback === 'function') { 127 | callback.apply(null, arguments); 128 | } 129 | }; 130 | } 131 | 132 | this.customProbe(request, bypass, wrappedCallback); 133 | }; 134 | 135 | Prober.prototype.customProbe = function probe(request, bypass, callback) { 136 | if (!callback) { 137 | callback = bypass; 138 | } 139 | 140 | if (!this.enabled) { 141 | return request(callback); 142 | } 143 | 144 | // If the backend is healthy, or it's been enough time 145 | // that we should check to see if the backend is no longer 146 | // sick, then make a request to the backend. 147 | if (this.isHealthy() || this._isPityProbe()) { 148 | if (this.statsd) { 149 | this.statsd.increment('prober.' + this.title + 150 | '.request.performed'); 151 | } 152 | 153 | try { 154 | request(callback); 155 | this.lastBackendRequest = this.now(); 156 | } catch (err) { 157 | this.lastBackendRequest = this.now(); 158 | this.notok(); 159 | 160 | throw err; 161 | } 162 | } else { 163 | if (this.statsd) { 164 | this.statsd.increment('prober.' + this.title + '.request.bypassed'); 165 | } 166 | 167 | if (bypass && typeof bypass === 'function') { 168 | bypass(new Error(this.title + ' backend is unhealthy')); 169 | } 170 | } 171 | }; 172 | 173 | Prober.prototype._addProbe = function addProbe(isOk) { 174 | var logger = this.logger; 175 | var statsd = this.statsd; 176 | 177 | var wasHealthy = this.isHealthy(); 178 | this.bitRing.push(isOk); 179 | var isHealthy = this.isHealthy(); 180 | 181 | if (wasHealthy && !isHealthy) { 182 | if (logger) { 183 | logger.warn(this.title + ' has gotten sick'); 184 | } 185 | if (statsd) { 186 | this.statsd.increment('prober.' + this.title + '.health.sick'); 187 | } 188 | } else if (!wasHealthy && isHealthy) { 189 | this.waitPeriod = this.defaultWaitPeriod; 190 | if (logger) { 191 | logger.warn(this.title + ' has returned to health'); 192 | } 193 | if (statsd) { 194 | this.statsd.increment('prober.' + this.title + '.health.recovered'); 195 | } 196 | } else if (!wasHealthy && !isHealthy) { 197 | if (statsd) { 198 | this.statsd.increment('prober.' + this.title + 199 | '.health.still-sick'); 200 | } 201 | 202 | if (isOk) { 203 | this.waitPeriod /= 2; 204 | if (logger) { 205 | logger.warn(this.title + ' is still sick but last probe was ' + 206 | 'healthy. Decreased wait period to ' + 207 | this.waitPeriod + 'ms'); 208 | } 209 | } else { 210 | this.waitPeriod *= 2; 211 | 212 | if (this.waitPeriod > this.maxWaitPeriod) { 213 | this.waitPeriod = this.maxWaitPeriod; 214 | if (logger) { 215 | logger.warn(this.title + ' is still sick. Wait period is ' + 216 | 'at its max, ' + this.waitPeriod + 'ms'); 217 | } 218 | } else if (logger) { 219 | logger.warn(this.title + ' is still sick. Increased wait ' + 220 | 'period to ' + this.waitPeriod + 'ms'); 221 | } 222 | } 223 | } else if (statsd) { 224 | this.statsd.increment('prober.' + this.title + '.health.still-healthy'); 225 | } 226 | }; 227 | 228 | Prober.prototype._isPityProbe = function _isPityProbe() { 229 | return this.lastBackendRequest && this.now() >= 230 | (this.lastBackendRequest + this.waitPeriod); 231 | }; 232 | 233 | module.exports = Prober; 234 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var timer = require('time-mock'); 3 | var times = require('lodash.times'); 4 | var test = global.it; 5 | 6 | var Prober = require('../index'); 7 | 8 | var exampleTchannelIsUnhealthyFunc = function isUnhealthy(err, resp) { 9 | if (err) { 10 | // not an exhaustive list, just an example 11 | var serverErrTypes = [ 12 | 'tchannel.request.timeout', 13 | 'tchannel.connection.close', 14 | 'tchannel.connection.reset', 15 | 'tchannel.connection.unknown-reset' 16 | ]; 17 | return serverErrTypes.indexOf(err.type) !== -1; 18 | } 19 | if (resp) { 20 | // not an exhaustive list, just an example 21 | var respServerErrTypes = [ 22 | 'tchannel.busy' 23 | ]; 24 | return resp.ok === false && respServerErrTypes.indexOf(resp.type) !== -1; 25 | } 26 | return false; 27 | }; 28 | 29 | test('Prober is a function', function (end) { 30 | assert.equal(typeof Prober, 'function'); 31 | end(); 32 | }); 33 | 34 | test('Prober should make request with no previous probes', function(end) { 35 | var wasCalled; 36 | var prober = new Prober(); 37 | prober.probe(function() { wasCalled = true; }); 38 | assert.ok(wasCalled); 39 | end(); 40 | }); 41 | 42 | test('can disabled prober', function (end) { 43 | var prober = new Prober({ enabled: false }); 44 | 45 | assert.equal(prober.enabled, false); 46 | 47 | end(); 48 | }); 49 | 50 | test('can reset to enable or disable prober', function(end) { 51 | var prober = new Prober(); 52 | assert.equal(prober.enabled, true); 53 | prober.setEnabled(false); 54 | assert.equal(prober.enabled, false); 55 | try { 56 | prober.setEnabled("false"); 57 | } catch(e) { 58 | assert(e.name === 'AssertionError'); 59 | assert.equal(prober.enabled, false); 60 | } 61 | prober.setEnabled(true); 62 | assert.equal(prober.enabled, true); 63 | end(); 64 | }); 65 | 66 | test('Prober should make request when amount of healthy probes are less than window', function(end) { 67 | var prober = new Prober(); 68 | times(prober.threshold, function() { prober.ok(); }); 69 | 70 | var wasCalled; 71 | 72 | prober.probe(function() { wasCalled = true; }); 73 | 74 | assert.ok(wasCalled); 75 | end(); 76 | }); 77 | 78 | test('should make request when amount of unhealthy probes are less than window', function(end) { 79 | var prober = new Prober(); 80 | times(prober.threshold, function() { prober.notok(); }); 81 | 82 | var wasCalled; 83 | 84 | prober.probe(function() { wasCalled = true; }); 85 | 86 | assert.ok(wasCalled); 87 | end(); 88 | }); 89 | 90 | test('should make request when amount of healthy requests is above threshold', function(end) { 91 | var prober = new Prober(); 92 | times(prober.threshold, function() { prober.ok(); }); 93 | times(prober.window - prober.threshold, function() { prober.notok(); }); 94 | 95 | var wasCalled; 96 | 97 | prober.probe(function() { wasCalled = true; }); 98 | 99 | assert.ok(wasCalled); 100 | end(); 101 | }); 102 | 103 | test('should bypass backend request when amount of unhealthy requests is above threshold', function(end) { 104 | var prober = new Prober(); 105 | times(prober.threshold, function() { prober.notok(); }); 106 | times(prober.window - prober.threshold, function() { prober.ok(); }); 107 | 108 | var backendWasCalled = false; 109 | var callbackWasCalled = false; 110 | 111 | prober.probe( 112 | function() { backendWasCalled = true; }, 113 | function() { callbackWasCalled = true; }); 114 | 115 | assert.ok(!prober.isHealthy()); 116 | assert.ok(!backendWasCalled); 117 | assert.ok(callbackWasCalled); 118 | end(); 119 | }); 120 | 121 | test('should bypass backend request until coming back to health', function(end) { 122 | var prober = new Prober(); 123 | times(prober.window, function() { prober.notok(); }); 124 | 125 | times(prober.threshold, function() { 126 | assert.ok(!prober.isHealthy()); 127 | prober.probe(assert.fail); 128 | prober.ok(); 129 | }); 130 | 131 | // After healthy threshold has been hit, backend should be healthy 132 | assert.ok(prober.isHealthy()); 133 | 134 | var wasCalled = false; 135 | prober.probe(function() { wasCalled = true; }); 136 | assert.ok(wasCalled); 137 | end(); 138 | }); 139 | 140 | test('should be healthy after returning to health', function(end) { 141 | var prober = new Prober(); 142 | times(prober.window, function() { prober.notok(); }); 143 | times(prober.threshold - 1, function() { prober.ok(); }); 144 | 145 | prober.probe(function() { }); 146 | 147 | assert.ok(!prober.isHealthy()); 148 | 149 | // Returns backend back to health 150 | prober.ok(); 151 | 152 | assert.ok(prober.isHealthy()); 153 | end(); 154 | }); 155 | 156 | test('should be unhealthy after getting sick', function(end) { 157 | var prober = new Prober(); 158 | times(prober.threshold, function() { prober.ok(); }); 159 | times(prober.window - prober.threshold, function() { prober.notok(); }); 160 | 161 | prober.probe(function() { }); 162 | 163 | assert.ok(prober.isHealthy()); 164 | 165 | // Gets sick 166 | prober.notok(); 167 | 168 | assert.ok(!prober.isHealthy()); 169 | end(); 170 | }); 171 | 172 | test('should have default wait period after becoming sick', function(end) { 173 | var prober = new Prober(); 174 | // Set to healthy 175 | times(prober.window, function() { prober.ok(); }); 176 | 177 | // Close to becoming sick 178 | times(prober.window - prober.threshold, function() { prober.notok(); }); 179 | 180 | assert.ok(prober.isHealthy()); 181 | 182 | prober.notok(); 183 | 184 | assert.ok(prober.isSick()); 185 | assert.equal(prober.waitPeriod, prober.defaultWaitPeriod); 186 | end(); 187 | }); 188 | 189 | test('should allow backend request only after wait period', function(end) { 190 | // create a fake timer. 191 | var clock = timer(Date.now()); 192 | var prober = new Prober({ 193 | // overwrite now to be a fake Date.now() based on our clock 194 | now: clock.now 195 | }); 196 | 197 | prober.waitPeriod = prober.maxWaitPeriod / 2; 198 | 199 | // Set to healthy 200 | times(prober.window, function() { prober.ok(); }); 201 | 202 | // Will set wait period to twice as long 203 | times(prober.window - prober.threshold + 2, function() { prober.notok(); }); 204 | 205 | // Should not call `assert.fail` 206 | prober.probe(assert.fail); 207 | 208 | // Simulate time after wait period 209 | clock.advance(prober.waitPeriod); 210 | 211 | var called = false; 212 | prober.probe(function () { 213 | called = true; 214 | }); 215 | 216 | // Backend request was made 217 | assert.ok(called); 218 | 219 | end(); 220 | }); 221 | 222 | test('should be unhealthy after request err', function(end) { 223 | var prober = new Prober(); 224 | prober.threshold = 1; 225 | prober.window = 1; 226 | 227 | prober.probe(function(fn) { 228 | fn(new Error('Some kind of bad going on')); 229 | }); 230 | 231 | assert.ok(prober.isSick()); 232 | end(); 233 | }); 234 | 235 | test('should be unhealthy after HTTP request server err', function(end) { 236 | var prober = new Prober(); 237 | prober.threshold = 1; 238 | prober.window = 1; 239 | 240 | prober.probe(function(fn) { 241 | fn(null, { 242 | statusCode: 500 243 | }); 244 | }); 245 | 246 | assert.ok(prober.isSick()); 247 | end(); 248 | }); 249 | 250 | test('should be healthy after HTTP request client err', function(end) { 251 | var prober = new Prober(); 252 | prober.threshold = 1; 253 | prober.window = 1; 254 | 255 | prober.probe(function(fn) { 256 | fn(null, { 257 | statusCode: 400 258 | }); 259 | }); 260 | 261 | assert.ok(prober.isHealthy()); 262 | end(); 263 | }); 264 | 265 | test('should be healthy after tchannel request server success', function(end) { 266 | var prober = new Prober({ 267 | threshold: 1, 268 | window: 1, 269 | isUnhealthyFunc: exampleTchannelIsUnhealthyFunc 270 | }); 271 | 272 | prober.probe(function(fn) { 273 | fn(null, { 274 | ok: true, 275 | body: {} 276 | }); 277 | }); 278 | 279 | assert.ok(prober.isHealthy()); 280 | end(); 281 | }); 282 | 283 | test('should be unhealthy after tchannel request server err', function(end) { 284 | var prober = new Prober({ 285 | threshold: 1, 286 | window: 1, 287 | isUnhealthyFunc: exampleTchannelIsUnhealthyFunc 288 | }); 289 | 290 | prober.probe(function(fn) { 291 | fn({ type: 'tchannel.request.timeout' }); 292 | }); 293 | 294 | assert.ok(prober.isSick()); 295 | end(); 296 | }); 297 | 298 | test('should be unhealthy after tchannel request server err from resp', function(end) { 299 | var prober = new Prober({ 300 | threshold: 1, 301 | window: 1, 302 | isUnhealthyFunc: exampleTchannelIsUnhealthyFunc 303 | }); 304 | 305 | prober.probe(function(fn) { 306 | fn(null, { 307 | ok: false, 308 | type: 'tchannel.busy' 309 | }); 310 | }); 311 | 312 | assert.ok(prober.isSick()); 313 | end(); 314 | }); 315 | 316 | test('should be healthy after custom handling expected error', function(end) { 317 | var prober = new Prober(); 318 | prober.threshold = 1; 319 | prober.window = 1; 320 | 321 | prober.customProbe(function(fn) { 322 | fn(new Error('Some kind of bad going on')); 323 | }, function() { 324 | prober.ok(); 325 | }); 326 | 327 | assert.ok(prober.isHealthy()); 328 | end(); 329 | }); 330 | 331 | --------------------------------------------------------------------------------