├── .gitignore ├── package.json ├── test ├── index.js ├── test_utils.js ├── stats.js ├── inline.js ├── failpoint.js └── failpoints.js ├── failpoint.js ├── README.md └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # Compiled binary addons (http://nodejs.org/api/addons.html) 20 | build/Release 21 | 22 | # Dependency directory 23 | # Commenting this out is preferred by some people, see 24 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 25 | node_modules 26 | 27 | # Users Environment Variables 28 | .lock-wscript 29 | 30 | # Other 31 | build 32 | 33 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "failpoints", 3 | "version": "2.2.0", 4 | "description": "A failpoints javascript library", 5 | "keywords": [], 6 | "repository": "git://github.com/uber/failpointsjs.git", 7 | "main": "index", 8 | "contributors": [ 9 | "Rob Skillington " 10 | ], 11 | "dependencies": { 12 | "es6-map": "^0.1.1", 13 | "individual": "^3.0.0", 14 | "robb": "^0.2.1", 15 | "uuid": "^2.0.1" 16 | }, 17 | "devDependencies": { 18 | "istanbul": "^0.3.14", 19 | "opn": "^1.0.2", 20 | "tap-spec": "^3.0.0", 21 | "tape": "^4.0.0", 22 | "uber-licence": "^1.5.1", 23 | "uber-standard": "^3.6.4" 24 | }, 25 | "scripts": { 26 | "test": "npm run lint && node test/index.js | tap-spec", 27 | "just-test": "node test/index.js | tap-spec", 28 | "add-licence": "uber-licence", 29 | "check-licence": "uber-licence --dry", 30 | "cover": "istanbul cover --print detail --report html test/index.js && istanbul check-coverage --branches=100", 31 | "just-cover": "istanbul cover --print detail --report html test/index.js", 32 | "lint": "uber-standard", 33 | "view-cover": "opn coverage/index.html" 34 | }, 35 | "pre-commit": [ 36 | "check-licence", 37 | "cover" 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | 'use strict'; 22 | 23 | require('./failpoint'); 24 | require('./failpoints'); 25 | require('./inline'); 26 | require('./stats'); 27 | -------------------------------------------------------------------------------- /test/test_utils.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | 'use strict'; 22 | /* eslint max-len: 0 */ 23 | 24 | module.exports.mockDateTypeWithFixedDateNow = function mockDateTypeWithFixedDateNow(fixed) { 25 | if (typeof fixed !== 'number') { 26 | fixed = 1; 27 | } 28 | return { 29 | now: function now() { 30 | return fixed; 31 | }, 32 | set: function set(value) { 33 | fixed = value; 34 | } 35 | }; 36 | }; 37 | -------------------------------------------------------------------------------- /test/stats.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | 'use strict'; 22 | /* eslint max-len: 0 */ 23 | 24 | var createFailpoints = require('../').create; 25 | var mockDateTypeWithFixedDateNow = require('./test_utils').mockDateTypeWithFixedDateNow; 26 | var test = require('tape'); 27 | 28 | test('failpoint setTime, triggerCount and lastTriggered all records correctly', function t(assert) { 29 | var myFailpointCallCount = 0; 30 | 31 | var failpoints = createFailpoints(); 32 | failpoints.Date = mockDateTypeWithFixedDateNow(); 33 | 34 | methodUnderTest(); 35 | 36 | failpoints.set('my_failpoint', true); 37 | 38 | methodUnderTest(); 39 | methodUnderTest(); 40 | 41 | assert.deepEqual(failpoints.get('my_failpoint'), { 42 | name: 'my_failpoint', 43 | probability: 1.0, 44 | maxCount: null, 45 | maxDurationMs: null, 46 | args: null, 47 | setTime: 1, 48 | triggerCount: 2, 49 | lastTriggered: 1 50 | }); 51 | 52 | failpoints.set('my_failpoint', false); 53 | 54 | methodUnderTest(); 55 | 56 | assert.equal(myFailpointCallCount, 2); 57 | 58 | assert.equal(failpoints.get('my_failpoint'), undefined); 59 | 60 | assert.end(); 61 | 62 | function methodUnderTest() { 63 | failpoints.inlineSync('my_failpoint', function onShouldFail() { 64 | myFailpointCallCount++; 65 | }); 66 | } 67 | }); 68 | -------------------------------------------------------------------------------- /test/inline.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | 'use strict'; 22 | /* eslint max-len: 0 */ 23 | 24 | var createFailpoints = require('../').create; 25 | var mockDateTypeWithFixedDateNow = require('./test_utils').mockDateTypeWithFixedDateNow; 26 | var test = require('tape'); 27 | 28 | test('inline failpoint can fail', function t(assert) { 29 | var myFailpointCallCount = 0; 30 | 31 | var failpoints = createFailpoints(); 32 | failpoints.Date = mockDateTypeWithFixedDateNow(); 33 | 34 | methodUnderTest(); 35 | 36 | failpoints.set('my_failpoint', true); 37 | 38 | methodUnderTest(); 39 | methodUnderTest(); 40 | 41 | failpoints.set('my_failpoint', false); 42 | 43 | methodUnderTest(); 44 | 45 | assert.equal(myFailpointCallCount, 2); 46 | 47 | assert.end(); 48 | 49 | function methodUnderTest() { 50 | failpoints.inline('my_failpoint', function onShouldFail() { 51 | myFailpointCallCount++; 52 | }); 53 | } 54 | }); 55 | 56 | test('inlineConditionally failpoint can fail', function t(assert) { 57 | var myFailpointCallCount = 0; 58 | var avoidedMyFailpointCallCount = 0; 59 | 60 | var failpoints = createFailpoints(); 61 | failpoints.Date = mockDateTypeWithFixedDateNow(); 62 | 63 | failpoints.set('my_maybe_failpoint', {probability: 1, args: {userId: 456}}); 64 | 65 | methodUnderTestWithUserId(123); 66 | methodUnderTestWithUserId(456); 67 | 68 | assert.equal(myFailpointCallCount, 1); 69 | assert.equal(avoidedMyFailpointCallCount, 1); 70 | 71 | assert.end(); 72 | 73 | function methodUnderTestWithUserId(userId) { 74 | failpoints.inlineConditionally('my_maybe_failpoint', function shouldAllow(args) { 75 | return args.userId === userId; 76 | }, function onShouldFail() { 77 | myFailpointCallCount++; 78 | }, function onNormally() { 79 | avoidedMyFailpointCallCount++; 80 | }); 81 | } 82 | }); 83 | -------------------------------------------------------------------------------- /test/failpoint.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | 'use strict'; 22 | /* eslint max-len: 0 */ 23 | 24 | var Failpoint = require('../failpoint'); 25 | var test = require('tape'); 26 | 27 | test('Failpoint throws when name not set correctly', function t(assert) { 28 | assert.throws(function tryPassBadName() { 29 | Failpoint(); 30 | }, /name/); 31 | assert.end(); 32 | }); 33 | 34 | test('Failpoint.setState throws on bad probability', function t(assert) { 35 | assert.throws(function tryPassBadProbability() { 36 | var failpoint = new Failpoint({name: 'test'}); 37 | failpoint.setState({probability: 2.0}); 38 | }); 39 | assert.end(); 40 | }); 41 | 42 | test('Failpoint.setState throws on bad maxCount', function t(assert) { 43 | assert.throws(function tryPassBadProbability() { 44 | var failpoint = new Failpoint({name: 'test'}); 45 | failpoint.setState({probability: 1.0, maxCount: -1}); 46 | }); 47 | assert.end(); 48 | }); 49 | 50 | test('Failpoint.setState throws on bad maxDurationMs', function t(assert) { 51 | assert.throws(function tryPassBadProbability() { 52 | var failpoint = new Failpoint({name: 'test'}); 53 | failpoint.setState({probability: 1.0, maxDurationMs: -1}); 54 | }); 55 | assert.end(); 56 | }); 57 | 58 | test('Failpoint.setState throws on bad args', function t(assert) { 59 | assert.throws(function tryPassBadProbability() { 60 | var failpoint = new Failpoint({name: 'test'}); 61 | failpoint.setState({probability: 1.0, args: 'args'}); 62 | }); 63 | assert.end(); 64 | }); 65 | 66 | test('Failpoint.shouldFail returns true if random() <= probability', function t(assert) { 67 | function MockMath() { } 68 | MockMath.random = function mockMathRandom() { 69 | return 0.2; 70 | }; 71 | 72 | var failpoint = new Failpoint({name: 'test', Math: MockMath}); 73 | failpoint.setState({probability: 0.5}); 74 | assert.equal(failpoint.shouldFail(), true); 75 | assert.end(); 76 | }); 77 | 78 | test('Failpoint.shouldFail returns false if random() > probability', function t(assert) { 79 | function MockMath() { } 80 | MockMath.random = function mockMathRandom() { 81 | return 0.8; 82 | }; 83 | 84 | var failpoint = new Failpoint({name: 'test', Math: MockMath}); 85 | failpoint.setState({probability: 0.5}); 86 | assert.equal(failpoint.shouldFail(), false); 87 | assert.end(); 88 | }); 89 | 90 | test('Failpoint.shouldFail returns false if exceeds maxCount', function t(assert) { 91 | var failpoint = new Failpoint({name: 'test'}); 92 | failpoint.setState({probability: 1.0, maxCount: 1}); 93 | assert.equal(failpoint.shouldFail(), true); 94 | assert.equal(failpoint.shouldFail(), false); 95 | assert.end(); 96 | }); 97 | 98 | test('Failpoint.shouldFail returns false if exceeds maxDurationMs', function t(assert) { 99 | var i = 0; 100 | var start = Date.now(); 101 | var failpoint = new Failpoint({ 102 | name: 'test', 103 | Date: { 104 | now: function mockNow() { 105 | var value = i++ > 1 ? start + 101 : start; 106 | return value; 107 | } 108 | } 109 | }); 110 | failpoint.setState({probability: 1.0, maxDurationMs: 100}); 111 | assert.equal(failpoint.shouldFail(), true); 112 | assert.equal(failpoint.shouldFail(), false); 113 | assert.end(); 114 | }); 115 | 116 | test('Failpoint.shouldFail returns false through hitMaxLimits fast path if exceeded maxCount', function t(assert) { 117 | var failpoint = new Failpoint({name: 'test'}); 118 | failpoint.setState({probability: 1.0, maxCount: 1}); 119 | assert.equal(failpoint.shouldFail(), true); 120 | assert.equal(failpoint.shouldFail(), false); 121 | assert.equal(failpoint.hitMaxLimits, true); 122 | assert.equal(failpoint.shouldFail(), false); 123 | assert.end(); 124 | }); 125 | -------------------------------------------------------------------------------- /failpoint.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | 'use strict'; 22 | /* eslint max-len: 0, max-statements: 0 */ 23 | 24 | module.exports = Failpoint; 25 | 26 | var EventEmitter = require('events').EventEmitter; 27 | var robb = require('robb/src/robb'); 28 | var util = require('util'); 29 | 30 | function Failpoint(options) { 31 | options = options || {}; 32 | 33 | if (typeof options.name !== 'string') { 34 | throw new Error('Requires `name` as string'); 35 | } 36 | 37 | this.name = options.name; 38 | this.failpoints = options.failpoints; 39 | this.Math = options.Math || Math; 40 | this.Date = options.Date || Date; 41 | this.probability = 0.0; 42 | this.maxCount = null; 43 | this.maxDurationMs = null; 44 | this.args = null; 45 | this.setTime = null; 46 | this.triggerCount = null; 47 | this.lastTriggered = null; 48 | this.hitMaxLimits = false; 49 | } 50 | 51 | util.inherits(Failpoint, EventEmitter); 52 | 53 | Failpoint.prototype.toJSON = function toJSON() { 54 | var self = this; 55 | return { 56 | name: self.name, 57 | probability: self.probability, 58 | maxCount: self.maxCount, 59 | maxDurationMs: self.maxDurationMs, 60 | args: self.args, 61 | setTime: self.setTime, 62 | triggerCount: self.triggerCount, 63 | lastTriggered: self.lastTriggered 64 | }; 65 | }; 66 | 67 | Failpoint.prototype.setState = function setState(options) { 68 | /* eslint complexity: 0 */ 69 | var self = this; 70 | if (typeof options === 'boolean') { 71 | if (options) { 72 | self.probability = 1.0; 73 | } else { 74 | self.probability = 0.0; 75 | } 76 | self.maxCount = null; 77 | self.maxDurationMs = null; 78 | self.args = null; 79 | return self._resetStatsVars(); 80 | } 81 | 82 | if (!options || 83 | typeof options.probability !== 'number' || 84 | !(options.probability >= 0.0 && options.probability <= 1.0)) { 85 | throw new Error('`options.probability` not between 0.0 to 1.0'); 86 | } 87 | 88 | if (options.maxCount !== null && 89 | options.maxCount !== undefined && 90 | (!robb.isInt(options.maxCount) || options.maxCount <= 0)) { 91 | throw new Error('`options.maxCount` not an integer greater than 0'); 92 | } 93 | 94 | if (options.maxDurationMs !== null && 95 | options.maxDurationMs !== undefined && 96 | (typeof options.maxDurationMs !== 'number' || options.maxDurationMs <= 0)) { 97 | throw new Error('`options.maxDurationMs` not a duration greater than 0ms'); 98 | } 99 | 100 | if (options.args !== null && 101 | options.args !== undefined && 102 | (typeof options.args !== 'object' || 103 | !Array.isArray(Object.keys(options.args)))) { 104 | throw new Error('`options.args` not an object with at least 1 key'); 105 | } 106 | 107 | self.probability = options.probability; 108 | self.maxCount = options.maxCount || null; 109 | self.maxDurationMs = options.maxDurationMs || null; 110 | self.args = options.args || null; 111 | self._resetStatsVars(); 112 | }; 113 | 114 | Failpoint.prototype._resetStatsVars = function resetStatsVars() { 115 | var self = this; 116 | self.hitMaxLimits = false; 117 | if (self.probability > 0.0) { 118 | self.emit('active', self); 119 | self.setTime = self.Date.now(); 120 | self.triggerCount = 0; 121 | self.lastTriggered = null; 122 | } else { 123 | self.emit('inactive', self); 124 | self.setTime = null; 125 | self.triggerCount = null; 126 | self.lastTriggered = null; 127 | } 128 | }; 129 | 130 | Failpoint.prototype.shouldFail = function shouldFail() { 131 | var self = this; 132 | 133 | if (self.hitMaxLimits) { 134 | // Fast path for avoiding counts & times 135 | return false; 136 | } 137 | 138 | if (self.maxCount !== null && self.triggerCount >= self.maxCount) { 139 | // Cache hitting the max count limit 140 | self.hitMaxLimits = true; 141 | self.emit('inactive', self); 142 | return false; 143 | } 144 | 145 | if (self.maxDurationMs !== null && (self.Date.now() - self.setTime) > self.maxDurationMs) { 146 | // Cache hitting the max duration limit 147 | self.hitMaxLimits = true; 148 | self.emit('inactive', self); 149 | return false; 150 | } 151 | 152 | var didPassProbabilityTest = false; 153 | if (self.probability === 1.0 || self.Math.random() <= self.probability) { 154 | didPassProbabilityTest = true; 155 | } 156 | 157 | if (!didPassProbabilityTest) { 158 | return false; 159 | } 160 | 161 | self.triggerCount++; 162 | self.lastTriggered = self.Date.now(); 163 | 164 | return true; 165 | }; 166 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # failpoints 2 | 3 | Add custom probabilistic and conditional failure points to your code and control when they execute. 4 | 5 | Why? Because sometimes you want to be ready for when things get funky. 6 | 7 | ## Motivation 8 | 9 | Failpoints becomes useful when there are many libraries and components that compose a piece of software and you want to namespace each one and toggle them on remotely. 10 | 11 | ## Example 12 | 13 | Dynamically set failpoints in your request handlers and also toggle on embedded library failpoints. 14 | 15 | Your server: 16 | 17 | ```js 18 | var Failpoints = require('failpoints'); 19 | var libraryThatUsesFailpoints = require('library-name'); 20 | var http = require('http'); 21 | var request = require('request'); 22 | 23 | var myServiceFailpoints = Failpoints.createWithNamespace('my-service'); 24 | 25 | var server = http.createServer(function onRequest(req, res) { 26 | if (myServiceFailpoints.shouldFail('all_requests')) { 27 | res.writeHead(500, {'Content-Type': 'application/json'}); 28 | return res.end(JSON.stringify({error: 'Failing on purpose'})); 29 | } 30 | 31 | libraryThatUsesFailpoints.doWork(req, function onWorkDone(err, result) { 32 | res.writeHead(200, {'Content-Type': 'application/json'}); 33 | res.end(JSON.stringify(result)); 34 | }); 35 | }); 36 | server.listen(8080); 37 | 38 | // Dynamically update failpoints by asking for installed failpoints for this service 39 | var since = 0; 40 | setInterval(function pollForFailpoints() { 41 | var options = { 42 | uri: 'http://cfgsrv/failpoints?service=my-service&since=' + since, 43 | json: true 44 | }; 45 | request(options, function onResponse(err, res, body) { 46 | if (err) { 47 | return console.error('Error: ' + err); 48 | } 49 | 50 | // Example response that fails all requests 20% of 51 | // the time and the library work 50% of the time: 52 | // { 53 | // "since": 1432928157, 54 | // "set": { 55 | // "my-service": [ 56 | // { 57 | // "name": "all_requests", 58 | // "options": { 59 | // "probability": 0.2, 60 | // "maxDurationMs": 1000 61 | // } 62 | // } 63 | // ], 64 | // "library-name": [ 65 | // { 66 | // "name": "all_work", 67 | // "options": { 68 | // "probability": 0.5, 69 | // "maxDurationMs": 1000 70 | // } 71 | // } 72 | // ] 73 | // } 74 | // } 75 | 76 | since = body.now; 77 | 78 | Object.keys(body.set).forEach(function eachNamespace(namespace) { 79 | var failpoints = Failpoints.getOrCreateFailpointsWithNamespace(namespace); 80 | var array = body.set[namespace]; 81 | array.forEach(function eachFailpointToSet(failpoint) { 82 | failpoints.set(failpoint.name, failpoint.options); 83 | }); 84 | }); 85 | }); 86 | }, 60000); 87 | ``` 88 | 89 | ## Usage 90 | 91 | ### Basic 92 | 93 | ```js 94 | var failpoints = require('failpoints').create(); 95 | failpoints.set('my_failpoint', {probability: 0.5, maxDurationMs: 100}); 96 | 97 | if (failpoints.shouldFail('my_failpoint')) { 98 | // 50-50 chance this will be entered in the next 100ms after setting the failpoint 99 | // Do things here instead doing your work, perhaps return early too 100 | } 101 | 102 | // Do normal things 103 | ``` 104 | 105 | #### Set options 106 | 107 | The second argument passed to `set` should be `Boolean` or `Object`. If `Object` the following keys can be used: 108 | 109 | ``` 110 | probability {Number} - Float between 0-1 with probability on each call to be on or off 111 | maxCount {Number} - Integer of many times at most to invoke as on before reducing to probability 0 112 | maxDurationMs {Number} - Integer of milliseconds to run at most before reducing probability to 0 113 | args {Object} - Arguments to pass to failpoint selection method if provided at failpoint, will also provide to failure method 114 | ``` 115 | 116 | The `set` method will also return `true` if failpoint state set or `false` if problem setting the failpoint. If `false` you can supply a logger to inspect any errors that might occur trying to set the failpoint: 117 | 118 | ```js 119 | myFailpointsInstance.logger = {info: ..., warn: ..., error: ...}; 120 | ``` 121 | 122 | ### Async inline example 123 | 124 | ```js 125 | var failpoints = require('failpoints').create(); 126 | 127 | // ... 128 | 129 | function getFileContents(file, callback) { 130 | failpoints.inline('read_file_failpoint', function onShouldFail() { 131 | // This method will only get called if should fail 132 | callback(new Error('Synthetic error from failpoint invocation')); 133 | 134 | }, function onNormally() { 135 | // This method will only get called if shouldn't fail 136 | fs.readFileSync(file, {encoding: 'utf8'}, callback); 137 | 138 | }); 139 | } 140 | 141 | getFileContents('./my_file.txt', function onDone(err, contents) { 142 | // Did get file contents 143 | 144 | failpoints.set('read_file_failpoint', true); 145 | 146 | getFileContents('./my_file.txt', function onDone(err, contents) { 147 | // Will fail 148 | 149 | // Can turn off further invocations 150 | failpoints.set('read_file_failpoint', false); 151 | }); 152 | }); 153 | ``` 154 | 155 | ### Set all on and off 156 | 157 | ```js 158 | var failpoints = require('failpoints').create(); 159 | 160 | // setup failpoints 161 | 162 | failpoints.setAll(true); 163 | failpoints.setAll({probability: 0.5}); 164 | failpoints.setAll(false); 165 | 166 | ``` 167 | 168 | ### Control failpoints across files and modules with namespaces 169 | 170 | somefile.js 171 | 172 | ```js 173 | var Failpoints = require('failpoints'); 174 | var failpoints = Failpoints.getOrCreateFailpointsWithNamespace('myNamespace'); 175 | 176 | function myMethod() { 177 | var didFail = failpoints.inlineSync('my_method_failpoint', function onShouldFail() { 178 | // Do bad things in sync 179 | }); 180 | } 181 | ``` 182 | 183 | otherfile.js 184 | 185 | ```js 186 | var Failpoints = require('failpoints'); 187 | var failpoints = Failpoints.getOrCreateFailpointsWithNamespace('myNamespace'); 188 | 189 | failpoints.set('my_method_failpoint', {probability: 0.5, maxDurationMs: 3000}); 190 | ``` 191 | 192 | You can see the tests for more advanced usages. 193 | 194 | ## Installation 195 | 196 | `npm install failpoints` 197 | 198 | ## Tests 199 | 200 | `npm test` 201 | 202 | ## MIT Licensed 203 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | 'use strict'; 22 | /* eslint max-len: 0, max-statements: 0 */ 23 | 24 | module.exports = Failpoints; 25 | 26 | /* eslint-disable */ 27 | var Map = require('es6-map'); 28 | /* eslint-enable */ 29 | 30 | var EventEmitter = require('events').EventEmitter; 31 | var Failpoint = require('./failpoint'); 32 | var Individual = require('individual'); 33 | var util = require('util'); 34 | var UUID = require('uuid'); 35 | 36 | var CONST = {}; 37 | CONST.DEFAULT_NAMESPACE = 'DEFAULT_NAMESPACE'; 38 | CONST.NO_ARGS = Object.freeze({}); 39 | CONST = Object.freeze(CONST); 40 | 41 | function Failpoints(options) { 42 | options = options || {}; 43 | 44 | if (typeof options.namespace !== 'string') { 45 | throw new Error('Requires `namespace` as string'); 46 | } 47 | 48 | this.namespace = options.namespace; 49 | this.activeFailpoints = new Map(); 50 | this.Math = options.Math || Math; 51 | this.Date = options.Date || Date; 52 | } 53 | 54 | util.inherits(Failpoints, EventEmitter); 55 | 56 | Failpoints.CONST = CONST; 57 | 58 | Failpoints.failpointsByNamespace = Individual('__FAILPOINTS_BY_NAMESPACE', new Map()); 59 | 60 | Failpoints.getOrCreateFailpointsWithNamespace = function getOrCreateFailpointsWithNamespace(namespace) { 61 | var byNamespace = Failpoints.failpointsByNamespace; 62 | var failpoints = byNamespace.get(namespace); 63 | if (!failpoints) { 64 | failpoints = new Failpoints({namespace: namespace}); 65 | byNamespace.set(namespace, failpoints); 66 | } 67 | return failpoints; 68 | }; 69 | 70 | Failpoints.getFailpointsWithDefaultNamespace = function getFailpointsWithDefaultNamespace() { 71 | return Failpoints.getOrCreateFailpointsWithNamespace(CONST.DEFAULT_NAMESPACE); 72 | }; 73 | 74 | Failpoints.createWithNamespace = function createWithNamespace(namespace) { 75 | return this.getOrCreateFailpointsWithNamespace(namespace); 76 | }; 77 | 78 | Failpoints.create = function createUntracked() { 79 | return new Failpoints({namespace: 'Failpoints-' + UUID.v4()}); 80 | }; 81 | 82 | /** 83 | * Set a failpoint's state. Can either be a registered failpoint or not-yet-registered failpoint. 84 | * 85 | * Will throw on bad options. 86 | * 87 | * @param {String} name - Name of the failpoint to enable/disable 88 | * @param {Boolean|Object} options - If boolean then turn it completely on or off, otherwise set as options object 89 | */ 90 | Failpoints.prototype.set = function set(name, options) { 91 | /* 92 | * options: 93 | * probability {Number} Float between 0-1 with probability on each call to be on or off 94 | * maxCount {Number} Integer of many times at most to invoke as on before reducing to probability 0 95 | * maxDurationMs {Number} Integer of milliseconds to run at most before reducing probability to 0 96 | * args {Object} Arguments to pass to failpoint selection method if provided at failpoint 97 | */ 98 | var self = this; 99 | 100 | var failpoint = self.activeFailpoints.get(name); 101 | if (!failpoint) { 102 | failpoint = new Failpoint({ 103 | name: name, 104 | failpoints: self, 105 | Math: self.Math, 106 | Date: self.Date 107 | }); 108 | failpoint.once('inactive', function onInactive() { 109 | // This failpoint has reached limit or set inactive, removing to increase performance 110 | var sizeBeforeDeletion = self.activeFailpoints.size; 111 | self.activeFailpoints.delete(name); 112 | var sizeAfterDeletion = self.activeFailpoints.size; 113 | self.emit('activeCount', sizeAfterDeletion); 114 | 115 | if (sizeAfterDeletion === 0 && sizeBeforeDeletion > 0) { 116 | self.emit('becameInactive'); 117 | } 118 | }); 119 | var sizeBeforeInsertion = self.activeFailpoints.size; 120 | self.activeFailpoints.set(name, failpoint); 121 | self.emit('activeCount', self.activeFailpoints.size); 122 | if (sizeBeforeInsertion === 0) { 123 | self.emit('becameActive'); 124 | } 125 | } 126 | 127 | failpoint.setState(options); 128 | }; 129 | 130 | /** 131 | * Set all known failpoints' state. 132 | * 133 | * Will throw on bad options. 134 | * 135 | * @param {Boolean|Object} options - If boolean then turn it completely on or off, otherwise set as options object 136 | */ 137 | Failpoints.prototype.setAll = function setAll(options) { 138 | var self = this; 139 | self.activeFailpoints.forEach(function eachFailpoint(failpoint, name) { 140 | failpoint.setState(options); 141 | }); 142 | }; 143 | 144 | Failpoints.prototype.get = function get(name) { 145 | var self = this; 146 | var failpoint = self.activeFailpoints.get(name); 147 | if (failpoint) { 148 | return failpoint.toJSON(); 149 | } 150 | return undefined; 151 | }; 152 | 153 | Failpoints.prototype.getArgs = function getArgs(name) { 154 | var self = this; 155 | var failpoint = self.activeFailpoints.get(name); 156 | if (!failpoint) { 157 | return undefined; 158 | } 159 | return failpoint.args || CONST.NO_ARGS; 160 | }; 161 | 162 | Failpoints.prototype.shouldFail = function shouldFail(name) { 163 | var self = this; 164 | var failpoint = self.activeFailpoints.get(name); 165 | if (!failpoint) { 166 | return false; 167 | } 168 | return failpoint.shouldFail(); 169 | }; 170 | 171 | Failpoints.prototype.shouldFailConditionally = function shouldFailConditionally(name, shouldAllow) { 172 | var self = this; 173 | if (typeof shouldAllow !== 'function') { 174 | throw new Error('shouldFailConditionally does not have shouldAllow callback set'); 175 | } 176 | var failpoint = self.activeFailpoints.get(name); 177 | if (!failpoint) { 178 | return false; 179 | } 180 | var args = failpoint.args || CONST.NO_ARGS; 181 | // Explicitly check hitMaxLimits first to avoid 182 | // potentially expensive shouldAllow function call 183 | return !failpoint.hitMaxLimits && shouldAllow(args) && failpoint.shouldFail(); 184 | }; 185 | 186 | Failpoints.prototype.inline = function inline(name, onShouldFail, onNormally) { 187 | var self = this; 188 | if (typeof onShouldFail !== 'function') { 189 | throw new Error('inline does not have onShouldFail callback set'); 190 | } 191 | if (self.shouldFail(name)) { 192 | onShouldFail(self.getArgs(name)); 193 | return true; 194 | } else if (typeof onNormally === 'function') { 195 | onNormally(); 196 | return false; 197 | } else { 198 | return false; 199 | } 200 | }; 201 | 202 | Failpoints.prototype.inlineConditionally = function inlineConditionally(name, shouldAllow, onShouldFail, onNormally) { 203 | var self = this; 204 | if (typeof onShouldFail !== 'function') { 205 | throw new Error('inlineConditionally does not have onShouldFail callback set'); 206 | } 207 | if (self.shouldFailConditionally(name, shouldAllow)) { 208 | onShouldFail(self.getArgs(name)); 209 | return true; 210 | } else if (typeof onNormally === 'function') { 211 | onNormally(); 212 | return false; 213 | } else { 214 | return false; 215 | } 216 | }; 217 | 218 | Failpoints.prototype.inlineSync = function inlineSync(name, onShouldFail) { 219 | var self = this; 220 | if (typeof onShouldFail !== 'function') { 221 | throw new Error('inlineSync does not have onShouldFail callback set'); 222 | } 223 | if (self.shouldFail(name)) { 224 | onShouldFail(self.getArgs(name)); 225 | return true; 226 | } else { 227 | return false; 228 | } 229 | }; 230 | 231 | Failpoints.prototype.inlineSyncConditionally = function inlineSyncConditionally(name, shouldAllow, onShouldFail) { 232 | var self = this; 233 | if (typeof onShouldFail !== 'function') { 234 | throw new Error('inlineSyncConditionally does not have onShouldFail callback set'); 235 | } 236 | if (self.shouldFailConditionally(name, shouldAllow)) { 237 | onShouldFail(self.getArgs(name)); 238 | return true; 239 | } else { 240 | return false; 241 | } 242 | }; 243 | -------------------------------------------------------------------------------- /test/failpoints.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | 'use strict'; 22 | /* eslint max-len: 0, max-statements: 0 */ 23 | 24 | var Failpoints = require('../'); 25 | var mockDateTypeWithFixedDateNow = require('./test_utils').mockDateTypeWithFixedDateNow; 26 | var test = require('tape'); 27 | var UUID = require('uuid'); 28 | 29 | test('Failpoints throws when name not set correctly', function t(assert) { 30 | assert.throws(function tryPassBadName() { 31 | Failpoints(); 32 | }, /name/); 33 | assert.end(); 34 | }); 35 | 36 | test('Failpoints.getOrCreateFailpointsWithNamespace can lazy create namespace', function t(assert) { 37 | var uuid = UUID.v4(); 38 | assert.ok(Failpoints.getOrCreateFailpointsWithNamespace('lazy' + uuid)); 39 | assert.end(); 40 | }); 41 | 42 | test('Failpoints.getOrCreateFailpointsWithNamespace can get created namespace', function t(assert) { 43 | var uuid = UUID.v4(); 44 | var created = Failpoints.getOrCreateFailpointsWithNamespace('created' + uuid); 45 | assert.ok(created); 46 | var retrieved = Failpoints.getOrCreateFailpointsWithNamespace('created' + uuid); 47 | assert.equal(retrieved, created); 48 | assert.end(); 49 | }); 50 | 51 | test('Failpoints emits activeCount when adding an active failpoint and when it becomes inactive', function t(assert) { 52 | var activeCountCalls = []; 53 | var becameActiveCallCount = 0; 54 | var becameInactiveCallCount = 0; 55 | var failpoints = new Failpoints({namespace: 'test'}); 56 | failpoints.on('activeCount', onActiveCount); 57 | failpoints.on('becameActive', onBecameActive); 58 | failpoints.on('becameInactive', onBecameInactive); 59 | 60 | failpoints.set('my_failpoint', {probability: 1.0, maxCount: 1}); 61 | assert.equal(becameActiveCallCount, 1); 62 | 63 | failpoints.set('my_second_failpoint', {probability: 1.0, maxCount: 1}); 64 | 65 | failpoints.shouldFail('my_failpoint'); 66 | failpoints.shouldFail('my_failpoint'); 67 | failpoints.shouldFail('my_second_failpoint'); 68 | failpoints.shouldFail('my_second_failpoint'); 69 | assert.equal(becameInactiveCallCount, 1); 70 | 71 | assert.deepEqual(activeCountCalls, [1, 2, 1, 0]); 72 | 73 | assert.end(); 74 | 75 | function onActiveCount(count) { 76 | activeCountCalls.push(count); 77 | } 78 | 79 | function onBecameActive() { 80 | becameActiveCallCount++; 81 | } 82 | 83 | function onBecameInactive() { 84 | becameInactiveCallCount++; 85 | } 86 | }); 87 | 88 | test('Failpoints.get can get failpoint as JSON', function t(assert) { 89 | var mockDate = mockDateTypeWithFixedDateNow(); 90 | var failpoints = new Failpoints({ 91 | namespace: 'test', 92 | Date: mockDate 93 | }); 94 | failpoints.set('my_failpoint', { 95 | probability: 0.5, 96 | maxDurationMs: 100 97 | }); 98 | assert.deepEqual(failpoints.get('my_failpoint'), { 99 | name: 'my_failpoint', 100 | probability: 0.5, 101 | maxCount: null, 102 | maxDurationMs: 100, 103 | args: null, 104 | setTime: mockDate.now(), 105 | triggerCount: 0, 106 | lastTriggered: null 107 | }); 108 | assert.end(); 109 | }); 110 | 111 | test('Failpoints.get returns undefined for unknown failpoint', function t(assert) { 112 | var failpoints = new Failpoints({namespace: 'test'}); 113 | assert.equal(failpoints.get('missing_failpoint'), undefined); 114 | assert.end(); 115 | }); 116 | 117 | test('Failpoints.getArgs returns args for failpoint', function t(assert) { 118 | var failpoints = new Failpoints({namespace: 'test'}); 119 | var args = {my: 'args'}; 120 | failpoints.set('my_failpoint', { 121 | probability: 1.0, 122 | maxDurationMs: 100, 123 | args: args 124 | }); 125 | assert.deepEqual(failpoints.getArgs('my_failpoint'), {my: 'args'}); 126 | assert.end(); 127 | }); 128 | 129 | test('Failpoints.getArgs returns undefined for unknown failpoint', function t(assert) { 130 | var failpoints = new Failpoints({namespace: 'test'}); 131 | assert.equal(failpoints.getArgs('missing_failpoint'), undefined); 132 | assert.end(); 133 | }); 134 | 135 | test('Failpoints.shouldFailConditionally returns empty args object if no args to shouldAllow', function t(assert) { 136 | var failpoints = new Failpoints({namespace: 'test'}); 137 | failpoints.set('my_failpoint', {probability: 1.0}); 138 | var result = failpoints.shouldFailConditionally('my_failpoint', function shouldAllow(args) { 139 | assert.deepEqual(args, {}); 140 | return true; 141 | }); 142 | assert.equal(result, true); 143 | assert.end(); 144 | }); 145 | 146 | test('Failpoints.shouldFailConditionally throws on shouldAllow not set', function t(assert) { 147 | assert.throws(function tryPassBadShouldAllow() { 148 | var failpoints = new Failpoints({namespace: 'test'}); 149 | failpoints.shouldFailConditionally('my_failpoint'); 150 | }); 151 | assert.end(); 152 | }); 153 | 154 | test('Failpoints.shouldFailConditionally returns false on unknown failpoint', function t(assert) { 155 | var failpoints = new Failpoints({namespace: 'test'}); 156 | assert.equal(failpoints.shouldFailConditionally('my_failpoint', shouldAllow), false); 157 | assert.end(); 158 | 159 | function shouldAllow() { 160 | return true; 161 | } 162 | }); 163 | 164 | test('Failpoints.inline calls onNormally when set', function t(assert) { 165 | var onNormallyCallCount = 0; 166 | var failpoints = new Failpoints({namespace: 'test'}); 167 | failpoints.inline('my_failpoint', function shouldAllow() { 168 | return true; 169 | }, function onNormally() { 170 | onNormallyCallCount++; 171 | }); 172 | assert.equal(onNormallyCallCount, 1); 173 | assert.end(); 174 | }); 175 | 176 | test('Failpoints.inline throws on onShouldFail not set', function t(assert) { 177 | assert.throws(function tryPassBadonShouldFail() { 178 | var failpoints = new Failpoints({namespace: 'test'}); 179 | failpoints.inline('my_failpoint'); 180 | }); 181 | assert.end(); 182 | }); 183 | 184 | test('Failpoints.inlineConditionally returns false when should fail without onNormallyCallCount', function t(assert) { 185 | var onShouldFailCallCount = 0; 186 | var failpoints = new Failpoints({namespace: 'test'}); 187 | var result = failpoints.inlineConditionally('my_failpoint', function shouldAllow() { 188 | return true; 189 | }, function onShouldFail() { 190 | onShouldFailCallCount++; 191 | }); 192 | assert.equal(result, false); 193 | assert.equal(onShouldFailCallCount, 0); 194 | assert.end(); 195 | }); 196 | 197 | test('Failpoints.inlineConditionally calls onShouldFail when set', function t(assert) { 198 | var onShouldFailCallCount = 0; 199 | var failpoints = new Failpoints({namespace: 'test'}); 200 | failpoints.set('my_failpoint', {probability: 1.0}); 201 | failpoints.inlineConditionally('my_failpoint', function shouldAllow() { 202 | return true; 203 | }, function onShouldFail() { 204 | onShouldFailCallCount++; 205 | }); 206 | assert.equal(onShouldFailCallCount, 1); 207 | assert.end(); 208 | }); 209 | 210 | test('Failpoints.inlineConditionally throws on onShouldFail not set', function t(assert) { 211 | assert.throws(function tryPassBadonShouldFail() { 212 | var failpoints = new Failpoints({namespace: 'test'}); 213 | failpoints.inlineConditionally('my_failpoint'); 214 | }); 215 | assert.end(); 216 | }); 217 | 218 | test('Failpoints.inlineSync throws on onShouldFail not set', function t(assert) { 219 | assert.throws(function tryPassBadonShouldFail() { 220 | var failpoints = new Failpoints({namespace: 'test'}); 221 | failpoints.inlineSync('my_failpoint'); 222 | }); 223 | assert.end(); 224 | }); 225 | 226 | test('Failpoints.inlineSyncConditionally calls onShouldFail when should fail', function t(assert) { 227 | var onShouldFailCallCount = 0; 228 | var failpoints = new Failpoints({namespace: 'test'}); 229 | failpoints.set('my_failpoint', {probability: 1.0}); 230 | failpoints.inlineSyncConditionally('my_failpoint', function shouldAllow() { 231 | return true; 232 | }, function onShouldFail() { 233 | onShouldFailCallCount++; 234 | }); 235 | assert.equal(onShouldFailCallCount, 1); 236 | assert.end(); 237 | }); 238 | 239 | test('Failpoints.inlineSyncConditionally does not call onShouldFail when should not fail', function t(assert) { 240 | var onShouldFailCallCount = 0; 241 | var failpoints = new Failpoints({namespace: 'test'}); 242 | failpoints.inlineSyncConditionally('my_failpoint', function shouldAllow() { 243 | return true; 244 | }, function onShouldFail() { 245 | onShouldFailCallCount++; 246 | }); 247 | assert.equal(onShouldFailCallCount, 0); 248 | assert.end(); 249 | }); 250 | 251 | test('Failpoints.inlineSyncConditionally throws on onShouldFail not set', function t(assert) { 252 | assert.throws(function tryPassBadonShouldFail() { 253 | var failpoints = new Failpoints({namespace: 'test'}); 254 | failpoints.inlineSyncConditionally('my_failpoint'); 255 | }); 256 | assert.end(); 257 | }); 258 | 259 | test('Failpoints.inlineSyncConditionally throws on onShouldFail not set', function t(assert) { 260 | assert.throws(function tryPassBadonShouldFail() { 261 | var failpoints = new Failpoints({namespace: 'test'}); 262 | failpoints.inlineSyncConditionally('my_failpoint'); 263 | }); 264 | assert.end(); 265 | }); 266 | --------------------------------------------------------------------------------