├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── example.js ├── index.js ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | sandbox.js 3 | .idea -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '8' 4 | - '10' 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fuzzbuzz 2 | 3 | Fuzz testing framework 4 | 5 | ``` 6 | npm install fuzzbuzz 7 | ``` 8 | 9 | ## Usage 10 | 11 | ``` js 12 | const FuzzBuzz = require('fuzzbuzz') 13 | 14 | const fuzz = new FuzzBuzz({ 15 | async setup () { 16 | console.log('do some initial setup') 17 | await sleep(100) 18 | }, 19 | async validate () { 20 | console.log('validate your state') 21 | await sleep(100) 22 | } 23 | }) 24 | 25 | // print out the random seed so things can be reproduced 26 | console.log('random seed is', fuzz.seed) 27 | 28 | // add an operation 29 | fuzz.add(1, async function () { 30 | console.log('hi') 31 | await sleep(100) 32 | }) 33 | 34 | // add another one that should be called ~10 times more 35 | fuzz.add(10, async function () { 36 | console.log('ho') 37 | await sleep(100) 38 | }) 39 | 40 | // run 20 operations 41 | fuzz.run(20) 42 | 43 | function sleep (n) { 44 | return new Promise(resolve => setTimeout(resolve, n)) 45 | } 46 | ``` 47 | 48 | ## API 49 | 50 | #### `fuzz = new FuzzBuzz([options])` 51 | 52 | Make a new fuzz tester. Options include: 53 | 54 | ``` js 55 | { 56 | seed: ..., // pass in a random seed here (32 byte buffer or hex string) 57 | async setup(), // pass in a section function that is run before operations 58 | async validate() // pass in validator that is run after all operations are done 59 | } 60 | ``` 61 | 62 | #### `fuzz.add(weight, async fn)` 63 | 64 | Add a testing operation. `weight` should be a positive integer indicating the ratio you wanna 65 | call this operation compared to other ones. 66 | 67 | `fn` should be a function that does some testing function. Internally it is `awaited` so it is 68 | same to make this an async function. If you are testing a callback API accept a callback in the fn 69 | signature like so `fn(callback)`. 70 | 71 | #### `fuzz.remove(weight, fn)` 72 | 73 | Remove an operation again. 74 | 75 | #### `promise = fuzz.run(times)` 76 | 77 | Run the fuzzer. First runs the setup function, then runs `times` operations where 78 | each operation is picked randomly based on their relative weight. After the operations 79 | are done, the validate function is run. 80 | 81 | The randomness that is used to pick each operation is based on the seed from the constructor 82 | so if you pass the same seed twice the order is deterministic. 83 | 84 | #### `promise = fuzz.bisect(maxTimes)` 85 | 86 | Using a bisection algorithm the fuzzer will find the minimum amount of runs 87 | for either your validation function to fail or for an operation to throw. 88 | 89 | The returned minimum amount of runs are returned afterwards. 90 | 91 | #### `item = fuzz.pick(array)` 92 | 93 | Pick a random element from an array. Uses the random seed as well. 94 | 95 | #### `num = fuzz.random()` 96 | 97 | Roll a random number between 0 and 1. Uses the random seed as well. 98 | 99 | #### `integer = fuzz.randomInt(max)` 100 | 101 | Helper to roll a random integer. 102 | 103 | #### `promise = fuzz.call(operations)` 104 | 105 | Call a random function from an array of weighted functions. Uses the random seed as well. 106 | 107 | The operations array should be an array of `[ weight, async fn ]` pairs. 108 | 109 | #### `fuzz.setup(async fn)` 110 | 111 | Set the setup function after constructing the fuzzer. 112 | 113 | #### `fuzz.validate(async fn)` 114 | 115 | Set the validate function after constructing the fuzzer. 116 | 117 | #### `fuzz.debug(msg)` 118 | 119 | Print out a debug message. Requires `options.debugging = true` or 120 | the DEBUG env var to contain `fuzzbuzz` to print it out. 121 | 122 | ## License 123 | 124 | MIT 125 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | const FuzzBuzz = require('./') 2 | 3 | const fuzz = new FuzzBuzz({ 4 | seed: process.argv[2] || '', 5 | async setup () { 6 | console.log('doing setup') 7 | await sleep(100) 8 | }, 9 | async validate () { 10 | console.log('validating state') 11 | await sleep(100) 12 | } 13 | }) 14 | 15 | console.log('seed is', fuzz.seed) 16 | 17 | fuzz.add(1, async function () { 18 | console.log('hi') 19 | await sleep(100) 20 | }) 21 | 22 | fuzz.add(10, async function () { 23 | console.log('ho') 24 | await sleep(100) 25 | }) 26 | 27 | fuzz.run(20) 28 | 29 | function sleep (n) { 30 | return new Promise(resolve => setTimeout(resolve, n)) 31 | } 32 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const random = require('math-random-seed') 2 | const { promisify } = require('util') 3 | const { randomBytes, createHash } = require('crypto') 4 | 5 | class FuzzBuzz { 6 | constructor (opts) { 7 | if (!opts) opts = {} 8 | 9 | this.seed = opts.seed || randomBytes(32).toString('hex') 10 | this.random = random(createHash('sha256').update(this.seed).digest()) 11 | this.operations = [] 12 | this.debugging = !!opts.debugging || (process.env.DEBUG || '').indexOf('fuzzbuzz') > -1 13 | 14 | if (opts.setup) this._setup = promisifyMaybe(opts.setup) 15 | if (opts.validate) this._validate = promisifyMaybe(opts.validate) 16 | 17 | const operations = opts.operations || [] 18 | for (const [ weight, fn ] of operations) this.add(weight, fn) 19 | 20 | this.debug('seed is ' + this.seed) 21 | } 22 | 23 | setup (fn) { 24 | this._setup = promisifyMaybe(fn) 25 | } 26 | 27 | validate (fn) { 28 | this._validate = promisifyMaybe(fn) 29 | } 30 | 31 | add (weight, fn) { 32 | this.operations.push([ weight, promisifyMaybe(fn), fn ]) 33 | } 34 | 35 | remove (weight, fn) { 36 | for (let i = 0; i < this.operations.length; i++) { 37 | const [ otherWeight, , otherFn ] = this.operations[i] 38 | 39 | if (weight === otherWeight && otherFn === fn) { 40 | this.operations.splice(i, 1) 41 | return true 42 | } 43 | } 44 | 45 | return false 46 | } 47 | 48 | async call (ops) { 49 | let totalWeight = 0 50 | for (const [ weight ] of ops) totalWeight += weight 51 | let n = this.randomInt(totalWeight) 52 | for (const [ weight, op ] of ops) { 53 | n -= weight 54 | if (n < 0) return op.call(this) 55 | } 56 | } 57 | 58 | randomInt (n) { 59 | return Math.floor(this.random() * n) 60 | } 61 | 62 | pick (items) { 63 | return items.length ? items[this.randomInt()] : null 64 | } 65 | 66 | async run (n, opts) { 67 | const validateAll = !!(opts && opts.validateAll) 68 | await this._setup() 69 | for (let i = 0; i < n; i++) { 70 | await this.call(this.operations) 71 | if (validateAll) await this._validate() 72 | } 73 | if (!validateAll) await this._validate() 74 | } 75 | 76 | debug (...msg) { 77 | if (this.debugging) console.log('fuzzbuzz:', ...msg) 78 | } 79 | 80 | async bisect (n) { 81 | let start = 0 82 | let end = n 83 | let ops = 0 84 | 85 | // galloping search ... 86 | while (start < end) { 87 | this.debug('bisecting start=' + start + ' and end=' + end) 88 | 89 | let dist = Math.min(1, end - start) 90 | let ptr = 0 91 | let i = 0 92 | 93 | this.random = random(this.random.seed) 94 | await this._setup() 95 | 96 | for (; i < n; i++) { 97 | try { 98 | ops++ 99 | await this.call(this.operations) 100 | } catch (_) { 101 | break 102 | } 103 | 104 | if (i < start) continue 105 | if (dist === ++ptr) { 106 | try { 107 | await this._validate() 108 | start = i + 1 109 | } catch (err) { 110 | break 111 | } 112 | dist *= 2 113 | } 114 | } 115 | end = i 116 | } 117 | 118 | // reset the state 119 | this.random = random(this.random.seed) 120 | this.debug('min amount of operations=' + (end + 1) + ' (ran a total of ' + ops + ' operations during bisect)') 121 | 122 | return end + 1 123 | } 124 | 125 | _setup () { 126 | // overwrite me 127 | } 128 | 129 | _validate () { 130 | // overwrite me 131 | } 132 | } 133 | 134 | module.exports = FuzzBuzz 135 | 136 | function promisifyMaybe (fn) { 137 | if (fn.length !== 1) return fn 138 | return promisify(fn) 139 | } 140 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fuzzbuzz", 3 | "version": "2.0.0", 4 | "description": "Fuzz testing framework", 5 | "main": "index.js", 6 | "dependencies": { 7 | "math-random-seed": "^2.0.0" 8 | }, 9 | "devDependencies": { 10 | "standard": "^12.0.1", 11 | "tape": "^4.9.2" 12 | }, 13 | "scripts": { 14 | "test": "standard && tape test.js" 15 | }, 16 | "repository": { 17 | "type": "git", 18 | "url": "https://github.com/mafintosh/fuzzbuzz.git" 19 | }, 20 | "author": "Mathias Buus (@mafintosh)", 21 | "license": "MIT", 22 | "bugs": { 23 | "url": "https://github.com/mafintosh/fuzzbuzz/issues" 24 | }, 25 | "homepage": "https://github.com/mafintosh/fuzzbuzz" 26 | } 27 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const FuzzBuzz = require('./') 3 | 4 | tape('seed works', function (assert) { 5 | const fuzz = new FuzzBuzz({ seed: Buffer.alloc(32) }) 6 | const rs = [ fuzz.random(), fuzz.random(), fuzz.random() ] 7 | 8 | assert.same(rs.map(n => typeof n), [ 'number', 'number', 'number' ]) 9 | 10 | const copy = new FuzzBuzz({ seed: fuzz.seed }) 11 | assert.same(copy.seed, fuzz.seed) 12 | assert.same([ copy.random(), copy.random(), copy.random() ], rs) 13 | assert.end() 14 | }) 15 | 16 | tape('add operation', async function (assert) { 17 | const fuzz = new FuzzBuzz() 18 | let tick = 0 19 | 20 | fuzz.add(1, function () { 21 | tick++ 22 | }) 23 | 24 | await fuzz.run(1000) 25 | 26 | assert.same(tick, 1000) 27 | assert.end() 28 | }) 29 | 30 | tape('setup and validate', async function (assert) { 31 | let tick = 0 32 | let setup = false 33 | let validate = false 34 | const fuzz = new FuzzBuzz({ 35 | async setup () { 36 | assert.notOk(setup) 37 | setup = true 38 | assert.same(tick, 0) 39 | }, 40 | async validate () { 41 | assert.notOk(validate) 42 | validate = true 43 | assert.same(tick, 100) 44 | } 45 | }) 46 | 47 | fuzz.add(1, function () { 48 | if (!setup) assert.fail('not setup') 49 | if (validate) assert.fail('premature validate') 50 | tick++ 51 | }) 52 | 53 | await fuzz.run(100) 54 | assert.ok(setup) 55 | assert.ok(validate) 56 | assert.same(tick, 100) 57 | assert.end() 58 | }) 59 | 60 | tape('setup and validate after construction', async function (assert) { 61 | let tick = 0 62 | let setup = false 63 | let validate = false 64 | const fuzz = new FuzzBuzz() 65 | 66 | fuzz.setup(async function () { 67 | assert.notOk(setup) 68 | setup = true 69 | assert.same(tick, 0) 70 | }) 71 | 72 | fuzz.validate(async function () { 73 | assert.notOk(validate) 74 | validate = true 75 | assert.same(tick, 100) 76 | }) 77 | 78 | fuzz.add(1, function () { 79 | if (!setup) assert.fail('not setup') 80 | if (validate) assert.fail('premature validate') 81 | tick++ 82 | }) 83 | 84 | await fuzz.run(100) 85 | assert.ok(setup) 86 | assert.ok(validate) 87 | assert.same(tick, 100) 88 | assert.end() 89 | }) 90 | 91 | tape('operations can be weighted', async function (assert) { 92 | const fuzz = new FuzzBuzz() 93 | 94 | let a = 0 95 | let b = 0 96 | 97 | fuzz.add(50, () => a++) 98 | fuzz.add(1, () => b++) 99 | 100 | await fuzz.run(10000) 101 | 102 | assert.ok(a > 10 * b) 103 | assert.end() 104 | }) 105 | 106 | tape('pass operations in constructor', async function (assert) { 107 | let a = 0 108 | let b = 0 109 | 110 | const fuzz = new FuzzBuzz({ 111 | operations: [ 112 | [ 50, () => a++ ], 113 | [ 1, () => b++ ] 114 | ] 115 | }) 116 | 117 | await fuzz.run(10000) 118 | 119 | assert.ok(a > 10 * b) 120 | assert.end() 121 | }) 122 | 123 | tape('async ops', async function (assert) { 124 | const fuzz = new FuzzBuzz() 125 | let active = 0 126 | let tick = 0 127 | 128 | fuzz.add(1, async function () { 129 | tick++ 130 | active++ 131 | await sleep(1) 132 | active-- 133 | }) 134 | 135 | fuzz.add(1, function (done) { 136 | tick++ 137 | active++ 138 | setTimeout(function () { 139 | active-- 140 | done() 141 | }, 1) 142 | }) 143 | 144 | assert.same(active, 0) 145 | await fuzz.run(500) 146 | assert.same(active, 0) 147 | assert.same(tick, 500) 148 | assert.end() 149 | 150 | function sleep (n) { 151 | return new Promise((resolve) => setTimeout(resolve, n)) 152 | } 153 | }) 154 | 155 | tape('operation randomness follows the seed', async function (assert) { 156 | const fuzz = new FuzzBuzz() 157 | const a = [] 158 | const b = [] 159 | 160 | fuzz.add(50, () => a.push('50')) 161 | fuzz.add(1, () => a.push('1')) 162 | 163 | await fuzz.run(10000) 164 | 165 | const copy = new FuzzBuzz({ seed: fuzz.seed }) 166 | 167 | copy.add(50, () => b.push('50')) 168 | copy.add(1, () => b.push('1')) 169 | 170 | await copy.run(10000) 171 | 172 | assert.same(a, b) 173 | assert.end() 174 | }) 175 | 176 | tape('operation + other apis randomness follows the seed', async function (assert) { 177 | const fuzz = new FuzzBuzz() 178 | const a = [] 179 | const b = [] 180 | 181 | fuzz.add(50, function () { 182 | a.push(fuzz.pick([ 10, 20, 30 ])) 183 | a.push(50) 184 | }) 185 | fuzz.add(1, function () { 186 | a.push(fuzz.pick([ 11, 12, 13 ])) 187 | a.push(1) 188 | }) 189 | 190 | await fuzz.run(10000) 191 | 192 | const copy = new FuzzBuzz({ seed: fuzz.seed }) 193 | 194 | copy.add(50, function () { 195 | b.push(copy.pick([ 10, 20, 30 ])) 196 | b.push(50) 197 | }) 198 | copy.add(1, function () { 199 | b.push(copy.pick([ 11, 12, 13 ])) 200 | b.push(1) 201 | }) 202 | 203 | await copy.run(10000) 204 | 205 | assert.same(a, b) 206 | assert.end() 207 | }) 208 | 209 | tape('can remove ops', async function (assert) { 210 | const fuzz = new FuzzBuzz() 211 | 212 | let ok = false 213 | const fn = () => assert.fail('nej tak') 214 | 215 | fuzz.add(5000, fn) 216 | fuzz.add(1, function () { 217 | ok = true 218 | }) 219 | 220 | fuzz.remove(5000, fn) 221 | await fuzz.run(1000) 222 | assert.ok(ok) 223 | assert.end() 224 | }) 225 | 226 | tape('fuzz the fuzzer', async function (assert) { 227 | const fuzz = new FuzzBuzz({ 228 | validate () { 229 | assert.same(other.operations.map(([ w, fn ]) => [ w, fn ]), ops) 230 | } 231 | }) 232 | 233 | assert.pass('fuzz seed is ' + fuzz.seed) 234 | 235 | const other = new FuzzBuzz() 236 | const ops = [] 237 | 238 | fuzz.add(10, async function add () { 239 | const fn = () => {} 240 | const n = fuzz.randomInt(100) 241 | ops.push([ n, fn ]) 242 | other.add(n, fn) 243 | }) 244 | 245 | fuzz.add(5, async function remove () { 246 | const n = fuzz.pick(ops) 247 | if (!n) return 248 | 249 | other.remove(n[0], n[1]) 250 | ops.splice(ops.indexOf(n), 1) 251 | }) 252 | 253 | await fuzz.run(1000) 254 | assert.end() 255 | }) 256 | 257 | tape('bisect', async function (assert) { 258 | const fuzz = new FuzzBuzz({ 259 | setup () { 260 | this.n = 0 261 | this.expected = 0 262 | }, 263 | validate () { 264 | if (this.n !== this.expected) throw new Error() 265 | }, 266 | operations: [ 267 | [ 10, add ], 268 | [ 10, sub ], 269 | [ 10, mul ], 270 | [ 1, faulty ] 271 | ] 272 | }) 273 | 274 | let faults = 0 275 | let error 276 | 277 | try { 278 | await fuzz.run(20000) 279 | } catch (err) { 280 | error = err 281 | } 282 | 283 | assert.ok(error, 'run failed') 284 | assert.ok(faults > 0, 'faulty op ran') 285 | 286 | const n = await fuzz.bisect(20000) 287 | assert.pass('bisect says we need ' + n + ' runs to fail') 288 | 289 | faults = 0 290 | error = null 291 | 292 | try { 293 | await fuzz.run(n) 294 | } catch (err) { 295 | error = err 296 | } 297 | 298 | assert.ok(error, 'still fails') 299 | assert.same(faults, 1, 'only one fault after rerunning') 300 | assert.end() 301 | 302 | function add () { 303 | const r = this.randomInt(10) 304 | this.n += r 305 | this.expected += r 306 | } 307 | 308 | function sub () { 309 | const r = this.randomInt(10) 310 | this.n -= r 311 | this.expected -= r 312 | } 313 | 314 | function mul () { 315 | const r = this.random() + 0.5 316 | this.n *= r 317 | this.expected *= r 318 | } 319 | 320 | function faulty () { 321 | const r = this.randomInt(10) 322 | this.n += r + 1 // faulty op 323 | this.expected += r 324 | faults++ 325 | } 326 | }) 327 | --------------------------------------------------------------------------------