├── .babelrc ├── .eslintrc.json ├── .flowconfig ├── .gitignore ├── LICENSE ├── README.md ├── bin ├── analyze-benchmark └── run-benchmark ├── docs └── Dimensions.md ├── examples ├── simplest-async │ ├── benchmark │ │ └── suite.js │ └── package.json └── simplest-sync │ ├── benchmark │ └── suite.js │ └── package.json ├── flow-typed └── npm │ └── jest_v18.x.x.js ├── package.json └── src ├── __tests__ ├── runner.js └── sampler.js ├── benchmark.js ├── dimension ├── __tests__ │ ├── debug.js │ ├── fifth.js │ ├── list.js │ ├── memory.js │ └── time.js ├── debug.js ├── fifth.js ├── list.js ├── memory.js ├── time.js └── type.js ├── index.js ├── report ├── __tests__ │ └── format.js ├── compare.js ├── format.js ├── report.js └── statistics.js ├── runner.js ├── sampler.js ├── scheduler └── legacy.js ├── tester.js └── warmer.js /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "env", { 5 | "targets": { 6 | node: 4 7 | } 8 | } 9 | ] 10 | ], 11 | plugins: [ 12 | "transform-flow-strip-types" 13 | ] 14 | } -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "node": true, 4 | "es6" : true, 5 | "jest" : true 6 | }, 7 | "extends": ["airbnb", "plugin:flowtype/recommended"], 8 | "parser": "babel-eslint", 9 | "parserOptions": { 10 | "ecmaVersion": 6, 11 | "sourceType": "module" 12 | }, 13 | "plugins": [ 14 | "flowtype" 15 | ], 16 | "rules": { 17 | "comma-dangle": "off", 18 | "no-plusplus": "off", 19 | "no-use-before-define": "off", 20 | "no-continue": "off", 21 | "no-console": "off", 22 | "import/prefer-default-export": "off", 23 | "no-param-reassign" : ["error", { "props": false }] 24 | } 25 | } -------------------------------------------------------------------------------- /.flowconfig: -------------------------------------------------------------------------------- 1 | [ignore] 2 | .*/dist/.* 3 | 4 | [include] 5 | 6 | [libs] 7 | 8 | [options] 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 18 | .grunt 19 | 20 | # node-waf configuration 21 | .lock-wscript 22 | 23 | # Compiled binary addons (http://nodejs.org/api/addons.html) 24 | build/Release 25 | 26 | # Dependency directory 27 | node_modules 28 | 29 | # Optional npm cache directory 30 | .npm 31 | 32 | # Optional REPL history 33 | .node_repl_history 34 | 35 | # IDE 36 | .idea 37 | 38 | # Build directories 39 | dist 40 | benchmark-results 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Jeff Moore 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Async Benchmark Runner 2 | A benchmark runner for node focusing on measuring elapsed time and memory usage 3 | for promise using asynchronous code. ABR is intended to be run as a part of a 4 | performance regression test suite. ABR is intended to help answer questions 5 | like "have performance characteristics changed between releases" or 6 | "does this change have an impact on performance?" 7 | 8 | ABR measures mean elapsed time, which does not give an accurate assessment of 9 | latency. It is not intended for load testing. 10 | 11 | ABR is intended to benchmark fine grained operations. The intent is that ABR 12 | should not only tell you that performance characteristcs have changed, but 13 | also help to pinpoint which operations have changed. The operations should 14 | have significance to your application. It generally possible to create more 15 | benchmarks of this type than it is to create load testing or system level 16 | benchmarks. 17 | 18 | Think of ABR as a performance unit test, not as a performance acceptance test. 19 | 20 | Candidate operations for benchmarking should be identified by profiling and 21 | monitoring your application. 22 | 23 | ABR is intended to be easy to use and produce repeatable results, sacrificing 24 | some level of accuracy toward this goal. 25 | 26 | ## Installing 27 | 28 | ``` 29 | npm install --save-dev async-benchmark-runner 30 | ``` 31 | 32 | ## Getting Started 33 | Here is an example of the simplest possible benchmark suite: 34 | ``` 35 | exports.name = 'Simple Suite'; 36 | 37 | exports.benchmarks = [ 38 | { 39 | name: 'NO-OP Synchronous', 40 | run: () => { 41 | return false; 42 | } 43 | } 44 | ]; 45 | ``` 46 | This creates a synchronous benchmark that does nothing. Place this in a file in 47 | your project called `benchmarks/suite.js`. 48 | 49 | Run the benchmark suite via the cli utility: 50 | ``` 51 | ./node_modules/.bin/run-benchmark 52 | ``` 53 | 54 | The results will be saved to a unique json file in the `benchmark-results` 55 | folder. The result location can be changed with a `--result-dir` parameter. 56 | This location can be changed by specifying an alternate location with 57 | via the `--suite` parameter. 58 | 59 | A result report will be output: 60 | 61 | ``` 62 | Running benchmark suite from benchmark/suite 63 | A Benchmark Time Memory 64 | - ---------------------------------------- ----------------- ------------------ 65 | NO-OP Synchronous 13 ns ± 10% 2 b ± 3% 66 | Writing results to benchmark-results/1460241638054-benchmark.json 67 | ``` 68 | Benchmark times are displayed in nanoseconds which is the unit of measure for 69 | node's [high resolution timer](https://nodejs.org/api/process.html#process_process_hrtime). 70 | Memory is displayed in bytes. The percentage indicates a [margin of error](https://en.wikipedia.org/wiki/Margin_of_error) 71 | calculated from a default [confidence level](https://en.wikipedia.org/wiki/Confidence_interval) of 0.95. 72 | This represents observed variation in timing, but cannot account for errors in 73 | benchmark design or environment setup. 74 | 75 | Here is the equivelent simplest possible asynchronous benchmark: 76 | ``` 77 | exports.name = 'Simple Suite'; 78 | 79 | exports.benchmarks = [ 80 | { 81 | name: 'NO-OP Asynchronous', 82 | startRunning: () => { 83 | return Promise.resolve(false); 84 | } 85 | } 86 | ]; 87 | ``` 88 | An asynchronous benchmark must return a promise. The measurement interval will 89 | not be completed until that promise resolves. 90 | 91 | The results of running this benchmark will look like this: 92 | 93 | ``` 94 | Running benchmark suite from benchmark/suite 95 | A Benchmark Time Memory 96 | - ---------------------------------------- ----------------- ------------------ 97 | * NO-OP Asynchronous 1194 ns ± 5% 1908 b ± 1% 98 | Writing results to benchmark-results/1460240762239-benchmark.json 99 | ``` 100 | 101 | Asynchronous benchmark names will be prefixed by an asterisk in the 102 | result report. 103 | 104 | ## Eliminating System Jitter 105 | Comparing different runs of the same benchmark requires that the conditions 106 | under which each are run to be similiar. You will acheive better results if you 107 | eliminate and exit any extranious programs on the machine you are running your 108 | tests on. Joe Bob's animated GIF storm in Slack may be amusing, but you do not 109 | want it to throw off your benchmarks. To check your environment, 110 | run the same benchmark suite twice 111 | ``` 112 | ./node_modules/.bin/run-benchmark 113 | ./node_modules/.bin/run-benchmark 114 | ``` 115 | 116 | The analyze-benchmark tool will automatically compare the last two benchmark 117 | results. It tests the statistical signficance of the two sets of results 118 | and will only show results from benchmarks that are significantly different. 119 | 120 | We can use this to measure environmental variation because two runs of 121 | the same benchmark should not be signficantly different. 122 | 123 | The default significance threshold is 5% (p < 0.05) and indicates less than 124 | 5% chance of false significance. To increase the sensativity of our 125 | test of our environment we increase the statisitical signficance threshold 126 | to 50% via the `--signficance-threshold` parameter. 127 | 128 | ``` 129 | ./node_modules/.bin/analyze-benchmark --significance-threshold=0.50 130 | ``` 131 | analyze-benchmark should report no significant difference between the runs, 132 | even at the weaker significance level. Here is an example report showing 133 | no significant difference: 134 | 135 | ``` 136 | TODO: example results here 137 | ``` 138 | If a difference is reported, look for ways to reduce the variation in your 139 | environment. Here is an example report from a noisy environment. 140 | ``` 141 | TODO: example results here 142 | ``` 143 | 144 | ## Comparing Benchmark Runs 145 | 146 | As seen in the last section the analyze-benchmark command can compare the 147 | last two benchmark runs. It can also be given explicit paths to two 148 | benchmark data files. 149 | ``` 150 | ./node_modules/.bin/analyze-benchmark 1-benchmark.json 2-benchmark.json 151 | ``` 152 | The order of the parameters do not matter, the comparison report always 153 | treats the older run as the baseline result and the newer run as the 154 | result under test. 155 | 156 | ### Comparing two branches 157 | 158 | Here is an example series of commands to compare two branches. 159 | ``` 160 | git checkout master 161 | ./node_modules/.bin/run-benchmark 162 | ./node_modules/.bin/run-benchmark 163 | ``` 164 | Switch to the master branch and take a baseline result. 165 | 166 | ``` 167 | ./node_modules/.bin/analyze-benchmark --significance-threshold=0.50 168 | ``` 169 | Check the current environment 170 | 171 | ``` 172 | git checkout feature-branch 173 | ./node_modules/.bin/run-benchmark 174 | ``` 175 | Take a result for the branch to be tested. 176 | 177 | ``` 178 | ./node_modules/.bin/analyze-benchmark 179 | ``` 180 | compare the two. 181 | 182 | ### Comparing work in progress 183 | 184 | Here is an example of benchmarking work which is not yet checked in. 185 | ``` 186 | git stash 187 | ./node_modules/.bin/run-benchmark 188 | ./node_modules/.bin/run-benchmark 189 | ``` 190 | Stash away pending changes and take a baseline result. 191 | 192 | ``` 193 | ./node_modules/.bin/analyze-benchmark --significance-threshold=0.50 194 | ``` 195 | Check the current environment 196 | 197 | ``` 198 | git apply 199 | ./node_modules/.bin/run-benchmark 200 | ``` 201 | Bring back the pending work and take a result for the branch to be tested. 202 | ``` 203 | ./node_modules/.bin/analyze-benchmark 204 | ``` 205 | compare the results. 206 | 207 | ### Comparing current work against a prior tagged release 208 | 209 | Here is an example of benchmarking new work against a tagged prior release. 210 | ``` 211 | git checkout -b v2benchmark v2.0.0 212 | ./node_modules/.bin/run-benchmark 213 | ./node_modules/.bin/run-benchmark 214 | ``` 215 | Create a new branch based on a prior version and take a baseline result. 216 | 217 | ``` 218 | ./node_modules/.bin/analyze-benchmark --significance-threshold=0.50 219 | ``` 220 | Check the current environment 221 | 222 | ``` 223 | git checkout master 224 | ./node_modules/.bin/run-benchmark 225 | ``` 226 | Test master against the prior release. 227 | ``` 228 | ./node_modules/.bin/analyze-benchmark 229 | ``` 230 | compare the results. 231 | 232 | ``` 233 | git branch -d v2benchmark 234 | ``` 235 | Clean up the temporary branch when done. 236 | 237 | ### Comparing the current version against the most recent prior version 238 | 239 | TODO 240 | 241 | ## Authoring a Benchmark Suite 242 | A benchmark suite is an array of benchmark definition objects. A benchmark 243 | definition is a simple javascript object. Here is an example of the simplest 244 | possible benchmark suite. 245 | 246 | ``` 247 | TODO 248 | ``` 249 | 250 | There are two types of benchmark, one for synchronous benchmarks and one for 251 | asynchronous benchmarks. Both types share the following fields: 252 | 253 | | field | Description | 254 | | --- | --- | 255 | | name | The name of the benchmark for reporting purposes. This is required. It must also be unique within a benchmark suite. | 256 | | setUp | An optional function which will be called prior to running the benchmark, outside of any measuring interval. Use to initialize any data required during the benchmark run. | 257 | | tearDown | An optional function which will be called after the benchmark has completed running, outside of any measuring interval. Use to free resources to make them available for other benchmarks. | 258 | 259 | Here is an example of a benchmark with setUp and tearDown 260 | 261 | ``` 262 | { 263 | name: 'example', 264 | setUp: () => { 265 | // TODO 266 | }, 267 | tearDown: () => { 268 | // TODO 269 | }, 270 | startRunning: () => { 271 | // TODO 272 | } 273 | ``` 274 | 275 | This interface [may change](https://github.com/JeffRMoore/async-benchmark-runner/issues/7). 276 | 277 | ### Benchmarking overhead 278 | 279 | TODO 280 | 281 | ## Benchmarking challenges under v8 282 | Javascript is a dynamic language. V8 gathers information about code as it runs, 283 | attempting to apply optimizations where they will have the most impact and 284 | trying not to let the cost of optimizing to outweigh the gains. This can make 285 | creating and interpreting benchmarks under node difficult. 286 | 287 | Vyacheslav Egorov explains some of the perils in benchmarking against the 288 | optimizer in [Benchmarking and Performance](https://www.youtube.com/watch?v=65-RbBwZQdU) 289 | 290 | V8 has several stages of optimization. Because ABR runs many cycles and takes many samples, 291 | it is not designed to benchmark code that does not also get repeatedly run in your 292 | application. Do not use ABR to benchmark "initialization" code. 293 | 294 | Use profiling to identify repeated "hot" areas and create benchmarks that mirror those 295 | portions of your codebase. 296 | 297 | ## Garbage Collection and Measuring Memory Usage 298 | 299 | Node is a garbage collected environment. This can be the source of significant 300 | jitter. This is bad not only for benchmarking but for application 301 | performance. The amount of memory consumed can lead to garbage collection 302 | pressure and increased jitter. 303 | 304 | When multiple tasks are running asynchronously, they hold system resources. An 305 | important such resource is memory. The more memory held by each task, 306 | the fewer concurrant tasks can be attempted. 307 | 308 | ABR operates under the hypothesis that measuring memory usage can act as a proxy 309 | for how many concurrant tasks can be attempted and how much jitter a system 310 | might experience due to garbage collection. 311 | 312 | In order to make accurate measurements of memory and to avoid garbage collection 313 | jitter, ABR attempts to control when garbage collection occurs, forcing it outside 314 | of measurement collecting periods. This is done by using the `gc` function 315 | available when node is run with the `--expose_gc` option. 316 | 317 | If running multiple operations during a sample triggers a garbage collection, 318 | the results of timing that sample will be less accurate, and the memory usage 319 | number for that sample will be wrong. Future versions of ABR will attempt 320 | to test memory usage and ensure that the number of operations per sample does 321 | not trigger a garbage collection. 322 | 323 | There is no way to programmatically detect if a garbage collection occurred 324 | during an interval, so ABR focuses on prevention. 325 | 326 | ABR has an option for debugging a benchmark to determine if garbage collection 327 | is happening during measurement. Passing the `--debug-gc` option to 328 | `run-benchmark` will trigger a debugging mode which outputs begining and ending 329 | indicators for measurement periods. Using the `--trace_gc` option for node, 330 | one can determine when garbage collection activity appears within a measurement 331 | interval. 332 | 333 | Example of an uninterrupted measurement interval: 334 | ``` 335 | TODO 336 | ``` 337 | Example of a measurement interval interrupted by garbage collection: 338 | ``` 339 | TODO 340 | ``` 341 | 342 | Note that this output cannot be piped to a file or other program as there is a 343 | buffering or flushing change in node which causes the output to be presented 344 | in a different order. This means that the debugging cannot at this time be 345 | scripted until the root cause of this behavior is discovered. 346 | 347 | Currently ABR cannot add the `--trace_gc` option during `run-benchmark`. To 348 | use this feature, you must edit the file directly adding the option to the 349 | shebang line for the script. Options on the shebang line are known to not 350 | be supoorted in Linux. Future ABR version will use a shell script instead 351 | of a node script to launch benchmarks to eliminate this issue. 352 | (see [#1](https://github.com/JeffRMoore/async-benchmark-runner/issues/1)) 353 | 354 | ## Building and Benchmarking your Application 355 | Using ABR in a non-trivial context is likely to require significant work on your 356 | project's build tooling. The benchmarks should be run against the final built 357 | version of your code. Benchmarks should also be run with `NODE_ENV` set to 358 | `production`. You may also have a specialized standard environment in which 359 | to run your benchmark suite. 360 | 361 | Tooling such as `node-babel`, while helpful for developement will significantly 362 | impact benchmark results. Thus, the benchmark suite should be run on the final 363 | built version of your code. This also impacts the development of the benchmarks 364 | themselves. If your benchmark code uses features that require tooling (such as 365 | es6), you will have to create a build process for the benchmarks, ensuring that 366 | a final build of the benchmarks run against the final build of the system under 367 | test. 368 | 369 | The development of the benchmark suite itself can be more difficult than 370 | developing code in the system under test. Tools that improve the interactivity 371 | of the code-test cycle may skew benchmark results and should be considered 372 | carefully. 373 | 374 | ## Testing your Benchmarks 375 | 376 | Benchmark code can have errors, too, especially when the output is not 377 | visible. Unit test your benchmarks. 378 | 379 | ABR provides helper functions for testing your benchmarks. The 380 | `runBenchmarkTest` function accepts a list of benchmarks and a benchmark 381 | name as parameters. It then runs the benchmark and returns the result 382 | of the run function. It is recommended that you construct your run 383 | function so that it returns a value which can be tested to determine if the 384 | benchmark is calculating the correct result. 385 | 386 | Here is an example of testing our simplest synchronous benchmark test. 387 | 388 | ``` 389 | import { benchmarks } from '../suite'; 390 | import { runBenchmarkTest } from 'async-benchmark-runner'; 391 | describe('NO-OP Synchronous', () => { 392 | it('returns false', () => { 393 | const result = runBenchmarkTest(benchmarks, 'NO-OP Synchronous'); 394 | expect(result).to.equal(false); 395 | }); 396 | }); 397 | ``` 398 | There is also a `startBenchmarkTest` function which returns a promise 399 | received from calling the startRunning function of an asynchronous 400 | benchmark. Similarly, construct your benchmark to resolve this promise 401 | to a testable value. Here is an example asynchronous benchmark test: 402 | 403 | ``` 404 | import { benchmarks } from '../suite'; 405 | import { startBenchmarkTest } from 'async-benchmark-runner'; 406 | describe('NO-OP Asynchronous', () => { 407 | it('returns false', async () => { 408 | const result = await startBenchmarkTest(benchmarks, 'NO-OP Asynchronous'); 409 | expect(result).to.equal(false); 410 | }); 411 | }); 412 | ``` 413 | 414 | ## CRITICAL BUGS 415 | 416 | [CLI fails on Linux](https://github.com/JeffRMoore/async-benchmark-runner/issues/2) 417 | -------------------------------------------------------------------------------- /bin/analyze-benchmark: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 'use strict'; 3 | 4 | const parseArgs = require('minimist'); 5 | const fs = require('fs'); 6 | const path = require('path'); 7 | 8 | const reportResult = require('../src/report/report').reportResult; 9 | const compareMemoryResults = require('../src/report/compare').compareMemoryResults; 10 | const compareTimeResults = require('../src/report/compare').compareTimeResults; 11 | 12 | const defaultSignificanceThreshold = 0.05; 13 | const defaultConfidenceLevel = 0.95; 14 | 15 | const argv = parseArgs(process.argv.slice(2)); 16 | 17 | if (argv._.length > 2) { 18 | console.log('Can only analyze 1 or 2 benchmark result files at a time'); 19 | process.exit(1); 20 | } 21 | 22 | let files = []; 23 | 24 | if (argv._.length === 0) { 25 | const resultDir = path.resolve( 26 | process.cwd(), 27 | argv['result-dir'] ? argv['result-dir'] : 'benchmark-results' 28 | ); 29 | const availableResults = fs.readdirSync(resultDir).sort(); 30 | if (availableResults.length > 0) { 31 | files.push( 32 | path.join(resultDir, 33 | availableResults[availableResults.length-1]) 34 | ); 35 | } 36 | if (availableResults.length > 1) { 37 | files.push( 38 | path.join(resultDir, 39 | availableResults[availableResults.length-2]) 40 | ); 41 | } 42 | } else { 43 | files = argv._; 44 | } 45 | 46 | const suiteResults = files.map(filename => { 47 | const contents = fs.readFileSync(filename, 'utf8'); 48 | return JSON.parse(contents); 49 | }); 50 | 51 | if (suiteResults.length === 1) { 52 | reportResult(suiteResults[0], console.log); 53 | } 54 | 55 | if (suiteResults.length === 2) { 56 | if (!compareArray(suiteResults[0].dimensions, suiteResults[1].dimensions)) { 57 | console.log('Cannot compare benchmarks with differing dimensions'); 58 | process.exit(1); 59 | } 60 | 61 | const significanceThreshold = 62 | argv['significance-threshold'] !== undefined ? 63 | Number(argv['significance-threshold']) : defaultSignificanceThreshold; 64 | const confidenceLevel = 65 | argv['confidence-level'] !== undefined ? 66 | Number(argv['confidence-level']) : defaultConfidenceLevel; 67 | compareMemoryResults( 68 | suiteResults[0], 69 | suiteResults[1], 70 | console.log, 71 | significanceThreshold, 72 | confidenceLevel 73 | ); 74 | compareTimeResults( 75 | suiteResults[0], 76 | suiteResults[1], 77 | console.log, 78 | significanceThreshold, 79 | confidenceLevel 80 | ); 81 | } 82 | 83 | function compareArray(array1, array2) { 84 | return (array1.length == array2.length) && array1.every(function(element, index) { 85 | return element === array2[index]; 86 | }); 87 | } -------------------------------------------------------------------------------- /bin/run-benchmark: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node --expose-gc 2 | 'use strict'; 3 | 4 | const fs = require('fs'); 5 | const path = require('path'); 6 | const mkdirp = require('mkdirp'); 7 | const parseArgs = require('minimist'); 8 | const startBenchmarking = require('../src/runner').startBenchmarking; 9 | const reportResult = require('../src/report/report').reportResult; 10 | 11 | const TimeDimension = require('../src/dimension/time').TimeDimension; 12 | const MemoryDimension = require('../src/dimension/memory').MemoryDimension; 13 | const DebugDimension = require('../src/dimension/debug').DebugDimension; 14 | 15 | const argv = parseArgs(process.argv.slice(2)); 16 | 17 | const suiteFile = path.resolve( 18 | process.cwd(), 19 | argv.suite ? argv.suite : 'benchmark/suite' 20 | ); 21 | const resultDir = path.resolve( 22 | process.cwd(), 23 | argv['result-dir'] ? argv['result-dir'] : 'benchmark-results' 24 | ); 25 | 26 | const benchmarkSuite = require(suiteFile); 27 | 28 | let isDebugging = false; 29 | 30 | if (argv['debug-gc'] !== undefined) { 31 | // --trace_gc 32 | isDebugging = true; 33 | } 34 | 35 | if (argv['debug-opt'] !== undefined) { 36 | // --trace_opt 37 | isDebugging = true; 38 | } 39 | 40 | if (argv['debug-deopt'] !== undefined) { 41 | // --trace_deopt 42 | isDebugging = true; 43 | } 44 | 45 | const dimensions = []; 46 | 47 | if (isDebugging) { 48 | dimensions.push(DebugDimension); 49 | } 50 | 51 | if (global.gc) { 52 | dimensions.push(MemoryDimension); 53 | } 54 | 55 | dimensions.push(TimeDimension); 56 | 57 | console.log( 58 | 'Running', 59 | benchmarkSuite.name, 60 | 'from', 61 | path.relative(process.cwd(), suiteFile) 62 | ); 63 | startBenchmarking( 64 | benchmarkSuite.name, 65 | benchmarkSuite.benchmarks, 66 | dimensions 67 | ).then( results => { 68 | reportResult(results, console.log); 69 | mkdirp(resultDir, (err) => { 70 | if (err) { 71 | console.log(err); 72 | process.exit(1); 73 | } 74 | const filename = path.join(resultDir, results.startTime + '-benchmark.json'); 75 | console.log('Writing results to', path.relative(process.cwd(), filename)); 76 | fs.writeFile(filename, JSON.stringify(results), (err) => { 77 | if (err) { 78 | console.log(err); 79 | process.exit(1); 80 | } 81 | }); 82 | }); 83 | }).catch( err => { 84 | console.log(err); 85 | process.exit(1); 86 | }); -------------------------------------------------------------------------------- /docs/Dimensions.md: -------------------------------------------------------------------------------- 1 | # Dimensions 2 | 3 | Many benchmarks focus on measuring a single aspect of performance, elasped time 4 | running a code sample. However, performance is often about making tradeoffs, making 5 | benchmarking a multi-dimensional problem. 6 | 7 | ABR uses a "Dimension" object to define a measurable aspect of the code under test, 8 | where measurements taken can be directly compared. 9 | 10 | # Custom Dimensions 11 | 12 | Some factors with significant impact on performance are difficult to measure. 13 | Some factors can have a significant impact on the stability and meaning of a 14 | benchmark. For example, benchmarking code that uses a network interface introduces 15 | an order of magnatude of complexity into the benchmarking process. It is difficult 16 | to create a benchmark for the purpose of CI that uses an uncertain network. The 17 | performance of the network might not be the aspect of performance that the 18 | benchmark is attempting to measure. 19 | 20 | One way to overcome this is to use a mock version of a sub-system with these 21 | characteristics and have instrument the mock to produce custom metrics. These 22 | metrics can be fed to an ABR Dimension definition if they are useful to measure. 23 | 24 | For example, code that is heavily reliant on the filesystem can be 25 | benchmarked against a virtual file system that operates out of memory, but that might 26 | track metrics such as blocks read or files opened. These can be captured in ABR via 27 | a custom dimension, allowing CI to fail if a dimension changes in a significant way. 28 | This does not give a true end-to-end acceptance test level understanding of 29 | performance, but might help to catch regressions in particular areas of code. -------------------------------------------------------------------------------- /examples/simplest-async/benchmark/suite.js: -------------------------------------------------------------------------------- 1 | exports.name = 'Simple Suite'; 2 | 3 | exports.benchmarks = [ 4 | { 5 | name: 'NO-OP Asynchronous', 6 | startRunning: () => { 7 | return Promise.resolve(false); 8 | } 9 | } 10 | ]; 11 | -------------------------------------------------------------------------------- /examples/simplest-async/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "async-benchmark-runner-simplest-async-example", 3 | "version": "1.0.0", 4 | "description": "An example of using async-benchmark-runner", 5 | "main": "benchmark/suite.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "async-benchmark-runner": "^0.1.1" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/simplest-sync/benchmark/suite.js: -------------------------------------------------------------------------------- 1 | exports.name = 'Simple Suite'; 2 | 3 | exports.benchmarks = [ 4 | { 5 | name: 'NO-OP Synchronous', 6 | run: () => { 7 | return false; 8 | } 9 | } 10 | ]; 11 | -------------------------------------------------------------------------------- /examples/simplest-sync/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "async-benchmark-runner-simplest-sync-example", 3 | "version": "1.0.0", 4 | "description": "An example of using async-benchmark-runner", 5 | "main": "benchmark/suite.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "async-benchmark-runner": "^0.1.1" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /flow-typed/npm/jest_v18.x.x.js: -------------------------------------------------------------------------------- 1 | // flow-typed signature: e49570b0f5e396c7206dda452bd6f004 2 | // flow-typed version: 1590d813f4/jest_v18.x.x/flow_>=v0.33.x 3 | 4 | type JestMockFn = { 5 | (...args: Array): any, 6 | /** 7 | * An object for introspecting mock calls 8 | */ 9 | mock: { 10 | /** 11 | * An array that represents all calls that have been made into this mock 12 | * function. Each call is represented by an array of arguments that were 13 | * passed during the call. 14 | */ 15 | calls: Array>, 16 | /** 17 | * An array that contains all the object instances that have been 18 | * instantiated from this mock function. 19 | */ 20 | instances: mixed, 21 | }, 22 | /** 23 | * Resets all information stored in the mockFn.mock.calls and 24 | * mockFn.mock.instances arrays. Often this is useful when you want to clean 25 | * up a mock's usage data between two assertions. 26 | */ 27 | mockClear(): Function, 28 | /** 29 | * Resets all information stored in the mock. This is useful when you want to 30 | * completely restore a mock back to its initial state. 31 | */ 32 | mockReset(): Function, 33 | /** 34 | * Accepts a function that should be used as the implementation of the mock. 35 | * The mock itself will still record all calls that go into and instances 36 | * that come from itself -- the only difference is that the implementation 37 | * will also be executed when the mock is called. 38 | */ 39 | mockImplementation(fn: Function): JestMockFn, 40 | /** 41 | * Accepts a function that will be used as an implementation of the mock for 42 | * one call to the mocked function. Can be chained so that multiple function 43 | * calls produce different results. 44 | */ 45 | mockImplementationOnce(fn: Function): JestMockFn, 46 | /** 47 | * Just a simple sugar function for returning `this` 48 | */ 49 | mockReturnThis(): void, 50 | /** 51 | * Deprecated: use jest.fn(() => value) instead 52 | */ 53 | mockReturnValue(value: any): JestMockFn, 54 | /** 55 | * Sugar for only returning a value once inside your mock 56 | */ 57 | mockReturnValueOnce(value: any): JestMockFn, 58 | } 59 | 60 | type JestAsymmetricEqualityType = { 61 | /** 62 | * A custom Jasmine equality tester 63 | */ 64 | asymmetricMatch(value: mixed): boolean, 65 | } 66 | 67 | type JestCallsType = { 68 | allArgs(): mixed, 69 | all(): mixed, 70 | any(): boolean, 71 | count(): number, 72 | first(): mixed, 73 | mostRecent(): mixed, 74 | reset(): void, 75 | } 76 | 77 | type JestClockType = { 78 | install(): void, 79 | mockDate(date: Date): void, 80 | tick(): void, 81 | uninstall(): void, 82 | } 83 | 84 | type JestMatcherResult = { 85 | message?: string | ()=>string, 86 | pass: boolean, 87 | } 88 | 89 | type JestMatcher = (actual: any, expected: any) => JestMatcherResult; 90 | 91 | type JestExpectType = { 92 | not: JestExpectType, 93 | /** 94 | * If you have a mock function, you can use .lastCalledWith to test what 95 | * arguments it was last called with. 96 | */ 97 | lastCalledWith(...args: Array): void, 98 | /** 99 | * toBe just checks that a value is what you expect. It uses === to check 100 | * strict equality. 101 | */ 102 | toBe(value: any): void, 103 | /** 104 | * Use .toHaveBeenCalled to ensure that a mock function got called. 105 | */ 106 | toBeCalled(): void, 107 | /** 108 | * Use .toBeCalledWith to ensure that a mock function was called with 109 | * specific arguments. 110 | */ 111 | toBeCalledWith(...args: Array): void, 112 | /** 113 | * Using exact equality with floating point numbers is a bad idea. Rounding 114 | * means that intuitive things fail. 115 | */ 116 | toBeCloseTo(num: number, delta: any): void, 117 | /** 118 | * Use .toBeDefined to check that a variable is not undefined. 119 | */ 120 | toBeDefined(): void, 121 | /** 122 | * Use .toBeFalsy when you don't care what a value is, you just want to 123 | * ensure a value is false in a boolean context. 124 | */ 125 | toBeFalsy(): void, 126 | /** 127 | * To compare floating point numbers, you can use toBeGreaterThan. 128 | */ 129 | toBeGreaterThan(number: number): void, 130 | /** 131 | * To compare floating point numbers, you can use toBeGreaterThanOrEqual. 132 | */ 133 | toBeGreaterThanOrEqual(number: number): void, 134 | /** 135 | * To compare floating point numbers, you can use toBeLessThan. 136 | */ 137 | toBeLessThan(number: number): void, 138 | /** 139 | * To compare floating point numbers, you can use toBeLessThanOrEqual. 140 | */ 141 | toBeLessThanOrEqual(number: number): void, 142 | /** 143 | * Use .toBeInstanceOf(Class) to check that an object is an instance of a 144 | * class. 145 | */ 146 | toBeInstanceOf(cls: Class<*>): void, 147 | /** 148 | * .toBeNull() is the same as .toBe(null) but the error messages are a bit 149 | * nicer. 150 | */ 151 | toBeNull(): void, 152 | /** 153 | * Use .toBeTruthy when you don't care what a value is, you just want to 154 | * ensure a value is true in a boolean context. 155 | */ 156 | toBeTruthy(): void, 157 | /** 158 | * Use .toBeUndefined to check that a variable is undefined. 159 | */ 160 | toBeUndefined(): void, 161 | /** 162 | * Use .toContain when you want to check that an item is in a list. For 163 | * testing the items in the list, this uses ===, a strict equality check. 164 | */ 165 | toContain(item: any): void, 166 | /** 167 | * Use .toContainEqual when you want to check that an item is in a list. For 168 | * testing the items in the list, this matcher recursively checks the 169 | * equality of all fields, rather than checking for object identity. 170 | */ 171 | toContainEqual(item: any): void, 172 | /** 173 | * Use .toEqual when you want to check that two objects have the same value. 174 | * This matcher recursively checks the equality of all fields, rather than 175 | * checking for object identity. 176 | */ 177 | toEqual(value: any): void, 178 | /** 179 | * Use .toHaveBeenCalled to ensure that a mock function got called. 180 | */ 181 | toHaveBeenCalled(): void, 182 | /** 183 | * Use .toHaveBeenCalledTimes to ensure that a mock function got called exact 184 | * number of times. 185 | */ 186 | toHaveBeenCalledTimes(number: number): void, 187 | /** 188 | * Use .toHaveBeenCalledWith to ensure that a mock function was called with 189 | * specific arguments. 190 | */ 191 | toHaveBeenCalledWith(...args: Array): void, 192 | /** 193 | * Check that an object has a .length property and it is set to a certain 194 | * numeric value. 195 | */ 196 | toHaveLength(number: number): void, 197 | /** 198 | * 199 | */ 200 | toHaveProperty(propPath: string, value?: any): void, 201 | /** 202 | * Use .toMatch to check that a string matches a regular expression. 203 | */ 204 | toMatch(regexp: RegExp): void, 205 | /** 206 | * Use .toMatchObject to check that a javascript object matches a subset of the properties of an object. 207 | */ 208 | toMatchObject(object: Object): void, 209 | /** 210 | * This ensures that a React component matches the most recent snapshot. 211 | */ 212 | toMatchSnapshot(name?: string): void, 213 | /** 214 | * Use .toThrow to test that a function throws when it is called. 215 | */ 216 | toThrow(message?: string | Error): void, 217 | /** 218 | * Use .toThrowError to test that a function throws a specific error when it 219 | * is called. The argument can be a string for the error message, a class for 220 | * the error, or a regex that should match the error. 221 | */ 222 | toThrowError(message?: string | Error | RegExp): void, 223 | /** 224 | * Use .toThrowErrorMatchingSnapshot to test that a function throws a error 225 | * matching the most recent snapshot when it is called. 226 | */ 227 | toThrowErrorMatchingSnapshot(): void, 228 | } 229 | 230 | type JestObjectType = { 231 | /** 232 | * Disables automatic mocking in the module loader. 233 | * 234 | * After this method is called, all `require()`s will return the real 235 | * versions of each module (rather than a mocked version). 236 | */ 237 | disableAutomock(): JestObjectType, 238 | /** 239 | * An un-hoisted version of disableAutomock 240 | */ 241 | autoMockOff(): JestObjectType, 242 | /** 243 | * Enables automatic mocking in the module loader. 244 | */ 245 | enableAutomock(): JestObjectType, 246 | /** 247 | * An un-hoisted version of enableAutomock 248 | */ 249 | autoMockOn(): JestObjectType, 250 | /** 251 | * Resets the state of all mocks. Equivalent to calling .mockReset() on every 252 | * mocked function. 253 | */ 254 | resetAllMocks(): JestObjectType, 255 | /** 256 | * Removes any pending timers from the timer system. 257 | */ 258 | clearAllTimers(): void, 259 | /** 260 | * The same as `mock` but not moved to the top of the expectation by 261 | * babel-jest. 262 | */ 263 | doMock(moduleName: string, moduleFactory?: any): JestObjectType, 264 | /** 265 | * The same as `unmock` but not moved to the top of the expectation by 266 | * babel-jest. 267 | */ 268 | dontMock(moduleName: string): JestObjectType, 269 | /** 270 | * Returns a new, unused mock function. Optionally takes a mock 271 | * implementation. 272 | */ 273 | fn(implementation?: Function): JestMockFn, 274 | /** 275 | * Determines if the given function is a mocked function. 276 | */ 277 | isMockFunction(fn: Function): boolean, 278 | /** 279 | * Given the name of a module, use the automatic mocking system to generate a 280 | * mocked version of the module for you. 281 | */ 282 | genMockFromModule(moduleName: string): any, 283 | /** 284 | * Mocks a module with an auto-mocked version when it is being required. 285 | * 286 | * The second argument can be used to specify an explicit module factory that 287 | * is being run instead of using Jest's automocking feature. 288 | * 289 | * The third argument can be used to create virtual mocks -- mocks of modules 290 | * that don't exist anywhere in the system. 291 | */ 292 | mock(moduleName: string, moduleFactory?: any): JestObjectType, 293 | /** 294 | * Resets the module registry - the cache of all required modules. This is 295 | * useful to isolate modules where local state might conflict between tests. 296 | */ 297 | resetModules(): JestObjectType, 298 | /** 299 | * Exhausts the micro-task queue (usually interfaced in node via 300 | * process.nextTick). 301 | */ 302 | runAllTicks(): void, 303 | /** 304 | * Exhausts the macro-task queue (i.e., all tasks queued by setTimeout(), 305 | * setInterval(), and setImmediate()). 306 | */ 307 | runAllTimers(): void, 308 | /** 309 | * Exhausts all tasks queued by setImmediate(). 310 | */ 311 | runAllImmediates(): void, 312 | /** 313 | * Executes only the macro task queue (i.e. all tasks queued by setTimeout() 314 | * or setInterval() and setImmediate()). 315 | */ 316 | runTimersToTime(msToRun: number): void, 317 | /** 318 | * Executes only the macro-tasks that are currently pending (i.e., only the 319 | * tasks that have been queued by setTimeout() or setInterval() up to this 320 | * point) 321 | */ 322 | runOnlyPendingTimers(): void, 323 | /** 324 | * Explicitly supplies the mock object that the module system should return 325 | * for the specified module. Note: It is recommended to use jest.mock() 326 | * instead. 327 | */ 328 | setMock(moduleName: string, moduleExports: any): JestObjectType, 329 | /** 330 | * Indicates that the module system should never return a mocked version of 331 | * the specified module from require() (e.g. that it should always return the 332 | * real module). 333 | */ 334 | unmock(moduleName: string): JestObjectType, 335 | /** 336 | * Instructs Jest to use fake versions of the standard timer functions 337 | * (setTimeout, setInterval, clearTimeout, clearInterval, nextTick, 338 | * setImmediate and clearImmediate). 339 | */ 340 | useFakeTimers(): JestObjectType, 341 | /** 342 | * Instructs Jest to use the real versions of the standard timer functions. 343 | */ 344 | useRealTimers(): JestObjectType, 345 | } 346 | 347 | type JestSpyType = { 348 | calls: JestCallsType, 349 | } 350 | 351 | /** Runs this function after every test inside this context */ 352 | declare function afterEach(fn: Function): void; 353 | /** Runs this function before every test inside this context */ 354 | declare function beforeEach(fn: Function): void; 355 | /** Runs this function after all tests have finished inside this context */ 356 | declare function afterAll(fn: Function): void; 357 | /** Runs this function before any tests have started inside this context */ 358 | declare function beforeAll(fn: Function): void; 359 | /** A context for grouping tests together */ 360 | declare function describe(name: string, fn: Function): void; 361 | 362 | /** An individual test unit */ 363 | declare var it: { 364 | /** 365 | * An individual test unit 366 | * 367 | * @param {string} Name of Test 368 | * @param {Function} Test 369 | */ 370 | (name: string, fn?: Function): ?Promise, 371 | /** 372 | * Only run this test 373 | * 374 | * @param {string} Name of Test 375 | * @param {Function} Test 376 | */ 377 | only(name: string, fn?: Function): ?Promise, 378 | /** 379 | * Skip running this test 380 | * 381 | * @param {string} Name of Test 382 | * @param {Function} Test 383 | */ 384 | skip(name: string, fn?: Function): ?Promise, 385 | /** 386 | * Run the test concurrently 387 | * 388 | * @param {string} Name of Test 389 | * @param {Function} Test 390 | */ 391 | concurrent(name: string, fn?: Function): ?Promise, 392 | }; 393 | declare function fit(name: string, fn: Function): ?Promise; 394 | /** An individual test unit */ 395 | declare var test: typeof it; 396 | /** A disabled group of tests */ 397 | declare var xdescribe: typeof describe; 398 | /** A focused group of tests */ 399 | declare var fdescribe: typeof describe; 400 | /** A disabled individual test */ 401 | declare var xit: typeof it; 402 | /** A disabled individual test */ 403 | declare var xtest: typeof it; 404 | 405 | /** The expect function is used every time you want to test a value */ 406 | declare var expect: { 407 | /** The object that you want to make assertions against */ 408 | (value: any): JestExpectType, 409 | /** Add additional Jasmine matchers to Jest's roster */ 410 | extend(matchers: {[name:string]: JestMatcher}): void, 411 | assertions(expectedAssertions: number): void, 412 | any(value: mixed): JestAsymmetricEqualityType, 413 | anything(): void, 414 | arrayContaining(value: Array): void, 415 | objectContaining(value: Object): void, 416 | stringMatching(value: string): void, 417 | }; 418 | 419 | // TODO handle return type 420 | // http://jasmine.github.io/2.4/introduction.html#section-Spies 421 | declare function spyOn(value: mixed, method: string): Object; 422 | 423 | /** Holds all functions related to manipulating test runner */ 424 | declare var jest: JestObjectType 425 | 426 | /** 427 | * The global Jamine object, this is generally not exposed as the public API, 428 | * using features inside here could break in later versions of Jest. 429 | */ 430 | declare var jasmine: { 431 | DEFAULT_TIMEOUT_INTERVAL: number, 432 | any(value: mixed): JestAsymmetricEqualityType, 433 | anything(): void, 434 | arrayContaining(value: Array): void, 435 | clock(): JestClockType, 436 | createSpy(name: string): JestSpyType, 437 | createSpyObj(baseName: string, methodNames: Array): {[methodName: string]: JestSpyType}, 438 | objectContaining(value: Object): void, 439 | stringMatching(value: string): void, 440 | } 441 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "async-benchmark-runner", 3 | "version": "0.1.2", 4 | "author": "Jeff Moore ", 5 | "description": "Benchmark runner for node focusing on measuring asynchronous code using promises.", 6 | "license": "MIT", 7 | "main": "src/index.js", 8 | "repository": { 9 | "type": "git", 10 | "url": "https://github.com/JeffRMoore/async-benchmark-runner.git" 11 | }, 12 | "jest": { 13 | "unmockedModulePathPatterns": [ 14 | "/node_modules/" 15 | ], 16 | "modulePathIgnorePatterns": [ 17 | "/dist/" 18 | ] 19 | }, 20 | "scripts": { 21 | "test": "npm run lint && npm run typecheck && npm run unit-tests", 22 | "unit-tests": "jest", 23 | "lint": "eslint src", 24 | "typecheck": "flow check", 25 | "build": "babel src --ignore __tests__ --out-dir dist/src && cp package.json dist/ && mkdir -p dist/bin && cp bin/analyze-benchmark dist/bin && cp bin/run-benchmark dist/bin" 26 | }, 27 | "bin": { 28 | "analyze-benchmark": "bin/analyze-benchmark", 29 | "run-benchmark": "bin/run-benchmark" 30 | }, 31 | "dependencies": { 32 | "experiments.js": "^0.1.0", 33 | "lodash.flatten": "^4.4.0", 34 | "minimalist": "^1.0.0", 35 | "mkdirp": "^0.5.1", 36 | "simple-statistics": "^2.2.0" 37 | }, 38 | "devDependencies": { 39 | "babel-cli": "^6.22.2", 40 | "babel-eslint": "7.1.1", 41 | "babel-jest": "^18.0.0", 42 | "babel-plugin-transform-flow-strip-types": "^6.22.0", 43 | "babel-polyfill": "^6.22.0", 44 | "babel-preset-env": "^1.1.8", 45 | "eslint": "^3.15.0", 46 | "eslint-config-airbnb": "^14.1.0", 47 | "eslint-plugin-flowtype": "^2.30.0", 48 | "eslint-plugin-import": "^2.2.0", 49 | "eslint-plugin-jsx-a11y": "^4.0.0", 50 | "eslint-plugin-react": "^6.9.0", 51 | "flow-bin": "^0.39.0", 52 | "jest-cli": "^18.1.0" 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/__tests__/runner.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import { 4 | startBenchmarking 5 | } from '../runner'; 6 | 7 | jest.unmock('../runner.js'); 8 | 9 | describe('Basic benchmark suite', () => { 10 | it('has the correct name', async () => { 11 | const expectedName = 'betty'; 12 | const result = await startBenchmarking(expectedName, [], []); 13 | expect(result.name).toBe(expectedName); 14 | }); 15 | 16 | it('has a start date in the future', async () => { 17 | const thePast = Date.now(); 18 | const result = await startBenchmarking('test', [], []); 19 | expect(result.startTime).not.toBeLessThan(thePast); 20 | }); 21 | }); 22 | 23 | describe('Consecutive benchmark runs', () => { 24 | it('have a consecutive start time', async () => { 25 | const result1 = await startBenchmarking('test', [], []); 26 | const result2 = await startBenchmarking('test', [], []); 27 | expect(result2.startTime).not.toBeLessThan(result1.startTime); 28 | }); 29 | }); 30 | -------------------------------------------------------------------------------- /src/__tests__/sampler.js: -------------------------------------------------------------------------------- 1 | /* Intentionally no Flow, some test case intantionally test bad type inputs */ 2 | 3 | import { 4 | setUpBenchmark, 5 | tearDownBenchmark, 6 | collectAsynchronousSample, 7 | collectSynchronousSample 8 | } from '../sampler'; 9 | 10 | import type { 11 | ASynchronousBenchmark, 12 | SynchronousBenchmark 13 | } from '../benchmark'; 14 | 15 | import type { 16 | DimensionList 17 | } from '../dimension/list'; 18 | 19 | import { 20 | FifthDimension 21 | } from '../dimension/fifth'; 22 | 23 | describe('Sampling', () => { 24 | const dimensions: DimensionList = [FifthDimension]; 25 | 26 | describe('a synchronous benchmark', () => { 27 | const syncBenchmark: SynchronousBenchmark = { 28 | name: 'test', 29 | run: () => false 30 | }; 31 | 32 | it('can collect a sample', () => { 33 | const errorCallback = jest.fn(); 34 | const completedCallback = jest.fn(); 35 | 36 | collectSynchronousSample( 37 | syncBenchmark, 38 | dimensions, 39 | 1, 40 | completedCallback, 41 | errorCallback 42 | ); 43 | expect(errorCallback).not.toBeCalled(); 44 | expect(completedCallback).toBeCalledWith([5]); 45 | }); 46 | 47 | describe('that does not define setUp or tearDown', () => { 48 | it('can setUp', () => { 49 | expect(setUpBenchmark(syncBenchmark)).toBe(false); 50 | }); 51 | 52 | it('can tearDown', () => { 53 | expect(tearDownBenchmark(syncBenchmark)).toBe(false); 54 | }); 55 | }); 56 | 57 | describe('that defines a setUp and tearDown', () => { 58 | const syncBenchmarkWithSetup: SynchronousBenchmark = { 59 | name: 'test', 60 | setUp: jest.fn(), 61 | run: () => false, 62 | tearDown: jest.fn() 63 | }; 64 | 65 | it('can setUp', () => { 66 | const result = setUpBenchmark(syncBenchmarkWithSetup); 67 | expect(result).toBe(false); 68 | expect(syncBenchmarkWithSetup.setUp).toBeCalled(); 69 | }); 70 | 71 | it('can tearDown', () => { 72 | const result = tearDownBenchmark(syncBenchmarkWithSetup); 73 | expect(result).toBe(false); 74 | expect(syncBenchmarkWithSetup.tearDown).toBeCalled(); 75 | }); 76 | }); 77 | 78 | describe('that defines a setUp with an error', () => { 79 | const syncBenchmarkWithSetUpError: SynchronousBenchmark = { 80 | name: 'test', 81 | setUp: () => { 82 | throw new Error('Oops'); 83 | }, 84 | run: () => false, 85 | }; 86 | 87 | it('can detect the error', () => { 88 | expect(setUpBenchmark(syncBenchmarkWithSetUpError)).toBeInstanceOf(Error); 89 | }); 90 | }); 91 | 92 | describe('that defines a setUp that is not a function', () => { 93 | const syncBenchmarkWithSetUpNotFunction: SynchronousBenchmark = { 94 | name: 'test', 95 | setUp: false, 96 | run: () => false, 97 | }; 98 | 99 | it('can detect the error', () => { 100 | expect(setUpBenchmark(syncBenchmarkWithSetUpNotFunction)).toBeInstanceOf(Error); 101 | }); 102 | }); 103 | 104 | describe('that defines a tearDown with an error', () => { 105 | const syncBenchmarkWithTearDownError: SynchronousBenchmark = { 106 | name: 'test', 107 | run: () => false, 108 | tearDown: () => { 109 | throw new Error('Oops'); 110 | }, 111 | }; 112 | 113 | it('can detect the error', () => { 114 | expect(tearDownBenchmark(syncBenchmarkWithTearDownError)).toBeInstanceOf(Error); 115 | }); 116 | }); 117 | 118 | describe('that defines a tearDown that is not a function', () => { 119 | const syncBenchmarkWithTearDownNotFunction: SynchronousBenchmark = { 120 | name: 'test', 121 | run: () => false, 122 | tearDown: false, 123 | }; 124 | 125 | it('can detect the error', () => { 126 | expect(tearDownBenchmark(syncBenchmarkWithTearDownNotFunction)).toBeInstanceOf(Error); 127 | }); 128 | }); 129 | 130 | describe('with an error', () => { 131 | const syncBenchmarkWithError: SynchronousBenchmark = { 132 | name: 'test', 133 | run: () => { 134 | throw new Error('Oops'); 135 | } 136 | }; 137 | 138 | it('can detect an error when collecting sample', () => { 139 | const errorCallback = jest.fn(); 140 | const completedCallback = jest.fn(); 141 | 142 | collectSynchronousSample( 143 | syncBenchmarkWithError, 144 | dimensions, 145 | 1, 146 | completedCallback, 147 | errorCallback 148 | ); 149 | expect(completedCallback).not.toBeCalled(); 150 | expect(errorCallback).toBeCalled(); 151 | }); 152 | }); 153 | }); 154 | 155 | describe('an asynchronous benchmark', () => { 156 | const asyncBenchmark: ASynchronousBenchmark = { 157 | name: 'test', 158 | startRunning: () => Promise.resolve(false) 159 | }; 160 | 161 | it('can collect a sample', () => { 162 | const task = new Promise((resolve, reject) => { 163 | collectAsynchronousSample( 164 | asyncBenchmark, 165 | dimensions, 166 | 1, 167 | resolve, 168 | reject 169 | ); 170 | }); 171 | return task.then(result => expect(result).toEqual([5])); 172 | }); 173 | 174 | describe('that does not define setUp or tearDown', () => { 175 | it('can setUp', () => { 176 | expect(setUpBenchmark(asyncBenchmark)).toBe(false); 177 | }); 178 | 179 | it('can tearDown', () => { 180 | expect(tearDownBenchmark(asyncBenchmark)).toBe(false); 181 | }); 182 | }); 183 | 184 | describe('that defines a setUp with an error', () => { 185 | const asyncBenchmarkSetUpError: ASynchronousBenchmark = { 186 | name: 'test', 187 | setUp: () => { 188 | throw new Error('Oops'); 189 | }, 190 | startRunning: () => true 191 | }; 192 | 193 | it('can detect the error', () => { 194 | const task = new Promise((resolve, reject) => { 195 | collectAsynchronousSample( 196 | asyncBenchmarkSetUpError, 197 | dimensions, 198 | 1, 199 | resolve, 200 | reject 201 | ); 202 | }); 203 | return task.catch(error => expect(error).toBeInstanceOf(Error)); 204 | }); 205 | }); 206 | 207 | describe('that defines a tearDown with an error', () => { 208 | const asyncBenchmarkTearDownError: ASynchronousBenchmark = { 209 | name: 'test', 210 | startRunning: () => true, 211 | tearDown: () => { 212 | throw new Error('Oops'); 213 | } 214 | }; 215 | 216 | it('can detect the error', () => { 217 | const task = new Promise((resolve, reject) => { 218 | collectAsynchronousSample( 219 | asyncBenchmarkTearDownError, 220 | dimensions, 221 | 1, 222 | resolve, 223 | reject 224 | ); 225 | }); 226 | return task.catch(error => expect(error).toBeInstanceOf(Error)); 227 | }); 228 | }); 229 | 230 | describe('that does not return a promise', () => { 231 | const asyncBenchmarkNoPromise: ASynchronousBenchmark = { 232 | name: 'test', 233 | startRunning: () => true 234 | }; 235 | 236 | it('can collect a sample', () => { 237 | const task = new Promise((resolve, reject) => { 238 | collectAsynchronousSample( 239 | asyncBenchmarkNoPromise, 240 | dimensions, 241 | 1, 242 | resolve, 243 | reject 244 | ); 245 | }); 246 | return task.then(result => expect(result).toEqual([5])); 247 | }); 248 | }); 249 | 250 | describe('that throws an error', () => { 251 | const asyncBenchmarkThrowsError: ASynchronousBenchmark = { 252 | name: 'test', 253 | startRunning: () => { 254 | throw new Error('Oops'); 255 | } 256 | }; 257 | 258 | it('can detect an error when collecting sample', () => { 259 | const task = new Promise((resolve, reject) => { 260 | collectAsynchronousSample( 261 | asyncBenchmarkThrowsError, 262 | dimensions, 263 | 1, 264 | resolve, 265 | reject 266 | ); 267 | }); 268 | return task.catch(error => expect(error).toBeInstanceOf(Error)); 269 | }); 270 | }); 271 | 272 | describe('that rejects a promise', () => { 273 | const asyncBenchmarkRejected: ASynchronousBenchmark = { 274 | name: 'test', 275 | startRunning: () => Promise.reject('Oops') 276 | }; 277 | 278 | it('can detect an error when collecting sample', () => { 279 | const task = new Promise((resolve, reject) => { 280 | collectAsynchronousSample( 281 | asyncBenchmarkRejected, 282 | dimensions, 283 | 1, 284 | resolve, 285 | reject 286 | ); 287 | }); 288 | return task.catch(error => expect(error).toEqual('Oops')); 289 | }); 290 | }); 291 | }); 292 | }); 293 | -------------------------------------------------------------------------------- /src/benchmark.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | /** 4 | * Describe a synchronous benchmark 5 | */ 6 | export type SynchronousBenchmark = 7 | { 8 | name: string; 9 | setUp?: () => void; 10 | tearDown?: () => void; 11 | run: () => any; 12 | } 13 | 14 | /** 15 | * Describe an asynchronous benchmark 16 | */ 17 | export type ASynchronousBenchmark = 18 | { 19 | name: string; 20 | setUp?: () => void; 21 | tearDown?: () => void; 22 | startRunning: () => Promise<*>; 23 | } 24 | 25 | /** 26 | * Supported benchmarking styles 27 | */ 28 | export type Benchmark = SynchronousBenchmark | ASynchronousBenchmark; 29 | 30 | /** 31 | * A set of sample measurements 32 | */ 33 | export type Samples = Array; 34 | 35 | /** 36 | * Result of running one benchmark 37 | */ 38 | export type BenchmarkResult = 39 | { 40 | name: string; 41 | isAsynchronous: boolean; 42 | opsPerSample: number; 43 | numSamples: number; 44 | samples: { [measurementName: string]: Samples }; 45 | } 46 | 47 | /** 48 | * Result of running a suite of benchmarks 49 | */ 50 | export type BenchmarkSuiteResult = 51 | { 52 | name: string; 53 | startTime: number; 54 | dimensions: Array; 55 | results: Array; 56 | } 57 | -------------------------------------------------------------------------------- /src/dimension/__tests__/debug.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import { 4 | DebugDimension 5 | } from '../debug'; 6 | 7 | describe('DebugDimension', () => { 8 | /* @TODO How do you expect() console output during a test? */ 9 | it('produces consequitive measurements', () => { 10 | expect(DebugDimension.stopMeasuring(DebugDimension.startMeasuring())).toBeGreaterThanOrEqual(0); 11 | }); 12 | }); 13 | -------------------------------------------------------------------------------- /src/dimension/__tests__/fifth.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import { 4 | FifthDimension 5 | } from '../fifth'; 6 | 7 | describe('FifthDimension', () => { 8 | it('produces positive numerical results', () => { 9 | expect(FifthDimension.stopMeasuring(FifthDimension.startMeasuring())).toBeGreaterThanOrEqual(0); 10 | }); 11 | }); 12 | -------------------------------------------------------------------------------- /src/dimension/__tests__/list.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import { 4 | startMeasuring, 5 | stopMeasuring 6 | } from '../list'; 7 | 8 | import { 9 | FifthDimension 10 | } from '../fifth'; 11 | 12 | import type { 13 | Dimension 14 | } from '../type'; 15 | 16 | export const FlatLandDimension: Dimension = { 17 | name: 'two', 18 | displayName: 'two', 19 | units: '-', 20 | startMeasuring: (): number => 1, 21 | stopMeasuring: (start: number): number => start + 1 22 | }; 23 | 24 | describe('startMeasuring', () => { 25 | it('produces measurements', () => { 26 | expect(startMeasuring([FifthDimension])).toEqual([3]); 27 | }); 28 | it('measures the first dimension in the DimensionList first', () => { 29 | expect(startMeasuring([FlatLandDimension, FifthDimension])).toEqual([1, 3]); 30 | }); 31 | }); 32 | 33 | describe('stopMeasuring', () => { 34 | it('produces measurements', () => { 35 | const starting = [3]; 36 | const ending = [0]; 37 | expect(stopMeasuring([FifthDimension], starting, ending)).toEqual([5]); 38 | }); 39 | 40 | it('updates elements of the ending parameter', () => { 41 | const starting = [3]; 42 | const ending = [0]; 43 | stopMeasuring([FifthDimension], starting, ending); 44 | expect(ending).toEqual([5]); 45 | }); 46 | 47 | it('measures the last dimension in the DimensionList first', () => { 48 | const starting = [1, 3]; 49 | const ending = [0, 0]; 50 | expect(stopMeasuring([FlatLandDimension, FifthDimension], starting, ending)) 51 | .toEqual([2, 5]); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /src/dimension/__tests__/memory.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import { 4 | MemoryDimension 5 | } from '../memory'; 6 | 7 | describe('MemoryDimension', () => { 8 | it('produces consequitive measurements', () => { 9 | expect(MemoryDimension.stopMeasuring(MemoryDimension.startMeasuring())) 10 | .toBeGreaterThanOrEqual(0); 11 | }); 12 | }); 13 | -------------------------------------------------------------------------------- /src/dimension/__tests__/time.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import { 4 | TimeDimension 5 | } from '../time'; 6 | 7 | describe('TimeDimension', () => { 8 | it('produces consequitive measurements', () => { 9 | expect(TimeDimension.stopMeasuring(TimeDimension.startMeasuring())).toBeGreaterThanOrEqual(0); 10 | }); 11 | }); 12 | -------------------------------------------------------------------------------- /src/dimension/debug.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import type { 4 | Dimension 5 | } from './type'; 6 | 7 | /** 8 | * Utility object to help debug node optimizations during measurement 9 | */ 10 | export const DebugDimension: Dimension = { 11 | name: 'debug', 12 | displayName: 'debugging', 13 | units: '-', 14 | startMeasuring: (): void => { 15 | process.stdout.write('[[[ START measuring --'); 16 | }, 17 | stopMeasuring: (): number => { 18 | process.stdout.write('-- STOP measuring ]]]\n'); 19 | return 0; 20 | } 21 | }; 22 | -------------------------------------------------------------------------------- /src/dimension/fifth.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import type { 4 | Dimension 5 | } from './type'; 6 | 7 | /** 8 | * The FifthDimension returns a constant measurement, 5, and is used for testing 9 | */ 10 | export const FifthDimension: Dimension = { 11 | name: 'fifth', 12 | displayName: 'fifth', 13 | units: '-', 14 | startMeasuring: (): number => 3, 15 | stopMeasuring: (start: number): number => start + 2 16 | }; 17 | -------------------------------------------------------------------------------- /src/dimension/list.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import type { 4 | Dimension 5 | } from './type'; 6 | 7 | /** 8 | * A list of dimensions to measure 9 | */ 10 | export type DimensionList = Array>; 11 | 12 | /** 13 | * A list of measurements corresponding to a list of dimensions 14 | */ 15 | export type Measurements = Array; 16 | 17 | /** 18 | * Start measuring all requested dimensions in order 19 | * @param dimensions The dimensions to start measuring 20 | */ 21 | export function startMeasuring(dimensions: DimensionList): Measurements { 22 | const startingMeasurements = new Array(dimensions.length); 23 | for (let i = 0; i < dimensions.length; i++) { 24 | startingMeasurements[i] = dimensions[i].startMeasuring(); 25 | } 26 | return startingMeasurements; 27 | } 28 | 29 | /** 30 | * Take the final measurement of all requested dimensions in the reverse order 31 | * from which they were started. 32 | * 33 | * Receives a pre-allocated array to prevent need to allocate memory in this 34 | * function. 35 | * 36 | * @param dimensions The dimensions to stop measuring 37 | * @param startingMeasurements A list of starting measurement for each dimension 38 | * @param endingMeasurements A pre-allocated Array to record ending measuremsnts into 39 | * @returns endingMeasurements 40 | */ 41 | export function stopMeasuring( 42 | dimensions: DimensionList, 43 | startingMeasurements: Measurements, 44 | endingMeasurements: Measurements 45 | ): Measurements { 46 | for (let i = dimensions.length - 1; i >= 0; i--) { 47 | endingMeasurements[i] = 48 | dimensions[i].stopMeasuring(startingMeasurements[i]); 49 | } 50 | return endingMeasurements; 51 | } 52 | -------------------------------------------------------------------------------- /src/dimension/memory.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import type { 4 | Dimension 5 | } from './type'; 6 | 7 | export type NodeProcessMemoryUsage = { 8 | rss: number, 9 | heapTotal: number, 10 | heapUsed: number 11 | /* 12 | Not defined in flow/lib/node.js 13 | but is defined at https://nodejs.org/api/process.html#process_process_memoryusage 14 | external: number 15 | */ 16 | }; 17 | 18 | /** 19 | * Utility object describing how to measure memory 20 | */ 21 | export const MemoryDimension: Dimension = { 22 | name: 'memory', 23 | displayName: 'Memory', 24 | units: 'b', 25 | startMeasuring: (): NodeProcessMemoryUsage => process.memoryUsage(), 26 | stopMeasuring: (startMemory: NodeProcessMemoryUsage): number => { 27 | const memory = process.memoryUsage(); 28 | return memory.heapUsed - startMemory.heapUsed; 29 | } 30 | }; 31 | -------------------------------------------------------------------------------- /src/dimension/time.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import type { 4 | Dimension 5 | } from './type'; 6 | 7 | export type NodeHighResolutionTime = [number, number]; 8 | 9 | /** 10 | * Utility object describing how to measure time 11 | */ 12 | export const TimeDimension: Dimension = { 13 | name: 'time', 14 | displayName: 'Elapsed Time', 15 | units: 'ns', 16 | startMeasuring: (): NodeHighResolutionTime => process.hrtime(), 17 | stopMeasuring: (startTime: NodeHighResolutionTime): number => { 18 | const nanoSecondsPerSecond = 1e9; 19 | const elapsed = process.hrtime(startTime); 20 | return (elapsed[0] * nanoSecondsPerSecond) + elapsed[1]; 21 | } 22 | }; 23 | -------------------------------------------------------------------------------- /src/dimension/type.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | /** 4 | * Describe a measurement dimension 5 | */ 6 | export type Dimension = { 7 | name: string; 8 | displayName: string; 9 | units: string; 10 | startMeasuring: () => T; 11 | stopMeasuring: (start: T) => number; 12 | } 13 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | export { 4 | startBenchmarking 5 | } from './runner'; 6 | 7 | export { 8 | runBenchmarkTest, 9 | startBenchmarkTest 10 | } from './tester'; 11 | 12 | export { 13 | compareMemoryResults, 14 | compareTimeResults 15 | } from './report/compare'; 16 | 17 | export { 18 | reportResult 19 | } from './report/report'; 20 | 21 | export { 22 | TimeDimension 23 | } from './dimension/time'; 24 | 25 | export { 26 | MemoryDimension 27 | } from './dimension/memory'; 28 | 29 | export { 30 | DebugDimension 31 | } from './dimension/debug'; 32 | 33 | -------------------------------------------------------------------------------- /src/report/__tests__/format.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import { 4 | formatLeft, 5 | formatRight 6 | } from '../format'; 7 | 8 | jest.unmock('../format'); 9 | 10 | describe('formatLeft', () => { 11 | it('pads a string to the requested length', () => { 12 | const expectedString = 'betty '; 13 | const result = formatLeft('betty', expectedString.length); 14 | expect(result).toBe(expectedString); 15 | }); 16 | 17 | it('converts numbers to strings', () => { 18 | const aNumber = 123; 19 | const expectedString = '123'; 20 | const result = formatLeft(aNumber, expectedString.length); 21 | expect(result).toBe(expectedString); 22 | }); 23 | 24 | it('indicates when value to format exceeds allowed length', () => { 25 | const value = 'betty'; 26 | const expectedString = '****'; 27 | const result = formatLeft(value, value.length - 1); 28 | expect(result).toBe(expectedString); 29 | }); 30 | 31 | it('does not pad for exact length', () => { 32 | const value = 'betty'; 33 | const result = formatLeft(value, value.length); 34 | expect(result).toBe(value); 35 | }); 36 | }); 37 | 38 | describe('formatRight', () => { 39 | it('pads a string to the requested length', () => { 40 | const expectedString = ' betty'; 41 | const result = formatRight('betty', expectedString.length); 42 | expect(result).toBe(expectedString); 43 | }); 44 | 45 | it('converts numbers to strings', () => { 46 | const aNumber = 123; 47 | const expectedString = '123'; 48 | const result = formatRight(aNumber, expectedString.length); 49 | expect(result).toBe(expectedString); 50 | }); 51 | 52 | it('indicates when value to format exceeds allowed length', () => { 53 | const value = 'betty'; 54 | const expectedString = '****'; 55 | const result = formatRight(value, value.length - 1); 56 | expect(result).toBe(expectedString); 57 | }); 58 | 59 | it('does not pad for exact length', () => { 60 | const value = 'betty'; 61 | const result = formatRight(value, value.length); 62 | expect(result).toBe(value); 63 | }); 64 | }); 65 | -------------------------------------------------------------------------------- /src/report/compare.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import { 3 | mean 4 | } from 'simple-statistics'; 5 | import { 6 | tTest 7 | } from 'experiments.js'; 8 | import { 9 | formatLeft, 10 | formatRight 11 | } from './format'; 12 | import type { 13 | BenchmarkSuiteResult 14 | } from '../benchmark'; 15 | 16 | const defaultSignificanceThreshold = 0.05; 17 | const defaultConfidenceLevel = 0.95; 18 | 19 | /** 20 | * Compare the results of two benchmarks, outputting the differences based 21 | * on statistical significance. 22 | */ 23 | export function compareMemoryResults( 24 | result1: BenchmarkSuiteResult, 25 | result2: BenchmarkSuiteResult, 26 | outputFn: (...x: any) => void, 27 | significanceThreshold: number = defaultSignificanceThreshold, 28 | confidenceLevel: number = defaultConfidenceLevel 29 | ): void { 30 | const marginOfErrorSize = 3; 31 | const marginOfErrorUnits = '%'; 32 | const marginOfErrorPrefix = ' ±'; 33 | const memorySize = 10; 34 | const memoryUnits = ' b'; 35 | const changeSize = 5; 36 | const changeUnits = '%'; 37 | const asyncColumnSize = 1; 38 | const benchmarkColumnSize = 40; 39 | const memoryColumnSize = 40 | memorySize + 41 | memoryUnits.length; 42 | const changeColumnSize = 43 | changeSize + 44 | changeUnits.length + 45 | marginOfErrorPrefix.length + 46 | marginOfErrorSize + 47 | marginOfErrorUnits.length; 48 | const toPercent = 100; 49 | 50 | // Sort the input results into a base result (before) and 51 | // test result (after) 52 | let baseResult; 53 | let testResult; 54 | if (result1.startTime > result2.startTime) { 55 | baseResult = result2; 56 | testResult = result1; 57 | } else { 58 | baseResult = result1; 59 | testResult = result2; 60 | } 61 | 62 | outputFn( 63 | formatLeft('A', asyncColumnSize), 64 | formatLeft('Benchmark', benchmarkColumnSize), 65 | formatRight('Memory', memoryColumnSize), 66 | formatRight('Change', changeColumnSize) 67 | ); 68 | outputFn( 69 | '-'.repeat(asyncColumnSize), 70 | '-'.repeat(benchmarkColumnSize), 71 | '-'.repeat(memoryColumnSize), 72 | '-'.repeat(changeColumnSize) 73 | ); 74 | 75 | let insignificantBenchmarks = 0; 76 | for (let i = 0; i < baseResult.results.length; i++) { 77 | const memory = tTest( 78 | baseResult.results[i].samples.memory, 79 | testResult.results[i].samples.memory, 80 | confidenceLevel 81 | ); 82 | 83 | if (memory.probabilityLevel > significanceThreshold) { 84 | insignificantBenchmarks += 1; 85 | continue; 86 | } 87 | 88 | const asyncColumn = 89 | formatLeft(baseResult.results[i].isAsynchronous ? '*' : '', asyncColumnSize); 90 | const benchmarkColumn = formatLeft( 91 | baseResult.results[i].name, benchmarkColumnSize 92 | ); 93 | const memoryDifference = Math.round(memory.meanDifference); 94 | const memoryColumn = 95 | formatRight(memoryDifference, memorySize) + 96 | memoryUnits; 97 | const baseMean = mean(baseResult.results[i].samples.memory); 98 | const change = Math.round((memory.meanDifference * toPercent) / baseMean); 99 | const interval = memory.confidenceInterval[1] - memory.meanDifference; 100 | const marginOfError = Math.round((interval * toPercent) / baseMean); 101 | const changeColumn = 102 | formatRight(change, changeSize) + 103 | changeUnits + 104 | marginOfErrorPrefix + 105 | formatRight(marginOfError, marginOfErrorSize) + 106 | marginOfErrorUnits; 107 | 108 | outputFn(asyncColumn, benchmarkColumn, memoryColumn, changeColumn); 109 | } 110 | 111 | if (insignificantBenchmarks > 0) { 112 | outputFn( 113 | ` ${insignificantBenchmarks} benchmarks not different ` + 114 | `(p > ${significanceThreshold})` 115 | ); 116 | } 117 | } 118 | 119 | /** 120 | * Compare the results of two benchmarks, outputting the differences based 121 | * on statistical significance. 122 | */ 123 | export function compareTimeResults( 124 | result1: BenchmarkSuiteResult, 125 | result2: BenchmarkSuiteResult, 126 | outputFn: (...x: any) => void, 127 | significanceThreshold: number = defaultSignificanceThreshold, 128 | confidenceLevel: number = defaultConfidenceLevel 129 | ): void { 130 | const marginOfErrorSize = 3; 131 | const marginOfErrorUnits = '%'; 132 | const marginOfErrorPrefix = ' ±'; 133 | const timeSize = 8; 134 | const timeUnits = ' ns'; 135 | const changeSize = 5; 136 | const changeUnits = '%'; 137 | const asyncColumnSize = 1; 138 | const benchmarkColumnSize = 40; 139 | const timeColumnSize = 140 | timeSize + 141 | timeUnits.length; 142 | const changeColumnSize = 143 | changeSize + 144 | changeUnits.length + 145 | marginOfErrorPrefix.length + 146 | marginOfErrorSize + 147 | marginOfErrorUnits.length; 148 | const toPercent = 100; 149 | 150 | // Sort the input results into a base result (before) and 151 | // test result (after) 152 | let baseResult; 153 | let testResult; 154 | if (result1.startTime > result2.startTime) { 155 | baseResult = result2; 156 | testResult = result1; 157 | } else { 158 | baseResult = result1; 159 | testResult = result2; 160 | } 161 | 162 | outputFn( 163 | formatLeft('A', asyncColumnSize), 164 | formatLeft('Benchmark', benchmarkColumnSize), 165 | formatRight('Time', timeColumnSize), 166 | formatRight('Change', changeColumnSize) 167 | ); 168 | outputFn( 169 | '-'.repeat(asyncColumnSize), 170 | '-'.repeat(benchmarkColumnSize), 171 | '-'.repeat(timeColumnSize), 172 | '-'.repeat(changeColumnSize) 173 | ); 174 | 175 | let insignificantBenchmarks = 0; 176 | for (let i = 0; i < baseResult.results.length; i++) { 177 | const time = tTest( 178 | baseResult.results[i].samples.time, 179 | testResult.results[i].samples.time, 180 | confidenceLevel 181 | ); 182 | 183 | if (time.probabilityLevel > significanceThreshold) { 184 | insignificantBenchmarks += 1; 185 | continue; 186 | } 187 | 188 | const asyncColumn = 189 | formatLeft(baseResult.results[i].isAsynchronous ? '*' : '', asyncColumnSize); 190 | const benchmarkColumn = formatLeft( 191 | baseResult.results[i].name, benchmarkColumnSize 192 | ); 193 | const timeDifference = Math.round(time.meanDifference); 194 | const timeColumn = 195 | formatRight(timeDifference, timeSize) + 196 | timeUnits; 197 | const baseMean = mean(baseResult.results[i].samples.time); 198 | const change = Math.round((time.meanDifference * toPercent) / baseMean); 199 | const interval = time.confidenceInterval[1] - time.meanDifference; 200 | const marginOfError = Math.round((interval * toPercent) / baseMean); 201 | const changeColumn = 202 | formatRight(change, changeSize) + 203 | changeUnits + 204 | marginOfErrorPrefix + 205 | formatRight(marginOfError, marginOfErrorSize) + 206 | marginOfErrorUnits; 207 | 208 | outputFn(asyncColumn, benchmarkColumn, timeColumn, changeColumn); 209 | } 210 | 211 | if (insignificantBenchmarks > 0) { 212 | outputFn( 213 | ` ${insignificantBenchmarks} benchmarks not different ` + 214 | `(p > ${significanceThreshold})` 215 | ); 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /src/report/format.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | /** 4 | * Left justify a value within a given length 5 | */ 6 | export function formatLeft( 7 | value: mixed, 8 | totalLength: number 9 | ): string { 10 | const str = String(value); 11 | if (str.length > totalLength) { 12 | return '*'.repeat(totalLength); 13 | } 14 | return str + ' '.repeat(totalLength - str.length); 15 | } 16 | 17 | /** 18 | * Right justify a value within a given length 19 | */ 20 | export function formatRight( 21 | value: mixed, 22 | totalLength: number 23 | ): string { 24 | const str = String(value); 25 | if (str.length > totalLength) { 26 | return '*'.repeat(totalLength); 27 | } 28 | return ' '.repeat(totalLength - str.length) + str; 29 | } 30 | -------------------------------------------------------------------------------- /src/report/report.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import { 4 | mean 5 | } from 'simple-statistics'; 6 | import { 7 | formatLeft, 8 | formatRight 9 | } from './format'; 10 | import type { 11 | BenchmarkSuiteResult 12 | } from '../benchmark'; 13 | import { 14 | marginOfError 15 | } from './statistics'; 16 | 17 | /** 18 | * 19 | */ 20 | export function reportResult( 21 | benchmarkResult: BenchmarkSuiteResult, 22 | outputFn: (...x: any) => void 23 | ): void { 24 | const marginOfErrorSize = 3; 25 | const marginOfErrorUnits = '%'; 26 | const marginOfErrorPrefix = ' ±'; 27 | const timeSize = 8; 28 | const timeUnits = ' ns'; 29 | const memorySize = 10; 30 | const memoryUnits = ' b'; 31 | const asyncColumnSize = 1; 32 | const benchmarkColumnSize = 40; 33 | const timeColumnSize = 34 | timeSize + 35 | timeUnits.length + 36 | marginOfErrorPrefix.length + 37 | marginOfErrorSize + 38 | marginOfErrorUnits.length; 39 | const memoryColumnSize = 40 | memorySize + 41 | memoryUnits.length + 42 | marginOfErrorPrefix.length + 43 | marginOfErrorSize + 44 | marginOfErrorUnits.length; 45 | 46 | outputFn( 47 | formatLeft('A', asyncColumnSize), 48 | formatLeft('Benchmark', benchmarkColumnSize), 49 | formatRight('Time', timeColumnSize), 50 | formatRight('Memory', memoryColumnSize) 51 | ); 52 | outputFn( 53 | '-'.repeat(asyncColumnSize), 54 | '-'.repeat(benchmarkColumnSize), 55 | '-'.repeat(timeColumnSize), 56 | '-'.repeat(memoryColumnSize) 57 | ); 58 | benchmarkResult.results.forEach((result) => { 59 | const time = Math.round(mean(result.samples.time)); 60 | const timeMoe = Math.round(marginOfError(result.samples.time)); 61 | const memory = Math.round(mean(result.samples.memory)); 62 | const memoryMoe = Math.round(marginOfError(result.samples.memory)); 63 | const asyncColumn = 64 | formatLeft(result.isAsynchronous ? '*' : '', asyncColumnSize); 65 | const benchmarkColumn = 66 | formatLeft(result.name, benchmarkColumnSize); 67 | const timeColumn = 68 | formatRight(time, timeSize) + 69 | timeUnits + 70 | marginOfErrorPrefix + 71 | formatRight(timeMoe, marginOfErrorSize) + 72 | marginOfErrorUnits; 73 | const memoryColumn = 74 | formatRight(memory, memorySize) + 75 | memoryUnits + 76 | marginOfErrorPrefix + 77 | formatRight(memoryMoe, marginOfErrorSize) + 78 | marginOfErrorUnits; 79 | outputFn(asyncColumn, benchmarkColumn, timeColumn, memoryColumn); 80 | }); 81 | } 82 | -------------------------------------------------------------------------------- /src/report/statistics.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import { 3 | mean, 4 | standardDeviation 5 | } from 'simple-statistics'; 6 | 7 | import { 8 | confidenceIntervalForT 9 | } from 'experiments.js'; 10 | 11 | /** 12 | * Calculate the margin of error for a set of samples 13 | */ 14 | export function marginOfError(samples: Array): number { 15 | const toPercent = 100; 16 | const confidenceLevel = 0.95; 17 | const sampleMean = mean(samples); 18 | const standardError = standardDeviation(samples) / Math.sqrt(samples.length); 19 | const degreesOfFreedom = samples.length - 1; 20 | const interval = confidenceIntervalForT( 21 | sampleMean, 22 | standardError, 23 | degreesOfFreedom, 24 | confidenceLevel 25 | ); 26 | return (((interval[1] - sampleMean) / sampleMean) * toPercent) || 0; 27 | } 28 | -------------------------------------------------------------------------------- /src/runner.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import flatten from 'lodash.flatten'; 4 | 5 | import type { 6 | Benchmark, 7 | BenchmarkSuiteResult 8 | } from './benchmark'; 9 | 10 | import type { 11 | DimensionList 12 | } from './dimension/list'; 13 | 14 | import { 15 | setDimensionList, 16 | scheduleNextBenchmark 17 | } from './scheduler/legacy'; 18 | 19 | import { 20 | populateInlineCaches 21 | } from './warmer'; 22 | 23 | /** 24 | * Start the process of running a suite of benchmarks 25 | */ 26 | export function startBenchmarking( 27 | name: string, 28 | benchmarkSuite: Array, 29 | dimensionList: DimensionList 30 | ): Promise<*> { 31 | setDimensionList(dimensionList); 32 | return new Promise((resolve, reject) => { 33 | const suiteResult : BenchmarkSuiteResult = { 34 | name, 35 | startTime: Date.now(), 36 | dimensions: dimensionList.map(dimension => dimension.name), 37 | results: [] 38 | }; 39 | const suite = flatten(benchmarkSuite); 40 | 41 | // impedance mismatch between async styles 42 | populateInlineCaches(suite).then(() => { 43 | scheduleNextBenchmark(resolve, reject, suite, suiteResult); 44 | }); 45 | }); 46 | } 47 | -------------------------------------------------------------------------------- /src/sampler.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import type { 4 | ASynchronousBenchmark, 5 | SynchronousBenchmark, 6 | Benchmark 7 | } from './benchmark'; 8 | 9 | import { 10 | startMeasuring, 11 | stopMeasuring, 12 | type Measurements, 13 | type DimensionList 14 | } from './dimension/list'; 15 | 16 | /** 17 | * Callback that is called with a Sample when benchmarking is complete 18 | */ 19 | type completeFn = (sample: Measurements) => void; 20 | 21 | /** 22 | * Callback that is called with an Error when a benchmark cannot be completed 23 | */ 24 | type rejectFn = (err: Error) => void; 25 | 26 | /** 27 | * Put memory into a stable state if we can to avoid triggering gc during a 28 | * sample run 29 | */ 30 | function cleanUpMemory() { 31 | if (global.gc) { 32 | global.gc(); 33 | } 34 | } 35 | 36 | /** 37 | * Setup benchmark resources before running 38 | * @TODO how to wait for an async setup? 39 | */ 40 | export function setUpBenchmark(benchmark: Benchmark): Error | false { 41 | if (typeof benchmark.setUp === 'function') { 42 | try { 43 | benchmark.setUp(); 44 | } catch (e) { 45 | return e; 46 | } 47 | } else if (typeof benchmark.setUp !== 'undefined') { 48 | return new Error('setUp must be either a function or undefined'); 49 | } 50 | return false; 51 | } 52 | 53 | /** 54 | * release benchmark resources after running 55 | * @TODO how to wait for an async tearDown? 56 | */ 57 | export function tearDownBenchmark(benchmark: Benchmark): Error | false { 58 | if (typeof benchmark.tearDown === 'function') { 59 | try { 60 | benchmark.tearDown(); 61 | } catch (e) { 62 | return e; 63 | } 64 | } else if (typeof benchmark.tearDown !== 'undefined') { 65 | return new Error('tearDown must be either a function or undefined'); 66 | } 67 | return false; 68 | } 69 | 70 | /** 71 | * Run an asynchronous benchmark for the desired number of operations and capture 72 | * the results 73 | */ 74 | export function collectAsynchronousSample( 75 | benchmark: ASynchronousBenchmark, 76 | dimensions: DimensionList, 77 | opsPerSample: number, 78 | complete: completeFn, 79 | reject: rejectFn 80 | ): void { 81 | // Pre-allocate to avoid allocating memory during run 82 | const promises = new Array(opsPerSample); 83 | const ending = new Array(dimensions.length); 84 | 85 | const setUpError = setUpBenchmark(benchmark); 86 | if (setUpError) { 87 | reject(setUpError); 88 | return; 89 | } 90 | 91 | cleanUpMemory(); 92 | 93 | const starting = startMeasuring(dimensions); 94 | 95 | try { 96 | // run the benchmark function and collect promises 97 | for (let i = 0; i < opsPerSample; i++) { 98 | promises[i] = benchmark.startRunning(); 99 | } 100 | } catch (e) { 101 | reject(e); 102 | return; 103 | } 104 | 105 | // take measurements when all promises have been resolved 106 | Promise.all(promises).then(() => { 107 | stopMeasuring(dimensions, starting, ending); 108 | 109 | const tearDownError = tearDownBenchmark(benchmark); 110 | if (tearDownError) { 111 | reject(tearDownError); 112 | return; 113 | } 114 | 115 | complete(ending); 116 | }).catch((error) => { 117 | reject(error); 118 | }); 119 | } 120 | 121 | /** 122 | * Run a synchronous benchmark for the desired number of operations and capture 123 | * the results 124 | */ 125 | export function collectSynchronousSample( 126 | benchmark: SynchronousBenchmark, 127 | dimensions: DimensionList, 128 | opsPerSample: number, 129 | complete: completeFn, 130 | reject: rejectFn 131 | ) { 132 | // Pre-allocate to avoid allocating memory during run 133 | const ending = new Array(dimensions.length); 134 | 135 | const setUpError = setUpBenchmark(benchmark); 136 | if (setUpError) { 137 | reject(setUpError); 138 | return; 139 | } 140 | 141 | cleanUpMemory(); 142 | 143 | const starting = startMeasuring(dimensions); 144 | 145 | try { 146 | // run the benchmark function 147 | for (let i = 0; i < opsPerSample; i++) { 148 | benchmark.run(); 149 | } 150 | } catch (e) { 151 | reject(e); 152 | return; 153 | } 154 | 155 | stopMeasuring(dimensions, starting, ending); 156 | 157 | const tearDownError = tearDownBenchmark(benchmark); 158 | if (tearDownError) { 159 | reject(tearDownError); 160 | return; 161 | } 162 | 163 | complete(ending); 164 | } 165 | -------------------------------------------------------------------------------- /src/scheduler/legacy.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import { 4 | collectSynchronousSample, 5 | collectAsynchronousSample, 6 | } from '../sampler'; 7 | 8 | import type { 9 | Benchmark, 10 | BenchmarkResult, 11 | BenchmarkSuiteResult 12 | } from '../benchmark'; 13 | 14 | import type { 15 | DimensionList 16 | } from '../dimension/list'; 17 | 18 | type resolveFn = (value: any) => void; 19 | 20 | type rejectFn = (value: any) => void; 21 | 22 | let dimensions: DimensionList = []; 23 | 24 | export function setDimensionList(dim: DimensionList) { 25 | dimensions = dim; 26 | } 27 | 28 | /** 29 | * Schedule a benchmark to be run at a later time 30 | */ 31 | export function scheduleNextBenchmark( 32 | resolve: resolveFn, 33 | reject: rejectFn, 34 | benchmarkSuite: Array, 35 | suiteResult: BenchmarkSuiteResult 36 | ): void { 37 | setImmediate(runBenchmark, resolve, reject, benchmarkSuite, suiteResult); 38 | } 39 | 40 | /** 41 | * Start the process of running a single benchmark 42 | */ 43 | function runBenchmark( 44 | resolve: resolveFn, 45 | reject: rejectFn, 46 | benchmarkSuite: Array, 47 | suiteResult: BenchmarkSuiteResult 48 | ): void { 49 | const benchmark: Benchmark = benchmarkSuite.shift(); 50 | 51 | // if there are no more benchmarks, stop 52 | if (!benchmark) { 53 | resolve(suiteResult); 54 | return; 55 | } 56 | 57 | if (!benchmark.name) { 58 | reject(new Error( 59 | `"${benchmark.name} does not provide a name` 60 | )); 61 | return; 62 | } 63 | 64 | /* eslint no-unneeded-ternary:0 */ 65 | const initialResult:BenchmarkResult = { 66 | name: benchmark.name, 67 | isAsynchronous: benchmark.startRunning ? true : false, 68 | opsPerSample: 1000, 69 | numSamples: 100, 70 | samples: dimensions.reduce((samples, dimension) => { 71 | samples[dimension.name] = []; 72 | return samples; 73 | }, Object.create(null)) 74 | }; 75 | 76 | const resolveBenchmark = (finalResult: BenchmarkResult) => { 77 | suiteResult.results.push(finalResult); 78 | scheduleNextBenchmark(resolve, reject, benchmarkSuite, suiteResult); 79 | }; 80 | 81 | scheduleNextSample(resolveBenchmark, reject, benchmark, initialResult); 82 | } 83 | 84 | /** 85 | * Schedule sample collection 86 | */ 87 | function scheduleNextSample( 88 | resolve: resolveFn, 89 | reject: rejectFn, 90 | benchmark: Benchmark, 91 | result: BenchmarkResult 92 | ): void { 93 | if (isSamplingComplete(result)) { 94 | resolve(result); 95 | return; 96 | } 97 | if (benchmark.run) { 98 | setImmediate( 99 | collectSynchronousSample, 100 | benchmark, 101 | dimensions, 102 | result.opsPerSample, 103 | (finalMeasurements) => { 104 | recordMeasurements(result, finalMeasurements); 105 | scheduleNextSample(resolve, reject, benchmark, result); 106 | }, 107 | reject 108 | ); 109 | } else if (benchmark.startRunning) { 110 | setImmediate( 111 | collectAsynchronousSample, 112 | benchmark, 113 | dimensions, 114 | result.opsPerSample, 115 | (finalMeasurements) => { 116 | recordMeasurements(result, finalMeasurements); 117 | scheduleNextSample(resolve, reject, benchmark, result); 118 | }, 119 | reject 120 | ); 121 | } else { 122 | reject(new Error( 123 | `"${benchmark.name} does not provide a run or startRunning function` 124 | )); 125 | } 126 | } 127 | 128 | /** 129 | * Does the result contain all required samples? 130 | */ 131 | function isSamplingComplete(result: BenchmarkResult) { 132 | return result.samples.time.length >= result.numSamples; 133 | } 134 | 135 | /** 136 | * Record the measurements from a set of dimensions 137 | */ 138 | function recordMeasurements( 139 | result: BenchmarkResult, 140 | finalMeasurements: Array 141 | ): void { 142 | for (let i = 0; i < dimensions.length; i++) { 143 | result.samples[dimensions[i].name].push( 144 | Math.round(finalMeasurements[i] / result.opsPerSample) 145 | ); 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /src/tester.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import flatten from 'lodash.flatten'; 4 | 5 | import type { 6 | Benchmark 7 | } from './benchmark'; 8 | 9 | /** 10 | * Find a benchmark matching a given name 11 | */ 12 | function findBenchmark( 13 | suite: Array, 14 | name: string 15 | ): ?Benchmark { 16 | return suite.find(benchmark => benchmark.name === name); 17 | } 18 | 19 | /** 20 | * Run a benchmark specified by name from the suite, returning 21 | * the value returned by calling run. 22 | */ 23 | export function runBenchmarkTest( 24 | suite: Array, 25 | name: string 26 | ): any { 27 | const benchmark = findBenchmark(flatten(suite), name); 28 | if (!benchmark) { 29 | throw new Error(`Benchmark not found "${name}"`); 30 | } 31 | 32 | if (benchmark.setUp) { 33 | benchmark.setUp(); 34 | } 35 | 36 | if (typeof benchmark.run !== 'function') { 37 | throw new Error(`Benchmark "${name}" does not define a run function`); 38 | } 39 | 40 | const result = benchmark.run(); 41 | 42 | if (benchmark.tearDown) { 43 | benchmark.tearDown(); 44 | } 45 | 46 | return result; 47 | } 48 | 49 | /** 50 | * Run a benchmark specified by name from the suite, returning 51 | * the value returned by calling run. 52 | */ 53 | export function startBenchmarkTest( 54 | suite: Array, 55 | name: string 56 | ): Promise<*> { 57 | const benchmark = findBenchmark(flatten(suite), name); 58 | if (!benchmark) { 59 | throw new Error(`Benchmark not found "${name}"`); 60 | } 61 | 62 | if (benchmark.setUp) { 63 | benchmark.setUp(); 64 | } 65 | 66 | if (typeof benchmark.startRunning !== 'function') { 67 | throw new Error(`Benchmark "${name}" does not define a startRunning function`); 68 | } 69 | 70 | return benchmark.startRunning().then((result) => { 71 | if (benchmark.tearDown) { 72 | benchmark.tearDown(); 73 | } 74 | return result; 75 | }); 76 | } 77 | -------------------------------------------------------------------------------- /src/warmer.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | 3 | import { 4 | setUpBenchmark, 5 | tearDownBenchmark 6 | } from './sampler'; 7 | 8 | import type { 9 | Benchmark 10 | } from './benchmark'; 11 | 12 | /** 13 | * Prevent variation by running each benchmark once, thus populating the 14 | * inline caches in the code under test with the full diversity of values 15 | * in the benchmark suite. Reduces sensitivity to benchmark ordering by 16 | * preventing optimization and de-optimization cycles as v8 learns new 17 | * run time information while cycling through benchmarks. 18 | */ 19 | export function populateInlineCaches(benchmarkSuite: Array) { 20 | const promises = []; 21 | for (let i = 0; i < benchmarkSuite.length; i++) { 22 | const benchmark = benchmarkSuite[i]; 23 | const setUpErr = setUpBenchmark(benchmark); 24 | if (setUpErr) { 25 | promises.push(Promise.reject(setUpErr)); 26 | } 27 | if (typeof benchmark.run === 'function') { 28 | benchmark.run(); 29 | const tearDownErr = tearDownBenchmark(benchmark); 30 | if (tearDownErr) { 31 | promises.push(Promise.reject(tearDownErr)); 32 | } 33 | } else if (typeof benchmark.startRunning === 'function') { 34 | promises.push(benchmark.startRunning().then(() => { 35 | const Err = tearDownBenchmark(benchmark); 36 | if (Err) { 37 | throw Err; 38 | } 39 | })); 40 | } else { 41 | throw Error( 42 | `"${benchmark.name} does not provide a run or startRunning function` 43 | ); 44 | } 45 | } 46 | return Promise.all(promises); 47 | } 48 | --------------------------------------------------------------------------------