├── .gitignore ├── LICENSE ├── README.md ├── autocannon-ci.js ├── example ├── autocannon-s3.yml ├── autocannon.yml └── server.js ├── help ├── autocannon-ci.txt ├── compare.txt └── run.txt ├── lib ├── compare.js ├── dashboard.js ├── get-backing.js ├── run-page.js ├── runner.js ├── storage.js └── template.js ├── package.json └── test ├── compare.test.js ├── meta.json ├── result-2.json ├── result.json ├── runner.test.js ├── server.js └── storage.test.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # nyc test coverage 18 | .nyc_output 19 | 20 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 21 | .grunt 22 | 23 | # node-waf configuration 24 | .lock-wscript 25 | 26 | # Compiled binary addons (http://nodejs.org/api/addons.html) 27 | build/Release 28 | 29 | # Dependency directories 30 | node_modules 31 | jspm_packages 32 | 33 | # Optional npm cache directory 34 | .npm 35 | 36 | # Optional REPL history 37 | .node_repl_history 38 | 39 | # test data 40 | perf-results 41 | 42 | # because Mac OS X 43 | .DS_Store 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Matteo Collina 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![banner](https://raw.githubusercontent.com/mcollina/autocannon/master/autocannon-banner.png) 2 | 3 | # autocannon-ci 4 | 5 | **autocannon-ci** can store the results and generate the 6 | [flamegraphs][0x] of your HTTP/1.1 benchmarks of Node.js server. 7 | Run your [autocannon][] benchmarks as 8 | part of your CI/dev flow, for Node.js. 9 | 10 | It can also generate a little website containing all the result of your 11 | benchmarking, including [flamegraphs with 0x][0x]: 12 | 13 | * [Dashboard](https://s3-us-west-2.amazonaws.com/autocannon-ci-test/index.html) 14 | * [Job page](https://s3-us-west-2.amazonaws.com/autocannon-ci-test/run-2/index.html) 15 | 16 | ## Install 17 | 18 | ```sh 19 | npm i autocannon-ci -g 20 | ``` 21 | 22 | ## Usage 23 | 24 | ```sh 25 | autocannon-ci -c autocannon.yml 26 | ``` 27 | 28 | ## Configuration Example 29 | 30 | ```yaml 31 | server: ./server.js 32 | benchmarks: 33 | root: 34 | connections: 100 35 | duration: 5 36 | url: localhost:3000 37 | b: 38 | connections: 100 39 | duration: 5 40 | url: localhost:3000/b 41 | storage: 42 | type: fs 43 | path: perf-results 44 | ``` 45 | 46 | ## Available commands and full options 47 | 48 | autocannon-ci is a tool to run multiple HTTP/1.1 benchmarks, and generate the relative 49 | flamegraphs, with the help of 0x. 50 | 51 | Available commands: 52 | 53 | * run (default) 54 | * compare 55 | * help 56 | 57 | ### Run 58 | 59 | ``` 60 | Usage: autocannon-ci run [OPTS] 61 | 62 | Runs the benchmarks configured in the autocannon-ci configuration file, and 63 | save them according to the storage configured in the config file. The job id 64 | is used to identify the single run. 65 | 66 | Options: 67 | 68 | --config/-c CONFIG Use the given config file; default: `autocannon.yml`. 69 | --job/-j ID Use the specific job id. 70 | --flamegrah/-F Generate and store flamegraphs. 71 | ``` 72 | 73 | ### Compare 74 | 75 | ``` 76 | Usage: autocannon-ci compare [OPTS] [A] [B] 77 | 78 | Compare the job with id A against the job id B. A and B are defaulted to the 79 | latest two jobs. 80 | 81 | Options: 82 | 83 | --config/-c CONFIG Use the given config file; default: `autocannon.yml`. 84 | Launch 'autocannon-ci help [command]' to know more about the commands. 85 | ``` 86 | 87 | ## Storage 88 | 89 | **autocannon-ci** can store the results and flamegraphs within a 90 | storage, which is configured in the config file. 91 | 92 | ### type: fs 93 | 94 | ```yaml 95 | storage: 96 | type: fs 97 | path: perf-results 98 | ``` 99 | 100 | ### type: s3 101 | 102 | ```yaml 103 | storage: 104 | type: s3 105 | bucket: autocannon-ci-test 106 | region: us-west-2 107 | ``` 108 | 109 | This will also require the environment variables `S3_ACCESS_KEY` and `S3_SECRET_KEY` 110 | containing the proper credentials to access S3. It uses the 111 | [aws-sdk](http://npm.im/aws-sdk), so any other way of configuring the 112 | credential for that will work for **autocannon-ci** as well. 113 | 114 | ## Acknowledgements 115 | 116 | This project was kindly sponsored by [nearForm](http://nearform.com). 117 | 118 | ## License 119 | 120 | MIT 121 | 122 | [autocannon]: https://github.com/mcollina/autocannon 123 | [0x]: https://github.com/davidmarkclements/0x 124 | -------------------------------------------------------------------------------- /autocannon-ci.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | 3 | 'use strict' 4 | 5 | const path = require('path') 6 | const debug = require('debug')('autocannon-ci:cli') 7 | const commist = require('commist')() 8 | const minimist = require('minimist') 9 | const fs = require('fs') 10 | const Runner = require('./lib/runner') 11 | const YAML = require('yamljs') 12 | const chalk = require('chalk') 13 | const autocannon = require('autocannon') 14 | const psTree = require('ps-tree') 15 | const help = require('help-me')({ 16 | dir: 'help', 17 | help: 'autocannon-ci.txt' 18 | }) 19 | const Storage = require('./lib/storage') 20 | const getBacking = require('./lib/get-backing') 21 | const compare = require('./lib/compare') 22 | const table = require('table') 23 | 24 | // get the current node path for clean windows support 25 | const isWin = /^win/.test(process.platform) 26 | const nodePath = isWin ? ('"' + process.argv[0] + '"') : process.argv[0] 27 | const zeroX = path.join(path.dirname(require.resolve('0x')), 'cmd.js') 28 | 29 | const res = commist 30 | .register('run', runCmd) 31 | .register('compare', compareCmd) 32 | .register('help', help.toStdout) 33 | .parse(process.argv.splice(2)) 34 | 35 | if (res) { 36 | runCmd(res) 37 | } 38 | 39 | process.on('uncaughtException', cleanup) 40 | process.on('beforeExit', cleanup) 41 | 42 | function hasFile (file) { 43 | try { 44 | fs.accessSync(file) 45 | return true 46 | } catch (err) { 47 | return false 48 | } 49 | } 50 | 51 | function runCmd (argv) { 52 | const args = minimist(argv, { 53 | boolean: ['flamegraph'], 54 | integer: ['job'], 55 | alias: { 56 | config: 'c', 57 | job: 'j', 58 | flamegraph: 'F' 59 | }, 60 | default: { 61 | config: path.resolve('autocannon.yml'), 62 | flamegraph: false 63 | } 64 | }) 65 | 66 | if (!hasFile(args.config)) { 67 | help.toStdout() 68 | return 69 | } 70 | 71 | // should never throw, we have just 72 | // checked if we can access this 73 | const data = fs.readFileSync(args.config, 'utf8') 74 | 75 | try { 76 | var config = YAML.parse(data) 77 | } catch (err) { 78 | console.error(err) 79 | process.exit(1) 80 | } 81 | 82 | var exec = nodePath 83 | 84 | if (args.flamegraph) { 85 | if (isWin) { 86 | console.error('flamegraphs are supported only on Linux and Mac OS X') 87 | process.exit(1) 88 | } 89 | 90 | exec = zeroX 91 | config.server = '--svg ' + config.server 92 | } 93 | 94 | const wd = path.dirname(path.resolve(args.config)) 95 | const backing = getBacking(config, wd) 96 | var storage 97 | var wire = function () {} 98 | var job = args.job 99 | 100 | if (backing) { 101 | storage = Storage(backing) 102 | wire = storage.wire 103 | storage.on('meta', function (meta) { 104 | const last = meta.runs[0] 105 | const prev = meta.runs[1] 106 | if (last && prev) { 107 | console.log(`==> Comparing ${last.id} against ${prev.id}`) 108 | console.log() 109 | printComparisonTable(compare(last.results, prev.results)) 110 | } 111 | }) 112 | } 113 | 114 | if (!job && storage) { 115 | storage.nextJobId(function (err, id) { 116 | if (err) { 117 | throw err 118 | } 119 | 120 | job = id 121 | 122 | wire(run(config, job, wd, exec, args.flamegraph)) 123 | }) 124 | return 125 | } 126 | 127 | wire(run(config, job || 1, wd, exec)) 128 | } 129 | 130 | function run (config, job, wd, exec, flamegraph) { 131 | console.log(chalk.yellow(`==> Running job ${job}`)) 132 | console.log() 133 | 134 | const runner = new Runner(config, job, wd, exec) 135 | 136 | // pass this over to the storage engine 137 | // we need this to generate the report afterwards 138 | runner.flamegraph = flamegraph 139 | 140 | runner.on('server', function (data) { 141 | console.log(chalk.green(`==> Started server`)) 142 | console.log(chalk.green(`url: ${data.url}`)) 143 | console.log(chalk.green(`cmd: ${data.cmd.join(' ')}`)) 144 | console.log(chalk.green(`exe: ${data.exe}`)) 145 | console.log() 146 | }) 147 | 148 | runner.on('warmup', function (url) { 149 | console.log(chalk.red(`==> warming up ${url} for 3s`)) 150 | console.log() 151 | }) 152 | 153 | runner.on('bench', function (data, cannon) { 154 | process.stdout.write('==> ') 155 | autocannon.track(cannon) 156 | cannon.on('done', function () { 157 | console.log() 158 | }) 159 | }) 160 | 161 | runner.on('error', function (err) { 162 | console.error(err.message) 163 | process.exit(1) 164 | }) 165 | 166 | return runner 167 | } 168 | 169 | function printComparisonTable (results, a, b) { 170 | const keys = Object.keys(results) 171 | const columns = Object.keys(results) 172 | 173 | const areEqual = keys.reduce(function (acc, k) { 174 | return acc && results[k].equal 175 | }, true) 176 | 177 | if (areEqual) { 178 | console.log(`The two jobs throughput is statistically ${chalk.bold('equal')}`) 179 | } else { 180 | console.log(`The two jobs throughput is ${chalk.bold('not')} statistically ${chalk.bold('equal')}`) 181 | } 182 | 183 | console.log('') 184 | 185 | columns.unshift('Stat') 186 | 187 | const out = table.table([ 188 | columns.map(k => chalk.cyan(k)), 189 | row('req/s', results, keys, 'requests', chalk.green, chalk.red), 190 | row('throughput', results, keys, 'throughput', chalk.green, chalk.red), 191 | row('latency', results, keys, 'latency', chalk.red, chalk.green) 192 | ], { 193 | border: table.getBorderCharacters('void'), 194 | columnDefault: { 195 | paddingLeft: 0, 196 | paddingRight: 1 197 | }, 198 | drawHorizontalLine: () => false 199 | }) 200 | 201 | console.log(out) 202 | } 203 | 204 | function row (title, results, keys, prop, positive, negative) { 205 | const res = keys.map(function (k) { 206 | const base = results[k][prop] 207 | const diff = parseFloat(base.difference) 208 | const aWins = !base.valid && diff > 5 209 | const bWins = !base.valid && diff < -5 210 | 211 | var color = noColor 212 | if (aWins) { 213 | color = positive 214 | } else if (bWins) { 215 | color = negative 216 | } 217 | 218 | return color(base.difference + ' ' + base.significant) 219 | }) 220 | 221 | res.unshift(chalk.bold(title)) 222 | 223 | return res 224 | } 225 | 226 | function compareCmd (argv) { 227 | const args = minimist(argv, { 228 | alias: { 229 | config: 'c' 230 | }, 231 | default: { 232 | config: path.resolve('autocannon.yml') 233 | } 234 | }) 235 | 236 | if (!hasFile(args.config)) { 237 | help.toStdout('compare') 238 | return 239 | } 240 | 241 | // should never throw, we have just 242 | // checked if we can access this 243 | const data = fs.readFileSync(args.config, 'utf8') 244 | 245 | try { 246 | var config = YAML.parse(data) 247 | } catch (err) { 248 | console.error(err) 249 | process.exit(1) 250 | } 251 | 252 | const wd = path.dirname(path.resolve(args.config)) 253 | const backing = getBacking(config, wd) 254 | 255 | if (!backing) { 256 | console.error('impossible to compare if there is no storage') 257 | process.exit(1) 258 | } 259 | 260 | const storage = Storage(backing) 261 | 262 | storage.fetchMeta(function (err, meta) { 263 | if (err) { 264 | throw err 265 | } 266 | 267 | const last = find(args._[0]) || meta.runs[0] 268 | const prev = find(args._[1]) || meta.runs[1] 269 | 270 | if (last && prev) { 271 | console.log(`Comparing ${last.id} against ${prev.id}`) 272 | console.log() 273 | printComparisonTable(compare(last.results, prev.results)) 274 | } else { 275 | console.log('No runs available to compare') 276 | } 277 | 278 | function find (num) { 279 | num = parseInt(num) 280 | return meta.runs.filter(r => r.id === num)[0] 281 | } 282 | }) 283 | } 284 | 285 | function noColor (a) { 286 | return a 287 | } 288 | 289 | function cleanup (err) { 290 | if (err) { 291 | console.error(err) 292 | } 293 | 294 | process.removeListener('uncaughtException', cleanup) 295 | process.removeListener('beforeExit', cleanup) 296 | 297 | // cleanup all the children processes 298 | psTree(process.pid, function (err2, children) { 299 | if (err2) { 300 | throw err2 301 | } 302 | 303 | children 304 | .map((p) => p.PID) 305 | .filter((p) => p !== process.pid) 306 | .forEach((p) => { 307 | try { 308 | process.kill(p, 'SIGKILL') 309 | } catch (err) { 310 | debug(err) 311 | } 312 | }) 313 | 314 | if (err) { 315 | process.emit('uncaughtException', err) 316 | } 317 | }) 318 | } 319 | -------------------------------------------------------------------------------- /example/autocannon-s3.yml: -------------------------------------------------------------------------------- 1 | server: ./server.js 2 | benchmarks: 3 | root: 4 | connections: 100 5 | duration: 5 6 | url: http://localhost:3000 7 | b: 8 | connections: 100 9 | duration: 5 10 | url: http://localhost:3000/b 11 | storage: 12 | type: s3 13 | bucket: autocannon-ci-test 14 | region: us-west-2 15 | -------------------------------------------------------------------------------- /example/autocannon.yml: -------------------------------------------------------------------------------- 1 | server: ./server.js 2 | benchmarks: 3 | root: 4 | connections: 100 5 | duration: 5 6 | url: http://localhost:3000 7 | b: 8 | connections: 100 9 | duration: 5 10 | url: http://localhost:3000/b 11 | storage: 12 | type: fs 13 | path: perf-results 14 | -------------------------------------------------------------------------------- /example/server.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const http = require('http') 4 | const server = http.createServer(handle) 5 | 6 | function handle (req, res) { 7 | if (req.url === '/b') { 8 | setTimeout(loop, 25, res) 9 | } else { 10 | setTimeout(loop, 5, res) 11 | } 12 | } 13 | 14 | function loop (res) { 15 | for (var i = 0; i < Math.pow(2, 8); i++) { 16 | // hurray! 17 | } 18 | res.end('loop finished!!!') 19 | } 20 | 21 | server.listen(3000) 22 | -------------------------------------------------------------------------------- /help/autocannon-ci.txt: -------------------------------------------------------------------------------- 1 | Usage: autocannon-ci [command] 2 | 3 | autocannon-ci is a tool to run multiple HTTP/1.1 benchmarks, and generate the relative 4 | flamegraphs, with the help of 0x. 5 | 6 | Available commands: 7 | 8 | * run (default) 9 | * compare 10 | * help 11 | 12 | Launch 'autocannon-ci help [command]' to know more about the commands. 13 | -------------------------------------------------------------------------------- /help/compare.txt: -------------------------------------------------------------------------------- 1 | Usage: autocannon-ci compare [OPTS] [A] [B] 2 | 3 | Compare the job with id A against the job id B. A and B are defaulted to the 4 | latest two jobs. 5 | 6 | Options: 7 | 8 | --config/-c CONFIG Use the given config file; default: `autocannon.yml`. 9 | -------------------------------------------------------------------------------- /help/run.txt: -------------------------------------------------------------------------------- 1 | Usage: autocannon-ci run [OPTS] 2 | 3 | Runs the benchmarks configured in the autocannon-ci configuration file, and 4 | save them according to the storage configured in the config file. The job id 5 | is used to identify the single run. 6 | 7 | Options: 8 | 9 | --config/-c CONFIG Use the given config file; default: `autocannon.yml`. 10 | --job/-j ID Use the specific job id. 11 | -------------------------------------------------------------------------------- /lib/compare.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const single = require('autocannon-compare') 4 | 5 | function compare (a, b) { 6 | const res = {} 7 | 8 | a.forEach(function (singleA) { 9 | b.forEach(function (singleB) { 10 | if (singleA.title === singleB.title) { 11 | res[singleA.title] = single(singleA, singleB) 12 | } 13 | }) 14 | }) 15 | 16 | return res 17 | } 18 | 19 | module.exports = compare 20 | -------------------------------------------------------------------------------- /lib/dashboard.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const html = require('bel') 4 | const template = require('./template') 5 | const compare = require('./compare') 6 | 7 | function render (data) { 8 | return template('Dashboard', body.bind(null, data)) 9 | } 10 | 11 | function body (data) { 12 | const columns = computeColumns(data) 13 | 14 | return html` 15 |
16 | 17 | 18 | 19 | ${columns.map(function (col) { 20 | return html` 21 |
22 |
23 | 24 | 25 | ` 26 | })} 27 | 28 | ${data.runs.map(function (run, i) { 29 | const prev = data.runs[i + 1] 30 | const compRes = prev && compare(run.results, prev.results) 31 | 32 | return html` 33 | 34 | 37 | ${columns.map(function (col) { 38 | const result = getResult(run.results, col) 39 | if (result) { 40 | return html` 41 |
42 |
43 | 44 | 45 | ` 46 | } 47 | })} 48 | 49 | ` 50 | })} 51 |
Job${col + ' req/s'}${col + ' diff'}
35 | ${run.id} 36 | ${result.requests.mean + ' \u00B1 ' + result.requests.stddev}${compRes && compRes[col] && compRes[col].requests.difference}
52 |
53 | ` 54 | } 55 | 56 | function asColor (compRes) { 57 | if (!compRes) { 58 | return '' 59 | } 60 | 61 | if (compRes.aWins) { 62 | return 'bg-green white' 63 | } else if (compRes.bWins) { 64 | return 'bg-red white' 65 | } 66 | 67 | return '' 68 | } 69 | 70 | function computeColumns (data) { 71 | const titles = new Set() 72 | 73 | data.runs.forEach(function (run) { 74 | run.results.forEach(function (result) { 75 | titles.add(result.title) 76 | }) 77 | }) 78 | 79 | return Array.from(titles) 80 | } 81 | 82 | function getResult (results, col) { 83 | return results.filter(result => result.title === col)[0] 84 | } 85 | 86 | module.exports = render 87 | 88 | if (require.main === module) { 89 | test() 90 | } 91 | 92 | function test () { 93 | console.log(render(require('../test/meta.json'))) 94 | } 95 | -------------------------------------------------------------------------------- /lib/get-backing.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const path = require('path') 4 | const bloomrun = require('bloomrun')() 5 | const fsBlobStorage = require('fs-blob-store') 6 | const s3BlobStorage = require('s3-blob-store') 7 | const aws = require('aws-sdk') 8 | 9 | bloomrun.add({ 10 | type: 'fs' 11 | }, function (storage, wd) { 12 | const dest = path.resolve(path.join(wd, storage.path || 'perf-results')) 13 | return fsBlobStorage(dest) 14 | }) 15 | 16 | bloomrun.add({ 17 | type: 's3', 18 | S3_ACCESS_KEY: /.+/, 19 | S3_SECRET_KEY: /.+/, 20 | bucket: /.+/, 21 | region: /.+/ 22 | }, function (storage) { 23 | const client = new aws.S3({ 24 | accessKeyId: process.env.S3_ACCESS_KEY, 25 | secretAccessKey: process.env.S3_SECRET_KEY, 26 | region: storage.region 27 | }) 28 | 29 | const store = s3BlobStorage({ 30 | client, 31 | bucket: storage.bucket 32 | }) 33 | 34 | return store 35 | }) 36 | 37 | bloomrun.add({ 38 | type: 's3', 39 | bucket: /.+/, 40 | region: /.+/ 41 | }, function (storage) { 42 | const client = new aws.S3({ 43 | region: storage.region 44 | }) 45 | 46 | const store = s3BlobStorage({ 47 | client, 48 | bucket: storage.bucket 49 | }) 50 | 51 | return store 52 | }) 53 | 54 | function getBacking (config, wd) { 55 | const toLookup = Object.assign({}, process.env, config.storage) 56 | const factory = bloomrun.lookup(toLookup) 57 | 58 | if (factory) { 59 | return factory(toLookup, wd) 60 | } 61 | 62 | return null 63 | } 64 | 65 | module.exports = getBacking 66 | -------------------------------------------------------------------------------- /lib/run-page.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const html = require('bel') 4 | const template = require('./template') 5 | const prettyBytes = require('pretty-bytes') 6 | 7 | function render (runner, data) { 8 | return template(`Job ${runner.jobId}`, body.bind(null, runner, data)) 9 | } 10 | 11 | function body (runner, data) { 12 | return html` 13 |
14 | ${data.map(bench, runner)} 15 |
16 | ` 17 | } 18 | 19 | function bench (data) { 20 | var flamegraph = '' 21 | 22 | if (this.flamegraph) { 23 | flamegraph = html` 24 | 25 | 26 | 27 | ` 28 | } 29 | 30 | return html` 31 |
32 |

${data.title}

33 | 34 | ${row('URL', data.url)} 35 | ${row('req/s', data.requests.mean + ' \u00B1 ' + data.requests.stddev)} 36 | ${row('latency', data.latency.mean + ' ms \u00B1 ' + data.latency.stddev)} 37 | ${row('throughput', 38 | prettyBytes(data.throughput.mean) + 39 | '/s \u00B1 ' + prettyBytes(data.throughput.stddev) + '/s')} 40 | ${row('duration', data.duration + ' s')} 41 | ${row('connections', data.connections)} 42 | ${row('pipelining', data.pipelining)} 43 | ${data.errors ? row('errors', data.errors) : ''} 44 |
45 | ${flamegraph} 46 |
47 | ` 48 | } 49 | 50 | function row (key, value) { 51 | return html` 52 | 53 | ${key} 54 | ${value} 55 | 56 | ` 57 | } 58 | 59 | module.exports = render 60 | 61 | if (require.main === module) { 62 | test() 63 | } 64 | 65 | function test () { 66 | console.log(render({ 67 | jobId: 42 68 | }, require('../test/result.json'))) 69 | } 70 | -------------------------------------------------------------------------------- /lib/runner.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('debug')('autocannon-ci:runner') 4 | const spawn = require('child_process').spawn 5 | const EE = require('events') 6 | const inherits = require('util').inherits 7 | const fastq = require('fastq') 8 | const parse = require('shell-quote').parse 9 | const request = require('request') 10 | const autocannon = require('autocannon') 11 | const psTree = require('ps-tree') 12 | const glob = require('glob') 13 | const path = require('path') 14 | 15 | function Runner (opts, jobId, cwd, nodePath) { 16 | if (!(this instanceof Runner)) { 17 | return new Runner(opts, jobId, cwd, nodePath) 18 | } 19 | 20 | EE.call(this) 21 | 22 | this.opts = opts 23 | this.jobId = jobId 24 | this.cwd = cwd 25 | this.nodePath = nodePath 26 | 27 | this._results = [] 28 | 29 | this.queue = fastq(this, work, 1) 30 | this.queue.pause() 31 | this.queue.drain = this.emit.bind(this, 'done', this._results) 32 | 33 | this._run() 34 | 35 | process.nextTick(this.queue.resume.bind(this.queue)) 36 | } 37 | 38 | inherits(Runner, EE) 39 | 40 | Runner.prototype._run = function () { 41 | if (typeof this.opts.benchmarks !== 'object') { 42 | this.emit('error', new Error('the benchmarks key in the config must be an object')) 43 | return 44 | } 45 | 46 | const benchmarks = Object.keys(this.opts.benchmarks) 47 | 48 | if (benchmarks.length === 0) { 49 | this.emit('error', new Error('no benchmarks required')) 50 | return 51 | } 52 | 53 | benchmarks.forEach((key) => { 54 | this.queue.push({ 55 | name: key, 56 | options: this.opts.benchmarks[key] 57 | }) 58 | }) 59 | } 60 | 61 | function work (data, cb) { 62 | const cmd = parse(this.opts.server) 63 | const server = spawn(this.nodePath, cmd, { 64 | cwd: this.cwd 65 | }) 66 | const toEmit = { 67 | cmd, 68 | exe: this.nodePath, 69 | url: data.options.url 70 | } 71 | 72 | var closed = false 73 | 74 | const cleanup = (err) => { 75 | if (closed) { 76 | return 77 | } 78 | 79 | var pids 80 | 81 | debug('cleanup', err) 82 | closed = true 83 | server.removeListener('close', cleanup) 84 | server.on('close', () => { 85 | debug('server closed gracefully') 86 | 87 | if (!pids || err) { 88 | setImmediate(cb, err) 89 | return 90 | } 91 | 92 | var p = path.join(this.cwd, 'profile-*') 93 | 94 | glob(p, (err, files) => { 95 | if (err) { 96 | debug(err) 97 | cb() 98 | 99 | // do nothing, there is no profile-* stuff in there 100 | // 0x did not work properly 101 | return 102 | } 103 | 104 | const match = files.filter(path => { 105 | return pids.some(pid => path.indexOf(pid) >= 0) 106 | })[0] 107 | 108 | if (match) { 109 | this.emit('flamegraph', data, match) 110 | } 111 | 112 | cb() 113 | }) 114 | }) 115 | 116 | if (cannon) { 117 | cannon.stop() 118 | } 119 | 120 | psTree(server.pid, function (err, procData) { 121 | if (err) { 122 | debug('psTree errored', err) 123 | } 124 | 125 | debug('server has children', pids) 126 | 127 | pids = procData.map(proc => proc.PID) 128 | 129 | server.kill('SIGINT') 130 | }) 131 | } 132 | 133 | server.on('close', function () { 134 | debug('cleanup from early close') 135 | cleanup() 136 | }) 137 | 138 | var cannon = null 139 | 140 | waitReady(data.options.url, (err) => { 141 | if (err) { 142 | cleanup(err) 143 | return 144 | } 145 | 146 | if (closed) { 147 | return 148 | } 149 | 150 | toEmit.server = server 151 | 152 | this.emit('server', toEmit) 153 | 154 | // warmup 155 | const warmupOpts = Object.assign({}, data.options, { duration: 3 }) 156 | 157 | this.emit('warmup', data.options.url) 158 | 159 | cannon = autocannon(warmupOpts, (err, result) => { 160 | cannon = null 161 | 162 | if (err) { 163 | cleanup(err) 164 | return 165 | } 166 | 167 | if (closed) { 168 | return 169 | } 170 | 171 | const opts = Object.assign({ title: data.name }, data.options) 172 | 173 | cannon = autocannon(opts, (err, result) => { 174 | cannon = null 175 | 176 | if (result) { 177 | this._results.push(result) 178 | } 179 | 180 | cleanup(err) 181 | }) 182 | 183 | this.emit('bench', data, cannon) 184 | }) 185 | }) 186 | } 187 | 188 | function waitReady (url, cb, tries) { 189 | debug('wait', tries) 190 | tries = tries || 0 191 | 192 | request(url, function (err) { 193 | tries++ 194 | 195 | if (err && tries === 10) { 196 | cb(err) 197 | return 198 | } else if (err) { 199 | setTimeout(waitReady, 1000, url, cb, tries) 200 | return 201 | } 202 | 203 | debug('server started') 204 | 205 | cb() 206 | }) 207 | } 208 | 209 | module.exports = Runner 210 | -------------------------------------------------------------------------------- /lib/storage.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('debug')('autocannon-ci:storage') 4 | const path = require('path') 5 | const concat = require('concat-stream') 6 | const rimraf = require('rimraf') 7 | const steed = require('steed') 8 | const glob = require('glob') 9 | const pump = require('pump') 10 | const fs = require('fs') 11 | const EE = require('events') 12 | const dashboard = require('./dashboard') 13 | const runPage = require('./run-page') 14 | 15 | function Storage (backing, runner) { 16 | const meta = 'meta.json' 17 | 18 | const res = new EE() 19 | res.nextJobId = nextJobId 20 | res.fetchMeta = fetchMeta 21 | res.wire = wire 22 | 23 | return res 24 | 25 | function wire (runner) { 26 | runner.on('bench', onBench) 27 | runner.on('done', runnerDone) 28 | runner.on('flamegraph', copyFlamegraph) 29 | } 30 | 31 | function nextJobId (cb) { 32 | fetchMeta(function (err, content) { 33 | if (err) { 34 | return cb(err) 35 | } 36 | 37 | cb(null, content.nextId || 1) 38 | }) 39 | } 40 | 41 | function fetchMeta (cb) { 42 | backing.createReadStream(meta) 43 | .on('error', function (err) { 44 | if (err) { 45 | debug('nextJobId error', err) 46 | } 47 | 48 | cb(null, 1) 49 | }) 50 | .pipe(concat(function (data) { 51 | try { 52 | const content = JSON.parse(data) 53 | cb(null, content) 54 | } catch (err) { 55 | cb(err) 56 | } 57 | })) 58 | .on('error', cb) 59 | } 60 | 61 | function onBench (data, cannon) { 62 | const base = `run-${this.jobId}` 63 | const metaToWrite = Object.assign({}, data, { server: undefined }) 64 | 65 | backing 66 | .createWriteStream(path.join(base, data.name, 'meta.json')) 67 | .on('error', handleErr) 68 | .end(JSON.stringify(metaToWrite, null, 2)) 69 | 70 | cannon.on('done', function (results) { 71 | backing 72 | .createWriteStream(path.join(base, data.name, 'results.json')) 73 | .on('error', handleErr) 74 | .end(JSON.stringify(results, null, 2)) 75 | }) 76 | } 77 | 78 | function runnerDone (results) { 79 | const id = this.jobId 80 | const base = `run-${this.jobId}` 81 | 82 | backing 83 | .createWriteStream(path.join(base, 'index.html')) 84 | .on('error', handleErr) 85 | .end(runPage(this, results)) 86 | 87 | backing.exists(meta, (err, exists) => { 88 | if (err) { 89 | this.emit('error', err) 90 | return 91 | } 92 | 93 | if (exists) { 94 | backing.createReadStream(meta) 95 | .pipe(concat(function (data) { 96 | const content = JSON.parse(data) 97 | content.runs.unshift({ id, path: base, results }) 98 | content.nextId = id + 1 99 | writeMeta(content) 100 | })) 101 | } else { 102 | writeMeta({ 103 | nextId: id + 1, 104 | runs: [{ id, path: base, results }] 105 | }) 106 | } 107 | }) 108 | } 109 | 110 | function writeMeta (content) { 111 | res.emit('meta', content) 112 | 113 | backing 114 | .createWriteStream(meta) 115 | .end(JSON.stringify(content, null, 2)) 116 | 117 | backing 118 | .createWriteStream('index.html') 119 | .end(dashboard(content)) 120 | } 121 | 122 | function copyFlamegraph (data, p) { 123 | const base = `run-${this.jobId}` 124 | debug('copyFlamegraph', data, p) 125 | glob(path.join(p, '*'), function (err, files) { 126 | handleErr(err) 127 | 128 | steed.each(files, (file, cb) => { 129 | const dest = path.join(base, data.name, path.basename(file)) 130 | debug('copying', file, dest) 131 | pump( 132 | fs.createReadStream(file), 133 | backing.createWriteStream(dest), 134 | function (err) { 135 | handleErr(err) 136 | cb() 137 | }) 138 | }, () => { 139 | debug('copy finished') 140 | rimraf(p, function () { 141 | debug 142 | }) 143 | }) 144 | }) 145 | } 146 | 147 | function handleErr (err) { 148 | if (err) { 149 | // if this happen we are seriously broken 150 | // better fold 151 | throw err 152 | } 153 | } 154 | } 155 | 156 | module.exports = Storage 157 | 158 | -------------------------------------------------------------------------------- /lib/template.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const html = require('bel') 4 | const doc = '' 5 | const pack = require('../package') 6 | 7 | function template (title, styles, func) { 8 | if (typeof styles === 'function') { 9 | func = styles 10 | styles = 'bg-white dark-gray' 11 | } 12 | return doc + html` 13 | 14 | ${title} 15 | 16 | 17 | 18 | 19 |
20 |

${title}

21 | 22 | 23 | 24 |
25 | ${func()} 26 | 27 | 28 | ` 29 | } 30 | 31 | module.exports = template 32 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "autocannon-ci", 3 | "version": "0.1.0", 4 | "description": "run your benchmarks as part of your dev flow, for Node.js", 5 | "main": "autocannon-ci.js", 6 | "bin": { 7 | "autocannon-ci": "./autocannon-ci.js" 8 | }, 9 | "scripts": { 10 | "test": "standard | snazzy && tap --cov test/*.test.js" 11 | }, 12 | "precommit": "test", 13 | "repository": { 14 | "type": "git", 15 | "url": "git+https://github.com/mcollina/autocannon-ci.git" 16 | }, 17 | "keywords": [ 18 | "perf", 19 | "ci", 20 | "flamegraph", 21 | "performance", 22 | "testing" 23 | ], 24 | "author": "Matteo Collina ", 25 | "license": "MIT", 26 | "bugs": { 27 | "url": "https://github.com/mcollina/autocannon-ci/issues" 28 | }, 29 | "homepage": "https://github.com/mcollina/autocannon-ci#readme", 30 | "devDependencies": { 31 | "pre-commit": "^1.2.2", 32 | "rimraf": "^2.5.4", 33 | "snazzy": "^6.0.0", 34 | "standard": "^8.6.0", 35 | "tap": "^9.0.3" 36 | }, 37 | "dependencies": { 38 | "0x": "^2.3.1", 39 | "abstract-blob-store": "^3.2.0", 40 | "autocannon": "^0.16.0", 41 | "autocannon-compare": "^0.2.0", 42 | "aws-sdk": "^2.12.0", 43 | "bel": "^4.5.1", 44 | "bloomrun": "^3.0.3", 45 | "chalk": "^1.1.3", 46 | "commist": "^1.0.0", 47 | "concat-stream": "^1.6.0", 48 | "debug": "^2.6.0", 49 | "fastq": "^1.5.0", 50 | "fs-blob-store": "^5.2.1", 51 | "glob": "^7.1.1", 52 | "help-me": "^1.0.1", 53 | "minimist": "^1.2.0", 54 | "pretty-bytes": "^4.0.2", 55 | "ps-tree": "^1.1.0", 56 | "pump": "^1.0.2", 57 | "request": "^2.79.0", 58 | "rimraf": "^2.5.4", 59 | "s3-blob-store": "^1.2.3", 60 | "shell-quote": "^1.6.1", 61 | "steed": "^1.1.3", 62 | "table": "^4.0.1", 63 | "yamljs": "^0.2.8" 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /test/compare.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const t = require('tap') 4 | const test = t.test 5 | const compare = require('../lib/compare') 6 | 7 | const results = require('./result') 8 | const results2 = require('./result-2') 9 | 10 | test('compare with itself should have no winner', function (t) { 11 | const res = compare(results, results2) 12 | 13 | t.match(res, { 14 | root: { 15 | 'requests': { 16 | 'difference': '89.27%', 17 | 'significant': '***' 18 | }, 19 | 'throughput': { 20 | 'difference': '83.76%', 21 | 'significant': '***' 22 | }, 23 | 'latency': { 24 | 'difference': '-50.06%', 25 | 'significant': '***' 26 | }, 27 | 'aWins': true, 28 | 'bWins': false, 29 | 'equal': false 30 | }, 31 | b: { 32 | 'requests': { 33 | 'difference': '-9.74%', 34 | 'significant': '***' 35 | }, 36 | 'throughput': { 37 | 'difference': '-10.14%', 38 | 'significant': '**' 39 | }, 40 | 'latency': { 41 | 'difference': '10.89%', 42 | 'significant': '***' 43 | }, 44 | 'aWins': false, 45 | 'bWins': true, 46 | 'equal': false 47 | } 48 | }) 49 | t.end() 50 | }) 51 | -------------------------------------------------------------------------------- /test/meta.json: -------------------------------------------------------------------------------- 1 | { 2 | "nextId": 7, 3 | "runs": [ 4 | { 5 | "id": 6, 6 | "path": "run-6", 7 | "results": [ 8 | { 9 | "title": "root", 10 | "url": "http://localhost:3000", 11 | "requests": { 12 | "average": 18648, 13 | "mean": 18648, 14 | "stddev": 700.58, 15 | "min": 17568, 16 | "max": 19727, 17 | "total": 93237, 18 | "sent": 93337 19 | }, 20 | "latency": { 21 | "average": 4.88, 22 | "mean": 4.88, 23 | "stddev": 1.99, 24 | "min": 0, 25 | "max": 58, 26 | "p50": 5, 27 | "p75": 5, 28 | "p90": 7, 29 | "p99": 11, 30 | "p999": 31, 31 | "p9999": 52, 32 | "p99999": 58 33 | }, 34 | "throughput": { 35 | "average": 2057830.4, 36 | "mean": 2057830.4, 37 | "stddev": 72977.81, 38 | "min": 1900544, 39 | "max": 2228223, 40 | "total": 10349307 41 | }, 42 | "errors": 0, 43 | "timeouts": 0, 44 | "duration": 5, 45 | "start": "2017-02-15T18:47:21.320Z", 46 | "finish": "2017-02-15T18:47:26.366Z", 47 | "connections": 100, 48 | "pipelining": 1, 49 | "non2xx": 0, 50 | "1xx": 0, 51 | "2xx": 93237, 52 | "3xx": 0, 53 | "4xx": 0, 54 | "5xx": 0 55 | }, 56 | { 57 | "title": "b", 58 | "url": "http://localhost:3000/b", 59 | "requests": { 60 | "average": 2641, 61 | "mean": 2641, 62 | "stddev": 48.99, 63 | "min": 2600, 64 | "max": 2701, 65 | "total": 13200, 66 | "sent": 13300 67 | }, 68 | "latency": { 69 | "average": 37.4, 70 | "mean": 37.4, 71 | "stddev": 2.41, 72 | "min": 31, 73 | "max": 68, 74 | "p50": 37, 75 | "p75": 38, 76 | "p90": 39, 77 | "p99": 42, 78 | "p999": 66, 79 | "p9999": 68, 80 | "p99999": 68 81 | }, 82 | "throughput": { 83 | "average": 309657.6, 84 | "mean": 309657.6, 85 | "stddev": 8026.49, 86 | "min": 294912, 87 | "max": 327679, 88 | "total": 1531200 89 | }, 90 | "errors": 0, 91 | "timeouts": 0, 92 | "duration": 5, 93 | "start": "2017-02-15T18:47:37.806Z", 94 | "finish": "2017-02-15T18:47:42.853Z", 95 | "connections": 100, 96 | "pipelining": 1, 97 | "non2xx": 0, 98 | "1xx": 0, 99 | "2xx": 13200, 100 | "3xx": 0, 101 | "4xx": 0, 102 | "5xx": 0 103 | } 104 | ] 105 | }, 106 | { 107 | "id": 5, 108 | "path": "run-5", 109 | "results": [ 110 | { 111 | "title": "root", 112 | "url": "http://localhost:3000", 113 | "requests": { 114 | "average": 16974.41, 115 | "mean": 16974.41, 116 | "stddev": 1878.67, 117 | "min": 14488, 118 | "max": 19391, 119 | "total": 84851, 120 | "sent": 84951 121 | }, 122 | "latency": { 123 | "average": 5.39, 124 | "mean": 5.39, 125 | "stddev": 2.37, 126 | "min": 0, 127 | "max": 73, 128 | "p50": 5, 129 | "p75": 6, 130 | "p90": 7, 131 | "p99": 11, 132 | "p999": 37, 133 | "p9999": 68, 134 | "p99999": 73 135 | }, 136 | "throughput": { 137 | "average": 1887436.8, 138 | "mean": 1887436.8, 139 | "stddev": 216962.77, 140 | "min": 1572864, 141 | "max": 2228223, 142 | "total": 9418461 143 | }, 144 | "errors": 0, 145 | "timeouts": 0, 146 | "duration": 5, 147 | "start": "2017-02-15T18:45:39.608Z", 148 | "finish": "2017-02-15T18:45:44.664Z", 149 | "connections": 100, 150 | "pipelining": 1, 151 | "non2xx": 0, 152 | "1xx": 0, 153 | "2xx": 84851, 154 | "3xx": 0, 155 | "4xx": 0, 156 | "5xx": 0 157 | }, 158 | { 159 | "title": "b", 160 | "url": "http://localhost:3000/b", 161 | "requests": { 162 | "average": 2684.2, 163 | "mean": 2684.2, 164 | "stddev": 42.06, 165 | "min": 2600, 166 | "max": 2717, 167 | "total": 13416, 168 | "sent": 13516 169 | }, 170 | "latency": { 171 | "average": 36.85, 172 | "mean": 36.85, 173 | "stddev": 4.37, 174 | "min": 29, 175 | "max": 100, 176 | "p50": 37, 177 | "p75": 38, 178 | "p90": 38, 179 | "p99": 41, 180 | "p999": 97, 181 | "p9999": 99, 182 | "p99999": 100 183 | }, 184 | "throughput": { 185 | "average": 316211.2, 186 | "mean": 316211.2, 187 | "stddev": 6553.6, 188 | "min": 294912, 189 | "max": 327679, 190 | "total": 1556256 191 | }, 192 | "errors": 0, 193 | "timeouts": 0, 194 | "duration": 5, 195 | "start": "2017-02-15T18:45:57.011Z", 196 | "finish": "2017-02-15T18:46:02.069Z", 197 | "connections": 100, 198 | "pipelining": 1, 199 | "non2xx": 0, 200 | "1xx": 0, 201 | "2xx": 13416, 202 | "3xx": 0, 203 | "4xx": 0, 204 | "5xx": 0 205 | } 206 | ] 207 | }, 208 | { 209 | "id": 4, 210 | "path": "run-4", 211 | "results": [ 212 | { 213 | "title": "root", 214 | "url": "http://localhost:3000", 215 | "requests": { 216 | "average": 15658.4, 217 | "mean": 15658.4, 218 | "stddev": 2219.84, 219 | "min": 12880, 220 | "max": 18527, 221 | "total": 78297, 222 | "sent": 78397 223 | }, 224 | "latency": { 225 | "average": 5.9, 226 | "mean": 5.9, 227 | "stddev": 3.94, 228 | "min": 0, 229 | "max": 118, 230 | "p50": 5, 231 | "p75": 7, 232 | "p90": 8, 233 | "p99": 15, 234 | "p999": 65, 235 | "p9999": 111, 236 | "p99999": 118 237 | }, 238 | "throughput": { 239 | "average": 1723596.8, 240 | "mean": 1723596.8, 241 | "stddev": 253481.19, 242 | "min": 1376256, 243 | "max": 2097151, 244 | "total": 8690967 245 | }, 246 | "errors": 0, 247 | "timeouts": 0, 248 | "duration": 5, 249 | "start": "2017-02-15T18:42:58.283Z", 250 | "finish": "2017-02-15T18:43:03.332Z", 251 | "connections": 100, 252 | "pipelining": 1, 253 | "non2xx": 0, 254 | "1xx": 0, 255 | "2xx": 78297, 256 | "3xx": 0, 257 | "4xx": 0, 258 | "5xx": 0 259 | }, 260 | { 261 | "title": "b", 262 | "url": "http://localhost:3000/b", 263 | "requests": { 264 | "average": 2681, 265 | "mean": 2681, 266 | "stddev": 40, 267 | "min": 2600, 268 | "max": 2701, 269 | "total": 13400, 270 | "sent": 13500 271 | }, 272 | "latency": { 273 | "average": 36.96, 274 | "mean": 36.96, 275 | "stddev": 2.38, 276 | "min": 29, 277 | "max": 63, 278 | "p50": 37, 279 | "p75": 38, 280 | "p90": 38, 281 | "p99": 43, 282 | "p999": 61, 283 | "p9999": 63, 284 | "p99999": 63 285 | }, 286 | "throughput": { 287 | "average": 316211.2, 288 | "mean": 316211.2, 289 | "stddev": 6553.6, 290 | "min": 294912, 291 | "max": 327679, 292 | "total": 1554400 293 | }, 294 | "errors": 0, 295 | "timeouts": 0, 296 | "duration": 5, 297 | "start": "2017-02-15T18:43:14.852Z", 298 | "finish": "2017-02-15T18:43:19.896Z", 299 | "connections": 100, 300 | "pipelining": 1, 301 | "non2xx": 0, 302 | "1xx": 0, 303 | "2xx": 13400, 304 | "3xx": 0, 305 | "4xx": 0, 306 | "5xx": 0 307 | } 308 | ] 309 | }, 310 | { 311 | "id": 3, 312 | "path": "run-3", 313 | "results": [ 314 | { 315 | "title": "root", 316 | "url": "http://localhost:3000", 317 | "requests": { 318 | "average": 17378.41, 319 | "mean": 17378.41, 320 | "stddev": 3024.69, 321 | "min": 11816, 322 | "max": 20879, 323 | "total": 86885, 324 | "sent": 86985 325 | }, 326 | "latency": { 327 | "average": 5.27, 328 | "mean": 5.27, 329 | "stddev": 3, 330 | "min": 0, 331 | "max": 53, 332 | "p50": 5, 333 | "p75": 6, 334 | "p90": 8, 335 | "p99": 15, 336 | "p999": 43, 337 | "p9999": 52, 338 | "p99999": 53 339 | }, 340 | "throughput": { 341 | "average": 1939865.6, 342 | "mean": 1939865.6, 343 | "stddev": 319987.54, 344 | "min": 1310720, 345 | "max": 2359295, 346 | "total": 9644235 347 | }, 348 | "errors": 0, 349 | "timeouts": 0, 350 | "duration": 5, 351 | "start": "2017-02-15T18:40:24.774Z", 352 | "finish": "2017-02-15T18:40:29.822Z", 353 | "connections": 100, 354 | "pipelining": 1, 355 | "non2xx": 0, 356 | "1xx": 0, 357 | "2xx": 86885, 358 | "3xx": 0, 359 | "4xx": 0, 360 | "5xx": 0 361 | }, 362 | { 363 | "title": "b", 364 | "url": "http://localhost:3000/b", 365 | "requests": { 366 | "average": 2681, 367 | "mean": 2681, 368 | "stddev": 40, 369 | "min": 2600, 370 | "max": 2701, 371 | "total": 13400, 372 | "sent": 13500 373 | }, 374 | "latency": { 375 | "average": 36.9, 376 | "mean": 36.9, 377 | "stddev": 2.12, 378 | "min": 30, 379 | "max": 60, 380 | "p50": 37, 381 | "p75": 37, 382 | "p90": 38, 383 | "p99": 41, 384 | "p999": 60, 385 | "p9999": 60, 386 | "p99999": 60 387 | }, 388 | "throughput": { 389 | "average": 316211.2, 390 | "mean": 316211.2, 391 | "stddev": 6553.6, 392 | "min": 294912, 393 | "max": 327679, 394 | "total": 1554400 395 | }, 396 | "errors": 0, 397 | "timeouts": 0, 398 | "duration": 5, 399 | "start": "2017-02-15T18:40:41.098Z", 400 | "finish": "2017-02-15T18:40:46.151Z", 401 | "connections": 100, 402 | "pipelining": 1, 403 | "non2xx": 0, 404 | "1xx": 0, 405 | "2xx": 13400, 406 | "3xx": 0, 407 | "4xx": 0, 408 | "5xx": 0 409 | } 410 | ] 411 | }, 412 | { 413 | "id": 2, 414 | "path": "run-2", 415 | "results": [ 416 | { 417 | "title": "root", 418 | "url": "http://localhost:3000", 419 | "requests": { 420 | "average": 18308, 421 | "mean": 18308, 422 | "stddev": 1196.84, 423 | "min": 16064, 424 | "max": 19407, 425 | "total": 91537, 426 | "sent": 91637 427 | }, 428 | "latency": { 429 | "average": 4.97, 430 | "mean": 4.97, 431 | "stddev": 2.06, 432 | "min": 0, 433 | "max": 53, 434 | "p50": 5, 435 | "p75": 5, 436 | "p90": 6, 437 | "p99": 11, 438 | "p999": 38, 439 | "p9999": 53, 440 | "p99999": 53 441 | }, 442 | "throughput": { 443 | "average": 2038169.6, 444 | "mean": 2038169.6, 445 | "stddev": 133346.04, 446 | "min": 1769472, 447 | "max": 2228223, 448 | "total": 10160607 449 | }, 450 | "errors": 0, 451 | "timeouts": 0, 452 | "duration": 5, 453 | "start": "2017-02-15T18:38:52.337Z", 454 | "finish": "2017-02-15T18:38:57.381Z", 455 | "connections": 100, 456 | "pipelining": 1, 457 | "non2xx": 0, 458 | "1xx": 0, 459 | "2xx": 91537, 460 | "3xx": 0, 461 | "4xx": 0, 462 | "5xx": 0 463 | }, 464 | { 465 | "title": "b", 466 | "url": "http://localhost:3000/b", 467 | "requests": { 468 | "average": 2680.6, 469 | "mean": 2680.6, 470 | "stddev": 31, 471 | "min": 2620, 472 | "max": 2701, 473 | "total": 13400, 474 | "sent": 13500 475 | }, 476 | "latency": { 477 | "average": 36.8, 478 | "mean": 36.8, 479 | "stddev": 2.3, 480 | "min": 29, 481 | "max": 64, 482 | "p50": 37, 483 | "p75": 37, 484 | "p90": 38, 485 | "p99": 40, 486 | "p999": 61, 487 | "p9999": 64, 488 | "p99999": 64 489 | }, 490 | "throughput": { 491 | "average": 312934.41, 492 | "mean": 312934.41, 493 | "stddev": 8026.49, 494 | "min": 294912, 495 | "max": 327679, 496 | "total": 1554400 497 | }, 498 | "errors": 0, 499 | "timeouts": 0, 500 | "duration": 5, 501 | "start": "2017-02-15T18:39:09.003Z", 502 | "finish": "2017-02-15T18:39:14.049Z", 503 | "connections": 100, 504 | "pipelining": 1, 505 | "non2xx": 0, 506 | "1xx": 0, 507 | "2xx": 13400, 508 | "3xx": 0, 509 | "4xx": 0, 510 | "5xx": 0 511 | } 512 | ] 513 | }, 514 | { 515 | "id": 1, 516 | "path": "run-1", 517 | "results": [ 518 | { 519 | "title": "root", 520 | "url": "http://localhost:3000", 521 | "requests": { 522 | "average": 18155.2, 523 | "mean": 18155.2, 524 | "stddev": 1446.4, 525 | "min": 16528, 526 | "max": 20255, 527 | "total": 90781, 528 | "sent": 90881 529 | }, 530 | "latency": { 531 | "average": 5.02, 532 | "mean": 5.02, 533 | "stddev": 2.25, 534 | "min": 0, 535 | "max": 55, 536 | "p50": 5, 537 | "p75": 5, 538 | "p90": 7, 539 | "p99": 12, 540 | "p999": 31, 541 | "p9999": 51, 542 | "p99999": 53 543 | }, 544 | "throughput": { 545 | "average": 2025062.4, 546 | "mean": 2025062.4, 547 | "stddev": 172647.26, 548 | "min": 1835008, 549 | "max": 2359295, 550 | "total": 10076691 551 | }, 552 | "errors": 0, 553 | "timeouts": 0, 554 | "duration": 5, 555 | "start": "2017-02-15T18:38:18.897Z", 556 | "finish": "2017-02-15T18:38:23.938Z", 557 | "connections": 100, 558 | "pipelining": 1, 559 | "non2xx": 0, 560 | "1xx": 0, 561 | "2xx": 90781, 562 | "3xx": 0, 563 | "4xx": 0, 564 | "5xx": 0 565 | }, 566 | { 567 | "title": "b", 568 | "url": "http://localhost:3000/b", 569 | "requests": { 570 | "average": 2661, 571 | "mean": 2661, 572 | "stddev": 48.99, 573 | "min": 2600, 574 | "max": 2701, 575 | "total": 13300, 576 | "sent": 13400 577 | }, 578 | "latency": { 579 | "average": 37.14, 580 | "mean": 37.14, 581 | "stddev": 3.33, 582 | "min": 32, 583 | "max": 86, 584 | "p50": 37, 585 | "p75": 38, 586 | "p90": 38, 587 | "p99": 42, 588 | "p999": 83, 589 | "p9999": 85, 590 | "p99999": 86 591 | }, 592 | "throughput": { 593 | "average": 312934.41, 594 | "mean": 312934.41, 595 | "stddev": 8026.49, 596 | "min": 294912, 597 | "max": 327679, 598 | "total": 1542800 599 | }, 600 | "errors": 0, 601 | "timeouts": 0, 602 | "duration": 5, 603 | "start": "2017-02-15T18:38:35.769Z", 604 | "finish": "2017-02-15T18:38:40.834Z", 605 | "connections": 100, 606 | "pipelining": 1, 607 | "non2xx": 0, 608 | "1xx": 0, 609 | "2xx": 13300, 610 | "3xx": 0, 611 | "4xx": 0, 612 | "5xx": 0 613 | } 614 | ] 615 | } 616 | ] 617 | } -------------------------------------------------------------------------------- /test/result-2.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "title": "root", 4 | "url": "http://localhost:3000", 5 | "requests": { 6 | "average": 11274.4, 7 | "mean": 11274.4, 8 | "stddev": 230.73, 9 | "min": 11048, 10 | "max": 11719, 11 | "total": 56361, 12 | "sent": 56461 13 | }, 14 | "latency": { 15 | "average": 8.41, 16 | "mean": 8.41, 17 | "stddev": 1.79, 18 | "min": 4, 19 | "max": 41, 20 | "p50": 8, 21 | "p75": 9, 22 | "p90": 10, 23 | "p99": 12, 24 | "p999": 31, 25 | "p9999": 41, 26 | "p99999": 41 27 | }, 28 | "throughput": { 29 | "average": 1291059.2, 30 | "mean": 1291059.2, 31 | "stddev": 26214.4, 32 | "min": 1245184, 33 | "max": 1376255, 34 | "total": 6537876 35 | }, 36 | "errors": 0, 37 | "timeouts": 0, 38 | "duration": 5, 39 | "start": "2017-02-17T08:15:37.770Z", 40 | "finish": "2017-02-17T08:15:42.815Z", 41 | "connections": 100, 42 | "pipelining": 1, 43 | "non2xx": 0, 44 | "1xx": 0, 45 | "2xx": 56361, 46 | "3xx": 0, 47 | "4xx": 0, 48 | "5xx": 0 49 | }, 50 | { 51 | "title": "b", 52 | "url": "http://localhost:3000/b", 53 | "requests": { 54 | "average": 3039, 55 | "mean": 3039, 56 | "stddev": 97.54, 57 | "min": 2900, 58 | "max": 3183, 59 | "total": 15190, 60 | "sent": 15290 61 | }, 62 | "latency": { 63 | "average": 32.5, 64 | "mean": 32.5, 65 | "stddev": 2.86, 66 | "min": 26, 67 | "max": 57, 68 | "p50": 32, 69 | "p75": 34, 70 | "p90": 35, 71 | "p99": 38, 72 | "p999": 56, 73 | "p9999": 57, 74 | "p99999": 57 75 | }, 76 | "throughput": { 77 | "average": 355532.8, 78 | "mean": 355532.8, 79 | "stddev": 12260.67, 80 | "min": 327680, 81 | "max": 376831, 82 | "total": 1762040 83 | }, 84 | "errors": 0, 85 | "timeouts": 0, 86 | "duration": 5, 87 | "start": "2017-02-17T08:15:46.990Z", 88 | "finish": "2017-02-17T08:15:52.033Z", 89 | "connections": 100, 90 | "pipelining": 1, 91 | "non2xx": 0, 92 | "1xx": 0, 93 | "2xx": 15190, 94 | "3xx": 0, 95 | "4xx": 0, 96 | "5xx": 0 97 | } 98 | ] 99 | -------------------------------------------------------------------------------- /test/result.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "title": "root", 3 | "url": "http://localhost:3000", 4 | "requests": { 5 | "average": 21339.2, 6 | "mean": 21339.2, 7 | "stddev": 627.68, 8 | "min": 20464, 9 | "max": 22319, 10 | "total": 106679, 11 | "sent": 106779 12 | }, 13 | "latency": { 14 | "average": 4.2, 15 | "mean": 4.2, 16 | "stddev": 1.95, 17 | "min": 0, 18 | "max": 48, 19 | "p50": 4, 20 | "p75": 5, 21 | "p90": 6, 22 | "p99": 10, 23 | "p999": 14, 24 | "p9999": 40, 25 | "p99999": 48 26 | }, 27 | "throughput": { 28 | "average": 2372403.21, 29 | "mean": 2372403.21, 30 | "stddev": 64211.91, 31 | "min": 2228224, 32 | "max": 2490367, 33 | "total": 11841369 34 | }, 35 | "errors": 0, 36 | "timeouts": 0, 37 | "duration": 5, 38 | "start": "2017-02-15T17:37:46.991Z", 39 | "finish": "2017-02-15T17:37:52.028Z", 40 | "connections": 100, 41 | "pipelining": 1, 42 | "non2xx": 0, 43 | "1xx": 0, 44 | "2xx": 106679, 45 | "3xx": 0, 46 | "4xx": 0, 47 | "5xx": 0 48 | }, { 49 | "title": "b", 50 | "url": "http://localhost:3000/b", 51 | "requests": { 52 | "average": 2743, 53 | "mean": 2743, 54 | "stddev": 39.62, 55 | "min": 2700, 56 | "max": 2801, 57 | "total": 13711, 58 | "sent": 13811 59 | }, 60 | "latency": { 61 | "average": 36.04, 62 | "mean": 36.04, 63 | "stddev": 3.08, 64 | "min": 29, 65 | "max": 61, 66 | "p50": 36, 67 | "p75": 38, 68 | "p90": 39, 69 | "p99": 42, 70 | "p999": 60, 71 | "p9999": 61, 72 | "p99999": 61 73 | }, 74 | "throughput": { 75 | "average": 319488, 76 | "mean": 319488, 77 | "stddev": 0, 78 | "min": 311296, 79 | "max": 327679, 80 | "total": 1590476 81 | }, 82 | "errors": 0, 83 | "timeouts": 0, 84 | "duration": 5, 85 | "start": "2017-02-15T17:37:56.188Z", 86 | "finish": "2017-02-15T17:38:01.238Z", 87 | "connections": 100, 88 | "pipelining": 1, 89 | "non2xx": 0, 90 | "1xx": 0, 91 | "2xx": 13711, 92 | "3xx": 0, 93 | "4xx": 0, 94 | "5xx": 0 95 | }] 96 | -------------------------------------------------------------------------------- /test/runner.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const t = require('tap') 4 | const Runner = require('../lib/runner') 5 | 6 | // get the current node path for clean windows support 7 | const isWin = /^win/.test(process.platform) 8 | const nodePath = isWin ? ('"' + process.argv[0] + '"') : process.argv[0] 9 | 10 | t.plan(13) 11 | 12 | const runner = new Runner({ 13 | server: './server.js', 14 | benchmarks: { 15 | something: { 16 | connections: 20, 17 | duration: 2, 18 | url: 'http://localhost:3000' 19 | }, 20 | else: { 21 | connections: 20, 22 | duration: 2, 23 | url: 'http://localhost:3000/b' 24 | } 25 | } 26 | }, 42, __dirname, nodePath) 27 | 28 | t.equal(runner.jobId, 42) 29 | 30 | const urls = [ 31 | 'http://localhost:3000', 32 | 'http://localhost:3000/b' 33 | ] 34 | 35 | const serverData = [{ 36 | cmd: ['./server.js'], 37 | exe: nodePath, 38 | url: 'http://localhost:3000' 39 | }, { 40 | cmd: ['./server.js'], 41 | exe: nodePath, 42 | url: 'http://localhost:3000/b' 43 | }] 44 | 45 | const results = [] 46 | 47 | runner.on('server', function (data) { 48 | const server = data.server 49 | delete data.server 50 | 51 | t.ok(server, 'server exists') 52 | t.deepEqual(data, serverData.shift()) 53 | }) 54 | 55 | runner.on('warmup', function (url) { 56 | t.equal(url, urls.shift(), 'url matches') 57 | }) 58 | 59 | runner.on('bench', function (data, cannon) { 60 | cannon.on('done', function (result) { 61 | t.ok('autocannon finished') 62 | t.ok(result.title, 'result has title') 63 | results.push(result) 64 | }) 65 | }) 66 | 67 | runner.on('done', function (_results) { 68 | t.pass('done emitted') 69 | t.deepEqual(_results, results) 70 | }) 71 | -------------------------------------------------------------------------------- /test/server.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const http = require('http') 4 | const server = http.createServer(handle) 5 | 6 | function handle (req, res) { 7 | if (req.url === '/b') { 8 | setTimeout(loop, 30, res) 9 | } else { 10 | res.end('hello world') 11 | } 12 | } 13 | 14 | function loop (res) { 15 | for (var i = 0; i < Math.pow(2, 8); i++) { 16 | // hurray! 17 | } 18 | res.end('loop finished!!!') 19 | } 20 | 21 | server.listen(3000) 22 | -------------------------------------------------------------------------------- /test/storage.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const t = require('tap') 4 | const Storage = require('../lib/storage') 5 | const abs = require('abstract-blob-store') 6 | const concat = require('concat-stream') 7 | const EE = require('events') 8 | 9 | const backing = abs() 10 | const results = require('./result') 11 | 12 | var storage = Storage(backing) 13 | 14 | t.test('nextJobId with no status', function (t) { 15 | t.plan(2) 16 | 17 | storage.nextJobId(function (err, id) { 18 | t.error(err) 19 | t.equal(id, 1) 20 | }) 21 | }) 22 | 23 | t.test('first run', function (t) { 24 | t.plan(6) 25 | 26 | const runner = new EE() 27 | runner.jobId = 42 28 | 29 | storage.wire(runner) 30 | 31 | var cannon = new EE() 32 | 33 | runner.emit('bench', { 34 | name: 'first', 35 | meta: 'data' 36 | }, cannon) 37 | 38 | cannon.emit('done', { 39 | are: 'results' 40 | }) 41 | 42 | cannon = new EE() 43 | 44 | runner.emit('bench', { 45 | name: 'second', 46 | meta: 'data2' 47 | }, cannon) 48 | 49 | cannon.emit('done', { 50 | are2: 'results' 51 | }) 52 | 53 | storage.once('meta', function (data) { 54 | t.deepEqual(data, { 55 | nextId: 43, 56 | runs: [{ 57 | id: 42, 58 | path: 'run-42', 59 | results 60 | }] 61 | }) 62 | }) 63 | 64 | runner.emit('done', results) 65 | 66 | // this will wait for all the process.nextTick 67 | // used by the dummy store 68 | setImmediate(function () { 69 | backing.createReadStream('meta.json') 70 | .pipe(concat(function (data) { 71 | t.deepEqual(JSON.parse(data), { 72 | nextId: 43, 73 | runs: [{ 74 | id: 42, 75 | path: 'run-42', 76 | results 77 | }] 78 | }) 79 | })) 80 | 81 | backing.createReadStream('run-42/first/meta.json') 82 | .pipe(concat(function (data) { 83 | t.deepEqual(JSON.parse(data), { 84 | name: 'first', 85 | meta: 'data' 86 | }) 87 | })) 88 | 89 | backing.createReadStream('run-42/first/results.json') 90 | .pipe(concat(function (data) { 91 | t.deepEqual(JSON.parse(data), { 92 | are: 'results' 93 | }) 94 | })) 95 | 96 | backing.createReadStream('run-42/second/meta.json') 97 | .pipe(concat(function (data) { 98 | t.deepEqual(JSON.parse(data), { 99 | name: 'second', 100 | meta: 'data2' 101 | }) 102 | })) 103 | 104 | backing.createReadStream('run-42/second/results.json') 105 | .pipe(concat(function (data) { 106 | t.deepEqual(JSON.parse(data), { 107 | are2: 'results' 108 | }) 109 | })) 110 | }) 111 | }) 112 | 113 | t.test('second run', function (t) { 114 | t.plan(6) 115 | 116 | const runner = new EE() 117 | runner.jobId = 43 118 | 119 | storage.wire(runner) 120 | 121 | var cannon = new EE() 122 | 123 | runner.emit('bench', { 124 | name: 'first', 125 | meta: 'data' 126 | }, cannon) 127 | 128 | cannon.emit('done', { 129 | are: 'results' 130 | }) 131 | 132 | cannon = new EE() 133 | 134 | runner.emit('bench', { 135 | name: 'second', 136 | meta: 'data2' 137 | }, cannon) 138 | 139 | cannon.emit('done', { 140 | are2: 'results' 141 | }) 142 | 143 | storage.once('meta', function (data) { 144 | t.deepEqual(data, { 145 | nextId: 44, 146 | runs: [{ 147 | id: 43, 148 | path: 'run-43', 149 | results 150 | }, { 151 | id: 42, 152 | path: 'run-42', 153 | results 154 | }] 155 | }) 156 | }) 157 | 158 | runner.emit('done', results) 159 | 160 | // this will wait for all the process.nextTick 161 | // used by the dummy store 162 | setImmediate(function () { 163 | backing.createReadStream('meta.json') 164 | .pipe(concat(function (data) { 165 | t.deepEqual(JSON.parse(data), { 166 | nextId: 44, 167 | runs: [{ 168 | id: 43, 169 | path: 'run-43', 170 | results 171 | }, { 172 | id: 42, 173 | path: 'run-42', 174 | results 175 | }] 176 | }) 177 | })) 178 | 179 | backing.createReadStream('run-43/first/meta.json') 180 | .pipe(concat(function (data) { 181 | t.deepEqual(JSON.parse(data), { 182 | name: 'first', 183 | meta: 'data' 184 | }) 185 | })) 186 | 187 | backing.createReadStream('run-43/first/results.json') 188 | .pipe(concat(function (data) { 189 | t.deepEqual(JSON.parse(data), { 190 | are: 'results' 191 | }) 192 | })) 193 | 194 | backing.createReadStream('run-43/second/meta.json') 195 | .pipe(concat(function (data) { 196 | t.deepEqual(JSON.parse(data), { 197 | name: 'second', 198 | meta: 'data2' 199 | }) 200 | })) 201 | 202 | backing.createReadStream('run-43/second/results.json') 203 | .pipe(concat(function (data) { 204 | t.deepEqual(JSON.parse(data), { 205 | are2: 'results' 206 | }) 207 | })) 208 | }) 209 | }) 210 | 211 | t.test('fetchMeta', function (t) { 212 | t.plan(2) 213 | 214 | storage.fetchMeta(function (err, meta) { 215 | t.error(err) 216 | t.equal(meta.runs.length, 2) 217 | }) 218 | }) 219 | 220 | t.test('nextJobId with status', function (t) { 221 | t.plan(2) 222 | 223 | storage.nextJobId(function (err, id) { 224 | t.error(err) 225 | t.equal(id, 44) 226 | }) 227 | }) 228 | --------------------------------------------------------------------------------