├── .babelrc ├── .gitignore ├── .editorconfig ├── functional ├── index.js ├── static │ └── index.html ├── configs │ └── load-page.js └── runners │ └── webdriverio.js ├── unit ├── runners │ ├── index.js │ ├── jest.js │ ├── mocha.js │ ├── ava.js │ ├── mocha-parallel-tests.js │ ├── tap.js │ ├── mocha.parallel.js │ ├── tape.js │ ├── lab.js │ ├── qunit.js │ └── jasmine.js ├── gen-configs.js ├── run-configs.js └── index.js ├── helpers ├── gen │ ├── randomizer.js │ ├── index.js │ └── generator.js ├── run │ ├── save-result.js │ ├── index.js │ ├── measurer.js │ └── report.js ├── expand-config.js └── utils.js ├── docs ├── index.css ├── data-labels-plugin.js ├── index.html ├── index.js └── data.js ├── package.json └── README.md /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["es2015"] 3 | } 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules/ 3 | npm-debug.log 4 | .idea 5 | 6 | /tests 7 | /temp 8 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | charset = utf-8 6 | insert_final_newline = true 7 | 8 | [*.{js,jsx,json}] 9 | indent_style = space 10 | indent_size = 2 11 | trim_trailing_whitespace = true 12 | -------------------------------------------------------------------------------- /functional/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Configs and runners for functional tests 3 | */ 4 | 5 | exports.configs = [ 6 | require('./configs/load-page'), 7 | ]; 8 | 9 | exports.runners = [ 10 | require('./runners/webdriverio'), 11 | ]; 12 | -------------------------------------------------------------------------------- /unit/runners/index.js: -------------------------------------------------------------------------------- 1 | module.exports = [ 2 | require('./mocha'), 3 | require('./mocha.parallel'), 4 | require('./mocha-parallel-tests'), 5 | require('./jasmine'), 6 | require('./qunit'), 7 | require('./lab'), 8 | require('./ava'), 9 | require('./tape'), 10 | require('./tap'), 11 | require('./jest'), 12 | ]; 13 | -------------------------------------------------------------------------------- /unit/runners/jest.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | name: 'jest', 3 | file: '', 4 | suite: `describe('{name}', () => { {content} });`, 5 | hooks: [ 6 | 'beforeAll', 7 | 'beforeEach', 8 | 'afterAll', 9 | 'afterEach', 10 | ], 11 | test: `test('{name}', {fn})`, 12 | syncFn: `function () { {content} }`, 13 | asyncFn: `function (done) { {content} }` 14 | }; 15 | -------------------------------------------------------------------------------- /unit/runners/mocha.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | name: 'mocha', 3 | file: '', 4 | suite: `describe('{name}', function () { {content} });`, 5 | hooks: [ 6 | 'before', 7 | 'beforeEach', 8 | 'after', 9 | 'afterEach', 10 | ], 11 | test: `it('{name}', {fn})`, 12 | syncFn: `function () { {content} }`, 13 | asyncFn: `function (done) { {content} }`, 14 | }; 15 | -------------------------------------------------------------------------------- /unit/runners/ava.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | name: 'ava', 3 | file: `import test from 'ava'; \n {content}`, 4 | suite: null, 5 | hooks: [ 6 | 'test.before', 7 | 'test.beforeEach', 8 | 'test.after', 9 | 'test.afterEach', 10 | ], 11 | test: `test('{name}', {fn})`, 12 | syncFn: `() => { {content} }`, 13 | asyncFn: `() => new Promise(done => { {content} })`, 14 | }; 15 | -------------------------------------------------------------------------------- /unit/runners/mocha-parallel-tests.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | name: 'mocha-parallel-tests', 3 | file: '', 4 | suite: `describe('{name}', function () { {content} });`, 5 | hooks: [ 6 | 'before', 7 | 'beforeEach', 8 | 'after', 9 | 'afterEach', 10 | ], 11 | test: `it('{name}', {fn})`, 12 | syncFn: `function () { {content} }`, 13 | asyncFn: `function (done) { {content} }`, 14 | }; 15 | -------------------------------------------------------------------------------- /unit/runners/tap.js: -------------------------------------------------------------------------------- 1 | 2 | module.exports = { 3 | name: 'tap', 4 | file: `require('tap').mochaGlobals(); \n {content}`, 5 | suite: `describe('{name}', function () { {content} });`, 6 | hooks: [ 7 | 'before', 8 | 'beforeEach', 9 | 'after', 10 | 'afterEach', 11 | ], 12 | test: `it('{name}', {fn})`, 13 | syncFn: `function () { {content} }`, 14 | asyncFn: `function (done) { {content} }`, 15 | }; 16 | -------------------------------------------------------------------------------- /functional/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Loading... 4 | 5 | 6 |

Loading...

7 | 15 | 16 | -------------------------------------------------------------------------------- /unit/runners/mocha.parallel.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | name: 'mocha.parallel', 3 | file: `const parallel = require('mocha.parallel'); \nparallel('{name}', function () { {content} });`, 4 | suite: null, 5 | hooks: [ 6 | 'before', 7 | 'beforeEach', 8 | 'after', 9 | 'afterEach', 10 | ], 11 | test: `it('{name}', {fn})`, 12 | syncFn: `function () { {content} }`, 13 | asyncFn: `function (done) { {content} }`, 14 | }; 15 | -------------------------------------------------------------------------------- /unit/runners/tape.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Tape. 3 | * 4 | * Does not support hooks: 5 | * https://github.com/substack/tape/issues/59 6 | * 7 | * Does not support parallel execution: 8 | * https://github.com/substack/tape/issues/78 9 | */ 10 | 11 | module.exports = { 12 | name: 'tape', 13 | file: `var test = require('tape'); \n {content}`, 14 | test: `test('{name}', {fn})`, 15 | syncFn: `function (t) { 16 | {content} 17 | t.end(); 18 | }`, 19 | asyncFn: `function (t) { 20 | var done = t.end; 21 | {content} 22 | }` 23 | }; 24 | -------------------------------------------------------------------------------- /unit/runners/lab.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Lab. 3 | * 4 | * Supports concurrency: 5 | * https://github.com/hapijs/lab/issues/68 6 | */ 7 | 8 | module.exports = { 9 | name: 'lab', 10 | file: `const Lab = require('lab'); const lab = exports.lab = Lab.script(); \n {content}`, 11 | suite: null, 12 | hooks: [ 13 | 'lab.before', 14 | 'lab.beforeEach', 15 | 'lab.after', 16 | 'lab.afterEach', 17 | ], 18 | test: `lab.test('{name}', {fn})`, 19 | syncFn: `function (done) { {content} \n done(); }`, 20 | asyncFn: `function (done) { {content} }` 21 | }; 22 | -------------------------------------------------------------------------------- /helpers/gen/randomizer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Pre-generates array of random values and returns it one by one 3 | */ 4 | 5 | class Randomizer { 6 | constructor() { 7 | this._values = []; 8 | this._index = -1; 9 | this._seed(1000); 10 | } 11 | reset() { 12 | this._index = -1; 13 | } 14 | getValue() { 15 | this._index = this._index < this._values.length - 1 ? this._index + 1 : 0; 16 | return this._values[this._index]; 17 | } 18 | _seed(count) { 19 | for (let i = 0; i < count; i++) { 20 | this._values.push(Math.random()); 21 | } 22 | } 23 | } 24 | 25 | module.exports = new Randomizer(); 26 | -------------------------------------------------------------------------------- /unit/runners/qunit.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Qunit does not support parallel execution: 3 | * https://github.com/qunitjs/qunit/issues/947 4 | */ 5 | 6 | module.exports = { 7 | name: 'qunit', 8 | suite: `QUnit.module('{name}', function (hooks) { {content} });`, 9 | hooks: [ 10 | 'hooks.before', 11 | 'hooks.beforeEach', 12 | 'hooks.after', 13 | 'hooks.afterEach', 14 | ], 15 | test: `QUnit.test('{name}', {fn})`, 16 | syncFn: `function (assert) { 17 | assert.expect(0); 18 | {content} 19 | }`, 20 | asyncFn: `function (assert) { 21 | var done = assert.async(); 22 | assert.expect(0); 23 | {content} 24 | }`, 25 | }; 26 | -------------------------------------------------------------------------------- /helpers/gen/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Generates tests for all configs and runners 3 | */ 4 | 5 | const fs = require('fs-extra'); 6 | const Generator = require('./generator'); 7 | 8 | module.exports = function gen(configs, runners) { 9 | fs.emptyDirSync(configs[0].basePath); 10 | configs.forEach(config => { 11 | runners.forEach(runner => { 12 | new Generator(config, runner).generate(); 13 | }); 14 | }); 15 | }; 16 | 17 | /* 18 | paths.setTestingType(testingType); 19 | fs.emptyDirSync(paths.getTestingTypePath()); 20 | const expandedConfigs = configUtils.getExpandedConfigs(configs); 21 | expandedConfigs.forEach(config => { 22 | runners.forEach(runner => { 23 | new Generator(config, runner).generate(); 24 | }); 25 | }); 26 | */ 27 | -------------------------------------------------------------------------------- /functional/configs/load-page.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Load-page config 3 | */ 4 | 5 | module.exports = { 6 | baseName: 'load-page', 7 | 8 | generate: { 9 | filesCount: 5, 10 | nestedSuites: 0, 11 | hooks: 0, 12 | testsInSuite: 3, 13 | suitesInSuite: 1, 14 | url: 'http://localhost:8080', 15 | delay: [0, '0-200'], 16 | testFn: 'loadPageFn', 17 | }, 18 | 19 | run: [ 20 | { 21 | runner: 'webdriverio', 22 | label: 'webdriverio (concurrency=3)', 23 | cmd: 'wdio temp/wdio.conf.js', 24 | config: { 25 | maxInstances: 3 26 | } 27 | }, 28 | { 29 | runner: 'webdriverio', 30 | label: 'webdriverio (concurrency=5)', 31 | cmd: 'wdio temp/wdio.conf.js', 32 | config: { 33 | maxInstances: 5 34 | } 35 | }, 36 | ] 37 | }; 38 | -------------------------------------------------------------------------------- /unit/gen-configs.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Config for generating test-files 3 | */ 4 | 5 | const expandConfig = require('../helpers/expand-config'); 6 | 7 | module.exports = expandConfig({ 8 | basePath: 'tests/unit', 9 | files: 50, 10 | nestedSuites: [false, true], 11 | hooks: false, 12 | // hooks does not have big impact on performance 13 | // hooks: [false, true], 14 | suitesInSuite: 2, 15 | testsInSuite: 5, 16 | test: [ 17 | {name: 'syncEmptyFn', value: ``}, 18 | {name: 'syncHeavyFn', value: ` 19 | eval(''); // disable V8 optimizations 20 | for (let i = 0; i < 10000; i++) { 21 | new Date(); 22 | } 23 | `}, 24 | {name: 'asyncEmptyFnZeroDelay', value: `setTimeout(done, 0)`}, 25 | {name: 'asyncEmptyFnRandomDelay', value: `setTimeout(done, Math.round({random} * 10))`}, 26 | ] 27 | }); 28 | -------------------------------------------------------------------------------- /unit/runners/jasmine.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs-extra'); 2 | const path = require('path'); 3 | 4 | module.exports = { 5 | name: 'jasmine', 6 | file: '', 7 | suite: `describe('{name}', function () { {content} });`, 8 | hooks: [ 9 | 'beforeAll', 10 | 'beforeEach', 11 | 'afterAll', 12 | 'afterEach', 13 | ], 14 | test: `it('{name}', {fn})`, 15 | syncFn: `function () { {content} }`, 16 | asyncFn: `function (done) { 17 | {content} 18 | }`, 19 | writeConfigFile 20 | }; 21 | 22 | function writeConfigFile(testsPath, runInfo) { 23 | const config = { 24 | spec_dir: testsPath, 25 | spec_files: [ 26 | '**/*.js' 27 | ] 28 | }; 29 | 30 | if (runInfo.babel) { 31 | config.helpers = [path.resolve('./node_modules/babel-register/lib/node.js')]; 32 | } 33 | 34 | const filename = './temp/jasmine.json'; 35 | fs.outputJsonSync(filename, config); 36 | } 37 | -------------------------------------------------------------------------------- /helpers/run/save-result.js: -------------------------------------------------------------------------------- 1 | 2 | const fs = require('fs-extra'); 3 | 4 | const filePath = 'docs/data.js'; 5 | const prefix = 'const data = '; 6 | 7 | /** 8 | * Saves result to docs/data.js 9 | */ 10 | module.exports = function saveResult(result) { 11 | fs.ensureFileSync(filePath); 12 | const data = readData(); 13 | data[result.name] = result; 14 | writeData(data); 15 | }; 16 | 17 | function readData() { 18 | const content = fs.readFileSync(filePath, 'utf8'); 19 | return parseContent(content); 20 | } 21 | 22 | function writeData(data) { 23 | const content = createContent(data); 24 | fs.writeFileSync(filePath, content, 'utf8'); 25 | } 26 | 27 | function parseContent(content) { 28 | const strJson = content.replace(prefix, '').trim(); 29 | return strJson ? JSON.parse(strJson) : {}; 30 | } 31 | 32 | function createContent(data) { 33 | return `${prefix}${JSON.stringify(data, false, 2)}`; 34 | } 35 | -------------------------------------------------------------------------------- /docs/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | color: #333333; 3 | font-family: Arial, "Times New Roman", serif; 4 | display: flex; 5 | flex-direction: column; 6 | align-items: center; 7 | padding: 15px; 8 | box-sizing: border-box; 9 | } 10 | 11 | h1 { 12 | padding-left: 15%; 13 | padding-right: 15%; 14 | } 15 | 16 | label { 17 | display: block; 18 | margin: 5px; 19 | } 20 | 21 | @media (min-width: 640px) { 22 | label { 23 | display: inline; 24 | } 25 | } 26 | 27 | .chart { 28 | position: relative; 29 | align-self: stretch; 30 | margin: 15px 0; 31 | } 32 | 33 | @media (min-width: 640px) { 34 | .chart { 35 | align-self: center; 36 | width: 70%; 37 | } 38 | } 39 | 40 | .chart-container { 41 | position: relative; 42 | height: 50vh; 43 | } 44 | 45 | .nodata { 46 | font-weight: bold; 47 | } 48 | 49 | footer { 50 | font-size: 0.9em; 51 | align-self: flex-end; 52 | } 53 | -------------------------------------------------------------------------------- /docs/data-labels-plugin.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Plugin to draw persistent label at the end of each bar 3 | */ 4 | Chart.plugins.register({ 5 | afterDatasetsDraw: function(ctrl) { 6 | var chart = ctrl.chart; 7 | var ctx = chart.ctx; 8 | ctrl.data.datasets.forEach(function (dataset, i) { 9 | var meta = ctrl.getDatasetMeta(i); 10 | if (!meta.hidden) { 11 | meta.data.forEach(function(element, index) { 12 | if (dataset.data[index] === 0) { 13 | return; 14 | } 15 | ctx.fillStyle = '#666'; 16 | var fontSize = 14; 17 | var fontStyle = 'normal'; 18 | var fontFamily = 'Helvetica Neue'; 19 | ctx.font = Chart.helpers.fontString(fontSize, fontStyle, fontFamily); 20 | var dataString = dataset.data[index].toString(); 21 | ctx.textAlign = 'left'; 22 | ctx.textBaseline = 'middle'; 23 | var position = element.tooltipPosition(); 24 | ctx.fillText(dataString, position.x + 5, position.y); 25 | }); 26 | } 27 | }); 28 | } 29 | }); 30 | -------------------------------------------------------------------------------- /helpers/run/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Runs bench for combination of genConfigs + runConfigs. 3 | */ 4 | 5 | const path = require('path'); 6 | const assert = require('assert'); 7 | const Measurer = require('./measurer'); 8 | const report = require('./report'); 9 | const saveResult = require('./save-result'); 10 | 11 | module.exports = function run(genConfigs, runConfigs, runners) { 12 | assert(genConfigs.length, `Empty genConfigs`); 13 | assert(runConfigs.length, `Empty runConfigs`); 14 | report.printHeader(); 15 | report.printRunners(runConfigs); 16 | measureRunners(genConfigs, runConfigs, runners); 17 | report.printFooter(); 18 | }; 19 | 20 | function measureRunners(genConfigs, runConfigs, runners) { 21 | genConfigs.forEach(genConfig => { 22 | runConfigs.forEach(runConfig => { 23 | const measurer = new Measurer(genConfig, runConfig, runners); 24 | report.printMeasurementHeader(measurer.result); 25 | measurer.measure(); 26 | report.printMeasurementResult(measurer.result); 27 | if (process.env.SAVE_BENCH) { 28 | saveResult(measurer.result); 29 | } 30 | }); 31 | }); 32 | } 33 | -------------------------------------------------------------------------------- /helpers/expand-config.js: -------------------------------------------------------------------------------- 1 | 2 | const path = require('path'); 3 | const utils = require('./utils'); 4 | 5 | /** 6 | * Expands config 7 | * 8 | * @returns {Array} 9 | */ 10 | module.exports = function expandConfig(config) { 11 | const expandedConfigs = utils.expandArrayProps(config); 12 | return expandedConfigs 13 | .map(sortPath) 14 | .map(setName) 15 | .map(setOutPath) 16 | .map(setTestsCount); 17 | }; 18 | 19 | function setTestsCount(config) { 20 | const {files, testsInSuite, suitesInSuite, nestedSuites} = config; 21 | const testsInFile = nestedSuites ? testsInSuite * suitesInSuite : testsInSuite; 22 | const tests = testsInFile * files; 23 | return Object.assign(config, {tests}); 24 | } 25 | 26 | function sortPath(config) { 27 | config._path.sort((a, b) => a > b ? -1 : 1).join('_'); 28 | return config; 29 | } 30 | 31 | function setName(config) { 32 | const name = config._path.join('_'); 33 | return Object.assign(config, {name}); 34 | } 35 | 36 | function setOutPath(config) { 37 | const outPath = path.join.apply(path, [config.basePath].concat(config._path)); 38 | return Object.assign(config, {outPath}); 39 | } 40 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "js-testrunners-bench", 3 | "version": "0.1.0", 4 | "description": "Javascript test-runners benchmark", 5 | "author": { 6 | "name": "Vitaliy Potapov", 7 | "email": "noginsk@rambler.ru" 8 | }, 9 | "scripts": { 10 | "selenium": "selenium-standalone start", 11 | "static": "http-server ./functional/static" 12 | }, 13 | "dependencies": { 14 | "ava": "^0.19.1", 15 | "babel-preset-es2015": "^6.24.1", 16 | "babel-register": "^6.24.1", 17 | "chalk": "^1.1.3", 18 | "columnify": "^1.5.4", 19 | "fs-extra": "^3.0.1", 20 | "http-server": "^0.10.0", 21 | "jasmine": "^2.6.0", 22 | "jest": "^20.0.0", 23 | "lab": "^13.1.0", 24 | "lab-babel": "^1.1.1", 25 | "mocha": "^3.3.0", 26 | "mocha-parallel-tests": "^1.2.9", 27 | "mocha.parallel": "^0.15.2", 28 | "qunitjs": "^2.3.3", 29 | "selenium-standalone": "^6.0.1", 30 | "selenium-webdriver": "^3.4.0", 31 | "shelljs": "^0.7.8", 32 | "tap": "^10.3.3", 33 | "tape": "^4.6.3", 34 | "wdio-mocha-framework": "^0.5.8", 35 | "webdriverio": "^4.6.2" 36 | }, 37 | "devDependencies": {}, 38 | "jest": { 39 | "testRegex": "\\.js$" 40 | }, 41 | "license": "MIT" 42 | } 43 | -------------------------------------------------------------------------------- /functional/runners/webdriverio.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs-extra'); 2 | const path = require('path'); 3 | 4 | module.exports = { 5 | name: 'webdriverio', 6 | hooks: [ 7 | 'before', 8 | 'beforeEach', 9 | 'after', 10 | 'afterEach', 11 | ], 12 | suite: `describe('{name}', function () { {content} });`, 13 | test: `it('{name}', {fn});`, 14 | loadPageFn: function () { 15 | browser.url('{url}?delay={delay}'); 16 | browser.waitUntil(function () { 17 | return browser.getTitle() === 'Loaded.'; 18 | }, 10000); 19 | }, 20 | writeConfigFile 21 | }; 22 | 23 | function writeConfigFile(testsPath, runInfo) { 24 | const config = Object.assign({ 25 | specs: [ 26 | `${testsPath}/*.js` 27 | ], 28 | maxInstances: 1, 29 | capabilities: [ 30 | {browserName: 'chrome'} 31 | ], 32 | sync: true, 33 | logLevel: 'silent', 34 | coloredLogs: true, 35 | waitforTimeout: 10000, 36 | connectionRetryTimeout: 90000, 37 | connectionRetryCount: 3, 38 | framework: 'mocha', 39 | mochaOpts: { 40 | ui: 'bdd' 41 | } 42 | }, runInfo.config); 43 | const filename = './temp/wdio.conf.js'; 44 | const content = `exports.config = ${JSON.stringify(config, false, 2)};`; 45 | fs.outputFileSync(filename, content); 46 | } 47 | -------------------------------------------------------------------------------- /unit/run-configs.js: -------------------------------------------------------------------------------- 1 | 2 | module.exports = [ 3 | { 4 | name: 'babel=false', 5 | runs: [ 6 | {runner: 'mocha', cmd: 'mocha {path}'}, 7 | {runner: 'jasmine', cmd: 'jasmine JASMINE_CONFIG_PATH=temp/jasmine.json'}, 8 | {runner: 'mocha.parallel', cmd: 'mocha {path}'}, 9 | {runner: 'mocha-parallel-tests', cmd: 'mocha-parallel-tests {path}'}, 10 | {runner: 'qunit', cmd: 'qunit {path}'}, 11 | {runner: 'tape', cmd: 'tape {path}/*.js'}, 12 | {runner: 'tap', cmd: 'tap {path} --jobs-auto'}, 13 | {runner: 'lab', cmd: 'lab --parallel {path}'}, 14 | {runner: 'ava', cmd: 'ava {path} --concurrency=4'}, 15 | {label: 'jest (jsdom)', runner: 'jest', cmd: 'jest --env=jsdom {path}'}, 16 | {label: 'jest (node)', runner: 'jest', cmd: 'jest --env=node {path}'}, 17 | ] 18 | }, 19 | { 20 | name: 'babel=true', 21 | runs: [ 22 | {runner: 'mocha', cmd: 'mocha {path} --compilers js:babel-register'}, 23 | {runner: 'mocha-parallel-tests', cmd: 'mocha-parallel-tests {path} --compilers js:babel-register'}, 24 | {runner: 'jasmine', cmd: 'jasmine JASMINE_CONFIG_PATH=temp/jasmine.json', babel: true}, 25 | {runner: 'lab', cmd: 'lab {path} -T node_modules/lab-babel'}, 26 | {runner: 'ava', cmd: 'ava {path} --concurrency=4'}, 27 | {label: 'jest (jsdom)', runner: 'jest', cmd: 'jest --env=jsdom {path}'}, 28 | {label: 'jest (node)', runner: 'jest', cmd: 'jest --env=node {path}'}, 29 | ] 30 | }, 31 | ]; 32 | -------------------------------------------------------------------------------- /helpers/run/measurer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Measures runners execution times for pair of genConfig and runConfig. 3 | */ 4 | 5 | const path = require('path'); 6 | const fs = require('fs-extra'); 7 | const chalk = require('chalk'); 8 | const utils = require('../utils'); 9 | 10 | module.exports = class Measurer { 11 | constructor(genConfig, runConfig, runners) { 12 | this._genConfig = genConfig; 13 | this._runConfig = runConfig; 14 | this._runners = runners; 15 | this._result = { 16 | name: `${genConfig.name}_${runConfig.name}`, 17 | genConfig, 18 | runConfig, 19 | runs: [], 20 | }; 21 | } 22 | 23 | get result() { 24 | return this._result; 25 | } 26 | 27 | measure() { 28 | this._measureRunners(); 29 | this._sortResults(); 30 | return this._result; 31 | } 32 | 33 | _measureRunners() { 34 | this._runConfig.runs.forEach(runInfo => this._measureRunner(runInfo)); 35 | } 36 | 37 | _measureRunner(runInfo) { 38 | const testsPath = path.join(this._genConfig.outPath, runInfo.runner); 39 | const label = runInfo.label || runInfo.runner; 40 | const cmd = runInfo.cmd.replace('{path}', testsPath); 41 | if (!fs.existsSync(testsPath)) { 42 | console.log(`${chalk.bold('Skipping:')} ${label}, ${chalk.bold('cmd:')} ${cmd}`); 43 | return; 44 | } 45 | console.log(`${chalk.bold('Running:')} ${label}, ${chalk.bold('cmd:')} ${cmd}`); 46 | this._writeRunnerConfigFile(testsPath, runInfo); 47 | const time = utils.measureCmd(cmd).toPrecision(3); 48 | const runnerResult = {label, time}; 49 | this._result.runs.push(runnerResult); 50 | } 51 | 52 | _writeRunnerConfigFile(runnerTestsPath, runInfo) { 53 | const runner = this._runners.find(runner => runner.name === runInfo.runner); 54 | if (runner.writeConfigFile) { 55 | runner.writeConfigFile(runnerTestsPath, runInfo); 56 | } 57 | } 58 | 59 | _sortResults() { 60 | this._result.runs.sort((a, b) => a.time - b.time); 61 | } 62 | }; 63 | -------------------------------------------------------------------------------- /unit/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Entry point for running tests. 3 | * @examples 4 | * node unit gen 5 | * node unit conf 6 | * node unit run test=syncEmptyFn_nestedSuites=false babel=false 7 | * node unit run test=syncEmptyFn_nestedSuites=false babel=true 8 | * node unit run test=syncHeavyFn_nestedSuites=false babel=false 9 | * node unit run test=syncHeavyFn_nestedSuites=true babel=false 10 | * node unit run test=asyncEmptyFnZeroDelay_nestedSuites=false babel=false 11 | * node unit run test=asyncEmptyFnRandomDelay_nestedSuites=false babel=false 12 | */ 13 | const chalk = require('chalk'); 14 | const genConfigs = require('./gen-configs'); 15 | const runConfigs = require('./run-configs'); 16 | const runners = require('./runners'); 17 | const run = require('../helpers/run'); 18 | const gen = require('../helpers/gen'); 19 | 20 | const command = process.argv[2]; 21 | 22 | switch (command) { 23 | case 'gen': 24 | gen(genConfigs, runners); 25 | break; 26 | case 'conf': 27 | printConfigs(); 28 | break; 29 | case 'run': 30 | const genFilter = process.argv[3]; 31 | const runFilter = process.argv[4]; 32 | runBench(genFilter, runFilter); 33 | break; 34 | case 'run-and-save': 35 | // run all benchs and save results to docs/data.js 36 | process.env.SAVE_BENCH = true; 37 | runBench(); 38 | break; 39 | default: 40 | console.log(`Unknown command: ${command}`); 41 | } 42 | 43 | function runBench(genConfigName, runConfigName) { 44 | const filteredGenConfigs = filterByName(genConfigs, genConfigName); 45 | const filteredRunConfigs = filterByName(runConfigs, runConfigName); 46 | run(filteredGenConfigs, filteredRunConfigs, runners); 47 | } 48 | 49 | function printConfigs() { 50 | console.log(chalk.bold('genConfigs:')); 51 | genConfigs.forEach(genConfig => console.log(genConfig.name)); 52 | console.log(chalk.bold('runConfigs:')); 53 | runConfigs.forEach(runConfig => console.log(runConfig.name)); 54 | } 55 | 56 | function filterByName(configs, value) { 57 | return configs.filter(config => !value || config.name.indexOf(value) >= 0); 58 | } 59 | -------------------------------------------------------------------------------- /helpers/run/report.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Reports results to console and html 3 | */ 4 | const path = require('path'); 5 | const os = require('os'); 6 | const chalk = require('chalk'); 7 | const columnify = require('columnify'); 8 | 9 | exports.printHeader = function () { 10 | console.log(chalk.bold(`JavaScript test-runners benchmark`)); 11 | printSystemInfo(); 12 | console.log(`${chalk.bold('Date:')} ${new Date().toDateString()}`); 13 | console.log(''); 14 | }; 15 | 16 | exports.printRunners = function (runConfigs) { 17 | const versions = []; 18 | runConfigs.forEach(runConfig => { 19 | runConfig.runs.forEach(runInfo => { 20 | const exists = versions.some(v => v.runner === runInfo.runner); 21 | if (!exists) { 22 | const npmModule = runInfo.runner === 'qunit' ? 'qunitjs' : runInfo.runner; 23 | versions.push({ 24 | runner: runInfo.runner, 25 | version: require(`${npmModule}/package.json`).version, 26 | }); 27 | } 28 | }); 29 | }); 30 | console.log(columnify(versions)); 31 | console.log(''); 32 | }; 33 | 34 | exports.printMeasurementHeader = function (result) { 35 | const {files, tests} = result.genConfig; 36 | console.log(`${chalk.bold('Bench type:')} ${result.name.replace(/_/g, ', ')}`); 37 | console.log(`${chalk.bold('Tests count:')} ${tests} (${files} files)`); 38 | }; 39 | 40 | exports.printMeasurementResult = function (result) { 41 | const runs = result.runs.slice(); 42 | if (runs.length) { 43 | const {label, time} = runs[0]; 44 | // mark top with bold 45 | runs[0] = { 46 | label: chalk.green.bold(label), 47 | time: chalk.green.bold(time), 48 | }; 49 | } 50 | console.log(chalk.bold('Result:')); 51 | console.log(columnify(runs)); 52 | console.log(''); 53 | }; 54 | 55 | exports.printFooter = function () { 56 | console.log(chalk.bold('Done.')); 57 | }; 58 | 59 | function printSystemInfo() { 60 | console.log([ 61 | `${chalk.bold('System:')} ${os.platform()} ${os.arch()} ${os.cpus().length} cpu(s) `, 62 | `${process.title} ${process.version}` 63 | ].join('')); 64 | } 65 | -------------------------------------------------------------------------------- /helpers/utils.js: -------------------------------------------------------------------------------- 1 | 2 | const {exec} = require('shelljs'); 3 | 4 | /** 5 | * Expands array props of object 6 | * Example: 7 | * {x: 1, y: [1, 2]} --> [{x:1, y: 1, _path: ['y=1']}, {x:1, y: 2, _path: ['y=2']}] 8 | * 9 | * @returns {Array} 10 | */ 11 | exports.expandArrayProps = function expandArrayProps(obj) { 12 | const PATH_KEY = '_path'; 13 | const subObjects = []; 14 | const keys = Object.keys(obj).sort(); 15 | for (let i = 0; i < keys.length; i++) { 16 | const prop = keys[i]; 17 | const value = obj[prop]; 18 | if (Array.isArray(value) && prop !== PATH_KEY) { 19 | value.forEach(item => { 20 | const itemName = item.name || item; 21 | const itemValue = item.value !== undefined ? item.value : item; 22 | const pathValue = (obj[PATH_KEY] || []).concat([`${prop}=${itemName}`]); 23 | const subObj = Object.assign({}, obj, { 24 | [prop]: itemValue, 25 | [PATH_KEY]: pathValue, 26 | }); 27 | subObjects.push(subObj); 28 | }); 29 | break; 30 | } 31 | } 32 | 33 | if (subObjects.length) { 34 | return subObjects.reduce((res, subObj) => { 35 | return res.concat(expandArrayProps(subObj)); 36 | }, []); 37 | } else { 38 | return [obj]; 39 | } 40 | }; 41 | 42 | /** 43 | * Measures CMD execution time 44 | * 45 | * @param {String} cmd 46 | * @param {Number} [count=1] count of runs 47 | * @returns {Number} 48 | */ 49 | exports.measureCmd = function measureCmd(cmd, count = 1) { 50 | const values = []; 51 | for (let i = 0; i < count; i++) { 52 | const proc = exec(`time ${cmd}`, {silent: true}); 53 | assertCmdOutput(cmd, proc.stderr); 54 | assertCmdOutput(cmd, proc.stdout); 55 | 56 | const matches = proc.stderr.match(/real\s+([0-9]+)m([0-9\.]+)s/); 57 | const minutes = parseInt(matches[1], 10); 58 | const seconds = parseFloat(matches[2]); 59 | const total = minutes * 60 + seconds; 60 | values.push(total); 61 | } 62 | const sum = values.reduce((r, value) => r + value, 0); 63 | return sum / count; 64 | }; 65 | 66 | function assertCmdOutput(cmd, output) { 67 | // if (!output) { 68 | // console.log(`Output is empty! COMMAND: ${cmd}`); 69 | // process.exit(1); 70 | // } 71 | if (/error|exception/i.test(output)) { 72 | console.log(output); 73 | console.log(`Error in output! COMMAND: ${cmd}`); 74 | process.exit(1); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | JavaScript Test-Runners Benchmark 7 | 8 | 9 | 10 | 11 | 12 | 13 | Fork me on GitHub 14 |

JavaScript Test-Runners Benchmark

15 |
16 |
17 | 26 | 33 | 40 |
41 |
42 | 45 | 46 | 47 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /docs/index.js: -------------------------------------------------------------------------------- 1 | 2 | function log(s) { 3 | document.querySelector('.log').textContent += String(s); 4 | } 5 | 6 | // generate colors: http://tools.medialab.sciences-po.fr/iwanthue/index.php 7 | const colors = [ 8 | "#3592ff", 9 | "#f74241", 10 | "#5ade5e", 11 | "#971da8", 12 | "#00d3a7", 13 | "#3447b5", 14 | "#ea6315", 15 | "#deb8f8", 16 | "#865e00", 17 | "#694673", 18 | "#ff8fa1", 19 | "#883c26", 20 | ]; 21 | 22 | const chart = document.querySelector('.chart'); 23 | const filterForm = document.querySelector('.filter'); 24 | 25 | submitOnChange(); 26 | const benchKey = getKeyFromUrl() || Object.keys(data)[0]; 27 | populateForm(benchKey); 28 | update(benchKey); 29 | 30 | function submitOnChange() { 31 | filterForm.addEventListener('change', () => { 32 | filterForm.submit(); 33 | }); 34 | } 35 | 36 | function populateForm(benchKey) { 37 | benchKey 38 | .split('_') 39 | .map(str => str.split('=')) 40 | .forEach(([name, value]) => filterForm.elements[name].value = value); 41 | } 42 | 43 | function update(benchKey) { 44 | chart.innerHTML = ''; 45 | const benchData = data[benchKey]; 46 | if (!benchData) { 47 | showNoBench(); 48 | } else { 49 | const title = getTitle(benchData); 50 | const datasets = getDatasets(benchData); 51 | const labels = getLabels(benchData); 52 | drawChart(title, labels, datasets); 53 | } 54 | } 55 | 56 | function getKeyFromUrl() { 57 | const url = new URL(location.href); 58 | return url.search.slice(1).replace(/\&/g, '_'); 59 | } 60 | 61 | function getTitle(benchData) { 62 | return `${benchData.genConfig.tests} ${benchData.name.replace(/_/g, ', ')}`; 63 | } 64 | 65 | function getLabels(benchData) { 66 | return benchData.runs.map(run => run.label); 67 | } 68 | 69 | function getDatasets(benchData) { 70 | return benchData.runs.map((run, index) => { 71 | return { 72 | label: run.label, 73 | data: getChartData(index, run.time), 74 | backgroundColor: colors[index], 75 | }; 76 | }); 77 | } 78 | 79 | function drawChart(title, labels, datasets) { 80 | const canvas = document.createElement('canvas'); 81 | chart.appendChild(canvas); 82 | new Chart(canvas, { 83 | type: 'horizontalBar', 84 | data: { 85 | labels, 86 | datasets, 87 | }, 88 | options: { 89 | tooltips: { 90 | enabled: false, 91 | }, 92 | title: { 93 | display: true, 94 | fontSize: 16, 95 | text: ' '.repeat(0) + title 96 | }, 97 | legend: { 98 | display: false, 99 | }, 100 | scales: { 101 | yAxes: [{ 102 | stacked: true, 103 | ticks: { 104 | fontSize: 14, 105 | } 106 | }], 107 | xAxes: [{ 108 | ticks: { 109 | beginAtZero: true, 110 | fontSize: 14, 111 | suggestedMax: datasets.slice(-1)[0].data.slice(-1)[0] * 1.1 112 | }, 113 | scaleLabel: { 114 | display: true, 115 | fontSize: 14, 116 | labelString: 'time (s)' 117 | } 118 | }] 119 | } 120 | } 121 | }); 122 | } 123 | 124 | function getChartData(index, value) { 125 | const data = []; 126 | for(let i = 0; i < index; i++) { 127 | data.push(0); 128 | } 129 | data.push(parseFloat(value)); 130 | return data; 131 | } 132 | 133 | function showNoBench() { 134 | chart.innerHTML = '
No data for that filter
'; 135 | } 136 | -------------------------------------------------------------------------------- /helpers/gen/generator.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Generates tests for particular config and runner 3 | */ 4 | const fs = require('fs-extra'); 5 | const path = require('path'); 6 | const randomizer = require('./randomizer'); 7 | 8 | module.exports = class Generator { 9 | constructor(config, runner) { 10 | this._config = config; 11 | this._runner = runner; 12 | this._outPath = path.join(config.outPath, runner.name); 13 | } 14 | 15 | generate() { 16 | if (this._isConfigSupportedByRunner()) { 17 | randomizer.reset(); 18 | this._createFiles(); 19 | } 20 | } 21 | 22 | _createFiles() { 23 | for (let i = 0; i < this._config.files; i++) { 24 | this._createFile(i); 25 | } 26 | console.log(`Generated ${this._config.files} files in ${this._outPath}`); 27 | } 28 | 29 | _createFile(index) { 30 | const fileName = `test-${index}.js`; 31 | const suiteName = `suite ${index}`; 32 | const suiteContent = this._createSuite(suiteName, 0); 33 | const tpl = this._runner.file || '{content}'; 34 | const fileContent = tpl 35 | .replace('{name}', fileName) 36 | .replace('{content}', suiteContent); 37 | 38 | const filePath = path.join(this._outPath, fileName); 39 | if (!fileContent) { 40 | throw new Error(`Empty file: ${filePath}`); 41 | } 42 | 43 | fs.outputFileSync(filePath, fileContent); 44 | } 45 | 46 | _createSuite(suiteName, suiteLevel) { 47 | const hooks = this._createHooks(); 48 | const isNestedSuites = this._config.nestedSuites && suiteLevel === 0; 49 | const children = isNestedSuites ? this._createSubSuites(suiteName, suiteLevel) : this._createTests(suiteLevel); 50 | const content = [hooks, children].join('\n'); 51 | const tpl = this._runner.suite || '{content}'; 52 | return tpl 53 | .replace('{name}', suiteName) 54 | .replace('{content}', content); 55 | } 56 | 57 | _createHooks() { 58 | if (this._config.hooks) { 59 | const hooks = this._runner.hooks.map(hook => this._createHook(hook)); 60 | return hooks.join('\n'); 61 | } else { 62 | return ''; 63 | } 64 | } 65 | 66 | _createHook(hookName) { 67 | const tpl = '{name}({fn});'; 68 | return tpl 69 | .replace('{name}', hookName) 70 | .replace('{fn}', this._createTestFn()); 71 | } 72 | 73 | _createSubSuites(suiteName, suiteLevel) { 74 | const subSuites = []; 75 | for (let i = 0; i < this._config.suitesInSuite; i++) { 76 | const name = `${suiteName} ${suiteLevel + 1} ${i}`; 77 | const subSuite = this._createSuite(name, suiteLevel + 1); 78 | subSuites.push(subSuite); 79 | } 80 | return subSuites.join('\n'); 81 | } 82 | 83 | _createTests(suiteLevel) { 84 | const tests = []; 85 | for (let i = 0; i < this._config.testsInSuite; i++) { 86 | const testName = `test ${suiteLevel} ${i}`; 87 | const test = this._createTest(testName); 88 | tests.push(test); 89 | } 90 | return tests.join('\n'); 91 | } 92 | 93 | _createTest(testName) { 94 | return this._runner.test 95 | .replace('{name}', testName) 96 | .replace('{fn}', this._createTestFn()); 97 | } 98 | 99 | _createTestFn() { 100 | const content = this._config.test; 101 | const isAsync = content.indexOf('done') >= 0; 102 | const tpl = isAsync ? this._runner.asyncFn : this._runner.syncFn; 103 | return tpl 104 | .replace('{content}', content) 105 | .replace('{random}', randomizer.getValue().toFixed(2)); 106 | } 107 | 108 | /** 109 | * Some configs are not supported by runners. For example, config with nested suites is not supported by AVA. 110 | * @returns {boolean} 111 | * @private 112 | */ 113 | _isConfigSupportedByRunner() { 114 | // nested suites are required but not supported by runner 115 | if (this._config.nestedSuites > 0 && !this._runner.suite) { 116 | return false; 117 | } 118 | // hooks are required but not supported by runner 119 | if (this._config.hooks > 0 && !this._runner.hooks) { 120 | return false; 121 | } 122 | // ? 123 | // if (!this._getFn()) { 124 | // return false; 125 | // } 126 | return true; 127 | } 128 | }; 129 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JavaScript test-runners benchmark 2 | The benchmark for running the same tests on the most popular JavaScript test-runners and measuring execution time. Tests are divided onto **unit** and **functional**. 3 | 4 | ## Unit tests 5 | 6 | ### Runners 7 | * [Mocha](https://github.com/mochajs/mocha) 8 | * [Jasmine](https://github.com/jasmine/jasmine) 9 | * [AVA](https://github.com/avajs/ava) 10 | * [Jest](https://github.com/facebook/jest) 11 | * [tape](https://github.com/substack/tape) 12 | * [tap](https://github.com/tapjs/node-tap) 13 | * [mocha-parallel-tests](https://github.com/yandex/mocha-parallel-tests) 14 | * [mocha.parallel](https://github.com/danielstjules/mocha.parallel) 15 | * [QUnit](https://github.com/qunitjs/qunit) 16 | * [lab](https://github.com/hapijs/lab) 17 | 18 | ### Benchmark conditions 19 | This benchmark measures execution time for every combination of following conditions: 20 | 21 | #### 1. Test types 22 | * Synchronous empty test: 23 | ```js 24 | function () {} 25 | ``` 26 | * Synchronous heavy test: 27 | ```js 28 | function () { 29 | for (let i = 0; i < 10000; i++) { 30 | new Date(); 31 | } 32 | } 33 | ``` 34 | * Asynchronous empty test with zero delay: 35 | ```js 36 | function (done) { 37 | setTimeout(done, 0); 38 | } 39 | ``` 40 | * Asynchronous empty test with random delay: 41 | ```js 42 | function (done) { 43 | setTimeout(done, Math.round(Math.random() * 10)); 44 | } 45 | ``` 46 | 47 | #### 2. Tests structure 48 | * With nested suites 49 | * Without nested suites 50 | 51 | #### 3. Run types 52 | * With Babel transpiling 53 | * Without Babel transpiling 54 | 55 | ### Latest results 56 | https://vitalets.github.io/js-testrunners-bench/index.html 57 | 58 | ### Run yourself 59 | 1. Clone the repo: 60 | ```bash 61 | git clone https://github.com/vitalets/js-testrunners-bench.git 62 | ``` 63 | 64 | 2. Install dependencies: 65 | ```bash 66 | cd js-testrunners-bench 67 | npm install 68 | ``` 69 | 70 | 3. Generate tests: 71 | ```bash 72 | node unit gen 73 | ``` 74 | After run check that `/tests` directory is created and filled with test-files. 75 | 76 | 4. Run benchmark: 77 | ```bash 78 | node unit run [testsType] [runType] 79 | ``` 80 | Examples: 81 | ```bash 82 | # Synchronous empty tests without nested suites and without Babel 83 | node unit run test=syncEmptyFn_nestedSuites=false babel=false 84 | 85 | # Asynchronous tests with random delay 0-10ms with nested suites and Babel 86 | node unit run test=asyncEmptyFnRandomDelay_nestedSuites=true babel=true 87 | 88 | etc.. 89 | ``` 90 | 91 |
92 | Example output: 93 | 94 | > node unit run test=syncEmptyFn_nestedSuites=false babel=false 95 | JavaScript test-runners benchmark 96 | System: darwin x64 4 cpu(s) node v7.2.0 97 | Date: Wed Jul 26 2017 98 | 99 | RUNNER VERSION 100 | mocha 3.4.2 101 | jasmine 2.6.0 102 | mocha.parallel 0.15.2 103 | mocha-parallel-tests 1.2.9 104 | qunit 2.3.3 105 | tape 4.6.3 106 | tap 10.3.3 107 | lab 13.1.0 108 | ava 0.19.1 109 | jest 20.0.4 110 | 111 | Bench type: test=syncEmptyFn, nestedSuites=false, babel=false 112 | Tests count: 250 (50 files) 113 | Running: mocha, cmd: mocha tests/unit/test=syncEmptyFn/nestedSuites=false/mocha 114 | Running: jasmine, cmd: jasmine JASMINE_CONFIG_PATH=temp/jasmine.json 115 | Running: mocha.parallel, cmd: mocha tests/unit/test=syncEmptyFn/nestedSuites=false/mocha.parallel 116 | Running: mocha-parallel-tests, cmd: mocha-parallel-tests tests/unit/test=syncEmptyFn/nestedSuites=false/mocha-parallel-tests 117 | Running: qunit, cmd: qunit tests/unit/test=syncEmptyFn/nestedSuites=false/qunit 118 | Running: tape, cmd: tape tests/unit/test=syncEmptyFn/nestedSuites=false/tape/*.js 119 | Running: tap, cmd: tap tests/unit/test=syncEmptyFn/nestedSuites=false/tap --jobs-auto 120 | Running: lab, cmd: lab --parallel tests/unit/test=syncEmptyFn/nestedSuites=false/lab 121 | Running: ava, cmd: ava tests/unit/test=syncEmptyFn/nestedSuites=false/ava --concurrency=4 122 | Running: jest (jsdom), cmd: jest --env=jsdom tests/unit/test=syncEmptyFn/nestedSuites=false/jest 123 | Running: jest (node), cmd: jest --env=node tests/unit/test=syncEmptyFn/nestedSuites=false/jest 124 | Result: 125 | LABEL TIME 126 | jasmine 0.205 127 | tape 0.273 128 | qunit 0.332 129 | mocha 0.346 130 | mocha.parallel 0.420 131 | lab 0.429 132 | mocha-parallel-tests 0.471 133 | jest (node) 1.84 134 | jest (jsdom) 3.78 135 | tap 6.32 136 | ava 8.34 137 | 138 | Done. 139 |
140 | 141 | ## Functional tests 142 | 143 | Not ready yet. 144 | 145 | ### Runners 146 | * todo [Webdriverio](http://webdriver.io) 147 | * todo [Mocha](https://github.com/mochajs/mocha) + [selenium-webdriver](https://www.npmjs.com/package/selenium-webdriver) 148 | * todo [CucumberJS](https://github.com/cucumber/cucumber-js) 149 | * todo [Nightwatch](https://github.com/nightwatchjs/nightwatch) 150 | * todo [TestCafe](https://github.com/DevExpress/testcafe) 151 | * todo [Nemo](https://github.com/paypal/nemo) 152 | 153 | ## Related links 154 | * [JavaScript Test-Runners Benchmark (Part 1, The Unit Testing)](https://medium.com/dailyjs/javascript-test-runners-benchmark-3a78d4117b4) 155 | * [JavaScript unit testing tools](https://mo.github.io/2017/06/05/javascript-unit-testing.html) 156 | * [An Overview of JavaScript Testing in 2017](https://medium.com/powtoon-engineering/a-complete-guide-to-testing-javascript-in-2017-a217b4cd5a2a) 157 | * [Picking Jest over Mocha – testing tools comparison](https://gziolo.pl/2017/06/17/picking-jest-over-mocha/) 158 | 159 | ## License 160 | MIT @ [Vitaliy Potapov](https://github.com/vitalets) 161 | -------------------------------------------------------------------------------- /docs/data.js: -------------------------------------------------------------------------------- 1 | const data = { 2 | "test=syncEmptyFn_nestedSuites=false_babel=false": { 3 | "name": "test=syncEmptyFn_nestedSuites=false_babel=false", 4 | "genConfig": { 5 | "basePath": "tests/unit", 6 | "files": 50, 7 | "nestedSuites": false, 8 | "hooks": false, 9 | "suitesInSuite": 2, 10 | "testsInSuite": 5, 11 | "test": "", 12 | "_path": [ 13 | "test=syncEmptyFn", 14 | "nestedSuites=false" 15 | ], 16 | "name": "test=syncEmptyFn_nestedSuites=false", 17 | "outPath": "tests/unit/test=syncEmptyFn/nestedSuites=false", 18 | "tests": 250 19 | }, 20 | "runConfig": { 21 | "name": "babel=false", 22 | "runs": [ 23 | { 24 | "runner": "mocha", 25 | "cmd": "mocha {path}" 26 | }, 27 | { 28 | "runner": "jasmine", 29 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json" 30 | }, 31 | { 32 | "runner": "mocha.parallel", 33 | "cmd": "mocha {path}" 34 | }, 35 | { 36 | "runner": "mocha-parallel-tests", 37 | "cmd": "mocha-parallel-tests {path}" 38 | }, 39 | { 40 | "runner": "qunit", 41 | "cmd": "qunit {path}" 42 | }, 43 | { 44 | "runner": "tape", 45 | "cmd": "tape {path}/*.js" 46 | }, 47 | { 48 | "runner": "tap", 49 | "cmd": "tap {path} --jobs-auto" 50 | }, 51 | { 52 | "runner": "lab", 53 | "cmd": "lab --parallel {path}" 54 | }, 55 | { 56 | "runner": "ava", 57 | "cmd": "ava {path} --concurrency=4" 58 | }, 59 | { 60 | "label": "jest (jsdom)", 61 | "runner": "jest", 62 | "cmd": "jest --env=jsdom {path}" 63 | }, 64 | { 65 | "label": "jest (node)", 66 | "runner": "jest", 67 | "cmd": "jest --env=node {path}" 68 | } 69 | ] 70 | }, 71 | "runs": [ 72 | { 73 | "label": "jasmine", 74 | "time": "0.204" 75 | }, 76 | { 77 | "label": "tape", 78 | "time": "0.266" 79 | }, 80 | { 81 | "label": "mocha", 82 | "time": "0.336" 83 | }, 84 | { 85 | "label": "qunit", 86 | "time": "0.339" 87 | }, 88 | { 89 | "label": "lab", 90 | "time": "0.370" 91 | }, 92 | { 93 | "label": "mocha.parallel", 94 | "time": "0.419" 95 | }, 96 | { 97 | "label": "mocha-parallel-tests", 98 | "time": "0.467" 99 | }, 100 | { 101 | "label": "jest (node)", 102 | "time": "1.84" 103 | }, 104 | { 105 | "label": "jest (jsdom)", 106 | "time": "3.46" 107 | }, 108 | { 109 | "label": "tap", 110 | "time": "5.64" 111 | }, 112 | { 113 | "label": "ava", 114 | "time": "8.26" 115 | } 116 | ] 117 | }, 118 | "test=asyncEmptyFnZeroDelay_nestedSuites=false_babel=false": { 119 | "name": "test=asyncEmptyFnZeroDelay_nestedSuites=false_babel=false", 120 | "genConfig": { 121 | "basePath": "tests/unit", 122 | "files": 50, 123 | "nestedSuites": false, 124 | "hooks": false, 125 | "suitesInSuite": 2, 126 | "testsInSuite": 5, 127 | "test": "setTimeout(done, 0)", 128 | "_path": [ 129 | "test=asyncEmptyFnZeroDelay", 130 | "nestedSuites=false" 131 | ], 132 | "name": "test=asyncEmptyFnZeroDelay_nestedSuites=false", 133 | "outPath": "tests/unit/test=asyncEmptyFnZeroDelay/nestedSuites=false", 134 | "tests": 250 135 | }, 136 | "runConfig": { 137 | "name": "babel=false", 138 | "runs": [ 139 | { 140 | "runner": "mocha", 141 | "cmd": "mocha {path}" 142 | }, 143 | { 144 | "runner": "jasmine", 145 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json" 146 | }, 147 | { 148 | "runner": "mocha.parallel", 149 | "cmd": "mocha {path}" 150 | }, 151 | { 152 | "runner": "mocha-parallel-tests", 153 | "cmd": "mocha-parallel-tests {path}" 154 | }, 155 | { 156 | "runner": "qunit", 157 | "cmd": "qunit {path}" 158 | }, 159 | { 160 | "runner": "tape", 161 | "cmd": "tape {path}/*.js" 162 | }, 163 | { 164 | "runner": "tap", 165 | "cmd": "tap {path} --jobs-auto" 166 | }, 167 | { 168 | "runner": "lab", 169 | "cmd": "lab --parallel {path}" 170 | }, 171 | { 172 | "runner": "ava", 173 | "cmd": "ava {path} --concurrency=4" 174 | }, 175 | { 176 | "label": "jest (jsdom)", 177 | "runner": "jest", 178 | "cmd": "jest --env=jsdom {path}" 179 | }, 180 | { 181 | "label": "jest (node)", 182 | "runner": "jest", 183 | "cmd": "jest --env=node {path}" 184 | } 185 | ] 186 | }, 187 | "runs": [ 188 | { 189 | "label": "lab", 190 | "time": "0.386" 191 | }, 192 | { 193 | "label": "mocha-parallel-tests", 194 | "time": "0.433" 195 | }, 196 | { 197 | "label": "mocha.parallel", 198 | "time": "0.504" 199 | }, 200 | { 201 | "label": "jasmine", 202 | "time": "0.594" 203 | }, 204 | { 205 | "label": "tape", 206 | "time": "0.614" 207 | }, 208 | { 209 | "label": "mocha", 210 | "time": "0.682" 211 | }, 212 | { 213 | "label": "jest (node)", 214 | "time": "1.96" 215 | }, 216 | { 217 | "label": "jest (jsdom)", 218 | "time": "3.12" 219 | }, 220 | { 221 | "label": "qunit", 222 | "time": "4.48" 223 | }, 224 | { 225 | "label": "tap", 226 | "time": "5.67" 227 | }, 228 | { 229 | "label": "ava", 230 | "time": "7.92" 231 | } 232 | ] 233 | }, 234 | "test=syncEmptyFn_nestedSuites=false_babel=true": { 235 | "name": "test=syncEmptyFn_nestedSuites=false_babel=true", 236 | "genConfig": { 237 | "basePath": "tests/unit", 238 | "files": 50, 239 | "nestedSuites": false, 240 | "hooks": false, 241 | "suitesInSuite": 2, 242 | "testsInSuite": 5, 243 | "test": "", 244 | "_path": [ 245 | "test=syncEmptyFn", 246 | "nestedSuites=false" 247 | ], 248 | "name": "test=syncEmptyFn_nestedSuites=false", 249 | "outPath": "tests/unit/test=syncEmptyFn/nestedSuites=false", 250 | "tests": 250 251 | }, 252 | "runConfig": { 253 | "name": "babel=true", 254 | "runs": [ 255 | { 256 | "runner": "mocha", 257 | "cmd": "mocha {path} --compilers js:babel-register" 258 | }, 259 | { 260 | "runner": "mocha-parallel-tests", 261 | "cmd": "mocha-parallel-tests {path} --compilers js:babel-register" 262 | }, 263 | { 264 | "runner": "jasmine", 265 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json", 266 | "babel": true 267 | }, 268 | { 269 | "runner": "lab", 270 | "cmd": "lab {path} -T node_modules/lab-babel" 271 | }, 272 | { 273 | "runner": "ava", 274 | "cmd": "ava {path} --concurrency=4" 275 | }, 276 | { 277 | "label": "jest (jsdom)", 278 | "runner": "jest", 279 | "cmd": "jest --env=jsdom {path}" 280 | }, 281 | { 282 | "label": "jest (node)", 283 | "runner": "jest", 284 | "cmd": "jest --env=node {path}" 285 | } 286 | ] 287 | }, 288 | "runs": [ 289 | { 290 | "label": "lab", 291 | "time": "1.09" 292 | }, 293 | { 294 | "label": "jasmine", 295 | "time": "1.75" 296 | }, 297 | { 298 | "label": "jest (node)", 299 | "time": "1.83" 300 | }, 301 | { 302 | "label": "mocha", 303 | "time": "2.23" 304 | }, 305 | { 306 | "label": "jest (jsdom)", 307 | "time": "3.07" 308 | }, 309 | { 310 | "label": "mocha-parallel-tests", 311 | "time": "4.27" 312 | }, 313 | { 314 | "label": "ava", 315 | "time": "8.17" 316 | } 317 | ] 318 | }, 319 | "test=syncHeavyFn_nestedSuites=false_babel=false": { 320 | "name": "test=syncHeavyFn_nestedSuites=false_babel=false", 321 | "genConfig": { 322 | "basePath": "tests/unit", 323 | "files": 50, 324 | "nestedSuites": false, 325 | "hooks": false, 326 | "suitesInSuite": 2, 327 | "testsInSuite": 5, 328 | "test": "\n eval(''); // disable V8 optimizations\n for (let i = 0; i < 10000; i++) {\n new Date();\n }\n ", 329 | "_path": [ 330 | "test=syncHeavyFn", 331 | "nestedSuites=false" 332 | ], 333 | "name": "test=syncHeavyFn_nestedSuites=false", 334 | "outPath": "tests/unit/test=syncHeavyFn/nestedSuites=false", 335 | "tests": 250 336 | }, 337 | "runConfig": { 338 | "name": "babel=false", 339 | "runs": [ 340 | { 341 | "runner": "mocha", 342 | "cmd": "mocha {path}" 343 | }, 344 | { 345 | "runner": "jasmine", 346 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json" 347 | }, 348 | { 349 | "runner": "mocha.parallel", 350 | "cmd": "mocha {path}" 351 | }, 352 | { 353 | "runner": "mocha-parallel-tests", 354 | "cmd": "mocha-parallel-tests {path}" 355 | }, 356 | { 357 | "runner": "qunit", 358 | "cmd": "qunit {path}" 359 | }, 360 | { 361 | "runner": "tape", 362 | "cmd": "tape {path}/*.js" 363 | }, 364 | { 365 | "runner": "tap", 366 | "cmd": "tap {path} --jobs-auto" 367 | }, 368 | { 369 | "runner": "lab", 370 | "cmd": "lab --parallel {path}" 371 | }, 372 | { 373 | "runner": "ava", 374 | "cmd": "ava {path} --concurrency=4" 375 | }, 376 | { 377 | "label": "jest (jsdom)", 378 | "runner": "jest", 379 | "cmd": "jest --env=jsdom {path}" 380 | }, 381 | { 382 | "label": "jest (node)", 383 | "runner": "jest", 384 | "cmd": "jest --env=node {path}" 385 | } 386 | ] 387 | }, 388 | "runs": [ 389 | { 390 | "label": "jasmine", 391 | "time": "0.676" 392 | }, 393 | { 394 | "label": "tape", 395 | "time": "0.707" 396 | }, 397 | { 398 | "label": "qunit", 399 | "time": "0.733" 400 | }, 401 | { 402 | "label": "lab", 403 | "time": "0.738" 404 | }, 405 | { 406 | "label": "mocha", 407 | "time": "0.744" 408 | }, 409 | { 410 | "label": "mocha.parallel", 411 | "time": "0.883" 412 | }, 413 | { 414 | "label": "mocha-parallel-tests", 415 | "time": "0.886" 416 | }, 417 | { 418 | "label": "jest (node)", 419 | "time": "2.52" 420 | }, 421 | { 422 | "label": "jest (jsdom)", 423 | "time": "3.73" 424 | }, 425 | { 426 | "label": "tap", 427 | "time": "5.85" 428 | }, 429 | { 430 | "label": "ava", 431 | "time": "8.71" 432 | } 433 | ] 434 | }, 435 | "test=syncHeavyFn_nestedSuites=false_babel=true": { 436 | "name": "test=syncHeavyFn_nestedSuites=false_babel=true", 437 | "genConfig": { 438 | "basePath": "tests/unit", 439 | "files": 50, 440 | "nestedSuites": false, 441 | "hooks": false, 442 | "suitesInSuite": 2, 443 | "testsInSuite": 5, 444 | "test": "\n eval(''); // disable V8 optimizations\n for (let i = 0; i < 10000; i++) {\n new Date();\n }\n ", 445 | "_path": [ 446 | "test=syncHeavyFn", 447 | "nestedSuites=false" 448 | ], 449 | "name": "test=syncHeavyFn_nestedSuites=false", 450 | "outPath": "tests/unit/test=syncHeavyFn/nestedSuites=false", 451 | "tests": 250 452 | }, 453 | "runConfig": { 454 | "name": "babel=true", 455 | "runs": [ 456 | { 457 | "runner": "mocha", 458 | "cmd": "mocha {path} --compilers js:babel-register" 459 | }, 460 | { 461 | "runner": "mocha-parallel-tests", 462 | "cmd": "mocha-parallel-tests {path} --compilers js:babel-register" 463 | }, 464 | { 465 | "runner": "jasmine", 466 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json", 467 | "babel": true 468 | }, 469 | { 470 | "runner": "lab", 471 | "cmd": "lab {path} -T node_modules/lab-babel" 472 | }, 473 | { 474 | "runner": "ava", 475 | "cmd": "ava {path} --concurrency=4" 476 | }, 477 | { 478 | "label": "jest (jsdom)", 479 | "runner": "jest", 480 | "cmd": "jest --env=jsdom {path}" 481 | }, 482 | { 483 | "label": "jest (node)", 484 | "runner": "jest", 485 | "cmd": "jest --env=node {path}" 486 | } 487 | ] 488 | }, 489 | "runs": [ 490 | { 491 | "label": "lab", 492 | "time": "1.64" 493 | }, 494 | { 495 | "label": "jasmine", 496 | "time": "2.16" 497 | }, 498 | { 499 | "label": "mocha", 500 | "time": "2.26" 501 | }, 502 | { 503 | "label": "jest (node)", 504 | "time": "2.52" 505 | }, 506 | { 507 | "label": "jest (jsdom)", 508 | "time": "3.69" 509 | }, 510 | { 511 | "label": "mocha-parallel-tests", 512 | "time": "4.69" 513 | }, 514 | { 515 | "label": "ava", 516 | "time": "8.32" 517 | } 518 | ] 519 | }, 520 | "test=asyncEmptyFnZeroDelay_nestedSuites=false_babel=true": { 521 | "name": "test=asyncEmptyFnZeroDelay_nestedSuites=false_babel=true", 522 | "genConfig": { 523 | "basePath": "tests/unit", 524 | "files": 50, 525 | "nestedSuites": false, 526 | "hooks": false, 527 | "suitesInSuite": 2, 528 | "testsInSuite": 5, 529 | "test": "setTimeout(done, 0)", 530 | "_path": [ 531 | "test=asyncEmptyFnZeroDelay", 532 | "nestedSuites=false" 533 | ], 534 | "name": "test=asyncEmptyFnZeroDelay_nestedSuites=false", 535 | "outPath": "tests/unit/test=asyncEmptyFnZeroDelay/nestedSuites=false", 536 | "tests": 250 537 | }, 538 | "runConfig": { 539 | "name": "babel=true", 540 | "runs": [ 541 | { 542 | "runner": "mocha", 543 | "cmd": "mocha {path} --compilers js:babel-register" 544 | }, 545 | { 546 | "runner": "mocha-parallel-tests", 547 | "cmd": "mocha-parallel-tests {path} --compilers js:babel-register" 548 | }, 549 | { 550 | "runner": "jasmine", 551 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json", 552 | "babel": true 553 | }, 554 | { 555 | "runner": "lab", 556 | "cmd": "lab {path} -T node_modules/lab-babel" 557 | }, 558 | { 559 | "runner": "ava", 560 | "cmd": "ava {path} --concurrency=4" 561 | }, 562 | { 563 | "label": "jest (jsdom)", 564 | "runner": "jest", 565 | "cmd": "jest --env=jsdom {path}" 566 | }, 567 | { 568 | "label": "jest (node)", 569 | "runner": "jest", 570 | "cmd": "jest --env=node {path}" 571 | } 572 | ] 573 | }, 574 | "runs": [ 575 | { 576 | "label": "lab", 577 | "time": "1.43" 578 | }, 579 | { 580 | "label": "jest (node)", 581 | "time": "1.92" 582 | }, 583 | { 584 | "label": "jasmine", 585 | "time": "2.19" 586 | }, 587 | { 588 | "label": "mocha", 589 | "time": "2.34" 590 | }, 591 | { 592 | "label": "jest (jsdom)", 593 | "time": "2.99" 594 | }, 595 | { 596 | "label": "mocha-parallel-tests", 597 | "time": "4.21" 598 | }, 599 | { 600 | "label": "ava", 601 | "time": "8.00" 602 | } 603 | ] 604 | }, 605 | "test=asyncEmptyFnRandomDelay_nestedSuites=false_babel=false": { 606 | "name": "test=asyncEmptyFnRandomDelay_nestedSuites=false_babel=false", 607 | "genConfig": { 608 | "basePath": "tests/unit", 609 | "files": 50, 610 | "nestedSuites": false, 611 | "hooks": false, 612 | "suitesInSuite": 2, 613 | "testsInSuite": 5, 614 | "test": "setTimeout(done, Math.round({random} * 10))", 615 | "_path": [ 616 | "test=asyncEmptyFnRandomDelay", 617 | "nestedSuites=false" 618 | ], 619 | "name": "test=asyncEmptyFnRandomDelay_nestedSuites=false", 620 | "outPath": "tests/unit/test=asyncEmptyFnRandomDelay/nestedSuites=false", 621 | "tests": 250 622 | }, 623 | "runConfig": { 624 | "name": "babel=false", 625 | "runs": [ 626 | { 627 | "runner": "mocha", 628 | "cmd": "mocha {path}" 629 | }, 630 | { 631 | "runner": "jasmine", 632 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json" 633 | }, 634 | { 635 | "runner": "mocha.parallel", 636 | "cmd": "mocha {path}" 637 | }, 638 | { 639 | "runner": "mocha-parallel-tests", 640 | "cmd": "mocha-parallel-tests {path}" 641 | }, 642 | { 643 | "runner": "qunit", 644 | "cmd": "qunit {path}" 645 | }, 646 | { 647 | "runner": "tape", 648 | "cmd": "tape {path}/*.js" 649 | }, 650 | { 651 | "runner": "tap", 652 | "cmd": "tap {path} --jobs-auto" 653 | }, 654 | { 655 | "runner": "lab", 656 | "cmd": "lab --parallel {path}" 657 | }, 658 | { 659 | "runner": "ava", 660 | "cmd": "ava {path} --concurrency=4" 661 | }, 662 | { 663 | "label": "jest (jsdom)", 664 | "runner": "jest", 665 | "cmd": "jest --env=jsdom {path}" 666 | }, 667 | { 668 | "label": "jest (node)", 669 | "runner": "jest", 670 | "cmd": "jest --env=node {path}" 671 | } 672 | ] 673 | }, 674 | "runs": [ 675 | { 676 | "label": "mocha-parallel-tests", 677 | "time": "0.450" 678 | }, 679 | { 680 | "label": "lab", 681 | "time": "0.777" 682 | }, 683 | { 684 | "label": "mocha.parallel", 685 | "time": "0.884" 686 | }, 687 | { 688 | "label": "jasmine", 689 | "time": "1.86" 690 | }, 691 | { 692 | "label": "tape", 693 | "time": "1.91" 694 | }, 695 | { 696 | "label": "mocha", 697 | "time": "1.99" 698 | }, 699 | { 700 | "label": "jest (node)", 701 | "time": "2.32" 702 | }, 703 | { 704 | "label": "jest (jsdom)", 705 | "time": "3.52" 706 | }, 707 | { 708 | "label": "tap", 709 | "time": "5.71" 710 | }, 711 | { 712 | "label": "qunit", 713 | "time": "5.72" 714 | }, 715 | { 716 | "label": "ava", 717 | "time": "7.98" 718 | } 719 | ] 720 | }, 721 | "test=asyncEmptyFnRandomDelay_nestedSuites=false_babel=true": { 722 | "name": "test=asyncEmptyFnRandomDelay_nestedSuites=false_babel=true", 723 | "genConfig": { 724 | "basePath": "tests/unit", 725 | "files": 50, 726 | "nestedSuites": false, 727 | "hooks": false, 728 | "suitesInSuite": 2, 729 | "testsInSuite": 5, 730 | "test": "setTimeout(done, Math.round({random} * 10))", 731 | "_path": [ 732 | "test=asyncEmptyFnRandomDelay", 733 | "nestedSuites=false" 734 | ], 735 | "name": "test=asyncEmptyFnRandomDelay_nestedSuites=false", 736 | "outPath": "tests/unit/test=asyncEmptyFnRandomDelay/nestedSuites=false", 737 | "tests": 250 738 | }, 739 | "runConfig": { 740 | "name": "babel=true", 741 | "runs": [ 742 | { 743 | "runner": "mocha", 744 | "cmd": "mocha {path} --compilers js:babel-register" 745 | }, 746 | { 747 | "runner": "mocha-parallel-tests", 748 | "cmd": "mocha-parallel-tests {path} --compilers js:babel-register" 749 | }, 750 | { 751 | "runner": "jasmine", 752 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json", 753 | "babel": true 754 | }, 755 | { 756 | "runner": "lab", 757 | "cmd": "lab {path} -T node_modules/lab-babel" 758 | }, 759 | { 760 | "runner": "ava", 761 | "cmd": "ava {path} --concurrency=4" 762 | }, 763 | { 764 | "label": "jest (jsdom)", 765 | "runner": "jest", 766 | "cmd": "jest --env=jsdom {path}" 767 | }, 768 | { 769 | "label": "jest (node)", 770 | "runner": "jest", 771 | "cmd": "jest --env=node {path}" 772 | } 773 | ] 774 | }, 775 | "runs": [ 776 | { 777 | "label": "jest (node)", 778 | "time": "2.34" 779 | }, 780 | { 781 | "label": "lab", 782 | "time": "2.72" 783 | }, 784 | { 785 | "label": "jest (jsdom)", 786 | "time": "3.39" 787 | }, 788 | { 789 | "label": "jasmine", 790 | "time": "3.45" 791 | }, 792 | { 793 | "label": "mocha", 794 | "time": "3.52" 795 | }, 796 | { 797 | "label": "mocha-parallel-tests", 798 | "time": "4.26" 799 | }, 800 | { 801 | "label": "ava", 802 | "time": "7.92" 803 | } 804 | ] 805 | }, 806 | "test=syncEmptyFn_nestedSuites=true_babel=false": { 807 | "name": "test=syncEmptyFn_nestedSuites=true_babel=false", 808 | "genConfig": { 809 | "basePath": "tests/unit", 810 | "files": 50, 811 | "nestedSuites": true, 812 | "hooks": false, 813 | "suitesInSuite": 2, 814 | "testsInSuite": 5, 815 | "test": "", 816 | "_path": [ 817 | "test=syncEmptyFn", 818 | "nestedSuites=true" 819 | ], 820 | "name": "test=syncEmptyFn_nestedSuites=true", 821 | "outPath": "tests/unit/test=syncEmptyFn/nestedSuites=true", 822 | "tests": 500 823 | }, 824 | "runConfig": { 825 | "name": "babel=false", 826 | "runs": [ 827 | { 828 | "runner": "mocha", 829 | "cmd": "mocha {path}" 830 | }, 831 | { 832 | "runner": "jasmine", 833 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json" 834 | }, 835 | { 836 | "runner": "mocha.parallel", 837 | "cmd": "mocha {path}" 838 | }, 839 | { 840 | "runner": "mocha-parallel-tests", 841 | "cmd": "mocha-parallel-tests {path}" 842 | }, 843 | { 844 | "runner": "qunit", 845 | "cmd": "qunit {path}" 846 | }, 847 | { 848 | "runner": "tape", 849 | "cmd": "tape {path}/*.js" 850 | }, 851 | { 852 | "runner": "tap", 853 | "cmd": "tap {path} --jobs-auto" 854 | }, 855 | { 856 | "runner": "lab", 857 | "cmd": "lab --parallel {path}" 858 | }, 859 | { 860 | "runner": "ava", 861 | "cmd": "ava {path} --concurrency=4" 862 | }, 863 | { 864 | "label": "jest (jsdom)", 865 | "runner": "jest", 866 | "cmd": "jest --env=jsdom {path}" 867 | }, 868 | { 869 | "label": "jest (node)", 870 | "runner": "jest", 871 | "cmd": "jest --env=node {path}" 872 | } 873 | ] 874 | }, 875 | "runs": [ 876 | { 877 | "label": "jasmine", 878 | "time": "0.233" 879 | }, 880 | { 881 | "label": "mocha", 882 | "time": "0.332" 883 | }, 884 | { 885 | "label": "qunit", 886 | "time": "0.344" 887 | }, 888 | { 889 | "label": "mocha-parallel-tests", 890 | "time": "0.495" 891 | }, 892 | { 893 | "label": "jest (node)", 894 | "time": "1.97" 895 | }, 896 | { 897 | "label": "jest (jsdom)", 898 | "time": "3.00" 899 | }, 900 | { 901 | "label": "tap", 902 | "time": "6.50" 903 | } 904 | ] 905 | }, 906 | "test=syncEmptyFn_nestedSuites=true_babel=true": { 907 | "name": "test=syncEmptyFn_nestedSuites=true_babel=true", 908 | "genConfig": { 909 | "basePath": "tests/unit", 910 | "files": 50, 911 | "nestedSuites": true, 912 | "hooks": false, 913 | "suitesInSuite": 2, 914 | "testsInSuite": 5, 915 | "test": "", 916 | "_path": [ 917 | "test=syncEmptyFn", 918 | "nestedSuites=true" 919 | ], 920 | "name": "test=syncEmptyFn_nestedSuites=true", 921 | "outPath": "tests/unit/test=syncEmptyFn/nestedSuites=true", 922 | "tests": 500 923 | }, 924 | "runConfig": { 925 | "name": "babel=true", 926 | "runs": [ 927 | { 928 | "runner": "mocha", 929 | "cmd": "mocha {path} --compilers js:babel-register" 930 | }, 931 | { 932 | "runner": "mocha-parallel-tests", 933 | "cmd": "mocha-parallel-tests {path} --compilers js:babel-register" 934 | }, 935 | { 936 | "runner": "jasmine", 937 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json", 938 | "babel": true 939 | }, 940 | { 941 | "runner": "lab", 942 | "cmd": "lab {path} -T node_modules/lab-babel" 943 | }, 944 | { 945 | "runner": "ava", 946 | "cmd": "ava {path} --concurrency=4" 947 | }, 948 | { 949 | "label": "jest (jsdom)", 950 | "runner": "jest", 951 | "cmd": "jest --env=jsdom {path}" 952 | }, 953 | { 954 | "label": "jest (node)", 955 | "runner": "jest", 956 | "cmd": "jest --env=node {path}" 957 | } 958 | ] 959 | }, 960 | "runs": [ 961 | { 962 | "label": "jasmine", 963 | "time": "1.80" 964 | }, 965 | { 966 | "label": "jest (node)", 967 | "time": "1.87" 968 | }, 969 | { 970 | "label": "mocha", 971 | "time": "1.89" 972 | }, 973 | { 974 | "label": "jest (jsdom)", 975 | "time": "2.97" 976 | }, 977 | { 978 | "label": "mocha-parallel-tests", 979 | "time": "4.47" 980 | } 981 | ] 982 | }, 983 | "test=syncHeavyFn_nestedSuites=true_babel=false": { 984 | "name": "test=syncHeavyFn_nestedSuites=true_babel=false", 985 | "genConfig": { 986 | "basePath": "tests/unit", 987 | "files": 50, 988 | "nestedSuites": true, 989 | "hooks": false, 990 | "suitesInSuite": 2, 991 | "testsInSuite": 5, 992 | "test": "\n eval(''); // disable V8 optimizations\n for (let i = 0; i < 10000; i++) {\n new Date();\n }\n ", 993 | "_path": [ 994 | "test=syncHeavyFn", 995 | "nestedSuites=true" 996 | ], 997 | "name": "test=syncHeavyFn_nestedSuites=true", 998 | "outPath": "tests/unit/test=syncHeavyFn/nestedSuites=true", 999 | "tests": 500 1000 | }, 1001 | "runConfig": { 1002 | "name": "babel=false", 1003 | "runs": [ 1004 | { 1005 | "runner": "mocha", 1006 | "cmd": "mocha {path}" 1007 | }, 1008 | { 1009 | "runner": "jasmine", 1010 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json" 1011 | }, 1012 | { 1013 | "runner": "mocha.parallel", 1014 | "cmd": "mocha {path}" 1015 | }, 1016 | { 1017 | "runner": "mocha-parallel-tests", 1018 | "cmd": "mocha-parallel-tests {path}" 1019 | }, 1020 | { 1021 | "runner": "qunit", 1022 | "cmd": "qunit {path}" 1023 | }, 1024 | { 1025 | "runner": "tape", 1026 | "cmd": "tape {path}/*.js" 1027 | }, 1028 | { 1029 | "runner": "tap", 1030 | "cmd": "tap {path} --jobs-auto" 1031 | }, 1032 | { 1033 | "runner": "lab", 1034 | "cmd": "lab --parallel {path}" 1035 | }, 1036 | { 1037 | "runner": "ava", 1038 | "cmd": "ava {path} --concurrency=4" 1039 | }, 1040 | { 1041 | "label": "jest (jsdom)", 1042 | "runner": "jest", 1043 | "cmd": "jest --env=jsdom {path}" 1044 | }, 1045 | { 1046 | "label": "jest (node)", 1047 | "runner": "jest", 1048 | "cmd": "jest --env=node {path}" 1049 | } 1050 | ] 1051 | }, 1052 | "runs": [ 1053 | { 1054 | "label": "jasmine", 1055 | "time": "1.15" 1056 | }, 1057 | { 1058 | "label": "qunit", 1059 | "time": "1.21" 1060 | }, 1061 | { 1062 | "label": "mocha", 1063 | "time": "1.25" 1064 | }, 1065 | { 1066 | "label": "mocha-parallel-tests", 1067 | "time": "1.37" 1068 | }, 1069 | { 1070 | "label": "jest (node)", 1071 | "time": "4.45" 1072 | }, 1073 | { 1074 | "label": "jest (jsdom)", 1075 | "time": "5.61" 1076 | }, 1077 | { 1078 | "label": "tap", 1079 | "time": "7.11" 1080 | } 1081 | ] 1082 | }, 1083 | "test=syncHeavyFn_nestedSuites=true_babel=true": { 1084 | "name": "test=syncHeavyFn_nestedSuites=true_babel=true", 1085 | "genConfig": { 1086 | "basePath": "tests/unit", 1087 | "files": 50, 1088 | "nestedSuites": true, 1089 | "hooks": false, 1090 | "suitesInSuite": 2, 1091 | "testsInSuite": 5, 1092 | "test": "\n eval(''); // disable V8 optimizations\n for (let i = 0; i < 10000; i++) {\n new Date();\n }\n ", 1093 | "_path": [ 1094 | "test=syncHeavyFn", 1095 | "nestedSuites=true" 1096 | ], 1097 | "name": "test=syncHeavyFn_nestedSuites=true", 1098 | "outPath": "tests/unit/test=syncHeavyFn/nestedSuites=true", 1099 | "tests": 500 1100 | }, 1101 | "runConfig": { 1102 | "name": "babel=true", 1103 | "runs": [ 1104 | { 1105 | "runner": "mocha", 1106 | "cmd": "mocha {path} --compilers js:babel-register" 1107 | }, 1108 | { 1109 | "runner": "mocha-parallel-tests", 1110 | "cmd": "mocha-parallel-tests {path} --compilers js:babel-register" 1111 | }, 1112 | { 1113 | "runner": "jasmine", 1114 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json", 1115 | "babel": true 1116 | }, 1117 | { 1118 | "runner": "lab", 1119 | "cmd": "lab {path} -T node_modules/lab-babel" 1120 | }, 1121 | { 1122 | "runner": "ava", 1123 | "cmd": "ava {path} --concurrency=4" 1124 | }, 1125 | { 1126 | "label": "jest (jsdom)", 1127 | "runner": "jest", 1128 | "cmd": "jest --env=jsdom {path}" 1129 | }, 1130 | { 1131 | "label": "jest (node)", 1132 | "runner": "jest", 1133 | "cmd": "jest --env=node {path}" 1134 | } 1135 | ] 1136 | }, 1137 | "runs": [ 1138 | { 1139 | "label": "jest (node)", 1140 | "time": "3.27" 1141 | }, 1142 | { 1143 | "label": "jasmine", 1144 | "time": "3.44" 1145 | }, 1146 | { 1147 | "label": "mocha", 1148 | "time": "3.57" 1149 | }, 1150 | { 1151 | "label": "jest (jsdom)", 1152 | "time": "4.36" 1153 | }, 1154 | { 1155 | "label": "mocha-parallel-tests", 1156 | "time": "6.22" 1157 | } 1158 | ] 1159 | }, 1160 | "test=asyncEmptyFnZeroDelay_nestedSuites=true_babel=false": { 1161 | "name": "test=asyncEmptyFnZeroDelay_nestedSuites=true_babel=false", 1162 | "genConfig": { 1163 | "basePath": "tests/unit", 1164 | "files": 50, 1165 | "nestedSuites": true, 1166 | "hooks": false, 1167 | "suitesInSuite": 2, 1168 | "testsInSuite": 5, 1169 | "test": "setTimeout(done, 0)", 1170 | "_path": [ 1171 | "test=asyncEmptyFnZeroDelay", 1172 | "nestedSuites=true" 1173 | ], 1174 | "name": "test=asyncEmptyFnZeroDelay_nestedSuites=true", 1175 | "outPath": "tests/unit/test=asyncEmptyFnZeroDelay/nestedSuites=true", 1176 | "tests": 500 1177 | }, 1178 | "runConfig": { 1179 | "name": "babel=false", 1180 | "runs": [ 1181 | { 1182 | "runner": "mocha", 1183 | "cmd": "mocha {path}" 1184 | }, 1185 | { 1186 | "runner": "jasmine", 1187 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json" 1188 | }, 1189 | { 1190 | "runner": "mocha.parallel", 1191 | "cmd": "mocha {path}" 1192 | }, 1193 | { 1194 | "runner": "mocha-parallel-tests", 1195 | "cmd": "mocha-parallel-tests {path}" 1196 | }, 1197 | { 1198 | "runner": "qunit", 1199 | "cmd": "qunit {path}" 1200 | }, 1201 | { 1202 | "runner": "tape", 1203 | "cmd": "tape {path}/*.js" 1204 | }, 1205 | { 1206 | "runner": "tap", 1207 | "cmd": "tap {path} --jobs-auto" 1208 | }, 1209 | { 1210 | "runner": "lab", 1211 | "cmd": "lab --parallel {path}" 1212 | }, 1213 | { 1214 | "runner": "ava", 1215 | "cmd": "ava {path} --concurrency=4" 1216 | }, 1217 | { 1218 | "label": "jest (jsdom)", 1219 | "runner": "jest", 1220 | "cmd": "jest --env=jsdom {path}" 1221 | }, 1222 | { 1223 | "label": "jest (node)", 1224 | "runner": "jest", 1225 | "cmd": "jest --env=node {path}" 1226 | } 1227 | ] 1228 | }, 1229 | "runs": [ 1230 | { 1231 | "label": "mocha-parallel-tests", 1232 | "time": "0.521" 1233 | }, 1234 | { 1235 | "label": "mocha", 1236 | "time": "1.07" 1237 | }, 1238 | { 1239 | "label": "jasmine", 1240 | "time": "1.10" 1241 | }, 1242 | { 1243 | "label": "jest (node)", 1244 | "time": "2.96" 1245 | }, 1246 | { 1247 | "label": "jest (jsdom)", 1248 | "time": "4.05" 1249 | }, 1250 | { 1251 | "label": "tap", 1252 | "time": "7.09" 1253 | }, 1254 | { 1255 | "label": "qunit", 1256 | "time": "8.78" 1257 | } 1258 | ] 1259 | }, 1260 | "test=asyncEmptyFnZeroDelay_nestedSuites=true_babel=true": { 1261 | "name": "test=asyncEmptyFnZeroDelay_nestedSuites=true_babel=true", 1262 | "genConfig": { 1263 | "basePath": "tests/unit", 1264 | "files": 50, 1265 | "nestedSuites": true, 1266 | "hooks": false, 1267 | "suitesInSuite": 2, 1268 | "testsInSuite": 5, 1269 | "test": "setTimeout(done, 0)", 1270 | "_path": [ 1271 | "test=asyncEmptyFnZeroDelay", 1272 | "nestedSuites=true" 1273 | ], 1274 | "name": "test=asyncEmptyFnZeroDelay_nestedSuites=true", 1275 | "outPath": "tests/unit/test=asyncEmptyFnZeroDelay/nestedSuites=true", 1276 | "tests": 500 1277 | }, 1278 | "runConfig": { 1279 | "name": "babel=true", 1280 | "runs": [ 1281 | { 1282 | "runner": "mocha", 1283 | "cmd": "mocha {path} --compilers js:babel-register" 1284 | }, 1285 | { 1286 | "runner": "mocha-parallel-tests", 1287 | "cmd": "mocha-parallel-tests {path} --compilers js:babel-register" 1288 | }, 1289 | { 1290 | "runner": "jasmine", 1291 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json", 1292 | "babel": true 1293 | }, 1294 | { 1295 | "runner": "lab", 1296 | "cmd": "lab {path} -T node_modules/lab-babel" 1297 | }, 1298 | { 1299 | "runner": "ava", 1300 | "cmd": "ava {path} --concurrency=4" 1301 | }, 1302 | { 1303 | "label": "jest (jsdom)", 1304 | "runner": "jest", 1305 | "cmd": "jest --env=jsdom {path}" 1306 | }, 1307 | { 1308 | "label": "jest (node)", 1309 | "runner": "jest", 1310 | "cmd": "jest --env=node {path}" 1311 | } 1312 | ] 1313 | }, 1314 | "runs": [ 1315 | { 1316 | "label": "jest (node)", 1317 | "time": "2.00" 1318 | }, 1319 | { 1320 | "label": "jest (jsdom)", 1321 | "time": "3.02" 1322 | }, 1323 | { 1324 | "label": "jasmine", 1325 | "time": "3.20" 1326 | }, 1327 | { 1328 | "label": "mocha", 1329 | "time": "3.26" 1330 | }, 1331 | { 1332 | "label": "mocha-parallel-tests", 1333 | "time": "5.10" 1334 | } 1335 | ] 1336 | }, 1337 | "test=asyncEmptyFnRandomDelay_nestedSuites=true_babel=false": { 1338 | "name": "test=asyncEmptyFnRandomDelay_nestedSuites=true_babel=false", 1339 | "genConfig": { 1340 | "basePath": "tests/unit", 1341 | "files": 50, 1342 | "nestedSuites": true, 1343 | "hooks": false, 1344 | "suitesInSuite": 2, 1345 | "testsInSuite": 5, 1346 | "test": "setTimeout(done, Math.round({random} * 10))", 1347 | "_path": [ 1348 | "test=asyncEmptyFnRandomDelay", 1349 | "nestedSuites=true" 1350 | ], 1351 | "name": "test=asyncEmptyFnRandomDelay_nestedSuites=true", 1352 | "outPath": "tests/unit/test=asyncEmptyFnRandomDelay/nestedSuites=true", 1353 | "tests": 500 1354 | }, 1355 | "runConfig": { 1356 | "name": "babel=false", 1357 | "runs": [ 1358 | { 1359 | "runner": "mocha", 1360 | "cmd": "mocha {path}" 1361 | }, 1362 | { 1363 | "runner": "jasmine", 1364 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json" 1365 | }, 1366 | { 1367 | "runner": "mocha.parallel", 1368 | "cmd": "mocha {path}" 1369 | }, 1370 | { 1371 | "runner": "mocha-parallel-tests", 1372 | "cmd": "mocha-parallel-tests {path}" 1373 | }, 1374 | { 1375 | "runner": "qunit", 1376 | "cmd": "qunit {path}" 1377 | }, 1378 | { 1379 | "runner": "tape", 1380 | "cmd": "tape {path}/*.js" 1381 | }, 1382 | { 1383 | "runner": "tap", 1384 | "cmd": "tap {path} --jobs-auto" 1385 | }, 1386 | { 1387 | "runner": "lab", 1388 | "cmd": "lab --parallel {path}" 1389 | }, 1390 | { 1391 | "runner": "ava", 1392 | "cmd": "ava {path} --concurrency=4" 1393 | }, 1394 | { 1395 | "label": "jest (jsdom)", 1396 | "runner": "jest", 1397 | "cmd": "jest --env=jsdom {path}" 1398 | }, 1399 | { 1400 | "label": "jest (node)", 1401 | "runner": "jest", 1402 | "cmd": "jest --env=node {path}" 1403 | } 1404 | ] 1405 | }, 1406 | "runs": [ 1407 | { 1408 | "label": "mocha-parallel-tests", 1409 | "time": "0.537" 1410 | }, 1411 | { 1412 | "label": "jasmine", 1413 | "time": "3.40" 1414 | }, 1415 | { 1416 | "label": "mocha", 1417 | "time": "3.50" 1418 | }, 1419 | { 1420 | "label": "jest (node)", 1421 | "time": "3.65" 1422 | }, 1423 | { 1424 | "label": "jest (jsdom)", 1425 | "time": "4.77" 1426 | }, 1427 | { 1428 | "label": "tap", 1429 | "time": "6.67" 1430 | }, 1431 | { 1432 | "label": "qunit", 1433 | "time": "11.2" 1434 | } 1435 | ] 1436 | }, 1437 | "test=asyncEmptyFnRandomDelay_nestedSuites=true_babel=true": { 1438 | "name": "test=asyncEmptyFnRandomDelay_nestedSuites=true_babel=true", 1439 | "genConfig": { 1440 | "basePath": "tests/unit", 1441 | "files": 50, 1442 | "nestedSuites": true, 1443 | "hooks": false, 1444 | "suitesInSuite": 2, 1445 | "testsInSuite": 5, 1446 | "test": "setTimeout(done, Math.round({random} * 10))", 1447 | "_path": [ 1448 | "test=asyncEmptyFnRandomDelay", 1449 | "nestedSuites=true" 1450 | ], 1451 | "name": "test=asyncEmptyFnRandomDelay_nestedSuites=true", 1452 | "outPath": "tests/unit/test=asyncEmptyFnRandomDelay/nestedSuites=true", 1453 | "tests": 500 1454 | }, 1455 | "runConfig": { 1456 | "name": "babel=true", 1457 | "runs": [ 1458 | { 1459 | "runner": "mocha", 1460 | "cmd": "mocha {path} --compilers js:babel-register" 1461 | }, 1462 | { 1463 | "runner": "mocha-parallel-tests", 1464 | "cmd": "mocha-parallel-tests {path} --compilers js:babel-register" 1465 | }, 1466 | { 1467 | "runner": "jasmine", 1468 | "cmd": "jasmine JASMINE_CONFIG_PATH=temp/jasmine.json", 1469 | "babel": true 1470 | }, 1471 | { 1472 | "runner": "lab", 1473 | "cmd": "lab {path} -T node_modules/lab-babel" 1474 | }, 1475 | { 1476 | "runner": "ava", 1477 | "cmd": "ava {path} --concurrency=4" 1478 | }, 1479 | { 1480 | "label": "jest (jsdom)", 1481 | "runner": "jest", 1482 | "cmd": "jest --env=jsdom {path}" 1483 | }, 1484 | { 1485 | "label": "jest (node)", 1486 | "runner": "jest", 1487 | "cmd": "jest --env=node {path}" 1488 | } 1489 | ] 1490 | }, 1491 | "runs": [ 1492 | { 1493 | "label": "jest (node)", 1494 | "time": "2.93" 1495 | }, 1496 | { 1497 | "label": "jest (jsdom)", 1498 | "time": "4.08" 1499 | }, 1500 | { 1501 | "label": "mocha-parallel-tests", 1502 | "time": "5.13" 1503 | }, 1504 | { 1505 | "label": "jasmine", 1506 | "time": "5.19" 1507 | }, 1508 | { 1509 | "label": "mocha", 1510 | "time": "5.35" 1511 | } 1512 | ] 1513 | } 1514 | } --------------------------------------------------------------------------------