├── .github └── workflows │ └── ci.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE.md ├── README.md ├── eslint.config.js ├── package.json ├── prettier.config.js ├── src ├── index.ts ├── models.ts ├── print.ts ├── runner.ts ├── tracker.ts └── worker.ts ├── test ├── asyncImport.test.ts ├── callbacks.test.ts ├── config │ ├── c8-ci.json │ ├── c8-local.json │ └── env ├── errorHandling.test.ts ├── fixture │ └── sample.ts ├── genericErrorHandling.test.ts ├── notTestExportedInWorkers.test.ts ├── optionsValidation.test.ts ├── print.test.ts ├── success.test.ts ├── testsCallbacks.test.ts ├── unhandledErrorHandling.test.ts └── worker.test.ts ├── tsconfig.json └── tsconfig.test.json /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: CI 3 | on: [push, pull_request, workflow_dispatch] 4 | jobs: 5 | ci: 6 | strategy: 7 | fail-fast: false 8 | matrix: 9 | os: [ubuntu-latest, windows-latest] 10 | node-version: [22, 23] 11 | runs-on: ${{ matrix.os }} 12 | steps: 13 | - name: Checkout 14 | uses: actions/checkout@v4 15 | - name: Use supported Node.js Version 16 | uses: actions/setup-node@v4 17 | with: 18 | node-version: ${{ matrix.node-version }} 19 | - name: Restore cached dependencies 20 | uses: actions/cache@v3 21 | with: 22 | path: ~/.pnpm-store 23 | key: node-modules-${{ hashFiles('package.json') }} 24 | - name: Setup pnpm 25 | uses: pnpm/action-setup@v2 26 | with: 27 | version: latest 28 | - name: Install dependencies 29 | run: pnpm install --shamefully-hoist 30 | - name: Run Tests 31 | run: pnpm run ci 32 | - name: Upload coverage to Codecov 33 | uses: codecov/codecov-action@v3 34 | with: 35 | file: ./coverage/coverage-final.json 36 | token: ${{ secrets.CODECOV_TOKEN }} 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | types/ 3 | coverage/ 4 | node_modules/ 5 | .eslintcache -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ### 2025-04-18 / 5.3.0 2 | 3 | - feat: Export Tracker. 4 | 5 | ### 2025-04-18 / 5.2.0 6 | 7 | - feat: Use tracker. 8 | 9 | ### 2025-04-18 / 5.1.0 10 | 11 | - feat: Export printResults method. 12 | 13 | ### 2025-04-18 / 5.0.0 14 | 15 | - fix: Fixed Windows compatibility. 16 | - feat!: Dropped compatibility for Node 20.x. 17 | - feat: Added skip option. 18 | 19 | ### 2025-01-19 / 4.0.3 20 | 21 | - feat: Merge pull request #7 from simoneb/main 22 | - chore: support windows 23 | 24 | ### 2025-01-12 / 4.0.2 25 | 26 | - fix: Better error handling on no test code. 27 | 28 | ### 2024-12-28 / 4.0.1 29 | 30 | - chore: Updated dependencies. 31 | 32 | ### 2024-10-21 / 4.0.0 33 | 34 | - feat!: Dropped support for Node 18 and updated dependencies. 35 | - fix: Fixed README example. 36 | - fix: Remove leftover. 37 | 38 | ### 2024-04-12 / 3.0.2 39 | 40 | - chore: Updated dependencies. 41 | - chore: Added missing dependency. 42 | 43 | ### 2024-02-07 / 3.0.1 44 | 45 | - chore: Updated dependencies. 46 | 47 | ### 2024-02-07 / 3.0.0 48 | 49 | - feat!: Replaced ts-node with @swc-node/register. 50 | 51 | ### 2024-01-27 / 2.0.2 52 | 53 | - chore: Updated dependencies. 54 | 55 | ### 2024-01-24 / 2.0.1 56 | 57 | - chore: Updated TypeScript configuration. 58 | 59 | ### 2023-12-20 / 2.0.0 60 | 61 | - chore: Updated dependencies. 62 | 63 | ### 2023-10-23 / 1.2.0 64 | 65 | - chore: Linted code. 66 | - chore: Updated dependencies and toolchain. 67 | - chore: Fixed compilation. 68 | - chore: CI improvement 69 | 70 | ### 2022-11-23 / 1.1.5 71 | 72 | - chore: Updated dependencies. 73 | - chore: Update package.json 74 | - fix: Fixed build script. 75 | 76 | ### 2022-10-12 / 1.1.4 77 | 78 | - chore: Updated dependencies. 79 | 80 | ### 2022-10-12 / 1.1.3 81 | 82 | - fix: Updated types layout. 83 | - chore: Updated compilation configuration. 84 | - chore: Remove lint rule. 85 | 86 | ### 2022-08-30 / 1.1.2 87 | 88 | - chore: Updated dependencies. 89 | 90 | ### 2022-08-29 / 1.1.1 91 | 92 | - chore: Updated dependencies. 93 | 94 | ### 2022-05-19 / 1.1.0 95 | 96 | - chore: Fixed CI. 97 | - feat: Added callbacks. 98 | - chore: Use sourcemaps with swc 99 | 100 | ### 2022-03-07 / 1.0.5 101 | 102 | - chore: Updated dependencies. 103 | 104 | ### 2022-03-07 / 1.0.4 105 | 106 | - chore: Updated build system. 107 | 108 | ### 2022-01-26 / 1.0.3 109 | 110 | - chore: Updated dependencies and linted code. 111 | - chore: Updated dependencies. 112 | - chore: Removed useless file. 113 | 114 | ### 2021-11-17 / 1.0.2 115 | 116 | 117 | ### 2021-11-16 / 1.0.1 118 | 119 | - fix: Added ESM note in the README.md 120 | - chore: Allow manual CI triggering. 121 | - chore: Updated badges. 122 | - fix: Fixed Typescript configuration. 123 | - chore: Renamed test files. 124 | - chore: Removed useless comments. 125 | 126 | ### 2021-08-14 / 1.0.0 127 | 128 | 129 | ### 2021-08-14 / 1.0.0-beta.1 130 | 131 | - feat: Allow main module to export a function. 132 | 133 | ### 2021-08-14 / 1.0.0-beta.0 134 | 135 | - chore: Increase tests timeout. 136 | - feat: Only export as ESM. 137 | - chore: Updated directive. 138 | 139 | ### 2021-01-04 / 0.8.0 140 | 141 | - feat: Export as ESM. 142 | 143 | ### 2021-01-03 / 0.7.1 144 | 145 | - chore: Updated linter config. 146 | - chore: Minor formatting. 147 | - chore: Updated config. 148 | 149 | ### 2021-01-02 / 0.7.0 150 | 151 | - test: Increased tests timeout. 152 | - chore: Updated dependencies, code and tools. 153 | 154 | ### 2020-09-18 / 0.6.0 155 | 156 | - feat: Remove setup option. Allow complex tests with before/after. 157 | 158 | ### 2020-09-18 / 0.5.0 159 | 160 | - feat: Add setup option to prepare tests. 161 | 162 | ### 2020-04-22 / 0.4.0 163 | 164 | - chore: Allow greater test timeouts. 165 | - feat: Removed useless debug infrastructure. 166 | - feat: Reimplemented warmup mode. 167 | - feat: Use worker_threads for execution. Tested everything. 168 | 169 | ### 2020-03-05 / 0.3.0 170 | 171 | - feat: Added warmup option. 172 | - fix: Correctly handle failures sorting. Fixes #1. 173 | 174 | ### 2020-01-29 / 0.2.2 175 | 176 | - fix: Fixed errorThreshold performances. 177 | 178 | ### 2020-01-29 / 0.2.1 179 | 180 | - feat: Added errorThreshold option. 181 | 182 | ### 2020-01-29 / 0.2.0 183 | 184 | - fix: Improve scheduling to avoid races. 185 | 186 | ### 2020-01-29 / 0.1.0 187 | 188 | - Initial version. 189 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | ISC License 2 | 3 | Copyright (c) 2020, and above Shogun 4 | 5 | Permission to use, copy, modify, and/or distribute this software for any 6 | purpose with or without fee is hereby granted, provided that the above 7 | copyright notice and this permission notice appear in all copies. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cronometro 2 | 3 | [![Version](https://img.shields.io/npm/v/cronometro.svg)](https://npm.im/cronometro) 4 | [![Dependencies](https://img.shields.io/librariesio/release/npm/cronometro)](https://libraries.io/npm/cronometro) 5 | [![Build](https://github.com/ShogunPanda/cronometro/workflows/CI/badge.svg)](https://github.com/ShogunPanda/cronometro/actions?query=workflow%3ACI) 6 | [![Coverage](https://img.shields.io/codecov/c/gh/ShogunPanda/cronometro?token=LhUiSgWHoI)](https://codecov.io/gh/ShogunPanda/cronometro) 7 | 8 | Simple benchmarking suite powered by HDR histograms. 9 | 10 | http://sw.cowtech.it/cronometro 11 | 12 | ## Requirements 13 | 14 | Cronometro uses [worker_threads](https://nodejs.org/dist/latest-v12.x/docs/api/worker_threads.html) to run tests in a isolated V8 enviroments to offer the most accurate benchmark. This imposes the restrictions described in the subsections below. 15 | 16 | ### Supported Node versions 17 | 18 | Only Node 12.x and above are supported. 19 | 20 | This package only supports to be directly imported in a ESM context. 21 | 22 | For informations on how to use it in a CommonJS context, please check [this page](https://gist.github.com/ShogunPanda/fe98fd23d77cdfb918010dbc42f4504d). 23 | 24 | ### Script invocation 25 | 26 | The main script which invokes cronometro must be executable without command line arguments, as it is how it will be called within a Worker Thread. 27 | 28 | If you need to configure the script at runtime, use environment variables and optionally configuration files. 29 | 30 | ### TypeScript 31 | 32 | cronometro can run on TypeScript files via [Node.js native types stripping](https://nodejs.org/dist/latest-v22.x/docs/api/typescript.html#type-stripping). 33 | 34 | ### API use 35 | 36 | If you use cronometro as an API and manipulate its return value, consider that the exact same code its executed in both the main thread and in worker threads. 37 | 38 | Inside worker threads, the cronometro function invocation will return no value and no callbacks are invoked. 39 | 40 | You can use `isMainThread` from Worker Threads API to check in which environment the script is running. 41 | 42 | If your main module returns a function, cronometro will execute it before running tests. The function can also return a promise and that will be awaited. 43 | 44 | ## Usage 45 | 46 | To run a benchmark, simply call the `cronometro` function with the set of tests you want to run, then optionally provide a options object and a Node's style callback. 47 | 48 | The return value of the cronometro function is a promise which will be resolved with a results object (see below). 49 | 50 | If the callback is provided, it will also be called with an error or the results object. 51 | 52 | The set of tests must a be a object whose property names are tests names, and property values are tests definitions. 53 | 54 | A test can be defined as a function containing the test to run or an object containing ore or more of the following properties: 55 | 56 | - `test`: The function containing the test to run. If omitted, the test will be a no-op. 57 | - `before`: A setup function to execute before starting test iteration. 58 | - `after`: A cleanup function to execute after all test iteration have been run. 59 | - `skip`: If a test should be skipped. 60 | 61 | Each of the `test` functions above can be either a function, a function accepting a Node style callback or a function returning a promise (hence also async functions). 62 | 63 | Each of the `before` or `after` functions above can be either a function accepting a Node style callback or a function returning a promise (hence also async functions). 64 | 65 | ## Options 66 | 67 | The supported options are the following: 68 | 69 | - `iterations`: The number of iterations to run for each test. Must be a positive number. The default is `10000`. 70 | - `errorThreshold`: If active, it stops the test run before the desider number of iterations if the standard error is below the provided value and at least 10% of the iterations have been run. Must be a number between `0` (which disables this option) and `100`. The default is `1`. 71 | - `warmup`: Run the suite twice, the first time without collecting results. The default is `true`. 72 | - `print`: If print results on the console in a pretty tabular way. The default is `true`. It can be a boolean or a printing options object. The supported printing options are: 73 | - `colors`: If use colors. Default is `true`. 74 | - `compare`: If compare tests in the output. Default is `false`. 75 | - `compareMode`: When comparing is enabled, this can be set to `base` in order to always compare a test to the slowest one. The default is to compare a test to the immediate slower one. 76 | - `onTestStart`: Callback invoked every time a test is started. 77 | - `onTestEnd`: Callback invoked every time a test has finished. 78 | - `onTestError`: Callback invoked every time a test could not be loaded. If the test function throws an error or rejects, `onTestEnd` will be invoked instead. 79 | 80 | ## Results structure 81 | 82 | The results object will a object whose property names are the tests names. 83 | 84 | Each property value is a object with the following properties: 85 | 86 | - `success`: A boolean indicating if the test was successful. If the test is not successful, only the property error will be present. 87 | - `error`: The first error, if any, thrown by a test iteration. 88 | - `size`: The number of iterations records. 89 | - `min`: The minimum execution time. 90 | - `max`: The maximum execution time. 91 | - `mean`: The average execution time. 92 | - `stddev`: The execution times standard deviation. 93 | - `standardError`: The execution times statistic standard error. 94 | - `percentiles`: The percentiles of the execution times. 95 | 96 | ## Example (tabular output) 97 | 98 | ```javascript 99 | import cronometro from 'cronometro' 100 | 101 | const results = cronometro({ 102 | async test1() { 103 | // Do something 104 | }, 105 | async test2() { 106 | // Do something else 107 | } 108 | }) 109 | ``` 110 | 111 | Output: 112 | 113 | ``` 114 | ╔══════════════╤══════════════════╤═══════════╗ 115 | ║ Test │ Result │ Tolerance ║ 116 | ╟──────────────┼──────────────────┼───────────╢ 117 | ║ test1 │ 161297.99 op/sec │ ± 0.65 % ║ 118 | ╟──────────────┼──────────────────┼───────────╢ 119 | ║ Fastest test │ Result │ Tolerance ║ 120 | ╟──────────────┼──────────────────┼───────────╢ 121 | ║ test2 │ 270642.97 op/sec │ ± 4.42 % ║ 122 | ╚══════════════╧══════════════════╧═══════════╝ 123 | ``` 124 | 125 | ## Example (results structure) 126 | 127 | ```javascript 128 | import cronometro from 'cronometro' 129 | 130 | const pattern = /[123]/g 131 | const replacements = { 1: 'a', 2: 'b', 3: 'c' } 132 | 133 | const subject = '123123123123123123123123123123123123123123123123' 134 | 135 | const results = cronometro( 136 | { 137 | single() { 138 | subject.replace(pattern, m => replacements[m]) 139 | }, 140 | multiple() { 141 | subject.replace(/1/g, 'a').replace(/2/g, 'b').replace(/3/g, 'c') 142 | } 143 | }, 144 | { 145 | setup: { 146 | single(cb) { 147 | cb() 148 | } 149 | }, 150 | print: { compare: true } 151 | }, 152 | (err, results) => { 153 | if (err) { 154 | throw err 155 | } 156 | 157 | console.log(JSON.stringify(results, null, 2)) 158 | } 159 | ) 160 | ``` 161 | 162 | Output: 163 | 164 | ``` 165 | { 166 | "single": { 167 | "success": true, 168 | "size": 5, 169 | "min": 29785, 170 | "max": 41506, 171 | "mean": 32894.2, 172 | "stddev": 4407.019555209621, 173 | "percentiles": { 174 | "1": 29785, 175 | "10": 29785, 176 | "25": 29861, 177 | "50": 30942, 178 | "75": 32377, 179 | "90": 41506, 180 | "99": 41506, 181 | "0.001": 29785, 182 | "0.01": 29785, 183 | "0.1": 29785, 184 | "2.5": 29785, 185 | "97.5": 41506, 186 | "99.9": 41506, 187 | "99.99": 41506, 188 | "99.999": 41506 189 | }, 190 | "standardError": 1970.87906072392 191 | }, 192 | "multiple": { 193 | "success": true, 194 | "size": 5, 195 | "min": 21881, 196 | "max": 33368, 197 | "mean": 27646.4, 198 | "stddev": 4826.189494829228, 199 | "percentiles": { 200 | "1": 21881, 201 | "10": 21881, 202 | "25": 23142, 203 | "50": 26770, 204 | "75": 33071, 205 | "90": 33368, 206 | "99": 33368, 207 | "0.001": 21881, 208 | "0.01": 21881, 209 | "0.1": 21881, 210 | "2.5": 21881, 211 | "97.5": 33368, 212 | "99.9": 33368, 213 | "99.99": 33368, 214 | "99.999": 33368 215 | }, 216 | "standardError": 2158.337556546705 217 | } 218 | } 219 | ``` 220 | 221 | ## Contributing to cronometro 222 | 223 | - Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet. 224 | - Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it. 225 | - Fork the project. 226 | - Start a feature/bugfix branch. 227 | - Commit and push until you are happy with your contribution. 228 | - Make sure to add tests for it. This is important so I don't break it in a future version unintentionally. 229 | 230 | ## Copyright 231 | 232 | Copyright (C) 2020 and above Shogun (shogun@cowtech.it). 233 | 234 | Licensed under the ISC license, which can be found at https://choosealicense.com/licenses/isc. 235 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | import { cowtech } from '@cowtech/eslint-config' 2 | 3 | export default [ 4 | ...cowtech, 5 | { 6 | languageOptions: { 7 | parserOptions: { 8 | project: './tsconfig.test.json' 9 | } 10 | } 11 | } 12 | ] 13 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cronometro", 3 | "version": "5.3.0", 4 | "description": "Simple benchmarking suite powered by HDR histograms.", 5 | "homepage": "https://sw.cowtech.it/cronometro", 6 | "repository": { 7 | "type": "git", 8 | "url": "git+https://github.com/ShogunPanda/cronometro.git" 9 | }, 10 | "keywords": [ 11 | "benchmark", 12 | "hdr" 13 | ], 14 | "bugs": { 15 | "url": "https://github.com/ShogunPanda/cronometro/issues" 16 | }, 17 | "author": "Shogun ", 18 | "license": "ISC", 19 | "private": false, 20 | "files": [ 21 | "dist", 22 | "CHANGELOG.md", 23 | "LICENSE.md", 24 | "README.md" 25 | ], 26 | "type": "module", 27 | "exports": "./dist/index.js", 28 | "types": "./dist/index.d.ts", 29 | "scripts": { 30 | "build": "tsc -p .", 31 | "postbuild": "npm run lint", 32 | "format": "prettier -w src test", 33 | "lint": "eslint --cache", 34 | "typecheck": "tsc -p . --noEmit", 35 | "test": "c8 -c test/config/c8-local.json node --env-file=test/config/env --test test/*.test.ts", 36 | "test:ci": "c8 -c test/config/c8-ci.json node --env-file=test/config/env --test test/*.test.ts", 37 | "ci": "npm run build && npm run test:ci", 38 | "prepublishOnly": "npm run ci", 39 | "postpublish": "git push origin && git push origin -f --tags" 40 | }, 41 | "dependencies": { 42 | "acquerello": "^3.0.1", 43 | "hdr-histogram-js": "^3.0.0", 44 | "table": "^6.9.0" 45 | }, 46 | "devDependencies": { 47 | "@cowtech/eslint-config": "10.4.0", 48 | "@types/node": "^22.10.2", 49 | "c8": "^10.1.3", 50 | "cleaner-spec-reporter": "^0.3.1", 51 | "cross-env": "^7.0.3", 52 | "eslint": "^9.17.0", 53 | "prettier": "^3.4.2", 54 | "proxyquire": "^2.1.3", 55 | "typescript": "^5.7.2" 56 | }, 57 | "engines": { 58 | "node": ">= 22.6.0" 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /prettier.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | printWidth: 120, 3 | semi: false, 4 | singleQuote: true, 5 | bracketSpacing: true, 6 | trailingComma: 'none', 7 | arrowParens: 'avoid' 8 | } 9 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { pathToFileURL } from 'node:url' 2 | import { isMainThread, Worker, workerData } from 'node:worker_threads' 3 | import { 4 | defaultOptions, 5 | runnerPath, 6 | type Callback, 7 | type Context, 8 | type Options, 9 | type PrintOptions, 10 | type Result, 11 | type Results, 12 | type Tests 13 | } from './models.ts' 14 | import { printResults } from './print.ts' 15 | 16 | type PromiseResolver = (value: T) => void 17 | type PromiseRejecter = (err: Error) => void 18 | 19 | export * from './models.ts' 20 | export { printResults } from './print.ts' 21 | export * from './tracker.ts' 22 | 23 | function scheduleNextTest(context: Context): void { 24 | // We still have work to do 25 | if (context.current < context.tests.length) { 26 | process.nextTick(() => { 27 | run(context) 28 | }) 29 | return 30 | } 31 | 32 | if (context.print) { 33 | const { colors, compare, compareMode }: PrintOptions = { 34 | colors: true, 35 | compare: false, 36 | compareMode: 'base', 37 | ...(context.print === true ? {} : context.print) 38 | } 39 | 40 | printResults(context.results, colors, compare, compareMode) 41 | } 42 | 43 | context.callback(null, context.results) 44 | } 45 | 46 | function run(context: Context): void { 47 | const name = context.tests[context.current][0] 48 | const workerData = { 49 | path: pathToFileURL(process.argv[1]).toString(), 50 | index: context.current, 51 | iterations: context.iterations, 52 | warmup: context.warmup, 53 | errorThreshold: context.errorThreshold 54 | } 55 | 56 | /* c8 ignore next 5 */ 57 | let nodeOptions = process.env.NODE_OPTIONS ?? '' 58 | const nodeMajor = Number(process.versions.node.split('.')[0]) 59 | if (nodeMajor < 23) { 60 | nodeOptions += ' --experimental-strip-types ' 61 | } 62 | 63 | const worker = new Worker(runnerPath, { 64 | workerData, 65 | env: { 66 | ...process.env, 67 | NODE_OPTIONS: nodeOptions 68 | } 69 | }) 70 | 71 | if (context.onTestStart) { 72 | context.onTestStart(name, workerData, worker) 73 | } 74 | 75 | worker.on('error', error => { 76 | context.results[name] = { 77 | success: false, 78 | error, 79 | size: 0, 80 | min: 0, 81 | max: 0, 82 | mean: 0, 83 | stddev: 0, 84 | percentiles: {}, 85 | standardError: 0 86 | } 87 | 88 | context.current++ 89 | 90 | if (context.onTestError) { 91 | context.onTestError(name, error, worker) 92 | } 93 | 94 | scheduleNextTest(context) 95 | }) 96 | 97 | worker.on('message', message => { 98 | if (message.type !== 'cronometro.result') { 99 | return 100 | } 101 | 102 | const result = message.payload 103 | 104 | context.results[name] = result 105 | context.current++ 106 | 107 | if (context.onTestEnd) { 108 | context.onTestEnd(name, result as Result, worker) 109 | } 110 | 111 | scheduleNextTest(context) 112 | }) 113 | } 114 | 115 | export function cronometro(tests: Tests): Promise | void 116 | export function cronometro(tests: Tests, options: Partial): Promise 117 | export function cronometro(tests: Tests, options: Partial, cb: Callback): undefined 118 | export function cronometro(tests: Tests, options: Callback): void 119 | export function cronometro( 120 | tests: Tests, 121 | options?: Partial | Callback, 122 | cb?: Callback 123 | ): Promise | void { 124 | if (!isMainThread) { 125 | workerData.tests = Object.entries(tests).filter(test => test[1]?.skip !== true) 126 | return 127 | } 128 | 129 | let promise: Promise | undefined 130 | let promiseResolve: PromiseResolver 131 | let promiseReject: PromiseRejecter 132 | 133 | if (typeof options === 'function') { 134 | cb = options 135 | options = {} 136 | } 137 | 138 | let callback = cb as (err: Error | null, results?: Results) => void 139 | 140 | if (!callback) { 141 | promise = new Promise((resolve, reject) => { 142 | promiseResolve = resolve 143 | promiseReject = reject 144 | }) 145 | 146 | callback = function (err: Error | null, results?: Results): void { 147 | if (err) { 148 | promiseReject(err) 149 | return 150 | } 151 | 152 | promiseResolve(results!) 153 | } 154 | } 155 | 156 | // Parse and validate options 157 | const { iterations, errorThreshold, print, warmup, onTestStart, onTestEnd, onTestError } = { 158 | ...defaultOptions, 159 | ...options 160 | } 161 | 162 | if (typeof iterations !== 'number' || iterations < 1) { 163 | callback(new Error('The iterations option must be a positive number.')) 164 | return promise 165 | } 166 | 167 | if (typeof errorThreshold !== 'number' || errorThreshold < 0 || errorThreshold > 100) { 168 | callback(new Error('The errorThreshold option must be a number between 0 and 100.')) 169 | return promise 170 | } 171 | 172 | if (onTestStart && typeof onTestStart !== 'function') { 173 | callback(new Error('The onTestStart option must be a function.')) 174 | return promise 175 | } 176 | 177 | if (onTestEnd && typeof onTestEnd !== 'function') { 178 | callback(new Error('The onTestEnd option must be a function.')) 179 | return promise 180 | } 181 | 182 | if (onTestError && typeof onTestError !== 'function') { 183 | callback(new Error('The onTestError option must be a function.')) 184 | return promise 185 | } 186 | 187 | // Prepare the test 188 | const context: Context = { 189 | warmup, 190 | iterations, 191 | errorThreshold: errorThreshold / 100, 192 | print, 193 | tests: Object.entries(tests).filter(test => test[1]?.skip !== true), // Convert tests to a easier to process [name, func] list, 194 | results: {}, 195 | current: 0, 196 | callback, 197 | onTestStart, 198 | onTestEnd, 199 | onTestError 200 | } 201 | 202 | process.nextTick(() => { 203 | run(context) 204 | }) 205 | return promise 206 | } 207 | 208 | export default cronometro 209 | -------------------------------------------------------------------------------- /src/models.ts: -------------------------------------------------------------------------------- 1 | import { fileURLToPath } from 'node:url' 2 | import { type Worker } from 'node:worker_threads' 3 | import { type Tracker } from './tracker.ts' 4 | 5 | export interface PrintOptions { 6 | colors?: boolean 7 | compare?: boolean 8 | compareMode?: 'base' | 'previous' 9 | } 10 | 11 | export type SetupFunctionCallback = (err?: Error | null) => void 12 | 13 | export type SetupFunction = (cb: SetupFunctionCallback) => Promise | void 14 | 15 | export interface Options { 16 | iterations: number 17 | setup: Record 18 | errorThreshold: number 19 | print: boolean | PrintOptions 20 | warmup: boolean 21 | onTestStart?: (name: string, data: object, worker: Worker) => void 22 | onTestEnd?: (name: string, result: Result, worker: Worker) => void 23 | onTestError?: (name: string, error: Error, worker: Worker) => void 24 | } 25 | 26 | export type StaticTest = () => any 27 | export type AsyncTest = (cb: Callback) => any 28 | export type PromiseTest = () => Promise 29 | export type TestFunction = (StaticTest | AsyncTest | PromiseTest) & { skip?: boolean } 30 | 31 | export interface Test { 32 | test?: TestFunction 33 | before?: SetupFunction 34 | after?: SetupFunction 35 | skip?: boolean 36 | } 37 | 38 | export type Percentiles = Record 39 | 40 | export interface Result { 41 | success: boolean 42 | error?: Error 43 | size: number 44 | min: number 45 | max: number 46 | mean: number 47 | stddev: number 48 | standardError: number 49 | percentiles: Percentiles 50 | } 51 | 52 | export type Callback = (err: Error | null, results: Results) => any 53 | 54 | export type Tests = Record 55 | 56 | export type Results = Record 57 | 58 | export interface Context { 59 | warmup: boolean 60 | iterations: number 61 | errorThreshold: number 62 | print: boolean | PrintOptions 63 | tests: [string, TestFunction | Test][] 64 | results: Results 65 | current: number 66 | callback: Callback 67 | onTestStart?: (name: string, data: object, worker: Worker) => void 68 | onTestEnd?: (name: string, result: Result, worker: Worker) => void 69 | onTestError?: (name: string, error: Error, worker: Worker) => void 70 | } 71 | 72 | export interface WorkerContext { 73 | path: string 74 | tests: [string, TestFunction | Test][] 75 | index: number 76 | iterations: number 77 | warmup: boolean 78 | errorThreshold: number 79 | } 80 | 81 | export interface TestContext { 82 | name: string 83 | test: TestFunction 84 | errorThreshold: number 85 | total: number 86 | executed: number 87 | tracker: Tracker 88 | start: bigint 89 | handler: (error?: Error | null) => void 90 | notifier: (value: any) => void 91 | callback: (result: Result) => void 92 | } 93 | 94 | export const defaultOptions = { 95 | iterations: 1e4, 96 | warmup: true, 97 | errorThreshold: 1, 98 | print: true 99 | } 100 | 101 | export const percentiles = [0.001, 0.01, 0.1, 1, 2.5, 10, 25, 50, 75, 90, 97.5, 99, 99.9, 99.99, 99.999] 102 | 103 | export const runnerPath = fileURLToPath(new URL(`./runner.${import.meta.url.slice(-2)}`, import.meta.url)) 104 | -------------------------------------------------------------------------------- /src/print.ts: -------------------------------------------------------------------------------- 1 | import { clean, colorize } from 'acquerello' 2 | import { table } from 'table' 3 | import { type Results } from './models.ts' 4 | 5 | const styles = ['red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white', 'gray'] 6 | 7 | interface PrintInfo { 8 | name: string 9 | size: number 10 | error: Error | null 11 | throughput: string 12 | standardError: string 13 | relative: string 14 | compared: string 15 | } 16 | 17 | let currentLogger: (message: string, ...params: any[]) => void = console.log 18 | 19 | export function setLogger(logger: (message: string, ...params: any[]) => void): void { 20 | currentLogger = logger 21 | } 22 | 23 | export function printResults(results: Results, colors: boolean, compare: boolean, mode: 'base' | 'previous'): void { 24 | const styler = colors ? colorize : clean 25 | 26 | // Sort results by least performant first, then compare relative performances and also printing padding 27 | let last = 0 28 | let compared = '' 29 | let standardErrorPadding = 0 30 | 31 | const entries: PrintInfo[] = Object.entries(results) 32 | .sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean)) 33 | .map(([name, result]) => { 34 | if (!result.success) { 35 | return { 36 | name, 37 | size: 0, 38 | error: result.error!, 39 | throughput: '', 40 | standardError: '', 41 | relative: '', 42 | compared: '' 43 | } 44 | } 45 | 46 | const { size, mean, standardError } = result 47 | const relative = last !== 0 ? (last / mean - 1) * 100 : 0 48 | 49 | if (mode === 'base') { 50 | if (last === 0) { 51 | last = mean 52 | compared = name 53 | } 54 | } else { 55 | last = mean 56 | compared = name 57 | } 58 | 59 | const standardErrorString = ((standardError / mean) * 100).toFixed(2) 60 | standardErrorPadding = Math.max(standardErrorPadding, standardErrorString.length) 61 | 62 | return { 63 | name, 64 | size, 65 | error: null, 66 | throughput: (1e9 / mean).toFixed(2), 67 | standardError: standardErrorString, 68 | relative: relative.toFixed(2), 69 | compared 70 | } 71 | }) 72 | 73 | let currentColor = 0 74 | 75 | const rows: string[][] = entries.map(entry => { 76 | if (entry.error) { 77 | const row = [ 78 | styler(`{{gray}}${entry.name}{{-}}`), 79 | styler(`{{gray}}${entry.size}{{-}}`), 80 | styler('{{gray}}Errored{{-}}'), 81 | styler('{{gray}}N/A{{-}}') 82 | ] 83 | 84 | if (compare) { 85 | row.push(styler('{{gray}}N/A{{-}}')) 86 | } 87 | 88 | return row 89 | } 90 | 91 | const { name, size, throughput, standardError, relative } = entry 92 | const color = styles[currentColor++ % styles.length] 93 | 94 | const row = [ 95 | styler(`{{${color}}}${name}{{-}}`), 96 | styler(`{{${color}}}${size}{{-}}`), 97 | styler(`{{${color}}}${throughput} op/sec{{-}}`), 98 | styler(`{{gray}}± ${standardError.padStart(standardErrorPadding, ' ')} %{{-}}`) 99 | ] 100 | 101 | if (compare) { 102 | if (/^[\s.0]+$/.test(relative)) { 103 | row.push('') 104 | } else { 105 | row.push(styler(`{{${color}}}+ ${relative} %{{-}}`)) 106 | } 107 | } 108 | 109 | return row 110 | }) 111 | 112 | const compareHeader = `Difference with ${mode === 'base' ? 'slowest' : 'previous'}` 113 | 114 | rows.unshift([ 115 | styler('{{bold white}}Slower tests{{-}}'), 116 | styler('{{bold white}}Samples{{-}}'), 117 | styler('{{bold white}}Result{{-}}'), 118 | styler('{{bold white}}Tolerance{{-}}') 119 | ]) 120 | 121 | rows.splice(-1, 0, [ 122 | styler('{{bold white}}Fastest test{{-}}'), 123 | styler('{{bold white}}Samples{{-}}'), 124 | styler('{{bold white}}Result{{-}}'), 125 | styler('{{bold white}}Tolerance{{-}}') 126 | ]) 127 | 128 | if (compare) { 129 | rows[0].push(styler(`{{bold white}}${compareHeader}{{-}}`)) 130 | rows.at(-2)!.push(styler(`{{bold white}}${compareHeader}{{-}}`)) 131 | } 132 | 133 | currentLogger( 134 | table(rows, { 135 | columns: { 136 | 0: { 137 | alignment: 'left' 138 | }, 139 | 1: { 140 | alignment: 'right' 141 | }, 142 | 2: { 143 | alignment: 'right' 144 | }, 145 | 3: { 146 | alignment: 'right' 147 | } 148 | }, 149 | drawHorizontalLine(index: number, size: number): boolean { 150 | return index < 2 || index > size - 3 151 | } 152 | }) 153 | ) 154 | } 155 | -------------------------------------------------------------------------------- /src/runner.ts: -------------------------------------------------------------------------------- 1 | /* c8 ignore start */ 2 | import { isMainThread, parentPort, workerData } from 'node:worker_threads' 3 | import { type WorkerContext } from './models.ts' 4 | import { runWorker } from './worker.ts' 5 | 6 | if (isMainThread) { 7 | throw new Error('Do not run this file as main script.') 8 | } 9 | 10 | // Require the script to set tests 11 | try { 12 | const module = await import(workerData.path) 13 | 14 | if (typeof module === 'function') { 15 | await module() 16 | } else if (typeof module.default === 'function') { 17 | await module.default() 18 | } 19 | 20 | // Run the worker 21 | runWorker( 22 | workerData as WorkerContext, 23 | value => { 24 | parentPort!.postMessage({ type: 'cronometro.result', payload: value }) 25 | }, 26 | (code: number) => process.exit(code) 27 | ) 28 | } catch (error) { 29 | process.nextTick(() => { 30 | throw error 31 | }) 32 | } 33 | /* c8 ignore stop */ 34 | -------------------------------------------------------------------------------- /src/tracker.ts: -------------------------------------------------------------------------------- 1 | import { build, type Histogram } from 'hdr-histogram-js' 2 | import { percentiles, type Result } from './models.ts' 3 | 4 | export class Tracker { 5 | iterations: number 6 | histogram: Histogram 7 | error: Error | undefined 8 | 9 | constructor() { 10 | this.iterations = 0 11 | this.error = undefined 12 | this.histogram = build({ 13 | lowestDiscernibleValue: 1, 14 | highestTrackableValue: 1e9, 15 | numberOfSignificantValueDigits: 5 16 | }) 17 | } 18 | 19 | get results(): Result { 20 | if (typeof this.error !== 'undefined') { 21 | return { 22 | success: false, 23 | error: this.error, 24 | size: 0, 25 | min: 0, 26 | max: 0, 27 | mean: 0, 28 | stddev: 0, 29 | percentiles: {}, 30 | standardError: 0 31 | } 32 | } 33 | 34 | const size = this.iterations 35 | const { minNonZeroValue: min, maxValue: max, mean, stdDeviation } = this.histogram 36 | 37 | return { 38 | success: true, 39 | size, 40 | min, 41 | max, 42 | mean, 43 | stddev: stdDeviation, 44 | percentiles: Object.fromEntries( 45 | percentiles.map(percentile => [percentile, this.histogram.getValueAtPercentile(percentile)]) 46 | ), 47 | standardError: stdDeviation / Math.sqrt(size) 48 | } 49 | } 50 | 51 | get standardError(): number { 52 | return this.histogram.stdDeviation / Math.sqrt(this.iterations) 53 | } 54 | 55 | track(start: bigint) { 56 | // Grab duration even in case of error to make sure we don't add any overhead to the benchmark 57 | const duration = Number(process.hrtime.bigint() - start) 58 | this.histogram.recordValue(duration) 59 | this.iterations++ 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/worker.ts: -------------------------------------------------------------------------------- 1 | import { type Result, type SetupFunction, type TestContext, type TestFunction, type WorkerContext } from './models.ts' 2 | import { Tracker } from './tracker.ts' 3 | 4 | function noOp(): void { 5 | // No-op 6 | } 7 | 8 | function noSetup(cb: (err?: Error | null) => void): void { 9 | cb() 10 | } 11 | 12 | function handleTestIteration(context: TestContext, error?: Error | null): void { 13 | // Handle error 14 | if (error) { 15 | context.tracker.error = error 16 | context.callback(context.tracker.results) 17 | return 18 | } 19 | 20 | // Get some parameters 21 | const { tracker, total, errorThreshold } = context 22 | 23 | // Track results 24 | tracker.track(context.start) 25 | context.executed++ 26 | 27 | // Check if stop earlier if we are below the error threshold 28 | const executed = context.executed 29 | let stop = false 30 | 31 | if (errorThreshold > 0) { 32 | const completedPercentage = Math.floor((executed / total) * 10_000) 33 | 34 | // Check if abort the test earlier. It is checked every 5% after 10% of the iterations 35 | if (completedPercentage >= 1000 && completedPercentage % 500 === 0) { 36 | const standardErrorPercentage = tracker.standardError / tracker.histogram.mean 37 | 38 | if (standardErrorPercentage < errorThreshold) { 39 | stop = true 40 | } 41 | } 42 | } 43 | 44 | // If the test is over 45 | if (stop || executed > total) { 46 | context.callback(tracker.results) 47 | return 48 | } 49 | 50 | // Schedule next iteration 51 | process.nextTick(() => { 52 | runTestIteration(context) 53 | }) 54 | } 55 | 56 | function runTestIteration(context: TestContext): void { 57 | // Execute the function and get the response time - Handle also promises 58 | try { 59 | context.start = process.hrtime.bigint() 60 | const callResult = context.test(context.handler) 61 | 62 | // It is a promise, handle it accordingly 63 | if (callResult && typeof callResult.then === 'function') { 64 | callResult.then(() => { 65 | context.handler(null) 66 | }, context.handler) 67 | } else if (context.test.length === 0) { 68 | // The function is not a promise and has no arguments, so it's sync 69 | context.handler(null) 70 | } 71 | } catch (error) { 72 | /* c8 ignore start */ 73 | // If a error was thrown, only handle if the original function length is 0, which means it's a sync error, otherwise propagate 74 | if (context.test.length === 0) { 75 | context.handler(error as Error) 76 | return 77 | } 78 | /* c8 ignore end */ 79 | 80 | throw error 81 | } 82 | } 83 | 84 | function beforeCallback(testContext: TestContext, err?: Error | null): void { 85 | if (err) { 86 | testContext.callback({ 87 | success: false, 88 | error: err, 89 | size: 0, 90 | min: 0, 91 | max: 0, 92 | mean: 0, 93 | stddev: 0, 94 | percentiles: {}, 95 | standardError: 0 96 | }) 97 | return 98 | } 99 | 100 | // Schedule the first run 101 | process.nextTick(() => { 102 | runTestIteration(testContext) 103 | }) 104 | } 105 | 106 | function afterCallback( 107 | result: Result, 108 | notifier: (value: any) => void, 109 | cb: (code: number) => void, 110 | err?: Error | null 111 | ): void { 112 | let notifierCode = result.success ? 0 : 1 113 | 114 | if (err) { 115 | notifier({ 116 | success: false, 117 | error: err, 118 | size: 0, 119 | min: 0, 120 | max: 0, 121 | mean: 0, 122 | stddev: 0, 123 | percentiles: {}, 124 | standardError: 0 125 | }) 126 | 127 | notifierCode = 1 128 | } else { 129 | notifier(result) 130 | } 131 | 132 | cb(notifierCode) 133 | } 134 | 135 | export function runWorker(context: WorkerContext, notifier: (value: any) => void, cb: (code: number) => void): void { 136 | const { warmup, tests, index, iterations, errorThreshold } = context 137 | 138 | // Require the original file to build tests 139 | const testToRun = tests?.[index] 140 | 141 | if (!testToRun) { 142 | throw new Error('No test code exported from the worker thread') 143 | } 144 | 145 | const [name, testDefinition] = testToRun 146 | 147 | // Prepare the test 148 | let test: TestFunction = noOp 149 | let before: SetupFunction = noSetup 150 | let after: SetupFunction = noSetup 151 | 152 | if (typeof testDefinition === 'function') { 153 | test = testDefinition 154 | } else { 155 | if (typeof testDefinition.test === 'function') { 156 | test = testDefinition.test 157 | } 158 | 159 | if (typeof testDefinition.before === 'function') { 160 | before = testDefinition.before 161 | } 162 | 163 | if (typeof testDefinition.after === 'function') { 164 | after = testDefinition.after 165 | } 166 | } 167 | 168 | // Prepare the context 169 | const testContext: TestContext = { 170 | name, 171 | test, 172 | errorThreshold, 173 | total: iterations - 1, 174 | executed: 0, 175 | tracker: new Tracker(), 176 | start: BigInt(0), 177 | handler: noOp, 178 | notifier, 179 | callback(result: Result): void { 180 | if (warmup) { 181 | context.warmup = false 182 | runWorker(context, notifier, cb) 183 | return 184 | } 185 | 186 | const callback = afterCallback.bind(null, result, notifier, cb) 187 | const afterResponse = after(callback) 188 | 189 | if (afterResponse && typeof afterResponse.then === 'function') { 190 | afterResponse.then(callback, callback) 191 | } 192 | } 193 | } 194 | 195 | // Bind the handler to the context 196 | testContext.handler = handleTestIteration.bind(null, testContext) 197 | 198 | // Run the test setup, then start the test 199 | const callback = beforeCallback.bind(null, testContext) 200 | const beforeResponse = before(callback) 201 | 202 | if (beforeResponse && typeof beforeResponse.then === 'function') { 203 | beforeResponse.then(callback, callback) 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /test/asyncImport.test.ts: -------------------------------------------------------------------------------- 1 | import { deepStrictEqual, ifError, ok } from 'node:assert' 2 | import { test } from 'node:test' 3 | import { isMainThread } from 'node:worker_threads' 4 | import { cronometro, percentiles } from '../src/index.ts' 5 | 6 | await new Promise(resolve => setTimeout(resolve, 100)) 7 | 8 | if (!isMainThread) { 9 | cronometro( 10 | { 11 | single() { 12 | Buffer.alloc(10) 13 | }, 14 | multiple() { 15 | Buffer.alloc(10) 16 | Buffer.alloc(20) 17 | } 18 | }, 19 | () => false 20 | ) 21 | } else { 22 | await test('Collecting results', async () => { 23 | const results = await cronometro( 24 | { 25 | single() { 26 | Buffer.alloc(10) 27 | }, 28 | multiple() { 29 | Buffer.alloc(10) 30 | Buffer.alloc(20) 31 | } 32 | }, 33 | { iterations: 10, print: false } 34 | ) 35 | 36 | deepStrictEqual(Object.keys(results), ['single', 'multiple']) 37 | 38 | for (const entry of Object.values(results)) { 39 | ok(entry.success) 40 | ifError(entry.error) 41 | deepStrictEqual(entry.size, 10) 42 | deepStrictEqual(typeof entry.min, 'number') 43 | deepStrictEqual(typeof entry.max, 'number') 44 | deepStrictEqual(typeof entry.mean, 'number') 45 | deepStrictEqual(typeof entry.stddev, 'number') 46 | deepStrictEqual(typeof entry.standardError, 'number') 47 | 48 | for (const percentile of percentiles) { 49 | ok(typeof entry.percentiles[percentile.toString()], 'number') 50 | } 51 | } 52 | }) 53 | } 54 | -------------------------------------------------------------------------------- /test/callbacks.test.ts: -------------------------------------------------------------------------------- 1 | import { deepStrictEqual, match, ok } from 'node:assert' 2 | import { test } from 'node:test' 3 | import { Worker, isMainThread, parentPort } from 'node:worker_threads' 4 | import { cronometro, type Result, type TestFunction } from '../src/index.ts' 5 | 6 | if (!isMainThread) { 7 | parentPort!.postMessage('another') 8 | 9 | cronometro( 10 | { 11 | single() { 12 | Buffer.alloc(10) 13 | }, 14 | multiple() { 15 | throw new Error('INVALID') 16 | }, 17 | missing: undefined as unknown as TestFunction, 18 | skipped: { 19 | test() {}, 20 | skip: true 21 | } 22 | }, 23 | () => false 24 | ) 25 | } else { 26 | test('Callbacks', async () => { 27 | await cronometro( 28 | { 29 | single() { 30 | Buffer.alloc(10) 31 | }, 32 | multiple() { 33 | throw new Error('INVALID') 34 | }, 35 | missing() { 36 | Buffer.alloc(10) 37 | }, 38 | skipped: { 39 | test() {}, 40 | skip: true 41 | } 42 | }, 43 | { 44 | iterations: 10, 45 | print: false, 46 | onTestStart(name: string, data: any, worker: Worker) { 47 | match(name, /single|multiple|missing/) 48 | ok(data.index < 3) 49 | ok(worker instanceof Worker) 50 | }, 51 | onTestEnd(name: string, result: Result, worker: Worker) { 52 | if (result.success) { 53 | deepStrictEqual(name, 'single') 54 | ok(result.size > 0) 55 | } else { 56 | deepStrictEqual(name, 'multiple') 57 | deepStrictEqual(result.error!.message, 'INVALID') 58 | } 59 | ok(worker instanceof Worker) 60 | }, 61 | onTestError(name: string, error: Error, worker: Worker) { 62 | deepStrictEqual(name, 'missing') 63 | deepStrictEqual(error.message, "Cannot read properties of undefined (reading 'test')") 64 | ok(worker instanceof Worker) 65 | } 66 | } 67 | ) 68 | }) 69 | } 70 | -------------------------------------------------------------------------------- /test/config/c8-ci.json: -------------------------------------------------------------------------------- 1 | { 2 | "check-coverage": true, 3 | "reporter": ["text", "json"], 4 | "branches": 90, 5 | "functions": 90, 6 | "lines": 90, 7 | "statements": 90 8 | } 9 | -------------------------------------------------------------------------------- /test/config/c8-local.json: -------------------------------------------------------------------------------- 1 | { 2 | "reporter": ["text", "html"] 3 | } 4 | -------------------------------------------------------------------------------- /test/config/env: -------------------------------------------------------------------------------- 1 | NODE_OPTIONS="--experimental-strip-types --test-reporter=cleaner-spec-reporter --disable-warning=ExperimentalWarning" -------------------------------------------------------------------------------- /test/errorHandling.test.ts: -------------------------------------------------------------------------------- 1 | import { deepStrictEqual, ifError, ok, rejects } from 'node:assert' 2 | import { test } from 'node:test' 3 | import { isMainThread } from 'node:worker_threads' 4 | import { cronometro, percentiles } from '../src/index.ts' 5 | 6 | if (!isMainThread) { 7 | cronometro( 8 | { 9 | single() { 10 | Buffer.alloc(10) 11 | }, 12 | multiple() { 13 | throw new Error('FAILED') 14 | } 15 | }, 16 | () => false 17 | ) 18 | } else { 19 | test('Errored tests handling', async () => { 20 | const results = await cronometro( 21 | { 22 | single() { 23 | Buffer.alloc(10) 24 | }, 25 | multiple() { 26 | throw new Error('FAILED') 27 | } 28 | }, 29 | { iterations: 10, print: false } 30 | ) 31 | 32 | deepStrictEqual(Object.keys(results), ['single', 'multiple']) 33 | 34 | ok(results.single.success) 35 | ifError(results.single.error) 36 | deepStrictEqual(results.single.size, 10) 37 | deepStrictEqual(typeof results.single.min, 'number') 38 | deepStrictEqual(typeof results.single.max, 'number') 39 | deepStrictEqual(typeof results.single.mean, 'number') 40 | deepStrictEqual(typeof results.single.stddev, 'number') 41 | deepStrictEqual(typeof results.single.standardError, 'number') 42 | 43 | for (const percentile of percentiles) { 44 | ok(typeof results.single.percentiles[percentile.toString()], 'number') 45 | } 46 | 47 | ok(!results.multiple.success) 48 | ok(results.multiple.error instanceof Error) 49 | deepStrictEqual(results.multiple.error.message, 'FAILED') 50 | deepStrictEqual(results.multiple.size, 0) 51 | deepStrictEqual(results.multiple.min, 0) 52 | deepStrictEqual(results.multiple.max, 0) 53 | deepStrictEqual(results.multiple.mean, 0) 54 | deepStrictEqual(results.multiple.stddev, 0) 55 | deepStrictEqual(results.multiple.standardError, 0) 56 | deepStrictEqual(results.multiple.percentiles, {}) 57 | }) 58 | 59 | test('Runner cannot be run in main thread', async () => { 60 | await rejects(import('../src/runner.ts'), { message: 'Do not run this file as main script.' }) 61 | }) 62 | 63 | test('Runner reports setup errors', async () => { 64 | const results = await cronometro( 65 | { 66 | notDefined() { 67 | Buffer.alloc(10) 68 | } 69 | }, 70 | { iterations: 10, print: false } 71 | ) 72 | 73 | deepStrictEqual(Object.keys(results), ['notDefined']) 74 | }) 75 | } 76 | -------------------------------------------------------------------------------- /test/fixture/sample.ts: -------------------------------------------------------------------------------- 1 | import { cronometro } from '../../dist/index.js' 2 | 3 | const pattern = /[1-3]/g 4 | const replacements = { 1: 'a', 2: 'b', 3: 'c' } 5 | 6 | const subject = 7 | '123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123123' 8 | 9 | cronometro( 10 | { 11 | single() { 12 | subject.replaceAll(pattern, m => replacements[m]) 13 | }, 14 | multiple() { 15 | subject.replaceAll('1', 'a').replaceAll('2', 'b').replaceAll('3', 'c') 16 | } 17 | }, 18 | { iterations: 5, errorThreshold: 0, print: { compare: true }, warmup: true }, 19 | (_, results) => { 20 | console.log(JSON.stringify(results, null, 2)) 21 | } 22 | ) 23 | -------------------------------------------------------------------------------- /test/genericErrorHandling.test.ts: -------------------------------------------------------------------------------- 1 | import { deepStrictEqual, ok } from 'node:assert' 2 | import { test } from 'node:test' 3 | import { isMainThread } from 'node:worker_threads' 4 | import { cronometro, type Tests } from '../src/index.ts' 5 | 6 | if (!isMainThread) { 7 | cronometro(undefined as unknown as Tests, () => false) 8 | } else { 9 | await test('Generic error handling', async () => { 10 | const results = await cronometro( 11 | { 12 | single() { 13 | Buffer.alloc(10) 14 | }, 15 | multiple() { 16 | throw new Error('FAILED') 17 | } 18 | }, 19 | { iterations: 10, print: false } 20 | ) 21 | 22 | ok(!results.single.success) 23 | ok(results.single.error instanceof Error) 24 | deepStrictEqual(results.single.error.message, 'Cannot convert undefined or null to object') 25 | 26 | ok(!results.single.success) 27 | ok(results.single.error instanceof Error) 28 | deepStrictEqual(results.single.error.message, 'Cannot convert undefined or null to object') 29 | }) 30 | } 31 | -------------------------------------------------------------------------------- /test/notTestExportedInWorkers.test.ts: -------------------------------------------------------------------------------- 1 | import { deepStrictEqual, ok } from 'node:assert' 2 | import { test } from 'node:test' 3 | import { isMainThread } from 'node:worker_threads' 4 | import { cronometro } from '../src/index.ts' 5 | 6 | if (isMainThread) { 7 | test('Errors are properly handled when no tests are exported in the worker threads', async () => { 8 | const results = await cronometro( 9 | { 10 | single() {}, 11 | multiple() {} 12 | }, 13 | { iterations: 10, print: false } 14 | ) 15 | 16 | ok(!results.single.success) 17 | ok(results.single.error instanceof Error) 18 | deepStrictEqual(results.single.error.message, 'No test code exported from the worker thread') 19 | deepStrictEqual(results.single.size, 0) 20 | deepStrictEqual(results.single.min, 0) 21 | deepStrictEqual(results.single.max, 0) 22 | deepStrictEqual(results.single.mean, 0) 23 | deepStrictEqual(results.single.stddev, 0) 24 | deepStrictEqual(results.single.standardError, 0) 25 | deepStrictEqual(results.single.percentiles, {}) 26 | 27 | ok(!results.multiple.success) 28 | ok(results.multiple.error instanceof Error) 29 | deepStrictEqual(results.multiple.error.message, 'No test code exported from the worker thread') 30 | deepStrictEqual(results.multiple.size, 0) 31 | deepStrictEqual(results.multiple.min, 0) 32 | deepStrictEqual(results.multiple.max, 0) 33 | deepStrictEqual(results.multiple.mean, 0) 34 | deepStrictEqual(results.multiple.stddev, 0) 35 | deepStrictEqual(results.multiple.standardError, 0) 36 | deepStrictEqual(results.multiple.percentiles, {}) 37 | }) 38 | } 39 | -------------------------------------------------------------------------------- /test/optionsValidation.test.ts: -------------------------------------------------------------------------------- 1 | import { deepStrictEqual, ok, rejects } from 'node:assert' 2 | import { test } from 'node:test' 3 | import { cronometro } from '../src/index.ts' 4 | 5 | test('Options validation', async () => { 6 | await rejects( 7 | () => 8 | cronometro( 9 | { 10 | single() { 11 | Buffer.alloc(10) 12 | }, 13 | multiple() { 14 | Buffer.alloc(10) 15 | Buffer.alloc(20) 16 | } 17 | }, 18 | { iterations: -1 } 19 | ), 20 | new Error('The iterations option must be a positive number.') 21 | ) 22 | 23 | cronometro( 24 | { 25 | single() { 26 | Buffer.alloc(10) 27 | }, 28 | multiple() { 29 | Buffer.alloc(10) 30 | Buffer.alloc(20) 31 | } 32 | }, 33 | { errorThreshold: -1 }, 34 | err => { 35 | ok(err instanceof Error) 36 | deepStrictEqual(err.message, 'The errorThreshold option must be a number between 0 and 100.') 37 | } 38 | ) 39 | 40 | cronometro( 41 | { 42 | single() { 43 | Buffer.alloc(10) 44 | }, 45 | multiple() { 46 | Buffer.alloc(10) 47 | Buffer.alloc(20) 48 | } 49 | }, 50 | { onTestStart: 1 as any }, 51 | err => { 52 | ok(err instanceof Error) 53 | deepStrictEqual(err.message, 'The onTestStart option must be a function.') 54 | } 55 | ) 56 | 57 | cronometro( 58 | { 59 | single() { 60 | Buffer.alloc(10) 61 | }, 62 | multiple() { 63 | Buffer.alloc(10) 64 | Buffer.alloc(20) 65 | } 66 | }, 67 | { onTestEnd: 1 as any }, 68 | err => { 69 | ok(err instanceof Error) 70 | deepStrictEqual(err.message, 'The onTestEnd option must be a function.') 71 | } 72 | ) 73 | 74 | cronometro( 75 | { 76 | single() { 77 | Buffer.alloc(10) 78 | }, 79 | multiple() { 80 | Buffer.alloc(10) 81 | Buffer.alloc(20) 82 | } 83 | }, 84 | { onTestError: 1 as any }, 85 | err => { 86 | ok(err instanceof Error) 87 | deepStrictEqual(err.message, 'The onTestError option must be a function.') 88 | } 89 | ) 90 | }) 91 | -------------------------------------------------------------------------------- /test/print.test.ts: -------------------------------------------------------------------------------- 1 | import { deepStrictEqual, ifError, match, ok } from 'node:assert' 2 | import { test } from 'node:test' 3 | import { isMainThread } from 'node:worker_threads' 4 | import { cronometro, defaultOptions, percentiles } from '../src/index.ts' 5 | import { setLogger } from '../src/print.ts' 6 | 7 | function removeStyle(source: string): string { 8 | // eslint-disable-next-line no-control-regex 9 | return source.replaceAll(/\u001B\[\d+m/g, '') 10 | } 11 | 12 | defaultOptions.iterations = 10 13 | 14 | if (!isMainThread) { 15 | cronometro( 16 | { 17 | single() { 18 | // No-op 19 | }, 20 | multiple() { 21 | // No-op 22 | }, 23 | error() { 24 | throw new Error('FAILED') 25 | } 26 | }, 27 | () => false 28 | ) 29 | } else { 30 | test('Printing - Default options', (t, done) => { 31 | const logger = t.mock.fn() 32 | setLogger(logger) 33 | 34 | cronometro( 35 | { 36 | single() { 37 | // No-op 38 | }, 39 | multiple() { 40 | // No-op 41 | }, 42 | error() { 43 | throw new Error('FAILED') 44 | } 45 | }, 46 | (err, results) => { 47 | ifError(err) 48 | deepStrictEqual(Object.keys(results), ['single', 'multiple', 'error']) 49 | 50 | ok(!results.error.success) 51 | ok(results.error.error instanceof Error) 52 | deepStrictEqual(results.error.error.message, 'FAILED') 53 | deepStrictEqual(results.error.size, 0) 54 | deepStrictEqual(results.error.min, 0) 55 | deepStrictEqual(results.error.max, 0) 56 | deepStrictEqual(results.error.mean, 0) 57 | deepStrictEqual(results.error.stddev, 0) 58 | deepStrictEqual(results.error.standardError, 0) 59 | deepStrictEqual(results.error.percentiles, {}) 60 | delete results.error 61 | 62 | for (const entry of Object.values(results)) { 63 | ok(entry.success) 64 | ifError(entry.error) 65 | deepStrictEqual(entry.size, 10) 66 | ok(typeof entry.min, 'number') 67 | ok(typeof entry.max, 'number') 68 | ok(typeof entry.mean, 'number') 69 | ok(typeof entry.stddev, 'number') 70 | ok(typeof entry.standardError, 'number') 71 | 72 | for (const percentile of percentiles) { 73 | ok(typeof entry.percentiles[percentile.toString()], 'number') 74 | } 75 | } 76 | 77 | const output = removeStyle(logger.mock.calls[0].arguments[0] as string) 78 | match(output, /║\s+Slower tests\s+|\s+Samples\s+|\s+Result\s+|\s+Tolerance\s+║/) 79 | match(output, /║\s+Faster test\s+|\s+Samples\s+|\s+Result\s+|\s+Tolerance\s+║/) 80 | match(output, /║\s+(single|multiple)\s+|\s+10\s+|\s+\d+\.\d{2}\sop\/sec\s+|\s+±\s\d+.\d{2}\s%\s+║/) 81 | match(output, /║\s+(error)\s+|\s+0\s+|\s+Errored\s+|\s+N\/A\s+║/) 82 | done() 83 | } 84 | ) 85 | }) 86 | 87 | test('Printing - No colors', (t, done) => { 88 | const logger = t.mock.fn() 89 | setLogger(logger) 90 | 91 | cronometro( 92 | { 93 | single() { 94 | // No-op 95 | }, 96 | multiple() { 97 | // No-op 98 | }, 99 | error() { 100 | throw new Error('FAILED') 101 | } 102 | }, 103 | { print: { colors: false } }, 104 | err => { 105 | ifError(err) 106 | 107 | const output = removeStyle(logger.mock.calls[0].arguments[0] as string) 108 | 109 | // eslint-disable-next-line no-control-regex 110 | ok(!output.match(/\u001B/)) 111 | done() 112 | } 113 | ) 114 | }) 115 | 116 | test('Printing - Base compare', (t, done) => { 117 | const logger = t.mock.fn() 118 | setLogger(logger) 119 | 120 | cronometro( 121 | { 122 | single() { 123 | // No-op 124 | }, 125 | multiple() { 126 | // No-op 127 | }, 128 | error() { 129 | throw new Error('FAILED') 130 | } 131 | }, 132 | { print: { compare: true } }, 133 | err => { 134 | ifError(err) 135 | 136 | const output = removeStyle(logger.mock.calls[0].arguments[0] as string) 137 | match(output, /║\s+Slower tests\s+|\s+Samples\s+|\s+Result\s+|\s+Tolerance\s+|\s+Difference with slowest║/) 138 | match(output, /║\s+Fastest test\s+|\s+Samples\s+|\s+Result\s+|\s+Tolerance\s+|\s+Difference with slowest║/) 139 | match(output, /║\s+(single|multiple)\s+|\s+10\s+|\s+\d+\.\d{2}\sop\/sec\s+|\s+±\s\d+.\d{2}\s%\s+|\s+║/) 140 | match( 141 | output, 142 | /║\s+(single|multiple)\s+|\s+10\s+|\s+\d+\.\d{2}\sop\/sec\s+|\s+±\s\d+.\d{2}\s%\s+|\s+\\+\s+\d+.\d{2}\s%\s+║/ 143 | ) 144 | match(output, /║\s+(error)\s+|\s+0\s+|\s+Errored\s+|\s+N\/A\s+|\s+N\/A\s+║/) 145 | done() 146 | } 147 | ) 148 | }) 149 | 150 | test('Printing - Previous compare', (t, done) => { 151 | const logger = t.mock.fn() 152 | setLogger(logger) 153 | 154 | cronometro( 155 | { 156 | single() { 157 | // No-op 158 | }, 159 | multiple() { 160 | // No-op 161 | }, 162 | error() { 163 | throw new Error('FAILED') 164 | } 165 | }, 166 | { print: { compare: true, compareMode: 'previous' } }, 167 | err => { 168 | ifError(err) 169 | 170 | const output = removeStyle(logger.mock.calls[0].arguments[0] as string) 171 | match(output, /║\s+Slower tests\s+|\s+Samples\s+|\s+Result\s+|\s+Tolerance\s+|\s+Difference with previous\s+║/) 172 | match(output, /║\s+Faster test\s+|\s+Samples\s+|\s+Result\s+|\s+Tolerance\s+|\s+Difference with previous\s+║/) 173 | match(output, /║\s+(single|multiple)\s+|\s+10\s+|\s+\d+\.\d{2}\sop\/sec\s+|\s+±\s\d+.\d{2}\s%\s+|\s+║/) 174 | match( 175 | output, 176 | /║\s+(single|multiple)\s+|\s+10\s+|\s+\d+\.\d{2}\sop\/sec\s+|\s+±\s\d+.\d{2}\s%\s+|\s+\\+\s+\d+.\d{2}\s%\s+║/ 177 | ) 178 | match(output, /║\s+(error)\s+|\s+0\s+|\s+Errored\s+|\s+N\/A\s+|\s+N\/A\s+║/) 179 | done() 180 | } 181 | ) 182 | }) 183 | } 184 | -------------------------------------------------------------------------------- /test/success.test.ts: -------------------------------------------------------------------------------- 1 | import { deepStrictEqual, ifError, ok } from 'node:assert' 2 | import { test } from 'node:test' 3 | import { isMainThread } from 'node:worker_threads' 4 | import { cronometro, percentiles } from '../src/index.ts' 5 | 6 | if (!isMainThread) { 7 | cronometro( 8 | { 9 | single() { 10 | Buffer.alloc(10) 11 | }, 12 | multiple() { 13 | Buffer.alloc(10) 14 | Buffer.alloc(20) 15 | } 16 | }, 17 | () => false 18 | ) 19 | } else { 20 | await test('Collecting results', async () => { 21 | const results = await cronometro( 22 | { 23 | single() { 24 | Buffer.alloc(10) 25 | }, 26 | multiple() { 27 | Buffer.alloc(10) 28 | Buffer.alloc(20) 29 | } 30 | }, 31 | { iterations: 10, print: false } 32 | ) 33 | 34 | deepStrictEqual(Object.keys(results), ['single', 'multiple']) 35 | 36 | for (const entry of Object.values(results)) { 37 | ok(entry.success) 38 | ifError(entry.error) 39 | deepStrictEqual(entry.size, 10) 40 | ok(typeof entry.min, 'number') 41 | ok(typeof entry.max, 'number') 42 | ok(typeof entry.mean, 'number') 43 | ok(typeof entry.stddev, 'number') 44 | ok(typeof entry.standardError, 'number') 45 | 46 | for (const percentile of percentiles) { 47 | ok(typeof entry.percentiles[percentile.toString()], 'number') 48 | } 49 | } 50 | }) 51 | } 52 | -------------------------------------------------------------------------------- /test/testsCallbacks.test.ts: -------------------------------------------------------------------------------- 1 | import { deepStrictEqual, ifError, ok } from 'node:assert' 2 | import { test } from 'node:test' 3 | import { isMainThread } from 'node:worker_threads' 4 | import { cronometro, type SetupFunctionCallback } from '../src/index.ts' 5 | 6 | if (!isMainThread) { 7 | cronometro( 8 | { 9 | first: { 10 | before(cb: SetupFunctionCallback) { 11 | cb() 12 | }, 13 | test() { 14 | Buffer.alloc(20) 15 | }, 16 | after(cb: SetupFunctionCallback) { 17 | cb() 18 | } 19 | }, 20 | second: { 21 | // eslint-disable-next-line @typescript-eslint/require-await 22 | async before() { 23 | throw new Error('FAILED ON BEFORE') 24 | }, 25 | test() { 26 | Buffer.alloc(20) 27 | } 28 | }, 29 | third: { 30 | test() { 31 | Buffer.alloc(20) 32 | }, 33 | after(cb: SetupFunctionCallback) { 34 | cb(new Error('FAILED ON AFTER')) 35 | } 36 | } 37 | }, 38 | () => false 39 | ) 40 | } else { 41 | test('Lifecycle Callbacks', async () => { 42 | const results = await cronometro( 43 | { 44 | first: { 45 | before(cb: SetupFunctionCallback) { 46 | cb() 47 | }, 48 | test() { 49 | Buffer.alloc(20) 50 | }, 51 | after(cb: SetupFunctionCallback) { 52 | cb() 53 | } 54 | }, 55 | second: { 56 | // eslint-disable-next-line @typescript-eslint/require-await 57 | async before() { 58 | throw new Error('FAILED ON BEFORE') 59 | }, 60 | test() { 61 | Buffer.alloc(20) 62 | } 63 | }, 64 | third: { 65 | test() { 66 | Buffer.alloc(20) 67 | }, 68 | after(cb: SetupFunctionCallback) { 69 | cb(new Error('FAILED ON AFTER')) 70 | } 71 | } 72 | }, 73 | { 74 | iterations: 10, 75 | print: false 76 | } 77 | ) 78 | 79 | ifError(results.first.error) 80 | ok(results.second.error instanceof Error) 81 | deepStrictEqual(results.second.error.message, 'FAILED ON BEFORE') 82 | 83 | ok(results.third.error instanceof Error) 84 | deepStrictEqual(results.third.error.message, 'FAILED ON AFTER') 85 | }) 86 | } 87 | -------------------------------------------------------------------------------- /test/unhandledErrorHandling.test.ts: -------------------------------------------------------------------------------- 1 | import { deepStrictEqual, ifError, ok } from 'node:assert' 2 | import { test } from 'node:test' 3 | import { isMainThread } from 'node:worker_threads' 4 | import { cronometro, percentiles, type Callback } from '../src/index.ts' 5 | 6 | if (!isMainThread) { 7 | cronometro( 8 | { 9 | single() { 10 | Buffer.alloc(10) 11 | }, 12 | multiple(_done: Callback) { 13 | Buffer.alloc(10) 14 | 15 | if (process.argv.length > 0) { 16 | throw new Error('FAILED') 17 | } 18 | } 19 | }, 20 | () => false 21 | ) 22 | } else { 23 | await test('Unhandled errored tests handling', async () => { 24 | const results = await cronometro( 25 | { 26 | single() { 27 | Buffer.alloc(10) 28 | }, 29 | multiple(_done: Callback) { 30 | Buffer.alloc(10) 31 | 32 | if (process.argv.length > 0) { 33 | throw new Error('FAILED') 34 | } 35 | } 36 | }, 37 | { iterations: 10, print: false } 38 | ) 39 | 40 | deepStrictEqual(Object.keys(results), ['single', 'multiple']) 41 | 42 | ok(results.single.success) 43 | ifError(results.single.error, 'undefined') 44 | deepStrictEqual(results.single.size, 10) 45 | ok(typeof results.single.min, 'number') 46 | ok(typeof results.single.max, 'number') 47 | ok(typeof results.single.mean, 'number') 48 | ok(typeof results.single.stddev, 'number') 49 | ok(typeof results.single.standardError, 'number') 50 | 51 | for (const percentile of percentiles) { 52 | ok(typeof results.single.percentiles[percentile.toString()], 'number') 53 | } 54 | 55 | ok(!results.multiple.success) 56 | ok(results.multiple.error instanceof Error) 57 | deepStrictEqual(results.multiple.error.message, 'FAILED') 58 | deepStrictEqual(results.multiple.size, 0) 59 | deepStrictEqual(results.multiple.min, 0) 60 | deepStrictEqual(results.multiple.max, 0) 61 | deepStrictEqual(results.multiple.mean, 0) 62 | deepStrictEqual(results.multiple.stddev, 0) 63 | deepStrictEqual(results.multiple.standardError, 0) 64 | deepStrictEqual(results.multiple.percentiles, {}) 65 | }) 66 | } 67 | -------------------------------------------------------------------------------- /test/worker.test.ts: -------------------------------------------------------------------------------- 1 | import { deepStrictEqual, ifError, ok } from 'node:assert' 2 | import { test } from 'node:test' 3 | import { percentiles, type AsyncTest, type Result } from '../src/index.ts' 4 | import { runWorker } from '../src/worker.ts' 5 | 6 | test('Worker execution - Handle sync functions that succeed', (t, done) => { 7 | let mainCalls = 0 8 | const notifier = t.mock.fn() 9 | 10 | function main(): void { 11 | mainCalls++ 12 | } 13 | 14 | runWorker( 15 | { 16 | path: 'fs', 17 | tests: [['main', main]], 18 | index: 0, 19 | iterations: 10_000, 20 | warmup: false, 21 | errorThreshold: 100 22 | }, 23 | notifier, 24 | code => { 25 | deepStrictEqual(code, 0) 26 | ok(mainCalls > 0) 27 | 28 | const result = notifier.mock.calls[0].arguments[0] as Result 29 | 30 | ok(result.success) 31 | ifError(result.error) 32 | deepStrictEqual(result.size, 1000) 33 | ok(typeof result.min, 'number') 34 | ok(typeof result.max, 'number') 35 | ok(typeof result.mean, 'number') 36 | ok(typeof result.stddev, 'number') 37 | ok(typeof result.standardError, 'number') 38 | 39 | for (const percentile of percentiles) { 40 | ok(typeof result.percentiles[percentile.toString()], 'number') 41 | } 42 | 43 | done() 44 | } 45 | ) 46 | }) 47 | 48 | test('Worker execution - Handle sync functions that throw errors', (t, done) => { 49 | let mainCalls = 0 50 | const notifier = t.mock.fn() 51 | 52 | /* eslint-disable-next-line @typescript-eslint/require-await */ 53 | async function main(): Promise { 54 | mainCalls++ 55 | throw new Error('FAILED') 56 | } 57 | 58 | runWorker( 59 | { 60 | path: 'fs', 61 | tests: [['main', main]], 62 | index: 0, 63 | iterations: 5, 64 | warmup: false, 65 | errorThreshold: 100 66 | }, 67 | notifier, 68 | code => { 69 | deepStrictEqual(code, 1) 70 | ok(mainCalls > 0) 71 | 72 | const result = notifier.mock.calls[0].arguments[0] as Result 73 | 74 | ok(!result.success) 75 | ok(result.error instanceof Error) 76 | deepStrictEqual(result.error.message, 'FAILED') 77 | deepStrictEqual(result.size, 0) 78 | deepStrictEqual(result.min, 0) 79 | deepStrictEqual(result.max, 0) 80 | deepStrictEqual(result.mean, 0) 81 | deepStrictEqual(result.stddev, 0) 82 | deepStrictEqual(result.standardError, 0) 83 | deepStrictEqual(result.percentiles, {}) 84 | 85 | done() 86 | } 87 | ) 88 | }) 89 | 90 | test('Worker execution - Handle callback functions that succeed', (t, done) => { 91 | let mainCalls = 0 92 | const notifier = t.mock.fn() 93 | 94 | function main(cb: (err?: Error) => void): void { 95 | mainCalls++ 96 | cb() 97 | } 98 | 99 | runWorker( 100 | { 101 | path: 'fs', 102 | tests: [['main', main as AsyncTest]], 103 | 104 | index: 0, 105 | iterations: 10_000, 106 | warmup: false, 107 | errorThreshold: 1e-9 108 | }, 109 | notifier, 110 | code => { 111 | deepStrictEqual(code, 0) 112 | ok(mainCalls > 0) 113 | 114 | const result = notifier.mock.calls[0].arguments[0] as Result 115 | 116 | ok(result.success) 117 | ifError(result.error) 118 | deepStrictEqual(result.size, 10_000) 119 | ok(typeof result.min, 'number') 120 | ok(typeof result.max, 'number') 121 | ok(typeof result.mean, 'number') 122 | ok(typeof result.stddev, 'number') 123 | ok(typeof result.standardError, 'number') 124 | 125 | for (const percentile of percentiles) { 126 | ok(typeof result.percentiles[percentile.toString()], 'number') 127 | } 128 | 129 | done() 130 | } 131 | ) 132 | }) 133 | 134 | test('Worker execution - Handle callback functions that throw errors', (t, done) => { 135 | let mainCalls = 0 136 | const notifier = t.mock.fn() 137 | 138 | function main(cb: (err?: Error) => void): void { 139 | mainCalls++ 140 | cb(new Error('FAILED')) 141 | } 142 | 143 | runWorker( 144 | { 145 | path: 'fs', 146 | tests: [['main', main as AsyncTest]], 147 | 148 | index: 0, 149 | iterations: 5, 150 | warmup: false, 151 | errorThreshold: 0 152 | }, 153 | notifier, 154 | code => { 155 | deepStrictEqual(code, 1) 156 | ok(mainCalls > 0) 157 | 158 | const result = notifier.mock.calls[0].arguments[0] as Result 159 | 160 | ok(!result.success) 161 | ok(result.error instanceof Error) 162 | deepStrictEqual(result.error.message, 'FAILED') 163 | deepStrictEqual(result.size, 0) 164 | deepStrictEqual(result.min, 0) 165 | deepStrictEqual(result.max, 0) 166 | deepStrictEqual(result.mean, 0) 167 | deepStrictEqual(result.stddev, 0) 168 | deepStrictEqual(result.standardError, 0) 169 | deepStrictEqual(result.percentiles, {}) 170 | 171 | done() 172 | } 173 | ) 174 | }) 175 | 176 | test('Worker execution - Handle promise functions that resolve', (t, done) => { 177 | let mainCalls = 0 178 | const notifier = t.mock.fn() 179 | 180 | /* eslint-disable-next-line @typescript-eslint/require-await */ 181 | async function main(): Promise { 182 | mainCalls++ 183 | } 184 | 185 | runWorker( 186 | { 187 | path: 'fs', 188 | tests: [['main', main]], 189 | 190 | index: 0, 191 | iterations: 5, 192 | warmup: false, 193 | errorThreshold: 0 194 | }, 195 | notifier, 196 | code => { 197 | deepStrictEqual(code, 0) 198 | ok(mainCalls > 0) 199 | 200 | const result = notifier.mock.calls[0].arguments[0] as Result 201 | 202 | ok(result.success) 203 | ifError(result.error) 204 | deepStrictEqual(result.size, 5) 205 | ok(typeof result.min, 'number') 206 | ok(typeof result.max, 'number') 207 | ok(typeof result.mean, 'number') 208 | ok(typeof result.stddev, 'number') 209 | ok(typeof result.standardError, 'number') 210 | 211 | for (const percentile of percentiles) { 212 | ok(typeof result.percentiles[percentile.toString()], 'number') 213 | } 214 | 215 | done() 216 | } 217 | ) 218 | }) 219 | 220 | test('Worker execution - Handle promise functions that reject', (t, done) => { 221 | let mainCalls = 0 222 | const notifier = t.mock.fn() 223 | 224 | /* eslint-disable-next-line @typescript-eslint/require-await */ 225 | async function main(): Promise { 226 | mainCalls++ 227 | throw new Error('FAILED') 228 | } 229 | 230 | runWorker( 231 | { 232 | path: 'fs', 233 | tests: [['main', main]], 234 | 235 | index: 0, 236 | iterations: 5, 237 | warmup: false, 238 | errorThreshold: 0 239 | }, 240 | notifier, 241 | code => { 242 | deepStrictEqual(code, 1) 243 | ok(mainCalls > 0) 244 | 245 | const result = notifier.mock.calls[0].arguments[0] as Result 246 | 247 | ok(!result.success) 248 | ok(result.error instanceof Error) 249 | deepStrictEqual(result.error.message, 'FAILED') 250 | deepStrictEqual(result.size, 0) 251 | deepStrictEqual(result.min, 0) 252 | deepStrictEqual(result.max, 0) 253 | deepStrictEqual(result.mean, 0) 254 | deepStrictEqual(result.stddev, 0) 255 | deepStrictEqual(result.standardError, 0) 256 | deepStrictEqual(result.percentiles, {}) 257 | 258 | done() 259 | } 260 | ) 261 | }) 262 | 263 | test('Worker execution - Handle warmup mode enabled', (t, done) => { 264 | let mainCalls = 0 265 | const notifier = t.mock.fn() 266 | 267 | function main(): void { 268 | mainCalls++ 269 | } 270 | 271 | runWorker( 272 | { 273 | path: 'fs', 274 | tests: [['main', main]], 275 | 276 | index: 0, 277 | iterations: 5, 278 | warmup: true, 279 | errorThreshold: 0 280 | }, 281 | notifier, 282 | code => { 283 | deepStrictEqual(code, 0) 284 | deepStrictEqual(mainCalls, 10) 285 | deepStrictEqual(notifier.mock.callCount(), 1) 286 | 287 | const result = notifier.mock.calls[0].arguments[0] as Result 288 | 289 | ok(result.success) 290 | ifError(result.error) 291 | deepStrictEqual(result.size, 5) 292 | ok(typeof result.min, 'number') 293 | ok(typeof result.max, 'number') 294 | ok(typeof result.mean, 'number') 295 | ok(typeof result.stddev, 'number') 296 | ok(typeof result.standardError, 'number') 297 | 298 | for (const percentile of percentiles) { 299 | ok(typeof result.percentiles[percentile.toString()], 'number') 300 | } 301 | 302 | done() 303 | } 304 | ) 305 | }) 306 | 307 | test('Worker execution - Handle warmup mode disabled', (t, done) => { 308 | let mainCalls = 0 309 | const notifier = t.mock.fn() 310 | 311 | function main(): void { 312 | mainCalls++ 313 | } 314 | 315 | runWorker( 316 | { 317 | path: 'fs', 318 | tests: [['main', main]], 319 | 320 | index: 0, 321 | iterations: 5, 322 | warmup: false, 323 | errorThreshold: 0 324 | }, 325 | notifier, 326 | code => { 327 | deepStrictEqual(code, 0) 328 | deepStrictEqual(mainCalls, 5) 329 | deepStrictEqual(notifier.mock.callCount(), 1) 330 | 331 | const result = notifier.mock.calls[0].arguments[0] as Result 332 | 333 | ok(result.success) 334 | ifError(result.error) 335 | deepStrictEqual(result.size, 5) 336 | ok(typeof result.min, 'number') 337 | ok(typeof result.max, 'number') 338 | ok(typeof result.mean, 'number') 339 | ok(typeof result.stddev, 'number') 340 | ok(typeof result.standardError, 'number') 341 | 342 | for (const percentile of percentiles) { 343 | ok(typeof result.percentiles[percentile.toString()], 'number') 344 | } 345 | 346 | done() 347 | } 348 | ) 349 | }) 350 | 351 | test('Worker setup - Handle callback before functions', (t, done) => { 352 | let mainCalls = 0 353 | let setupCalls = 0 354 | const notifier = t.mock.fn() 355 | 356 | function main(): void { 357 | mainCalls++ 358 | } 359 | 360 | runWorker( 361 | { 362 | path: 'fs', 363 | tests: [ 364 | [ 365 | 'main', 366 | { 367 | test: main, 368 | before(cb: (err?: Error | null) => void): void { 369 | setupCalls++ 370 | cb() 371 | } 372 | } 373 | ] 374 | ], 375 | index: 0, 376 | iterations: 10_000, 377 | warmup: false, 378 | errorThreshold: 100 379 | }, 380 | notifier, 381 | code => { 382 | deepStrictEqual(code, 0) 383 | deepStrictEqual(setupCalls, 1) 384 | ok(mainCalls > 0) 385 | deepStrictEqual(notifier.mock.callCount(), 1) 386 | 387 | const result = notifier.mock.calls[0].arguments[0] as Result 388 | 389 | ok(result.success) 390 | ifError(result.error) 391 | deepStrictEqual(result.size, 1000) 392 | ok(typeof result.min, 'number') 393 | ok(typeof result.max, 'number') 394 | ok(typeof result.mean, 'number') 395 | ok(typeof result.stddev, 'number') 396 | ok(typeof result.standardError, 'number') 397 | 398 | for (const percentile of percentiles) { 399 | ok(typeof result.percentiles[percentile.toString()], 'number') 400 | } 401 | 402 | done() 403 | } 404 | ) 405 | }) 406 | 407 | test('Worker setup - Handle callback before functions that throw errors', (t, done) => { 408 | let mainCalls = 0 409 | const notifier = t.mock.fn() 410 | 411 | function main(): void { 412 | mainCalls++ 413 | } 414 | 415 | runWorker( 416 | { 417 | path: 'fs', 418 | tests: [ 419 | [ 420 | 'main', 421 | { 422 | test: main, 423 | before(cb: (err?: Error | null) => void): void { 424 | cb(new Error('FAILED')) 425 | } 426 | } 427 | ] 428 | ], 429 | index: 0, 430 | iterations: 10_000, 431 | warmup: false, 432 | errorThreshold: 100 433 | }, 434 | notifier, 435 | code => { 436 | deepStrictEqual(code, 1) 437 | ok(!mainCalls) 438 | 439 | const result = notifier.mock.calls[0].arguments[0] as Result 440 | 441 | ok(!result.success) 442 | ok(result.error instanceof Error) 443 | deepStrictEqual(result.error.message, 'FAILED') 444 | deepStrictEqual(result.size, 0) 445 | deepStrictEqual(result.min, 0) 446 | deepStrictEqual(result.max, 0) 447 | deepStrictEqual(result.mean, 0) 448 | deepStrictEqual(result.stddev, 0) 449 | deepStrictEqual(result.standardError, 0) 450 | deepStrictEqual(result.percentiles, {}) 451 | 452 | done() 453 | } 454 | ) 455 | }) 456 | 457 | test('Worker setup - Handle promise before functions that resolve', (t, done) => { 458 | let mainCalls = 0 459 | let setupCalls = 0 460 | const notifier = t.mock.fn() 461 | 462 | function main(): void { 463 | mainCalls++ 464 | } 465 | 466 | runWorker( 467 | { 468 | path: 'fs', 469 | tests: [ 470 | [ 471 | 'main', 472 | { 473 | test: main, 474 | before() { 475 | setupCalls++ 476 | return Promise.resolve() 477 | } 478 | } 479 | ] 480 | ], 481 | index: 0, 482 | iterations: 10_000, 483 | warmup: false, 484 | errorThreshold: 100 485 | }, 486 | notifier, 487 | code => { 488 | deepStrictEqual(code, 0) 489 | deepStrictEqual(setupCalls, 1) 490 | ok(mainCalls > 0) 491 | deepStrictEqual(notifier.mock.callCount(), 1) 492 | 493 | const result = notifier.mock.calls[0].arguments[0] as Result 494 | 495 | ok(result.success) 496 | ifError(result.error) 497 | deepStrictEqual(result.size, 1000) 498 | ok(typeof result.min, 'number') 499 | ok(typeof result.max, 'number') 500 | ok(typeof result.mean, 'number') 501 | ok(typeof result.stddev, 'number') 502 | ok(typeof result.standardError, 'number') 503 | 504 | for (const percentile of percentiles) { 505 | ok(typeof result.percentiles[percentile.toString()], 'number') 506 | } 507 | 508 | done() 509 | } 510 | ) 511 | }) 512 | 513 | test('Worker setup - Handle promise before functions that reject', (t, done) => { 514 | let mainCalls = 0 515 | const notifier = t.mock.fn() 516 | 517 | function main(): void { 518 | mainCalls++ 519 | } 520 | 521 | runWorker( 522 | { 523 | path: 'fs', 524 | tests: [ 525 | [ 526 | 'main', 527 | { 528 | test: main, 529 | before() { 530 | return Promise.reject(new Error('FAILED')) 531 | } 532 | } 533 | ] 534 | ], 535 | index: 0, 536 | iterations: 10_000, 537 | warmup: false, 538 | errorThreshold: 100 539 | }, 540 | notifier, 541 | code => { 542 | deepStrictEqual(code, 1) 543 | ok(!mainCalls) 544 | 545 | const result = notifier.mock.calls[0].arguments[0] as Result 546 | 547 | ok(!result.success) 548 | ok(result.error instanceof Error) 549 | deepStrictEqual(result.error.message, 'FAILED') 550 | deepStrictEqual(result.size, 0) 551 | deepStrictEqual(result.min, 0) 552 | deepStrictEqual(result.max, 0) 553 | deepStrictEqual(result.mean, 0) 554 | deepStrictEqual(result.stddev, 0) 555 | deepStrictEqual(result.standardError, 0) 556 | deepStrictEqual(result.percentiles, {}) 557 | 558 | done() 559 | } 560 | ) 561 | }) 562 | 563 | test('Worker setup - Handle callback after functions', (t, done) => { 564 | let mainCalls = 0 565 | let setupCalls = 0 566 | const notifier = t.mock.fn() 567 | 568 | function main(): void { 569 | mainCalls++ 570 | } 571 | 572 | runWorker( 573 | { 574 | path: 'fs', 575 | tests: [ 576 | [ 577 | 'main', 578 | { 579 | test: main, 580 | after(cb: (err?: Error | null) => void): void { 581 | setupCalls++ 582 | cb() 583 | } 584 | } 585 | ] 586 | ], 587 | index: 0, 588 | iterations: 10_000, 589 | warmup: false, 590 | errorThreshold: 100 591 | }, 592 | notifier, 593 | code => { 594 | deepStrictEqual(code, 0) 595 | deepStrictEqual(setupCalls, 1) 596 | ok(mainCalls > 0) 597 | deepStrictEqual(notifier.mock.callCount(), 1) 598 | 599 | const result = notifier.mock.calls[0].arguments[0] as Result 600 | 601 | ok(result.success) 602 | ifError(result.error) 603 | deepStrictEqual(result.size, 1000) 604 | ok(typeof result.min, 'number') 605 | ok(typeof result.max, 'number') 606 | ok(typeof result.mean, 'number') 607 | ok(typeof result.stddev, 'number') 608 | ok(typeof result.standardError, 'number') 609 | 610 | for (const percentile of percentiles) { 611 | ok(typeof result.percentiles[percentile.toString()], 'number') 612 | } 613 | 614 | done() 615 | } 616 | ) 617 | }) 618 | 619 | test('Worker setup - Handle callback after functions that throw errors', (t, done) => { 620 | let mainCalls = 0 621 | const notifier = t.mock.fn() 622 | 623 | function main(): void { 624 | mainCalls++ 625 | } 626 | 627 | runWorker( 628 | { 629 | path: 'fs', 630 | tests: [ 631 | [ 632 | 'main', 633 | { 634 | test: main, 635 | after(cb: (err?: Error | null) => void): void { 636 | cb(new Error('FAILED')) 637 | } 638 | } 639 | ] 640 | ], 641 | index: 0, 642 | iterations: 10_000, 643 | warmup: false, 644 | errorThreshold: 100 645 | }, 646 | notifier, 647 | code => { 648 | deepStrictEqual(code, 1) 649 | ok(mainCalls > 0) 650 | 651 | const result = notifier.mock.calls[0].arguments[0] as Result 652 | 653 | ok(!result.success) 654 | ok(result.error instanceof Error) 655 | deepStrictEqual(result.error.message, 'FAILED') 656 | deepStrictEqual(result.size, 0) 657 | deepStrictEqual(result.min, 0) 658 | deepStrictEqual(result.max, 0) 659 | deepStrictEqual(result.mean, 0) 660 | deepStrictEqual(result.stddev, 0) 661 | deepStrictEqual(result.standardError, 0) 662 | deepStrictEqual(result.percentiles, {}) 663 | 664 | done() 665 | } 666 | ) 667 | }) 668 | 669 | test('Worker setup - Handle promise after functions that resolve', (t, done) => { 670 | let mainCalls = 0 671 | let setupCalls = 0 672 | const notifier = t.mock.fn() 673 | 674 | function main(): void { 675 | mainCalls++ 676 | } 677 | 678 | runWorker( 679 | { 680 | path: 'fs', 681 | tests: [ 682 | [ 683 | 'main', 684 | { 685 | test: main, 686 | after() { 687 | setupCalls++ 688 | return Promise.resolve() 689 | } 690 | } 691 | ] 692 | ], 693 | index: 0, 694 | iterations: 10_000, 695 | warmup: false, 696 | errorThreshold: 100 697 | }, 698 | notifier, 699 | code => { 700 | deepStrictEqual(code, 0) 701 | deepStrictEqual(setupCalls, 1) 702 | ok(mainCalls > 0) 703 | deepStrictEqual(notifier.mock.callCount(), 1) 704 | 705 | const result = notifier.mock.calls[0].arguments[0] as Result 706 | 707 | ok(result.success) 708 | ifError(result.error) 709 | deepStrictEqual(result.size, 1000) 710 | ok(typeof result.min, 'number') 711 | ok(typeof result.max, 'number') 712 | ok(typeof result.mean, 'number') 713 | ok(typeof result.stddev, 'number') 714 | ok(typeof result.standardError, 'number') 715 | 716 | for (const percentile of percentiles) { 717 | ok(typeof result.percentiles[percentile.toString()], 'number') 718 | } 719 | 720 | done() 721 | } 722 | ) 723 | }) 724 | 725 | test('Worker setup - Handle promise after functions that reject', (t, done) => { 726 | let mainCalls = 0 727 | const notifier = t.mock.fn() 728 | 729 | function main(): void { 730 | mainCalls++ 731 | } 732 | 733 | runWorker( 734 | { 735 | path: 'fs', 736 | tests: [ 737 | [ 738 | 'main', 739 | { 740 | test: main, 741 | after() { 742 | return Promise.reject(new Error('FAILED')) 743 | } 744 | } 745 | ] 746 | ], 747 | index: 0, 748 | iterations: 10_000, 749 | warmup: false, 750 | errorThreshold: 100 751 | }, 752 | notifier, 753 | code => { 754 | deepStrictEqual(code, 1) 755 | ok(mainCalls > 0) 756 | 757 | const result = notifier.mock.calls[0].arguments[0] as Result 758 | 759 | ok(!result.success) 760 | ok(result.error instanceof Error) 761 | deepStrictEqual(result.error.message, 'FAILED') 762 | deepStrictEqual(result.size, 0) 763 | deepStrictEqual(result.min, 0) 764 | deepStrictEqual(result.max, 0) 765 | deepStrictEqual(result.mean, 0) 766 | deepStrictEqual(result.stddev, 0) 767 | deepStrictEqual(result.standardError, 0) 768 | deepStrictEqual(result.percentiles, {}) 769 | 770 | done() 771 | } 772 | ) 773 | }) 774 | 775 | test('Worker execution - Handle empty tests', (t, done) => { 776 | const notifier = t.mock.fn() 777 | 778 | runWorker( 779 | { 780 | path: 'fs', 781 | tests: [['main', {}]], 782 | index: 0, 783 | iterations: 10_000, 784 | warmup: false, 785 | errorThreshold: 100 786 | }, 787 | notifier, 788 | code => { 789 | deepStrictEqual(code, 0) 790 | 791 | const result = notifier.mock.calls[0].arguments[0] as Result 792 | 793 | ok(result.success) 794 | ifError(result.error) 795 | deepStrictEqual(result.size, 1000) 796 | ok(typeof result.min, 'number') 797 | ok(typeof result.max, 'number') 798 | ok(typeof result.mean, 'number') 799 | ok(typeof result.stddev, 'number') 800 | ok(typeof result.standardError, 'number') 801 | 802 | for (const percentile of percentiles) { 803 | ok(typeof result.percentiles[percentile.toString()], 'number') 804 | } 805 | 806 | done() 807 | } 808 | ) 809 | }) 810 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "module": "NodeNext", 5 | "moduleResolution": "NodeNext", 6 | "jsx": "preserve", 7 | "declaration": true, 8 | "outDir": "dist", 9 | "allowJs": false, 10 | "allowSyntheticDefaultImports": true, 11 | "esModuleInterop": true, 12 | "strict": true, 13 | "skipLibCheck": true, 14 | "noImplicitAny": true, 15 | "noUnusedLocals": true, 16 | "noUnusedParameters": true, 17 | "strictNullChecks": true, 18 | "useUnknownInCatchVariables": false, 19 | "allowImportingTsExtensions": true, 20 | "rewriteRelativeImportExtensions": true 21 | }, 22 | "include": ["src/*.ts"] 23 | } 24 | -------------------------------------------------------------------------------- /tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "include": ["src/*.ts", "test/**/*.ts"] 4 | } 5 | --------------------------------------------------------------------------------