├── .prettierignore ├── .eslintignore ├── .prettierrc ├── jest.config.js ├── .gitignore ├── .eslintrc.js ├── tsconfig.json ├── tsconfig.build.json ├── LICENSE ├── package.json ├── src ├── index.ts └── index.spec.ts └── README.md /.prettierignore: -------------------------------------------------------------------------------- 1 | lib/ 2 | coverage/ 3 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | lib/ 3 | coverage/ 4 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "trailingComma": "none" 4 | } 5 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line no-undef 2 | module.exports = { 3 | preset: 'ts-jest', 4 | testEnvironment: 'node' 5 | }; 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | *.seed 3 | *.log 4 | *.csv 5 | *.dat 6 | *.out 7 | *.pid 8 | *.gz 9 | *.swp 10 | 11 | pids 12 | logs 13 | results 14 | tmp 15 | 16 | # Coverage reports 17 | coverage 18 | 19 | # API keys and secrets 20 | .env 21 | 22 | # Dependency directory 23 | node_modules 24 | 25 | # Editors 26 | .idea 27 | *.iml 28 | .vscode 29 | 30 | # OS metadata 31 | .DS_Store 32 | Thumbs.db 33 | 34 | # Ignore built ts files 35 | lib/**/* 36 | 37 | # ignore yarn.lock 38 | yarn.lock 39 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | parser: '@typescript-eslint/parser', 4 | parserOptions: { 5 | ecmaVersion: 2017, 6 | project: './tsconfig.json' 7 | }, 8 | plugins: ['@typescript-eslint'], 9 | extends: [ 10 | 'eslint:recommended', 11 | 'plugin:@typescript-eslint/eslint-recommended', 12 | 'plugin:@typescript-eslint/recommended', 13 | 'prettier' 14 | ], 15 | rules: { 16 | '@typescript-eslint/no-floating-promises': ['error'], 17 | '@typescript-eslint/no-explicit-any': 'off' 18 | } 19 | }; 20 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2017", 4 | "lib": ["ES2017"], 5 | "module": "commonjs", 6 | "declaration": true, 7 | "outDir": "lib", 8 | "removeComments": false, 9 | "strict": true, 10 | "noUnusedLocals": true, 11 | "noUnusedParameters": true, 12 | "noImplicitReturns": true, 13 | "noFallthroughCasesInSwitch": true, 14 | "moduleResolution": "node", 15 | "esModuleInterop": true, 16 | "forceConsistentCasingInFileNames": true 17 | }, 18 | "include": ["src/**/*"] 19 | } 20 | -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2017", 4 | "lib": ["ES2017"], 5 | "module": "commonjs", 6 | "declaration": true, 7 | "outDir": "lib", 8 | "removeComments": false, 9 | "strict": true, 10 | "noUnusedLocals": true, 11 | "noUnusedParameters": true, 12 | "noImplicitReturns": true, 13 | "noFallthroughCasesInSwitch": true, 14 | "moduleResolution": "node", 15 | "esModuleInterop": true, 16 | "forceConsistentCasingInFileNames": true 17 | }, 18 | "include": ["src/**/*"], 19 | "exclude": ["src/**/*.spec.ts"] 20 | } 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 William E. Sorensen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tzientist", 3 | "version": "3.2.0", 4 | "description": "Scientist-like library for Node.js in TypeScript", 5 | "main": "lib/index.js", 6 | "typings": "lib/index.d.ts", 7 | "files": [ 8 | "lib" 9 | ], 10 | "scripts": { 11 | "test": "jest", 12 | "cover": "jest --coverage --coverageProvider=v8", 13 | "lint": "eslint \"src/**/*.ts\"", 14 | "prepare": "npm run build", 15 | "build": "tsc -p tsconfig.build.json" 16 | }, 17 | "engines": { 18 | "node": ">=14" 19 | }, 20 | "repository": { 21 | "type": "git", 22 | "url": "git+https://github.com/TrueWill/tzientist.git" 23 | }, 24 | "author": "William E. Sorensen", 25 | "license": "MIT", 26 | "keywords": [ 27 | "scientist", 28 | "refactoring", 29 | "typescript", 30 | "nodejs" 31 | ], 32 | "bugs": { 33 | "url": "https://github.com/TrueWill/tzientist/issues" 34 | }, 35 | "homepage": "https://github.com/TrueWill/tzientist#tzientist", 36 | "devDependencies": { 37 | "@types/jest": "^29.5.5", 38 | "@types/node": "^20.7.0", 39 | "@typescript-eslint/eslint-plugin": "^6.7.3", 40 | "@typescript-eslint/parser": "^6.7.3", 41 | "eslint": "^8.15.0", 42 | "eslint-config-prettier": "^9.0.0", 43 | "jest": "^29.7.0", 44 | "prettier": "3.0.3", 45 | "ts-jest": "^29.1.1", 46 | "typescript": "5.2.2" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export type ExperimentFunction = ( 2 | ...args: TParams 3 | ) => TResult; 4 | 5 | export type ExperimentAsyncFunction< 6 | TParams extends any[], 7 | TResult 8 | > = ExperimentFunction>; 9 | 10 | export interface Results { 11 | experimentName: string; 12 | experimentArguments: TParams; 13 | controlResult?: TResult; 14 | candidateResult?: TResult; 15 | controlError?: any; 16 | candidateError?: any; 17 | controlTimeMs?: number; 18 | candidateTimeMs?: number; 19 | } 20 | 21 | export interface Options { 22 | publish?: (results: Results) => void; 23 | enabled?: (...args: TParams) => boolean; 24 | } 25 | 26 | export type OptionsSync = Options< 27 | TParams, 28 | TResult 29 | >; 30 | 31 | export type OptionsAsync = Options< 32 | TParams, 33 | TResult 34 | > & { 35 | inParallel?: boolean; 36 | }; 37 | 38 | function hrtimeToMs(hrtime: [number, number]): number { 39 | const MS_PER_SEC = 1000; 40 | const NS_PER_MS = 1e6; 41 | const [seconds, nanoseconds] = hrtime; 42 | return seconds * MS_PER_SEC + nanoseconds / NS_PER_MS; 43 | } 44 | 45 | function defaultPublish( 46 | results: Results 47 | ): void { 48 | if ( 49 | results.candidateResult !== results.controlResult || 50 | (results.candidateError && !results.controlError) || 51 | (!results.candidateError && results.controlError) 52 | ) { 53 | console.warn(`Experiment ${results.experimentName}: difference found`); 54 | } 55 | } 56 | 57 | const defaultOptionsSync = { 58 | publish: defaultPublish 59 | }; 60 | 61 | /** 62 | * A factory that creates an experiment function. 63 | * 64 | * @param name - The name of the experiment, typically for use in publish. 65 | * @param control - The legacy function you are trying to replace. 66 | * @param candidate - The new function intended to replace the control. 67 | * @param [options] - Options for the experiment. You will usually want to specify a publish function. 68 | * @returns A function that acts like the control while also running the candidate and publishing results. 69 | */ 70 | export function experiment({ 71 | name, 72 | control, 73 | candidate, 74 | options = defaultOptionsSync 75 | }: { 76 | name: string; 77 | control: ExperimentFunction; 78 | candidate: ExperimentFunction; 79 | options?: OptionsSync; 80 | }): ExperimentFunction { 81 | const publish = options.publish || defaultPublish; 82 | 83 | return (...args): TResult => { 84 | let controlResult: TResult | undefined; 85 | let candidateResult: TResult | undefined; 86 | let controlError: any; 87 | let candidateError: any; 88 | let controlTimeMs: number; 89 | let candidateTimeMs: number; 90 | const isEnabled: boolean = !options.enabled || options.enabled(...args); 91 | 92 | function publishResults(): void { 93 | if (isEnabled) { 94 | publish({ 95 | experimentName: name, 96 | experimentArguments: args, 97 | controlResult, 98 | candidateResult, 99 | controlError, 100 | candidateError, 101 | controlTimeMs, 102 | candidateTimeMs 103 | }); 104 | } 105 | } 106 | 107 | if (isEnabled) { 108 | try { 109 | // Not using bigint version of hrtime for Node 8 compatibility 110 | const candidateStartTime = process.hrtime(); 111 | candidateResult = candidate(...args); 112 | candidateTimeMs = hrtimeToMs(process.hrtime(candidateStartTime)); 113 | } catch (e) { 114 | candidateError = e; 115 | } 116 | } 117 | 118 | try { 119 | const controlStartTime = process.hrtime(); 120 | controlResult = control(...args); 121 | controlTimeMs = hrtimeToMs(process.hrtime(controlStartTime)); 122 | } catch (e) { 123 | controlError = e; 124 | publishResults(); 125 | throw e; 126 | } 127 | 128 | publishResults(); 129 | return controlResult; 130 | }; 131 | } 132 | 133 | async function executeAndTime( 134 | controlOrCandidate: ExperimentAsyncFunction, 135 | args: TParams 136 | ): Promise<[TResult, number]> { 137 | // Not using bigint version of hrtime for Node 8 compatibility 138 | const startTime = process.hrtime(); 139 | const result = await controlOrCandidate(...args); 140 | const timeMs = hrtimeToMs(process.hrtime(startTime)); 141 | return [result, timeMs]; 142 | } 143 | 144 | const defaultOptionsAsync = { 145 | ...defaultOptionsSync, 146 | inParallel: true 147 | }; 148 | 149 | /** 150 | * A factory that creates an asynchronous experiment function. 151 | * 152 | * @param name - The name of the experiment, typically for use in publish. 153 | * @param control - The legacy async function you are trying to replace. 154 | * @param candidate - The new async function intended to replace the control. 155 | * @param [options] - Options for the experiment. You will usually want to specify a publish function. 156 | * @returns An async function that acts like the control while also running the candidate and publishing results. 157 | */ 158 | export function experimentAsync({ 159 | name, 160 | control, 161 | candidate, 162 | options = defaultOptionsAsync 163 | }: { 164 | name: string; 165 | control: ExperimentAsyncFunction; 166 | candidate: ExperimentAsyncFunction; 167 | options?: OptionsAsync; 168 | }): ExperimentAsyncFunction { 169 | const publish = options.publish ?? defaultOptionsAsync.publish; 170 | const inParallel = options.inParallel ?? defaultOptionsAsync.inParallel; 171 | 172 | return async (...args): Promise => { 173 | let controlResult: TResult | undefined; 174 | let candidateResult: TResult | undefined; 175 | let controlError: any; 176 | let candidateError: any; 177 | let controlTimeMs: number | undefined; 178 | let candidateTimeMs: number | undefined; 179 | const isEnabled: boolean = !options.enabled || options.enabled(...args); 180 | 181 | function publishResults(): void { 182 | if (isEnabled) { 183 | publish({ 184 | experimentName: name, 185 | experimentArguments: args, 186 | controlResult, 187 | candidateResult, 188 | controlError, 189 | candidateError, 190 | controlTimeMs, 191 | candidateTimeMs 192 | }); 193 | } 194 | } 195 | 196 | if (isEnabled) { 197 | const runFunctions = getRunFunctions(inParallel); 198 | [[candidateResult, candidateTimeMs], [controlResult, controlTimeMs]] = 199 | await runFunctions( 200 | () => 201 | executeAndTime(candidate, args).catch((e) => { 202 | candidateError = e; 203 | return [undefined, undefined]; 204 | }), 205 | () => 206 | executeAndTime(control, args).catch((e) => { 207 | controlError = e; 208 | return [undefined, undefined]; 209 | }) 210 | ); 211 | } else { 212 | controlResult = await control(...args).catch((e) => { 213 | controlError = e; 214 | return undefined; 215 | }); 216 | } 217 | 218 | publishResults(); 219 | 220 | if (controlError) { 221 | throw controlError; 222 | } 223 | 224 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion 225 | return controlResult!; 226 | }; 227 | } 228 | 229 | function getRunFunctions(inParallel: boolean) { 230 | return async ( 231 | function1: () => Promise<[TResult, number] | [undefined, undefined]>, 232 | function2: () => Promise<[TResult, number] | [undefined, undefined]> 233 | ) => { 234 | if (inParallel) { 235 | return Promise.all([function1(), function2()]); 236 | } 237 | return [await function1(), await function2()]; 238 | }; 239 | } 240 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tzientist 2 | 3 | A Scientist-like library for Node.js, implemented in TypeScript. 4 | 5 | It permits comparing legacy and refactored code paths in production environments, 6 | verifying both functional and non-functional requirements. 7 | This is also known as the Parallel Run pattern. 8 | 9 | ## Installation 10 | 11 | `npm i tzientist` 12 | 13 | or 14 | 15 | `yarn add tzientist` 16 | 17 | ## Getting started 18 | 19 | ```TypeScript 20 | import * as scientist from 'tzientist'; 21 | 22 | const experiment = scientist.experiment({ 23 | name: 'trial1', 24 | control: (s: string) => 'Control ' + s, 25 | candidate: (s: string) => 'not quite right ' + s 26 | }); 27 | 28 | console.log(experiment('C')); 29 | ``` 30 | 31 | This uses the default options and prints: 32 | 33 | ```Text 34 | Experiment trial1: difference found 35 | Control C 36 | ``` 37 | 38 | Note that `scientist.experiment` is a factory; it returns a function (named `experiment` in the example) that matches the signature of the `control` and the `candidate`. 39 | 40 | The `control` is the source of truth. It's typically the legacy code you're trying to replace. The `experiment` (the function returned by `scientist.experiment`) will always return whatever the `control` returns (**or will throw if the `control` throws**). You would replace the original call to `control` in your codebase with a call to `experiment`. 41 | 42 | The `candidate` is the new code you're testing that's intended to replace the `control` eventually. The `experiment` runs this code and publishes the result (along with the `control` result). The `experiment` will swallow any errors thrown by the `candidate`. 43 | 44 | The `experiment` runs both the `control` and the `candidate`, and it publishes the results to a callback function. Normally you will provide a custom `publish` function in the options that will report the results to some location for later analysis. 45 | 46 | ### Publishing results 47 | 48 | ```TypeScript 49 | function publish(results: scientist.Results<[string], string>): void { 50 | if (results.candidateResult !== results.controlResult) { 51 | console.log( 52 | `Experiment ${results.experimentName}: expected "${results.controlResult}" but got "${results.candidateResult}"` 53 | ); 54 | } 55 | } 56 | 57 | const experiment = scientist.experiment({ 58 | name: 'trial2', 59 | control: (s: string) => 'Control ' + s, 60 | candidate: (s: string) => 'not quite right ' + s, 61 | options: { publish } 62 | }); 63 | 64 | console.log(experiment('C')); 65 | ``` 66 | 67 | This prints: 68 | 69 | ```Text 70 | Experiment trial2: expected "Control C" but got "not quite right C" 71 | Control C 72 | ``` 73 | 74 | You will probably want to check `results.candidateError` and `results.controlError` as well. 75 | 76 | Typically you would replace `console.log` in `publish` with a call to a logging framework, persisting to a database, sending metrics to Grafana, etc. 77 | 78 | The results include the arguments passed to the experiment (`experimentArguments`). 79 | 80 | ### Sampling 81 | 82 | Running experiments can be expensive. Both the control and the candidate execute. If either may be slow or if the experiment runs in a performance-sensitive context, you may want to run the experiment on a percentage of traffic. You can provide a custom `enabled` function in the options. If `enabled` returns `false`, the experiment will still return what the control returns but it will not call the candidate nor will it publish results. If `enabled` returns `true`, the experiment will run normally. Tzientist passes the arguments to the experiment to the `enabled` function in case you want to base the sampling on them. 83 | 84 | Note: `enabled` receives the same arguments as the experiment, where `publish` receives a `Results` object with `experimentArguments` and other properties. 85 | 86 | ```TypeScript 87 | function enabled(_: string): boolean { 88 | // Run candidate 25% of the time 89 | return Math.floor(Math.random() * 100 + 1) <= 25; 90 | } 91 | 92 | const experiment = scientist.experiment({ 93 | name: 'trial3', 94 | control: (s: string) => 'Control ' + s, 95 | candidate: (s: string) => 'not quite right ' + s, 96 | options: { enabled } 97 | }); 98 | ``` 99 | 100 | ### Asynchronous code 101 | 102 | If your functions are async (returning a Promise), use `experimentAsync`. The resulting experiment function will return a Promise. 103 | 104 | ```TypeScript 105 | const experiment = scientist.experimentAsync({ 106 | name: 'async trial1', 107 | control: myAsyncControl, 108 | candidate: myAsyncCandidate, 109 | options: { publish } 110 | }); 111 | 112 | const result: number = await experiment(1, 2); 113 | ``` 114 | 115 | The `control` and the `candidate` will be run in parallel (that is, concurrently) by default. Options are the same as for a normal `experiment` with one exception - you can specify `inParallel: false` to run in serial (the candidate will run first). 116 | 117 | Note that Node applications run on a single thread, so if the functions are CPU-intensive then the experiment may take significantly longer than just running the original code. 118 | 119 | If your functions use callbacks, look at wrapping them with [util.promisify](https://nodejs.org/api/util.html#util_util_promisify_original). 120 | 121 | ### Timing / profiling 122 | 123 | Published results now include timings for both the control and the candidate. Timings are in milliseconds (ms). Note that other queued tasks could affect asynchronous timings, at least in theory. 124 | 125 | ## FAQ 126 | 127 | Q. Why would I use this library? 128 | 129 | A. You want to refactor or replace existing code, but that code is difficult or impossible to test with automated unit or integration tests. Perhaps it's nondeterministic. It might rely on data or on user input that is only available in a production environment. It could be a combinatorial explosion of states that requires too many test cases. Typically you would use this for high-risk changes, since you'll want to run the experiment for some time in production and check the results. 130 | 131 | --- 132 | 133 | Q. What if my candidate or control have side effects (such as updating a database)? 134 | 135 | A. In general, don't use Tzientist in those cases. 136 | 137 | --- 138 | 139 | Q. My candidate and control take different parameters. How do I handle that? 140 | 141 | A. Create a facade for one or both so that the parameters match. You don't need to use all of the parameters in both functions. 142 | 143 | --- 144 | 145 | Q. How do I configure custom compare, clean, or ignore functions? 146 | 147 | A. Tzientist always publishes results, so you can do all of the above in your `publish` function. `publish` can also delegate to other functions. 148 | 149 | --- 150 | 151 | Q. How do I configure a custom run_if function to conditionally disable an experiment? 152 | 153 | A. Tzientist passes the arguments to the experiment to the `enabled` function (if this is present in the options). If `enabled` returns `false`, the experiment will still return what the control returns but it will not call the candidate nor will it publish results. 154 | 155 | --- 156 | 157 | Q. What are some guidelines for writing `publish` and `enabled` functions? 158 | 159 | A. 160 | 161 | - Both `publish` and `enabled` should be fast 162 | - Both `publish` and `enabled` should **not** throw (they should catch any errors) 163 | - Both `publish` and `enabled` should **not** mutate the results (particularly the `controlResult`). Tzientist does not do any deep cloning on its own. If you want to alter the results for a custom compare, consider using something like [Lodash's](https://lodash.com/) `cloneDeep` first. 164 | 165 | --- 166 | 167 | Q. Why doesn't Tzientist randomize the order in which the control and the candidate are run? 168 | 169 | A. Because those functions should not have side effects. 170 | 171 | --- 172 | 173 | Q. What if the results always differ due to the data containing timestamps, GUIDs, etc.? 174 | 175 | A. One technique is to match those with regular expressions and replace them with a placeholder before comparing. 176 | 177 | --- 178 | 179 | Q. Will this work with [Deno](https://deno.land/)? 180 | 181 | A. No; instead, please check out [paleontologist](https://github.com/TrueWill/paleontologist). 182 | 183 | ## Why 184 | 185 | GitHub's [Scientist](https://github.com/github/scientist) Ruby library is a brilliant concept. Unfortunately the Node.js alternatives aren't very TypeScript-friendly. 186 | 187 | The goals of this project: 188 | 189 | - Simplicity 190 | - TypeScript support 191 | - Easy setup 192 | - Reasonable defaults 193 | - Good documentation 194 | - High test coverage 195 | 196 | Feature parity with Scientist is _not_ a goal. 197 | 198 | ## Contributing 199 | 200 | ### Technology stack 201 | 202 | - TypeScript v5.2 203 | - Node v18 (_may_ work on older versions) 204 | - npm v10 (I like yarn, but not everyone does) 205 | - [Prettier](https://prettier.io/) 206 | - ESLint 207 | - Jest 208 | 209 | ### Standards 210 | 211 | - TypeScript strict option 212 | - No classes 213 | - Prettier 3 with single quotes and no trailing commas 214 | - No ESLint warnings/errors 215 | - All tests pass 216 | - No dependencies (other than devDependencies) 217 | - [Semantic Versioning 2.0.0](https://semver.org/) 218 | 219 | **Note**: I use Mac OS, which uses Unix style (LF) line breaks. I haven't added a .gitattributes file yet. 220 | 221 | ## Thanks to 222 | 223 | - GitHub and all contributors for [Scientist](https://github.com/github/scientist) 224 | - Microsoft and all contributors for TypeScript 225 | - @mathieug for his contribution to this project! (The `inParallel` option.) 😺 226 | - Rashauna, Dani, TC, Jon, and many others from Redox, Inc. for educating me about Scientist and TypeScript 227 | - Jon for his feedback 228 | - Titian Cernicova-Dragomir for a key [Stack Overflow answer](https://stackoverflow.com/a/60469374/161457) on types 229 | - The rest of the TypeScript community (particularly the maintainers of [DefinitelyTyped](https://github.com/DefinitelyTyped/DefinitelyTyped)) 230 | - Michal Zalecki for the article [Creating a TypeScript library with a minimal setup](https://michalzalecki.com/creating-typescript-library-with-a-minimal-setup/) 231 | - Linus Unnebäck for a [Stack Overflow answer](https://stackoverflow.com/a/50466512/161457) 232 | - Higor Ramos for contributing 233 | - All of the creators, contributors, and maintainers of the open source used here 234 | - Sam Newman for discussing the Parallel Run pattern in the book [Monolith to Microservices](https://samnewman.io/books/monolith-to-microservices/) 235 | - OpenJS Foundation for Node.js 236 | - Future contributors 😺 237 | 238 | ## About the name 239 | 240 | I love puns, gaming, and vampires. Tzientist is named after the Tzimisce vampire clan from the game Vampire: The Masquerade; they are the ultimate scientists. 241 | 242 | ## Legal 243 | 244 | Tzimisce and Vampire: The Masquerade are copyrighted by or registered trademarks of CCP hf. 245 | 246 | Node.js is a trademark of Joyent, Inc. 247 | 248 | Tzientist is not published by, affiliated with, or endorsed by any of these organizations. 249 | -------------------------------------------------------------------------------- /src/index.spec.ts: -------------------------------------------------------------------------------- 1 | import * as scientist from './index'; 2 | 3 | describe('experiment', () => { 4 | const publishMock: jest.Mock]> = jest.fn< 5 | void, 6 | [scientist.Results] 7 | >(); 8 | 9 | afterEach(() => { 10 | publishMock.mockClear(); 11 | }); 12 | 13 | describe('when functions are equivalent', () => { 14 | function sum(a: number, b: number): number { 15 | return a + b; 16 | } 17 | 18 | function sum2(a: number, b: number): number { 19 | return b + a; 20 | } 21 | 22 | it('should return result', () => { 23 | const experiment = scientist.experiment({ 24 | name: 'equivalent1', 25 | control: sum, 26 | candidate: sum2, 27 | options: { 28 | publish: publishMock 29 | } 30 | }); 31 | 32 | const result: number = experiment(1, 2); 33 | 34 | expect(result).toBe(3); 35 | }); 36 | 37 | it('should publish results', () => { 38 | const experiment = scientist.experiment({ 39 | name: 'equivalent2', 40 | control: sum, 41 | candidate: sum2, 42 | options: { 43 | publish: publishMock 44 | } 45 | }); 46 | 47 | experiment(1, 2); 48 | 49 | expect(publishMock.mock.calls.length).toBe(1); 50 | const results = publishMock.mock.calls[0][0]; 51 | expect(results.experimentName).toBe('equivalent2'); 52 | expect(results.experimentArguments).toEqual([1, 2]); 53 | expect(results.controlResult).toBe(3); 54 | expect(results.candidateResult).toBe(3); 55 | expect(results.controlError).toBeUndefined(); 56 | expect(results.candidateError).toBeUndefined(); 57 | expect(results.controlTimeMs).toBeDefined(); 58 | expect(results.controlTimeMs).toBeGreaterThan(0); 59 | expect(results.candidateTimeMs).toBeDefined(); 60 | expect(results.candidateTimeMs).toBeGreaterThan(0); 61 | }); 62 | }); 63 | 64 | describe('when function results differ', () => { 65 | function ctrl(s: string): string { 66 | return `Ctrl+${s}`; 67 | } 68 | 69 | function candi(s: string): string { 70 | return s; 71 | } 72 | 73 | it('should return result of control', () => { 74 | const experiment = scientist.experiment({ 75 | name: 'differ1', 76 | control: ctrl, 77 | candidate: candi, 78 | options: { 79 | publish: publishMock 80 | } 81 | }); 82 | 83 | const result: string = experiment('C'); 84 | 85 | expect(result).toBe('Ctrl+C'); 86 | }); 87 | 88 | it('should publish results', () => { 89 | const experiment = scientist.experiment({ 90 | name: 'differ2', 91 | control: ctrl, 92 | candidate: candi, 93 | options: { 94 | publish: publishMock 95 | } 96 | }); 97 | 98 | experiment('C'); 99 | 100 | expect(publishMock.mock.calls.length).toBe(1); 101 | const results = publishMock.mock.calls[0][0]; 102 | expect(results.experimentName).toBe('differ2'); 103 | expect(results.experimentArguments).toEqual(['C']); 104 | expect(results.controlResult).toBe('Ctrl+C'); 105 | expect(results.candidateResult).toBe('C'); 106 | expect(results.controlError).toBeUndefined(); 107 | expect(results.candidateError).toBeUndefined(); 108 | expect(results.controlTimeMs).toBeDefined(); 109 | expect(results.controlTimeMs).toBeGreaterThan(0); 110 | expect(results.candidateTimeMs).toBeDefined(); 111 | expect(results.candidateTimeMs).toBeGreaterThan(0); 112 | }); 113 | }); 114 | 115 | describe('when candidate throws', () => { 116 | function ctrl(): string { 117 | return 'Everything is under control'; 118 | } 119 | 120 | function candi(): string { 121 | throw new Error("Candy I can't let you go"); 122 | } 123 | 124 | it('should return result of control', () => { 125 | const experiment = scientist.experiment({ 126 | name: 'throw1', 127 | control: ctrl, 128 | candidate: candi, 129 | options: { 130 | publish: publishMock 131 | } 132 | }); 133 | 134 | const result: string = experiment(); 135 | 136 | expect(result).toBe('Everything is under control'); 137 | }); 138 | 139 | it('should publish results', () => { 140 | const experiment = scientist.experiment({ 141 | name: 'throw2', 142 | control: ctrl, 143 | candidate: candi, 144 | options: { 145 | publish: publishMock 146 | } 147 | }); 148 | 149 | experiment(); 150 | 151 | expect(publishMock.mock.calls.length).toBe(1); 152 | const results = publishMock.mock.calls[0][0]; 153 | expect(results.experimentName).toBe('throw2'); 154 | expect(results.experimentArguments).toEqual([]); 155 | expect(results.controlResult).toBe('Everything is under control'); 156 | expect(results.candidateResult).toBeUndefined(); 157 | expect(results.controlError).toBeUndefined(); 158 | expect(results.candidateError).toBeDefined(); 159 | expect(results.candidateError.message).toBe("Candy I can't let you go"); 160 | expect(results.controlTimeMs).toBeDefined(); 161 | expect(results.controlTimeMs).toBeGreaterThan(0); 162 | expect(results.candidateTimeMs).toBeUndefined(); 163 | }); 164 | }); 165 | 166 | describe('when control throws', () => { 167 | function ctrl(): string { 168 | throw new Error('Kaos!'); 169 | } 170 | 171 | function candi(): string { 172 | return 'Kane'; 173 | } 174 | 175 | it('should throw', () => { 176 | const experiment = scientist.experiment({ 177 | name: 'cthrow1', 178 | control: ctrl, 179 | candidate: candi, 180 | options: { 181 | publish: publishMock 182 | } 183 | }); 184 | 185 | expect(() => experiment()).toThrowError('Kaos!'); 186 | }); 187 | 188 | it('should publish results', () => { 189 | const experiment = scientist.experiment({ 190 | name: 'cthrow2', 191 | control: ctrl, 192 | candidate: candi, 193 | options: { 194 | publish: publishMock 195 | } 196 | }); 197 | 198 | try { 199 | experiment(); 200 | } catch { 201 | // swallow error 202 | } 203 | 204 | expect(publishMock.mock.calls.length).toBe(1); 205 | const results = publishMock.mock.calls[0][0]; 206 | expect(results.experimentName).toBe('cthrow2'); 207 | expect(results.experimentArguments).toEqual([]); 208 | expect(results.controlResult).toBeUndefined(); 209 | expect(results.candidateResult).toBe('Kane'); 210 | expect(results.controlError).toBeDefined(); 211 | expect(results.controlError.message).toBe('Kaos!'); 212 | expect(results.candidateError).toBeUndefined(); 213 | expect(results.controlTimeMs).toBeUndefined(); 214 | expect(results.candidateTimeMs).toBeDefined(); 215 | expect(results.candidateTimeMs).toBeGreaterThan(0); 216 | }); 217 | }); 218 | 219 | describe('when both throw', () => { 220 | function ctrl(): string { 221 | throw new Error('Kaos!'); 222 | } 223 | 224 | function candi(): string { 225 | throw new Error("Candy I can't let you go"); 226 | } 227 | 228 | it('should throw control error', () => { 229 | const experiment = scientist.experiment({ 230 | name: 'bothrow1', 231 | control: ctrl, 232 | candidate: candi, 233 | options: { 234 | publish: publishMock 235 | } 236 | }); 237 | 238 | expect(() => experiment()).toThrowError('Kaos!'); 239 | }); 240 | 241 | it('should publish results', () => { 242 | const experiment = scientist.experiment({ 243 | name: 'bothrow2', 244 | control: ctrl, 245 | candidate: candi, 246 | options: { 247 | publish: publishMock 248 | } 249 | }); 250 | 251 | try { 252 | experiment(); 253 | } catch { 254 | // swallow error 255 | } 256 | 257 | expect(publishMock.mock.calls.length).toBe(1); 258 | const results = publishMock.mock.calls[0][0]; 259 | expect(results.experimentName).toBe('bothrow2'); 260 | expect(results.experimentArguments).toEqual([]); 261 | expect(results.controlResult).toBeUndefined(); 262 | expect(results.candidateResult).toBeUndefined(); 263 | expect(results.controlError).toBeDefined(); 264 | expect(results.controlError.message).toBe('Kaos!'); 265 | expect(results.candidateError).toBeDefined(); 266 | expect(results.candidateError.message).toBe("Candy I can't let you go"); 267 | expect(results.controlTimeMs).toBeUndefined(); 268 | expect(results.candidateTimeMs).toBeUndefined(); 269 | }); 270 | }); 271 | 272 | describe('when enabled option is specified', () => { 273 | const candidateMock: jest.Mock = jest.fn< 274 | string, 275 | [string] 276 | >(); 277 | 278 | afterEach(() => { 279 | candidateMock.mockClear(); 280 | }); 281 | 282 | describe('when control does not throw', () => { 283 | function ctrl(s: string): string { 284 | return `Ctrl+${s}`; 285 | } 286 | 287 | describe('when enabled returns false', () => { 288 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 289 | function enabled(_: string): boolean { 290 | return false; 291 | } 292 | 293 | it('should not run candidate', () => { 294 | const experiment = scientist.experiment({ 295 | name: 'disabled1', 296 | control: ctrl, 297 | candidate: candidateMock, 298 | options: { 299 | publish: publishMock, 300 | enabled 301 | } 302 | }); 303 | 304 | experiment('C'); 305 | 306 | expect(candidateMock.mock.calls.length).toBe(0); 307 | }); 308 | 309 | it('should return result of control', () => { 310 | const experiment = scientist.experiment({ 311 | name: 'disabled2', 312 | control: ctrl, 313 | candidate: candidateMock, 314 | options: { 315 | publish: publishMock, 316 | enabled 317 | } 318 | }); 319 | 320 | const result: string = experiment('C'); 321 | 322 | expect(result).toBe('Ctrl+C'); 323 | }); 324 | 325 | it('should not publish results', () => { 326 | const experiment = scientist.experiment({ 327 | name: 'disabled3', 328 | control: ctrl, 329 | candidate: candidateMock, 330 | options: { 331 | publish: publishMock, 332 | enabled 333 | } 334 | }); 335 | 336 | experiment('C'); 337 | 338 | expect(publishMock.mock.calls.length).toBe(0); 339 | }); 340 | }); 341 | 342 | describe('when enabled returns true', () => { 343 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 344 | function enabled(_: string): boolean { 345 | return true; 346 | } 347 | 348 | it('should run candidate', () => { 349 | const experiment = scientist.experiment({ 350 | name: 'enabled1', 351 | control: ctrl, 352 | candidate: candidateMock, 353 | options: { 354 | publish: publishMock, 355 | enabled 356 | } 357 | }); 358 | 359 | experiment('C'); 360 | 361 | expect(candidateMock.mock.calls.length).toBe(1); 362 | }); 363 | 364 | it('should return result of control', () => { 365 | const experiment = scientist.experiment({ 366 | name: 'enabled2', 367 | control: ctrl, 368 | candidate: candidateMock, 369 | options: { 370 | publish: publishMock, 371 | enabled 372 | } 373 | }); 374 | 375 | const result: string = experiment('C'); 376 | 377 | expect(result).toBe('Ctrl+C'); 378 | }); 379 | 380 | it('should publish results', () => { 381 | const experiment = scientist.experiment({ 382 | name: 'enabled3', 383 | control: ctrl, 384 | candidate: candidateMock, 385 | options: { 386 | publish: publishMock, 387 | enabled 388 | } 389 | }); 390 | 391 | experiment('C'); 392 | 393 | expect(publishMock.mock.calls.length).toBe(1); 394 | }); 395 | }); 396 | 397 | describe('when enabled function specified', () => { 398 | it('should pass experiment params to enabled', () => { 399 | const enabledMock: jest.Mock = jest 400 | .fn() 401 | .mockReturnValue(false); 402 | 403 | const experiment = scientist.experiment({ 404 | name: 'paramsToEnabled', 405 | control: ctrl, 406 | candidate: candidateMock, 407 | options: { 408 | publish: publishMock, 409 | enabled: enabledMock 410 | } 411 | }); 412 | 413 | experiment('myparam'); 414 | 415 | expect(enabledMock.mock.calls.length).toBe(1); 416 | expect(enabledMock.mock.calls[0][0]).toBe('myparam'); 417 | }); 418 | }); 419 | }); 420 | 421 | describe('when control throws', () => { 422 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 423 | function ctrl(_: string): string { 424 | throw new Error('Kaos!'); 425 | } 426 | 427 | describe('when enabled returns false', () => { 428 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 429 | function enabled(_: string): boolean { 430 | return false; 431 | } 432 | 433 | it('should throw', () => { 434 | const experiment = scientist.experiment({ 435 | name: 'disabledthrow1', 436 | control: ctrl, 437 | candidate: candidateMock, 438 | options: { 439 | publish: publishMock, 440 | enabled 441 | } 442 | }); 443 | 444 | expect(() => experiment('C')).toThrowError('Kaos!'); 445 | }); 446 | 447 | it('should not run candidate', () => { 448 | const experiment = scientist.experiment({ 449 | name: 'disabledthrow2', 450 | control: ctrl, 451 | candidate: candidateMock, 452 | options: { 453 | publish: publishMock, 454 | enabled 455 | } 456 | }); 457 | 458 | try { 459 | experiment('C'); 460 | } catch { 461 | // swallow error 462 | } 463 | 464 | expect(candidateMock.mock.calls.length).toBe(0); 465 | }); 466 | 467 | it('should not publish results', () => { 468 | const experiment = scientist.experiment({ 469 | name: 'disabledthrow3', 470 | control: ctrl, 471 | candidate: candidateMock, 472 | options: { 473 | publish: publishMock, 474 | enabled 475 | } 476 | }); 477 | 478 | try { 479 | experiment('C'); 480 | } catch { 481 | // swallow error 482 | } 483 | 484 | expect(publishMock.mock.calls.length).toBe(0); 485 | }); 486 | }); 487 | }); 488 | }); 489 | 490 | describe('when default options are used', () => { 491 | function ctrl(): number { 492 | return 1; 493 | } 494 | 495 | function candi(): number { 496 | return 2; 497 | } 498 | 499 | let consoleSpy: jest.SpyInstance; 500 | 501 | beforeEach(() => { 502 | // eslint-disable-next-line @typescript-eslint/no-empty-function 503 | consoleSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); 504 | }); 505 | 506 | afterEach(() => { 507 | jest.restoreAllMocks(); 508 | }); 509 | 510 | describe('when no options are specified', () => { 511 | it('should use sensible defaults', () => { 512 | const experiment = scientist.experiment({ 513 | name: 'no1', 514 | control: ctrl, 515 | candidate: candi 516 | }); 517 | 518 | experiment(); 519 | 520 | expect(consoleSpy.mock.calls.length).toBe(1); 521 | expect(consoleSpy.mock.calls[0][0]).toBe( 522 | 'Experiment no1: difference found' 523 | ); 524 | }); 525 | }); 526 | 527 | describe('when only publish option is specified', () => { 528 | it('should enable experiment', () => { 529 | const experiment = scientist.experiment({ 530 | name: 'opt1', 531 | control: ctrl, 532 | candidate: candi, 533 | options: { 534 | publish: publishMock 535 | } 536 | }); 537 | 538 | experiment(); 539 | 540 | expect(publishMock.mock.calls.length).toBe(1); 541 | const results = publishMock.mock.calls[0][0]; 542 | expect(results.controlResult).toBe(1); 543 | expect(results.candidateResult).toBe(2); 544 | }); 545 | }); 546 | 547 | describe('when only enabled option is specified', () => { 548 | it('should use default publish', () => { 549 | const experiment = scientist.experiment({ 550 | name: 'opt2', 551 | control: ctrl, 552 | candidate: candi, 553 | options: { 554 | enabled: (): boolean => true 555 | } 556 | }); 557 | 558 | experiment(); 559 | 560 | expect(consoleSpy.mock.calls.length).toBe(1); 561 | expect(consoleSpy.mock.calls[0][0]).toBe( 562 | 'Experiment opt2: difference found' 563 | ); 564 | }); 565 | 566 | it('should respect enabled', () => { 567 | const candidateMock: jest.Mock = jest.fn(); 568 | 569 | const experiment = scientist.experiment({ 570 | name: 'opt3', 571 | control: ctrl, 572 | candidate: candidateMock, 573 | options: { 574 | enabled: (): boolean => false 575 | } 576 | }); 577 | 578 | experiment(); 579 | 580 | expect(consoleSpy.mock.calls.length).toBe(0); 581 | expect(candidateMock.mock.calls.length).toBe(0); 582 | }); 583 | }); 584 | }); 585 | }); 586 | 587 | describe('experimentAsync', () => { 588 | const sleep = (ms: number): Promise => 589 | new Promise((resolve) => setTimeout(resolve, ms)); 590 | 591 | describe.each([ 592 | { inParallel: undefined }, 593 | { inParallel: true }, 594 | { inParallel: false } 595 | ])('when inParallel is $inParallel', ({ inParallel }) => { 596 | describe('when functions are equivalent', () => { 597 | const publishMock: jest.Mock< 598 | void, 599 | [scientist.Results<[number, number], number>] 600 | > = jest.fn]>(); 601 | 602 | afterEach(() => { 603 | publishMock.mockClear(); 604 | }); 605 | 606 | async function sum(a: number, b: number): Promise { 607 | await sleep(250); 608 | return a + b; 609 | } 610 | 611 | async function sum2(a: number, b: number): Promise { 612 | await sleep(125); 613 | return b + a; 614 | } 615 | 616 | it('should await result', async () => { 617 | const experiment = scientist.experimentAsync({ 618 | name: 'async equivalent1', 619 | control: sum, 620 | candidate: sum2, 621 | options: { 622 | publish: publishMock, 623 | inParallel 624 | } 625 | }); 626 | 627 | const result: number = await experiment(1, 2); 628 | 629 | expect(result).toBe(3); 630 | }); 631 | 632 | it('should publish results', async () => { 633 | const experiment = scientist.experimentAsync({ 634 | name: 'async equivalent2', 635 | control: sum, 636 | candidate: sum2, 637 | options: { 638 | publish: publishMock, 639 | inParallel 640 | } 641 | }); 642 | 643 | await experiment(1, 2); 644 | 645 | expect(publishMock.mock.calls.length).toBe(1); 646 | const results = publishMock.mock.calls[0][0]; 647 | expect(results.experimentName).toBe('async equivalent2'); 648 | expect(results.experimentArguments).toEqual([1, 2]); 649 | expect(results.controlResult).toBe(3); 650 | expect(results.candidateResult).toBe(3); 651 | expect(results.controlError).toBeUndefined(); 652 | expect(results.candidateError).toBeUndefined(); 653 | expect(results.controlTimeMs).toBeDefined(); 654 | expect(results.controlTimeMs).toBeGreaterThan(0); 655 | expect(results.candidateTimeMs).toBeDefined(); 656 | expect(results.candidateTimeMs).toBeGreaterThan(0); 657 | }); 658 | }); 659 | 660 | describe('when function results differ', () => { 661 | const publishMock: jest.Mock< 662 | void, 663 | [scientist.Results<[string], string>] 664 | > = jest.fn]>(); 665 | 666 | afterEach(() => { 667 | publishMock.mockClear(); 668 | }); 669 | 670 | async function ctrl(s: string): Promise { 671 | await sleep(250); 672 | return `Ctrl+${s}`; 673 | } 674 | 675 | async function candi(s: string): Promise { 676 | await sleep(125); 677 | return s; 678 | } 679 | 680 | it('should await result of control', async () => { 681 | const experiment = scientist.experimentAsync({ 682 | name: 'async differ1', 683 | control: ctrl, 684 | candidate: candi, 685 | options: { 686 | publish: publishMock, 687 | inParallel 688 | } 689 | }); 690 | 691 | const result: string = await experiment('C'); 692 | 693 | expect(result).toBe('Ctrl+C'); 694 | }); 695 | 696 | it('should publish results', async () => { 697 | const experiment = scientist.experimentAsync({ 698 | name: 'async differ2', 699 | control: ctrl, 700 | candidate: candi, 701 | options: { 702 | publish: publishMock, 703 | inParallel 704 | } 705 | }); 706 | 707 | await experiment('C'); 708 | 709 | expect(publishMock.mock.calls.length).toBe(1); 710 | const results = publishMock.mock.calls[0][0]; 711 | expect(results.experimentName).toBe('async differ2'); 712 | expect(results.experimentArguments).toEqual(['C']); 713 | expect(results.controlResult).toBe('Ctrl+C'); 714 | expect(results.candidateResult).toBe('C'); 715 | expect(results.controlError).toBeUndefined(); 716 | expect(results.candidateError).toBeUndefined(); 717 | expect(results.controlTimeMs).toBeDefined(); 718 | expect(results.controlTimeMs).toBeGreaterThan(0); 719 | expect(results.candidateTimeMs).toBeDefined(); 720 | expect(results.candidateTimeMs).toBeGreaterThan(0); 721 | }); 722 | }); 723 | 724 | describe('when candidate rejects', () => { 725 | const publishMock: jest.Mock]> = 726 | jest.fn]>(); 727 | 728 | afterEach(() => { 729 | publishMock.mockClear(); 730 | }); 731 | 732 | async function ctrl(): Promise { 733 | await sleep(125); 734 | return 'Everything is under control'; 735 | } 736 | 737 | async function candi(): Promise { 738 | return Promise.reject(new Error("Candy I can't let you go")); 739 | } 740 | 741 | it('should await result of control', async () => { 742 | const experiment = scientist.experimentAsync({ 743 | name: 'async throw1', 744 | control: ctrl, 745 | candidate: candi, 746 | options: { 747 | publish: publishMock, 748 | inParallel 749 | } 750 | }); 751 | 752 | const result: string = await experiment(); 753 | 754 | expect(result).toBe('Everything is under control'); 755 | }); 756 | 757 | it('should publish results', async () => { 758 | const experiment = scientist.experimentAsync({ 759 | name: 'async throw2', 760 | control: ctrl, 761 | candidate: candi, 762 | options: { 763 | publish: publishMock, 764 | inParallel 765 | } 766 | }); 767 | 768 | await experiment(); 769 | 770 | expect(publishMock.mock.calls.length).toBe(1); 771 | const results = publishMock.mock.calls[0][0]; 772 | expect(results.experimentName).toBe('async throw2'); 773 | expect(results.experimentArguments).toEqual([]); 774 | expect(results.controlResult).toBe('Everything is under control'); 775 | expect(results.candidateResult).toBeUndefined(); 776 | expect(results.controlError).toBeUndefined(); 777 | expect(results.candidateError).toBeDefined(); 778 | expect(results.candidateError.message).toBe("Candy I can't let you go"); 779 | expect(results.controlTimeMs).toBeDefined(); 780 | expect(results.controlTimeMs).toBeGreaterThan(0); 781 | expect(results.candidateTimeMs).toBeUndefined(); 782 | }); 783 | }); 784 | 785 | describe('when control rejects', () => { 786 | const publishMock: jest.Mock]> = 787 | jest.fn]>(); 788 | 789 | afterEach(() => { 790 | publishMock.mockClear(); 791 | }); 792 | 793 | async function ctrl(): Promise { 794 | throw new Error('Kaos!'); 795 | } 796 | 797 | async function candi(): Promise { 798 | await sleep(125); 799 | return 'Kane'; 800 | } 801 | 802 | it('should reject', () => { 803 | const experiment = scientist.experimentAsync({ 804 | name: 'async cthrow1', 805 | control: ctrl, 806 | candidate: candi, 807 | options: { 808 | publish: publishMock, 809 | inParallel 810 | } 811 | }); 812 | 813 | return expect(experiment()).rejects.toMatchObject({ message: 'Kaos!' }); 814 | }); 815 | 816 | it('should publish results', async () => { 817 | const experiment = scientist.experimentAsync({ 818 | name: 'async cthrow2', 819 | control: ctrl, 820 | candidate: candi, 821 | options: { 822 | publish: publishMock, 823 | inParallel 824 | } 825 | }); 826 | 827 | try { 828 | await experiment(); 829 | } catch { 830 | // swallow error 831 | } 832 | 833 | expect(publishMock.mock.calls.length).toBe(1); 834 | const results = publishMock.mock.calls[0][0]; 835 | expect(results.experimentName).toBe('async cthrow2'); 836 | expect(results.experimentArguments).toEqual([]); 837 | expect(results.controlResult).toBeUndefined(); 838 | expect(results.candidateResult).toBe('Kane'); 839 | expect(results.controlError).toBeDefined(); 840 | expect(results.controlError.message).toBe('Kaos!'); 841 | expect(results.candidateError).toBeUndefined(); 842 | expect(results.controlTimeMs).toBeUndefined(); 843 | expect(results.candidateTimeMs).toBeDefined(); 844 | expect(results.candidateTimeMs).toBeGreaterThan(0); 845 | }); 846 | }); 847 | 848 | describe('when both reject', () => { 849 | const publishMock: jest.Mock]> = 850 | jest.fn]>(); 851 | 852 | afterEach(() => { 853 | publishMock.mockClear(); 854 | }); 855 | 856 | async function ctrl(): Promise { 857 | throw new Error('Kaos!'); 858 | } 859 | 860 | async function candi(): Promise { 861 | return Promise.reject(new Error("Candy I can't let you go")); 862 | } 863 | 864 | it('should reject with control error', () => { 865 | const experiment = scientist.experimentAsync({ 866 | name: 'async bothrow1', 867 | control: ctrl, 868 | candidate: candi, 869 | options: { 870 | publish: publishMock, 871 | inParallel 872 | } 873 | }); 874 | 875 | return expect(experiment()).rejects.toMatchObject({ message: 'Kaos!' }); 876 | }); 877 | 878 | it('should publish results', async () => { 879 | const experiment = scientist.experimentAsync({ 880 | name: 'async bothrow2', 881 | control: ctrl, 882 | candidate: candi, 883 | options: { 884 | publish: publishMock, 885 | inParallel 886 | } 887 | }); 888 | 889 | try { 890 | await experiment(); 891 | } catch { 892 | // swallow error 893 | } 894 | 895 | expect(publishMock.mock.calls.length).toBe(1); 896 | const results = publishMock.mock.calls[0][0]; 897 | expect(results.experimentName).toBe('async bothrow2'); 898 | expect(results.experimentArguments).toEqual([]); 899 | expect(results.controlResult).toBeUndefined(); 900 | expect(results.candidateResult).toBeUndefined(); 901 | expect(results.controlError).toBeDefined(); 902 | expect(results.controlError.message).toBe('Kaos!'); 903 | expect(results.candidateError).toBeDefined(); 904 | expect(results.candidateError.message).toBe("Candy I can't let you go"); 905 | expect(results.controlTimeMs).toBeUndefined(); 906 | expect(results.candidateTimeMs).toBeUndefined(); 907 | }); 908 | }); 909 | 910 | describe('when enabled option is specified', () => { 911 | const publishMock: jest.Mock< 912 | void, 913 | [scientist.Results<[string], string>] 914 | > = jest.fn]>(); 915 | 916 | const candidateMock: jest.Mock, [string]> = jest.fn< 917 | Promise, 918 | [string] 919 | >(); 920 | 921 | afterEach(() => { 922 | publishMock.mockClear(); 923 | candidateMock.mockClear(); 924 | }); 925 | 926 | describe('when enabled returns false', () => { 927 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 928 | function enabled(_: string): boolean { 929 | return false; 930 | } 931 | 932 | describe('when control resolves', () => { 933 | async function ctrl(s: string): Promise { 934 | await sleep(125); 935 | return `Ctrl+${s}`; 936 | } 937 | 938 | it('should not run candidate', async () => { 939 | const experiment = scientist.experimentAsync({ 940 | name: 'async disabled1', 941 | control: ctrl, 942 | candidate: candidateMock, 943 | options: { 944 | publish: publishMock, 945 | enabled, 946 | inParallel 947 | } 948 | }); 949 | 950 | await experiment('C'); 951 | 952 | expect(candidateMock.mock.calls.length).toBe(0); 953 | }); 954 | 955 | it('should await result of control', async () => { 956 | const experiment = scientist.experimentAsync({ 957 | name: 'async disabled2', 958 | control: ctrl, 959 | candidate: candidateMock, 960 | options: { 961 | publish: publishMock, 962 | enabled, 963 | inParallel 964 | } 965 | }); 966 | 967 | const result: string = await experiment('C'); 968 | 969 | expect(result).toBe('Ctrl+C'); 970 | }); 971 | 972 | it('should not publish results', async () => { 973 | const experiment = scientist.experimentAsync({ 974 | name: 'async disabled3', 975 | control: ctrl, 976 | candidate: candidateMock, 977 | options: { 978 | publish: publishMock, 979 | enabled, 980 | inParallel 981 | } 982 | }); 983 | 984 | await experiment('C'); 985 | 986 | expect(publishMock.mock.calls.length).toBe(0); 987 | }); 988 | }); 989 | 990 | describe('when control rejects', () => { 991 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 992 | async function ctrl(_: string): Promise { 993 | throw new Error('Kaos!'); 994 | } 995 | 996 | it('should reject', () => { 997 | const experiment = scientist.experimentAsync({ 998 | name: 'async cthrow1', 999 | control: ctrl, 1000 | candidate: candidateMock, 1001 | options: { 1002 | publish: publishMock, 1003 | enabled, 1004 | inParallel 1005 | } 1006 | }); 1007 | 1008 | return expect(experiment('C')).rejects.toMatchObject({ 1009 | message: 'Kaos!' 1010 | }); 1011 | }); 1012 | 1013 | it('should not run candidate', async () => { 1014 | const experiment = scientist.experimentAsync({ 1015 | name: 'async disabledthrow2', 1016 | control: ctrl, 1017 | candidate: candidateMock, 1018 | options: { 1019 | publish: publishMock, 1020 | enabled, 1021 | inParallel 1022 | } 1023 | }); 1024 | 1025 | try { 1026 | await experiment('C'); 1027 | } catch { 1028 | // swallow error 1029 | } 1030 | 1031 | expect(candidateMock.mock.calls.length).toBe(0); 1032 | }); 1033 | 1034 | it('should not publish results', async () => { 1035 | const experiment = scientist.experimentAsync({ 1036 | name: 'async disabledthrow3', 1037 | control: ctrl, 1038 | candidate: candidateMock, 1039 | options: { 1040 | publish: publishMock, 1041 | enabled, 1042 | inParallel 1043 | } 1044 | }); 1045 | 1046 | try { 1047 | await experiment('C'); 1048 | } catch { 1049 | // swallow error 1050 | } 1051 | 1052 | expect(publishMock.mock.calls.length).toBe(0); 1053 | }); 1054 | }); 1055 | }); 1056 | }); 1057 | }); 1058 | 1059 | describe.each([ 1060 | { inParallel: undefined, msPerFunction: 1000, expectedElapsedTimeMs: 1000 }, 1061 | { inParallel: true, msPerFunction: 1000, expectedElapsedTimeMs: 1000 }, 1062 | { inParallel: false, msPerFunction: 1000, expectedElapsedTimeMs: 2000 } 1063 | ])( 1064 | 'when functions are slow and inParallel is $inParallel', 1065 | ({ inParallel, msPerFunction, expectedElapsedTimeMs }) => { 1066 | const publishMock: jest.Mock]> = 1067 | jest.fn]>(); 1068 | 1069 | afterEach(() => { 1070 | publishMock.mockClear(); 1071 | }); 1072 | 1073 | async function ctrl(): Promise { 1074 | await sleep(msPerFunction); 1075 | return 'Control'; 1076 | } 1077 | 1078 | async function candi(): Promise { 1079 | await sleep(msPerFunction); 1080 | return 'Candidate'; 1081 | } 1082 | 1083 | it('should run functions', async () => { 1084 | const nsPerMs = 1000000; 1085 | const allowedOverhead = 125; 1086 | 1087 | const experiment = scientist.experimentAsync({ 1088 | name: 'async parallel1', 1089 | control: ctrl, 1090 | candidate: candi, 1091 | options: { 1092 | publish: publishMock, 1093 | inParallel 1094 | } 1095 | }); 1096 | 1097 | const start = process.hrtime.bigint(); 1098 | await experiment(); 1099 | const end = process.hrtime.bigint(); 1100 | 1101 | const elapsedMs = Number((end - start) / BigInt(nsPerMs)); 1102 | 1103 | expect(elapsedMs).toBeLessThan(expectedElapsedTimeMs + allowedOverhead); 1104 | }); 1105 | 1106 | it('should publish individual timings', async () => { 1107 | const allowedVarianceMs = 125; 1108 | const minMs = msPerFunction - allowedVarianceMs; 1109 | const maxMs = msPerFunction + allowedVarianceMs; 1110 | const experiment = scientist.experimentAsync({ 1111 | name: 'async parallel2', 1112 | control: ctrl, 1113 | candidate: candi, 1114 | options: { 1115 | publish: publishMock, 1116 | inParallel 1117 | } 1118 | }); 1119 | 1120 | await experiment(); 1121 | 1122 | expect(publishMock.mock.calls.length).toBe(1); 1123 | const results = publishMock.mock.calls[0][0]; 1124 | expect(results.controlTimeMs).toBeDefined(); 1125 | expect(results.controlTimeMs).toBeGreaterThan(minMs); 1126 | expect(results.controlTimeMs).toBeLessThan(maxMs); 1127 | expect(results.candidateTimeMs).toBeDefined(); 1128 | expect(results.candidateTimeMs).toBeGreaterThan(minMs); 1129 | expect(results.candidateTimeMs).toBeLessThan(maxMs); 1130 | }); 1131 | } 1132 | ); 1133 | }); 1134 | --------------------------------------------------------------------------------