├── .eslintrc.json ├── .github └── workflows │ ├── ci.yml │ ├── codspeed.yml │ └── release.yml ├── .gitignore ├── .gitmodules ├── .husky ├── commit-msg └── pre-commit ├── .moon ├── tasks.yml ├── toolchain.yml └── workspace.yml ├── .npmrc ├── .nvmrc ├── .prettierignore ├── .prettierrc.json ├── .vscode ├── c_cpp_properties.json ├── launch.json └── settings.json ├── CLAUDE.md ├── LICENSE ├── README.md ├── commitlint.config.js ├── docs └── introspection.md ├── examples ├── with-javascript-esm │ ├── benchmark-js.js │ ├── package.json │ └── tinybench.js ├── with-typescript-cjs │ ├── bench │ │ ├── benchmark.js │ │ │ ├── fibo.bench.ts │ │ │ ├── foobarbaz.bench.ts │ │ │ └── index.bench.ts │ │ └── tinybench │ │ │ ├── fibo.bench.ts │ │ │ ├── foobarbaz.bench.ts │ │ │ └── index.bench.ts │ ├── package.json │ ├── src │ │ ├── fibonacci.ts │ │ └── foobarbaz.ts │ └── tsconfig.json ├── with-typescript-esm │ ├── bench │ │ ├── benchmark.js │ │ │ ├── fibo.bench.ts │ │ │ ├── foobarbaz.bench.ts │ │ │ └── index.bench.ts │ │ └── tinybench │ │ │ ├── fibo.bench.ts │ │ │ ├── foobarbaz.bench.ts │ │ │ └── index.bench.ts │ ├── package.json │ ├── src │ │ ├── fibonacci.bench.ts │ │ ├── fibonacci.test.ts │ │ ├── fibonacci.ts │ │ └── foobarbaz.ts │ ├── tsconfig.json │ └── vitest.config.ts ├── with-typescript-simple-cjs │ ├── benchmark-js.ts │ ├── package.json │ └── tinybench.ts └── with-typescript-simple-esm │ ├── benchmark-js.ts │ ├── package.json │ └── tinybench.ts ├── flake.lock ├── flake.nix ├── lerna.json ├── package.json ├── packages ├── benchmark.js-plugin │ ├── README.md │ ├── babel.config.js │ ├── benches │ │ ├── parsePr.ts │ │ └── sample.ts │ ├── jest.config.integ.js │ ├── jest.config.js │ ├── moon.yml │ ├── package.json │ ├── rollup.config.ts │ ├── src │ │ ├── __tests__ │ │ │ └── buildSuiteAdd.test.ts │ │ ├── buildSuiteAdd.ts │ │ ├── getCallingFile.ts │ │ ├── index.ts │ │ └── types.ts │ ├── tests │ │ ├── __snapshots__ │ │ │ └── index.integ.test.ts.snap │ │ ├── index.integ.test.ts │ │ ├── registerBenchmarks.ts │ │ └── registerOtherBenchmarks.ts │ ├── tsconfig.json │ └── tsconfig.test.json ├── core │ ├── .gitignore │ ├── README.md │ ├── binding.gyp │ ├── jest.config.integ.js │ ├── jest.config.js │ ├── moon.yml │ ├── package.json │ ├── rollup.config.ts │ ├── src │ │ ├── index.ts │ │ ├── introspection.ts │ │ ├── mongoMeasurement.ts │ │ ├── native_core │ │ │ ├── index.ts │ │ │ ├── instruments │ │ │ │ ├── hooks.ts │ │ │ │ ├── hooks_wrapper.cc │ │ │ │ └── hooks_wrapper.h │ │ │ ├── linux_perf │ │ │ │ ├── linux_perf.cc │ │ │ │ ├── linux_perf.h │ │ │ │ ├── linux_perf.ts │ │ │ │ ├── linux_perf_listener.cc │ │ │ │ └── utils.h │ │ │ └── native_core.cc │ │ ├── optimization.ts │ │ ├── utils.ts │ │ └── walltime │ │ │ ├── index.ts │ │ │ ├── interfaces.ts │ │ │ ├── quantiles.ts │ │ │ └── utils.ts │ ├── tests │ │ └── index.integ.test.ts │ ├── tracer.spec.json │ ├── tsconfig.json │ └── tsconfig.test.json ├── tinybench-plugin │ ├── README.md │ ├── benches │ │ ├── parsePr.ts │ │ ├── sample.ts │ │ └── timing.ts │ ├── jest.config.integ.js │ ├── jest.config.js │ ├── moon.yml │ ├── package.json │ ├── rollup.config.ts │ ├── src │ │ ├── index.ts │ │ ├── index.unit.test.ts │ │ ├── instrumented.ts │ │ ├── shared.ts │ │ ├── uri.ts │ │ └── walltime.ts │ ├── tests │ │ ├── __snapshots__ │ │ │ └── index.integ.test.ts.snap │ │ ├── index.integ.test.ts │ │ ├── registerBenchmarks.ts │ │ └── registerOtherBenchmarks.ts │ ├── tsconfig.json │ ├── tsconfig.test.json │ └── vitest.config.ts └── vitest-plugin │ ├── README.md │ ├── benches │ ├── flat.bench.ts │ ├── hooks.bench.ts │ ├── parsePr.bench.ts │ ├── parsePr.ts │ └── timing.bench.ts │ ├── moon.yml │ ├── package.json │ ├── rollup.config.ts │ ├── src │ ├── __tests__ │ │ ├── globalSetup.test.ts │ │ ├── index.test.ts │ │ └── instrumented.test.ts │ ├── common.ts │ ├── globalSetup.ts │ ├── index.ts │ ├── instrumented.ts │ ├── runner.ts │ └── walltime │ │ ├── index.ts │ │ └── utils.ts │ ├── tsconfig.json │ ├── tsconfig.test.json │ └── vitest.config.ts ├── pnpm-lock.yaml ├── pnpm-workspace.yaml ├── rollup.options.js ├── scripts └── release.sh ├── tsconfig.base.json └── tsconfig.json /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "extends": [ 4 | "eslint:recommended", 5 | "plugin:@typescript-eslint/recommended", 6 | "plugin:import/recommended", 7 | "plugin:import/typescript" 8 | ], 9 | "ignorePatterns": [ 10 | "**/dist/**", 11 | "**/node_modules/**", 12 | "**/rollup.config.ts", 13 | "**/jest.config.js", 14 | "packages/core/src/native_core/instruments/hooks/**" 15 | ], 16 | "settings": { 17 | "import/parsers": { 18 | "@typescript-eslint/parser": [".ts", ".tsx"] 19 | }, 20 | "import/resolver": { 21 | "typescript": { 22 | "alwaysTryTypes": true, 23 | "project": [ 24 | "tsconfig.json", 25 | "packages/*/tsconfig.json", 26 | "packages/*/tsconfig.*.json" 27 | ] 28 | } 29 | } 30 | }, 31 | "rules": { 32 | "import/no-named-as-default": "off", 33 | "import/no-named-as-default-member": "off" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: "CI" 2 | on: 3 | push: 4 | branches: 5 | - "main" 6 | pull_request: 7 | workflow_dispatch: 8 | 9 | jobs: 10 | check: 11 | strategy: 12 | matrix: 13 | os: ["ubuntu-latest", "codspeedhq-arm64-ubuntu-22.04"] 14 | runs-on: ${{ matrix.os }} 15 | steps: 16 | - uses: "actions/checkout@v4" 17 | with: 18 | fetch-depth: 0 19 | submodules: true 20 | - uses: pnpm/action-setup@v2 21 | - uses: actions/setup-node@v3 22 | with: 23 | cache: pnpm 24 | node-version-file: .nvmrc 25 | - run: pnpm install --frozen-lockfile --prefer-offline 26 | - run: pnpm moon check --all 27 | 28 | list-examples: 29 | runs-on: "ubuntu-latest" 30 | name: List examples 31 | outputs: 32 | examples: ${{ steps.list-examples.outputs.examples }} 33 | steps: 34 | - uses: "actions/checkout@v4" 35 | with: 36 | fetch-depth: 0 37 | submodules: true 38 | # list the directories in ./examples and output them to a github action workflow variables as a JSON array 39 | - run: | 40 | examples=$(find ./examples -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | jq -R -s -c 'split("\n") | map(select(length > 0))') 41 | echo "examples=$examples" >> $GITHUB_OUTPUT 42 | id: list-examples 43 | 44 | node-versions: 45 | runs-on: "ubuntu-latest" 46 | name: "${{ matrix.example }} on Node ${{ matrix.node-version }}" 47 | needs: list-examples 48 | strategy: 49 | matrix: 50 | node-version: ["18", "20.5.1"] 51 | example: ${{ fromJson(needs.list-examples.outputs.examples) }} 52 | fail-fast: false 53 | steps: 54 | - uses: "actions/checkout@v4" 55 | with: 56 | fetch-depth: 0 57 | submodules: true 58 | - uses: pnpm/action-setup@v2 59 | - uses: actions/setup-node@v3 60 | with: 61 | cache: pnpm 62 | node-version: ${{ matrix.node-version }} 63 | - run: pnpm install --frozen-lockfile --prefer-offline 64 | - run: pnpm moon run :build 65 | 66 | - name: Run benchmarks with tinybench-plugin 67 | # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2` 68 | uses: CodSpeedHQ/action@main 69 | with: 70 | mode: instrumentation 71 | run: pnpm --filter ${{ matrix.example }} bench-tinybench 72 | env: 73 | CODSPEED_SKIP_UPLOAD: true 74 | CODSPEED_DEBUG: true 75 | - name: Run benchmarks with benchmark.js-plugin 76 | # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2` 77 | uses: CodSpeedHQ/action@main 78 | with: 79 | mode: instrumentation 80 | run: pnpm --filter ${{ matrix.example }} bench-benchmark-js 81 | env: 82 | CODSPEED_SKIP_UPLOAD: true 83 | CODSPEED_DEBUG: true 84 | -------------------------------------------------------------------------------- /.github/workflows/codspeed.yml: -------------------------------------------------------------------------------- 1 | name: CodSpeed 2 | on: 3 | push: 4 | branches: 5 | - "main" 6 | pull_request: 7 | workflow_dispatch: 8 | 9 | jobs: 10 | codspeed-instrumented: 11 | name: Run CodSpeed instrumented 12 | runs-on: "ubuntu-latest" 13 | steps: 14 | - uses: "actions/checkout@v4" 15 | with: 16 | fetch-depth: 0 17 | submodules: true 18 | - uses: pnpm/action-setup@v2 19 | - uses: actions/setup-node@v3 20 | with: 21 | cache: pnpm 22 | node-version-file: .nvmrc 23 | - run: pnpm install --frozen-lockfile --prefer-offline 24 | - run: pnpm moon run :build 25 | 26 | - name: Run benchmarks 27 | # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2` 28 | uses: CodSpeedHQ/action@main 29 | with: 30 | mode: instrumentation 31 | run: | 32 | pnpm moon run tinybench-plugin:bench 33 | pnpm moon run vitest-plugin:bench 34 | pnpm moon run benchmark.js-plugin:bench 35 | pnpm --workspace-concurrency 1 -r bench-tinybench 36 | pnpm --workspace-concurrency 1 -r bench-benchmark-js 37 | pnpm --workspace-concurrency 1 -r bench-vitest 38 | 39 | codspeed-walltime: 40 | name: Run CodSpeed walltime 41 | runs-on: "codspeed-macro" 42 | steps: 43 | - uses: "actions/checkout@v4" 44 | with: 45 | fetch-depth: 0 46 | submodules: true 47 | - uses: pnpm/action-setup@v2 48 | - uses: actions/setup-node@v3 49 | with: 50 | cache: pnpm 51 | node-version-file: .nvmrc 52 | - run: pnpm install --frozen-lockfile --prefer-offline 53 | - run: pnpm moon run :build 54 | 55 | - name: Run benchmarks 56 | # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2` 57 | uses: CodSpeedHQ/action@main 58 | with: 59 | mode: walltime 60 | run: | 61 | pnpm moon run tinybench-plugin:bench 62 | pnpm moon run vitest-plugin:bench 63 | pnpm moon run benchmark.js-plugin:bench 64 | pnpm --workspace-concurrency 1 -r bench-tinybench 65 | pnpm --workspace-concurrency 1 -r bench-vitest 66 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release on tag 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" 7 | 8 | permissions: 9 | id-token: write 10 | contents: write 11 | 12 | jobs: 13 | build-native-arm: 14 | runs-on: codspeedhq-arm64-ubuntu-22.04 15 | 16 | steps: 17 | - uses: actions/checkout@v4 18 | with: 19 | fetch-depth: 0 20 | submodules: true 21 | - uses: pnpm/action-setup@v2 22 | - uses: actions/setup-node@v3 23 | with: 24 | cache: pnpm 25 | node-version-file: .nvmrc 26 | - run: pnpm install --frozen-lockfile --prefer-offline 27 | - name: Build native code on ARM 28 | run: pnpm moon core:build-native-addon 29 | - name: Upload ARM prebuilds 30 | uses: actions/upload-artifact@v4 31 | with: 32 | name: arm-prebuilds 33 | path: packages/core/prebuilds 34 | 35 | build: 36 | runs-on: ubuntu-latest 37 | needs: build-native-arm 38 | 39 | steps: 40 | - uses: actions/checkout@v4 41 | with: 42 | fetch-depth: 0 43 | submodules: true 44 | - uses: pnpm/action-setup@v2 45 | - uses: actions/setup-node@v3 46 | with: 47 | cache: pnpm 48 | node-version-file: .nvmrc 49 | registry-url: "https://registry.npmjs.org" 50 | - run: pnpm install --frozen-lockfile --prefer-offline 51 | - name: Build the libraries 52 | run: pnpm moon run :build 53 | 54 | - name: Download ARM prebuilds 55 | uses: actions/download-artifact@v4 56 | with: 57 | name: arm-prebuilds 58 | path: packages/core/prebuilds 59 | 60 | - name: Publish the libraries 61 | run: | 62 | if [[ "${{ github.ref }}" == *"-alpha"* ]]; then 63 | pnpm publish -r --access=public --no-git-checks --tag=alpha 64 | else 65 | pnpm publish -r --access=public --no-git-checks 66 | fi 67 | env: 68 | NPM_CONFIG_PROVENANCE: true 69 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 70 | 71 | - name: Create a draft release 72 | run: | 73 | NEW_VERSION=$(pnpm lerna list --json | jq -r '.[] | select(.name == "@codspeed/core") | .version') 74 | gh release create v$NEW_VERSION --title "v$NEW_VERSION" --generate-notes -d 75 | env: 76 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 77 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | .pnpm-debug.log* 9 | 10 | # JIT dumps 11 | jit-*.dump 12 | 13 | # Diagnostic reports (https://nodejs.org/api/report.html) 14 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 15 | 16 | # Runtime data 17 | pids 18 | *.pid 19 | *.seed 20 | *.pid.lock 21 | 22 | # Directory for instrumented libs generated by jscoverage/JSCover 23 | lib-cov 24 | 25 | # Coverage directory used by tools like istanbul 26 | coverage 27 | *.lcov 28 | 29 | # nyc test coverage 30 | .nyc_output 31 | 32 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 33 | .grunt 34 | 35 | # Bower dependency directory (https://bower.io/) 36 | bower_components 37 | 38 | # node-waf configuration 39 | .lock-wscript 40 | 41 | # Compiled binary addons (https://nodejs.org/api/addons.html) 42 | build/Release 43 | 44 | # Dependency directories 45 | node_modules/ 46 | jspm_packages/ 47 | 48 | # Snowpack dependency directory (https://snowpack.dev/) 49 | web_modules/ 50 | 51 | # TypeScript cache 52 | *.tsbuildinfo 53 | 54 | # Optional npm cache directory 55 | .npm 56 | 57 | # Optional eslint cache 58 | .eslintcache 59 | 60 | # Optional stylelint cache 61 | .stylelintcache 62 | 63 | # Microbundle cache 64 | .rpt2_cache/ 65 | .rts2_cache_cjs/ 66 | .rts2_cache_es/ 67 | .rts2_cache_umd/ 68 | 69 | # Optional REPL history 70 | .node_repl_history 71 | 72 | # Output of 'npm pack' 73 | *.tgz 74 | 75 | # Yarn Integrity file 76 | .yarn-integrity 77 | 78 | # dotenv environment variable files 79 | .env 80 | .env.development.local 81 | .env.test.local 82 | .env.production.local 83 | .env.local 84 | 85 | # parcel-bundler cache (https://parceljs.org/) 86 | .cache 87 | .parcel-cache 88 | 89 | # Next.js build output 90 | .next 91 | out 92 | 93 | # Nuxt.js build / generate output 94 | .nuxt 95 | dist 96 | 97 | # Gatsby files 98 | .cache/ 99 | # Comment in the public line in if your project uses Gatsby and not Next.js 100 | # https://nextjs.org/blog/next-9-1#public-directory-support 101 | # public 102 | 103 | # vuepress build output 104 | .vuepress/dist 105 | 106 | # vuepress v2.x temp and cache directory 107 | .temp 108 | .cache 109 | 110 | # Serverless directories 111 | .serverless/ 112 | 113 | # FuseBox cache 114 | .fusebox/ 115 | 116 | # DynamoDB Local files 117 | .dynamodb/ 118 | 119 | # TernJS port file 120 | .tern-port 121 | 122 | # Stores VSCode versions used for testing VSCode extensions 123 | .vscode-test 124 | 125 | 126 | pnpm-global 127 | packages/app/.env 128 | 129 | .vercel 130 | .DS_Store 131 | 132 | # moon 133 | .moon/cache 134 | .moon/docker 135 | .rollup.cache/ 136 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "packages/core/src/native_core/instruments/hooks"] 2 | path = packages/core/src/native_core/instruments/hooks 3 | url = git@github.com:CodSpeedHQ/instrument-hooks.git 4 | -------------------------------------------------------------------------------- /.husky/commit-msg: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | . "$(dirname "$0")/_/husky.sh" 3 | 4 | pnpm commitlint --edit ${1} 5 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | . "$(dirname "$0")/_/husky.sh" 3 | 4 | pnpm moon run :lint :format :typecheck --affected --status=staged 5 | 6 | -------------------------------------------------------------------------------- /.moon/tasks.yml: -------------------------------------------------------------------------------- 1 | # https://moonrepo.dev/docs/config/global-project 2 | $schema: "https://moonrepo.dev/schemas/global-project.json" 3 | 4 | fileGroups: 5 | configs: 6 | - "*.config.{js,cjs,mjs,ts}" 7 | - ".eslintrc.js" 8 | - "tsconfig.*.json" 9 | 10 | sources: 11 | - "src/**/*" 12 | - "types/**/*" 13 | tests: 14 | - "tests/**/*.test.*" 15 | - "**/__tests__/**/*" 16 | dist: 17 | - "dist/**/*" 18 | 19 | tasks: 20 | format: 21 | command: "prettier --config @in(0) --ignore-path @in(1) --check ." 22 | inputs: 23 | - "/.prettierrc.json" 24 | - "/.prettierignore" 25 | - "@globs(sources)" 26 | - "@globs(tests)" 27 | - "@globs(configs)" 28 | fix-format: 29 | local: true 30 | command: "prettier --config @in(0) --ignore-path @in(1) --write ." 31 | inputs: 32 | - "/.prettierrc.json" 33 | - "/.prettierignore" 34 | - "@globs(sources)" 35 | - "@globs(tests)" 36 | - "@globs(configs)" 37 | lint: 38 | command: "eslint ." 39 | inputs: 40 | - "@globs(sources)" 41 | - "@globs(tests)" 42 | - ".eslintignore" 43 | - ".eslintrc.js" 44 | - "/.eslintrc.js" 45 | - "tsconfig.json" 46 | - "tsconfig.*.json" 47 | deps: 48 | - "build" 49 | 50 | typecheck: 51 | command: "tsc --noEmit --pretty" 52 | inputs: 53 | - "@globs(sources)" 54 | - "@globs(tests)" 55 | - "tsconfig.json" 56 | - "/tsconfig.json" 57 | - "/tsconfig.base.json" 58 | deps: 59 | - "build" 60 | 61 | build: 62 | command: "rollup -c rollup.config.ts --configPlugin typescript" 63 | inputs: 64 | - "@globs(sources)" 65 | - "rollup.config.ts" 66 | outputs: 67 | - "dist/" 68 | deps: 69 | - "^:build" 70 | env: 71 | NODE_NO_WARNINGS: "1" 72 | 73 | test: 74 | command: "jest --passWithNoTests --silent" 75 | inputs: 76 | - "@globs(sources)" 77 | - "@globs(tests)" 78 | - "@globs(configs)" 79 | - "tsconfig.json" 80 | - "/tsconfig.json" 81 | - "/tsconfig.base.json" 82 | deps: 83 | - "build" 84 | - "test/integ" 85 | 86 | test/integ: 87 | command: "jest --passWithNoTests --silent -c jest.config.integ.js" 88 | inputs: 89 | - "@globs(sources)" 90 | - "@globs(tests)" 91 | - "@globs(configs)" 92 | - "tsconfig.json" 93 | - "/tsconfig.json" 94 | - "/tsconfig.base.json" 95 | deps: 96 | - "build" 97 | 98 | clean: 99 | command: "rm -rf" 100 | args: 101 | - dist 102 | local: true 103 | options: 104 | cache: false 105 | platform: system 106 | -------------------------------------------------------------------------------- /.moon/toolchain.yml: -------------------------------------------------------------------------------- 1 | # https://moonrepo.dev/docs/config/toolchain 2 | $schema: "https://moonrepo.dev/schemas/toolchain.json" 3 | 4 | node: 5 | packageManager: "pnpm" 6 | pnpm: 7 | version: "10.12.4" 8 | 9 | dedupeOnLockfileChange: false 10 | dependencyVersionFormat: "workspace" 11 | inferTasksFromScripts: true 12 | syncProjectWorkspaceDependencies: true 13 | 14 | typescript: 15 | createMissingConfig: false 16 | rootConfigFileName: "tsconfig.json" 17 | rootOptionsConfigFileName: "tsconfig.base.json" 18 | 19 | routeOutDirToCache: false 20 | syncProjectReferences: false 21 | syncProjectReferencesToPaths: false 22 | -------------------------------------------------------------------------------- /.moon/workspace.yml: -------------------------------------------------------------------------------- 1 | # https://moonrepo.dev/docs/config/workspace 2 | $schema: 'https://moonrepo.dev/schemas/workspace.json' 3 | 4 | projects: 5 | - 'packages/*' 6 | 7 | vcs: 8 | manager: 'git' 9 | defaultBranch: 'main' 10 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | auto-install-peers=true 2 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 20.5.1 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .rollup.cache 3 | dist 4 | generated 5 | packages/core/src/native_core/instruments/hooks 6 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "plugins": ["prettier-plugin-organize-imports"] 3 | } 4 | -------------------------------------------------------------------------------- /.vscode/c_cpp_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "Linux", 5 | "includePath": [ 6 | "${workspaceFolder}/**", 7 | "/usr/include/node" 8 | ], 9 | "defines": [], 10 | "compilerPath": "/usr/bin/clang", 11 | "cStandard": "c17", 12 | "cppStandard": "c++14", 13 | "intelliSenseMode": "linux-clang-x64", 14 | "configurationProvider": "ms-vscode.makefile-tools" 15 | } 16 | ], 17 | "version": 4 18 | } -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "type": "node", 6 | "request": "attach", 7 | "name": "Attach to Node", 8 | "port": 9229 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "C_Cpp.default.configurationProvider": "ms-vscode.makefile-tools", 3 | "makefile.makeDirectory": "${workspaceRoot}/packages/core/build/", 4 | "files.exclude": {}, 5 | "editor.defaultFormatter": "esbenp.prettier-vscode", 6 | "editor.formatOnSave": true, 7 | "editor.codeActionsOnSave": { 8 | "source.organizeImports": "never" // Import sorting is handled by prettier 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /CLAUDE.md: -------------------------------------------------------------------------------- 1 | # CodSpeed Node Repository Layout 2 | 3 | ## Repository Structure 4 | 5 | This is a monorepo containing CodSpeed plugins for various Node.js benchmarking frameworks. 6 | 7 | ### Root Level 8 | - `package.json` - Root package configuration 9 | - `pnpm-workspace.yaml` - PNPM workspace configuration 10 | - `lerna.json` - Lerna monorepo configuration 11 | - `tsconfig.base.json` - Base TypeScript configuration 12 | - `rollup.options.js` - Rollup bundler configuration 13 | - `scripts/` - Build and release scripts 14 | - `docs/` - Documentation files 15 | - `examples/` - Example projects using the plugins 16 | 17 | ### Packages (`packages/`) 18 | 19 | #### Core Package (`packages/core/`) 20 | - **Purpose**: Core measurement and instrumentation functionality 21 | - **Key files**: 22 | - `src/index.ts` - Main exports, setupCore/teardownCore functions 23 | - `src/mongoMeasurement.ts` - MongoDB measurement handling 24 | - `src/optimization.ts` - Function optimization utilities 25 | - `src/native_core/` - Native C++ bindings for performance measurement 26 | - `src/introspection.ts` - V8 flags and runtime introspection 27 | 28 | #### Tinybench Plugin (`packages/tinybench-plugin/`) 29 | - **Purpose**: CodSpeed integration for tinybench framework 30 | - **Key files**: 31 | - `src/index.ts` - Main plugin implementation with `withCodSpeed()` function 32 | - `tests/index.integ.test.ts` - Integration tests 33 | - `benches/` - Benchmark examples 34 | 35 | #### Benchmark.js Plugin (`packages/benchmark.js-plugin/`) 36 | - **Purpose**: CodSpeed integration for benchmark.js framework 37 | - **Key files**: 38 | - `src/index.ts` - Main plugin implementation 39 | - `src/buildSuiteAdd.ts` - Suite building utilities 40 | 41 | #### Vitest Plugin (`packages/vitest-plugin/`) 42 | - **Purpose**: CodSpeed integration for Vitest framework 43 | - **Key files**: 44 | - `src/index.ts` - Main plugin implementation 45 | - `src/runner.ts` - Custom test runner 46 | - `src/globalSetup.ts` - Global setup configuration 47 | 48 | ### Examples Directory (`examples/`) 49 | - `with-javascript-cjs/` - CommonJS JavaScript examples 50 | - `with-javascript-esm/` - ESM JavaScript examples 51 | - `with-typescript-cjs/` - CommonJS TypeScript examples 52 | - `with-typescript-esm/` - ESM TypeScript examples 53 | - `with-typescript-simple-cjs/` - Simple CommonJS TypeScript examples 54 | - `with-typescript-simple-esm/` - Simple ESM TypeScript examples 55 | 56 | ## Tinybench Plugin Architecture 57 | 58 | ### Current Stats/Measurements Access 59 | 60 | The tinybench plugin currently has **limited stats exposure**: 61 | 62 | 1. **No direct stats API**: The `withCodSpeed()` function wraps a tinybench instance but doesn't expose measurement results 63 | 2. **Console-only output**: Results are only printed to console via `console.log()` 64 | 3. **Core measurement**: Uses `@codspeed/core` for actual measurement via: 65 | - `mongoMeasurement.start(uri)` / `mongoMeasurement.stop(uri)` 66 | - `Measurement.startInstrumentation()` / `Measurement.stopInstrumentation(uri)` 67 | 68 | ### Current Workflow 69 | 1. User calls `withCodSpeed(new Bench())` to wrap their tinybench instance 70 | 2. Plugin intercepts `bench.run()` to add CodSpeed instrumentation 71 | 3. Each benchmark task runs with measurement instrumentation 72 | 4. Results are logged to console but not returned as structured data 73 | 74 | ### Key Functions in tinybench plugin 75 | - `withCodSpeed(bench: Bench): Bench` - Main wrapper function 76 | - `setupInstruments(body)` - Dynamic instrument setup 77 | - `getCallingFile()` - Helper to generate unique URIs for benchmarks 78 | 79 | ## Potential Enhancement Areas 80 | 81 | Based on the codebase analysis, to add stats access features: 82 | 83 | 1. **Extend return value**: Modify `bench.run()` to return structured measurement data 84 | 2. **Add stats methods**: Add methods like `getStats()`, `getResults()`, `getLastRunStats()` 85 | 3. **Integrate with core**: Leverage `@codspeed/core` measurement data 86 | 4. **Maintain tinybench compatibility**: Ensure existing `bench.table()` still works 87 | 88 | ## Repository Management Memories 89 | 90 | - Use pnpm instead of npm 91 | - To run tests in a package use moon :test -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2023 CodSpeed 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 |

codspeed-node

3 | 4 | Node.js libraries to create CodSpeed benchmarks 5 | 6 | [![CI](https://github.com/CodSpeedHQ/codspeed-node/actions/workflows/ci.yml/badge.svg)](https://github.com/CodSpeedHQ/codspeed-node/actions/workflows/ci.yml) 7 | [![npm (scoped)](https://img.shields.io/npm/v/@codspeed/core)](https://www.npmjs.com/org/codspeed) 8 | [![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF) 9 | [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/CodSpeedHQ/codspeed-node) 10 | 11 |
12 | 13 | ## Documentation 14 | 15 | Check out the [documentation](https://docs.codspeed.io/benchmarks/nodejs) for complete integration instructions. 16 | 17 | ## Packages 18 | 19 | This mono-repo contains the integration packages for using CodSpeed with Node.js: 20 | 21 | - [`@codspeed/vitest-plugin`](./packages/vitest-plugin): vitest compatibility layer for CodSpeed 22 | - [`@codspeed/tinybench-plugin`](./packages/tinybench-plugin): tinybench compatibility layer for CodSpeed 23 | - [`@codspeed/benchmark.js-plugin`](./packages/benchmark.js-plugin): Benchmark.js compatibility layer for CodSpeed 24 | - [`@codspeed/core`](./packages/core): The core library used to integrate with Codspeed runners 25 | -------------------------------------------------------------------------------- /commitlint.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { extends: ["@commitlint/config-conventional"] }; 2 | -------------------------------------------------------------------------------- /docs/introspection.md: -------------------------------------------------------------------------------- 1 | ## Testing introspection locally 2 | 3 | 1. Inside `codspeed-node` directory, run `export PATH="$pwd/../action/dist/bin:$PATH"`. 4 | This will ensure that the action's `dist/bin/node` file will be used instead of the 5 | system's `node` binary. 6 | 7 | 2. Replace the `CodSpeedHQ/action` grep filter with `CodSpeedHQ` in the `dist/bin/node`. 8 | Since we used `../action` in the `export PATH=...` command, the original grep filter will 9 | not work. 10 | 11 | 3. Run your command with the correct flags in `codspeed-node`, for example 12 | 13 | ```bash 14 | CI=1 CODSPEED_DEBUG=true pnpm --filter with-typescript-esm bench-tinybench 15 | ``` 16 | -------------------------------------------------------------------------------- /examples/with-javascript-esm/benchmark-js.js: -------------------------------------------------------------------------------- 1 | import { withCodSpeed } from "@codspeed/benchmark.js-plugin"; 2 | import Benchmark from "benchmark"; 3 | 4 | const suite = withCodSpeed(new Benchmark.Suite()); 5 | 6 | suite 7 | .add("RegExp#test", function () { 8 | /o/.test("Hello World!"); 9 | }) 10 | .add("String#indexOf", function () { 11 | "Hello World!".indexOf("o") > -1; 12 | }) 13 | // add listeners 14 | .on("cycle", function (event) { 15 | console.log(String(event.target)); 16 | }) 17 | // run async 18 | .run({ async: true }); 19 | -------------------------------------------------------------------------------- /examples/with-javascript-esm/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "with-javascript-esm", 3 | "private": true, 4 | "type": "module", 5 | "scripts": { 6 | "bench-benchmark-js": "node benchmark-js.js", 7 | "bench-tinybench": "node tinybench.js" 8 | }, 9 | "devDependencies": { 10 | "@codspeed/benchmark.js-plugin": "workspace:*", 11 | "@codspeed/tinybench-plugin": "workspace:*", 12 | "benchmark": "^2.1.4", 13 | "tinybench": "^4.0.1" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /examples/with-javascript-esm/tinybench.js: -------------------------------------------------------------------------------- 1 | import { withCodSpeed } from "@codspeed/tinybench-plugin"; 2 | import { Bench } from "tinybench"; 3 | 4 | const bench = withCodSpeed(new Bench({ time: 100 })); 5 | 6 | bench 7 | .add("switch 1", () => { 8 | let a = 1; 9 | let b = 2; 10 | const c = a; 11 | a = b; 12 | b = c; 13 | }) 14 | .add("switch 2", () => { 15 | let a = 1; 16 | let b = 10; 17 | a = b + a; 18 | b = a - b; 19 | a = b - a; 20 | }); 21 | 22 | bench.run().then(() => { 23 | console.table(bench.table()); 24 | }); 25 | -------------------------------------------------------------------------------- /examples/with-typescript-cjs/bench/benchmark.js/fibo.bench.ts: -------------------------------------------------------------------------------- 1 | import type { WithCodSpeedSuite } from "@codspeed/benchmark.js-plugin"; 2 | import { 3 | iterativeFibonacci, 4 | recursiveCachedFibonacci, 5 | recursiveFibonacci, 6 | } from "../../src/fibonacci"; 7 | 8 | export function registerFiboBenchmarks(suite: WithCodSpeedSuite) { 9 | suite 10 | .add("test_recursive_fibo_10", () => { 11 | recursiveFibonacci(10); 12 | }) 13 | .add("test_recursive_fibo_20", () => { 14 | recursiveFibonacci(20); 15 | }); 16 | 17 | suite 18 | .add("test_recursive_cached_fibo_10", () => { 19 | recursiveCachedFibonacci(10); 20 | }) 21 | .add("test_recursive_cached_fibo_20", () => { 22 | recursiveCachedFibonacci(20); 23 | }) 24 | .add("test_recursive_cached_fibo_30", () => { 25 | recursiveCachedFibonacci(30); 26 | }); 27 | 28 | suite 29 | .add("test_iterative_fibo_10", () => { 30 | iterativeFibonacci(10); 31 | }) 32 | .add("test_iterative_fibo_100", () => { 33 | iterativeFibonacci(100); 34 | }); 35 | } 36 | -------------------------------------------------------------------------------- /examples/with-typescript-cjs/bench/benchmark.js/foobarbaz.bench.ts: -------------------------------------------------------------------------------- 1 | import type { WithCodSpeedSuite } from "@codspeed/benchmark.js-plugin"; 2 | import { baz } from "../../src/foobarbaz"; 3 | 4 | export function registerFoobarbazBenchmarks(suite: WithCodSpeedSuite) { 5 | suite 6 | .add("test sync baz 10", () => { 7 | baz(10); 8 | }) 9 | .add("test sync baz 100", () => { 10 | baz(100); 11 | }); 12 | } 13 | -------------------------------------------------------------------------------- /examples/with-typescript-cjs/bench/benchmark.js/index.bench.ts: -------------------------------------------------------------------------------- 1 | import { withCodSpeed } from "@codspeed/benchmark.js-plugin"; 2 | import Benchmark from "benchmark"; 3 | import { registerFiboBenchmarks } from "./fibo.bench"; 4 | import { registerFoobarbazBenchmarks } from "./foobarbaz.bench"; 5 | 6 | export const suite = withCodSpeed(new Benchmark.Suite()); 7 | 8 | (async () => { 9 | registerFiboBenchmarks(suite); 10 | registerFoobarbazBenchmarks(suite); 11 | 12 | suite.on("cycle", function (event: Benchmark.Event) { 13 | console.log(String(event.target)); 14 | }); 15 | 16 | await suite.run({ async: true }); 17 | })(); 18 | -------------------------------------------------------------------------------- /examples/with-typescript-cjs/bench/tinybench/fibo.bench.ts: -------------------------------------------------------------------------------- 1 | import { Bench } from "tinybench"; 2 | import { 3 | iterativeFibonacci, 4 | recursiveCachedFibonacci, 5 | recursiveFibonacci, 6 | } from "../../src/fibonacci"; 7 | 8 | export function registerFiboBenchmarks(bench: Bench) { 9 | bench 10 | .add("test_recursive_fibo_10", () => { 11 | recursiveFibonacci(10); 12 | }) 13 | .add("test_recursive_fibo_20", () => { 14 | recursiveFibonacci(20); 15 | }); 16 | 17 | bench 18 | .add("test_recursive_cached_fibo_10", () => { 19 | recursiveCachedFibonacci(10); 20 | }) 21 | .add("test_recursive_cached_fibo_20", () => { 22 | recursiveCachedFibonacci(20); 23 | }) 24 | .add("test_recursive_cached_fibo_30", () => { 25 | recursiveCachedFibonacci(30); 26 | }); 27 | 28 | bench 29 | .add("test_iterative_fibo_10", () => { 30 | iterativeFibonacci(10); 31 | }) 32 | .add("test_iterative_fibo_100", () => { 33 | iterativeFibonacci(100); 34 | }); 35 | } 36 | -------------------------------------------------------------------------------- /examples/with-typescript-cjs/bench/tinybench/foobarbaz.bench.ts: -------------------------------------------------------------------------------- 1 | import { Bench } from "tinybench"; 2 | import { baz } from "../../src/foobarbaz"; 3 | 4 | export function registerFoobarbazBenchmarks(bench: Bench) { 5 | bench 6 | .add("test sync baz 10", () => { 7 | baz(10); 8 | }) 9 | .add("test sync baz 100", () => { 10 | baz(100); 11 | }); 12 | } 13 | -------------------------------------------------------------------------------- /examples/with-typescript-cjs/bench/tinybench/index.bench.ts: -------------------------------------------------------------------------------- 1 | import { withCodSpeed } from "@codspeed/tinybench-plugin"; 2 | import { Bench } from "tinybench"; 3 | import { registerFiboBenchmarks } from "./fibo.bench"; 4 | import { registerFoobarbazBenchmarks } from "./foobarbaz.bench"; 5 | 6 | export const bench = withCodSpeed(new Bench()); 7 | 8 | (async () => { 9 | registerFiboBenchmarks(bench); 10 | registerFoobarbazBenchmarks(bench); 11 | 12 | await bench.run(); 13 | console.table(bench.table()); 14 | })(); 15 | -------------------------------------------------------------------------------- /examples/with-typescript-cjs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "with-typescript-cjs", 3 | "private": true, 4 | "scripts": { 5 | "bench-benchmark-js": "node -r esbuild-register bench/benchmark.js/index.bench.ts", 6 | "bench-tinybench": "node -r esbuild-register bench/tinybench/index.bench.ts" 7 | }, 8 | "devDependencies": { 9 | "@codspeed/benchmark.js-plugin": "workspace:*", 10 | "@codspeed/tinybench-plugin": "workspace:*", 11 | "@types/benchmark": "^2.1.2", 12 | "benchmark": "^2.1.4", 13 | "esbuild-register": "^3.4.2", 14 | "tinybench": "^4.0.1", 15 | "typescript": "^5.1.3" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /examples/with-typescript-cjs/src/fibonacci.ts: -------------------------------------------------------------------------------- 1 | export function recursiveFibonacci(n: number): number { 2 | if (n < 2) { 3 | return n; 4 | } 5 | return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); 6 | } 7 | 8 | export function recursiveCachedFibonacci(n: number) { 9 | const cache: Record = { 0: 0, 1: 1 }; 10 | const fiboInner = (n: number) => { 11 | if (n in cache) { 12 | return cache[n]; 13 | } 14 | cache[n] = fiboInner(n - 1) + fiboInner(n - 2); 15 | return cache[n]; 16 | }; 17 | return fiboInner(n); 18 | } 19 | 20 | export function iterativeFibonacci(n: number) { 21 | let a = 0; 22 | let b = 1; 23 | let c = 0; 24 | for (let i = 0; i < n; i++) { 25 | c = a + b; 26 | a = b; 27 | b = c; 28 | } 29 | return a; 30 | } 31 | -------------------------------------------------------------------------------- /examples/with-typescript-cjs/src/foobarbaz.ts: -------------------------------------------------------------------------------- 1 | // Sync version 2 | function foo(n: number) { 3 | let result = 0; 4 | for (let i = 0; i < n; i++) { 5 | result += 1; 6 | } 7 | return result; 8 | } 9 | 10 | function bar(n: number) { 11 | foo(n); 12 | } 13 | 14 | export function baz(n: number) { 15 | bar(n); 16 | } 17 | -------------------------------------------------------------------------------- /examples/with-typescript-cjs/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["es2023"], 4 | "module": "Node16", 5 | "target": "es2022", 6 | "strict": true, 7 | "esModuleInterop": true, 8 | "skipLibCheck": true, 9 | "forceConsistentCasingInFileNames": true, 10 | "moduleResolution": "Node" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /examples/with-typescript-esm/bench/benchmark.js/fibo.bench.ts: -------------------------------------------------------------------------------- 1 | import type { WithCodSpeedSuite } from "@codspeed/benchmark.js-plugin"; 2 | import { 3 | iterativeFibonacci, 4 | recursiveCachedFibonacci, 5 | recursiveFibonacci, 6 | } from "../../src/fibonacci"; 7 | 8 | export function registerFiboBenchmarks(suite: WithCodSpeedSuite) { 9 | suite 10 | .add("test_recursive_fibo_10", () => { 11 | recursiveFibonacci(10); 12 | }) 13 | .add("test_recursive_fibo_20", () => { 14 | recursiveFibonacci(20); 15 | }); 16 | 17 | suite 18 | .add("test_recursive_cached_fibo_10", () => { 19 | recursiveCachedFibonacci(10); 20 | }) 21 | .add("test_recursive_cached_fibo_20", () => { 22 | recursiveCachedFibonacci(20); 23 | }) 24 | .add("test_recursive_cached_fibo_30", () => { 25 | recursiveCachedFibonacci(30); 26 | }); 27 | 28 | suite 29 | .add("test_iterative_fibo_10", () => { 30 | iterativeFibonacci(10); 31 | }) 32 | .add("test_iterative_fibo_100", () => { 33 | iterativeFibonacci(100); 34 | }); 35 | } 36 | -------------------------------------------------------------------------------- /examples/with-typescript-esm/bench/benchmark.js/foobarbaz.bench.ts: -------------------------------------------------------------------------------- 1 | import type { WithCodSpeedSuite } from "@codspeed/benchmark.js-plugin"; 2 | import { baz } from "../../src/foobarbaz"; 3 | 4 | export function registerFoobarbazBenchmarks(suite: WithCodSpeedSuite) { 5 | suite 6 | .add("test sync baz 10", () => { 7 | baz(10); 8 | }) 9 | .add("test sync baz 100", () => { 10 | baz(100); 11 | }); 12 | } 13 | -------------------------------------------------------------------------------- /examples/with-typescript-esm/bench/benchmark.js/index.bench.ts: -------------------------------------------------------------------------------- 1 | import { withCodSpeed } from "@codspeed/benchmark.js-plugin"; 2 | import Benchmark from "benchmark"; 3 | import { registerFiboBenchmarks } from "./fibo.bench"; 4 | import { registerFoobarbazBenchmarks } from "./foobarbaz.bench"; 5 | 6 | export const suite = withCodSpeed(new Benchmark.Suite()); 7 | 8 | (async () => { 9 | registerFiboBenchmarks(suite); 10 | registerFoobarbazBenchmarks(suite); 11 | 12 | suite.on("cycle", function (event: Benchmark.Event) { 13 | console.log(String(event.target)); 14 | }); 15 | 16 | await suite.run({ async: true }); 17 | })(); 18 | -------------------------------------------------------------------------------- /examples/with-typescript-esm/bench/tinybench/fibo.bench.ts: -------------------------------------------------------------------------------- 1 | import { Bench } from "tinybench"; 2 | import { 3 | iterativeFibonacci, 4 | recursiveCachedFibonacci, 5 | recursiveFibonacci, 6 | } from "../../src/fibonacci"; 7 | 8 | export function registerFiboBenchmarks(bench: Bench) { 9 | bench 10 | .add("test_recursive_fibo_10", () => { 11 | recursiveFibonacci(10); 12 | }) 13 | .add("test_recursive_fibo_20", () => { 14 | recursiveFibonacci(20); 15 | }); 16 | 17 | bench 18 | .add("test_recursive_cached_fibo_10", () => { 19 | recursiveCachedFibonacci(10); 20 | }) 21 | .add("test_recursive_cached_fibo_20", () => { 22 | recursiveCachedFibonacci(20); 23 | }) 24 | .add("test_recursive_cached_fibo_30", () => { 25 | recursiveCachedFibonacci(30); 26 | }); 27 | 28 | bench 29 | .add("test_iterative_fibo_10", () => { 30 | iterativeFibonacci(10); 31 | }) 32 | .add("test_iterative_fibo_100", () => { 33 | iterativeFibonacci(100); 34 | }); 35 | } 36 | -------------------------------------------------------------------------------- /examples/with-typescript-esm/bench/tinybench/foobarbaz.bench.ts: -------------------------------------------------------------------------------- 1 | import { Bench } from "tinybench"; 2 | import { baz } from "../../src/foobarbaz"; 3 | 4 | export function registerFoobarbazBenchmarks(bench: Bench) { 5 | bench 6 | .add("test sync baz 10", () => { 7 | baz(10); 8 | }) 9 | .add("test sync baz 100", () => { 10 | baz(100); 11 | }); 12 | } 13 | -------------------------------------------------------------------------------- /examples/with-typescript-esm/bench/tinybench/index.bench.ts: -------------------------------------------------------------------------------- 1 | import { withCodSpeed } from "@codspeed/tinybench-plugin"; 2 | import { Bench } from "tinybench"; 3 | import { registerFiboBenchmarks } from "./fibo.bench"; 4 | import { registerFoobarbazBenchmarks } from "./foobarbaz.bench"; 5 | 6 | export const bench = withCodSpeed(new Bench()); 7 | 8 | (async () => { 9 | registerFiboBenchmarks(bench); 10 | registerFoobarbazBenchmarks(bench); 11 | 12 | await bench.run(); 13 | console.table(bench.table()); 14 | })(); 15 | -------------------------------------------------------------------------------- /examples/with-typescript-esm/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "with-typescript-esm", 3 | "private": true, 4 | "type": "module", 5 | "scripts": { 6 | "bench-benchmark-js": "node --loader esbuild-register/loader -r esbuild-register bench/benchmark.js/index.bench.ts", 7 | "bench-tinybench": "node --loader esbuild-register/loader -r esbuild-register bench/tinybench/index.bench.ts", 8 | "bench-vitest": "vitest bench --run" 9 | }, 10 | "devDependencies": { 11 | "@codspeed/benchmark.js-plugin": "workspace:*", 12 | "@codspeed/tinybench-plugin": "workspace:*", 13 | "@codspeed/vitest-plugin": "workspace:*", 14 | "@types/benchmark": "^2.1.2", 15 | "benchmark": "^2.1.4", 16 | "esbuild-register": "^3.4.2", 17 | "tinybench": "^4.0.1", 18 | "typescript": "^5.1.3", 19 | "vitest": "^3.2.4" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /examples/with-typescript-esm/src/fibonacci.bench.ts: -------------------------------------------------------------------------------- 1 | import { bench, describe } from "vitest"; 2 | import { recursiveFibonacci } from "./fibonacci"; 3 | 4 | describe("recursiveFibonacci", () => { 5 | bench("fibo 30", () => { 6 | recursiveFibonacci(30); 7 | }); 8 | }); 9 | -------------------------------------------------------------------------------- /examples/with-typescript-esm/src/fibonacci.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, it } from "vitest"; 2 | import { iterativeFibonacci } from "./fibonacci"; 3 | 4 | describe("iterativeFibonacci", () => { 5 | it("should return the correct value", () => { 6 | expect(iterativeFibonacci(1)).toBe(1); 7 | }); 8 | }); 9 | -------------------------------------------------------------------------------- /examples/with-typescript-esm/src/fibonacci.ts: -------------------------------------------------------------------------------- 1 | export function recursiveFibonacci(n: number): number { 2 | if (n < 2) { 3 | return n; 4 | } 5 | return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); 6 | } 7 | 8 | export function recursiveCachedFibonacci(n: number) { 9 | const cache: Record = { 0: 0, 1: 1 }; 10 | const fiboInner = (n: number) => { 11 | if (n in cache) { 12 | return cache[n]; 13 | } 14 | cache[n] = fiboInner(n - 1) + fiboInner(n - 2); 15 | return cache[n]; 16 | }; 17 | return fiboInner(n); 18 | } 19 | 20 | export function iterativeFibonacci(n: number) { 21 | let a = 0; 22 | let b = 1; 23 | let c = 0; 24 | for (let i = 0; i < n; i++) { 25 | c = a + b; 26 | a = b; 27 | b = c; 28 | } 29 | return a; 30 | } 31 | -------------------------------------------------------------------------------- /examples/with-typescript-esm/src/foobarbaz.ts: -------------------------------------------------------------------------------- 1 | // Sync version 2 | function foo(n: number) { 3 | let result = 0; 4 | for (let i = 0; i < n; i++) { 5 | result += 1; 6 | } 7 | return result; 8 | } 9 | 10 | function bar(n: number) { 11 | foo(n); 12 | } 13 | 14 | export function baz(n: number) { 15 | bar(n); 16 | } 17 | -------------------------------------------------------------------------------- /examples/with-typescript-esm/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["es2023"], 4 | "module": "ESNext", 5 | "verbatimModuleSyntax": true, 6 | "target": "es2022", 7 | "strict": true, 8 | "esModuleInterop": true, 9 | "skipLibCheck": true, 10 | "forceConsistentCasingInFileNames": true, 11 | "moduleResolution": "Node" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /examples/with-typescript-esm/vitest.config.ts: -------------------------------------------------------------------------------- 1 | import codspeedPlugin from "@codspeed/vitest-plugin"; 2 | import { defineConfig } from "vitest/config"; 3 | 4 | export default defineConfig({ 5 | plugins: [codspeedPlugin()], 6 | test: { 7 | benchmark: { 8 | exclude: ["**/bench/**/*", "**/node_modules/**/*"], 9 | }, 10 | }, 11 | }); 12 | -------------------------------------------------------------------------------- /examples/with-typescript-simple-cjs/benchmark-js.ts: -------------------------------------------------------------------------------- 1 | import { withCodSpeed } from "@codspeed/benchmark.js-plugin"; 2 | import Benchmark from "benchmark"; 3 | 4 | const suite = withCodSpeed(new Benchmark.Suite()); 5 | 6 | suite 7 | .add("RegExp#test", function () { 8 | /o/.test("Hello World!"); 9 | }) 10 | .add("String#indexOf", function () { 11 | "Hello World!".indexOf("o") > -1; 12 | }) 13 | // add listeners 14 | .on("cycle", function (event: Benchmark.Event) { 15 | console.log(String(event.target)); 16 | }) 17 | // run async 18 | .run({ async: true }); 19 | -------------------------------------------------------------------------------- /examples/with-typescript-simple-cjs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "with-typescript-simple-cjs", 3 | "private": true, 4 | "scripts": { 5 | "bench-benchmark-js": "node -r esbuild-register benchmark-js.ts", 6 | "bench-tinybench": "node -r esbuild-register tinybench.ts" 7 | }, 8 | "devDependencies": { 9 | "@codspeed/benchmark.js-plugin": "workspace:*", 10 | "@codspeed/tinybench-plugin": "workspace:*", 11 | "@types/benchmark": "^2.1.2", 12 | "benchmark": "^2.1.4", 13 | "esbuild-register": "^3.4.2", 14 | "tinybench": "^4.0.1", 15 | "typescript": "^5.1.3" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /examples/with-typescript-simple-cjs/tinybench.ts: -------------------------------------------------------------------------------- 1 | import { withCodSpeed } from "@codspeed/tinybench-plugin"; 2 | import { Bench } from "tinybench"; 3 | 4 | const bench = withCodSpeed(new Bench({ time: 100 })); 5 | 6 | bench 7 | .add("switch 1", () => { 8 | let a = 1; 9 | let b = 2; 10 | const c = a; 11 | a = b; 12 | b = c; 13 | }) 14 | .add("switch 2", () => { 15 | let a = 1; 16 | let b = 10; 17 | a = b + a; 18 | b = a - b; 19 | a = b - a; 20 | }); 21 | 22 | bench.run().then(() => { 23 | console.table(bench.table()); 24 | }); 25 | -------------------------------------------------------------------------------- /examples/with-typescript-simple-esm/benchmark-js.ts: -------------------------------------------------------------------------------- 1 | import { withCodSpeed } from "@codspeed/benchmark.js-plugin"; 2 | import Benchmark from "benchmark"; 3 | 4 | const suite = withCodSpeed(new Benchmark.Suite()); 5 | 6 | suite 7 | .add("RegExp#test", function () { 8 | /o/.test("Hello World!"); 9 | }) 10 | .add("String#indexOf", function () { 11 | "Hello World!".indexOf("o") > -1; 12 | }) 13 | // add listeners 14 | .on("cycle", function (event: Benchmark.Event) { 15 | console.log(String(event.target)); 16 | }) 17 | // run async 18 | .run({ async: true }); 19 | -------------------------------------------------------------------------------- /examples/with-typescript-simple-esm/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "with-typescript-simple-esm", 3 | "private": true, 4 | "type": "module", 5 | "scripts": { 6 | "bench-benchmark-js": "node --loader esbuild-register/loader -r esbuild-register benchmark-js.ts", 7 | "bench-tinybench": "node --loader esbuild-register/loader -r esbuild-register tinybench.ts" 8 | }, 9 | "devDependencies": { 10 | "@codspeed/benchmark.js-plugin": "workspace:*", 11 | "@codspeed/tinybench-plugin": "workspace:*", 12 | "@types/benchmark": "^2.1.2", 13 | "benchmark": "^2.1.4", 14 | "esbuild-register": "^3.4.2", 15 | "tinybench": "^4.0.1", 16 | "typescript": "^5.1.3" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /examples/with-typescript-simple-esm/tinybench.ts: -------------------------------------------------------------------------------- 1 | import { withCodSpeed } from "@codspeed/tinybench-plugin"; 2 | import { Bench } from "tinybench"; 3 | 4 | const bench = withCodSpeed(new Bench({ time: 100 })); 5 | 6 | bench 7 | .add("switch 1", () => { 8 | let a = 1; 9 | let b = 2; 10 | const c = a; 11 | a = b; 12 | b = c; 13 | }) 14 | .add("switch 2", () => { 15 | let a = 1; 16 | let b = 10; 17 | a = b + a; 18 | b = a - b; 19 | a = b - a; 20 | }); 21 | 22 | bench.run().then(() => { 23 | console.table(bench.table()); 24 | }); 25 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "flake-utils": { 4 | "inputs": { 5 | "systems": "systems" 6 | }, 7 | "locked": { 8 | "lastModified": 1731533236, 9 | "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", 10 | "owner": "numtide", 11 | "repo": "flake-utils", 12 | "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", 13 | "type": "github" 14 | }, 15 | "original": { 16 | "owner": "numtide", 17 | "repo": "flake-utils", 18 | "type": "github" 19 | } 20 | }, 21 | "nixpkgs": { 22 | "locked": { 23 | "lastModified": 1752620740, 24 | "narHash": "sha256-f3pO+9lg66mV7IMmmIqG4PL3223TYMlnlw+pnpelbss=", 25 | "owner": "NixOS", 26 | "repo": "nixpkgs", 27 | "rev": "32a4e87942101f1c9f9865e04dc3ddb175f5f32e", 28 | "type": "github" 29 | }, 30 | "original": { 31 | "owner": "NixOS", 32 | "ref": "nixos-25.05", 33 | "repo": "nixpkgs", 34 | "type": "github" 35 | } 36 | }, 37 | "root": { 38 | "inputs": { 39 | "flake-utils": "flake-utils", 40 | "nixpkgs": "nixpkgs" 41 | } 42 | }, 43 | "systems": { 44 | "locked": { 45 | "lastModified": 1681028828, 46 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 47 | "owner": "nix-systems", 48 | "repo": "default", 49 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 50 | "type": "github" 51 | }, 52 | "original": { 53 | "owner": "nix-systems", 54 | "repo": "default", 55 | "type": "github" 56 | } 57 | } 58 | }, 59 | "root": "root", 60 | "version": 7 61 | } 62 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "CodSpeed Node development environment"; 3 | 4 | inputs = { 5 | nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05"; 6 | flake-utils.url = "github:numtide/flake-utils"; 7 | }; 8 | 9 | outputs = 10 | { 11 | self, 12 | nixpkgs, 13 | flake-utils, 14 | }: 15 | flake-utils.lib.eachDefaultSystem ( 16 | system: 17 | let 18 | pkgs = import nixpkgs { inherit system; }; 19 | commonBuildInputs = with pkgs; [ 20 | # Needed for node-gyp 21 | (python314.withPackages ( 22 | ps: with ps; [ 23 | setuptools 24 | ] 25 | )) 26 | ]; 27 | 28 | in 29 | { 30 | devShells = { 31 | default = pkgs.mkShell { 32 | buildInputs = commonBuildInputs; 33 | shellHook = '' 34 | echo "CodSpeed Node development environment" 35 | ''; 36 | }; 37 | 38 | lsp = pkgs.mkShell { 39 | buildInputs = 40 | with pkgs; 41 | [ 42 | typescript-language-server 43 | ] 44 | ++ commonBuildInputs; 45 | shellHook = '' 46 | echo "CodSpeed Node development environment with LSP" 47 | ''; 48 | }; 49 | }; 50 | } 51 | ); 52 | } 53 | -------------------------------------------------------------------------------- /lerna.json: -------------------------------------------------------------------------------- 1 | { 2 | "npmClient": "pnpm", 3 | "useWorkspaces": true, 4 | "packages": ["packages/*"], 5 | "$schema": "node_modules/lerna/schemas/lerna-schema.json", 6 | "version": "5.0.1" 7 | } 8 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "root", 3 | "private": true, 4 | "devDependencies": { 5 | "@commitlint/cli": "^17.5.1", 6 | "@commitlint/config-conventional": "^17.4.4", 7 | "@moonrepo/cli": "1.37.3", 8 | "@rollup/plugin-commonjs": "^25.0.7", 9 | "@rollup/plugin-json": "^6.0.1", 10 | "@rollup/plugin-node-resolve": "^15.2.3", 11 | "@rollup/plugin-typescript": "^11.1.5", 12 | "@types/jest": "^29.5.0", 13 | "@types/node": "^20.5.1", 14 | "@typescript-eslint/eslint-plugin": "^5.58.0", 15 | "@typescript-eslint/parser": "^5.58.0", 16 | "esbuild": "^0.17.16", 17 | "esbuild-register": "^3.4.2", 18 | "eslint": "^7.32.0", 19 | "eslint-import-resolver-typescript": "^3.5.5", 20 | "eslint-plugin-import": "^2.27.5", 21 | "husky": "^7.0.4", 22 | "jest": "^29.5.0", 23 | "jest-config": "^29.5.0", 24 | "lerna": "^6.6.1", 25 | "prettier": "^2.8.7", 26 | "prettier-plugin-organize-imports": "^3.2.2", 27 | "rollup": "^4.47.1", 28 | "rollup-plugin-dts": "^6.1.0", 29 | "rollup-plugin-esbuild": "^6.1.0", 30 | "ts-jest": "^29.1.0", 31 | "tslib": "^2.5.0", 32 | "typescript": "4.9.4" 33 | }, 34 | "packageManager": "pnpm@10.12.4", 35 | "engines": { 36 | "node": "20.5.1" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/README.md: -------------------------------------------------------------------------------- 1 |
2 |

@codspeed/benchmark.js-plugin

3 | 4 | Benchmark.js compatibility layer for CodSpeed 5 | 6 | [![CI](https://github.com/CodSpeedHQ/codspeed-node/actions/workflows/ci.yml/badge.svg)](https://github.com/CodSpeedHQ/codspeed-node/actions/workflows/ci.yml) 7 | [![npm (scoped)](https://img.shields.io/npm/v/@codspeed/benchmark.js-plugin)](https://www.npmjs.com/package/@codspeed/benchmark.js-plugin) 8 | [![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF) 9 | [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/CodSpeedHQ/codspeed-node) 10 | 11 |
12 | 13 | ## Documentation 14 | 15 | Check out the [documentation](https://docs.codspeed.io/benchmarks/nodejs/benchmark.js) for complete integration instructions. 16 | 17 | ## Installation 18 | 19 | First, install the plugin [`@codspeed/benchmark.js-plugin`](https://www.npmjs.com/package/@codspeed/benchmark.js-plugin) and `benchmark.js` (if not already installed): 20 | 21 | ```sh 22 | npm install --save-dev @codspeed/benchmark.js-plugin benchmark.js 23 | ``` 24 | 25 | or with `yarn`: 26 | 27 | ```sh 28 | yarn add --dev @codspeed/benchmark.js-plugin benchmark.js 29 | ``` 30 | 31 | or with `pnpm`: 32 | 33 | ```sh 34 | pnpm add --save-dev @codspeed/benchmark.js-plugin benchmark.js 35 | ``` 36 | 37 | ## Usage 38 | 39 | Let's create a fibonacci function and benchmark it with benchmark.js and the CodSpeed plugin: 40 | 41 | ```js title="benches/bench.mjs" 42 | import Benchmark from "benchmark"; 43 | import { withCodSpeed } from "@codspeed/benchmark.js-plugin"; 44 | 45 | function fibonacci(n) { 46 | if (n < 2) { 47 | return n; 48 | } 49 | return fibonacci(n - 1) + fibonacci(n - 2); 50 | } 51 | 52 | const suite = withCodSpeed(new Benchmark.Suite()); 53 | 54 | suite 55 | .add("fibonacci10", () => { 56 | fibonacci(10); 57 | }) 58 | .add("fibonacci15", () => { 59 | fibonacci(15); 60 | }) 61 | .on("cycle", function (event: Benchmark.Event) { 62 | console.log(String(event.target)); 63 | }) 64 | .run(); 65 | ``` 66 | 67 | Here, a few things are happening: 68 | 69 | - We create a simple recursive fibonacci function. 70 | - We create a new `Benchmark.Suite` instance with CodSpeed support by using the **`withCodSpeed`** helper. This step is **critical** to enable CodSpeed on your benchmarks. 71 | - We add two benchmarks to the suite and launch it, benching our `fibonacci` function with 10 and 15. 72 | 73 | Now, we can run our benchmarks locally to make sure everything is working as expected: 74 | 75 | ```sh 76 | $ node benches/bench.mjs 77 | [CodSpeed] 2 benches detected but no instrumentation found 78 | [CodSpeed] falling back to benchmark.js 79 | 80 | fibonacci10 x 2,155,187 ops/sec ±0.50% (96 runs sampled) 81 | fibonacci15 x 194,742 ops/sec ±0.48% (95 runs sampled) 82 | ``` 83 | 84 | And... Congrats🎉, CodSpeed is installed in your benchmarking suite! Locally, CodSpeed will fallback to tinybench since the instrumentation is only available in the CI environment for now. 85 | 86 | You can now [run those benchmarks in your CI](https://docs.codspeed.io/benchmarks/nodejs/benchmark.js#running-the-benchmarks-in-your-ci) to continuously get consistent performance measurements. 87 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/babel.config.js: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line no-undef 2 | module.exports = { 3 | presets: [["@babel/preset-env", { targets: { node: "current" } }]], 4 | }; 5 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/benches/parsePr.ts: -------------------------------------------------------------------------------- 1 | interface PullRequest { 2 | number: number; 3 | title: string; 4 | body: string; 5 | } 6 | 7 | function sendEvent(numberOfOperations: number): void { 8 | for (let i = 0; i < numberOfOperations; i++) { 9 | let a = i; 10 | a = a + 1; 11 | } 12 | } 13 | 14 | function logMetrics( 15 | numberOfOperations: number, 16 | numberOfDeepOperations: number 17 | ): void { 18 | for (let i = 0; i < numberOfOperations; i++) { 19 | for (let i = 0; i < numberOfOperations; i++) { 20 | let a = i; 21 | a = a + 1; 22 | a = a + 1; 23 | } 24 | sendEvent(numberOfDeepOperations); 25 | } 26 | } 27 | 28 | function parseTitle(title: string): void { 29 | logMetrics(10, 10); 30 | modifyTitle(title); 31 | } 32 | 33 | function modifyTitle(title: string): void { 34 | for (let i = 0; i < 100; i++) { 35 | let a = i; 36 | a = a + 1 + title.length; 37 | } 38 | } 39 | 40 | function prepareParsingBody(body: string): void { 41 | for (let i = 0; i < 100; i++) { 42 | let a = i; 43 | a = a + 1; 44 | } 45 | parseBody(body); 46 | } 47 | 48 | function parseBody(body: string): void { 49 | logMetrics(10, 10); 50 | for (let i = 0; i < 200; i++) { 51 | let a = i; 52 | a = a + 1; 53 | } 54 | parseIssueFixed(body); 55 | } 56 | 57 | function parseIssueFixed(body: string): number | null { 58 | const prefix = "fixes #"; 59 | const index = body.indexOf(prefix); 60 | if (index === -1) { 61 | return null; 62 | } 63 | 64 | const start = index + prefix.length; 65 | let end = start; 66 | while (end < body.length && /\d/.test(body[end])) { 67 | end += 1; 68 | } 69 | return parseInt(body.slice(start, end)); 70 | } 71 | 72 | export default function parsePr(pullRequest: PullRequest): void { 73 | parseTitle(pullRequest.title); 74 | prepareParsingBody(pullRequest.body); 75 | } 76 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/benches/sample.ts: -------------------------------------------------------------------------------- 1 | import Benchmark from "benchmark"; 2 | import { withCodSpeed } from ".."; 3 | import parsePr from "./parsePr"; 4 | 5 | const LONG_BODY = 6 | new Array(1_000) 7 | .fill( 8 | "Lorem ipsum dolor sit amet consectetur adipisicing elit. Sunt, earum. Atque architecto vero veniam est tempora fugiat sint quo praesentium quia. Autem, veritatis omnis beatae iste delectus recusandae animi non." 9 | ) 10 | .join("\n") + "fixes #123"; 11 | 12 | const suite = withCodSpeed(new Benchmark.Suite()); 13 | 14 | suite 15 | .add("RegExp#test", function () { 16 | /o/.test("Hello World!"); 17 | }) 18 | .add("String#indexOf", function () { 19 | "Hello World!".indexOf("o") > -1; 20 | }) 21 | .add("short body", () => { 22 | parsePr({ body: "fixes #123", title: "test", number: 124 }); 23 | }) 24 | .add("long body", () => { 25 | parsePr({ body: LONG_BODY, title: "test", number: 124 }); 26 | }) 27 | .add("short body 2", () => { 28 | parsePr({ body: "fixes #123", title: "test", number: 124 }); 29 | }) 30 | .add("short body 3", () => { 31 | parsePr({ body: "fixes #123", title: "test", number: 124 }); 32 | }) 33 | .add("short body 4", () => { 34 | parsePr({ body: "fixes #123", title: "test", number: 124 }); 35 | }) 36 | .add("short body 5", () => { 37 | parsePr({ body: "fixes #123", title: "test", number: 124 }); 38 | }) 39 | // add listeners 40 | .on("cycle", function (event: Benchmark.Event) { 41 | console.log(String(event.target)); 42 | }) 43 | // run async 44 | .run({ async: true }); 45 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/jest.config.integ.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest').JestConfigWithTsJest} */ 2 | // eslint-disable-next-line no-undef 3 | module.exports = { 4 | preset: "ts-jest", 5 | testEnvironment: "node", 6 | transform: { 7 | "^.+\\.tsx?$": [ 8 | "ts-jest", 9 | { 10 | tsconfig: "tsconfig.test.json", 11 | }, 12 | ], 13 | }, 14 | testPathIgnorePatterns: [ 15 | "/node_modules/", 16 | "/src/", 17 | "/.rollup.cache/", 18 | ], 19 | }; 20 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/jest.config.js: -------------------------------------------------------------------------------- 1 | const esmModules = [ 2 | "find-up", 3 | "locate-path", 4 | "p-locate", 5 | "p-limit", 6 | "yocto-queue", 7 | "path-exists", 8 | "stack-trace", 9 | ]; 10 | 11 | /** @type {import('ts-jest').JestConfigWithTsJest} */ 12 | module.exports = { 13 | preset: "ts-jest", 14 | testEnvironment: "node", 15 | transform: { 16 | "^.+\\.tsx?$": ["ts-jest"], 17 | // transform js with babel-jest 18 | "^.+\\.js$": "babel-jest", 19 | }, 20 | testPathIgnorePatterns: [ 21 | "/node_modules/", 22 | "/tests/", 23 | "/.rollup.cache/", 24 | ], 25 | transformIgnorePatterns: [ 26 | `node_modules/(?!(?:.pnpm/)?(${esmModules.join("|")}))`, 27 | ], 28 | }; 29 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/moon.yml: -------------------------------------------------------------------------------- 1 | tasks: 2 | bench: 3 | command: node -r esbuild-register benches/sample.ts 4 | inputs: 5 | - "benches/**" 6 | local: true 7 | platform: "system" 8 | options: 9 | cache: false 10 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@codspeed/benchmark.js-plugin", 3 | "version": "5.0.1", 4 | "description": "Benchmark.js compatibility layer for CodSpeed", 5 | "keywords": [ 6 | "codspeed", 7 | "benchmark", 8 | "benchmark.js", 9 | "performance" 10 | ], 11 | "main": "dist/index.cjs.js", 12 | "module": "dist/index.es5.js", 13 | "types": "dist/index.d.ts", 14 | "files": [ 15 | "dist" 16 | ], 17 | "author": "Arthur Pastel ", 18 | "repository": "https://github.com/CodSpeedHQ/codspeed-node", 19 | "homepage": "https://codspeed.io", 20 | "license": "Apache-2.0", 21 | "devDependencies": { 22 | "@babel/preset-env": "^7.22.5", 23 | "@types/benchmark": "^2.1.2", 24 | "@types/lodash": "^4.14.195", 25 | "@types/stack-trace": "^0.0.30", 26 | "benchmark": "^2.1.4", 27 | "jest-mock-extended": "^3.0.4" 28 | }, 29 | "dependencies": { 30 | "@codspeed/core": "workspace:^5.0.1", 31 | "lodash": "^4.17.10", 32 | "stack-trace": "1.0.0-pre2" 33 | }, 34 | "peerDependencies": { 35 | "benchmark": "^2.1.0" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/rollup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "rollup"; 2 | import { declarationsPlugin, jsPlugins } from "../../rollup.options"; 3 | import pkg from "./package.json" assert { type: "json" }; 4 | 5 | const entrypoint = "src/index.ts"; 6 | 7 | export default defineConfig([ 8 | { 9 | input: entrypoint, 10 | output: [ 11 | { 12 | file: pkg.types, 13 | format: "es", 14 | sourcemap: true, 15 | }, 16 | ], 17 | plugins: declarationsPlugin({ compilerOptions: { composite: false } }), 18 | }, 19 | { 20 | input: entrypoint, 21 | output: [ 22 | { 23 | file: pkg.main, 24 | format: "cjs", 25 | sourcemap: true, 26 | }, 27 | { file: pkg.module, format: "es", sourcemap: true }, 28 | ], 29 | plugins: jsPlugins(pkg.version), 30 | external: ["@codspeed/core"], 31 | }, 32 | ]); 33 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/src/__tests__/buildSuiteAdd.test.ts: -------------------------------------------------------------------------------- 1 | import { Suite } from "benchmark"; 2 | import buildSuiteAdd from "../buildSuiteAdd"; 3 | import { CodSpeedBenchmark } from "../types"; 4 | 5 | describe("buildSuiteAdd", () => { 6 | let emptyBench: () => void; 7 | let suite: Suite; 8 | 9 | beforeEach(() => { 10 | emptyBench = () => { 11 | return; 12 | }; 13 | suite = new Suite(); 14 | }); 15 | 16 | it("should register benchmark name when using (options: Options)", () => { 17 | suite.add = buildSuiteAdd(suite); 18 | suite.add({ name: "test", fn: emptyBench }); 19 | suite.forEach((bench: CodSpeedBenchmark) => 20 | expect(bench.uri).toBe( 21 | "packages/benchmark.js-plugin/src/__tests__/buildSuiteAdd.test.ts::test" 22 | ) 23 | ); 24 | }); 25 | 26 | it("should register benchmark name when using (fn: Function, options?: Options)", () => { 27 | suite.add = buildSuiteAdd(suite); 28 | suite.add(emptyBench, { name: "test" }); 29 | suite.forEach((bench: CodSpeedBenchmark) => 30 | expect(bench.uri).toBe( 31 | "packages/benchmark.js-plugin/src/__tests__/buildSuiteAdd.test.ts::test" 32 | ) 33 | ); 34 | }); 35 | 36 | it("should register benchmark name when using (name: string, options?: Options)", () => { 37 | suite.add = buildSuiteAdd(suite); 38 | suite.add("test", { fn: emptyBench }); 39 | suite.forEach((bench: CodSpeedBenchmark) => 40 | expect(bench.uri).toBe( 41 | "packages/benchmark.js-plugin/src/__tests__/buildSuiteAdd.test.ts::test" 42 | ) 43 | ); 44 | }); 45 | 46 | it("should register benchmark name when using (name: string, fn: Function, options?: Options)", () => { 47 | suite.add = buildSuiteAdd(suite); 48 | suite.add("test", emptyBench); 49 | suite.forEach((bench: CodSpeedBenchmark) => 50 | expect(bench.uri).toBe( 51 | "packages/benchmark.js-plugin/src/__tests__/buildSuiteAdd.test.ts::test" 52 | ) 53 | ); 54 | }); 55 | 56 | it("should register benchmark name when suite name is defined", () => { 57 | suite.name = "suite"; 58 | suite.add = buildSuiteAdd(suite); 59 | suite.add("test", emptyBench); 60 | suite.forEach((bench: CodSpeedBenchmark) => 61 | expect(bench.uri).toBe( 62 | "packages/benchmark.js-plugin/src/__tests__/buildSuiteAdd.test.ts::suite::test" 63 | ) 64 | ); 65 | }); 66 | 67 | it("should call rawAdd with options object", () => { 68 | const rawAdd = jest.fn(); 69 | suite.add = rawAdd; 70 | suite.add = buildSuiteAdd(suite); 71 | const options = { name: "test", delay: 100 }; 72 | suite.add(options); 73 | expect(rawAdd).toHaveBeenCalledWith(options); 74 | }); 75 | 76 | it("should call rawAdd with function and options object", () => { 77 | const rawAdd = jest.fn(); 78 | suite.add = rawAdd; 79 | suite.add = buildSuiteAdd(suite); 80 | const fn = emptyBench; 81 | const options = { name: "test", delay: 100 }; 82 | suite.add("test", fn, options); 83 | expect(rawAdd).toHaveBeenCalledWith("test", fn, { 84 | ...options, 85 | uri: "packages/benchmark.js-plugin/src/__tests__/buildSuiteAdd.test.ts::test", 86 | }); 87 | }); 88 | 89 | it("should call rawAdd with name and options object", () => { 90 | const rawAdd = jest.fn(); 91 | suite.add = rawAdd; 92 | suite.add = buildSuiteAdd(suite); 93 | const options = { name: "test", delay: 100 }; 94 | suite.add("test", options); 95 | expect(rawAdd).toHaveBeenCalledWith("test", options); 96 | }); 97 | 98 | it("should call rawAdd with function and undefined options", () => { 99 | const rawAdd = jest.fn(); 100 | suite.add = rawAdd; 101 | suite.add = buildSuiteAdd(suite); 102 | const fn = emptyBench; 103 | suite.add("test", fn); 104 | expect(rawAdd).toHaveBeenCalledWith("test", fn, { 105 | uri: "packages/benchmark.js-plugin/src/__tests__/buildSuiteAdd.test.ts::test", 106 | }); 107 | }); 108 | }); 109 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/src/buildSuiteAdd.ts: -------------------------------------------------------------------------------- 1 | import { Options, Suite } from "benchmark"; 2 | import { isFunction, isPlainObject } from "lodash"; 3 | import getCallingFile from "./getCallingFile"; 4 | 5 | function isOptions(options: unknown): options is Options { 6 | return isPlainObject(options); 7 | } 8 | 9 | export default function buildSuiteAdd(suite: Suite) { 10 | const rawAdd = suite.add; 11 | const suiteName = suite.name; 12 | 13 | function registerBenchmarkName(name: string) { 14 | const callingFile = getCallingFile(2); // [here, suite.add, actual caller] 15 | let uri = callingFile; 16 | if (suiteName !== undefined) { 17 | uri += `::${suiteName}`; 18 | } 19 | uri += `::${name}`; 20 | 21 | return uri; 22 | } 23 | 24 | function add(options: Options): Suite; 25 | // eslint-disable-next-line @typescript-eslint/ban-types 26 | function add(fn: Function, options?: Options): Suite; 27 | function add(name: string, options?: Options): Suite; 28 | // eslint-disable-next-line @typescript-eslint/ban-types 29 | function add(name: string, fn: Function, options?: Options): Suite; 30 | function add(name: unknown, fn?: unknown, opts?: unknown) { 31 | // 1 argument: (options: Options) 32 | if (isOptions(name)) { 33 | if (name.name !== undefined) { 34 | const rawFn = name.fn; 35 | if (typeof rawFn === "function") { 36 | const uri = registerBenchmarkName(name.name); 37 | const options = Object.assign({}, name, { uri }); 38 | return rawAdd.bind(suite)(options); 39 | } 40 | } 41 | return rawAdd.bind(suite)(name); 42 | } 43 | 44 | // 2 arguments: (fn: Function, options?: Options) 45 | if (isFunction(name) && (isOptions(fn) || fn === undefined)) { 46 | if (fn !== undefined) { 47 | if (fn.name !== undefined) { 48 | const uri = registerBenchmarkName(fn.name); 49 | const options = Object.assign({}, fn, { uri }); 50 | return rawAdd.bind(suite)(name, options); 51 | } 52 | } 53 | return rawAdd.bind(suite)(name, fn); 54 | } 55 | 56 | // 2 arguments: (name: string, options?: Options) 57 | if (typeof name === "string" && (isOptions(fn) || fn === undefined)) { 58 | if (fn !== undefined && typeof fn.fn === "function") { 59 | const uri = registerBenchmarkName(name); 60 | const options = Object.assign({}, fn, { uri }); 61 | return rawAdd.bind(suite)(name, options); 62 | } 63 | return rawAdd.bind(suite)(name, fn); 64 | } 65 | 66 | // 3 arguments: (name: string, fn: Function, options?: Options) 67 | if ( 68 | typeof name === "string" && 69 | isFunction(fn) && 70 | (isOptions(opts) || opts === undefined) 71 | ) { 72 | const uri = registerBenchmarkName(name); 73 | const options = Object.assign({}, opts ?? {}, { uri }); 74 | return rawAdd.bind(suite)(name, fn, options); 75 | } 76 | } 77 | 78 | return add; 79 | } 80 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/src/getCallingFile.ts: -------------------------------------------------------------------------------- 1 | import { getGitDir } from "@codspeed/core"; 2 | import path from "path"; 3 | import { get as getStackTrace } from "stack-trace"; 4 | import { fileURLToPath } from "url"; 5 | 6 | export default function getCallingFile(depth: number): string { 7 | const stack = getStackTrace(); 8 | let callingFile = stack[depth + 1].getFileName(); 9 | const gitDir = getGitDir(callingFile); 10 | if (gitDir === undefined) { 11 | throw new Error("Could not find a git repository"); 12 | } 13 | if (callingFile.startsWith("file://")) { 14 | callingFile = fileURLToPath(callingFile); 15 | } 16 | return path.relative(gitDir, callingFile); 17 | } 18 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/src/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | InstrumentHooks, 3 | mongoMeasurement, 4 | optimizeFunction, 5 | optimizeFunctionSync, 6 | setupCore, 7 | SetupInstrumentsRequestBody, 8 | SetupInstrumentsResponse, 9 | teardownCore, 10 | tryIntrospect, 11 | } from "@codspeed/core"; 12 | import Benchmark from "benchmark"; 13 | import buildSuiteAdd from "./buildSuiteAdd"; 14 | import getCallingFile from "./getCallingFile"; 15 | import { CodSpeedBenchmark } from "./types"; 16 | 17 | declare const __VERSION__: string; 18 | 19 | tryIntrospect(); 20 | 21 | interface WithCodSpeedBenchmark 22 | extends Omit< 23 | Benchmark, 24 | "run" | "abort" | "clone" | "compare" | "emit" | "off" | "on" | "reset" 25 | > { 26 | abort(): WithCodSpeedBenchmark; 27 | clone(options: Benchmark.Options): WithCodSpeedBenchmark; 28 | compare(benchmark: Benchmark): number; 29 | off( 30 | type?: string, 31 | listener?: CallableFunction 32 | ): Benchmark | Promise; 33 | off(types: string[]): WithCodSpeedBenchmark; 34 | on(type?: string, listener?: CallableFunction): WithCodSpeedBenchmark; 35 | on(types: string[]): WithCodSpeedBenchmark; 36 | reset(): WithCodSpeedBenchmark; 37 | // Makes run an async function 38 | run(options?: Benchmark.Options): Benchmark | Promise; 39 | } 40 | 41 | export interface WithCodSpeedSuite 42 | extends Omit< 43 | Benchmark.Suite, 44 | | "run" 45 | | "abort" 46 | | "clone" 47 | | "compare" 48 | | "emit" 49 | | "off" 50 | | "on" 51 | | "reset" 52 | | "add" 53 | | "filter" 54 | | "each" 55 | | "forEach" 56 | > { 57 | abort(): WithCodSpeedSuite; 58 | add( 59 | name: string, 60 | fn: CallableFunction | string, 61 | options?: Benchmark.Options 62 | ): WithCodSpeedSuite; 63 | add( 64 | fn: CallableFunction | string, 65 | options?: Benchmark.Options 66 | ): WithCodSpeedSuite; 67 | add(name: string, options?: Benchmark.Options): WithCodSpeedSuite; 68 | add(options: Benchmark.Options): WithCodSpeedSuite; 69 | clone(options: Benchmark.Options): WithCodSpeedSuite; 70 | filter(callback: CallableFunction | string): WithCodSpeedSuite; 71 | off(type?: string, callback?: CallableFunction): WithCodSpeedSuite; 72 | off(types: string[]): WithCodSpeedSuite; 73 | on(type?: string, callback?: CallableFunction): WithCodSpeedSuite; 74 | on(types: string[]): WithCodSpeedSuite; 75 | reset(): WithCodSpeedSuite; 76 | each(callback: CallableFunction): WithCodSpeedSuite; 77 | forEach(callback: CallableFunction): WithCodSpeedSuite; 78 | 79 | run(options?: Benchmark.Options): Benchmark.Suite | Promise; 80 | } 81 | 82 | export function withCodSpeed(suite: Benchmark): WithCodSpeedBenchmark; 83 | export function withCodSpeed(suite: Benchmark.Suite): WithCodSpeedSuite; 84 | export function withCodSpeed(item: unknown): unknown { 85 | if ((item as { length?: number }).length === undefined) { 86 | return withCodSpeedBenchmark(item as Benchmark); 87 | } else { 88 | return withCodSpeedSuite(item as Benchmark.Suite); 89 | } 90 | } 91 | 92 | function withCodSpeedBenchmark(bench: Benchmark): WithCodSpeedBenchmark { 93 | if (!InstrumentHooks.isInstrumented()) { 94 | const rawRun = bench.run; 95 | bench.run = (options?: Benchmark.Options) => { 96 | console.warn( 97 | `[CodSpeed] bench detected but no instrumentation found, falling back to benchmark.js` 98 | ); 99 | return rawRun.bind(bench)(options); 100 | }; 101 | return bench; 102 | } 103 | const callingFile = getCallingFile(2); // [here, withCodSpeed, actual caller] 104 | const codspeedBench = bench as BenchmarkWithOptions; 105 | if (codspeedBench.name !== undefined) { 106 | codspeedBench.uri = `${callingFile}::${bench.name}`; 107 | } 108 | // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/ban-ts-comment 109 | // @ts-ignore 110 | bench.run = async function (options?: Benchmark.Options): Promise { 111 | await runBenchmarks({ 112 | benches: [codspeedBench], 113 | baseUri: callingFile, 114 | benchmarkCompletedListeners: bench.listeners("complete"), 115 | options, 116 | }); 117 | return bench; 118 | }; 119 | return bench; 120 | } 121 | 122 | function withCodSpeedSuite(suite: Benchmark.Suite): WithCodSpeedSuite { 123 | if (!InstrumentHooks.isInstrumented()) { 124 | const rawRun = suite.run; 125 | suite.run = (options?: Benchmark.Options) => { 126 | console.warn( 127 | `[CodSpeed] ${suite.length} benches detected but no instrumentation found, falling back to benchmark.js` 128 | ); 129 | return rawRun.bind(suite)(options); 130 | }; 131 | return suite as WithCodSpeedSuite; 132 | } 133 | suite.add = buildSuiteAdd(suite); 134 | const callingFile = getCallingFile(2); // [here, withCodSpeed, actual caller] 135 | // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/ban-ts-comment 136 | // @ts-ignore 137 | suite.run = async function ( 138 | options?: Benchmark.Options 139 | ): Promise { 140 | const suiteName = suite.name; 141 | const benches = this as unknown as BenchmarkWithOptions[]; 142 | let baseUri = callingFile; 143 | if (suiteName !== undefined) { 144 | baseUri += `::${suiteName}`; 145 | } 146 | await runBenchmarks({ 147 | benches, 148 | baseUri, 149 | benchmarkCompletedListeners: suite.listeners("complete"), 150 | options, 151 | }); 152 | return suite; 153 | }; 154 | return suite as WithCodSpeedSuite; 155 | } 156 | 157 | type BenchmarkWithOptions = CodSpeedBenchmark & { options: Benchmark.Options }; 158 | 159 | interface RunBenchmarksOptions { 160 | benches: BenchmarkWithOptions[]; 161 | baseUri: string; 162 | benchmarkCompletedListeners: CallableFunction[]; 163 | options?: Benchmark.Options; 164 | } 165 | 166 | async function runBenchmarks({ 167 | benches, 168 | baseUri, 169 | benchmarkCompletedListeners, 170 | }: RunBenchmarksOptions): Promise { 171 | console.log(`[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`); 172 | setupCore(); 173 | for (let i = 0; i < benches.length; i++) { 174 | const bench = benches[i]; 175 | const uri = bench.uri ?? `${baseUri}::unknown_${i}`; 176 | const isAsync = bench.options.async || bench.options.defer; 177 | let benchPayload; 178 | if (bench.options.defer) { 179 | benchPayload = () => { 180 | return new Promise((resolve, reject) => { 181 | (bench.fn as CallableFunction)({ resolve, reject }); 182 | }); 183 | }; 184 | } else if (bench.options.async) { 185 | benchPayload = async () => { 186 | await (bench.fn as CallableFunction)(); 187 | }; 188 | } else { 189 | benchPayload = bench.fn as CallableFunction; 190 | } 191 | 192 | if (typeof bench.options.setup === "function") { 193 | await bench.options.setup(); 194 | } 195 | 196 | if (isAsync) { 197 | await optimizeFunction(benchPayload); 198 | await mongoMeasurement.start(uri); 199 | global.gc?.(); 200 | await (async function __codspeed_root_frame__() { 201 | InstrumentHooks.startBenchmark(); 202 | await benchPayload(); 203 | InstrumentHooks.stopBenchmark(); 204 | InstrumentHooks.setExecutedBenchmark(process.pid, uri); 205 | })(); 206 | await mongoMeasurement.stop(uri); 207 | } else { 208 | optimizeFunctionSync(benchPayload); 209 | await mongoMeasurement.start(uri); 210 | (function __codspeed_root_frame__() { 211 | InstrumentHooks.startBenchmark(); 212 | benchPayload(); 213 | InstrumentHooks.stopBenchmark(); 214 | InstrumentHooks.setExecutedBenchmark(process.pid, uri); 215 | })(); 216 | await mongoMeasurement.stop(uri); 217 | } 218 | 219 | if (typeof bench.options.teardown === "function") { 220 | await bench.options.teardown(); 221 | } 222 | 223 | console.log(` ✔ Measured ${uri}`); 224 | benchmarkCompletedListeners.forEach((listener) => listener()); 225 | } 226 | teardownCore(); 227 | console.log(`[CodSpeed] Done running ${benches.length} benches.`); 228 | } 229 | 230 | /** 231 | * Dynamically setup the CodSpeed instruments. 232 | */ 233 | export async function setupInstruments( 234 | body: SetupInstrumentsRequestBody 235 | ): Promise { 236 | if (!InstrumentHooks.isInstrumented()) { 237 | console.warn("[CodSpeed] No instrumentation found, using default mongoUrl"); 238 | 239 | return { remoteAddr: body.mongoUrl }; 240 | } 241 | 242 | return await mongoMeasurement.setupInstruments(body); 243 | } 244 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/src/types.ts: -------------------------------------------------------------------------------- 1 | import Benchmark, { Options } from "benchmark"; 2 | 3 | export interface CodSpeedBenchOptions extends Options { 4 | uri: string; 5 | } 6 | 7 | export interface CodSpeedBenchmark extends Benchmark { 8 | uri: string; 9 | } 10 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/tests/__snapshots__/index.integ.test.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`Benchmark check console output(instrumented=false) 1`] = ` 4 | { 5 | "log": [], 6 | "warn": [ 7 | [ 8 | "[CodSpeed] bench detected but no instrumentation found, falling back to benchmark.js", 9 | ], 10 | ], 11 | } 12 | `; 13 | 14 | exports[`Benchmark check console output(instrumented=true) 1`] = ` 15 | { 16 | "log": [ 17 | [ 18 | " ✔ Measured packages/benchmark.js-plugin/tests/index.integ.test.ts::RegExpSingle", 19 | ], 20 | [ 21 | "[CodSpeed] Done running 1 benches.", 22 | ], 23 | ], 24 | "warn": [], 25 | } 26 | `; 27 | 28 | exports[`Benchmark.Suite check console output(instrumented=false) 1`] = ` 29 | { 30 | "log": [], 31 | "warn": [ 32 | [ 33 | "[CodSpeed] 2 benches detected but no instrumentation found, falling back to benchmark.js", 34 | ], 35 | ], 36 | } 37 | `; 38 | 39 | exports[`Benchmark.Suite check console output(instrumented=true) 1`] = ` 40 | { 41 | "log": [ 42 | [ 43 | " ✔ Measured packages/benchmark.js-plugin/tests/index.integ.test.ts::thesuite::RegExp", 44 | ], 45 | [ 46 | " ✔ Measured packages/benchmark.js-plugin/tests/index.integ.test.ts::thesuite::unknown_1", 47 | ], 48 | [ 49 | "[CodSpeed] Done running 2 benches.", 50 | ], 51 | ], 52 | "warn": [], 53 | } 54 | `; 55 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/tests/registerBenchmarks.ts: -------------------------------------------------------------------------------- 1 | import type { WithCodSpeedSuite } from ".."; 2 | 3 | export function registerBenchmarks(suite: WithCodSpeedSuite) { 4 | suite.add( 5 | "RegExp", 6 | function () { 7 | /o/.test("Hello World!"); 8 | }, 9 | { maxTime: 0.1 } 10 | ); 11 | } 12 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/tests/registerOtherBenchmarks.ts: -------------------------------------------------------------------------------- 1 | import type { WithCodSpeedSuite } from ".."; 2 | 3 | export function registerOtherBenchmarks(suite: WithCodSpeedSuite) { 4 | suite.add( 5 | "RegExp", 6 | function () { 7 | /o/.test("Hello World!"); 8 | }, 9 | { maxTime: 0.1 } 10 | ); 11 | } 12 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.base.json", 3 | "compilerOptions": { 4 | "outDir": "dist", 5 | "rootDir": "src", 6 | "types": ["jest", "node"], 7 | "typeRoots": ["node_modules/@types", "../../node_modules/@types"] 8 | }, 9 | "references": [{ "path": "../core" }], 10 | "include": ["src/**/*.ts"] 11 | } 12 | -------------------------------------------------------------------------------- /packages/benchmark.js-plugin/tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "types": ["jest", "node"] 5 | }, 6 | "include": ["tests/**/*.ts", "benches/**/*.ts", "jest.config.ts"] 7 | } 8 | -------------------------------------------------------------------------------- /packages/core/.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | prebuilds 3 | generated -------------------------------------------------------------------------------- /packages/core/README.md: -------------------------------------------------------------------------------- 1 |
2 |

@codspeed/core

3 | 4 | The core Node library used to integrate with Codspeed runners 5 | 6 | [![CI](https://github.com/CodSpeedHQ/codspeed-node/actions/workflows/ci.yml/badge.svg)](https://github.com/CodSpeedHQ/codspeed-node/actions/workflows/ci.yml) 7 | [![npm (scoped)](https://img.shields.io/npm/v/@codspeed/core)](https://www.npmjs.com/package/@codspeed/core) 8 | [![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF) 9 | [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/CodSpeedHQ/codspeed-node) 10 | 11 |
12 | 13 | For now, this package should not be used directly. Instead, use one of the integration packages: 14 | 15 | - [`@codspeed/vitest-plugin`](../vitest-plugin): vitest compatibility layer for CodSpeed 16 | - [`@codspeed/tinybench-plugin`](../tinybench-plugin): tinybench compatibility layer for CodSpeed 17 | - [`@codspeed/benchmark.js-plugin`](../benchmark.js-plugin): Benchmark.js compatibility layer for CodSpeed 18 | -------------------------------------------------------------------------------- /packages/core/binding.gyp: -------------------------------------------------------------------------------- 1 | { 2 | "targets": [ 3 | { 4 | "target_name": "native_core", 5 | "cflags!": [ 6 | "-fno-exceptions" 7 | ], 8 | "cflags_cc!": [ 9 | "-fno-exceptions" 10 | ], 11 | "cflags": [ 12 | "-g", 13 | "-Wno-maybe-uninitialized", 14 | "-Wno-unused-variable", 15 | "-Wno-unused-parameter", 16 | "-Wno-unused-but-set-variable", 17 | "-Wno-type-limits" 18 | ], 19 | "cflags_cc": [ 20 | "-Wno-maybe-uninitialized", 21 | "-Wno-unused-variable", 22 | "-Wno-unused-parameter", 23 | "-Wno-unused-but-set-variable", 24 | "-Wno-type-limits" 25 | ], 26 | "sources": [ 27 | "src/native_core/linux_perf/linux_perf.cc", 28 | "src/native_core/linux_perf/linux_perf_listener.cc", 29 | "src/native_core/instruments/hooks_wrapper.cc", 30 | "src/native_core/instruments/hooks/dist/core.c", 31 | "src/native_core/native_core.cc" 32 | ], 33 | "include_dirs": [ 34 | "/src/", 17 | "/.rollup.cache/", 18 | ], 19 | }; 20 | -------------------------------------------------------------------------------- /packages/core/jest.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest').JestConfigWithTsJest} */ 2 | module.exports = { 3 | preset: "ts-jest", 4 | testEnvironment: "node", 5 | transform: { 6 | "^.+\\.tsx?$": [ 7 | "ts-jest", 8 | { 9 | tsconfig: "tsconfig.test.json", 10 | }, 11 | ], 12 | }, 13 | testPathIgnorePatterns: [ 14 | "/node_modules/", 15 | "/tests/", 16 | "/.rollup.cache/", 17 | ], 18 | }; 19 | -------------------------------------------------------------------------------- /packages/core/moon.yml: -------------------------------------------------------------------------------- 1 | tasks: 2 | clean: 3 | args: 4 | - build 5 | - generated/openapi 6 | build: 7 | deps: 8 | - build-native-addon 9 | - build-tracer-client 10 | 11 | build-native-addon: 12 | command: prebuildify --napi --strip 13 | inputs: 14 | - "src/native_core/**/*.cc" 15 | - "src/native_core/**/*.c" 16 | - "src/native_core/**/*.h" 17 | - "binding.gyp" 18 | outputs: 19 | - "prebuilds" 20 | 21 | build-tracer-client: 22 | inputs: 23 | - "./tracer.spec.json" 24 | outputs: 25 | - "src/generated/openapi" 26 | command: openapi --client axios --input ./tracer.spec.json --name MongoTracer --output ./src/generated/openapi 27 | 28 | typecheck: 29 | deps: 30 | - build-tracer-client 31 | 32 | lint: 33 | deps: 34 | - build-tracer-client 35 | -------------------------------------------------------------------------------- /packages/core/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@codspeed/core", 3 | "version": "5.0.1", 4 | "description": "The core Node library used to integrate with Codspeed runners", 5 | "keywords": [ 6 | "codspeed", 7 | "benchmark", 8 | "performance" 9 | ], 10 | "files": [ 11 | "dist", 12 | "prebuilds" 13 | ], 14 | "main": "dist/index.cjs.js", 15 | "module": "dist/index.es5.js", 16 | "types": "dist/index.d.ts", 17 | "gypfile": true, 18 | "author": "Arthur Pastel ", 19 | "repository": "https://github.com/CodSpeedHQ/codspeed-node", 20 | "homepage": "https://codspeed.io", 21 | "license": "Apache-2.0", 22 | "devDependencies": { 23 | "@types/find-up": "^4.0.0", 24 | "node-addon-api": "^5.1.0", 25 | "node-gyp": "^9.3.1", 26 | "openapi-typescript-codegen": "^0.23.0", 27 | "prebuildify": "^5.0.1" 28 | }, 29 | "dependencies": { 30 | "axios": "^1.4.0", 31 | "find-up": "^6.3.0", 32 | "form-data": "^4.0.4", 33 | "node-gyp-build": "^4.6.0" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /packages/core/rollup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "rollup"; 2 | import { declarationsPlugin, jsPlugins } from "../../rollup.options"; 3 | 4 | import pkg from "./package.json" assert { type: "json" }; 5 | const entrypoint = "src/index.ts"; 6 | 7 | export default defineConfig([ 8 | { 9 | input: entrypoint, 10 | output: [ 11 | { 12 | file: pkg.types, 13 | format: "es", 14 | sourcemap: true, 15 | }, 16 | ], 17 | plugins: declarationsPlugin({ compilerOptions: { composite: false } }), 18 | }, 19 | { 20 | input: entrypoint, 21 | output: [ 22 | { 23 | file: pkg.main, 24 | format: "cjs", 25 | sourcemap: true, 26 | }, 27 | { file: pkg.module, format: "es", sourcemap: true }, 28 | ], 29 | plugins: jsPlugins(pkg.version), 30 | }, 31 | ]); 32 | -------------------------------------------------------------------------------- /packages/core/src/index.ts: -------------------------------------------------------------------------------- 1 | import { checkV8Flags } from "./introspection"; 2 | import { MongoMeasurement } from "./mongoMeasurement"; 3 | import native_core from "./native_core"; 4 | 5 | declare const __VERSION__: string; 6 | 7 | const linuxPerf = new native_core.LinuxPerf(); 8 | 9 | export const isBound = native_core.isBound; 10 | 11 | export const mongoMeasurement = new MongoMeasurement(); 12 | 13 | type CodSpeedRunnerMode = "disabled" | "instrumented" | "walltime"; 14 | 15 | export function getCodspeedRunnerMode(): CodSpeedRunnerMode { 16 | const isCodSpeedEnabled = process.env.CODSPEED_ENV !== undefined; 17 | if (!isCodSpeedEnabled) { 18 | return "disabled"; 19 | } 20 | 21 | // If CODSPEED_ENV is set, check CODSPEED_RUNNER_MODE 22 | const codspeedRunnerMode = process.env.CODSPEED_RUNNER_MODE; 23 | if (codspeedRunnerMode === "instrumentation") { 24 | return "instrumented"; 25 | } else if (codspeedRunnerMode === "walltime") { 26 | return "walltime"; 27 | } 28 | 29 | console.warn( 30 | `Unknown codspeed runner mode: ${codspeedRunnerMode}, defaulting to disabled` 31 | ); 32 | return "disabled"; 33 | } 34 | 35 | export const setupCore = () => { 36 | if (!native_core.isBound) { 37 | throw new Error( 38 | "Native core module is not bound, CodSpeed integration will not work properly" 39 | ); 40 | } 41 | 42 | native_core.InstrumentHooks.setIntegration("codspeed-node", __VERSION__); 43 | linuxPerf.start(); 44 | checkV8Flags(); 45 | }; 46 | 47 | export const teardownCore = () => { 48 | linuxPerf.stop(); 49 | }; 50 | 51 | export type { 52 | SetupInstrumentsRequestBody, 53 | SetupInstrumentsResponse, 54 | } from "./generated/openapi"; 55 | export { getV8Flags, tryIntrospect } from "./introspection"; 56 | export { optimizeFunction, optimizeFunctionSync } from "./optimization"; 57 | export * from "./utils"; 58 | export * from "./walltime"; 59 | export const InstrumentHooks = native_core.InstrumentHooks; 60 | -------------------------------------------------------------------------------- /packages/core/src/introspection.ts: -------------------------------------------------------------------------------- 1 | import { writeFileSync } from "fs"; 2 | import { getCodspeedRunnerMode } from "."; 3 | 4 | const CUSTOM_INTROSPECTION_EXIT_CODE = 0; 5 | 6 | export const getV8Flags = () => { 7 | const nodeVersionMajor = parseInt(process.version.slice(1).split(".")[0]); 8 | const codspeedRunnerMode = getCodspeedRunnerMode(); 9 | 10 | const flags = ["--interpreted-frames-native-stack", "--allow-natives-syntax"]; 11 | 12 | if (codspeedRunnerMode === "instrumented") { 13 | flags.push( 14 | ...[ 15 | "--hash-seed=1", 16 | "--random-seed=1", 17 | "--no-opt", 18 | "--predictable", 19 | "--predictable-gc-schedule", 20 | "--expose-gc", 21 | "--no-concurrent-sweeping", 22 | "--max-old-space-size=4096", 23 | ] 24 | ); 25 | if (nodeVersionMajor < 18) { 26 | flags.push("--no-randomize-hashes"); 27 | } 28 | if (nodeVersionMajor < 20) { 29 | flags.push("--no-scavenge-task"); 30 | } 31 | } 32 | 33 | return flags; 34 | }; 35 | 36 | export const tryIntrospect = () => { 37 | if (process.env.__CODSPEED_NODE_CORE_INTROSPECTION_PATH__ !== undefined) { 38 | const introspectionMetadata = { 39 | flags: getV8Flags(), 40 | }; 41 | writeFileSync( 42 | process.env.__CODSPEED_NODE_CORE_INTROSPECTION_PATH__, 43 | JSON.stringify(introspectionMetadata) 44 | ); 45 | process.exit(CUSTOM_INTROSPECTION_EXIT_CODE); 46 | } 47 | }; 48 | 49 | export const checkV8Flags = () => { 50 | const requiredFlags = getV8Flags(); 51 | const actualFlags = process.execArgv; 52 | const missingFlags = requiredFlags.filter( 53 | (flag) => !actualFlags.includes(flag) 54 | ); 55 | if (missingFlags.length > 0) { 56 | console.warn( 57 | `[CodSpeed] missing required flags: ${missingFlags.join(", ")}` 58 | ); 59 | } 60 | }; 61 | -------------------------------------------------------------------------------- /packages/core/src/mongoMeasurement.ts: -------------------------------------------------------------------------------- 1 | import { 2 | MongoTracer, 3 | SetupInstrumentsRequestBody, 4 | SetupInstrumentsResponse, 5 | } from "./generated/openapi"; 6 | 7 | export type { SetupInstrumentsRequestBody }; 8 | 9 | export class MongoMeasurement { 10 | private tracerClient: MongoTracer | undefined; 11 | 12 | constructor() { 13 | const serverUrl = process.env.CODSPEED_MONGO_INSTR_SERVER_ADDRESS; 14 | 15 | if (serverUrl !== undefined) { 16 | this.tracerClient = new MongoTracer({ 17 | BASE: serverUrl, 18 | }); 19 | } 20 | } 21 | 22 | public async setupInstruments( 23 | body: SetupInstrumentsRequestBody 24 | ): Promise { 25 | if (this.tracerClient === undefined) { 26 | throw new Error("MongoDB Instrumentation is not enabled"); 27 | } 28 | return await this.tracerClient.instruments.setup(body); 29 | } 30 | 31 | public async start(uri: string) { 32 | if (this.tracerClient !== undefined) { 33 | await this.tracerClient.instrumentation.start({ 34 | uri, 35 | }); 36 | } 37 | } 38 | 39 | public async stop(uri: string) { 40 | if (this.tracerClient !== undefined) { 41 | await this.tracerClient.instrumentation.stop({ 42 | uri, 43 | }); 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /packages/core/src/native_core/index.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import { logDebug } from "../utils"; 3 | import { InstrumentHooks } from "./instruments/hooks"; 4 | import { LinuxPerf } from "./linux_perf/linux_perf"; 5 | interface NativeCore { 6 | InstrumentHooks: InstrumentHooks; 7 | LinuxPerf: typeof LinuxPerf; 8 | } 9 | 10 | interface NativeCoreWithBindingStatus extends NativeCore { 11 | isBound: boolean; 12 | } 13 | 14 | let native_core: NativeCoreWithBindingStatus; 15 | try { 16 | // eslint-disable-next-line @typescript-eslint/no-var-requires 17 | const nativeCore = require("node-gyp-build")( 18 | path.dirname(__dirname) 19 | ) as NativeCore; 20 | native_core = { 21 | ...nativeCore, 22 | isBound: true, 23 | }; 24 | } catch (e) { 25 | logDebug("Failed to bind native core, instruments will not work."); 26 | logDebug(e); 27 | native_core = { 28 | LinuxPerf: class LinuxPerf { 29 | start() { 30 | return false; 31 | } 32 | stop() { 33 | return false; 34 | } 35 | }, 36 | InstrumentHooks: { 37 | isInstrumented: () => { 38 | return false; 39 | }, 40 | startBenchmark: () => { 41 | return 0; 42 | }, 43 | stopBenchmark: () => { 44 | return 0; 45 | }, 46 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 47 | setExecutedBenchmark: (_pid: number, _uri: string) => { 48 | return 0; 49 | }, 50 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 51 | setIntegration: (_name: string, _version: string) => { 52 | return 0; 53 | }, 54 | __codspeed_root_frame__: (callback: () => T): T => { 55 | return callback(); 56 | }, 57 | }, 58 | isBound: false, 59 | }; 60 | } 61 | 62 | export default native_core; 63 | -------------------------------------------------------------------------------- /packages/core/src/native_core/instruments/hooks.ts: -------------------------------------------------------------------------------- 1 | export interface InstrumentHooks { 2 | /** 3 | * Check if instrumentation is enabled 4 | */ 5 | isInstrumented(): boolean; 6 | 7 | /** 8 | * Start benchmark measurement 9 | * @returns 0 on success, non-zero on error 10 | */ 11 | startBenchmark(): number; 12 | 13 | /** 14 | * Stop benchmark measurement 15 | * @returns 0 on success, non-zero on error 16 | */ 17 | stopBenchmark(): number; 18 | 19 | /** 20 | * Set the executed benchmark metadata 21 | * @param pid Process ID 22 | * @param uri Benchmark URI/identifier 23 | * @returns 0 on success, non-zero on error 24 | */ 25 | setExecutedBenchmark(pid: number, uri: string): number; 26 | 27 | /** 28 | * Set integration metadata 29 | * @param name Integration name 30 | * @param version Integration version 31 | * @returns 0 on success, non-zero on error 32 | */ 33 | setIntegration(name: string, version: string): number; 34 | 35 | /** 36 | * Execute a callback function with __codspeed_root_frame__ in its stack trace 37 | * @param callback Function to execute 38 | */ 39 | __codspeed_root_frame__(callback: () => T): T; 40 | } 41 | -------------------------------------------------------------------------------- /packages/core/src/native_core/instruments/hooks_wrapper.cc: -------------------------------------------------------------------------------- 1 | #include "hooks_wrapper.h" 2 | #include "hooks/includes/core.h" 3 | #include 4 | 5 | namespace codspeed_native { 6 | namespace instruments { 7 | namespace hooks_wrapper { 8 | 9 | static InstrumentHooks *hooks = nullptr; 10 | 11 | void InitializeGlobal() { 12 | if (!hooks) { 13 | hooks = instrument_hooks_init(); 14 | } 15 | } 16 | 17 | Napi::Boolean IsInstrumented(const Napi::CallbackInfo &info) { 18 | Napi::Env env = info.Env(); 19 | 20 | bool instrumented = instrument_hooks_is_instrumented(hooks); 21 | return Napi::Boolean::New(env, instrumented); 22 | } 23 | 24 | Napi::Number StartBenchmark(const Napi::CallbackInfo &info) { 25 | Napi::Env env = info.Env(); 26 | 27 | uint8_t result = instrument_hooks_start_benchmark(hooks); 28 | return Napi::Number::New(env, result); 29 | } 30 | 31 | Napi::Number StopBenchmark(const Napi::CallbackInfo &info) { 32 | Napi::Env env = info.Env(); 33 | 34 | uint8_t result = instrument_hooks_stop_benchmark(hooks); 35 | return Napi::Number::New(env, result); 36 | } 37 | 38 | Napi::Number SetExecutedBenchmark(const Napi::CallbackInfo &info) { 39 | Napi::Env env = info.Env(); 40 | 41 | if (info.Length() != 2) { 42 | Napi::TypeError::New(env, "Expected 2 arguments: pid and uri") 43 | .ThrowAsJavaScriptException(); 44 | return Napi::Number::New(env, 1); 45 | } 46 | 47 | if (!info[0].IsNumber() || !info[1].IsString()) { 48 | Napi::TypeError::New(env, "Expected number (pid) and string (uri)") 49 | .ThrowAsJavaScriptException(); 50 | return Napi::Number::New(env, 1); 51 | } 52 | 53 | uint32_t pid = info[0].As().Uint32Value(); 54 | std::string uri = info[1].As().Utf8Value(); 55 | 56 | uint8_t result = 57 | instrument_hooks_set_executed_benchmark(hooks, pid, uri.c_str()); 58 | return Napi::Number::New(env, result); 59 | } 60 | 61 | Napi::Number SetIntegration(const Napi::CallbackInfo &info) { 62 | Napi::Env env = info.Env(); 63 | 64 | if (info.Length() != 2) { 65 | Napi::TypeError::New(env, "Expected 2 arguments: name and version") 66 | .ThrowAsJavaScriptException(); 67 | return Napi::Number::New(env, 1); 68 | } 69 | 70 | if (!info[0].IsString() || !info[1].IsString()) { 71 | Napi::TypeError::New(env, "Expected string (name) and string (version)") 72 | .ThrowAsJavaScriptException(); 73 | return Napi::Number::New(env, 1); 74 | } 75 | 76 | std::string name = info[0].As().Utf8Value(); 77 | std::string version = info[1].As().Utf8Value(); 78 | 79 | uint8_t result = 80 | instrument_hooks_set_integration(hooks, name.c_str(), version.c_str()); 81 | return Napi::Number::New(env, result); 82 | } 83 | 84 | Napi::Value __attribute__ ((noinline)) __codspeed_root_frame__(const Napi::CallbackInfo &info) { 85 | Napi::Env env = info.Env(); 86 | 87 | if (info.Length() != 1) { 88 | Napi::TypeError::New(env, "Expected 1 argument: callback function") 89 | .ThrowAsJavaScriptException(); 90 | return env.Undefined(); 91 | } 92 | 93 | if (!info[0].IsFunction()) { 94 | Napi::TypeError::New(env, "Expected function argument") 95 | .ThrowAsJavaScriptException(); 96 | return env.Undefined(); 97 | } 98 | 99 | Napi::Function callback = info[0].As(); 100 | Napi::Value result = callback.Call(env.Global(), {}); 101 | 102 | return result; 103 | } 104 | 105 | Napi::Object Initialize(Napi::Env env, Napi::Object exports) { 106 | Napi::Object instrumentHooksObj = Napi::Object::New(env); 107 | 108 | InitializeGlobal(); 109 | 110 | instrumentHooksObj.Set(Napi::String::New(env, "isInstrumented"), 111 | Napi::Function::New(env, IsInstrumented)); 112 | instrumentHooksObj.Set(Napi::String::New(env, "startBenchmark"), 113 | Napi::Function::New(env, StartBenchmark)); 114 | instrumentHooksObj.Set(Napi::String::New(env, "stopBenchmark"), 115 | Napi::Function::New(env, StopBenchmark)); 116 | instrumentHooksObj.Set(Napi::String::New(env, "setExecutedBenchmark"), 117 | Napi::Function::New(env, SetExecutedBenchmark)); 118 | instrumentHooksObj.Set(Napi::String::New(env, "setIntegration"), 119 | Napi::Function::New(env, SetIntegration)); 120 | instrumentHooksObj.Set(Napi::String::New(env, "__codspeed_root_frame__"), 121 | Napi::Function::New(env, __codspeed_root_frame__)); 122 | 123 | exports.Set(Napi::String::New(env, "InstrumentHooks"), instrumentHooksObj); 124 | 125 | return exports; 126 | } 127 | 128 | } // namespace hooks_wrapper 129 | } // namespace instruments 130 | } // namespace codspeed_native 131 | -------------------------------------------------------------------------------- /packages/core/src/native_core/instruments/hooks_wrapper.h: -------------------------------------------------------------------------------- 1 | #ifndef INSTRUMENTS_HOOKS_WRAPPER_H 2 | #define INSTRUMENTS_HOOKS_WRAPPER_H 3 | 4 | #include 5 | 6 | namespace codspeed_native { 7 | namespace instruments { 8 | namespace hooks_wrapper { 9 | 10 | // Global initialization 11 | // WARNING: InitializeGlobal() must be called before using any other functions 12 | // All API functions assume the global instance is already initialized 13 | void InitializeGlobal(); 14 | 15 | Napi::Boolean IsInstrumented(const Napi::CallbackInfo &info); 16 | Napi::Number StartBenchmark(const Napi::CallbackInfo &info); 17 | Napi::Number StopBenchmark(const Napi::CallbackInfo &info); 18 | Napi::Number SetExecutedBenchmark(const Napi::CallbackInfo &info); 19 | Napi::Number SetIntegration(const Napi::CallbackInfo &info); 20 | Napi::Object Initialize(Napi::Env env, Napi::Object exports); 21 | 22 | } // namespace hooks_wrapper 23 | } // namespace instruments 24 | } // namespace codspeed_native 25 | 26 | #endif // INSTRUMENTS_HOOKS_WRAPPER_H 27 | -------------------------------------------------------------------------------- /packages/core/src/native_core/linux_perf/linux_perf.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "linux_perf.h" 4 | #include 5 | 6 | namespace codspeed_native { 7 | 8 | Napi::Object LinuxPerf::Initialize(Napi::Env env, Napi::Object exports) { 9 | Napi::Function func = DefineClass(env, "LinuxPerf", 10 | {InstanceMethod("start", &LinuxPerf::Start), 11 | InstanceMethod("stop", &LinuxPerf::Stop)}); 12 | 13 | exports.Set("LinuxPerf", func); 14 | return exports; 15 | } 16 | 17 | LinuxPerf::LinuxPerf(const Napi::CallbackInfo &info) 18 | : Napi::ObjectWrap(info) { 19 | handler = nullptr; 20 | } 21 | 22 | Napi::Value LinuxPerf::Start(const Napi::CallbackInfo &info) { 23 | if (handler == nullptr) { 24 | v8::Isolate *isolate = v8::Isolate::GetCurrent(); 25 | handler = new LinuxPerfHandler(isolate); 26 | handler->Enable(); 27 | return Napi::Boolean::New(info.Env(), true); 28 | } 29 | return Napi::Boolean::New(info.Env(), false); 30 | } 31 | 32 | Napi::Value LinuxPerf::Stop(const Napi::CallbackInfo &info) { 33 | if (handler != nullptr) { 34 | handler->Disable(); 35 | delete handler; 36 | handler = nullptr; 37 | return Napi::Boolean::New(info.Env(), true); 38 | } 39 | return Napi::Boolean::New(info.Env(), false); 40 | } 41 | 42 | } // namespace codspeed_native 43 | -------------------------------------------------------------------------------- /packages/core/src/native_core/linux_perf/linux_perf.h: -------------------------------------------------------------------------------- 1 | #ifndef __LINUX_PERF_H 2 | #define __LINUX_PERF_H 3 | 4 | #include "v8-profiler.h" 5 | #include 6 | #include 7 | #include 8 | 9 | namespace codspeed_native { 10 | 11 | class LinuxPerfHandler : public v8::CodeEventHandler { 12 | public: 13 | explicit LinuxPerfHandler(v8::Isolate *isolate); 14 | ~LinuxPerfHandler() override; 15 | 16 | void Handle(v8::CodeEvent *code_event) override; 17 | 18 | private: 19 | std::ofstream mapFile; 20 | std::string FormatName(v8::CodeEvent *code_event); 21 | v8::Isolate *isolate_; 22 | }; 23 | 24 | class LinuxPerf : public Napi::ObjectWrap { 25 | public: 26 | static Napi::Object Initialize(Napi::Env env, Napi::Object exports); 27 | 28 | LinuxPerf(const Napi::CallbackInfo &info); 29 | ~LinuxPerf() = default; 30 | 31 | Napi::Value Start(const Napi::CallbackInfo &info); 32 | Napi::Value Stop(const Napi::CallbackInfo &info); 33 | 34 | LinuxPerfHandler *handler; 35 | }; 36 | 37 | } // namespace codspeed_native 38 | 39 | #endif // __LINUX_PERF_H -------------------------------------------------------------------------------- /packages/core/src/native_core/linux_perf/linux_perf.ts: -------------------------------------------------------------------------------- 1 | export declare class LinuxPerf { 2 | constructor(); 3 | start(): boolean; 4 | stop(): boolean; 5 | } 6 | -------------------------------------------------------------------------------- /packages/core/src/native_core/linux_perf/linux_perf_listener.cc: -------------------------------------------------------------------------------- 1 | #include "linux_perf.h" 2 | #include "utils.h" 3 | #include 4 | #include 5 | 6 | namespace codspeed_native { 7 | 8 | LinuxPerfHandler::LinuxPerfHandler(v8::Isolate *isolate) 9 | : v8::CodeEventHandler(isolate) { 10 | isolate_ = isolate; 11 | int pid = static_cast(uv_os_getpid()); 12 | mapFile.open("/tmp/perf-" + std::to_string(pid) + ".map"); 13 | } 14 | 15 | LinuxPerfHandler::~LinuxPerfHandler() { mapFile.close(); } 16 | 17 | std::string LinuxPerfHandler::FormatName(v8::CodeEvent *code_event) { 18 | std::string name = std::string(code_event->GetComment()); 19 | if (name.empty()) { 20 | name = v8LocalStringToString(code_event->GetFunctionName()); 21 | } 22 | return name; 23 | } 24 | 25 | void LinuxPerfHandler::Handle(v8::CodeEvent *code_event) { 26 | mapFile << std::hex << code_event->GetCodeStartAddress() << " " 27 | << code_event->GetCodeSize() << " "; 28 | mapFile << v8::CodeEvent::GetCodeEventTypeName(code_event->GetCodeType()) 29 | << ":" << FormatName(code_event) << " " 30 | << v8LocalStringToString(code_event->GetScriptName()) << std::dec 31 | << ":" << code_event->GetScriptLine() << ":" 32 | << code_event->GetScriptColumn() << std::endl; 33 | } 34 | 35 | } // namespace codspeed_native 36 | -------------------------------------------------------------------------------- /packages/core/src/native_core/linux_perf/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef LINUX_PERF_UTILS_H 2 | #define LINUX_PERF_UTILS_H 3 | 4 | #include "v8-profiler.h" 5 | 6 | static inline std::string 7 | v8LocalStringToString(v8::Local v8String) { 8 | std::string buffer(v8String->Utf8Length(v8::Isolate::GetCurrent()) + 1, 0); 9 | v8String->WriteUtf8(v8::Isolate::GetCurrent(), &buffer[0], 10 | v8String->Utf8Length(v8::Isolate::GetCurrent()) + 1); 11 | // Sanitize name, removing unwanted \0 resulted from WriteUtf8 12 | return std::string(buffer.c_str()); 13 | } 14 | 15 | #endif // LINUX_PERF_UTILS_H -------------------------------------------------------------------------------- /packages/core/src/native_core/native_core.cc: -------------------------------------------------------------------------------- 1 | #include "linux_perf/linux_perf.h" 2 | #include "instruments/hooks_wrapper.h" 3 | #include 4 | 5 | namespace codspeed_native { 6 | 7 | Napi::Object Initialize(Napi::Env env, Napi::Object exports) { 8 | codspeed_native::LinuxPerf::Initialize(env, exports); 9 | codspeed_native::instruments::hooks_wrapper::Initialize(env, exports); 10 | 11 | return exports; 12 | } 13 | 14 | NODE_API_MODULE(native_core, Initialize) 15 | 16 | } // namespace codspeed_native 17 | -------------------------------------------------------------------------------- /packages/core/src/optimization.ts: -------------------------------------------------------------------------------- 1 | export const optimizeFunction = async (fn: CallableFunction) => { 2 | // Source: https://github.com/petkaantonov/bluebird/wiki/Optimization-killers#optimization-killers 3 | // a total of 7 calls seems to be the sweet spot 4 | await fn(); 5 | await fn(); 6 | await fn(); 7 | await fn(); 8 | await fn(); 9 | await fn(); 10 | eval("%OptimizeFunctionOnNextCall(fn)"); 11 | await fn(); // optimize 12 | }; 13 | 14 | export const optimizeFunctionSync = (fn: CallableFunction) => { 15 | // Source: https://github.com/petkaantonov/bluebird/wiki/Optimization-killers#optimization-killers 16 | // a total of 7 calls seems to be the sweet spot 17 | fn(); 18 | fn(); 19 | fn(); 20 | fn(); 21 | fn(); 22 | fn(); 23 | eval("%OptimizeFunctionOnNextCall(fn)"); 24 | fn(); // optimize 25 | }; 26 | -------------------------------------------------------------------------------- /packages/core/src/utils.ts: -------------------------------------------------------------------------------- 1 | import { findUpSync, Options as FindupOptions } from "find-up"; 2 | import { dirname } from "path"; 3 | 4 | export function getGitDir(path: string): string | undefined { 5 | const dotGitPath = findUpSync(".git", { 6 | cwd: path, 7 | type: "directory", 8 | } as FindupOptions); 9 | return dotGitPath ? dirname(dotGitPath) : undefined; 10 | } 11 | 12 | /** 13 | * Log debug messages if the environment variable `CODSPEED_DEBUG` is set. 14 | */ 15 | export function logDebug(...args: unknown[]) { 16 | if (process.env.CODSPEED_DEBUG) { 17 | console.log(...args); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /packages/core/src/walltime/index.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | import { Benchmark, ResultData } from "./interfaces"; 4 | 5 | declare const __VERSION__: string; 6 | 7 | export function getProfileFolder(): string | null { 8 | return process.env.CODSPEED_PROFILE_FOLDER || null; 9 | } 10 | 11 | export function writeWalltimeResults( 12 | benchmarks: Benchmark[], 13 | asyncWarning = false 14 | ): void { 15 | const profileFolder = getProfileFolder(); 16 | 17 | const resultDir = (() => { 18 | if (profileFolder) { 19 | return path.join(profileFolder, "results"); 20 | } else { 21 | // Fallback: write to .codspeed in current working directory 22 | return path.join(process.cwd(), ".codspeed"); 23 | } 24 | })(); 25 | fs.mkdirSync(resultDir, { recursive: true }); 26 | const resultPath = path.join(resultDir, `${process.pid}.json`); 27 | 28 | // Check if file already exists and merge benchmarks 29 | let existingBenchmarks: Benchmark[] = []; 30 | if (fs.existsSync(resultPath)) { 31 | try { 32 | const existingData = JSON.parse( 33 | fs.readFileSync(resultPath, "utf-8") 34 | ) as ResultData; 35 | existingBenchmarks = existingData.benchmarks || []; 36 | } catch (error) { 37 | console.warn(`[CodSpeed] Failed to read existing results file: ${error}`); 38 | } 39 | } 40 | 41 | const data: ResultData = { 42 | creator: { 43 | name: "codspeed-node", 44 | version: __VERSION__, 45 | pid: process.pid, 46 | }, 47 | instrument: { type: "walltime" }, 48 | benchmarks: [...existingBenchmarks, ...benchmarks], 49 | metadata: asyncWarning 50 | ? { 51 | async_warning: "Profiling is inaccurate due to async operations", 52 | } 53 | : undefined, 54 | }; 55 | 56 | fs.writeFileSync(resultPath, JSON.stringify(data, null, 2)); 57 | console.log( 58 | `[CodSpeed] Results written to ${resultPath} (${data.benchmarks.length} total benchmarks)` 59 | ); 60 | } 61 | 62 | export * from "./interfaces"; 63 | export * from "./quantiles"; 64 | export * from "./utils"; 65 | -------------------------------------------------------------------------------- /packages/core/src/walltime/interfaces.ts: -------------------------------------------------------------------------------- 1 | export interface BenchmarkStats { 2 | min_ns: number; 3 | max_ns: number; 4 | mean_ns: number; 5 | stdev_ns: number; 6 | q1_ns: number; 7 | median_ns: number; 8 | q3_ns: number; 9 | rounds: number; 10 | total_time: number; 11 | iqr_outlier_rounds: number; 12 | stdev_outlier_rounds: number; 13 | iter_per_round: number; 14 | warmup_iters: number; 15 | } 16 | 17 | export interface BenchmarkConfig { 18 | warmup_time_ns: number | null; 19 | min_round_time_ns: number | null; 20 | max_time_ns?: number | null; 21 | max_rounds?: number | null; 22 | } 23 | 24 | export interface Benchmark { 25 | name: string; 26 | uri: string; 27 | config: BenchmarkConfig; 28 | stats: BenchmarkStats; 29 | } 30 | 31 | export interface InstrumentInfo { 32 | type: string; 33 | } 34 | 35 | export interface ResultData { 36 | creator: { 37 | name: string; 38 | version: string; 39 | pid: number; 40 | }; 41 | instrument: { type: "walltime" }; 42 | benchmarks: Benchmark[]; 43 | metadata?: Record; 44 | } 45 | -------------------------------------------------------------------------------- /packages/core/src/walltime/quantiles.ts: -------------------------------------------------------------------------------- 1 | export function calculateQuantiles({ 2 | sortedTimesNs, 3 | stdevNs, 4 | meanNs, 5 | }: { 6 | sortedTimesNs: number[]; 7 | stdevNs: number; 8 | meanNs: number; 9 | }): { 10 | q1_ns: number; 11 | median_ns: number; 12 | q3_ns: number; 13 | iqr_outlier_rounds: number; 14 | stdev_outlier_rounds: number; 15 | } { 16 | const IQR_OUTLIER_FACTOR = 1.5; 17 | const STDEV_OUTLIER_FACTOR = 3; 18 | 19 | const n = sortedTimesNs.length; 20 | if (n === 0) { 21 | throw new Error("Cannot calculate quantiles for empty array"); 22 | } 23 | if (n === 1) { 24 | return { 25 | q1_ns: sortedTimesNs[0], 26 | median_ns: sortedTimesNs[0], 27 | q3_ns: sortedTimesNs[0], 28 | iqr_outlier_rounds: 0, 29 | stdev_outlier_rounds: 0, 30 | }; 31 | } 32 | 33 | // Use same quantile calculation as Python's statistics.quantiles(n=4) 34 | const q1Index = (n - 1) * 0.25; 35 | const q2Index = (n - 1) * 0.5; 36 | const q3Index = (n - 1) * 0.75; 37 | 38 | const q1_ns = interpolateQuantile(sortedTimesNs, q1Index); 39 | const median = interpolateQuantile(sortedTimesNs, q2Index); 40 | const q3_ns = interpolateQuantile(sortedTimesNs, q3Index); 41 | const iqr_ns = q3_ns - q1_ns; 42 | 43 | // Calculate outliers using same algorithm as pytest-codspeed 44 | const iqr_outlier_rounds = sortedTimesNs.filter( 45 | (t) => 46 | t < q1_ns - IQR_OUTLIER_FACTOR * iqr_ns || 47 | t > q3_ns + IQR_OUTLIER_FACTOR * iqr_ns 48 | ).length; 49 | 50 | const stdev_outlier_rounds = sortedTimesNs.filter( 51 | (t) => 52 | t < meanNs - STDEV_OUTLIER_FACTOR * stdevNs || 53 | t > meanNs + STDEV_OUTLIER_FACTOR * stdevNs 54 | ).length; 55 | 56 | return { 57 | q1_ns, 58 | median_ns: median, 59 | q3_ns, 60 | iqr_outlier_rounds, 61 | stdev_outlier_rounds, 62 | }; 63 | } 64 | 65 | function interpolateQuantile(sortedArray: number[], index: number): number { 66 | const lowerIndex = Math.floor(index); 67 | const upperIndex = Math.ceil(index); 68 | 69 | if (lowerIndex === upperIndex) { 70 | return sortedArray[lowerIndex]; 71 | } 72 | 73 | const lowerValue = sortedArray[lowerIndex]; 74 | const upperValue = sortedArray[upperIndex]; 75 | const fraction = index - lowerIndex; 76 | 77 | return lowerValue + fraction * (upperValue - lowerValue); 78 | } 79 | -------------------------------------------------------------------------------- /packages/core/src/walltime/utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Converts nanoseconds to milliseconds. 3 | * @param ns - the nanoseconds to convert 4 | * @returns the milliseconds 5 | */ 6 | export const nsToMs = (ns: number) => ns / 1e6; 7 | 8 | /** 9 | * Converts milliseconds to nanoseconds. 10 | * @param ms - the milliseconds to convert 11 | * @returns the nanoseconds 12 | */ 13 | export const msToNs = (ms: number) => ms * 1e6; 14 | 15 | /** 16 | * Converts milliseconds to seconds. 17 | * @param ms - the milliseconds to convert 18 | * @returns the seconds 19 | */ 20 | export const msToS = (ms: number) => ms / 1e3; 21 | -------------------------------------------------------------------------------- /packages/core/tests/index.integ.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-var-requires */ 2 | export {}; // Make this a module 3 | 4 | beforeEach(() => { 5 | jest.resetModules(); 6 | }); 7 | 8 | describe("with bindings", () => { 9 | it("should be bound", () => { 10 | const isBound = require("..").isBound as boolean; 11 | expect(isBound).toBe(true); 12 | }); 13 | }); 14 | 15 | describe("without bindings", () => { 16 | const initialEnv = process.env; 17 | beforeAll(() => { 18 | process.env.npm_config_arch = "unknown"; 19 | // Prevent node-gyp from falling back to a local version of the native core in packages/core/build 20 | process.env.PREBUILDS_ONLY = "1"; 21 | }); 22 | afterAll(() => { 23 | process.env = initialEnv; 24 | }); 25 | it("should not be bound", () => { 26 | const isBound = require("..").isBound as boolean; 27 | expect(isBound).toBe(false); 28 | }); 29 | 30 | it("should throw when calling setupCore", () => { 31 | const setupCore = require("..").setupCore as () => unknown; 32 | expect(setupCore).toThrowError("Native core module is not bound"); 33 | }); 34 | }); 35 | -------------------------------------------------------------------------------- /packages/core/tracer.spec.json: -------------------------------------------------------------------------------- 1 | { 2 | "openapi": "3.0.3", 3 | "info": { 4 | "title": "CodSpeed MongoDB Tracer", 5 | "description": "Instrumentation API for CodSpeed Tracer", 6 | "version": "0.2.0" 7 | }, 8 | "paths": { 9 | "/benchmark/start": { 10 | "post": { 11 | "tags": ["instrumentation"], 12 | "operationId": "start", 13 | "requestBody": { 14 | "content": { 15 | "application/json": { 16 | "schema": { 17 | "$ref": "#/components/schemas/InstrumentationRequestBody" 18 | } 19 | } 20 | }, 21 | "required": true 22 | }, 23 | "responses": { 24 | "200": { 25 | "description": "successful operation", 26 | "content": { 27 | "application/json": { 28 | "schema": { 29 | "$ref": "#/components/schemas/InstrumentationStatus" 30 | } 31 | } 32 | } 33 | }, 34 | "4XX": { 35 | "$ref": "#/components/responses/Error" 36 | }, 37 | "5XX": { 38 | "$ref": "#/components/responses/Error" 39 | } 40 | } 41 | } 42 | }, 43 | "/benchmark/stop": { 44 | "post": { 45 | "tags": ["instrumentation"], 46 | "operationId": "stop", 47 | "requestBody": { 48 | "content": { 49 | "application/json": { 50 | "schema": { 51 | "$ref": "#/components/schemas/InstrumentationRequestBody" 52 | } 53 | } 54 | }, 55 | "required": true 56 | }, 57 | "responses": { 58 | "200": { 59 | "description": "successful operation", 60 | "content": { 61 | "application/json": { 62 | "schema": { 63 | "$ref": "#/components/schemas/InstrumentationStatus" 64 | } 65 | } 66 | } 67 | }, 68 | "4XX": { 69 | "$ref": "#/components/responses/Error" 70 | }, 71 | "5XX": { 72 | "$ref": "#/components/responses/Error" 73 | } 74 | } 75 | } 76 | }, 77 | "/instruments/setup": { 78 | "post": { 79 | "tags": ["instruments"], 80 | "summary": "Start the instruments (proxy and aggregator) for the given `body.mongo_url`.", 81 | "description": "If other endpoints of the instrumentation server are called before this one, they will likely fail as the proxy and aggregator are not running yet.", 82 | "operationId": "setup", 83 | "requestBody": { 84 | "content": { 85 | "application/json": { 86 | "schema": { 87 | "$ref": "#/components/schemas/SetupInstrumentsRequestBody" 88 | } 89 | } 90 | }, 91 | "required": true 92 | }, 93 | "responses": { 94 | "200": { 95 | "description": "successful operation", 96 | "content": { 97 | "application/json": { 98 | "schema": { 99 | "$ref": "#/components/schemas/SetupInstrumentsResponse" 100 | } 101 | } 102 | } 103 | }, 104 | "4XX": { 105 | "$ref": "#/components/responses/Error" 106 | }, 107 | "5XX": { 108 | "$ref": "#/components/responses/Error" 109 | } 110 | } 111 | } 112 | }, 113 | "/status": { 114 | "get": { 115 | "tags": ["instrumentation"], 116 | "operationId": "status", 117 | "responses": { 118 | "200": { 119 | "description": "successful operation", 120 | "content": { 121 | "application/json": { 122 | "schema": { 123 | "$ref": "#/components/schemas/InstrumentationStatus" 124 | } 125 | } 126 | } 127 | }, 128 | "4XX": { 129 | "$ref": "#/components/responses/Error" 130 | }, 131 | "5XX": { 132 | "$ref": "#/components/responses/Error" 133 | } 134 | } 135 | } 136 | }, 137 | "/terminate": { 138 | "post": { 139 | "tags": ["instrumentation"], 140 | "operationId": "terminate", 141 | "responses": { 142 | "200": { 143 | "description": "successful operation", 144 | "content": { 145 | "application/json": { 146 | "schema": { 147 | "$ref": "#/components/schemas/AggregatorStore" 148 | } 149 | } 150 | } 151 | }, 152 | "4XX": { 153 | "$ref": "#/components/responses/Error" 154 | }, 155 | "5XX": { 156 | "$ref": "#/components/responses/Error" 157 | } 158 | } 159 | } 160 | } 161 | }, 162 | "components": { 163 | "responses": { 164 | "Error": { 165 | "description": "Error", 166 | "content": { 167 | "application/json": { 168 | "schema": { 169 | "$ref": "#/components/schemas/Error" 170 | } 171 | } 172 | } 173 | } 174 | }, 175 | "schemas": { 176 | "AggregatorMetadata": { 177 | "type": "object", 178 | "properties": { 179 | "name": { 180 | "type": "string" 181 | }, 182 | "version": { 183 | "type": "string" 184 | } 185 | }, 186 | "required": ["name", "version"] 187 | }, 188 | "AggregatorStore": { 189 | "type": "object", 190 | "properties": { 191 | "metadata": { 192 | "nullable": true, 193 | "allOf": [ 194 | { 195 | "$ref": "#/components/schemas/AggregatorMetadata" 196 | } 197 | ] 198 | }, 199 | "queries": { 200 | "type": "object", 201 | "additionalProperties": { 202 | "type": "array", 203 | "items": { 204 | "$ref": "#/components/schemas/MongoQuery" 205 | } 206 | } 207 | } 208 | }, 209 | "required": ["queries"] 210 | }, 211 | "Document": { 212 | "type": "object" 213 | }, 214 | "Error": { 215 | "description": "Error information from a response.", 216 | "type": "object", 217 | "properties": { 218 | "error_code": { 219 | "type": "string" 220 | }, 221 | "message": { 222 | "type": "string" 223 | }, 224 | "request_id": { 225 | "type": "string" 226 | } 227 | }, 228 | "required": ["message", "request_id"] 229 | }, 230 | "InstrumentationRequestBody": { 231 | "type": "object", 232 | "properties": { 233 | "uri": { 234 | "type": "string" 235 | } 236 | }, 237 | "required": ["uri"] 238 | }, 239 | "InstrumentationStatus": { 240 | "type": "object", 241 | "properties": { 242 | "currentUri": { 243 | "nullable": true, 244 | "type": "string" 245 | } 246 | } 247 | }, 248 | "MongoQuery": { 249 | "type": "object", 250 | "properties": { 251 | "collection": { 252 | "type": "string" 253 | }, 254 | "database": { 255 | "type": "string" 256 | }, 257 | "explanation": { 258 | "nullable": true, 259 | "allOf": [ 260 | { 261 | "$ref": "#/components/schemas/Document" 262 | } 263 | ] 264 | }, 265 | "op": { 266 | "type": "string" 267 | }, 268 | "query_documents": { 269 | "type": "array", 270 | "items": { 271 | "$ref": "#/components/schemas/Document" 272 | } 273 | }, 274 | "response_documents": { 275 | "type": "array", 276 | "items": { 277 | "$ref": "#/components/schemas/Document" 278 | } 279 | } 280 | }, 281 | "required": [ 282 | "collection", 283 | "database", 284 | "op", 285 | "query_documents", 286 | "response_documents" 287 | ] 288 | }, 289 | "SetupInstrumentsRequestBody": { 290 | "type": "object", 291 | "properties": { 292 | "mongoUrl": { 293 | "description": "The full `MONGO_URL` that is usually used to connect to the database.", 294 | "type": "string" 295 | } 296 | }, 297 | "required": ["mongoUrl"] 298 | }, 299 | "SetupInstrumentsResponse": { 300 | "type": "object", 301 | "properties": { 302 | "remoteAddr": { 303 | "description": "The patched `MONGO_URL` that should be used to connect to the database.", 304 | "type": "string" 305 | } 306 | }, 307 | "required": ["remoteAddr"] 308 | } 309 | } 310 | }, 311 | "tags": [ 312 | { 313 | "name": "instrumentation" 314 | }, 315 | { 316 | "name": "instruments" 317 | } 318 | ] 319 | } 320 | -------------------------------------------------------------------------------- /packages/core/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.base.json", 3 | "compilerOptions": { 4 | "outDir": "dist", 5 | "rootDir": "src" 6 | }, 7 | "include": ["src/**/*.ts"] 8 | } 9 | -------------------------------------------------------------------------------- /packages/core/tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "types": ["jest", "node"] 5 | }, 6 | "include": ["tests/**/*.ts", "benches/**/*.ts", "jest.config.ts"] 7 | } 8 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/README.md: -------------------------------------------------------------------------------- 1 |
2 |

@codspeed/tinybench-plugin

3 | 4 | tinybench compatibility layer for CodSpeed 5 | 6 | [![CI](https://github.com/CodSpeedHQ/codspeed-node/actions/workflows/ci.yml/badge.svg)](https://github.com/CodSpeedHQ/codspeed-node/actions/workflows/ci.yml) 7 | [![npm (scoped)](https://img.shields.io/npm/v/@codspeed/tinybench-plugin)](https://www.npmjs.com/package/@codspeed/tinybench-plugin) 8 | [![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF) 9 | [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/CodSpeedHQ/codspeed-node) 10 | 11 |
12 | 13 | ## Documentation 14 | 15 | Check out the [documentation](https://docs.codspeed.io/benchmarks/nodejs/tinybench) for complete integration instructions. 16 | 17 | ## Installation 18 | 19 | First, install the plugin [`@codspeed/tinybench-plugin`](https://www.npmjs.com/package/@codspeed/tinybench-plugin) and `tinybench` (if not already installed): 20 | 21 | ```sh 22 | npm install --save-dev @codspeed/tinybench-plugin tinybench 23 | ``` 24 | 25 | or with `yarn`: 26 | 27 | ```sh 28 | yarn add --dev @codspeed/tinybench-plugin tinybench 29 | ``` 30 | 31 | or with `pnpm`: 32 | 33 | ```sh 34 | pnpm add --save-dev @codspeed/tinybench-plugin tinybench 35 | ``` 36 | 37 | ## Usage 38 | 39 | Let's create a fibonacci function and benchmark it with tinybench and the CodSpeed plugin: 40 | 41 | ```js title="benches/bench.mjs" 42 | import { Bench } from "tinybench"; 43 | import { withCodSpeed } from "@codspeed/tinybench-plugin"; 44 | 45 | function fibonacci(n) { 46 | if (n < 2) { 47 | return n; 48 | } 49 | return fibonacci(n - 1) + fibonacci(n - 2); 50 | } 51 | 52 | const bench = withCodSpeed(new Bench()); 53 | 54 | bench 55 | .add("fibonacci10", () => { 56 | fibonacci(10); 57 | }) 58 | .add("fibonacci15", () => { 59 | fibonacci(15); 60 | }); 61 | 62 | await bench.run(); 63 | console.table(bench.table()); 64 | ``` 65 | 66 | Here, a few things are happening: 67 | 68 | - We create a simple recursive fibonacci function. 69 | - We create a new `Bench` instance with CodSpeed support by using the **`withCodSpeed`** helper. This step is **critical** to enable CodSpeed on your benchmarks. 70 | 71 | - We add two benchmarks to the suite and launch it, benching our `fibonacci` function for 10 and 15. 72 | 73 | Now, we can run our benchmarks locally to make sure everything is working as expected: 74 | 75 | ```sh 76 | $ node benches/bench.mjs 77 | [CodSpeed] 2 benches detected but no instrumentation found 78 | [CodSpeed] falling back to tinybench 79 | 80 | ┌─────────┬───────────────┬─────────────┬───────────────────┬──────────┬─────────┐ 81 | │ (index) │ Task Name │ ops/sec │ Average Time (ns) │ Margin │ Samples │ 82 | ├─────────┼───────────────┼─────────────┼───────────────────┼──────────┼─────────┤ 83 | │ 0 │ 'fibonacci10' │ '1,810,236' │ 552.4139857896414 │ '±0.18%' │ 905119 │ 84 | │ 1 │ 'fibonacci15' │ '177,516' │ 5633.276191749634 │ '±0.14%' │ 88759 │ 85 | └─────────┴───────────────┴─────────────┴───────────────────┴──────────┴─────────┘ 86 | ``` 87 | 88 | And... Congrats🎉, CodSpeed is installed in your benchmarking suite! Locally, CodSpeed will fallback to tinybench since the instrumentation is only available in the CI environment for now. 89 | 90 | You can now [run those benchmarks in your CI](https://docs.codspeed.io/benchmarks/nodejs/tinybench#running-the-benchmarks-in-your-ci) to continuously get consistent performance measurements. 91 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/benches/parsePr.ts: -------------------------------------------------------------------------------- 1 | interface PullRequest { 2 | number: number; 3 | title: string; 4 | body: string; 5 | } 6 | 7 | function sendEvent(numberOfOperations: number): void { 8 | for (let i = 0; i < numberOfOperations; i++) { 9 | let a = i; 10 | a = a + 1; 11 | } 12 | } 13 | 14 | function logMetrics( 15 | numberOfOperations: number, 16 | numberOfDeepOperations: number 17 | ): void { 18 | for (let i = 0; i < numberOfOperations; i++) { 19 | for (let i = 0; i < numberOfOperations; i++) { 20 | let a = i; 21 | a = a + 1; 22 | a = a + 1; 23 | } 24 | sendEvent(numberOfDeepOperations); 25 | } 26 | } 27 | 28 | function parseTitle(title: string): void { 29 | logMetrics(10, 10); 30 | modifyTitle(title); 31 | } 32 | 33 | function modifyTitle(title: string): void { 34 | for (let i = 0; i < 100; i++) { 35 | let a = i; 36 | a = a + 1 + title.length; 37 | } 38 | } 39 | 40 | function prepareParsingBody(body: string): void { 41 | for (let i = 0; i < 100; i++) { 42 | let a = i; 43 | a = a + 1; 44 | } 45 | parseBody(body); 46 | } 47 | 48 | function parseBody(body: string): void { 49 | logMetrics(10, 10); 50 | for (let i = 0; i < 200; i++) { 51 | let a = i; 52 | a = a + 1; 53 | } 54 | parseIssueFixed(body); 55 | } 56 | 57 | function parseIssueFixed(body: string): number | null { 58 | const prefix = "fixes #"; 59 | const index = body.indexOf(prefix); 60 | if (index === -1) { 61 | return null; 62 | } 63 | 64 | const start = index + prefix.length; 65 | let end = start; 66 | while (end < body.length && /\d/.test(body[end])) { 67 | end += 1; 68 | } 69 | return parseInt(body.slice(start, end)); 70 | } 71 | 72 | export default function parsePr(pullRequest: PullRequest): void { 73 | parseTitle(pullRequest.title); 74 | prepareParsingBody(pullRequest.body); 75 | } 76 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/benches/sample.ts: -------------------------------------------------------------------------------- 1 | import { Bench } from "tinybench"; 2 | import { withCodSpeed } from ".."; 3 | import parsePr from "./parsePr"; 4 | import { registerTimingBenchmarks } from "./timing"; 5 | 6 | const LONG_BODY = 7 | new Array(1_000) 8 | .fill( 9 | "Lorem ipsum dolor sit amet consectetur adipisicing elit. Sunt, earum. Atque architecto vero veniam est tempora fugiat sint quo praesentium quia. Autem, veritatis omnis beatae iste delectus recusandae animi non." 10 | ) 11 | .join("\n") + "fixes #123"; 12 | 13 | const bench = withCodSpeed(new Bench({ time: 100 })); 14 | 15 | bench 16 | .add("switch 1", () => { 17 | let a = 1; 18 | let b = 2; 19 | const c = a; 20 | a = b; 21 | b = c; 22 | }) 23 | .add("switch 2", () => { 24 | let a = 1; 25 | let b = 10; 26 | a = b + a; 27 | b = a - b; 28 | a = b - a; 29 | }) 30 | .add("short body", () => { 31 | parsePr({ body: "fixes #123", title: "test", number: 124 }); 32 | }) 33 | .add("long body", () => { 34 | parsePr({ body: LONG_BODY, title: "test", number: 124 }); 35 | }); 36 | 37 | (async () => { 38 | bench.runSync(); 39 | console.table(bench.table()); 40 | 41 | const timingBench = withCodSpeed( 42 | new Bench({ name: "timing", iterations: 5, warmup: false }) 43 | ); 44 | 45 | registerTimingBenchmarks(timingBench); 46 | 47 | timingBench.runSync(); 48 | console.table(timingBench.table()); 49 | })(); 50 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/benches/timing.ts: -------------------------------------------------------------------------------- 1 | import type { Bench } from "tinybench"; 2 | 3 | const busySleep = (ms: number): void => { 4 | const end = performance.now() + ms; 5 | while (performance.now() < end) { 6 | // Busy wait 7 | } 8 | }; 9 | 10 | export function registerTimingBenchmarks(bench: Bench) { 11 | bench.add("wait 1ms", () => { 12 | busySleep(1); 13 | }); 14 | 15 | bench.add("wait 500ms", () => { 16 | busySleep(500); 17 | }); 18 | 19 | bench.add("wait 1sec", () => { 20 | busySleep(1000); 21 | }); 22 | } 23 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/jest.config.integ.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest').JestConfigWithTsJest} */ 2 | // eslint-disable-next-line no-undef 3 | module.exports = { 4 | preset: "ts-jest", 5 | testEnvironment: "node", 6 | transform: { 7 | "^.+\\.tsx?$": [ 8 | "ts-jest", 9 | { 10 | tsconfig: "tsconfig.test.json", 11 | }, 12 | ], 13 | }, 14 | testPathIgnorePatterns: [ 15 | "/node_modules/", 16 | "/src/", 17 | "/.rollup.cache/", 18 | ], 19 | }; 20 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/jest.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest').JestConfigWithTsJest} */ 2 | module.exports = { 3 | preset: "ts-jest", 4 | testEnvironment: "node", 5 | transform: { 6 | "^.+\\.tsx?$": [ 7 | "ts-jest", 8 | { 9 | tsconfig: "tsconfig.test.json", 10 | }, 11 | ], 12 | }, 13 | testPathIgnorePatterns: [ 14 | "/node_modules/", 15 | "/tests/", 16 | "/.rollup.cache/", 17 | ], 18 | }; 19 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/moon.yml: -------------------------------------------------------------------------------- 1 | tasks: 2 | bench: 3 | command: node --loader esbuild-register/loader -r esbuild-register benches/sample.ts 4 | inputs: 5 | - "benches/**" 6 | local: true 7 | platform: "system" 8 | options: 9 | cache: false 10 | deps: 11 | - build 12 | 13 | test: 14 | command: vitest --run 15 | inputs: 16 | - "./vitest.config.ts" 17 | 18 | test/integ: 19 | command: noop 20 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@codspeed/tinybench-plugin", 3 | "version": "5.0.1", 4 | "description": "tinybench compatibility layer for CodSpeed", 5 | "keywords": [ 6 | "codspeed", 7 | "benchmark", 8 | "tinybench", 9 | "performance" 10 | ], 11 | "main": "dist/index.cjs", 12 | "module": "dist/index.es5.js", 13 | "types": "dist/index.d.ts", 14 | "type": "module", 15 | "exports": { 16 | "types": "./dist/index.d.ts", 17 | "import": "./dist/index.es5.js", 18 | "require": "./dist/index.cjs" 19 | }, 20 | "files": [ 21 | "dist" 22 | ], 23 | "author": "Arthur Pastel ", 24 | "repository": "https://github.com/CodSpeedHQ/codspeed-node", 25 | "homepage": "https://codspeed.io", 26 | "license": "Apache-2.0", 27 | "devDependencies": { 28 | "@types/stack-trace": "^0.0.30", 29 | "esbuild-register": "^3.4.2", 30 | "tinybench": "^4.0.1", 31 | "vitest": "^3.2.4" 32 | }, 33 | "dependencies": { 34 | "@codspeed/core": "workspace:^5.0.1", 35 | "stack-trace": "1.0.0-pre2" 36 | }, 37 | "peerDependencies": { 38 | "tinybench": ">=4.0.1" 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/rollup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "rollup"; 2 | import { declarationsPlugin, jsPlugins } from "../../rollup.options"; 3 | import pkg from "./package.json" assert { type: "json" }; 4 | 5 | const entrypoint = "src/index.ts"; 6 | 7 | export default defineConfig([ 8 | { 9 | input: entrypoint, 10 | output: [ 11 | { 12 | file: pkg.types, 13 | format: "es", 14 | sourcemap: true, 15 | }, 16 | ], 17 | plugins: declarationsPlugin({ compilerOptions: { composite: false } }), 18 | }, 19 | { 20 | input: entrypoint, 21 | output: [ 22 | { 23 | file: pkg.main, 24 | format: "cjs", 25 | sourcemap: true, 26 | }, 27 | { file: pkg.module, format: "es", sourcemap: true }, 28 | ], 29 | plugins: jsPlugins(pkg.version), 30 | external: ["@codspeed/core"], 31 | }, 32 | ]); 33 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/src/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | getCodspeedRunnerMode, 3 | getGitDir, 4 | InstrumentHooks, 5 | mongoMeasurement, 6 | SetupInstrumentsRequestBody, 7 | SetupInstrumentsResponse, 8 | tryIntrospect, 9 | } from "@codspeed/core"; 10 | import path from "path"; 11 | import { get as getStackTrace } from "stack-trace"; 12 | import { Bench } from "tinybench"; 13 | import { fileURLToPath } from "url"; 14 | import { setupCodspeedInstrumentedBench } from "./instrumented"; 15 | import { getOrCreateUriMap } from "./uri"; 16 | import { setupCodspeedWalltimeBench } from "./walltime"; 17 | 18 | tryIntrospect(); 19 | 20 | export function withCodSpeed(bench: Bench): Bench { 21 | const codspeedRunnerMode = getCodspeedRunnerMode(); 22 | if (codspeedRunnerMode === "disabled") { 23 | return bench; 24 | } 25 | 26 | const rootCallingFile = getCallingFile(); 27 | 28 | // Compute and register URI for bench 29 | const uriMap = getOrCreateUriMap(bench); 30 | const rawAdd = bench.add; 31 | bench.add = (name, fn, opts?) => { 32 | const callingFile = getCallingFile(); 33 | let uri = callingFile; 34 | if (bench.name !== undefined) { 35 | uri += `::${bench.name}`; 36 | } 37 | uri += `::${name}`; 38 | uriMap.set(name, uri); 39 | return rawAdd.bind(bench)(name, fn, opts); 40 | }; 41 | 42 | if (codspeedRunnerMode === "instrumented") { 43 | setupCodspeedInstrumentedBench(bench, rootCallingFile); 44 | } else if (codspeedRunnerMode === "walltime") { 45 | setupCodspeedWalltimeBench(bench, rootCallingFile); 46 | } 47 | 48 | return bench; 49 | } 50 | 51 | function getCallingFile(): string { 52 | const stack = getStackTrace(); 53 | let callingFile = stack[2].getFileName(); // [here, withCodSpeed, actual caller] 54 | const gitDir = getGitDir(callingFile); 55 | if (gitDir === undefined) { 56 | throw new Error("Could not find a git repository"); 57 | } 58 | if (callingFile.startsWith("file://")) { 59 | callingFile = fileURLToPath(callingFile); 60 | } 61 | return path.relative(gitDir, callingFile); 62 | } 63 | 64 | /** 65 | * Dynamically setup the CodSpeed instruments. 66 | */ 67 | export async function setupInstruments( 68 | body: SetupInstrumentsRequestBody 69 | ): Promise { 70 | if (!InstrumentHooks.isInstrumented()) { 71 | console.warn("[CodSpeed] No instrumentation found, using default mongoUrl"); 72 | 73 | return { remoteAddr: body.mongoUrl }; 74 | } 75 | 76 | return await mongoMeasurement.setupInstruments(body); 77 | } 78 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/src/index.unit.test.ts: -------------------------------------------------------------------------------- 1 | import { Bench } from "tinybench"; 2 | import { beforeEach, describe, expect, it, vi } from "vitest"; 3 | import { withCodSpeed } from "."; 4 | 5 | const mockInstrumented = vi.hoisted(() => ({ 6 | setupCodspeedInstrumentedBench: vi.fn(), 7 | })); 8 | 9 | vi.mock("./instrumented", () => ({ 10 | ...mockInstrumented, 11 | })); 12 | 13 | const mockWalltime = vi.hoisted(() => ({ 14 | setupCodspeedWalltimeBench: vi.fn(), 15 | })); 16 | 17 | vi.mock("./walltime", () => ({ 18 | ...mockWalltime, 19 | })); 20 | 21 | describe("withCodSpeed behavior without different codspeed modes", () => { 22 | beforeEach(() => { 23 | vi.clearAllMocks(); 24 | // Ensure no CODSPEED env vars are set 25 | delete process.env.CODSPEED_ENV; 26 | delete process.env.CODSPEED_RUNNER_MODE; 27 | }); 28 | 29 | it("should return the same bench instance unchanged when no CODSPEED_ENV", async () => { 30 | const originalBench = new Bench({ iterations: 10, time: 10 }); 31 | const wrappedBench = withCodSpeed(originalBench); 32 | const shouldBeCalled = vi.fn(); 33 | wrappedBench.add("test task", shouldBeCalled); 34 | await wrappedBench.run(); 35 | 36 | // Should return the exact same instance 37 | expect(wrappedBench).toBe(originalBench); 38 | expect(shouldBeCalled.mock.calls.length).toBeGreaterThan(1000); 39 | }); 40 | 41 | it("should run in instrumented mode when CODSPEED_RUNNER_MODE=instrumentation", async () => { 42 | process.env.CODSPEED_ENV = "true"; 43 | process.env.CODSPEED_RUNNER_MODE = "instrumentation"; 44 | 45 | withCodSpeed(new Bench()); 46 | 47 | expect(mockInstrumented.setupCodspeedInstrumentedBench).toHaveBeenCalled(); 48 | expect(mockWalltime.setupCodspeedWalltimeBench).not.toHaveBeenCalled(); 49 | }); 50 | 51 | it("should run in walltime mode when CODSPEED_RUNNER_MODE=walltime", async () => { 52 | process.env.CODSPEED_ENV = "true"; 53 | process.env.CODSPEED_RUNNER_MODE = "walltime"; 54 | 55 | withCodSpeed(new Bench()); 56 | 57 | expect( 58 | mockInstrumented.setupCodspeedInstrumentedBench 59 | ).not.toHaveBeenCalled(); 60 | expect(mockWalltime.setupCodspeedWalltimeBench).toHaveBeenCalled(); 61 | }); 62 | }); 63 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/src/instrumented.ts: -------------------------------------------------------------------------------- 1 | import { 2 | InstrumentHooks, 3 | mongoMeasurement, 4 | optimizeFunction, 5 | } from "@codspeed/core"; 6 | import { Bench, Fn, FnOptions, Task } from "tinybench"; 7 | import { BaseBenchRunner } from "./shared"; 8 | 9 | export function setupCodspeedInstrumentedBench( 10 | bench: Bench, 11 | rootCallingFile: string 12 | ): void { 13 | const runner = new InstrumentedBenchRunner(bench, rootCallingFile); 14 | runner.setupBenchMethods(); 15 | } 16 | 17 | class InstrumentedBenchRunner extends BaseBenchRunner { 18 | protected getModeName(): string { 19 | return "instrumented mode"; 20 | } 21 | 22 | private taskCompletionMessage() { 23 | return InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; 24 | } 25 | 26 | private wrapFunctionWithFrame(fn: Fn, isAsync: boolean): Fn { 27 | if (isAsync) { 28 | return async function __codspeed_root_frame__() { 29 | await fn(); 30 | }; 31 | } else { 32 | return function __codspeed_root_frame__() { 33 | fn(); 34 | }; 35 | } 36 | } 37 | 38 | protected async runTaskAsync(task: Task, uri: string): Promise { 39 | const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn }; 40 | 41 | await fnOpts?.beforeAll?.call(task, "run"); 42 | await optimizeFunction(async () => { 43 | await fnOpts?.beforeEach?.call(task, "run"); 44 | await fn(); 45 | await fnOpts?.afterEach?.call(task, "run"); 46 | }); 47 | await fnOpts?.beforeEach?.call(task, "run"); 48 | await mongoMeasurement.start(uri); 49 | 50 | global.gc?.(); 51 | await this.wrapWithInstrumentHooksAsync( 52 | this.wrapFunctionWithFrame(fn, true), 53 | uri 54 | ); 55 | 56 | await mongoMeasurement.stop(uri); 57 | await fnOpts?.afterEach?.call(task, "run"); 58 | await fnOpts?.afterAll?.call(task, "run"); 59 | 60 | this.logTaskCompletion(uri, this.taskCompletionMessage()); 61 | } 62 | 63 | protected runTaskSync(task: Task, uri: string): void { 64 | const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn }; 65 | 66 | fnOpts?.beforeAll?.call(task, "run"); 67 | fnOpts?.beforeEach?.call(task, "run"); 68 | 69 | this.wrapWithInstrumentHooks(this.wrapFunctionWithFrame(fn, false), uri); 70 | 71 | fnOpts?.afterEach?.call(task, "run"); 72 | fnOpts?.afterAll?.call(task, "run"); 73 | 74 | this.logTaskCompletion(uri, this.taskCompletionMessage()); 75 | } 76 | 77 | protected finalizeAsyncRun(): Task[] { 78 | return this.finalizeBenchRun(); 79 | } 80 | 81 | protected finalizeSyncRun(): Task[] { 82 | return this.finalizeBenchRun(); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/src/shared.ts: -------------------------------------------------------------------------------- 1 | import { InstrumentHooks, setupCore, teardownCore } from "@codspeed/core"; 2 | import { Bench, Fn, Task } from "tinybench"; 3 | import { getTaskUri } from "./uri"; 4 | 5 | declare const __VERSION__: string; 6 | 7 | export abstract class BaseBenchRunner { 8 | protected bench: Bench; 9 | protected rootCallingFile: string; 10 | 11 | constructor(bench: Bench, rootCallingFile: string) { 12 | this.bench = bench; 13 | this.rootCallingFile = rootCallingFile; 14 | } 15 | 16 | private setupBenchRun(): void { 17 | setupCore(); 18 | this.logStart(); 19 | } 20 | 21 | private logStart(): void { 22 | console.log( 23 | `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (${this.getModeName()})` 24 | ); 25 | } 26 | 27 | protected getTaskUri(task: Task): string { 28 | return getTaskUri(this.bench, task.name, this.rootCallingFile); 29 | } 30 | 31 | protected logTaskCompletion(uri: string, status: string): void { 32 | console.log(`[CodSpeed] ${status} ${uri}`); 33 | } 34 | 35 | protected finalizeBenchRun(): Task[] { 36 | teardownCore(); 37 | console.log(`[CodSpeed] Done running ${this.bench.tasks.length} benches.`); 38 | return this.bench.tasks; 39 | } 40 | 41 | protected wrapWithInstrumentHooks(fn: () => T, uri: string): T { 42 | InstrumentHooks.startBenchmark(); 43 | const result = fn(); 44 | InstrumentHooks.stopBenchmark(); 45 | InstrumentHooks.setExecutedBenchmark(process.pid, uri); 46 | return result; 47 | } 48 | 49 | protected async wrapWithInstrumentHooksAsync( 50 | fn: Fn, 51 | uri: string 52 | ): Promise { 53 | InstrumentHooks.startBenchmark(); 54 | const result = await fn(); 55 | InstrumentHooks.stopBenchmark(); 56 | InstrumentHooks.setExecutedBenchmark(process.pid, uri); 57 | return result; 58 | } 59 | 60 | protected abstract getModeName(): string; 61 | protected abstract runTaskAsync(task: Task, uri: string): Promise; 62 | protected abstract runTaskSync(task: Task, uri: string): void; 63 | protected abstract finalizeAsyncRun(): Task[]; 64 | protected abstract finalizeSyncRun(): Task[]; 65 | 66 | public setupBenchMethods(): void { 67 | this.bench.run = async () => { 68 | this.setupBenchRun(); 69 | 70 | for (const task of this.bench.tasks) { 71 | const uri = this.getTaskUri(task); 72 | await this.runTaskAsync(task, uri); 73 | } 74 | 75 | return this.finalizeAsyncRun(); 76 | }; 77 | 78 | this.bench.runSync = () => { 79 | this.setupBenchRun(); 80 | 81 | for (const task of this.bench.tasks) { 82 | const uri = this.getTaskUri(task); 83 | this.runTaskSync(task, uri); 84 | } 85 | 86 | return this.finalizeSyncRun(); 87 | }; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/src/uri.ts: -------------------------------------------------------------------------------- 1 | import { Bench } from "tinybench"; 2 | 3 | // Store URI mapping externally since fnOpts is private 4 | export const taskUriMap = new WeakMap>(); 5 | 6 | export function getTaskUri( 7 | bench: Bench, 8 | taskName: string, 9 | rootCallingFile: string 10 | ): string { 11 | const uriMap = taskUriMap.get(bench); 12 | return uriMap?.get(taskName) || `${rootCallingFile}::${taskName}`; 13 | } 14 | 15 | export function getOrCreateUriMap(bench: Bench): Map { 16 | let uriMap = taskUriMap.get(bench); 17 | if (!uriMap) { 18 | uriMap = new Map(); 19 | taskUriMap.set(bench, uriMap); 20 | } 21 | return uriMap; 22 | } 23 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/src/walltime.ts: -------------------------------------------------------------------------------- 1 | import { 2 | calculateQuantiles, 3 | mongoMeasurement, 4 | msToNs, 5 | msToS, 6 | writeWalltimeResults, 7 | type Benchmark as CodspeedBenchmark, 8 | type BenchmarkStats, 9 | } from "@codspeed/core"; 10 | import { Bench, Fn, Task, TaskResult } from "tinybench"; 11 | import { BaseBenchRunner } from "./shared"; 12 | 13 | export function setupCodspeedWalltimeBench( 14 | bench: Bench, 15 | rootCallingFile: string 16 | ): void { 17 | const runner = new WalltimeBenchRunner(bench, rootCallingFile); 18 | runner.setupBenchMethods(); 19 | } 20 | 21 | class WalltimeBenchRunner extends BaseBenchRunner { 22 | private codspeedBenchmarks: CodspeedBenchmark[] = []; 23 | 24 | protected getModeName(): string { 25 | return "walltime mode"; 26 | } 27 | 28 | protected async runTaskAsync(task: Task, uri: string): Promise { 29 | // Override the function under test to add a static frame 30 | this.wrapTaskFunction(task, true); 31 | 32 | // run the warmup of the task right before its actual run 33 | if (this.bench.opts.warmup) { 34 | await task.warmup(); 35 | } 36 | 37 | await mongoMeasurement.start(uri); 38 | await this.wrapWithInstrumentHooksAsync(() => task.run(), uri); 39 | await mongoMeasurement.stop(uri); 40 | 41 | this.registerCodspeedBenchmarkFromTask(task); 42 | } 43 | 44 | protected runTaskSync(task: Task, uri: string): void { 45 | // Override the function under test to add a static frame 46 | this.wrapTaskFunction(task, false); 47 | 48 | if (this.bench.opts.warmup) { 49 | task.warmup(); 50 | } 51 | 52 | this.wrapWithInstrumentHooks(() => task.runSync(), uri); 53 | 54 | this.registerCodspeedBenchmarkFromTask(task); 55 | } 56 | 57 | protected finalizeAsyncRun(): Task[] { 58 | return this.finalizeWalltimeRun(true); 59 | } 60 | 61 | protected finalizeSyncRun(): Task[] { 62 | return this.finalizeWalltimeRun(false); 63 | } 64 | 65 | private wrapTaskFunction(task: Task, isAsync: boolean): void { 66 | const { fn } = task as unknown as { fn: Fn }; 67 | if (isAsync) { 68 | // eslint-disable-next-line no-inner-declarations 69 | async function __codspeed_root_frame__() { 70 | await fn(); 71 | } 72 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 73 | (task as any).fn = __codspeed_root_frame__; 74 | } else { 75 | // eslint-disable-next-line no-inner-declarations 76 | function __codspeed_root_frame__() { 77 | fn(); 78 | } 79 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 80 | (task as any).fn = __codspeed_root_frame__; 81 | } 82 | } 83 | 84 | private registerCodspeedBenchmarkFromTask(task: Task): void { 85 | const uri = this.getTaskUri(task); 86 | 87 | if (!task.result) { 88 | console.warn(` ⚠ No result data available for ${uri}`); 89 | return; 90 | } 91 | 92 | const warmupIterations = this.bench.opts.warmup 93 | ? this.bench.opts.warmupIterations ?? TINYBENCH_WARMUP_DEFAULT 94 | : 0; 95 | const stats = convertTinybenchResultToBenchmarkStats( 96 | task.result, 97 | warmupIterations 98 | ); 99 | 100 | this.codspeedBenchmarks.push({ 101 | name: task.name, 102 | uri, 103 | config: { 104 | max_rounds: this.bench.opts.iterations ?? null, 105 | max_time_ns: this.bench.opts.time ? msToNs(this.bench.opts.time) : null, 106 | min_round_time_ns: null, // tinybench does not have an option for this 107 | warmup_time_ns: 108 | this.bench.opts.warmup && this.bench.opts.warmupTime 109 | ? msToNs(this.bench.opts.warmupTime) 110 | : null, 111 | }, 112 | stats, 113 | }); 114 | 115 | this.logTaskCompletion(uri, "Collected walltime data for"); 116 | } 117 | 118 | private finalizeWalltimeRun(isAsync: boolean): Task[] { 119 | // Write results to JSON file using core function 120 | if (this.codspeedBenchmarks.length > 0) { 121 | writeWalltimeResults(this.codspeedBenchmarks, isAsync); 122 | } 123 | 124 | console.log( 125 | `[CodSpeed] Done collecting walltime data for ${this.bench.tasks.length} benches.` 126 | ); 127 | return this.bench.tasks; 128 | } 129 | } 130 | 131 | const TINYBENCH_WARMUP_DEFAULT = 16; 132 | 133 | function convertTinybenchResultToBenchmarkStats( 134 | result: TaskResult, 135 | warmupIterations: number 136 | ): BenchmarkStats { 137 | const { min, max, mean, sd, samples } = result.latency; 138 | 139 | // Get individual sample times in nanoseconds and sort them 140 | const sortedTimesNs = samples.map(msToNs).sort((a, b) => a - b); 141 | const meanNs = msToNs(mean); 142 | const stdevNs = msToNs(sd); 143 | 144 | const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = 145 | calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); 146 | 147 | return { 148 | min_ns: msToNs(min), 149 | max_ns: msToNs(max), 150 | mean_ns: meanNs, 151 | stdev_ns: stdevNs, 152 | q1_ns, 153 | median_ns, 154 | q3_ns, 155 | total_time: msToS(result.totalTime), 156 | iter_per_round: 1, // as there is only one round in tinybench, we define that there were n rounds of 1 iteration 157 | rounds: sortedTimesNs.length, 158 | iqr_outlier_rounds, 159 | stdev_outlier_rounds, 160 | warmup_iters: warmupIterations, 161 | }; 162 | } 163 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/tests/__snapshots__/index.integ.test.ts.snap: -------------------------------------------------------------------------------- 1 | // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html 2 | 3 | exports[`Benchmark.Suite > check console output(instrumented=%p) false 1`] = ` 4 | { 5 | "log": [ 6 | [ 7 | "[CodSpeed] running with @codspeed/tinybench v1.0.0 (instrumented mode)", 8 | ], 9 | [ 10 | "[CodSpeed] Checked packages/tinybench-plugin/tests/index.integ.test.ts::RegExp", 11 | ], 12 | [ 13 | "[CodSpeed] Checked packages/tinybench-plugin/tests/index.integ.test.ts::RegExp2", 14 | ], 15 | [ 16 | "[CodSpeed] Done running 2 benches.", 17 | ], 18 | ], 19 | "warn": [], 20 | } 21 | `; 22 | 23 | exports[`Benchmark.Suite > check console output(instrumented=%p) true 1`] = ` 24 | { 25 | "log": [ 26 | [ 27 | "[CodSpeed] Measured packages/tinybench-plugin/tests/index.integ.test.ts::RegExp", 28 | ], 29 | [ 30 | "[CodSpeed] Measured packages/tinybench-plugin/tests/index.integ.test.ts::RegExp2", 31 | ], 32 | [ 33 | "[CodSpeed] Done running 2 benches.", 34 | ], 35 | ], 36 | "warn": [], 37 | } 38 | `; 39 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/tests/index.integ.test.ts: -------------------------------------------------------------------------------- 1 | import { Bench } from "tinybench"; 2 | import { beforeEach, describe, expect, it, vi } from "vitest"; 3 | import { withCodSpeed } from "../src"; 4 | import { registerBenchmarks } from "./registerBenchmarks"; 5 | import { registerOtherBenchmarks } from "./registerOtherBenchmarks"; 6 | 7 | const mockCore = vi.hoisted(() => { 8 | return { 9 | mongoMeasurement: { 10 | start: vi.fn(), 11 | stop: vi.fn(), 12 | setupInstruments: vi.fn(), 13 | }, 14 | InstrumentHooks: { 15 | isInstrumented: vi.fn(), 16 | startBenchmark: vi.fn(), 17 | stopBenchmark: vi.fn(), 18 | setExecutedBenchmark: vi.fn(), 19 | }, 20 | optimizeFunction: vi 21 | .fn() 22 | .mockImplementation(async (fn: () => Promise) => { 23 | await fn(); 24 | }), 25 | setupCore: vi.fn(), 26 | teardownCore: vi.fn(), 27 | }; 28 | }); 29 | 30 | vi.mock("@codspeed/core", async (importOriginal) => { 31 | const actual = await importOriginal(); 32 | return { 33 | ...actual, 34 | ...mockCore, 35 | }; 36 | }); 37 | 38 | beforeEach(() => { 39 | process.env.CODSPEED_ENV = "true"; 40 | process.env.CODSPEED_RUNNER_MODE = "instrumentation"; 41 | vi.clearAllMocks(); 42 | }); 43 | 44 | describe("Benchmark.Suite", () => { 45 | it("check core methods are called", async () => { 46 | mockCore.InstrumentHooks.isInstrumented.mockReturnValue(true); 47 | await withCodSpeed(new Bench()) 48 | .add("RegExp", function () { 49 | /o/.test("Hello World!"); 50 | }) 51 | .run(); 52 | 53 | expect(mockCore.mongoMeasurement.start).toHaveBeenCalledWith( 54 | "packages/tinybench-plugin/tests/index.integ.test.ts::RegExp" 55 | ); 56 | expect(mockCore.mongoMeasurement.stop).toHaveBeenCalledTimes(1); 57 | expect(mockCore.InstrumentHooks.startBenchmark).toHaveBeenCalled(); 58 | expect(mockCore.InstrumentHooks.stopBenchmark).toHaveBeenCalled(); 59 | expect(mockCore.InstrumentHooks.setExecutedBenchmark).toHaveBeenCalledWith( 60 | process.pid, 61 | "packages/tinybench-plugin/tests/index.integ.test.ts::RegExp" 62 | ); 63 | }); 64 | it("check suite name is in the uri", async () => { 65 | mockCore.InstrumentHooks.isInstrumented.mockReturnValue(true); 66 | await withCodSpeed(new Bench()) 67 | .add("RegExp", function () { 68 | /o/.test("Hello World!"); 69 | }) 70 | .add("RegExp2", () => { 71 | /o/.test("Hello World!"); 72 | }) 73 | .run(); 74 | 75 | expect(mockCore.mongoMeasurement.start).toHaveBeenCalledWith( 76 | "packages/tinybench-plugin/tests/index.integ.test.ts::RegExp" 77 | ); 78 | expect(mockCore.mongoMeasurement.start).toHaveBeenCalledWith( 79 | "packages/tinybench-plugin/tests/index.integ.test.ts::RegExp2" 80 | ); 81 | expect(mockCore.mongoMeasurement.stop).toHaveBeenCalledTimes(2); 82 | expect(mockCore.InstrumentHooks.startBenchmark).toHaveBeenCalledTimes(2); 83 | expect(mockCore.InstrumentHooks.stopBenchmark).toHaveBeenCalledTimes(2); 84 | expect(mockCore.InstrumentHooks.setExecutedBenchmark).toHaveBeenCalledWith( 85 | process.pid, 86 | "packages/tinybench-plugin/tests/index.integ.test.ts::RegExp" 87 | ); 88 | expect(mockCore.InstrumentHooks.setExecutedBenchmark).toHaveBeenCalledWith( 89 | process.pid, 90 | "packages/tinybench-plugin/tests/index.integ.test.ts::RegExp2" 91 | ); 92 | }); 93 | it("check error handling", async () => { 94 | mockCore.InstrumentHooks.isInstrumented.mockReturnValue(true); 95 | const bench = withCodSpeed(new Bench()); 96 | bench.add("throwing", async () => { 97 | throw new Error("test"); 98 | }); 99 | await expect(bench.run()).rejects.toThrowError("test"); 100 | }); 101 | it.each([true, false])( 102 | "check console output(instrumented=%p) ", 103 | async (instrumented) => { 104 | const logSpy = vi.spyOn(console, "log"); 105 | const warnSpy = vi.spyOn(console, "warn"); 106 | mockCore.InstrumentHooks.isInstrumented.mockReturnValue(instrumented); 107 | await withCodSpeed(new Bench({ time: 100 })) 108 | .add("RegExp", function () { 109 | /o/.test("Hello World!"); 110 | }) 111 | .add("RegExp2", () => { 112 | /o/.test("Hello World!"); 113 | }) 114 | .run(); 115 | // Check that the first log contains "[CodSpeed] running with @codspeed/tinybench v" 116 | if (instrumented) { 117 | expect(logSpy).toHaveBeenCalledWith( 118 | expect.stringContaining( 119 | "[CodSpeed] running with @codspeed/tinybench v" 120 | ) 121 | ); 122 | expect({ 123 | log: logSpy.mock.calls.slice(1), 124 | warn: warnSpy.mock.calls, 125 | }).toMatchSnapshot(); 126 | } else { 127 | expect({ 128 | log: logSpy.mock.calls, 129 | warn: warnSpy.mock.calls, 130 | }).toMatchSnapshot(); 131 | } 132 | } 133 | ); 134 | it("check nested file path is in the uri when bench is registered in another file", async () => { 135 | mockCore.InstrumentHooks.isInstrumented.mockReturnValue(true); 136 | const bench = withCodSpeed(new Bench()); 137 | registerBenchmarks(bench); 138 | await bench.run(); 139 | expect(mockCore.InstrumentHooks.startBenchmark).toHaveBeenCalled(); 140 | expect(mockCore.InstrumentHooks.stopBenchmark).toHaveBeenCalled(); 141 | expect(mockCore.InstrumentHooks.setExecutedBenchmark).toHaveBeenCalledWith( 142 | process.pid, 143 | "packages/tinybench-plugin/tests/registerBenchmarks.ts::RegExp" 144 | ); 145 | }); 146 | // TODO: this is not supported at the moment as tinybench does not support tasks with same name 147 | // remove `.fails` when tinybench supports it 148 | it.fails( 149 | "check that benchmarks with same name have different URIs when registered in different files", 150 | async () => { 151 | mockCore.InstrumentHooks.isInstrumented.mockReturnValue(true); 152 | const bench = withCodSpeed(new Bench()); 153 | registerBenchmarks(bench); 154 | registerOtherBenchmarks(bench); 155 | await bench.run(); 156 | expect(mockCore.InstrumentHooks.startBenchmark).toHaveBeenCalled(); 157 | expect(mockCore.InstrumentHooks.stopBenchmark).toHaveBeenCalled(); 158 | expect( 159 | mockCore.InstrumentHooks.setExecutedBenchmark 160 | ).toHaveBeenCalledWith( 161 | process.pid, 162 | "packages/tinybench-plugin/tests/registerBenchmarks.ts::RegExp" 163 | ); 164 | expect( 165 | mockCore.InstrumentHooks.setExecutedBenchmark 166 | ).toHaveBeenCalledWith( 167 | process.pid, 168 | "packages/tinybench-plugin/tests/registerOtherBenchmarks.ts::RegExp" 169 | ); 170 | } 171 | ); 172 | 173 | it("should run before and after hooks", async () => { 174 | mockCore.InstrumentHooks.isInstrumented.mockReturnValue(true); 175 | mockCore.optimizeFunction.mockImplementation(async (fn) => { 176 | await fn(); 177 | }); 178 | const beforeAll = vi.fn(); 179 | const beforeEach = vi.fn(); 180 | const afterEach = vi.fn(); 181 | const afterAll = vi.fn(); 182 | 183 | await withCodSpeed(new Bench()) 184 | .add( 185 | "RegExp", 186 | function () { 187 | /o/.test("Hello World!"); 188 | }, 189 | { afterAll, afterEach, beforeAll, beforeEach } 190 | ) 191 | .add( 192 | "RegExp2", 193 | () => { 194 | /o/.test("Hello World!"); 195 | }, 196 | { afterAll, afterEach, beforeAll, beforeEach } 197 | ) 198 | .run(); 199 | 200 | // since the optimization is running the benchmark once before the actual run, the each hooks are called twice 201 | expect(beforeEach).toHaveBeenCalledTimes(4); 202 | expect(afterEach).toHaveBeenCalledTimes(4); 203 | 204 | expect(beforeAll).toHaveBeenCalledTimes(2); 205 | expect(afterAll).toHaveBeenCalledTimes(2); 206 | }); 207 | 208 | it("should call setupCore and teardownCore only once", async () => { 209 | mockCore.InstrumentHooks.isInstrumented.mockReturnValue(true); 210 | const bench = withCodSpeed(new Bench()) 211 | .add("RegExp", function () { 212 | /o/.test("Hello World!"); 213 | }) 214 | .add("RegExp2", () => { 215 | /o/.test("Hello World!"); 216 | }); 217 | 218 | expect(mockCore.setupCore).not.toHaveBeenCalled(); 219 | expect(mockCore.teardownCore).not.toHaveBeenCalled(); 220 | 221 | await bench.run(); 222 | 223 | expect(mockCore.setupCore).toHaveBeenCalledTimes(1); 224 | expect(mockCore.teardownCore).toHaveBeenCalledTimes(1); 225 | }); 226 | }); 227 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/tests/registerBenchmarks.ts: -------------------------------------------------------------------------------- 1 | import type { Bench } from "tinybench"; 2 | 3 | export function registerBenchmarks(bench: Bench) { 4 | bench.add("RegExp", function () { 5 | /o/.test("Hello World!"); 6 | }); 7 | } 8 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/tests/registerOtherBenchmarks.ts: -------------------------------------------------------------------------------- 1 | import type { Bench } from "tinybench"; 2 | 3 | export function registerOtherBenchmarks(bench: Bench) { 4 | bench.add("RegExp", function () { 5 | /o/.test("Hello World!"); 6 | }); 7 | } 8 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.base.json", 3 | "compilerOptions": { 4 | "outDir": "dist", 5 | "rootDir": "src", 6 | "typeRoots": ["node_modules/@types", "../../node_modules/@types"] 7 | }, 8 | "references": [{ "path": "../core" }], 9 | "include": ["src/**/*.ts"] 10 | } 11 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "types": ["vitest/globals", "node"] 5 | }, 6 | "include": ["tests/**/*.ts", "benches/**/*.ts", "jest.config.ts"] 7 | } 8 | -------------------------------------------------------------------------------- /packages/tinybench-plugin/vitest.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "vitest/config"; 2 | 3 | export default defineConfig({ 4 | define: { 5 | __VERSION__: JSON.stringify("1.0.0"), 6 | }, 7 | test: { 8 | exclude: ["**/node_modules/**", "**/.rollup.cache/**"], 9 | mockReset: true, 10 | }, 11 | }); 12 | -------------------------------------------------------------------------------- /packages/vitest-plugin/README.md: -------------------------------------------------------------------------------- 1 |
2 |

@codspeed/vitest-plugin

3 | 4 | [Vitest](https://vitest.dev) plugin for [CodSpeed](https://codspeed.io) 5 | 6 | [![CI](https://github.com/CodSpeedHQ/codspeed-node/actions/workflows/ci.yml/badge.svg)](https://github.com/CodSpeedHQ/codspeed-node/actions/workflows/ci.yml) 7 | [![npm (scoped)](https://img.shields.io/npm/v/@codspeed/tinybench-plugin)](https://www.npmjs.com/package/@codspeed/tinybench-plugin) 8 | [![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF) 9 | [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/CodSpeedHQ/codspeed-node) 10 | 11 |
12 | 13 | ## Documentation 14 | 15 | Check out the [documentation](https://docs.codspeed.io/benchmarks/nodejs/vitest) for complete integration instructions. 16 | 17 | ## Installation 18 | 19 | First, install the plugin [`@codspeed/vitest-plugin`](https://www.npmjs.com/package/@codspeed/vitest-plugin) and `vitest` (if not already installed): 20 | 21 | > [!NOTE] 22 | > The CodSpeed plugin is only compatible with 23 | > [vitest@3.2](https://www.npmjs.com/package/vitest/v/3.2.4) 24 | > and above. 25 | 26 | ```sh 27 | npm install --save-dev @codspeed/vitest-plugin vitest 28 | ``` 29 | 30 | or with `yarn`: 31 | 32 | ```sh 33 | yarn add --dev @codspeed/vitest-plugin vitest 34 | ``` 35 | 36 | or with `pnpm`: 37 | 38 | ```sh 39 | pnpm add --save-dev @codspeed/vitest-plugin vitest 40 | ``` 41 | 42 | ## Usage 43 | 44 | Let's create a fibonacci function and benchmark it with `vitest.bench`: 45 | 46 | ```ts title="benches/fibo.bench.ts" 47 | import { describe, bench } from "vitest"; 48 | 49 | function fibonacci(n: number): number { 50 | if (n < 2) { 51 | return n; 52 | } 53 | return fibonacci(n - 1) + fibonacci(n - 2); 54 | } 55 | 56 | describe("fibonacci", () => { 57 | bench("fibonacci10", () => { 58 | fibonacci(10); 59 | }); 60 | 61 | bench("fibonacci15", () => { 62 | fibonacci(15); 63 | }); 64 | }); 65 | ``` 66 | 67 | Create or update your `vitest.config.ts` file to use the CodSpeed runner: 68 | 69 | ```ts title="vitest.config.ts" 70 | import { defineConfig } from "vitest/config"; 71 | import codspeedPlugin from "@codspeed/vitest-plugin"; 72 | 73 | export default defineConfig({ 74 | plugins: [codspeedPlugin()], 75 | // ... 76 | }); 77 | ``` 78 | 79 | Finally, run your benchmarks (here with `pnpm`): 80 | 81 | ```bash 82 | $ pnpm vitest bench --run 83 | [CodSpeed] bench detected but no instrumentation found, falling back to default vitest runner 84 | 85 | ... Regular `vitest bench` output 86 | ``` 87 | 88 | And... Congrats 🎉, CodSpeed is installed in your benchmarking suite! Locally, CodSpeed will fallback to vitest since the instrumentation is only available in the CI environment for now. 89 | 90 | You can now [run those benchmarks in your CI](https://docs.codspeed.io/benchmarks/nodejs/vitest#running-the-benchmarks-in-your-ci) to continuously get consistent performance measurements. 91 | -------------------------------------------------------------------------------- /packages/vitest-plugin/benches/flat.bench.ts: -------------------------------------------------------------------------------- 1 | import { bench, describe } from "vitest"; 2 | import parsePr from "./parsePr"; 3 | 4 | const LONG_BODY = 5 | new Array(1_000) 6 | .fill( 7 | "Lorem ipsum dolor sit amet consectetur adipisicing elit. Sunt, earum. Atque architecto vero veniam est tempora fugiat sint quo praesentium quia. Autem, veritatis omnis beatae iste delectus recusandae animi non." 8 | ) 9 | .join("\n") + "fixes #123"; 10 | 11 | describe("parsePr", () => { 12 | bench("short body", () => { 13 | parsePr({ body: "fixes #123", title: "test-1", number: 1 }); 14 | }); 15 | 16 | bench("long body", () => { 17 | parsePr({ body: LONG_BODY, title: "test-2", number: 2 }); 18 | }); 19 | }); 20 | 21 | function fibo(n: number): number { 22 | if (n < 2) return 1; 23 | return fibo(n - 1) + fibo(n - 2); 24 | } 25 | 26 | describe("fibo", () => { 27 | bench("fibo 10", () => { 28 | fibo(10); 29 | }); 30 | bench("fibo 15", () => { 31 | fibo(15); 32 | }); 33 | }); 34 | -------------------------------------------------------------------------------- /packages/vitest-plugin/benches/hooks.bench.ts: -------------------------------------------------------------------------------- 1 | import { 2 | afterAll, 3 | afterEach, 4 | beforeAll, 5 | beforeEach, 6 | bench, 7 | describe, 8 | expect, 9 | } from "vitest"; 10 | 11 | describe("hooks", () => { 12 | let count = 0; 13 | describe("run", () => { 14 | beforeAll(() => { 15 | count += 10; 16 | }); 17 | beforeEach(() => { 18 | count += 1; 19 | }); 20 | afterEach(() => { 21 | count -= 1; 22 | }); 23 | afterAll(() => { 24 | count -= 10; 25 | }); 26 | 27 | bench("one", () => { 28 | expect(count).toBe(11); 29 | }); 30 | bench("two", () => { 31 | expect(count).toBe(11); 32 | }); 33 | }); 34 | bench("end", () => { 35 | expect(count).toBe(0); 36 | }); 37 | }); 38 | -------------------------------------------------------------------------------- /packages/vitest-plugin/benches/parsePr.bench.ts: -------------------------------------------------------------------------------- 1 | import { bench, describe } from "vitest"; 2 | import parsePr from "./parsePr"; 3 | 4 | const LONG_BODY = 5 | new Array(1_000) 6 | .fill( 7 | "Lorem ipsum dolor sit amet consectetur adipisicing elit. Sunt, earum. Atque architecto vero veniam est tempora fugiat sint quo praesentium quia. Autem, veritatis omnis beatae iste delectus recusandae animi non." 8 | ) 9 | .join("\n") + "fixes #123"; 10 | 11 | describe("parsePr", () => { 12 | bench("short body", () => { 13 | parsePr({ body: "fixes #123", title: "test", number: 124 }); 14 | }); 15 | 16 | bench("long body", () => { 17 | parsePr({ body: LONG_BODY, title: "test", number: 124 }); 18 | }); 19 | 20 | describe("nested suite", () => { 21 | bench("short body", () => { 22 | parsePr({ body: "fixes #123", title: "test", number: 124 }); 23 | }); 24 | 25 | bench("long body", () => { 26 | parsePr({ body: LONG_BODY, title: "test", number: 124 }); 27 | }); 28 | 29 | describe("deeply nested suite", () => { 30 | bench("short body", () => { 31 | parsePr({ body: "fixes #123", title: "test", number: 124 }); 32 | }); 33 | }); 34 | }); 35 | }); 36 | 37 | describe("another parsePr", () => { 38 | bench("short body", () => { 39 | parsePr({ body: "fixes #123", title: "test", number: 124 }); 40 | }); 41 | 42 | bench("long body", () => { 43 | parsePr({ body: LONG_BODY, title: "test", number: 124 }); 44 | }); 45 | 46 | describe("nested suite", () => { 47 | bench("short body", () => { 48 | parsePr({ body: "fixes #123", title: "test", number: 124 }); 49 | }); 50 | }); 51 | }); 52 | -------------------------------------------------------------------------------- /packages/vitest-plugin/benches/parsePr.ts: -------------------------------------------------------------------------------- 1 | interface PullRequest { 2 | number: number; 3 | title: string; 4 | body: string; 5 | } 6 | 7 | function sendEvent(numberOfOperations: number): void { 8 | for (let i = 0; i < numberOfOperations; i++) { 9 | let a = i; 10 | a = a + 1; 11 | } 12 | } 13 | 14 | function logMetrics( 15 | numberOfOperations: number, 16 | numberOfDeepOperations: number 17 | ): void { 18 | for (let i = 0; i < numberOfOperations; i++) { 19 | for (let i = 0; i < numberOfOperations; i++) { 20 | let a = i; 21 | a = a + 1; 22 | a = a + 1; 23 | } 24 | sendEvent(numberOfDeepOperations); 25 | } 26 | } 27 | 28 | function parseTitle(title: string): void { 29 | logMetrics(10, 10); 30 | modifyTitle(title); 31 | } 32 | 33 | function modifyTitle(title: string): void { 34 | for (let i = 0; i < 100; i++) { 35 | let a = i; 36 | a = a + 1 + title.length; 37 | } 38 | } 39 | 40 | function prepareParsingBody(body: string): void { 41 | for (let i = 0; i < 100; i++) { 42 | let a = i; 43 | a = a + 1; 44 | } 45 | parseBody(body); 46 | } 47 | 48 | function parseBody(body: string): void { 49 | logMetrics(10, 10); 50 | for (let i = 0; i < 200; i++) { 51 | let a = i; 52 | a = a + 1; 53 | } 54 | parseIssueFixed(body); 55 | } 56 | 57 | function parseIssueFixed(body: string): number | null { 58 | const prefix = "fixes #"; 59 | const index = body.indexOf(prefix); 60 | if (index === -1) { 61 | return null; 62 | } 63 | 64 | const start = index + prefix.length; 65 | let end = start; 66 | while (end < body.length && /\d/.test(body[end])) { 67 | end += 1; 68 | } 69 | return parseInt(body.slice(start, end)); 70 | } 71 | 72 | export default function parsePr(pullRequest: PullRequest): void { 73 | parseTitle(pullRequest.title); 74 | prepareParsingBody(pullRequest.body); 75 | } 76 | -------------------------------------------------------------------------------- /packages/vitest-plugin/benches/timing.bench.ts: -------------------------------------------------------------------------------- 1 | import { bench, describe, type BenchOptions } from "vitest"; 2 | 3 | const busySleep = (ms: number): void => { 4 | const end = performance.now() + ms; 5 | while (performance.now() < end) { 6 | // Busy wait 7 | } 8 | }; 9 | 10 | const timingBenchOptions: BenchOptions = { 11 | iterations: 5, 12 | warmupIterations: 0, 13 | }; 14 | 15 | describe("timing tests", () => { 16 | bench( 17 | "wait 1ms", 18 | async () => { 19 | busySleep(1); 20 | }, 21 | timingBenchOptions 22 | ); 23 | 24 | bench( 25 | "wait 500ms", 26 | async () => { 27 | busySleep(500); 28 | }, 29 | timingBenchOptions 30 | ); 31 | 32 | bench( 33 | "wait 1sec", 34 | async () => { 35 | busySleep(1_000); 36 | }, 37 | timingBenchOptions 38 | ); 39 | }); 40 | -------------------------------------------------------------------------------- /packages/vitest-plugin/moon.yml: -------------------------------------------------------------------------------- 1 | tasks: 2 | bench: 3 | command: vitest bench --run 4 | inputs: 5 | - "benches/**" 6 | local: true 7 | options: 8 | cache: false 9 | deps: 10 | - build 11 | 12 | test: 13 | command: vitest --run 14 | inputs: 15 | - "./vitest.config.ts" 16 | 17 | test/integ: 18 | command: noop 19 | -------------------------------------------------------------------------------- /packages/vitest-plugin/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@codspeed/vitest-plugin", 3 | "version": "5.0.1", 4 | "description": "vitest plugin for CodSpeed", 5 | "keywords": [ 6 | "codspeed", 7 | "benchmark", 8 | "vitest", 9 | "performance" 10 | ], 11 | "module": "./dist/index.mjs", 12 | "types": "./dist/index.d.ts", 13 | "exports": { 14 | ".": { 15 | "types": "./dist/index.d.ts", 16 | "import": "./dist/index.mjs" 17 | } 18 | }, 19 | "type": "module", 20 | "files": [ 21 | "dist" 22 | ], 23 | "author": "Adrien Cacciaguerra ", 24 | "repository": "https://github.com/CodSpeedHQ/codspeed-node", 25 | "homepage": "https://codspeed.io", 26 | "license": "Apache-2.0", 27 | "scripts": { 28 | "bench": "vitest bench" 29 | }, 30 | "dependencies": { 31 | "@codspeed/core": "workspace:^5.0.1" 32 | }, 33 | "peerDependencies": { 34 | "tinybench": ">=2.9.0", 35 | "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0", 36 | "vitest": ">=3.2" 37 | }, 38 | "devDependencies": { 39 | "@total-typescript/shoehorn": "^0.1.1", 40 | "execa": "^8.0.1", 41 | "tinybench": "^2.9.0", 42 | "vite": "^7.0.0", 43 | "vitest": "^3.2.4" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /packages/vitest-plugin/rollup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "rollup"; 2 | import { declarationsPlugin, jsPlugins } from "../../rollup.options"; 3 | import pkg from "./package.json" assert { type: "json" }; 4 | 5 | export default defineConfig([ 6 | { 7 | input: "src/index.ts", 8 | output: { file: pkg.module, format: "es" }, 9 | plugins: jsPlugins(pkg.version), 10 | external: ["@codspeed/core", /^vitest/], 11 | }, 12 | { 13 | input: "src/index.ts", 14 | output: { file: pkg.types, format: "es" }, 15 | plugins: declarationsPlugin({ compilerOptions: { composite: false } }), 16 | }, 17 | { 18 | input: "src/globalSetup.ts", 19 | output: { file: "dist/globalSetup.mjs", format: "es" }, 20 | plugins: jsPlugins(pkg.version), 21 | external: ["@codspeed/core", /^vitest/], 22 | }, 23 | { 24 | input: "src/instrumented.ts", 25 | output: { file: "dist/instrumented.mjs", format: "es" }, 26 | plugins: jsPlugins(pkg.version), 27 | external: ["@codspeed/core", /^vitest/], 28 | }, 29 | { 30 | input: "src/walltime/index.ts", 31 | output: { file: "dist/walltime.mjs", format: "es" }, 32 | plugins: jsPlugins(pkg.version), 33 | external: ["@codspeed/core", /^vitest/], 34 | }, 35 | ]); 36 | -------------------------------------------------------------------------------- /packages/vitest-plugin/src/__tests__/globalSetup.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, it, vi } from "vitest"; 2 | import globalSetup from "../globalSetup"; 3 | 4 | console.log = vi.fn(); 5 | 6 | describe("globalSetup", () => { 7 | it("should log the correct message on setup and teardown, and fail when teardown is called twice", async () => { 8 | const teardown = globalSetup(); 9 | 10 | expect(console.log).toHaveBeenCalledWith( 11 | "[CodSpeed] @codspeed/vitest-plugin v1.0.0 - setup" 12 | ); 13 | 14 | teardown(); 15 | 16 | expect(console.log).toHaveBeenCalledWith( 17 | "[CodSpeed] @codspeed/vitest-plugin v1.0.0 - teardown" 18 | ); 19 | 20 | expect(() => teardown()).toThrowError("teardown called twice"); 21 | }); 22 | }); 23 | -------------------------------------------------------------------------------- /packages/vitest-plugin/src/__tests__/index.test.ts: -------------------------------------------------------------------------------- 1 | import { fromPartial } from "@total-typescript/shoehorn"; 2 | import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; 3 | import codspeedPlugin from "../index"; 4 | 5 | const coreMocks = vi.hoisted(() => { 6 | return { 7 | InstrumentHooks: { 8 | isInstrumented: vi.fn(), 9 | }, 10 | }; 11 | }); 12 | 13 | const resolvedCodSpeedPlugin = codspeedPlugin(); 14 | const applyPluginFunction = resolvedCodSpeedPlugin.apply; 15 | if (typeof applyPluginFunction !== "function") 16 | throw new Error("applyPluginFunction is not a function"); 17 | 18 | vi.mock("@codspeed/core", async (importOriginal) => { 19 | const mod = await importOriginal(); 20 | return { ...mod, ...coreMocks }; 21 | }); 22 | 23 | console.warn = vi.fn(); 24 | 25 | describe("codSpeedPlugin", () => { 26 | beforeAll(() => { 27 | // Set environment variables to trigger instrumented mode 28 | process.env.CODSPEED_ENV = "1"; 29 | process.env.CODSPEED_RUNNER_MODE = "instrumentation"; 30 | }); 31 | 32 | afterAll(() => { 33 | // Clean up environment variables 34 | delete process.env.CODSPEED_ENV; 35 | delete process.env.CODSPEED_RUNNER_MODE; 36 | }); 37 | 38 | it("should have a name", async () => { 39 | expect(resolvedCodSpeedPlugin.name).toBe("codspeed:vitest"); 40 | }); 41 | 42 | it("should enforce to run after the other plugins", async () => { 43 | expect(resolvedCodSpeedPlugin.enforce).toBe("post"); 44 | }); 45 | 46 | describe("apply", () => { 47 | it("should not apply the plugin when the mode is not benchmark", async () => { 48 | const applyPlugin = applyPluginFunction( 49 | {}, 50 | fromPartial({ mode: "test" }) 51 | ); 52 | 53 | expect(applyPlugin).toBe(false); 54 | }); 55 | 56 | it("should apply the plugin when there is no instrumentation", async () => { 57 | coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(false); 58 | 59 | const applyPlugin = applyPluginFunction( 60 | {}, 61 | fromPartial({ mode: "benchmark" }) 62 | ); 63 | 64 | expect(console.warn).toHaveBeenCalledWith( 65 | "[CodSpeed] bench detected but no instrumentation found" 66 | ); 67 | expect(applyPlugin).toBe(true); 68 | }); 69 | 70 | it("should apply the plugin when there is instrumentation", async () => { 71 | coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(true); 72 | 73 | const applyPlugin = applyPluginFunction( 74 | {}, 75 | fromPartial({ mode: "benchmark" }) 76 | ); 77 | 78 | expect(applyPlugin).toBe(true); 79 | }); 80 | }); 81 | 82 | it("should apply the codspeed config", async () => { 83 | const config = resolvedCodSpeedPlugin.config; 84 | if (typeof config !== "function") 85 | throw new Error("config is not a function"); 86 | 87 | expect(config.call({} as never, {}, fromPartial({}))).toStrictEqual({ 88 | test: { 89 | globalSetup: [ 90 | expect.stringContaining("packages/vitest-plugin/src/globalSetup.ts"), 91 | ], 92 | pool: "forks", 93 | poolOptions: { 94 | forks: { 95 | execArgv: [ 96 | "--interpreted-frames-native-stack", 97 | "--allow-natives-syntax", 98 | "--hash-seed=1", 99 | "--random-seed=1", 100 | "--no-opt", 101 | "--predictable", 102 | "--predictable-gc-schedule", 103 | "--expose-gc", 104 | "--no-concurrent-sweeping", 105 | "--max-old-space-size=4096", 106 | ], 107 | }, 108 | }, 109 | runner: expect.stringContaining( 110 | "packages/vitest-plugin/src/instrumented.ts" 111 | ), 112 | }, 113 | }); 114 | }); 115 | }); 116 | -------------------------------------------------------------------------------- /packages/vitest-plugin/src/__tests__/instrumented.test.ts: -------------------------------------------------------------------------------- 1 | import { fromPartial } from "@total-typescript/shoehorn"; 2 | import { describe, expect, it, vi, type RunnerTestSuite } from "vitest"; 3 | import { getBenchFn } from "vitest/suite"; 4 | import { InstrumentedRunner as CodSpeedRunner } from "../instrumented"; 5 | 6 | const coreMocks = vi.hoisted(() => { 7 | return { 8 | InstrumentHooks: { 9 | startBenchmark: vi.fn(), 10 | stopBenchmark: vi.fn(), 11 | setExecutedBenchmark: vi.fn(), 12 | }, 13 | setupCore: vi.fn(), 14 | teardownCore: vi.fn(), 15 | mongoMeasurement: { 16 | start: vi.fn(), 17 | stop: vi.fn(), 18 | }, 19 | }; 20 | }); 21 | 22 | global.eval = vi.fn(); 23 | 24 | vi.mock("@codspeed/core", async (importOriginal) => { 25 | const mod = await importOriginal(); 26 | return { ...mod, ...coreMocks }; 27 | }); 28 | 29 | console.log = vi.fn(); 30 | 31 | vi.mock("vitest/suite", async (importOriginal) => { 32 | const actual = await importOriginal(); 33 | return { 34 | ...actual, 35 | getBenchFn: vi.fn(), 36 | }; 37 | }); 38 | const mockedGetBenchFn = vi.mocked(getBenchFn); 39 | 40 | describe("CodSpeedRunner", () => { 41 | it("should run the bench function", async () => { 42 | const benchFn = vi.fn(); 43 | mockedGetBenchFn.mockReturnValue(benchFn); 44 | 45 | const runner = new CodSpeedRunner(fromPartial({})); 46 | const suite = fromPartial({ 47 | file: { filepath: __filename }, 48 | name: "test suite", 49 | tasks: [ 50 | { 51 | type: "test", 52 | mode: "run", 53 | meta: { benchmark: true }, 54 | name: "test bench", 55 | }, 56 | ], 57 | }); 58 | await runner.runSuite(suite); 59 | 60 | // setup 61 | expect(coreMocks.setupCore).toHaveBeenCalledTimes(1); 62 | expect(console.log).toHaveBeenCalledWith( 63 | "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/instrumented.test.ts" 64 | ); 65 | 66 | // run 67 | expect(coreMocks.mongoMeasurement.start).toHaveBeenCalledWith( 68 | "packages/vitest-plugin/src/__tests__/instrumented.test.ts::test bench" 69 | ); 70 | expect(coreMocks.InstrumentHooks.startBenchmark).toHaveBeenCalledTimes(1); 71 | expect(benchFn).toHaveBeenCalledTimes(8); 72 | expect(coreMocks.InstrumentHooks.stopBenchmark).toHaveBeenCalledTimes(1); 73 | expect(coreMocks.mongoMeasurement.stop).toHaveBeenCalledTimes(1); 74 | expect(console.log).toHaveBeenCalledWith( 75 | "[CodSpeed] packages/vitest-plugin/src/__tests__/instrumented.test.ts::test bench done" 76 | ); 77 | 78 | // teardown 79 | expect(console.log).toHaveBeenCalledWith( 80 | "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/instrumented.test.ts done" 81 | ); 82 | expect(coreMocks.teardownCore).toHaveBeenCalledTimes(1); 83 | }); 84 | 85 | it("should run nested suites", async () => { 86 | const benchFn = vi.fn(); 87 | mockedGetBenchFn.mockReturnValue(benchFn); 88 | 89 | const runner = new CodSpeedRunner(fromPartial({})); 90 | const rootSuite = fromPartial({ 91 | file: { filepath: __filename }, 92 | name: "test suite", 93 | tasks: [ 94 | { 95 | type: "suite", 96 | name: "nested suite", 97 | mode: "run", 98 | tasks: [ 99 | { 100 | type: "test", 101 | mode: "run", 102 | meta: { benchmark: true }, 103 | name: "test bench", 104 | }, 105 | ], 106 | }, 107 | ], 108 | }); 109 | 110 | await runner.runSuite(rootSuite); 111 | 112 | // setup 113 | expect(coreMocks.setupCore).toHaveBeenCalledTimes(1); 114 | expect(console.log).toHaveBeenCalledWith( 115 | "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/instrumented.test.ts" 116 | ); 117 | 118 | // run 119 | expect(coreMocks.mongoMeasurement.start).toHaveBeenCalledWith( 120 | "packages/vitest-plugin/src/__tests__/instrumented.test.ts::nested suite::test bench" 121 | ); 122 | expect(coreMocks.InstrumentHooks.startBenchmark).toHaveBeenCalledTimes(1); 123 | expect(benchFn).toHaveBeenCalledTimes(8); 124 | expect(coreMocks.InstrumentHooks.stopBenchmark).toHaveBeenCalledTimes(1); 125 | expect(coreMocks.mongoMeasurement.stop).toHaveBeenCalledTimes(1); 126 | expect(console.log).toHaveBeenCalledWith( 127 | "[CodSpeed] packages/vitest-plugin/src/__tests__/instrumented.test.ts::nested suite::test bench done" 128 | ); 129 | 130 | // teardown 131 | expect(console.log).toHaveBeenCalledWith( 132 | "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/instrumented.test.ts done" 133 | ); 134 | expect(coreMocks.teardownCore).toHaveBeenCalledTimes(1); 135 | }); 136 | }); 137 | -------------------------------------------------------------------------------- /packages/vitest-plugin/src/common.ts: -------------------------------------------------------------------------------- 1 | import { getGitDir } from "@codspeed/core"; 2 | import path from "path"; 3 | import { Benchmark, type RunnerTask, type RunnerTestSuite } from "vitest"; 4 | import { getHooks } from "vitest/suite"; 5 | type SuiteHooks = ReturnType; 6 | 7 | function getSuiteHooks(suite: RunnerTestSuite, name: keyof SuiteHooks) { 8 | return getHooks(suite)?.[name] ?? []; 9 | } 10 | 11 | export async function callSuiteHook( 12 | suite: RunnerTestSuite, 13 | currentTask: RunnerTask, 14 | name: T 15 | ): Promise { 16 | if (name === "beforeEach" && suite?.suite) { 17 | await callSuiteHook(suite.suite, currentTask, name); 18 | } 19 | 20 | const hooks = getSuiteHooks(suite, name); 21 | 22 | // @ts-expect-error TODO: add support for hooks parameters 23 | await Promise.all(hooks.map((fn) => fn())); 24 | 25 | if (name === "afterEach" && suite?.suite) { 26 | await callSuiteHook(suite.suite, currentTask, name); 27 | } 28 | } 29 | 30 | export function patchRootSuiteWithFullFilePath(suite: RunnerTestSuite) { 31 | const gitDir = getGitDir(suite.file.filepath); 32 | if (gitDir === undefined) { 33 | throw new Error("Could not find a git repository"); 34 | } 35 | suite.name = path.relative(gitDir, suite.file.filepath); 36 | } 37 | 38 | export function isVitestTaskBenchmark(task: RunnerTask): task is Benchmark { 39 | return task.type === "test" && task.meta.benchmark === true; 40 | } 41 | -------------------------------------------------------------------------------- /packages/vitest-plugin/src/globalSetup.ts: -------------------------------------------------------------------------------- 1 | declare const __VERSION__: string; 2 | 3 | /** 4 | * @deprecated 5 | * TODO: try to use something like `updateTask` from `@vitest/runner` instead to use the output 6 | * of vitest instead console.log but at the moment, `updateTask` is not exposed 7 | */ 8 | function logCodSpeed(message: string) { 9 | console.log(`[CodSpeed] ${message}`); 10 | } 11 | 12 | let teardownHappened = false; 13 | 14 | export default function () { 15 | logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - setup`); 16 | 17 | return () => { 18 | if (teardownHappened) throw new Error("teardown called twice"); 19 | teardownHappened = true; 20 | 21 | logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - teardown`); 22 | }; 23 | } 24 | -------------------------------------------------------------------------------- /packages/vitest-plugin/src/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | getCodspeedRunnerMode, 3 | getV8Flags, 4 | InstrumentHooks, 5 | mongoMeasurement, 6 | SetupInstrumentsRequestBody, 7 | SetupInstrumentsResponse, 8 | } from "@codspeed/core"; 9 | import { join } from "path"; 10 | import { Plugin } from "vite"; 11 | import { type ViteUserConfig } from "vitest/config"; 12 | 13 | // get this file's directory path from import.meta.url 14 | const __dirname = new URL(".", import.meta.url).pathname; 15 | const isFileInTs = import.meta.url.endsWith(".ts"); 16 | 17 | function getCodSpeedFileFromName(name: string) { 18 | const fileExtension = isFileInTs ? "ts" : "mjs"; 19 | 20 | return join(__dirname, `${name}.${fileExtension}`); 21 | } 22 | 23 | function getRunnerFile(): string | undefined { 24 | const codspeedRunnerMode = getCodspeedRunnerMode(); 25 | if (codspeedRunnerMode === "disabled") { 26 | return undefined; 27 | } 28 | 29 | return getCodSpeedFileFromName(codspeedRunnerMode); 30 | } 31 | 32 | export default function codspeedPlugin(): Plugin { 33 | return { 34 | name: "codspeed:vitest", 35 | apply(_, { mode }) { 36 | if (mode !== "benchmark") { 37 | return false; 38 | } 39 | if ( 40 | getCodspeedRunnerMode() == "instrumented" && 41 | !InstrumentHooks.isInstrumented() 42 | ) { 43 | console.warn("[CodSpeed] bench detected but no instrumentation found"); 44 | } 45 | return true; 46 | }, 47 | enforce: "post", 48 | config(): ViteUserConfig { 49 | const runnerFile = getRunnerFile(); 50 | const runnerMode = getCodspeedRunnerMode(); 51 | 52 | const config: ViteUserConfig = { 53 | test: { 54 | pool: "forks", 55 | poolOptions: { 56 | forks: { 57 | execArgv: getV8Flags(), 58 | }, 59 | }, 60 | globalSetup: [getCodSpeedFileFromName("globalSetup")], 61 | ...(runnerFile && { 62 | runner: runnerFile, 63 | }), 64 | ...(runnerMode === "walltime" && { 65 | benchmark: { 66 | includeSamples: true, 67 | }, 68 | }), 69 | }, 70 | }; 71 | 72 | return config; 73 | }, 74 | }; 75 | } 76 | 77 | /** 78 | * Dynamically setup the CodSpeed instruments. 79 | */ 80 | export async function setupInstruments( 81 | body: SetupInstrumentsRequestBody 82 | ): Promise { 83 | if (!InstrumentHooks.isInstrumented()) { 84 | console.warn("[CodSpeed] No instrumentation found, using default mongoUrl"); 85 | 86 | return { remoteAddr: body.mongoUrl }; 87 | } 88 | 89 | return await mongoMeasurement.setupInstruments(body); 90 | } 91 | -------------------------------------------------------------------------------- /packages/vitest-plugin/src/instrumented.ts: -------------------------------------------------------------------------------- 1 | import { 2 | InstrumentHooks, 3 | logDebug, 4 | mongoMeasurement, 5 | optimizeFunction, 6 | setupCore, 7 | teardownCore, 8 | } from "@codspeed/core"; 9 | import { Benchmark, type RunnerTestSuite } from "vitest"; 10 | import { NodeBenchmarkRunner } from "vitest/runners"; 11 | import { getBenchFn } from "vitest/suite"; 12 | import { 13 | callSuiteHook, 14 | isVitestTaskBenchmark, 15 | patchRootSuiteWithFullFilePath, 16 | } from "./common"; 17 | 18 | const currentFileName = 19 | typeof __filename === "string" 20 | ? __filename 21 | : new URL("instrumented.mjs", import.meta.url).pathname; 22 | 23 | /** 24 | * @deprecated 25 | * TODO: try to use something like `updateTask` from `@vitest/runner` instead to use the output 26 | * of vitest instead console.log but at the moment, `updateTask` is not exposed 27 | */ 28 | function logCodSpeed(message: string) { 29 | console.log(`[CodSpeed] ${message}`); 30 | } 31 | 32 | async function runInstrumentedBench( 33 | benchmark: Benchmark, 34 | suite: RunnerTestSuite, 35 | currentSuiteName: string 36 | ) { 37 | const uri = `${currentSuiteName}::${benchmark.name}`; 38 | const fn = getBenchFn(benchmark); 39 | 40 | await optimizeFunction(async () => { 41 | await callSuiteHook(suite, benchmark, "beforeEach"); 42 | // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench 43 | await fn(); 44 | await callSuiteHook(suite, benchmark, "afterEach"); 45 | }); 46 | 47 | await callSuiteHook(suite, benchmark, "beforeEach"); 48 | await mongoMeasurement.start(uri); 49 | global.gc?.(); 50 | await (async function __codspeed_root_frame__() { 51 | InstrumentHooks.startBenchmark(); 52 | // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench 53 | await fn(); 54 | InstrumentHooks.stopBenchmark(); 55 | InstrumentHooks.setExecutedBenchmark(process.pid, uri); 56 | })(); 57 | await mongoMeasurement.stop(uri); 58 | await callSuiteHook(suite, benchmark, "afterEach"); 59 | 60 | logCodSpeed(`${uri} done`); 61 | } 62 | 63 | async function runInstrumentedBenchmarkSuite( 64 | suite: RunnerTestSuite, 65 | parentSuiteName?: string 66 | ) { 67 | const currentSuiteName = parentSuiteName 68 | ? parentSuiteName + "::" + suite.name 69 | : suite.name; 70 | 71 | await callSuiteHook(suite, suite, "beforeAll"); 72 | 73 | for (const task of suite.tasks) { 74 | if (task.mode !== "run") continue; 75 | 76 | if (isVitestTaskBenchmark(task)) { 77 | await runInstrumentedBench(task, suite, currentSuiteName); 78 | } else if (task.type === "suite") { 79 | await runInstrumentedBenchmarkSuite(task, currentSuiteName); 80 | } 81 | } 82 | 83 | await callSuiteHook(suite, suite, "afterAll"); 84 | } 85 | 86 | export class InstrumentedRunner extends NodeBenchmarkRunner { 87 | async runSuite(suite: RunnerTestSuite): Promise { 88 | logDebug(`PROCESS PID: ${process.pid} in ${currentFileName}`); 89 | setupCore(); 90 | 91 | patchRootSuiteWithFullFilePath(suite); 92 | 93 | logCodSpeed(`running suite ${suite.name}`); 94 | await runInstrumentedBenchmarkSuite(suite); 95 | logCodSpeed(`running suite ${suite.name} done`); 96 | 97 | teardownCore(); 98 | } 99 | } 100 | 101 | export default InstrumentedRunner; 102 | -------------------------------------------------------------------------------- /packages/vitest-plugin/src/runner.ts: -------------------------------------------------------------------------------- 1 | import { InstrumentedRunner } from "./instrumented"; 2 | 3 | export default InstrumentedRunner; 4 | -------------------------------------------------------------------------------- /packages/vitest-plugin/src/walltime/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | InstrumentHooks, 3 | setupCore, 4 | writeWalltimeResults, 5 | } from "@codspeed/core"; 6 | import { Fn } from "tinybench"; 7 | import { 8 | RunnerTaskEventPack, 9 | RunnerTaskResultPack, 10 | type RunnerTestSuite, 11 | } from "vitest"; 12 | import { NodeBenchmarkRunner } from "vitest/runners"; 13 | import { patchRootSuiteWithFullFilePath } from "../common"; 14 | import { extractBenchmarkResults } from "./utils"; 15 | 16 | /** 17 | * WalltimeRunner uses Vitest's default benchmark execution 18 | * and extracts results from the suite after completion 19 | */ 20 | export class WalltimeRunner extends NodeBenchmarkRunner { 21 | private isTinybenchHookedWithCodspeed = false; 22 | private suiteUris = new Map(); 23 | /// Suite ID of the currently running suite, to allow constructing the URI in the context of tinybench tasks 24 | private currentSuiteId: string | null = null; 25 | 26 | async runSuite(suite: RunnerTestSuite): Promise { 27 | patchRootSuiteWithFullFilePath(suite); 28 | this.populateBenchmarkUris(suite); 29 | 30 | setupCore(); 31 | 32 | await super.runSuite(suite); 33 | 34 | const benchmarks = await extractBenchmarkResults(suite); 35 | 36 | if (benchmarks.length > 0) { 37 | writeWalltimeResults(benchmarks); 38 | console.log( 39 | `[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.` 40 | ); 41 | } else { 42 | console.warn( 43 | `[CodSpeed] No benchmark results found after suite execution` 44 | ); 45 | } 46 | } 47 | 48 | private populateBenchmarkUris(suite: RunnerTestSuite, parentPath = ""): void { 49 | const currentPath = 50 | parentPath !== "" ? `${parentPath}::${suite.name}` : suite.name; 51 | 52 | for (const task of suite.tasks) { 53 | if (task.type === "suite") { 54 | this.suiteUris.set(task.id, `${currentPath}::${task.name}`); 55 | this.populateBenchmarkUris(task, currentPath); 56 | } 57 | } 58 | } 59 | 60 | async importTinybench(): Promise { 61 | const tinybench = await super.importTinybench(); 62 | 63 | if (this.isTinybenchHookedWithCodspeed) { 64 | return tinybench; 65 | } 66 | this.isTinybenchHookedWithCodspeed = true; 67 | 68 | const originalRun = tinybench.Task.prototype.run; 69 | 70 | const getSuiteUri = (): string => { 71 | if (this.currentSuiteId === null) { 72 | throw new Error("currentSuiteId is null - something went wrong"); 73 | } 74 | return this.suiteUris.get(this.currentSuiteId) || ""; 75 | }; 76 | 77 | tinybench.Task.prototype.run = async function () { 78 | const { fn } = this as { fn: Fn }; 79 | const suiteUri = getSuiteUri(); 80 | 81 | function __codspeed_root_frame__() { 82 | return fn(); 83 | } 84 | (this as { fn: Fn }).fn = __codspeed_root_frame__; 85 | 86 | InstrumentHooks.startBenchmark(); 87 | await originalRun.call(this); 88 | InstrumentHooks.stopBenchmark(); 89 | 90 | // Look up the URI by task name 91 | const uri = `${suiteUri}::${this.name}`; 92 | InstrumentHooks.setExecutedBenchmark(process.pid, uri); 93 | 94 | return this; 95 | }; 96 | 97 | return tinybench; 98 | } 99 | 100 | // Allow tinybench to retrieve the path to the currently running suite 101 | async onTaskUpdate( 102 | _: RunnerTaskResultPack[], 103 | events: RunnerTaskEventPack[] 104 | ): Promise { 105 | events.map((event) => { 106 | const [id, eventName] = event; 107 | 108 | if (eventName === "suite-prepare") { 109 | this.currentSuiteId = id; 110 | } 111 | }); 112 | } 113 | } 114 | 115 | export default WalltimeRunner; 116 | -------------------------------------------------------------------------------- /packages/vitest-plugin/src/walltime/utils.ts: -------------------------------------------------------------------------------- 1 | import { 2 | calculateQuantiles, 3 | msToNs, 4 | msToS, 5 | type Benchmark, 6 | type BenchmarkStats, 7 | } from "@codspeed/core"; 8 | import { 9 | type Benchmark as VitestBenchmark, 10 | type RunnerTaskResult, 11 | type RunnerTestSuite, 12 | } from "vitest"; 13 | import { getBenchOptions } from "vitest/suite"; 14 | import { isVitestTaskBenchmark } from "../common"; 15 | 16 | export async function extractBenchmarkResults( 17 | suite: RunnerTestSuite, 18 | parentPath = "" 19 | ): Promise { 20 | const benchmarks: Benchmark[] = []; 21 | const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; 22 | 23 | for (const task of suite.tasks) { 24 | if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { 25 | const benchmark = await processBenchmarkTask(task, currentPath); 26 | if (benchmark) { 27 | benchmarks.push(benchmark); 28 | } 29 | } else if (task.type === "suite") { 30 | const nestedBenchmarks = await extractBenchmarkResults(task, currentPath); 31 | benchmarks.push(...nestedBenchmarks); 32 | } 33 | } 34 | 35 | return benchmarks; 36 | } 37 | 38 | async function processBenchmarkTask( 39 | task: VitestBenchmark, 40 | suitePath: string 41 | ): Promise { 42 | const uri = `${suitePath}::${task.name}`; 43 | 44 | const result = task.result; 45 | if (!result) { 46 | console.warn(` ⚠ No result data available for ${uri}`); 47 | return null; 48 | } 49 | 50 | try { 51 | // Get tinybench configuration options from vitest 52 | const benchOptions = getBenchOptions(task); 53 | 54 | const stats = convertVitestResultToBenchmarkStats(result, benchOptions); 55 | 56 | if (stats === null) { 57 | console.log(` ✔ No walltime data to collect for ${uri}`); 58 | return null; 59 | } 60 | 61 | const coreBenchmark: Benchmark = { 62 | name: task.name, 63 | uri, 64 | config: { 65 | max_rounds: benchOptions.iterations ?? null, 66 | max_time_ns: benchOptions.time ? msToNs(benchOptions.time) : null, 67 | min_round_time_ns: null, // tinybench does not have an option for this 68 | warmup_time_ns: 69 | benchOptions.warmupIterations !== 0 && benchOptions.warmupTime 70 | ? msToNs(benchOptions.warmupTime) 71 | : null, 72 | }, 73 | stats, 74 | }; 75 | 76 | console.log(` ✔ Collected walltime data for ${uri}`); 77 | return coreBenchmark; 78 | } catch (error) { 79 | console.warn(` ⚠ Failed to process benchmark result for ${uri}:`, error); 80 | return null; 81 | } 82 | } 83 | 84 | function convertVitestResultToBenchmarkStats( 85 | result: RunnerTaskResult, 86 | benchOptions: { 87 | time?: number; 88 | warmupTime?: number; 89 | warmupIterations?: number; 90 | iterations?: number; 91 | } 92 | ): BenchmarkStats | null { 93 | const benchmark = result.benchmark; 94 | 95 | if (!benchmark) { 96 | throw new Error("No benchmark data available in result"); 97 | } 98 | 99 | const { totalTime, min, max, mean, sd, samples } = benchmark; 100 | 101 | // Get individual sample times in nanoseconds and sort them 102 | const sortedTimesNs = samples.map(msToNs).sort((a, b) => a - b); 103 | const meanNs = msToNs(mean); 104 | const stdevNs = msToNs(sd); 105 | 106 | if (sortedTimesNs.length == 0) { 107 | // Sometimes the benchmarks can be completely optimized out and not even run, but its beforeEach and afterEach hooks are still executed, and the task is still considered a success. 108 | // This is the case for the hooks.bench.ts example in this package 109 | return null; 110 | } 111 | 112 | const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = 113 | calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); 114 | 115 | return { 116 | min_ns: msToNs(min), 117 | max_ns: msToNs(max), 118 | mean_ns: meanNs, 119 | stdev_ns: stdevNs, 120 | q1_ns, 121 | median_ns, 122 | q3_ns, 123 | total_time: msToS(totalTime), 124 | iter_per_round: 1, // as there is only one round in tinybench, we define that there were n rounds of 1 iteration 125 | rounds: sortedTimesNs.length, 126 | iqr_outlier_rounds, 127 | stdev_outlier_rounds, 128 | warmup_iters: benchOptions.warmupIterations ?? 0, 129 | }; 130 | } 131 | -------------------------------------------------------------------------------- /packages/vitest-plugin/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.base.json", 3 | "compilerOptions": { 4 | "outDir": "dist", 5 | "rootDir": "src", 6 | "typeRoots": ["node_modules/@types", "../../node_modules/@types"] 7 | }, 8 | "references": [{ "path": "../core" }], 9 | "include": ["src/**/*.ts"] 10 | } 11 | -------------------------------------------------------------------------------- /packages/vitest-plugin/tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "types": ["jest", "node"] 5 | }, 6 | "include": ["tests/**/*.ts", "benches/**/*.ts", "jest.config.ts"] 7 | } 8 | -------------------------------------------------------------------------------- /packages/vitest-plugin/vitest.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "vitest/config"; 2 | import codspeedPlugin from "./dist/index.mjs"; 3 | 4 | export default defineConfig({ 5 | // @ts-expect-error - TODO: investigate why importing from '.' wants to import only "main" field and thus fail 6 | plugins: [codspeedPlugin()], 7 | define: { 8 | __VERSION__: JSON.stringify("1.0.0"), 9 | }, 10 | test: { 11 | exclude: ["**/tests/**/*", "**/.rollup.cache/**/*"], 12 | mockReset: true, 13 | }, 14 | }); 15 | -------------------------------------------------------------------------------- /pnpm-workspace.yaml: -------------------------------------------------------------------------------- 1 | packages: 2 | - examples/* 3 | - packages/* 4 | 5 | onlyBuiltDependencies: 6 | - '@moonrepo/cli' 7 | -------------------------------------------------------------------------------- /rollup.options.js: -------------------------------------------------------------------------------- 1 | import commonjs from "@rollup/plugin-commonjs"; 2 | import json from "@rollup/plugin-json"; 3 | import resolve from "@rollup/plugin-node-resolve"; 4 | import dts from "rollup-plugin-dts"; 5 | import esbuild from "rollup-plugin-esbuild"; 6 | 7 | /** 8 | * @typedef {import('rollup-plugin-dts').Options} DtsOptions 9 | */ 10 | 11 | /** 12 | * @param {DtsOptions} options 13 | */ 14 | export const declarationsPlugin = (options) => [dts(options)]; 15 | 16 | export const jsPlugins = (version) => [ 17 | json(), 18 | esbuild({ 19 | define: { 20 | __VERSION__: '"' + version + '"', 21 | }, 22 | }), 23 | commonjs(), 24 | resolve(), 25 | ]; 26 | -------------------------------------------------------------------------------- /scripts/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Usage: ./scripts/release.sh 3 | set -ex 4 | 5 | # Fail if not on main 6 | if [ "$(git rev-parse --abbrev-ref HEAD)" != "main" ]; then 7 | echo "Not on default branch" 8 | exit 1 9 | fi 10 | 11 | if [ $# -ne 1 ]; then 12 | echo "Usage: ./release.sh " 13 | exit 1 14 | fi 15 | 16 | # Fail if there are any unstaged changes left 17 | git diff --exit-code 18 | 19 | pnpm lerna version $1 --force-publish --no-private 20 | -------------------------------------------------------------------------------- /tsconfig.base.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "composite": true, 4 | "moduleResolution": "node", 5 | "target": "ESNext", 6 | "module": "ESNext", 7 | "lib": ["esnext"], 8 | "types": ["node"], 9 | "strict": true, 10 | "allowSyntheticDefaultImports": true, 11 | "experimentalDecorators": true, 12 | "sourceMap": true, 13 | "emitDecoratorMetadata": true, 14 | "skipLibCheck": true, 15 | "forceConsistentCasingInFileNames": true, 16 | "esModuleInterop": true, 17 | "resolveJsonModule": true, 18 | "isolatedModules": true 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [], 3 | "include": [], 4 | "exclude": ["**/node_modules"], 5 | "references": [ 6 | { 7 | "path": "packages/benchmark.js-plugin" 8 | }, 9 | { 10 | "path": "packages/tinybench-plugin" 11 | }, 12 | { 13 | "path": "packages/vitest-plugin" 14 | }, 15 | { 16 | "path": "packages/core" 17 | } 18 | ] 19 | } 20 | --------------------------------------------------------------------------------