├── .dockerignore ├── tests ├── data │ ├── pack-json.sh │ ├── large-payload.json.bz2 │ ├── expected-results │ │ ├── charts │ │ │ ├── jssom.png │ │ │ ├── jssom-som.png │ │ │ ├── trufflesom.png │ │ │ ├── trufflesom-macro-startup.png │ │ │ ├── trufflesom-macro-steady.png │ │ │ ├── trufflesom-micro-somsom.png │ │ │ ├── trufflesom-micro-startup.png │ │ │ └── trufflesom-micro-steady.png │ │ ├── stats-data-prep │ │ │ ├── tsom │ │ │ │ └── overview.png │ │ │ └── jssom │ │ │ │ └── overview.png │ │ ├── compare-view │ │ │ ├── navigation-jssom.html │ │ │ ├── stats-row-version-missing.html │ │ │ ├── stats-row-across-version.html │ │ │ ├── stats-tbl-header.html │ │ │ ├── stats-row-button-info.html │ │ │ ├── stats-summary.html │ │ │ ├── stats-row-across-exes.html │ │ │ ├── stats-row-version-one-criteria-missing.html │ │ │ ├── stats-row-version.html │ │ │ ├── stats-row-exe.html │ │ │ ├── stats-tbl.html │ │ │ ├── navigation-tsom.html │ │ │ └── compare-versions.html │ │ ├── project │ │ │ ├── get-exp-data.html │ │ │ └── project-data.html │ │ └── main │ │ │ └── index.html │ └── small-payload.json ├── rebench-integration │ ├── test-vm.py │ ├── rebench.conf │ └── check-data.js ├── shared │ ├── aesthetics.test.ts │ ├── single-requester.test.ts │ ├── fast-or-precise.ts │ └── ui.test.ts ├── backend │ ├── compare │ │ └── db-measurements.test.ts │ ├── common │ │ └── standard-responses.test.ts │ ├── project │ │ └── project.test.ts │ ├── db │ │ └── cache-validity.test.ts │ ├── perf-tracker.test.ts │ └── rebench │ │ └── api.test.ts ├── payload.ts └── views │ └── helpers.test.ts ├── .prettierrc ├── src ├── views │ ├── common-menu.html │ ├── header.html │ └── theme-switcher-btn.html ├── backend │ ├── db │ │ ├── schema-updates │ │ │ ├── migration.013.sql │ │ │ ├── migration.009.sql │ │ │ ├── migration.007.sql │ │ │ ├── migration.008.sql │ │ │ ├── migration.005.sql │ │ │ ├── migration.006.sql │ │ │ ├── cleanup-zeros.sql │ │ │ ├── migration.010.sql │ │ │ ├── migration.003.sql │ │ │ ├── migration.002.sql │ │ │ ├── migration.001.sql │ │ │ ├── migration.011.sql │ │ │ └── migration.012.sql │ │ ├── timed-cache-validity.ts │ │ ├── database-with-pool.ts │ │ └── has-profile.ts │ ├── admin │ │ └── operations.ts │ ├── request-check.ts │ ├── compare │ │ └── html │ │ │ ├── navigation.html │ │ │ ├── compare-exes.html │ │ │ ├── compare-versions.html │ │ │ ├── stats-tbl-header.html │ │ │ ├── stats-summary.html │ │ │ ├── stats-tbl.html │ │ │ ├── stats-row-across-versions.html │ │ │ ├── stats-row-buttons-info.html │ │ │ ├── stats-row-across-exes.html │ │ │ ├── refresh-menu.html │ │ │ ├── stats-row.html │ │ │ └── gen-index.html │ ├── rebench │ │ ├── api-validator.ts │ │ └── results.ts │ ├── templates.ts │ ├── logging.ts │ ├── common │ │ ├── standard-responses.ts │ │ └── api-v1.ts │ ├── timeline │ │ ├── timeline-calc-worker.ts │ │ ├── timeline.ts │ │ └── timeline.html │ ├── project │ │ ├── get-exp-data.html │ │ ├── project-data.html │ │ ├── project.html │ │ ├── project.ts │ │ └── data-export.ts │ ├── dev-server │ │ └── server.ts │ ├── util.ts │ ├── perf-tracker.ts │ └── main │ │ └── index.html ├── shared │ ├── util.ts │ ├── errors.ts │ ├── single-requester.ts │ ├── helpers.ts │ ├── aesthetics.ts │ └── data-format.ts ├── download.ts ├── frontend │ ├── project.ts │ ├── project-data.ts │ ├── index.ts │ ├── timeline.ts │ ├── filter.ts │ └── theme.ts ├── vendored │ └── chartjs-node-canvas │ │ └── src │ │ └── backgroundColourPlugin.ts └── benchmarks │ ├── rebenchdb-benchmark.ts │ ├── benchmark.ts │ ├── store-results.ts │ ├── fetch-results.ts │ ├── render-report.ts │ └── compute-timeline.ts ├── Dockerfile.rebench ├── .github └── workflows │ ├── container.yml │ └── ci.yml ├── docker-compose.yml ├── .gitlab-ci.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── patches └── @sgratzl+chartjs-chart-boxplot+4.4.5.patch ├── eslint.config.mjs ├── rebench.conf ├── README.md ├── package.json └── tsconfig.json /.dockerignore: -------------------------------------------------------------------------------- 1 | **/node_modules 2 | /dist 3 | /resources 4 | 5 | -------------------------------------------------------------------------------- /tests/data/pack-json.sh: -------------------------------------------------------------------------------- 1 | rm -f large-payload.json.bz2 2 | bzip2 -9 -k large-payload.json 3 | -------------------------------------------------------------------------------- /tests/data/large-payload.json.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smarr/ReBenchDB/HEAD/tests/data/large-payload.json.bz2 -------------------------------------------------------------------------------- /tests/data/expected-results/charts/jssom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smarr/ReBenchDB/HEAD/tests/data/expected-results/charts/jssom.png -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": true, 3 | "trailingComma": "none", 4 | "singleQuote": true, 5 | "printWidth": 80, 6 | "tabWidth": 2 7 | } 8 | -------------------------------------------------------------------------------- /tests/data/expected-results/charts/jssom-som.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smarr/ReBenchDB/HEAD/tests/data/expected-results/charts/jssom-som.png -------------------------------------------------------------------------------- /tests/data/expected-results/charts/trufflesom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smarr/ReBenchDB/HEAD/tests/data/expected-results/charts/trufflesom.png -------------------------------------------------------------------------------- /src/views/common-menu.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | {%- include('theme-switcher-btn.html') %} 4 |
5 |
-------------------------------------------------------------------------------- /src/backend/db/schema-updates/migration.013.sql: -------------------------------------------------------------------------------- 1 | -- Remove the Unit table 2 | ALTER TABLE Criterion DROP CONSTRAINT IF EXISTS criterion_unit_fkey; 3 | DROP TABLE Unit; 4 | -------------------------------------------------------------------------------- /tests/data/expected-results/stats-data-prep/tsom/overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smarr/ReBenchDB/HEAD/tests/data/expected-results/stats-data-prep/tsom/overview.png -------------------------------------------------------------------------------- /tests/data/expected-results/charts/trufflesom-macro-startup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smarr/ReBenchDB/HEAD/tests/data/expected-results/charts/trufflesom-macro-startup.png -------------------------------------------------------------------------------- /tests/data/expected-results/charts/trufflesom-macro-steady.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smarr/ReBenchDB/HEAD/tests/data/expected-results/charts/trufflesom-macro-steady.png -------------------------------------------------------------------------------- /tests/data/expected-results/charts/trufflesom-micro-somsom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smarr/ReBenchDB/HEAD/tests/data/expected-results/charts/trufflesom-micro-somsom.png -------------------------------------------------------------------------------- /tests/data/expected-results/charts/trufflesom-micro-startup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smarr/ReBenchDB/HEAD/tests/data/expected-results/charts/trufflesom-micro-startup.png -------------------------------------------------------------------------------- /tests/data/expected-results/charts/trufflesom-micro-steady.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smarr/ReBenchDB/HEAD/tests/data/expected-results/charts/trufflesom-micro-steady.png -------------------------------------------------------------------------------- /tests/data/expected-results/stats-data-prep/jssom/overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smarr/ReBenchDB/HEAD/tests/data/expected-results/stats-data-prep/jssom/overview.png -------------------------------------------------------------------------------- /src/backend/db/schema-updates/migration.009.sql: -------------------------------------------------------------------------------- 1 | -- Add a column to indicate whether we want notifications on github for this 2 | -- project. 3 | ALTER TABLE Project ADD COLUMN githubNotification bool DEFAULT true; 4 | -------------------------------------------------------------------------------- /src/backend/db/schema-updates/migration.007.sql: -------------------------------------------------------------------------------- 1 | -- Add a position column, which is now used to order the projects 2 | -- where ever db.getAllProjects() is used 3 | ALTER TABLE Project ADD COLUMN position integer DEFAULT 0; 4 | -------------------------------------------------------------------------------- /tests/data/expected-results/compare-view/navigation-jssom.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/backend/db/schema-updates/migration.008.sql: -------------------------------------------------------------------------------- 1 | -- remove the timeline-related database objects 2 | -- they are not needed anymore, because we do everything inside of Node 3 | DROP TABLE IF EXISTS TimelineCalcJob; 4 | DROP SEQUENCE IF EXISTS TimelineJobId; 5 | -------------------------------------------------------------------------------- /tests/data/expected-results/compare-view/stats-row-version-missing.html: -------------------------------------------------------------------------------- 1 | 2 | my-benchmark 3 | No matching configuration for
aaa: total 4 | 5 | 6 | -------------------------------------------------------------------------------- /src/shared/util.ts: -------------------------------------------------------------------------------- 1 | const pathRegex = /^([^\s]*)\/([^\s]+\s.*$)/; 2 | 3 | export function simplifyCmdline(cmdline: string): string { 4 | // remove the beginning of the path, leaving only the last element of it 5 | return cmdline.replace(pathRegex, '$2'); 6 | } 7 | -------------------------------------------------------------------------------- /src/backend/db/schema-updates/migration.005.sql: -------------------------------------------------------------------------------- 1 | -- add slug column 2 | ALTER TABLE Project ADD COLUMN slug varchar unique; 3 | 4 | -- populate the slug column by replacing all non-save-characters with a dash 5 | UPDATE Project SET slug = regexp_replace(name, '[^0-9a-zA-Z-]', '-', 'g'); 6 | -------------------------------------------------------------------------------- /Dockerfile.rebench: -------------------------------------------------------------------------------- 1 | # Used for benchmarking 2 | FROM rebenchdb-app:latest 3 | 4 | RUN apt-get update && apt-get install -y git python3-pip 5 | RUN pip install --break-system-packages git+https://github.com/smarr/ReBench.git 6 | 7 | RUN npm run pretest 8 | 9 | ENTRYPOINT ["rebench"] 10 | CMD ["--help"] 11 | -------------------------------------------------------------------------------- /src/backend/db/schema-updates/migration.006.sql: -------------------------------------------------------------------------------- 1 | -- add the envId to the unique constraint 2 | -- or rather, drop the old constraint, and add a new one 3 | ALTER TABLE Trial 4 | DROP CONSTRAINT trial_username_envid_starttime_key; 5 | 6 | ALTER TABLE Trial 7 | ADD CONSTRAINT trial_username_envid_expid_starttime_key 8 | UNIQUE(username, envId, expId, startTime); 9 | -------------------------------------------------------------------------------- /src/backend/db/schema-updates/cleanup-zeros.sql: -------------------------------------------------------------------------------- 1 | SELECT count(*) as cnt, value, criterion, c.name, c.unit FROM Measurement JOIN Criterion c ON c.id = criterion GROUP BY criterion, value, c.name, c.unit HAVING count(value) > 1000 ORDER BY cnt DESC, criterion ASC; 2 | 3 | DELETE FROM Measurement WHERE criterion = 2 AND value = 0; 4 | DELETE FROM Measurement WHERE criterion = 3 AND value = 0; -------------------------------------------------------------------------------- /src/backend/db/schema-updates/migration.010.sql: -------------------------------------------------------------------------------- 1 | -- Add NOT NULL constraints to the Run table 2 | ALTER TABLE Run ALTER COLUMN benchmarkId SET NOT NULL; 3 | ALTER TABLE Run ALTER COLUMN suiteId SET NOT NULL; 4 | ALTER TABLE Run ALTER COLUMN execId SET NOT NULL; 5 | ALTER TABLE Run ALTER COLUMN cmdline SET NOT NULL; 6 | ALTER TABLE Run ALTER COLUMN maxInvocationTime SET NOT NULL; 7 | ALTER TABLE Run ALTER COLUMN minIterationTime SET NOT NULL; 8 | -------------------------------------------------------------------------------- /src/backend/db/schema-updates/migration.003.sql: -------------------------------------------------------------------------------- 1 | -- migration to add support for storing profiling data 2 | CREATE TABLE ProfileData ( 3 | runId smallint, 4 | trialId smallint, 5 | invocation smallint, 6 | numIterations smallint, 7 | 8 | value text NOT NULL, 9 | 10 | primary key (numIterations, invocation, runId, trialId), 11 | foreign key (trialId) references Trial (id), 12 | foreign key (runId) references Run (id) 13 | ); 14 | -------------------------------------------------------------------------------- /src/download.ts: -------------------------------------------------------------------------------- 1 | import { createWriteStream } from 'node:fs'; 2 | import { argv } from 'node:process'; 3 | import { Readable } from 'node:stream'; 4 | import { finished } from 'node:stream/promises'; 5 | 6 | const url = argv[2]; 7 | const targetFile = argv[3]; 8 | 9 | console.log(`Downloading ${url} to ${targetFile}`); 10 | 11 | const file = createWriteStream(targetFile); 12 | const { body } = await fetch(url); 13 | await finished(Readable.fromWeb(body).pipe(file)); 14 | -------------------------------------------------------------------------------- /src/backend/admin/operations.ts: -------------------------------------------------------------------------------- 1 | import { ParameterizedContext } from 'koa'; 2 | 3 | import { Database } from '../db/db.js'; 4 | 5 | export async function submitTimelineUpdateJobs( 6 | ctx: ParameterizedContext, 7 | db: Database 8 | ): Promise { 9 | db 10 | .getTimelineUpdater() 11 | ?.submitUpdateJobs() 12 | .then((n) => n) 13 | .catch((e) => e); 14 | ctx.body = 'update process started'; 15 | ctx.type = 'text'; 16 | ctx.status = 200; 17 | } 18 | -------------------------------------------------------------------------------- /src/frontend/project.ts: -------------------------------------------------------------------------------- 1 | import { renderChanges, renderAllResults } from './render.js'; 2 | 3 | $(async () => { 4 | const projectId = $('#project-id').attr('value'); 5 | const showChanges = $('#project-showchanges').attr('value') === 'true'; 6 | const allResults = $('#project-allresults').attr('value') === 'true'; 7 | 8 | if (showChanges) { 9 | renderChanges(projectId); 10 | } 11 | 12 | if (allResults) { 13 | renderAllResults(projectId); 14 | } 15 | }); 16 | -------------------------------------------------------------------------------- /src/backend/request-check.ts: -------------------------------------------------------------------------------- 1 | import { ParameterizedContext } from 'koa'; 2 | 3 | export function getNumberOrError( 4 | ctx: ParameterizedContext, 5 | paramName: string 6 | ): number | null { 7 | const value = Number(ctx.params[paramName]); 8 | 9 | if (isNaN(value)) { 10 | ctx.status = 400; 11 | ctx.body = { 12 | error: `Invalid ${paramName} provided. Received "${ctx.params.runId}".` 13 | }; 14 | 15 | return null; 16 | } 17 | 18 | return value; 19 | } 20 | -------------------------------------------------------------------------------- /src/frontend/project-data.ts: -------------------------------------------------------------------------------- 1 | import { renderProjectDataOverview } from './render.js'; 2 | 3 | const projectId = $('#project-id').attr('value'); 4 | const projectSlug = $('#project-slug').attr('value'); 5 | const dataOverviewP = fetch(`/rebenchdb/dash/${projectId}/data-overview`); 6 | 7 | $(async () => { 8 | const dataOverviewResponse = await dataOverviewP; 9 | const data = (await dataOverviewResponse.json()).data; 10 | renderProjectDataOverview(data, projectSlug); 11 | }); 12 | -------------------------------------------------------------------------------- /tests/data/expected-results/compare-view/stats-row-across-version.html: -------------------------------------------------------------------------------- 1 | 43 2 | 3 | 0.33 4 | 54600 5 | 0 6 | 14600 7 | 222b 8 | 64600 -------------------------------------------------------------------------------- /tests/rebench-integration/test-vm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | 4 | if len(sys.argv) != 3: 5 | print("Usage: test-vm.py ") 6 | sys.exit(1) 7 | 8 | criteria = { 9 | "mem": {"unit": "MB", "step": 3}, 10 | "compile": {"unit": "ms", "step": 7}, 11 | "total": {"unit": "ms", "step": 1}, 12 | } 13 | 14 | benchmark_name = sys.argv[1] 15 | num_iterations = int(sys.argv[2]) 16 | 17 | for i in range(1, num_iterations + 1): 18 | for n, c in criteria.items(): 19 | if i % c["step"] == 0: 20 | print(f"{benchmark_name}: {n}: {i}{c['unit']}") -------------------------------------------------------------------------------- /.github/workflows/container.yml: -------------------------------------------------------------------------------- 1 | name: Build Container 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - docker** 8 | - compose** 9 | 10 | jobs: 11 | docker: 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - name: Checkout ReBench 16 | uses: actions/checkout@v5 17 | 18 | - name: Build and Test Docker Image and Composition 19 | run: | 20 | docker compose -f ./docker-compose.yml up --detach 21 | curl --retry 5 --retry-delay 5 --retry-all-errors http://localhost:33333/status 22 | docker compose -f ./docker-compose.yml down 23 | -------------------------------------------------------------------------------- /src/backend/compare/html/navigation.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/data/expected-results/compare-view/stats-tbl-header.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | #M 6 | median time
in ms 7 | time diff % 8 | median GC
in ms 9 | GC diff % 10 | median allocated
in bytes 11 | allocated diff % 12 | 13 | 14 | -------------------------------------------------------------------------------- /tests/data/expected-results/compare-view/stats-row-button-info.html: -------------------------------------------------------------------------------- 1 | 3 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | rebenchdb-app: 5 | build: . 6 | image: rebenchdb-app 7 | command: npm run start 8 | expose: 9 | - 33333 10 | environment: 11 | RDB_HOST: postgres 12 | RDB_USER: docker 13 | RDB_PASS: docker 14 | RDB_DB: rebenchdb 15 | RDB_PORT: 5432 16 | REFRESH_SECRET: refresh 17 | DEV: true 18 | depends_on: 19 | - postgres 20 | ports: 21 | - '33333:33333' 22 | 23 | postgres: 24 | image: postgres:18-alpine 25 | environment: 26 | POSTGRES_USER: docker 27 | POSTGRES_PASSWORD: docker 28 | POSTGRES_DB: rebenchdb 29 | ports: 30 | - '5432:5432' 31 | -------------------------------------------------------------------------------- /src/backend/compare/html/compare-exes.html: -------------------------------------------------------------------------------- 1 |

Executor Comparisons

2 | {% 3 | for (const [s, suite] of it.acrossExes) { 4 | %} 5 |

{%= s %}

6 |

Baseline: {%= suite.baselineExeName %}

7 | 8 | 9 | {%- include('stats-tbl.html', { 10 | config: it.config, 11 | criteria: suite.criteria, 12 | benchmarks: suite.benchmarks, 13 | environments: it.environments, 14 | dataFormatters: it.dataFormatters, 15 | viewHelpers: it.viewHelpers, 16 | isAcrossExes: true 17 | }) %} 18 | {% } %} -------------------------------------------------------------------------------- /src/vendored/chartjs-node-canvas/src/backgroundColourPlugin.ts: -------------------------------------------------------------------------------- 1 | import { Chart as ChartJS, Plugin as ChartJSPlugin } from 'chart.js'; 2 | 3 | export class BackgroundColourPlugin implements ChartJSPlugin { 4 | public readonly id: string = 5 | 'chartjs-plugin-chartjs-node-canvas-background-colour'; 6 | 7 | public constructor( 8 | private readonly _width: number, 9 | private readonly _height: number, 10 | private readonly _fillStyle: string 11 | ) {} 12 | 13 | public beforeDraw(chart: ChartJS): boolean | void { 14 | const ctx = chart.ctx; 15 | ctx.save(); 16 | ctx.fillStyle = this._fillStyle; 17 | ctx.fillRect(0, 0, this._width, this._height); 18 | ctx.restore(); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /tests/data/expected-results/compare-view/stats-summary.html: -------------------------------------------------------------------------------- 1 |

Result Overview

2 |
3 | 4 |
5 |
6 |
Number of Run Configurations
7 |
232
8 |
Change of Run time
9 |
median 50% (min. 10%, max. 110%)
10 |
Change of gcTime
11 |
median 250% (min. 210%, max. 310%)
12 |
Change of allocated
13 |
median 450% (min. 410%, max. 510%)
14 |
-------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | - build-and-benchmark 3 | 4 | build-and-benchmark: 5 | stage: build-and-benchmark 6 | tags: [yuria2] 7 | script: 8 | - podman build . -f Dockerfile -t rebenchdb-app 9 | - podman build . -f Dockerfile.rebench -t bench-rdb 10 | - podman run --hostname postgres -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=rebenchdb --cidfile=postgres.pid --publish=5432:5432 --detach postgres:18-alpine 11 | - sleep 5 12 | - podman run --network=host bench-rdb:latest -c --experiment="CI ID $CI_PIPELINE_ID" --branch="$CI_COMMIT_REF_NAME" rebench.conf 13 | after_script: 14 | - POSTGRES_PID=$(cat postgres.pid) 15 | - podman stop $POSTGRES_PID 16 | - podman rm $POSTGRES_PID 17 | - rm postgres.pid 18 | -------------------------------------------------------------------------------- /src/backend/rebench/api-validator.ts: -------------------------------------------------------------------------------- 1 | import Ajv, { ValidateFunction } from 'ajv'; 2 | import { 3 | getProgramFromFiles, 4 | generateSchema, 5 | CompilerOptions, 6 | PartialArgs 7 | } from 'typescript-json-schema'; 8 | import { robustPath } from '../util.js'; 9 | 10 | export function createValidator(): ValidateFunction { 11 | const compilerOptions: CompilerOptions = { 12 | strictNullChecks: true 13 | }; 14 | 15 | const settings: PartialArgs = { 16 | required: true 17 | }; 18 | 19 | const api = robustPath('shared/api.ts'); 20 | 21 | const program = getProgramFromFiles([api], compilerOptions); 22 | const schema = generateSchema(program, 'BenchmarkData', settings); 23 | 24 | const ajv = new Ajv({ allErrors: true, allowUnionTypes: true }); 25 | return ajv.compile(schema); 26 | } 27 | -------------------------------------------------------------------------------- /src/backend/compare/html/compare-versions.html: -------------------------------------------------------------------------------- 1 |

Performance Changes between Versions

2 | {% 3 | for (const [exeName, suites] of it.acrossVersions.allMeasurements.entries()) { 4 | for (const [suiteName, suite] of suites.entries()) { 5 | if (suite.benchmarks.length > 0) { 6 | %} 7 |
8 |

{%= suiteName %}

9 |
Executor: {%= exeName %}
10 | 11 | {%- include('stats-tbl.html', { 12 | config: it.config, 13 | criteria: suite.criteria, 14 | benchmarks: suite.benchmarks, 15 | environments: it.environments, 16 | dataFormatters: it.dataFormatters, 17 | viewHelpers: it.viewHelpers, 18 | isAcrossExes: false 19 | }) %} 20 | 21 |
22 | {% 23 | } 24 | } 25 | } 26 | %} -------------------------------------------------------------------------------- /src/frontend/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | populateStatistics, 3 | renderAllResults, 4 | renderChanges 5 | } from './render.js'; 6 | 7 | async function showStatistics(): Promise { 8 | const statsP = fetch(`/rebenchdb/stats`); 9 | await populateStatistics(statsP); 10 | $('#stats-table-button').hide(); 11 | } 12 | 13 | $(async () => { 14 | $('#stats-table-button').on('click', showStatistics); 15 | $('.project-data').each((_i, elem) => { 16 | const elemJq = $(elem); 17 | const showChanges = elemJq.data('showchanges'); 18 | const allResults = elemJq.data('allresults'); 19 | const projectId = elemJq.data('id'); 20 | 21 | if (showChanges) { 22 | renderChanges(projectId); 23 | } 24 | 25 | if (allResults) { 26 | renderAllResults(projectId); 27 | } 28 | }); 29 | }); 30 | -------------------------------------------------------------------------------- /src/backend/compare/html/stats-tbl-header.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% if (it.isAcrossExes) { %}Exe{% } %} 5 | #M 6 | {% for (const i of it.criteriaOrder) { 7 | const name = it.criteria[i].name; 8 | const unit = it.criteria[i].unit; 9 | const longLabel = it.dataFormatters.smartLower(name === 'total' ? 'time' : name); 10 | const shortLabel = name === 'total' ? 'time' : it.dataFormatters.shortenCriteria(longLabel); 11 | %}median {%= shortLabel %}
in {%= unit %} 12 | {%= shortLabel %} diff % 13 | {% } %} 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/backend/templates.ts: -------------------------------------------------------------------------------- 1 | import { readFileSync } from 'fs'; 2 | import { robustPath } from './util.js'; 3 | import { compile, TemplateFunction, Options } from 'ejs'; 4 | 5 | const ejsConfig: Options = { 6 | openDelimiter: '{', 7 | closeDelimiter: '}', 8 | root: robustPath('views'), 9 | views: [robustPath('views')], 10 | _with: false, 11 | strict: true, 12 | localsName: 'it' 13 | }; 14 | 15 | export function prepareTemplate( 16 | filename: string, 17 | rmWhitespace = false, 18 | templateRoot: string | undefined = undefined 19 | ): TemplateFunction { 20 | const fileContent = readFileSync(filename).toString(); 21 | const config = { ...ejsConfig, rmWhitespace }; 22 | 23 | if (templateRoot) { 24 | config.root = templateRoot; 25 | config.views?.push(templateRoot); 26 | } 27 | return compile(fileContent, config); 28 | } 29 | -------------------------------------------------------------------------------- /src/backend/compare/html/stats-summary.html: -------------------------------------------------------------------------------- 1 |

Result Overview

2 |
3 | {% 4 | const d = it.dataFormatters; 5 | const s = it.stats; 6 | 7 | for (const url of it.overviewSvgUrls) { 8 | %}{% 9 | } 10 | 11 | %} 12 |
13 |
14 |
Number of Run Configurations
15 |
{%= it.numRunConfigs %}
16 | 17 | {% for (let name in s) { 18 | const data = s[name]; 19 | const label = name === 'total' ? 'Run time' : name; 20 | %}
Change of {%= label %}
21 |
median {%= d.per(data.median) %}% (min. {%= d.per(data.min) %}%, max. {%= d.per(data.max) %}%)
22 | {% } 23 | %}
24 | -------------------------------------------------------------------------------- /tests/data/expected-results/compare-view/stats-row-across-exes.html: -------------------------------------------------------------------------------- 1 | 2 | ast 3 |
bc 4 | 5 | 6 | 43 7 |
12 8 | 9 | 10 | 11 | 0.33 12 |
0.45 13 | 14 | 15 | 54600 16 |
3400 17 | 18 | 19 | 0 20 |
0 21 | 22 | 23 | 14600 24 |
232300 25 | 26 | 27 | 222b 28 |
675b 29 | 30 | 31 | 64600 32 |
604600 33 | -------------------------------------------------------------------------------- /src/backend/logging.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from 'tslog'; 2 | import { DEV, isRunningTests } from './util.js'; 3 | 4 | // 0: silly, 1: trace, 2: debug, 3: info, 4: warn, 5: error, 6: fatal 5 | const trace = 2; 6 | const info = 3; 7 | const error = 5; 8 | 9 | function getLoggingLevel(): number { 10 | if (isRunningTests) { 11 | return error; 12 | } 13 | 14 | if (DEV) { 15 | return trace; 16 | } 17 | 18 | return info; 19 | } 20 | 21 | const minLevel = getLoggingLevel(); 22 | 23 | export const log = new Logger({ name: 'index', minLevel }); 24 | 25 | export function assert( 26 | condition: boolean, 27 | message: string | undefined = undefined 28 | ): void { 29 | if (!condition) { 30 | const stack = new Error().stack; 31 | if (message) { 32 | log.error('Assertion failed', message, stack); 33 | } else { 34 | log.error('Assertion failed', stack); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/backend/compare/html/stats-tbl.html: -------------------------------------------------------------------------------- 1 | 2 | {% const criteria = Object.keys(it.criteria); 3 | it.viewHelpers.sortTotalToFront(criteria); 4 | %}{%- include('stats-tbl-header.html', { 5 | criteria: it.criteria, 6 | criteriaOrder: criteria, 7 | isAcrossExes: it.isAcrossExes, 8 | dataFormatters: it.dataFormatters, 9 | }) %} 10 | 11 | {% it.benchmarks.sort(it.viewHelpers.sortByNameAndArguments); 12 | for (const benchmark of it.benchmarks) { 13 | %}{%- include('stats-row.html', { 14 | config: it.config, 15 | stats: benchmark, 16 | environments: it.environments, 17 | dataFormatters: it.dataFormatters, 18 | viewHelpers: it.viewHelpers, 19 | isAcrossExes: it.isAcrossExes, 20 | criteriaOrder: criteria, 21 | criteria: it.criteria, 22 | }) %} 23 | {% } 24 | %} 25 |
26 | -------------------------------------------------------------------------------- /src/backend/compare/html/stats-row-across-versions.html: -------------------------------------------------------------------------------- 1 | {% 2 | const f = it.dataFormatters; 3 | const s = it.stats; 4 | 5 | %}{%= s.total.samples %} 6 | {% for (const i of it.criteriaOrder) { 7 | const cssClassForTotal = i === 'total' ? ' stats-total' : ''; 8 | let median; 9 | if (!s[i]) { 10 | median = ''; 11 | } else if (i === 'total') { 12 | median = f.r2(s[i].median); 13 | } else { 14 | switch (it.criteria[i].unit) { 15 | case 'bytes': 16 | median = f.asHumanMem(s[i].median); 17 | break; 18 | default: 19 | median = f.r0(s[i].median); 20 | break; 21 | } 22 | } 23 | 24 | const change = (!s[i]) ? '' : f.per(s[i].change_m); 25 | %} 26 | {%= median %} 27 | {%= change %}{% 28 | } %} -------------------------------------------------------------------------------- /tests/rebench-integration/rebench.conf: -------------------------------------------------------------------------------- 1 | default_experiment: benchmarks 2 | default_data_file: 'rebench.data' 3 | 4 | reporting: 5 | # Benchmark results will be reported to ReBenchDB 6 | rebenchdb: 7 | # this url needs to point to the API endpoint 8 | db_url: http://localhost:33333/rebenchdb 9 | repo_url: https://github.com/smarr/rebenchdb 10 | record_all: true # make sure everything is recorded 11 | project_name: ReBenchDB-integration-test 12 | 13 | benchmark_suites: 14 | test-suite: 15 | gauge_adapter: RebenchLog 16 | command: "%(benchmark)s %(iterations)s" 17 | iterations: 100 18 | invocations: 1 19 | benchmarks: 20 | - Test 21 | 22 | executors: 23 | TestVM: 24 | path: . 25 | executable: ./test-vm.py 26 | 27 | experiments: 28 | benchmarks: 29 | description: Test Benchmark 30 | suites: 31 | - test-suite 32 | executions: 33 | - TestVM 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.Rproj 2 | *.qs 3 | *.swp 4 | /.Rhistory 5 | /.Rproj.user 6 | /.vscode 7 | /coverage 8 | /dist 9 | /logo 10 | /node_modules 11 | /resources/*.js 12 | /resources/exp-data 13 | /resources/reports 14 | /resources/uPlot.min.css 15 | /src/views/somns.html 16 | /src/views/test.html 17 | /tests/data/actual-results 18 | /tests/data/large-payload.json 19 | /tmp 20 | test.figures 21 | test.html 22 | 23 | # ignore most generated plots to make it easier to handle 24 | /tests/data/expected-results/stats-data-prep/jssom/inline-[0-9].svg 25 | /tests/data/expected-results/stats-data-prep/jssom/inline-[1-2][0-9].svg 26 | /tests/data/expected-results/stats-data-prep/tsom/inline-[0-9].svg 27 | /tests/data/expected-results/stats-data-prep/tsom/inline-[0-9][0-9].svg 28 | /tests/data/expected-results/stats-data-prep/tsom/inline-1[0-9][0-9].svg 29 | /tests/data/expected-results/stats-data-prep/tsom/inline-exe-*.svg 30 | 31 | # ignore developer-specific helper scripts 32 | /*.sh 33 | 34 | -------------------------------------------------------------------------------- /src/backend/compare/html/stats-row-buttons-info.html: -------------------------------------------------------------------------------- 1 | {% 2 | const d = it.details; 3 | const b = it.benchId; 4 | const f = it.dataFormatters; 5 | %} 7 | {% 8 | if (it.environments) { 9 | %} 11 | {% } 12 | 13 | if (d.hasWarmup) { 14 | %} 15 | {% 16 | } 17 | 18 | if (d.profiles) { 19 | %} 20 | {% 21 | } 22 | %} 24 | -------------------------------------------------------------------------------- /tests/shared/aesthetics.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, it } from '@jest/globals'; 2 | import { siteAesthetics } from '../../src/shared/aesthetics.js'; 3 | 4 | describe('lighten()', () => { 5 | it('should lighten a color', () => { 6 | expect(siteAesthetics.lighten('#2e3436')).toEqual('#555753'); 7 | expect(siteAesthetics.lighten('#f57900')).toEqual('#fcaf3e'); 8 | expect(siteAesthetics.lighten('#97c4f0')).toEqual('#daeeff'); 9 | }); 10 | 11 | it('should lighten a color, also if it does not start with a #', () => { 12 | expect(siteAesthetics.lighten('2e3436')).toEqual('#555753'); 13 | expect(siteAesthetics.lighten('f57900')).toEqual('#fcaf3e'); 14 | expect(siteAesthetics.lighten('97c4f0')).toEqual('#daeeff'); 15 | }); 16 | 17 | it('when lighten the last color, it should just stay the same', () => { 18 | expect(siteAesthetics.lighten('#daeeff')).toEqual('#daeeff'); 19 | expect(siteAesthetics.lighten('#ffcccc')).toEqual('#ffcccc'); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /src/backend/db/timed-cache-validity.ts: -------------------------------------------------------------------------------- 1 | export class TimedCacheValidity { 2 | private valid: boolean; 3 | private scheduledInvalidation: boolean; 4 | 5 | /** Delay in milliseconds. */ 6 | private readonly invalidationDelay: number; 7 | 8 | constructor(invalidationDelay: number) { 9 | this.invalidationDelay = invalidationDelay; 10 | this.valid = true; 11 | this.scheduledInvalidation = false; 12 | } 13 | 14 | public invalidateAndNew(): TimedCacheValidity { 15 | if (!this.scheduledInvalidation) { 16 | this.scheduledInvalidation = true; 17 | if (this.invalidationDelay === 0) { 18 | this.valid = false; 19 | } else { 20 | setTimeout(() => { 21 | this.valid = false; 22 | }, this.invalidationDelay); 23 | } 24 | } 25 | 26 | if (this.valid) { 27 | return this; 28 | } 29 | return new TimedCacheValidity(this.invalidationDelay); 30 | } 31 | 32 | public isValid(): boolean { 33 | return this.valid; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/shared/errors.ts: -------------------------------------------------------------------------------- 1 | import { log } from '../backend/logging.js'; 2 | 3 | export function reportConnectionRefused(e: any): void { 4 | if (e.errors && e.errors.length > 0) { 5 | for (const currentE of e.errors) { 6 | if (currentE.code == 'ECONNREFUSED' && currentE.port) { 7 | log.error( 8 | `Unable to connect to database on port ` + 9 | `${currentE.address}:${currentE.port}.\n` 10 | ); 11 | } 12 | } 13 | log.error( 14 | 'Connection refused.\n' + 15 | 'ReBenchDB requires a Postgres database to work.' 16 | ); 17 | } else { 18 | log.error( 19 | `Unable to connect to database on port ${e.address}:${e.port}.\n` + 20 | 'Connection refused.\n' + 21 | 'ReBenchDB requires a Postgres database to work.' 22 | ); 23 | } 24 | } 25 | 26 | export function reportDatabaseInUse(e: any): void { 27 | log.error(e.message); 28 | log.error(e.detail); 29 | } 30 | 31 | export function reportOtherErrors(e: any): void { 32 | log.error('benchmark failed unexpectedly', e); 33 | } 34 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:24-bookworm 2 | # this allows the setup to ignore all of the ubuntu OS setup 3 | # thats not needed for this docker image (Time Zone for example) 4 | ARG DEBIAN_FRONTEND=noninteractive 5 | 6 | # tools needed for docker setup 7 | RUN apt-get update && apt-get install -y apt-utils bash sudo 8 | 9 | ENV TZ=UTC 10 | RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 11 | 12 | # Node.js, PostgreSQL, headers for R packages 13 | RUN apt-get update && apt-get install -y \ 14 | build-essential unzip \ 15 | libfontconfig1-dev \ 16 | libpq-dev 17 | 18 | # Copy only package*.json, which are likely unchanged, allowing caching 19 | COPY package.json package-lock.json /project/ 20 | COPY patches /project/patches 21 | 22 | # Set the working dir to the project & install and compile all dependency 23 | WORKDIR /project/ 24 | 25 | RUN SKIP_COMPILE=true npm ci --ignore-scripts=false --foreground-scripts 26 | 27 | # copy all files, which likely have changed, and prevent caching 28 | COPY . /project/ 29 | 30 | RUN npm run compile 31 | -------------------------------------------------------------------------------- /src/backend/db/schema-updates/migration.002.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE Run ALTER COLUMN warmup TYPE int; 2 | ALTER TABLE Run ALTER COLUMN maxInvocationTime TYPE int; 3 | ALTER TABLE Run ALTER COLUMN minIterationTime TYPE int; 4 | ALTER TABLE Timeline ALTER COLUMN numSamples TYPE int; 5 | 6 | -- details on system settings that influence noise level for measurements 7 | ALTER TABLE Trial ADD COLUMN denoise jsonb; 8 | 9 | -- set sensible defaults for projects 10 | ALTER TABLE Project ALTER COLUMN showChanges SET DEFAULT true; 11 | ALTER TABLE Project ALTER COLUMN allResults SET DEFAULT false; 12 | 13 | -- add the TimelineCalcJob table 14 | CREATE SEQUENCE TimelineJobId AS smallint CYCLE; 15 | 16 | CREATE TABLE TimelineCalcJob ( 17 | timelineJobId smallint NOT NULL DEFAULT nextval('TimelineJobId') PRIMARY KEY, 18 | trialId smallint, 19 | runId smallint, 20 | criterion smallint 21 | ); 22 | 23 | ALTER SEQUENCE TimelineJobId OWNED BY TimelineCalcJob.timelineJobId; 24 | 25 | -- add the things needed for experiment comparisons 26 | ALTER TABLE Project ADD COLUMN baseBranch varchar; 27 | -------------------------------------------------------------------------------- /src/backend/db/database-with-pool.ts: -------------------------------------------------------------------------------- 1 | import pg, { PoolConfig, QueryConfig, QueryResult, QueryResultRow } from 'pg'; 2 | 3 | import { Database } from './db.js'; 4 | import { BatchingTimelineUpdater } from '../timeline/timeline-calc.js'; 5 | 6 | export class DatabaseWithPool extends Database { 7 | private pool: pg.Pool; 8 | 9 | constructor( 10 | config: PoolConfig, 11 | numBootstrapSamples = 1000, 12 | timelineEnabled = false, 13 | cacheInvalidationDelay = 0 14 | ) { 15 | super( 16 | config, 17 | timelineEnabled ? new BatchingTimelineUpdater(numBootstrapSamples) : null, 18 | cacheInvalidationDelay 19 | ); 20 | this.pool = new pg.Pool(config); 21 | } 22 | 23 | public async query( 24 | queryConfig: QueryConfig 25 | ): Promise> { 26 | return this.pool.query(queryConfig); 27 | } 28 | 29 | public async close(): Promise { 30 | await super.close(); 31 | this.statsValid.invalidateAndNew(); 32 | await this.pool.end(); 33 | (this).pool = null; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/backend/db/has-profile.ts: -------------------------------------------------------------------------------- 1 | import { BenchmarkId } from '../../shared/api.js'; 2 | import { AvailableProfile } from './types.js'; 3 | 4 | function sameBenchId(a: BenchmarkId, b: BenchmarkId): boolean { 5 | // using here == instead of === is intentional 6 | // otherwise we don't equate null and undefined 7 | return ( 8 | a.b == b.b && 9 | a.e == b.e && 10 | a.s == b.s && 11 | a.v == b.v && 12 | a.c == b.c && 13 | a.i == b.i && 14 | a.ea == b.ea 15 | ); 16 | } 17 | 18 | export class HasProfile { 19 | /** 20 | * This is expected to be sorted by runid, commitid 21 | * as coming from the database. 22 | * This ensures that base and change profile availability is paired up. 23 | */ 24 | private readonly availableProfiles: AvailableProfile[]; 25 | 26 | constructor(availableProfiles: AvailableProfile[]) { 27 | this.availableProfiles = availableProfiles; 28 | } 29 | 30 | public has(benchId: BenchmarkId): boolean { 31 | const idx = this.availableProfiles.findIndex((id) => 32 | sameBenchId(id, benchId) 33 | ); 34 | return idx >= 0; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/backend/db/schema-updates/migration.001.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE SchemaVersion ( 2 | updateDate timestamp with time zone, 3 | version smallint primary key 4 | ); 5 | 6 | INSERT INTO SchemaVersion (version, updateDate) VALUES (1, now()); 7 | 8 | ALTER TABLE Project ADD COLUMN showChanges bool DEFAULT true; 9 | 10 | ALTER TABLE Project ADD COLUMN allResults bool DEFAULT false; 11 | 12 | UPDATE Project SET showChanges = true, allResults = false WHERE name = 'SOMns'; 13 | UPDATE Project SET showChanges = false, allResults = true WHERE name = 'ReBenchDB Self-Tracking'; 14 | 15 | CREATE TABLE Timeline ( 16 | runId smallint, 17 | trialId smallint, 18 | criterion smallint, 19 | 20 | numSamples smallint, 21 | 22 | minVal float4, 23 | maxVal float4, 24 | sdVal float4, 25 | mean float4, 26 | median float4, 27 | 28 | -- bootstrap confidence interval 95%-tile 29 | bci95low float4, 30 | bci95up float4, 31 | 32 | primary key (runId, trialId, criterion), 33 | foreign key (trialId) references Trial (id), 34 | foreign key (runId) references Run (id), 35 | foreign key (criterion) references Criterion (id) 36 | ); 37 | -------------------------------------------------------------------------------- /src/views/header.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Stefan Marr 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /src/backend/common/standard-responses.ts: -------------------------------------------------------------------------------- 1 | import { ParameterizedContext } from 'koa'; 2 | 3 | export function respondProjectIdNotFound( 4 | ctx: ParameterizedContext, 5 | projectId: number 6 | ): void { 7 | ctx.body = `Requested project with id ${projectId} not found`; 8 | ctx.status = 404; 9 | ctx.type = 'text'; 10 | } 11 | 12 | export function respondProjectNotFound( 13 | ctx: ParameterizedContext, 14 | projectSlug: string 15 | ): void { 16 | ctx.body = `Requested project "${projectSlug}" not found`; 17 | ctx.status = 404; 18 | ctx.type = 'text'; 19 | } 20 | 21 | export function respondProjectAndSourceNotFound( 22 | ctx: ParameterizedContext, 23 | projectSlug: string, 24 | sourceId: string 25 | ): void { 26 | ctx.body = 27 | `Requested combination of project "${projectSlug}"` + 28 | ` and source ${sourceId} not found`; 29 | ctx.status = 404; 30 | ctx.type = 'text'; 31 | } 32 | 33 | export function respondExpIdNotFound( 34 | ctx: ParameterizedContext, 35 | expId: string 36 | ): void { 37 | ctx.body = `Requested experiment ${expId} not found`; 38 | ctx.status = 404; 39 | ctx.type = 'text'; 40 | } 41 | -------------------------------------------------------------------------------- /src/backend/timeline/timeline-calc-worker.ts: -------------------------------------------------------------------------------- 1 | import { parentPort, workerData } from 'node:worker_threads'; 2 | import { calculateSummaryStatistics } from '../../shared/stats.js'; 3 | import type { 4 | ComputeRequest, 5 | ComputeResult, 6 | ComputeResults 7 | } from './timeline-calc.js'; 8 | 9 | parentPort?.on('message', (message) => { 10 | if (message === 'exit') { 11 | parentPort?.postMessage('exiting'); 12 | parentPort?.close(); 13 | return; 14 | } 15 | 16 | const request: ComputeRequest = message; 17 | const results: ComputeResult[] = []; 18 | 19 | for (const req of request.jobs) { 20 | const stats = calculateSummaryStatistics( 21 | req.dataForCriterion, 22 | workerData.numBootstrapSamples 23 | ); 24 | 25 | const result: ComputeResult = { 26 | runId: req.runId, 27 | trialId: req.trialId, 28 | criterion: req.criterion, 29 | stats 30 | }; 31 | results.push(result); 32 | } 33 | 34 | const result: ComputeResults = { 35 | results, 36 | requestStart: request.requestStart, 37 | requestId: request.requestId 38 | }; 39 | parentPort?.postMessage(result); 40 | }); 41 | -------------------------------------------------------------------------------- /tests/data/expected-results/compare-view/stats-row-version-one-criteria-missing.html: -------------------------------------------------------------------------------- 1 | 2 | my-benchmark 3 | No matching configuration for
aaa: some missing criterion 4 | 5 | 6 | 7 | my-benchmark 8 | 9 | 43 10 | 11 | 0.33 12 | 54600 13 | 15 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /tests/data/expected-results/compare-view/stats-row-version.html: -------------------------------------------------------------------------------- 1 | 2 | my-benchmark 3 | 4 | 43 5 | 6 | 0.33 7 | 54600 8 | 0 9 | 14600 10 | 222b 11 | 64600 12 | 14 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/frontend/timeline.ts: -------------------------------------------------------------------------------- 1 | import type { TimelineResponse } from '../shared/api.js'; 2 | import { initializeFilters } from './filter.js'; 3 | import { renderTimelinePlot } from './plots.js'; 4 | 5 | const projectId = $('#project-id').attr('value'); 6 | const projectSlug = $('#project-slug').attr('value'); 7 | 8 | async function loadPlotOnce(this: any) { 9 | const thisJq = $(this); 10 | if (thisJq.data('requested')) { 11 | return; 12 | } 13 | 14 | thisJq.data('requested', true); 15 | 16 | const runId = thisJq.data('runid'); 17 | const timelineP = await fetch( 18 | `/rebenchdb/dash/${projectId}/timeline/${runId}` 19 | ); 20 | const response = await timelineP.json(); 21 | renderTimelinePlot(response, thisJq, projectSlug); 22 | thisJq.off('appear', onPlotAppearing); 23 | } 24 | 25 | function onPlotAppearing(_event, allAppearedElements) { 26 | allAppearedElements.each(loadPlotOnce); 27 | } 28 | 29 | $(async () => { 30 | const timelineJq = $('.timeline-plot'); 31 | // activate the appear event 32 | (timelineJq).appear(); 33 | timelineJq.on('appear', onPlotAppearing); 34 | 35 | timelineJq.each((i, e) => { 36 | if (i < 10) { 37 | loadPlotOnce.call(e); 38 | } 39 | }); 40 | 41 | initializeFilters('.benchmark > h4'); 42 | }); 43 | -------------------------------------------------------------------------------- /tests/shared/single-requester.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, it } from '@jest/globals'; 2 | import { SingleRequestOnly } from '../../src/shared/single-requester.js'; 3 | 4 | describe('Basic functionality of SingleRequestOnly', () => { 5 | it('should execute the request on a triggering', async () => { 6 | let executed = 0; 7 | const sro = new SingleRequestOnly(async () => { 8 | executed += 1; 9 | }); 10 | sro.trigger(); 11 | 12 | const promise = sro.getQuiescencePromise(); 13 | expect(promise).not.toBeUndefined(); 14 | 15 | await promise; 16 | 17 | expect(executed).toEqual(1); 18 | expect(sro.getQuiescencePromise()).toBeUndefined(); 19 | }); 20 | 21 | it('should execute request only once, even if triggered twice', async () => { 22 | let executed = 0; 23 | const sro = new SingleRequestOnly(async () => { 24 | executed += 1; 25 | }); 26 | 27 | // Since the request is executed asynchronously, 28 | // we do execute it only once 29 | sro.trigger(); 30 | sro.trigger(); 31 | 32 | const promise = sro.getQuiescencePromise(); 33 | expect(promise).not.toBeUndefined(); 34 | 35 | await promise; 36 | 37 | expect(executed).toEqual(1); 38 | expect(sro.getQuiescencePromise()).toBeUndefined(); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /patches/@sgratzl+chartjs-chart-boxplot+4.4.5.patch: -------------------------------------------------------------------------------- 1 | diff --git a/node_modules/@sgratzl/chartjs-chart-boxplot/build/index.js b/node_modules/@sgratzl/chartjs-chart-boxplot/build/index.js 2 | index 29e5b08..203670f 100644 3 | --- a/node_modules/@sgratzl/chartjs-chart-boxplot/build/index.js 4 | +++ b/node_modules/@sgratzl/chartjs-chart-boxplot/build/index.js 5 | @@ -71,7 +71,7 @@ function determineStatsOptions(options) { 6 | }; 7 | } 8 | function boxplotStats(arr, options) { 9 | - const vs = window.Float64Array != null && !(arr instanceof Float32Array || arr instanceof Float64Array) 10 | + const vs = typeof Float64Array !== 'undefined' && !(arr instanceof Float32Array || arr instanceof Float64Array) 11 | ? Float64Array.from(arr) 12 | : arr; 13 | const r = boxplot(vs, determineStatsOptions(options)); 14 | @@ -104,7 +104,7 @@ function violinStats(arr, options) { 15 | if (arr.length === 0) { 16 | return undefined; 17 | } 18 | - const vs = window.Float64Array != null && !(arr instanceof Float32Array || arr instanceof Float64Array) 19 | + const vs = typeof Float64Array !== 'undefined' && !(arr instanceof Float32Array || arr instanceof Float64Array) 20 | ? Float64Array.from(arr) 21 | : arr; 22 | const stats = boxplot(vs, determineStatsOptions(options)); 23 | -------------------------------------------------------------------------------- /src/benchmarks/rebenchdb-benchmark.ts: -------------------------------------------------------------------------------- 1 | import { BenchmarkData } from '../shared/api.js'; 2 | import { 3 | closeMainDb, 4 | createAndInitializeDB, 5 | TestDatabase 6 | } from '../../tests/backend/db/db-testing.js'; 7 | import { Benchmark } from './benchmark.js'; 8 | import { loadLargePayloadApiV1 } from '../../tests/payload.js'; 9 | 10 | export class RebenchDbBenchmark extends Benchmark { 11 | protected readonly testData: BenchmarkData; 12 | protected db: TestDatabase | null; 13 | protected problemSize: string; 14 | protected enableTimeline: boolean; 15 | 16 | constructor() { 17 | super(); 18 | this.db = null; 19 | this.problemSize = ''; 20 | this.enableTimeline = false; 21 | 22 | this.testData = loadLargePayloadApiV1(); 23 | } 24 | 25 | public async oneTimeSetup(problemSize: string): Promise { 26 | this.problemSize = problemSize; 27 | this.db = await createAndInitializeDB( 28 | 'rdb_benchmark', 29 | 100, 30 | this.enableTimeline, 31 | false 32 | ); 33 | 34 | if (!this.db) { 35 | throw new Error('ReBenchDB connection was not initialized'); 36 | } 37 | } 38 | 39 | public async oneTimeTeardown(): Promise { 40 | if (this.db) { 41 | await this.db?.close(); 42 | await closeMainDb(); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/backend/db/schema-updates/migration.011.sql: -------------------------------------------------------------------------------- 1 | -- This migration denormalizes the Run table by adding the benchmark, suite, and executor columns. 2 | -- add new columns 3 | ALTER TABLE Run ADD COLUMN benchmark varchar DEFAULT NULL; 4 | ALTER TABLE Run ADD COLUMN suite varchar DEFAULT NULL; 5 | ALTER TABLE Run ADD COLUMN executor varchar DEFAULT NULL; 6 | 7 | -- update the new columns based on the existing data 8 | UPDATE Run SET benchmark = Benchmark.name FROM Benchmark WHERE Run.benchmarkId = Benchmark.id; 9 | UPDATE Run SET suite = Suite.name FROM Suite WHERE Run.suiteId = Suite.id; 10 | UPDATE Run SET executor = Executor.name FROM Executor WHERE Run.execId = Executor.id; 11 | 12 | -- make the new columns not null 13 | ALTER TABLE Run ALTER COLUMN suite SET NOT NULL; 14 | ALTER TABLE Run ALTER COLUMN executor SET NOT NULL; 15 | ALTER TABLE Run ALTER COLUMN benchmark SET NOT NULL; 16 | 17 | -- remove the old id columns 18 | ALTER TABLE Run DROP COLUMN benchmarkId; 19 | ALTER TABLE Run DROP COLUMN suiteId; 20 | ALTER TABLE Run DROP COLUMN execId; 21 | 22 | -- remove foreign keys 23 | ALTER TABLE Run DROP CONSTRAINT IF EXISTS run_benchmarkid_fkey; 24 | ALTER TABLE Run DROP CONSTRAINT IF EXISTS run_suiteid_fkey; 25 | ALTER TABLE Run DROP CONSTRAINT IF EXISTS run_execid_fkey; 26 | 27 | -- remove the old tables 28 | DROP TABLE Benchmark; 29 | DROP TABLE Suite; 30 | DROP TABLE Executor; 31 | -------------------------------------------------------------------------------- /tests/data/small-payload.json: -------------------------------------------------------------------------------- 1 | {"env":{"userName":"smarr","hostName":"Artemis","manualRun":true,"memory":17179869184,"osType":"Darwin","clockSpeed":2800000000,"cpu":"Intel(R) Core(TM) i7-7700HQ CPU @ 2.80GHz","software":[{"version":"Darwin Kernel Version 18.7.0: Sat Oct 12 00:02:19 PDT 2019; root:xnu-4903.278.12~1/RELEASE_X86_64","name":"kernel"},{"version":"18.7.0","name":"kernel-release"},{"version":"x86_64","name":"architecture"}]},"projectName":"Small Example Project","source":{"commitId":"58666d1c84c652306f930daa72e7a47c58478e86","committerName":"Stefan Marr","commitMsg":"Clean out data only when actually sent\n\nSigned-off-by: Stefan Marr \n","authorName":"Stefan Marr","committerEmail":"git@stefan-marr.de","repoURL":"http://repo.git","branchOrTag":"HEAD -> rebenchdb, smarr/rebenchdb","authorEmail":"git@stefan-marr.de"},"startTime":"2019-12-13T22:49:56","criteria":[{"i":0,"c":"total","u":"ms"}],"data":[{"runId":{"varValue":null,"inputSize":null,"benchmark":{"suite":{"desc":null,"name":"macro-startup","executor":{"name":"SOMns-graal","desc":null}},"name":"NBody","runDetails":{"maxInvocationTime":600,"minIterationTime":5,"warmup":null}},"extraArgs":"1 0 10000","cmdline":"/Users/smarr/Projects/ReBench/tests/small/som -t1 core-lib/Benchmarks/Harness.ns NBody 1 0 10000","location":"/Users/smarr/Projects/ReBench/tests/small","cores":1},"d":[{"m":[[383.821]],"in":1},{"m":[[432.783]],"in":2},{"m":[[482.53]],"in":3}]}],"experimentName":"Small Test Case"} -------------------------------------------------------------------------------- /tests/backend/compare/db-measurements.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, it, beforeAll, afterAll } from '@jest/globals'; 2 | 3 | import { 4 | TestDatabase, 5 | closeMainDb, 6 | createAndInitializeDB 7 | } from '../db/db-testing.js'; 8 | import { loadLargePayload } from '../../payload.js'; 9 | import { getMeasurements } from '../../../src/backend/compare/compare.js'; 10 | 11 | describe('compare view: getMeasurements()', () => { 12 | let db: TestDatabase; 13 | 14 | beforeAll(async () => { 15 | db = await createAndInitializeDB('compare_view_get_m', 0, false); 16 | const largeTestData = loadLargePayload(); 17 | await db.recordAllData(largeTestData); 18 | }); 19 | 20 | afterAll(async () => { 21 | if (db) return db.close(); 22 | }); 23 | 24 | it('should give expected results for runId 1', async () => { 25 | const results = await getMeasurements( 26 | 'Large-Example-Project', 27 | 10, 28 | '58666d1c84c652306f930daa72e7a47c58478e86', 29 | '58666d1c84c652306f930daa72e7a47c58478e86', 30 | db 31 | ); 32 | 33 | expect(results).toHaveLength(1); 34 | if (results === null) return; 35 | expect(results[0].data).toHaveLength(1); 36 | expect(results[0].data[0].criterion).toEqual('total'); 37 | expect(results[0].data[0].values).toHaveLength(1); 38 | expect(results[0].data[0].values[0]).toHaveLength(1000); 39 | }); 40 | }); 41 | 42 | afterAll(async () => { 43 | return closeMainDb(); 44 | }); 45 | -------------------------------------------------------------------------------- /tests/shared/fast-or-precise.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * A little script to compare the results of the fast and precise confidence 3 | * interval calculations. 4 | */ 5 | import Decimal from 'decimal.js'; 6 | import { 7 | confidenceSliceIndicesFast, 8 | confidenceSliceIndicesPrecise 9 | } from '../../src/shared/stats.js'; 10 | 11 | const confidenceLevels = ['0.8', '0.85', '0.9', '0.99', '0.999', '0.99999']; 12 | 13 | for (let i = 1; i <= 7; i += 1) { 14 | confidenceLevels.push('0.' + i); 15 | confidenceLevels.push('0.' + i + '5'); 16 | } 17 | 18 | const differences: any = []; 19 | 20 | for (const level of confidenceLevels) { 21 | const levelPrecise = new Decimal(level); 22 | const levelFast = parseFloat(level); 23 | const diffs: number[] = []; 24 | 25 | for (let i = 1; i < 100_000; i += 1) { 26 | const fast = confidenceSliceIndicesFast(i, levelFast); 27 | const precise = confidenceSliceIndicesPrecise(i, levelPrecise); 28 | 29 | if ( 30 | fast.low !== precise.low || 31 | fast.mid[0] !== precise.mid[0] || 32 | fast.mid[1] !== precise.mid[1] || 33 | fast.high !== precise.high 34 | ) { 35 | // differences.push({ 36 | // level, 37 | // i, 38 | // fast, 39 | // fastMean: fast.mean, 40 | // precise, 41 | // preciseMean: precise.mean 42 | // }); 43 | // break; 44 | diffs.push(i); 45 | } 46 | } 47 | 48 | if (diffs.length > 0) { 49 | differences.push({ level, diffs }); 50 | } 51 | } 52 | 53 | console.log(differences); 54 | -------------------------------------------------------------------------------- /src/backend/project/get-exp-data.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ReBenchDB {%= it.project %}: Preparing Data For Download 6 | 7 | {%- include('header.html', { rebenchVersion: it.rebenchVersion }) %} 8 | 9 | 10 | 11 | 12 | 13 |
14 |
15 |

ReBenchDB for {%= it.project %}

16 |

Preparing Data for download of {%= it.expName %}

17 |
18 | {%- include('common-menu.html', it) %} 19 |
20 | 21 | {% if (it.preparingData) { %} 22 | 36 | {% } %} 37 | 38 | {% if (it.generationFailed) { %} 39 | 42 | {% } %} 43 | 44 | 45 | -------------------------------------------------------------------------------- /tests/backend/common/standard-responses.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, it } from '@jest/globals'; 2 | import { 3 | respondExpIdNotFound, 4 | respondProjectAndSourceNotFound, 5 | respondProjectIdNotFound, 6 | respondProjectNotFound 7 | } from '../../../src/backend/common/standard-responses.js'; 8 | 9 | describe('respondProjectIdNotFound', () => { 10 | it('should set status to 404 and respond with text', () => { 11 | const response: any = {}; 12 | respondProjectIdNotFound(response, 42); 13 | expect(response.status).toEqual(404); 14 | expect(response.type).toEqual('text'); 15 | }); 16 | }); 17 | 18 | describe('respondProjectNotFound', () => { 19 | it('should set status to 404 and respond with text', () => { 20 | const response: any = {}; 21 | respondProjectNotFound(response, 'project-slug'); 22 | expect(response.status).toEqual(404); 23 | expect(response.type).toEqual('text'); 24 | }); 25 | }); 26 | 27 | describe('respondProjectAndSourceNotFound', () => { 28 | it('should set status to 404 and respond with text', () => { 29 | const response: any = {}; 30 | respondProjectAndSourceNotFound(response, 'project-slug', 'sha-commit-id'); 31 | expect(response.status).toEqual(404); 32 | expect(response.type).toEqual('text'); 33 | }); 34 | }); 35 | 36 | describe('respondExpIdNotFound', () => { 37 | it('should set status to 404 and respond with text', () => { 38 | const response: any = {}; 39 | respondExpIdNotFound(response, 'exp-id'); 40 | expect(response.status).toEqual(404); 41 | expect(response.type).toEqual('text'); 42 | }); 43 | }); 44 | -------------------------------------------------------------------------------- /tests/data/expected-results/compare-view/stats-row-exe.html: -------------------------------------------------------------------------------- 1 | 2 | my-benchmark 3 | 4 | 5 | ast 6 |
bc 7 | 8 | 9 | 43 10 |
12 11 | 12 | 13 | 14 | 0.33 15 |
0.45 16 | 17 | 18 | 54600 19 |
3400 20 | 21 | 22 | 0 23 |
0 24 | 25 | 26 | 14600 27 |
232300 28 | 29 | 30 | 222b 31 |
675b 32 | 33 | 34 | 64600 35 |
604600 36 | 37 | 39 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /tests/backend/project/project.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, it } from '@jest/globals'; 2 | import { initJestMatchers } from '../../helpers.js'; 3 | import { Project } from '../../../src/backend/db/types.js'; 4 | import { prepareTemplate } from '../../../src/backend/templates.js'; 5 | import { robustPath } from '../../../src/backend/util'; 6 | 7 | initJestMatchers(); 8 | 9 | describe('renderProjectDataPage', () => { 10 | it('should render the page', () => { 11 | const project: Project = { 12 | id: 1, 13 | name: 'Test Project', 14 | slug: 'test-project', 15 | description: 'desc', 16 | logo: 'logo.png', 17 | showchanges: true, 18 | allresults: true, 19 | githubnotification: false, 20 | basebranch: 'base' 21 | }; 22 | 23 | const tpl = prepareTemplate( 24 | robustPath('backend/project/project-data.html'), 25 | true 26 | ); 27 | const html = tpl({ project, rebenchVersion: 'testVersion' }); 28 | expect(html).toEqualHtmlFragment('project/project-data'); 29 | }); 30 | }); 31 | 32 | describe('renderDataExport', () => { 33 | it('should render the page', () => { 34 | const data = { 35 | project: 'Test Project', 36 | expName: 'Test Experiment', 37 | preparingData: true, 38 | currentTime: 'Some Date', 39 | generationFailed: true, 40 | rebenchVersion: 'testVersion' 41 | }; 42 | const tpl = prepareTemplate( 43 | robustPath('backend/project/get-exp-data.html'), 44 | true 45 | ); 46 | const html = tpl(data); 47 | expect(html).toEqualHtmlFragment('project/get-exp-data'); 48 | }); 49 | }); 50 | -------------------------------------------------------------------------------- /src/backend/compare/html/stats-row-across-exes.html: -------------------------------------------------------------------------------- 1 | {% 2 | const h = it.viewHelpers; 3 | const f = it.dataFormatters; 4 | const commonStart = h.commonStringStart(it.exes.map((e) => e.exeName)); 5 | %} 6 | {% 7 | let itOut = new h.PerIterationOutput('', '
'); 8 | for (const e of it.exes) { 9 | %}{%- itOut.next() 10 | %}{%= h.withoutStart(commonStart, e.exeName) %} 11 | {% } 12 | %} 13 | 14 | {% 15 | itOut = new h.PerIterationOutput('', '
'); 16 | for (const e of it.exes) { 17 | %}{%- itOut.next() 18 | %}{%= e.criteria.total.samples %} 19 | {% } 20 | %} 21 | {% for (const i of it.criteriaOrder) { 22 | const cssClassForTotal = i === 'total' ? ' stats-total' : ''; 23 | let medianFmtFn; 24 | if (i === 'total') { 25 | medianFmtFn = f.r2; 26 | } else { 27 | switch (it.criteria[i].unit) { 28 | case 'bytes': 29 | medianFmtFn = f.asHumanMem; 30 | break; 31 | default: 32 | medianFmtFn = f.r0; 33 | break; 34 | } 35 | } 36 | 37 | const changeFmtFn = f.per; 38 | %} 39 | 40 | {% 41 | itOut = new h.PerIterationOutput('', '
'); 42 | for (const e of it.exes) { 43 | %}{%- itOut.next() 44 | %}{%= (!e.criteria[i]) ? '' : medianFmtFn(e.criteria[i].median) %} 45 | {% } 46 | %} 47 | 48 | {% 49 | itOut = new h.PerIterationOutput('', '
'); 50 | for (const e of it.exes) { 51 | %}{%- itOut.next() 52 | %}{%= (!e.criteria[i]) ? '' : changeFmtFn(e.criteria[i].change_m) %} 53 | {% } 54 | %}{% 55 | } %} 56 | -------------------------------------------------------------------------------- /src/backend/project/project-data.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ReBench: {%= it.project.name %} 8 | {%- include('header.html', { rebenchVersion: it.rebenchVersion }) %} 9 | 10 | 11 | 12 | 13 |
14 |
15 |

{%= it.project.name %}

16 | {% if (it.project.description) { %} 17 |

{%= it.project.description %}

18 | {% } %} 19 |
20 | {%- include('common-menu.html', it) %} 21 |
22 | 23 |
24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 48 | 49 |
ExperimentDescriptionStart/EndUserCommitMachine#runs#measurements
50 |
51 | 52 | 53 | -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import { defineConfig, globalIgnores } from 'eslint/config'; 2 | 3 | import js from '@eslint/js'; 4 | import { parser as _parser, plugin } from 'typescript-eslint'; 5 | import eslintConfigPrettier from 'eslint-config-prettier/flat'; 6 | import prettierPlugin from 'eslint-plugin-prettier'; 7 | 8 | import jest from 'eslint-plugin-jest'; 9 | import globals from 'globals'; 10 | 11 | export default defineConfig([ 12 | eslintConfigPrettier, 13 | { 14 | languageOptions: { 15 | parser: _parser, 16 | globals: { 17 | ...globals.browser, 18 | ...globals.node, 19 | ...globals.jquery 20 | } 21 | }, 22 | 23 | plugins: { 24 | '@typescript-eslint': plugin, 25 | jest, 26 | js, 27 | prettier: prettierPlugin 28 | }, 29 | 30 | extends: [ 31 | 'js/recommended', 32 | '@typescript-eslint/recommended', 33 | '@typescript-eslint/eslint-recommended', 34 | 'jest/recommended' 35 | ], 36 | 37 | rules: { 38 | 'max-len': [ 39 | 'error', 40 | { 41 | code: 80 42 | } 43 | ], 44 | 45 | quotes: [ 46 | 'error', 47 | 'single', 48 | { 49 | allowTemplateLiterals: true 50 | } 51 | ], 52 | 53 | '@typescript-eslint/no-unused-vars': [ 54 | 'error', 55 | { 56 | argsIgnorePattern: '^_', 57 | varsIgnorePattern: '^_', 58 | caughtErrorsIgnorePattern: '^_' 59 | } 60 | ], 61 | 62 | '@typescript-eslint/no-explicit-any': 'off', 63 | 64 | '@typescript-eslint/explicit-module-boundary-types': [ 65 | 'error', 66 | { 67 | allowArgumentsExplicitlyTypedAsAny: true 68 | } 69 | ], 70 | 71 | 'jest/no-conditional-expect': 'off' 72 | } 73 | }, 74 | globalIgnores(['**/node_modules', '**/dist', '**/resources', '**/coverage']) 75 | ]); 76 | -------------------------------------------------------------------------------- /tests/data/expected-results/compare-view/stats-tbl.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 35 | 36 | 37 |
#Mmedian time
in ms		time diff %median gctime
in ms		gctime diff %median allocated
in bytes		allocated diff %
my-benchmark430.3354600014600222b64600 30 | 32 | 33 | 34 |
-------------------------------------------------------------------------------- /src/views/theme-switcher-btn.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/data/expected-results/compare-view/navigation-tsom.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rebench.conf: -------------------------------------------------------------------------------- 1 | # -*- mode: yaml -*- 2 | # Config file for ReBench 3 | default_experiment: benchmarks 4 | default_data_file: 'rebench.data' 5 | 6 | reporting: 7 | # Benchmark results will be reported to ReBenchDB 8 | rebenchdb: 9 | # this url needs to point to the API endpoint 10 | db_url: https://rebench.stefan-marr.de/rebenchdb 11 | repo_url: https://github.com/smarr/ReBenchDB 12 | record_all: true # make sure everything is recorded 13 | project_name: ReBenchDB 14 | 15 | runs: 16 | max_invocation_time: 6000 17 | min_iteration_time: 1 18 | 19 | benchmark_suites: 20 | normal: 21 | gauge_adapter: RebenchLog 22 | command: "harness.js %(benchmark)s %(iterations)s 1 %(input)s " 23 | location: dist/src/benchmarks 24 | iterations: 10 25 | invocations: 1 26 | input_sizes: 27 | - small 28 | - medium 29 | - large 30 | benchmarks: 31 | - store-results 32 | - fetch-results 33 | - compute-timeline 34 | - render-report 35 | full: 36 | gauge_adapter: RebenchLog 37 | command: "harness.js %(benchmark)s %(iterations)s 1 %(input)s " 38 | location: dist/src/benchmarks 39 | iterations: 1 40 | invocations: 1 41 | input_sizes: 42 | - full 43 | benchmarks: 44 | - store-results 45 | - compute-timeline 46 | - render-report 47 | 48 | executors: 49 | benchmarks: 50 | executable: /usr/local/bin/node 51 | env: 52 | RDB_USER: docker 53 | RDB_PASS: docker 54 | RDB_HOST: localhost 55 | RDB_DB: rebenchdb 56 | NODE_ENV: test 57 | 58 | # define the benchmarks to be executed for a re-executable benchmark run 59 | experiments: 60 | benchmarks: 61 | description: All benchmarks 62 | suites: 63 | - normal 64 | - full 65 | executions: 66 | - benchmarks 67 | -------------------------------------------------------------------------------- /tests/rebench-integration/check-data.js: -------------------------------------------------------------------------------- 1 | import { readFileSync } from 'node:fs'; 2 | 3 | let allCorrect = true; 4 | 5 | function assert(values, criterion, step) { 6 | for (let i = 1; i < values.length; i += 1) { 7 | const val = values[i]; 8 | if (i % step === 0) { 9 | console.assert( 10 | val === i, 11 | `${criterion} at ${i}: Expected ${i}, got ${val}` 12 | ); 13 | allCorrect = allCorrect && val === i; 14 | } else { 15 | console.assert( 16 | val == null, 17 | `${criterion} at ${i}: Expected null, got ${val}` 18 | ); 19 | allCorrect = allCorrect && val == null; 20 | } 21 | } 22 | } 23 | 24 | function getJsonData() { 25 | const data = JSON.parse(readFileSync('actual.json', 'utf-8')); 26 | 27 | const byCriterion = { mem: [], compile: [], total: [] }; 28 | 29 | for (const e of data) { 30 | byCriterion[e.criterion][e.iteration] = e.value; 31 | } 32 | 33 | return byCriterion; 34 | } 35 | 36 | function getCsvData() { 37 | const data = readFileSync('actual.csv', 'utf-8'); 38 | 39 | const lines = data.split('\n'); 40 | const columnArr = lines.shift().split(','); 41 | const criterionIdx = columnArr.indexOf('criterion'); 42 | const iterationIdx = columnArr.indexOf('iteration'); 43 | const valueIdx = columnArr.indexOf('value'); 44 | 45 | const byCriterion = { mem: [], compile: [], total: [] }; 46 | 47 | for (const line of lines) { 48 | if (line === '') { 49 | continue; 50 | } 51 | const columns = line.split(','); 52 | byCriterion[columns[criterionIdx]][columns[iterationIdx]] = parseInt( 53 | columns[valueIdx] 54 | ); 55 | } 56 | 57 | return byCriterion; 58 | } 59 | 60 | function check(byCriterion) { 61 | for (const [c, step] of [ 62 | ['mem', 3], 63 | ['compile', 7], 64 | ['total', 1] 65 | ]) { 66 | assert(byCriterion[c], c, step); 67 | } 68 | } 69 | 70 | check(getJsonData()); 71 | check(getCsvData()); 72 | 73 | if (allCorrect) { 74 | process.exit(0); 75 | } else { 76 | process.exit(1); 77 | } 78 | -------------------------------------------------------------------------------- /src/benchmarks/benchmark.ts: -------------------------------------------------------------------------------- 1 | // This code is derived from the SOM benchmarks, see AUTHORS.md file. 2 | // 3 | // Copyright (c) 2015-2016 Stefan Marr 4 | // 5 | // Permission is hereby granted, free of charge, to any person obtaining a copy 6 | // of this software and associated documentation files (the 'Software'), to deal 7 | // in the Software without restriction, including without limitation the rights 8 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | // copies of the Software, and to permit persons to whom the Software is 10 | // furnished to do so, subject to the following conditions: 11 | // 12 | // The above copyright notice and this permission notice shall be included in 13 | // all copies or substantial portions of the Software. 14 | // 15 | // THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | // THE SOFTWARE. 22 | export class Benchmark { 23 | public async innerBenchmarkLoop(innerIterations: number): Promise { 24 | for (let i = 0; i < innerIterations; i++) { 25 | const result = await this.benchmark(); 26 | if (!this.verifyResult(result)) { 27 | return false; 28 | } 29 | } 30 | return true; 31 | } 32 | 33 | public async benchmark(): Promise { 34 | throw 'benchmark is subclass responsibility'; 35 | } 36 | 37 | public verifyResult(_result: any): boolean { 38 | throw 'verifyResult is subclass responsibility'; 39 | } 40 | 41 | public async oneTimeSetup(_problemSize: string): Promise { 42 | /* implemented in subclasses */ 43 | } 44 | 45 | public async oneTimeTeardown(): Promise { 46 | /* implemented in subclasses */ 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /tests/backend/db/cache-validity.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, it } from '@jest/globals'; 2 | // eslint-disable-next-line max-len 3 | import { TimedCacheValidity } from '../../../src/backend/db/timed-cache-validity.js'; 4 | 5 | async function delayOf(ms): Promise { 6 | return new Promise((resolve) => { 7 | setTimeout(resolve, ms); 8 | }); 9 | } 10 | 11 | describe('Timed Cache Validity', () => { 12 | it('should be valid on creation', () => { 13 | const v = new TimedCacheValidity(0); 14 | expect(v.isValid()).toBeTruthy(); 15 | }); 16 | 17 | it('should be immediately invalid when delay is 0', () => { 18 | const v = new TimedCacheValidity(0); 19 | v.invalidateAndNew(); 20 | expect(v.isValid()).toBeFalsy(); 21 | }); 22 | 23 | it('should return a new validity object on immediate invalidation', () => { 24 | const v = new TimedCacheValidity(0); 25 | const v2 = v.invalidateAndNew(); 26 | expect(v2).not.toBe(v); 27 | }); 28 | 29 | it('should not be immediately invalid if delay is set', () => { 30 | const v = new TimedCacheValidity(10); 31 | expect(v.isValid()).toBeTruthy(); 32 | }); 33 | 34 | it('should not be invalid if delay is set, after delay is over', async () => { 35 | const v = new TimedCacheValidity(10); 36 | expect(v.isValid()).toBeTruthy(); 37 | 38 | v.invalidateAndNew(); 39 | await delayOf(20); 40 | expect(v.isValid()).toBeFalsy(); 41 | }); 42 | 43 | it('should not return new validity while not yet invalid', async () => { 44 | const v = new TimedCacheValidity(10); 45 | expect(v.isValid()).toBeTruthy(); 46 | 47 | const vPre = v.invalidateAndNew(); 48 | const validPre = v.isValid(); 49 | 50 | await delayOf(5); 51 | 52 | const vPre2 = v.invalidateAndNew(); 53 | const validPre2 = v.isValid(); 54 | 55 | await delayOf(10); 56 | 57 | expect(validPre).toBeTruthy(); 58 | expect(vPre).toBe(v); 59 | 60 | expect(validPre2).toBeTruthy(); 61 | expect(vPre2).toBe(v); 62 | 63 | expect(v.isValid()).toBeFalsy(); 64 | expect(v.invalidateAndNew()).not.toBe(v); 65 | }); 66 | }); 67 | -------------------------------------------------------------------------------- /tests/data/expected-results/compare-view/compare-versions.html: -------------------------------------------------------------------------------- 1 |

Performance Changes between Versions

2 | 3 |
4 |

suite1

5 |
Executor: exe1
6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 40 | 41 | 42 |
#Mmedian time
in ms		time diff %median gctime
in ms		gctime diff %median allocated
in bytes		allocated diff %
my-benchmark430.3354600014600222b64600 35 | 37 | 38 | 39 |
43 |
44 | -------------------------------------------------------------------------------- /src/backend/compare/html/refresh-menu.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | 10 | 16 | {%- include('theme-switcher-btn.html') %} 17 | 18 | 24 |
25 | 27 |
28 |
29 |
-------------------------------------------------------------------------------- /src/benchmarks/store-results.ts: -------------------------------------------------------------------------------- 1 | import { convertToCurrentApi } from '../backend/common/api-v1.js'; 2 | import { RebenchDbBenchmark } from './rebenchdb-benchmark.js'; 3 | 4 | export default class StoreResults extends RebenchDbBenchmark { 5 | private iteration: number; 6 | 7 | constructor() { 8 | super(); 9 | this.iteration = 0; 10 | } 11 | 12 | public async oneTimeSetup(problemSize: string): Promise { 13 | await super.oneTimeSetup(problemSize); 14 | 15 | if (problemSize === 'full') { 16 | // just use the testData as is 17 | } else if (problemSize === 'large') { 18 | this.testData.data.length = 50; 19 | } else if (problemSize === 'medium') { 20 | this.testData.data.length = 20; 21 | for (const run of this.testData.data) { 22 | if (run.d) { 23 | run.d.length = 200; 24 | } 25 | } 26 | } else if (problemSize === 'small') { 27 | this.testData.data.length = 10; 28 | for (const run of this.testData.data) { 29 | if (run.d) { 30 | run.d.length = 15; 31 | } 32 | } 33 | } else { 34 | throw new Error('Unsupported problem size given: ' + problemSize); 35 | } 36 | (this).testData = convertToCurrentApi(this.testData); 37 | } 38 | 39 | public async benchmark(): Promise { 40 | this.iteration += 1; 41 | this.testData.experimentName = 'Benchmark ' + this.iteration; 42 | this.testData.source.commitId = 'commit-' + this.iteration; 43 | return this.db?.recordAllData(this.testData); 44 | } 45 | 46 | public verifyResult(result: any): boolean { 47 | const [recMs, recPs] = result; 48 | 49 | if (this.problemSize === 'full') { 50 | return recMs === 460 && recPs === 0; 51 | } else if (this.problemSize === 'large') { 52 | return recMs === 76 && recPs === 0; 53 | } else if (this.problemSize === 'medium') { 54 | return recMs === 26 && recPs === 0; 55 | } else if (this.problemSize === 'small') { 56 | return recMs === 12 && recPs === 0; 57 | } else { 58 | throw new Error('Unsupported problem size given: ' + this.problemSize); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/backend/db/schema-updates/migration.012.sql: -------------------------------------------------------------------------------- 1 | -- First Create an index that helps with the data migration 2 | BEGIN; 3 | 4 | CREATE INDEX runid_trialid_criterion_invocation_idx 5 | ON Measurement (runId, trialId, criterion, invocation); 6 | 7 | CREATE TEMPORARY TABLE temp_series_details AS ( 8 | SELECT runId, trialId, criterion, invocation, max(iteration) as max FROM Measurement 9 | GROUP BY runId, trialId, criterion, invocation 10 | ); 11 | 12 | CREATE TABLE TempMeasurement AS 13 | SELECT 14 | sd.runId, sd.trialId, sd.criterion, sd.invocation, 15 | array_agg(value order by iteration) as values 16 | FROM temp_series_details sd 17 | CROSS JOIN generate_series(1, sd.max::INTEGER) as g(iteration) 18 | LEFT JOIN Measurement m USING (iteration, runId, trialId, criterion, invocation) 19 | GROUP BY sd.runId, sd.trialId, sd.criterion, sd.invocation; 20 | 21 | -- put new table into place 22 | DROP TABLE Measurement; 23 | ALTER TABLE TempMeasurement RENAME TO Measurement; 24 | 25 | -- add the PK and FK constraints 26 | ALTER TABLE Measurement 27 | ADD PRIMARY KEY (invocation, runId, trialId, criterion); 28 | ALTER TABLE Measurement 29 | ADD FOREIGN KEY (trialId) REFERENCES Trial (id); 30 | ALTER TABLE Measurement 31 | ADD FOREIGN KEY (runId) REFERENCES Run (id); 32 | ALTER TABLE Measurement 33 | ADD FOREIGN KEY (criterion) REFERENCES Criterion (id); 34 | 35 | -- Add the now used recordAdditionalMeasurement. 36 | -- It is used by ReBenchDB's perf-tracker, for self-performance tracking 37 | CREATE PROCEDURE recordAdditionalMeasurement( 38 | aRunId smallint, 39 | aTrialId smallint, 40 | aCriterionId smallint, 41 | aValue float4) 42 | LANGUAGE plpgsql 43 | AS $$ 44 | BEGIN 45 | UPDATE Measurement m 46 | SET values = array_append(values, aValue) 47 | WHERE 48 | m.runId = aRunId AND 49 | m.trialId = aTrialId AND 50 | m.criterion = aCriterionId AND 51 | m.invocation = 1; 52 | 53 | IF NOT FOUND THEN 54 | INSERT INTO Measurement (runId, trialId, criterion, invocation, values) 55 | VALUES (aRunId, aTrialId, aCriterionId, 1, ARRAY[aValue]); 56 | END IF; 57 | END; 58 | $$; 59 | 60 | 61 | COMMIT; 62 | -------------------------------------------------------------------------------- /src/backend/common/api-v1.ts: -------------------------------------------------------------------------------- 1 | import type { BenchmarkData, DataPoint } from '../../shared/api.js'; 2 | 3 | export interface MeasureV1 { 4 | /** Criterion id. */ 5 | c: number; 6 | 7 | /** Value */ 8 | v: number; 9 | } 10 | 11 | export interface DataPointV1 { 12 | /** Invocation */ 13 | in: number; 14 | 15 | /** Iteration */ 16 | it: number; 17 | 18 | m: MeasureV1[]; 19 | } 20 | 21 | function convertDataPointsToCurrentApi(oldDs: any): DataPoint[] { 22 | const result: Map = new Map(); 23 | const oldDataPoints = oldDs as DataPointV1[]; 24 | 25 | for (const oldD of oldDataPoints) { 26 | if (!result.has(oldD.in)) { 27 | result.set(oldD.in, { 28 | in: oldD.in, 29 | m: [] 30 | }); 31 | } 32 | 33 | const newDP = result.get(oldD.in)!; 34 | const iteration = oldD.it; 35 | for (const measure of oldD.m) { 36 | if (newDP.m[measure.c] === undefined || newDP.m[measure.c] === null) { 37 | newDP.m[measure.c] = []; 38 | 39 | // mark criteria we have not seen yet explicitly with null 40 | for (let i = 0; i < measure.c; i += 1) { 41 | if (newDP.m[i] === undefined) { 42 | newDP.m[i] = null; 43 | } 44 | } 45 | } 46 | 47 | // iteration 1 is at index 0, etc 48 | newDP.m[measure.c]![iteration - 1] = measure.v; 49 | } 50 | } 51 | 52 | const newDataPoints = [...result.values()]; 53 | 54 | // turn undefined into null, to have a consistent absent value 55 | for (const dp of newDataPoints) { 56 | for (const criterionMs of dp.m) { 57 | if (criterionMs !== null) { 58 | const cMs = criterionMs as (number | null)[]; 59 | for (let i = 0; i < cMs.length; i += 1) { 60 | if (cMs[i] === undefined) { 61 | cMs[i] = null; 62 | } 63 | } 64 | } 65 | } 66 | } 67 | 68 | return newDataPoints; 69 | } 70 | 71 | export function convertToCurrentApi(data: BenchmarkData): BenchmarkData { 72 | for (const run of data.data) { 73 | if (!run.d) { 74 | continue; 75 | } 76 | 77 | run.d = convertDataPointsToCurrentApi(run.d); 78 | } 79 | return data; 80 | } 81 | -------------------------------------------------------------------------------- /src/benchmarks/fetch-results.ts: -------------------------------------------------------------------------------- 1 | import { convertToCurrentApi } from '../backend/common/api-v1.js'; 2 | import { getLast100Measurements } from '../backend/main/main.js'; 3 | import { RebenchDbBenchmark } from './rebenchdb-benchmark.js'; 4 | 5 | export default class RenderReport extends RebenchDbBenchmark { 6 | public async oneTimeSetup(problemSize: string): Promise { 7 | await super.oneTimeSetup(problemSize); 8 | 9 | if (problemSize === 'large') { 10 | // just use the testData as is 11 | } else if (problemSize === 'medium') { 12 | this.testData.data.length = 20; 13 | for (const run of this.testData.data) { 14 | if (run.d) { 15 | run.d.length = 200; 16 | } 17 | } 18 | } else if (problemSize === 'small') { 19 | this.testData.data.length = 10; 20 | for (const run of this.testData.data) { 21 | if (run.d) { 22 | run.d.length = 15; 23 | } 24 | } 25 | } else { 26 | throw new Error('Unsupported problem size given: ' + problemSize); 27 | } 28 | (this).testData = convertToCurrentApi(this.testData); 29 | 30 | for (let i = 1; i <= 2; i += 1) { 31 | this.testData.experimentName = 'Benchmark ' + i; 32 | this.testData.source.commitId = 'commit-' + i; 33 | await this.db?.recordAllData(this.testData); 34 | } 35 | } 36 | 37 | public async benchmark(): Promise { 38 | if (!this.db) { 39 | throw new Error('this.db not initialized'); 40 | } 41 | this.db.getStatsCacheValidity().invalidateAndNew(); 42 | return getLast100Measurements(1, this.db); 43 | } 44 | 45 | public verifyResult(result: any): boolean { 46 | if (this.problemSize === 'small') { 47 | return ( 48 | result.length === 9 && 49 | result.every((r) => r.values.length === 30 || r.values.length === 60) 50 | ); 51 | } 52 | 53 | if (this.problemSize === 'medium') { 54 | return ( 55 | result.length === 15 && result.every((r) => r.values.length === 100) 56 | ); 57 | } 58 | 59 | if (this.problemSize === 'large') { 60 | return ( 61 | result.length === 32 && 62 | result.every((r) => r.values.length === 128 || r.values.length === 100) 63 | ); 64 | } 65 | console.log(result, result.length, result[1].values.length); 66 | 67 | return false; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/shared/single-requester.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * At any given time, there should be only a single instance of the request 3 | * be running via this class. 4 | * 5 | * Triggering new runs are postponed until after the current run is completed. 6 | * It will only be triggered once after it is postponed this way. 7 | * Though, if the postponed run is active, it may be triggered again. 8 | */ 9 | export class SingleRequestOnly { 10 | private readonly request: () => Promise; 11 | private requestInProgress: boolean; 12 | private rerunRequested: boolean; 13 | 14 | private quiescencePromise?: Promise; 15 | private resolve?: (value: any) => void; 16 | private reject?: () => void; 17 | 18 | constructor(request: () => Promise) { 19 | this.request = request; 20 | this.requestInProgress = false; 21 | this.rerunRequested = false; 22 | 23 | this.initializePromise(); 24 | } 25 | 26 | private initializePromise() { 27 | this.quiescencePromise = new Promise((resolve, reject) => { 28 | this.resolve = resolve; 29 | this.reject = reject; 30 | }); 31 | } 32 | 33 | public trigger(): void { 34 | if (this.requestInProgress) { 35 | this.rerunRequested = true; 36 | return; 37 | } 38 | 39 | if (!this.quiescencePromise) { 40 | this.initializePromise(); 41 | } 42 | 43 | this.performRequestAsynchronously(); 44 | } 45 | 46 | private performRequestAsynchronously() { 47 | this.requestInProgress = true; 48 | 49 | setTimeout(() => { 50 | this.rerunRequested = false; 51 | this.attachPromiseHandlers(this.request()); 52 | }, 0); 53 | } 54 | 55 | private afterRequest(failed: boolean) { 56 | if (this.rerunRequested) { 57 | this.performRequestAsynchronously(); 58 | } else { 59 | this.quiescencePromise = undefined; 60 | if (this.resolve && this.reject) { 61 | if (failed) { 62 | this.reject(); 63 | } else { 64 | this.resolve(null); 65 | } 66 | } 67 | this.resolve = undefined; 68 | this.reject = undefined; 69 | this.requestInProgress = false; 70 | } 71 | } 72 | 73 | private attachPromiseHandlers(promise) { 74 | promise 75 | .then(() => { 76 | this.afterRequest(false); 77 | }) 78 | .catch(() => { 79 | this.afterRequest(true); 80 | }); 81 | } 82 | 83 | public getQuiescencePromise(): Promise | undefined { 84 | return this.quiescencePromise; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/frontend/filter.ts: -------------------------------------------------------------------------------- 1 | export function initializeFilters(benchmarkNameSelector: string): void { 2 | const allGroups = $('.exe-suite-group'); 3 | const byName = new Map(); 4 | const groups: string[][] = []; 5 | 6 | allGroups.each((_, group) => { 7 | const namesInGroup: string[] = []; 8 | groups.push(namesInGroup); 9 | 10 | $(group) 11 | .find(benchmarkNameSelector) 12 | .each((_, element) => { 13 | const name = element.textContent?.trim(); 14 | if (!byName.has(name)) { 15 | byName.set(name, []); 16 | namesInGroup.push(name); 17 | } 18 | 19 | byName.get(name).push(element); 20 | }); 21 | 22 | // make sure we don't have empty groups 23 | if (namesInGroup.length == 0) { 24 | groups.pop(); 25 | } 26 | }); 27 | 28 | let nameCheckBoxes = ''; 29 | 30 | for (const group of groups) { 31 | nameCheckBoxes += '
'; 32 | nameCheckBoxes += '
'; 33 | 34 | for (const name of group) { 35 | nameCheckBoxes += ` 36 |
37 | 39 | 40 |
`; 41 | } 42 | 43 | nameCheckBoxes += '
'; 44 | } 45 | 46 | $('#filter-groups').html(nameCheckBoxes); 47 | $('#filter-groups .card-body input').on('change', (e) => { 48 | const checkBoxJQ = $(e.currentTarget); 49 | const name = checkBoxJQ.val(); 50 | const nameElements = byName.get(name); 51 | if (checkBoxJQ.is(':checked')) { 52 | for (const nameElem of nameElements) { 53 | const nameElemJQ = $(nameElem); 54 | nameElemJQ.parent().show(); 55 | nameElemJQ.closest('.exe-suite-group').show(); 56 | } 57 | } else { 58 | for (const nameElem of nameElements) { 59 | const nameElemJQ = $(nameElem); 60 | nameElemJQ.parent().hide(); 61 | if (nameElemJQ.closest('tbody').find('tr:visible').length == 0) { 62 | nameElemJQ.closest('.exe-suite-group').hide(); 63 | } 64 | } 65 | } 66 | }); 67 | 68 | $('#filter-all').on('click', () => 69 | $('#filter-groups input').prop('checked', true).trigger('change') 70 | ); 71 | $('#filter-none').on('click', () => 72 | $('#filter-groups input').prop('checked', false).trigger('change') 73 | ); 74 | } 75 | -------------------------------------------------------------------------------- /src/shared/helpers.ts: -------------------------------------------------------------------------------- 1 | import type { CompareStatsRow } from './view-types.js'; 2 | 3 | /** 4 | * Returns the common start of a list of strings. 5 | */ 6 | export function commonStringStart(strings: string[]): string { 7 | if (strings.length === 0) { 8 | return ''; 9 | } 10 | 11 | const sorted = strings.sort(); 12 | const n = Math.min(...sorted.map((s) => s.length)); 13 | const first = sorted[0].slice(0, n); 14 | const last = sorted[sorted.length - 1].slice(0, n); 15 | 16 | let i = 0; 17 | while (i < n && first[i] === last[i]) { 18 | i += 1; 19 | } 20 | 21 | return first.slice(0, i); 22 | } 23 | 24 | /** 25 | * Remove a prefix from a string. 26 | */ 27 | export function withoutStart(prefix: string, str: string): string { 28 | if (str.startsWith(prefix)) { 29 | return str.slice(prefix.length); 30 | } 31 | return str; 32 | } 33 | 34 | /** 35 | * Return a string based on the iteration a loop is in. 36 | */ 37 | export class PerIterationOutput { 38 | private readonly first: string; 39 | private readonly allButFirst: string; 40 | 41 | private isFirst: boolean; 42 | 43 | constructor(first: string, allButFirst: string) { 44 | this.first = first; 45 | this.allButFirst = allButFirst; 46 | this.isFirst = true; 47 | } 48 | 49 | public next(): string { 50 | if (this.isFirst) { 51 | this.isFirst = false; 52 | return this.first; 53 | } 54 | return this.allButFirst; 55 | } 56 | } 57 | 58 | export function getBenchmarkArgumentsAsNamePart( 59 | benchmark: CompareStatsRow 60 | ): string { 61 | let args = ''; 62 | 63 | if (benchmark.details.numV > 1) { 64 | args += benchmark.benchId.v + ' '; 65 | } 66 | if (benchmark.details.numC > 1) { 67 | args += benchmark.benchId.c + ' '; 68 | } 69 | if (benchmark.details.numI > 1) { 70 | args += benchmark.benchId.i + ' '; 71 | } 72 | if (benchmark.details.numEa > 1) { 73 | args += benchmark.benchId.ea + ' '; 74 | } 75 | 76 | return args.trim(); 77 | } 78 | 79 | export function sortByNameAndArguments( 80 | a: CompareStatsRow, 81 | b: CompareStatsRow 82 | ): number { 83 | const result = a.benchId.b.localeCompare(b.benchId.b, undefined, { 84 | numeric: true 85 | }); 86 | 87 | if (result !== 0) { 88 | return result; 89 | } 90 | 91 | return a.argumentsForDisplay.localeCompare(b.argumentsForDisplay, undefined, { 92 | numeric: true 93 | }); 94 | } 95 | 96 | export function sortTotalToFront(criteria: string[]): string[] { 97 | const i = criteria.indexOf('total'); 98 | if (i !== -1) { 99 | criteria.splice(i, 1); 100 | criteria.unshift('total'); 101 | } 102 | return criteria; 103 | } 104 | -------------------------------------------------------------------------------- /src/backend/compare/html/stats-row.html: -------------------------------------------------------------------------------- 1 | {% 2 | const stats = it.stats; 3 | 4 | let args = ''; 5 | if (stats.argumentsForDisplay.length > 0) { 6 | args = `${stats.argumentsForDisplay}`; 7 | } 8 | 9 | if (stats.missing) { 10 | %} 11 | %}{%= stats.benchId.b %}{%- args %} 12 | No matching configuration for {% 13 | const byCommitId = new Map(); 14 | for (const m of stats.missing) { 15 | let byCommit = byCommitId.get(m.commitId); 16 | if (!byCommit) { 17 | byCommit = []; 18 | byCommitId.set(m.commitId, byCommit); 19 | } 20 | byCommit.push(m.criterion.name); 21 | } 22 | 23 | for (const [commitId, byCommit] of byCommitId) { 24 | %}
{%= commitId %}: {%= byCommit.join(', ') %}{% 25 | } 26 | %} 27 | 28 | 29 | {% 30 | } 31 | 32 | let hasTotal = false; 33 | if (stats.exeStats && stats.exeStats.every((e) => e.criteria.total)) { 34 | hasTotal = true; 35 | } else if (stats.versionStats && stats.versionStats.total) { 36 | hasTotal = true; 37 | } 38 | if (!stats.missing || hasTotal) { 39 | 40 | 41 | if (hasTotal) { 42 | let inconsistent = ''; 43 | if (stats.inconsistentRunIds) { 44 | inconsistent = ``; 49 | } 50 | %} 51 | {%= stats.benchId.b %}{%- args %}{%- inconsistent %} 52 | 53 | {% if (stats.exeStats) { 54 | %}{%- include('stats-row-across-exes.html', { 55 | exes: stats.exeStats, 56 | dataFormatters: it.dataFormatters, 57 | viewHelpers: it.viewHelpers, 58 | criteriaOrder: it.criteriaOrder, 59 | criteria: it.criteria} 60 | ) %} 61 | {% } else { 62 | %}{%- include('stats-row-across-versions.html', { 63 | stats: stats.versionStats, 64 | dataFormatters: it.dataFormatters, 65 | criteriaOrder: it.criteriaOrder, 66 | criteria: it.criteria} 67 | ) %} 68 | {% } 69 | %}{%- include('stats-row-buttons-info.html', { 70 | benchId: stats.benchId, 71 | details: stats.details, 72 | environments: it.environments, 73 | dataFormatters: it.dataFormatters} 74 | ) %} 75 | {% } 76 | %}{% 77 | } %} -------------------------------------------------------------------------------- /src/benchmarks/render-report.ts: -------------------------------------------------------------------------------- 1 | import { mkdtempSync, rmSync } from 'fs'; 2 | import { tmpdir } from 'os'; 3 | import * as path from 'path'; 4 | import { RebenchDbBenchmark } from './rebenchdb-benchmark.js'; 5 | import { renderCompareViewToString } from '../backend/compare/report.js'; 6 | import { convertToCurrentApi } from '../backend/common/api-v1.js'; 7 | 8 | export default class RenderReport extends RebenchDbBenchmark { 9 | private baseHash: string | null = null; 10 | private changeHash: string | null = null; 11 | private tmpDir: string | null = null; 12 | 13 | public async oneTimeSetup(problemSize: string): Promise { 14 | await super.oneTimeSetup(problemSize); 15 | 16 | if (problemSize === 'full') { 17 | // use as is 18 | } else if (problemSize === 'large') { 19 | this.testData.data.length = 50; 20 | } else if (problemSize === 'medium') { 21 | this.testData.data.length = 20; 22 | for (const run of this.testData.data) { 23 | if (run.d) { 24 | run.d.length = 200; 25 | } 26 | } 27 | } else if (problemSize === 'small') { 28 | this.testData.data.length = 10; 29 | for (const run of this.testData.data) { 30 | if (run.d) { 31 | run.d.length = 15; 32 | } 33 | } 34 | } else { 35 | throw new Error('Unsupported problem size given: ' + problemSize); 36 | } 37 | (this).testData = convertToCurrentApi(this.testData); 38 | 39 | this.testData.experimentName = 'Benchmark 1'; 40 | this.baseHash = this.testData.source.commitId = 'commit-1'; 41 | await this.db?.recordAllData(this.testData); 42 | 43 | this.testData.experimentName = 'Benchmark 2'; 44 | this.changeHash = this.testData.source.commitId = 'commit-2'; 45 | await this.db?.recordAllData(this.testData); 46 | 47 | this.tmpDir = mkdtempSync(path.join(tmpdir(), '/rebenchdb-tmp')); 48 | } 49 | 50 | public async oneTimeTeardown(): Promise { 51 | super.oneTimeTeardown(); 52 | if (this.tmpDir) { 53 | rmSync(this.tmpDir, { recursive: true, force: true }); 54 | } 55 | } 56 | 57 | public async benchmark(): Promise { 58 | if (!this.baseHash || !this.changeHash || !this.tmpDir || !this.db) { 59 | throw new Error( 60 | 'RenderReport.oneTimeSetup did not set baseHash or changeHash' 61 | ); 62 | } 63 | 64 | return await renderCompareViewToString( 65 | this.baseHash, 66 | this.changeHash, 67 | 'Large-Example-Project', 68 | this.db 69 | ); 70 | } 71 | 72 | public verifyResult(result: any): boolean { 73 | let r = true; 74 | 75 | for (const run of this.testData.data) { 76 | r &&= result.includes(run.runId.benchmark.name); 77 | } 78 | 79 | return r; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /tests/shared/ui.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable max-len */ 2 | 3 | import { describe, expect, it } from '@jest/globals'; 4 | import { simplifyCmdline } from '../../src/shared/util.js'; 5 | 6 | describe('Helper Functions for Rendering', () => { 7 | describe('Simplifying the command-line for display', () => { 8 | it('Should remove the beginning of the command-line from all command-lines', () => { 9 | expect( 10 | simplifyCmdline( 11 | '/data/home/gitlab-runner/builds/71gxYod2/0/sm951/awfy-runs/awfy/benchmarks/CSharp/bin/Release/net6.0/Benchmarks Queens 1000 1000' 12 | ) 13 | ).toBe('Benchmarks Queens 1000 1000'); 14 | 15 | expect( 16 | simplifyCmdline( 17 | '/data/home/gitlab-runner/builds/71gxYod2/0/sm951/truffleruby/bin/jt --use jvm-ce ruby --experimental-options --engine.Compilation=false harness.rb Activesupport 1 30' 18 | ) 19 | ).toBe( 20 | 'jt --use jvm-ce ruby --experimental-options --engine.Compilation=false harness.rb Activesupport 1 30' 21 | ); 22 | 23 | expect( 24 | simplifyCmdline( 25 | '/data/home/gitlab-runner/builds/d258e35c/0/sm951/truffleruby/truffleruby-jvm-ce/bin/truffleruby harness.rb Richards 1 1' 26 | ) 27 | ).toBe('truffleruby harness.rb Richards 1 1'); 28 | 29 | expect( 30 | simplifyCmdline('/usr/bin/ruby2.7 harness.rb Permute 10 1000') 31 | ).toBe('ruby2.7 harness.rb Permute 10 1000'); 32 | 33 | const somCmdLines = [ 34 | '/data/home/gitlab-runner/builds/71gxYod2/0/sm951/awfy-runs/awfy/implementations/SOM/som.sh -cp .:Core:CD:DeltaBlue:Havlak:Json:NBody:Richards:../../implementations/TruffleSOM/Smalltalk Harness.som Richards 10 100', 35 | '/data/home/gitlab-runner/builds/71gxYod2/0/sm951/benchmark-runs/som-ast-interp -cp Smalltalk:Examples/Benchmarks/LanguageFeatures:Examples/Benchmarks/TestSuite Examples/Benchmarks/BenchmarkHarness.som Fibonacci 1 0 10', 36 | '/data/home/gitlab-runner/builds/d258e35c/0/sm951/SOMns/som -G -t1 core-lib/Benchmarks/AsyncHarness.ns Savina.AStarSearch 50 0 100:10', 37 | '/home/gitlab-runner/builds/d258e35c/0/sm951/SOMpp/SOM++ -cp Smalltalk:Examples/Benchmarks/Richards:Examples/Benchmarks/DeltaBlue:Examples/Benchmarks/NBody:Examples/Benchmarks/Json:Examples/Benchmarks/GraphSearch Examples/Benchmarks/BenchmarkHarness.som DeltaBlue 10 0 50' 38 | ]; 39 | 40 | const simplified = somCmdLines.map((c) => simplifyCmdline(c)); 41 | 42 | for (const c of simplified) { 43 | expect(c).toMatch(/^som/i); 44 | } 45 | 46 | expect(simplifyCmdline('generate-report')).toBe('generate-report'); 47 | expect(simplifyCmdline('get-exp-data')).toBe('get-exp-data'); 48 | expect(simplifyCmdline('python3.10 harness.py List 25 1500')).toBe( 49 | 'python3.10 harness.py List 25 1500' 50 | ); 51 | }); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /src/frontend/theme.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Based on: 3 | * Color mode toggler for Bootstrap's docs (https://getbootstrap.com/) 4 | * Copyright 2011-2024 The Bootstrap Authors 5 | * Licensed under the Creative Commons Attribution 3.0 Unported License. 6 | */ 7 | 8 | function getStoredTheme(): string | null { 9 | return localStorage.getItem('theme'); 10 | } 11 | 12 | function setStoredTheme(theme: string) { 13 | localStorage.setItem('theme', theme); 14 | } 15 | 16 | function getSystemPreferredTheme(): string { 17 | return window.matchMedia('(prefers-color-scheme: dark)').matches 18 | ? 'dark' 19 | : 'light'; 20 | } 21 | 22 | function getPreferredTheme(): string { 23 | const storedTheme = getStoredTheme(); 24 | if (storedTheme) { 25 | return storedTheme; 26 | } 27 | 28 | return getSystemPreferredTheme(); 29 | } 30 | 31 | function setTheme(theme: string) { 32 | document.documentElement.setAttribute('data-bs-theme', theme); 33 | } 34 | 35 | function showActiveTheme(theme: string) { 36 | const themeSwitcher = document.querySelector('#theme-switcher'); 37 | 38 | if (!themeSwitcher) { 39 | return; 40 | } 41 | 42 | const themeIcons = document.querySelectorAll('.theme-icon'); 43 | for (const themeIcon of themeIcons) { 44 | themeIcon.classList.remove('active'); 45 | } 46 | 47 | const iconToActivate = document.querySelector(`#theme-icon-${theme}`); 48 | 49 | if (!iconToActivate) { 50 | return; 51 | } 52 | 53 | iconToActivate.classList.add('active'); 54 | } 55 | 56 | function toggleTheme() { 57 | const themeIcons = document.querySelectorAll('.theme-icon'); 58 | 59 | let currentTheme = getPreferredTheme(); // that's the fallback 60 | 61 | // let's see what the current user setting is 62 | for (const themeIcon of themeIcons) { 63 | if (themeIcon.classList.contains('active')) { 64 | if (themeIcon.id === 'theme-icon-light') { 65 | currentTheme = 'light'; 66 | } else { 67 | currentTheme = 'dark'; 68 | } 69 | break; 70 | } 71 | } 72 | 73 | // flip the theme 74 | const newTheme = currentTheme === 'light' ? 'dark' : 'light'; 75 | 76 | setStoredTheme(newTheme); 77 | setTheme(newTheme); 78 | showActiveTheme(newTheme); 79 | } 80 | 81 | (() => { 82 | setTheme(getPreferredTheme()); 83 | 84 | window 85 | .matchMedia('(prefers-color-scheme: dark)') 86 | .addEventListener('change', () => { 87 | const newTheme = getSystemPreferredTheme(); 88 | setTheme(newTheme); 89 | showActiveTheme(newTheme); 90 | }); 91 | 92 | window.addEventListener('DOMContentLoaded', () => { 93 | showActiveTheme(getPreferredTheme()); 94 | 95 | const themeSwitcher = document.querySelector('#theme-switcher'); 96 | if (!themeSwitcher) { 97 | return; 98 | } 99 | 100 | themeSwitcher.addEventListener('click', toggleTheme); 101 | }); 102 | })(); 103 | -------------------------------------------------------------------------------- /src/backend/project/project.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ReBench: {%= it.name %} 6 | 7 | 8 | 9 | {%- include('header.html', { rebenchVersion: it.rebenchVersion }) %} 10 | 11 | 12 | 13 | 14 |
15 |
16 |

{%= it.name %}

17 | {% if (it.description) { %} 18 |

{%= it.description%}

19 | {% } %} 20 |
21 | {%- include('common-menu.html', it) %} 22 |
23 | 24 |
25 |
26 |
27 | 28 |
29 |
30 | Most Used 31 | Alphabetical 32 | Most Recent 33 |
34 |
35 |
36 |
37 |
{%= it.name %}
39 |
40 | {% if (it.showchanges) { %} 41 |
Changes
42 |
43 |
44 |
46 |
47 |
48 |
49 |
50 |
51 | 52 | Compare 53 | {% } %} 54 | {% if (it.allresults) { %} 55 |
56 | {% } %} 57 |
58 | Timeline 59 |
60 |
61 |
62 |
63 | 64 |
65 |
66 | Most Used 67 | Alphabetical 68 | Most Recent 69 |
70 |
71 |
72 |
73 | 74 | 75 | -------------------------------------------------------------------------------- /src/shared/aesthetics.ts: -------------------------------------------------------------------------------- 1 | import type { CanvasSettings } from '../backend/compare/charts.js'; 2 | 3 | const inlinePlot: CanvasSettings = { 4 | width: 336, 5 | height: 38, 6 | outputType: 'svg', 7 | plotType: 'boxplot' 8 | }; 9 | 10 | const allTangoColorsAsString = ` 11 | 2e3436 555753 888a85 babdb6 d3d7cf ecf0eb f7f8f5 12 | 291e00 725000 c4a000 edd400 fce94f fffc9c feffd0 13 | 301700 8c3700 ce5c00 f57900 fcaf3e ffd797 fff0d7 14 | 271700 503000 8f5902 c17d11 e9b96e efd0a7 faf0d7 15 | 173000 2a5703 4e9a06 73d216 8ae234 b7f774 e4ffc7 16 | 00202a 0a3050 204a87 3465a4 729fcf 97c4f0 daeeff 17 | 170720 371740 5c3566 75507b ad7fa8 e0c0e4 fce0ff 18 | 270000 600000 a40000 cc0000 ef2929 f78787 ffcccc`; 19 | 20 | const allTangoShadesPerColor = allTangoColorsAsString 21 | .split('\n') 22 | .map((line) => line.split(' ')); 23 | 24 | // colors are from the Extended Tango Palette 25 | // https://emilis.info/other/extended_tango/ 26 | export const siteAesthetics = { 27 | changeColor: '#e9b96e', 28 | baseColor: '#729fcf', 29 | 30 | changeColorLight: '#efd0a7', 31 | baseColorLight: '#97c4f0', 32 | 33 | fastColor: '#e4ffc7', 34 | slowColor: '#ffcccc', 35 | 36 | backgroundColor: '#ffffff', 37 | 38 | overviewPlotWidth: 432, 39 | 40 | inlinePlot, 41 | 42 | baselineColorGradient: [ 43 | // '#00202a', '#0a3050', 44 | '#204a87', 45 | '#3465a4', 46 | '#729fcf', 47 | '#97c4f0' 48 | // '#daeeff' 49 | ], 50 | changeColorGradient: [ 51 | // '#271700', '#503000', 52 | '#8f5902', 53 | '#c17d11', 54 | '#e9b96e', 55 | '#efd0a7' 56 | // '#faf0d7' 57 | ], 58 | exeColors: [ 59 | // the colors are from the Extended Tango Palette, columns 4-6. 60 | // the order is randomized 61 | '#f78787', 62 | '#3465a4', 63 | '#ffd797', 64 | 65 | '#ad7fa8', 66 | '#b7f774', 67 | '#ef2929', 68 | 69 | '#e9b96e', 70 | '#729fcf', 71 | '#8ae234', 72 | 73 | '#edd400', 74 | '#fce94f', 75 | '#cc0000', 76 | 77 | '#fffc9c', 78 | '#75507b', 79 | '#efd0a7', 80 | 81 | '#73d216', 82 | '#fcaf3e', 83 | '#c17d11', 84 | 85 | '#97c4f0', 86 | '#f57900', 87 | '#e0c0e4' 88 | ], 89 | lighten(color: string): string { 90 | const colorWithoutHash = color[0] === '#' ? color.slice(1) : color; 91 | 92 | for (const colorShades of allTangoShadesPerColor) { 93 | const index = colorShades.indexOf(colorWithoutHash); 94 | if (index !== -1) { 95 | const nextIndex = Math.min(index + 1, colorShades.length - 1); 96 | return `#${colorShades[nextIndex]}`; 97 | } 98 | } 99 | throw Error(`Color ${color} not found in allTangoShadesPerColor`); 100 | }, 101 | getColorsForExecutors(executors: Set): Map { 102 | const colors = new Map(); 103 | let i = 0; 104 | for (const exe of executors) { 105 | colors.set(exe, this.exeColors[i]); 106 | i++; 107 | } 108 | return colors; 109 | } 110 | } as const; 111 | -------------------------------------------------------------------------------- /tests/backend/perf-tracker.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, beforeAll, afterAll, it } from '@jest/globals'; 2 | import { 3 | TestDatabase, 4 | closeMainDb, 5 | createAndInitializeDB 6 | } from './db/db-testing.js'; 7 | import { 8 | _completeRequest, 9 | initPerfTracker, 10 | startRequest 11 | } from '../../src/backend/perf-tracker.js'; 12 | 13 | describe('Test basic performance self-tracking', () => { 14 | let db: TestDatabase; 15 | 16 | beforeAll(async () => { 17 | db = await createAndInitializeDB('perf_tracker', 0, false, false); 18 | }); 19 | 20 | it('should start with empty trials etc tables', async () => { 21 | let result = await db.query({ text: 'SELECT * FROM Trial' }); 22 | expect(result.rowCount).toEqual(0); 23 | 24 | result = await db.query({ text: 'SELECT * FROM Experiment' }); 25 | expect(result.rowCount).toEqual(0); 26 | 27 | result = await db.query({ text: 'SELECT * FROM Run' }); 28 | expect(result.rowCount).toEqual(0); 29 | 30 | result = await db.query({ text: 'SELECT * FROM Measurement' }); 31 | expect(result.rowCount).toEqual(0); 32 | }); 33 | 34 | it('should create trial on initialization', async () => { 35 | await initPerfTracker(db); 36 | 37 | let result = await db.query({ text: 'SELECT * FROM Trial' }); 38 | expect(result.rowCount).toEqual(1); 39 | expect(result.rows[0].starttime).not.toBeNull(); 40 | 41 | result = await db.query({ text: 'SELECT * FROM Experiment' }); 42 | expect(result.rowCount).toEqual(1); 43 | expect(result.rows[0].name).toEqual('monitoring'); 44 | 45 | result = await db.query({ text: 'SELECT * FROM Run' }); 46 | expect(result.rowCount).toEqual(11); 47 | 48 | result = await db.query({ text: 'SELECT * FROM Measurement' }); 49 | expect(result.rowCount).toEqual(0); 50 | }); 51 | 52 | it('should create measurement on demand', async () => { 53 | const time = startRequest(); 54 | await _completeRequest(time, db, 'get-results'); 55 | 56 | const result = await db.query({ text: 'SELECT * FROM Measurement' }); 57 | expect(result.rowCount).toEqual(1); 58 | expect(result.rows[0].invocation).toEqual(1); 59 | expect(result.rows[0].values).toHaveLength(1); 60 | expect(typeof result.rows[0].values[0]).toBe('number'); 61 | }); 62 | 63 | it('should append to the array on subsequent calls', async () => { 64 | const time = startRequest(); 65 | await _completeRequest(time, db, 'get-results'); 66 | await _completeRequest(time, db, 'get-results'); 67 | 68 | const result = await db.query({ text: 'SELECT * FROM Measurement' }); 69 | expect(result.rowCount).toEqual(1); 70 | expect(result.rows[0].invocation).toEqual(1); 71 | expect(result.rows[0].values).toHaveLength(3); 72 | expect(typeof result.rows[0].values[0]).toBe('number'); 73 | expect(typeof result.rows[0].values[1]).toBe('number'); 74 | expect(typeof result.rows[0].values[2]).toBe('number'); 75 | }); 76 | 77 | afterAll(async () => { 78 | return db.close(); 79 | }); 80 | }); 81 | 82 | afterAll(async () => { 83 | return closeMainDb(); 84 | }); 85 | -------------------------------------------------------------------------------- /src/backend/dev-server/server.ts: -------------------------------------------------------------------------------- 1 | import { ParameterizedContext } from 'koa'; 2 | import { readFileSync } from 'node:fs'; 3 | 4 | import { log } from '../logging.js'; 5 | import { robustPath, robustSrcPath, siteConfig } from '../util.js'; 6 | 7 | export async function serveStaticResource( 8 | ctx: ParameterizedContext 9 | ): Promise { 10 | const filename = ctx.params.filename; 11 | log.debug(`serve ${filename}`); 12 | let path: string; 13 | 14 | if (filename.endsWith('.css')) { 15 | ctx.type = 'css'; 16 | path = robustPath(`../resources/${filename}`); 17 | } else if (filename.endsWith('.js')) { 18 | ctx.type = 'application/javascript'; 19 | if (filename.includes('uPlot')) { 20 | path = robustPath(`../resources/${filename}`); 21 | } else { 22 | path = robustSrcPath(`frontend/${filename}`); 23 | } 24 | } else if (filename.endsWith('.map')) { 25 | ctx.type = 'application/json'; 26 | path = robustSrcPath(`frontend/${filename}`); 27 | } else if (filename.endsWith('.svg')) { 28 | ctx.type = 'image/svg+xml'; 29 | path = robustPath(`../resources/${filename}`); 30 | } else if (filename.endsWith('.json.gz')) { 31 | ctx.type = 'application/json'; 32 | ctx.set('Content-Encoding', 'gzip'); 33 | path = `${siteConfig.dataExportPath}/${filename}`; 34 | } else if (filename.endsWith('.csv.gz')) { 35 | ctx.type = 'text/csv'; 36 | ctx.set('Content-Encoding', 'gzip'); 37 | path = `${siteConfig.dataExportPath}/${filename}`; 38 | } else { 39 | throw new Error(`Unsupported file type. Filename: ${filename}`); 40 | } 41 | ctx.body = readFileSync(path); 42 | } 43 | 44 | export async function serveStaticSharedResource( 45 | ctx: ParameterizedContext 46 | ): Promise { 47 | const filename = ctx.params.filename; 48 | log.debug(`serve ${filename}`); 49 | let path: string; 50 | 51 | if (filename.endsWith('.js')) { 52 | ctx.type = 'application/javascript'; 53 | path = robustSrcPath(`shared/${filename}`); 54 | } else if (filename.endsWith('.map')) { 55 | ctx.type = 'application/json'; 56 | path = robustSrcPath(`shared/${filename}`); 57 | } else { 58 | throw new Error(`Unsupported file type. Filename: ${filename}`); 59 | } 60 | ctx.body = readFileSync(path); 61 | } 62 | 63 | export async function serveViewJs(ctx: ParameterizedContext): Promise { 64 | log.debug(`serve ${ctx.params.filename}`); 65 | let path: string; 66 | if (ctx.params.filename.endsWith('.ts')) { 67 | ctx.type = 'application/typescript'; 68 | path = robustPath(`frontend/${ctx.params.filename}`); 69 | } else { 70 | throw new Error(`Unsupported file type ${ctx.params.filename}`); 71 | } 72 | ctx.body = readFileSync(path); 73 | } 74 | 75 | export async function serveReport(ctx: ParameterizedContext): Promise { 76 | log.debug(`serve ${ctx.params.filename}`); 77 | const reportPath = robustPath(`../resources/reports`); 78 | ctx.body = readFileSync( 79 | `${reportPath}/${ctx.params.change}/figure-html/${ctx.params.filename}` 80 | ); 81 | if (ctx.params.filename.endsWith('.svg')) { 82 | ctx.type = 'svg'; 83 | } else if (ctx.params.filename.endsWith('.css')) { 84 | ctx.type = 'text/css'; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/backend/compare/html/gen-index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ReBenchDB for {%= it.project %}: Comparing {%= it.baselineHash6 %} with {%= it.changeHash6 %} 6 | 7 | {%- include('header.html', { rebenchVersion: it.rebenchVersion }) %} 8 | 9 | {% if (it.generatingReport) { %} 10 | 11 | {% } %} 12 | 13 | 14 | 15 |
16 |
17 |

ReBenchDB for {%= it.project %}

18 | {% if (it.revisionFound) { %} 19 |

Comparing {%= it.baselineHash6%} with {%= it.changeHash6%}

20 | {% } else { %} 21 |

Comparing {%= it.baselineHash6%} with {%= it.changeHash6%}

22 | {% } %} 23 |
24 | {%- include('common-menu.html', it) %} 25 |
26 | 27 | {% if (it.revisionFound) { %} 28 |
29 |

Version Details

30 |
31 |
Baseline
32 |
33 | {%= it.baselineHash6%} {%= it.base.branchortag%}
34 | {%= it.base.authorname%} 35 | 
{%= it.base.commitmessage%}
36 | 37 | {%= it.base.name%} 38 |
39 |
Change
40 |
41 | {%= it.changeHash6%} {%= it.change.branchortag%}
42 | {%= it.change.authorname%} 43 | 
{%= it.change.commitmessage%}
44 | 45 | {%= it.change.name%} 46 |
47 |
Significant Change
48 |
49 | 50 | 51 |
52 |
53 |
54 | {% } %} 55 | 56 | {% if (it.generatingReport) { %} 57 | 69 | {% } %} 70 | 71 | {% if (it.generationFailed) { %} 72 | 78 | {% } %} 79 | 80 | 81 | -------------------------------------------------------------------------------- /tests/payload.ts: -------------------------------------------------------------------------------- 1 | import { readFileSync } from 'node:fs'; 2 | import { convertToCurrentApi } from '../src/backend/common/api-v1.js'; 3 | import { robustPath } from '../src/backend/util.js'; 4 | 5 | import type { BenchmarkData } from '../src/shared/api.js'; 6 | import type { 7 | MeasurementData, 8 | MeasurementDataOld 9 | } from '../src/backend/db/types.js'; 10 | import { assert } from '../src/backend/logging.js'; 11 | 12 | export function loadSmallPayload(): BenchmarkData { 13 | return JSON.parse( 14 | readFileSync(robustPath('../tests/data/small-payload.json')).toString() 15 | ); 16 | } 17 | 18 | export function loadLargePayload(): BenchmarkData { 19 | const testData = JSON.parse( 20 | readFileSync(robustPath('../tests/data/large-payload.json')).toString() 21 | ); 22 | 23 | return convertToCurrentApi(testData); 24 | } 25 | 26 | export function loadLargePayloadApiV1(): any { 27 | return JSON.parse( 28 | readFileSync(robustPath('../tests/data/large-payload.json')).toString() 29 | ); 30 | } 31 | 32 | function convertMeasurementDataToCurrentApi( 33 | oldMs: MeasurementDataOld[] 34 | ): MeasurementData[] { 35 | const result: MeasurementData[] = []; 36 | 37 | let lastMD: MeasurementData | null = null; 38 | let lastExpId = -1; 39 | let lastRunId = -1; 40 | let lastTrialId = -1; 41 | let lastCriterion = ''; 42 | let lastInvocation = -1; 43 | 44 | for (const oldM of oldMs) { 45 | if ( 46 | oldM.expid !== lastExpId || 47 | oldM.runid !== lastRunId || 48 | oldM.trialid !== lastTrialId || 49 | oldM.criterion !== lastCriterion || 50 | oldM.invocation !== lastInvocation 51 | ) { 52 | lastMD = { 53 | expid: oldM.expid, 54 | runid: oldM.runid, 55 | trialid: oldM.trialid, 56 | commitid: oldM.commitid, 57 | bench: oldM.bench, 58 | exe: oldM.exe, 59 | suite: oldM.suite, 60 | cmdline: oldM.cmdline, 61 | varvalue: oldM.varvalue, 62 | cores: oldM.cores, 63 | inputsize: oldM.inputsize, 64 | extraargs: oldM.extraargs, 65 | invocation: oldM.invocation, 66 | warmup: oldM.warmup, 67 | criterion: oldM.criterion, 68 | unit: oldM.unit, 69 | values: [], 70 | envid: oldM.envid 71 | }; 72 | 73 | result.push(lastMD); 74 | lastExpId = oldM.expid; 75 | lastRunId = oldM.runid; 76 | lastTrialId = oldM.trialid; 77 | lastCriterion = oldM.criterion; 78 | lastInvocation = oldM.invocation; 79 | } 80 | assert(lastMD!.values[oldM.iteration - 1] == null, 'iteration already set'); 81 | lastMD!.values[oldM.iteration - 1] = oldM.value; 82 | } 83 | 84 | return result; 85 | } 86 | 87 | export function loadCompareViewJsSomPayload(): MeasurementData[] { 88 | const testData = JSON.parse( 89 | readFileSync( 90 | robustPath('../tests/data/compare-view-data-jssom.json') 91 | ).toString() 92 | ).results; 93 | 94 | return convertMeasurementDataToCurrentApi(testData); 95 | } 96 | 97 | export function loadCompareViewTSomPayload(): MeasurementData[] { 98 | const testData = JSON.parse( 99 | readFileSync( 100 | robustPath('../tests/data/compare-view-data-trufflesom.json') 101 | ).toString() 102 | ).results; 103 | 104 | return convertMeasurementDataToCurrentApi(testData); 105 | } 106 | -------------------------------------------------------------------------------- /src/backend/rebench/results.ts: -------------------------------------------------------------------------------- 1 | import { ParameterizedContext } from 'koa'; 2 | import { ValidateFunction } from 'ajv'; 3 | 4 | import { BenchmarkData } from '../../shared/api.js'; 5 | import { Database } from '../db/db.js'; 6 | import { createValidator } from './api-validator.js'; 7 | import { DEBUG } from '../util.js'; 8 | import { log } from '../logging.js'; 9 | import { 10 | completeRequestAndHandlePromise, 11 | startRequest 12 | } from '../perf-tracker.js'; 13 | 14 | const validateFn: ValidateFunction = DEBUG ? createValidator() : undefined; 15 | 16 | const rebenchdbApiVersion = '2.0.0'; 17 | 18 | function validateSchema(data: BenchmarkData, ctx: ParameterizedContext) { 19 | const result = validateFn(data); 20 | if (!result) { 21 | log.error('Data validation failed.', validateFn.errors); 22 | ctx.status = 500; 23 | ctx.body = `Request does not validate: 24 | ${validateFn.errors}`; 25 | } else { 26 | log.debug('Data validated successfully.'); 27 | } 28 | } 29 | 30 | export async function reportResultApiVersion( 31 | ctx: ParameterizedContext 32 | ): Promise { 33 | ctx.set('X-ReBenchDB-Result-API-Version', rebenchdbApiVersion); 34 | ctx.set('Allow', 'PUT'); 35 | ctx.status = 200; 36 | ctx.body = ''; 37 | } 38 | 39 | function isUsingV2Api(data: BenchmarkData): boolean { 40 | if (data.data.length === 0) { 41 | return true; // no data, no problem 42 | } 43 | 44 | const firstRun = data.data[0]; 45 | if (!firstRun.d || firstRun.d.length === 0) { 46 | return true; // no data, no problem 47 | } 48 | 49 | const firstDataPoint = firstRun.d[0]; 50 | 51 | // the old API had an 'it' field, for the iteration number 52 | return !Object.hasOwn(firstDataPoint, 'it'); 53 | } 54 | 55 | export async function acceptResultData( 56 | ctx: ParameterizedContext, 57 | db: Database 58 | ): Promise { 59 | const start = startRequest(); 60 | 61 | const data: BenchmarkData = await ctx.request.body; 62 | ctx.type = 'text'; 63 | 64 | if (DEBUG) { 65 | validateSchema(data, ctx); 66 | } 67 | 68 | if (!data.startTime) { 69 | ctx.body = `Request misses a startTime setting, 70 | which is needed to store results correctly.`; 71 | ctx.status = 400; 72 | return; 73 | } 74 | 75 | if (!isUsingV2Api(data)) { 76 | log.info(`/rebenchdb/results: Request with old API version`); 77 | ctx.body = `Only API version ${rebenchdbApiVersion} is supported.`; 78 | ctx.status = 400; // Bad Request 79 | return; 80 | } 81 | 82 | try { 83 | const recRunsPromise = db.recordMetaDataAndRuns(data); 84 | log.info(`/rebenchdb/results: Content-Length=${ctx.request.length}`); 85 | const recordedRuns = await recRunsPromise; 86 | db.recordAllData(data) 87 | .then(([recMs, recPs]) => 88 | log.info( 89 | // eslint-disable-next-line max-len 90 | `/rebenchdb/results: stored ${recMs} sets of measurements, ${recPs} profiles` 91 | ) 92 | ) 93 | .catch((e) => { 94 | log.error('/rebenchdb/results failed to store measurements:', e.stack); 95 | }); 96 | 97 | ctx.body = 98 | `Meta data for ${recordedRuns} stored.` + 99 | ' Storing of measurements is ongoing'; 100 | ctx.status = 201; 101 | } catch (e: any) { 102 | ctx.status = 500; 103 | ctx.body = `${e.stack}`; 104 | log.error(e, e.stack); 105 | } 106 | 107 | completeRequestAndHandlePromise(start, db, 'put-results'); 108 | } 109 | -------------------------------------------------------------------------------- /src/benchmarks/compute-timeline.ts: -------------------------------------------------------------------------------- 1 | import { convertToCurrentApi } from '../backend/common/api-v1.js'; 2 | import { 3 | BatchingTimelineUpdater, 4 | ComputeJob 5 | } from '../backend/timeline/timeline-calc.js'; 6 | import { RebenchDbBenchmark } from './rebenchdb-benchmark.js'; 7 | 8 | export default class ComputeTimeline extends RebenchDbBenchmark { 9 | private updater: BatchingTimelineUpdater | null = null; 10 | private jobs: ComputeJob[] | null = null; 11 | 12 | public async oneTimeSetup(problemSize: string): Promise { 13 | this.enableTimeline = true; 14 | await super.oneTimeSetup(problemSize); 15 | 16 | if (!this.db) { 17 | throw new Error('Database is not initialized'); 18 | } 19 | 20 | if (problemSize === 'full') { 21 | // just use the testData as is 22 | } else if (problemSize === 'large') { 23 | this.testData.data.length = 50; 24 | } else if (problemSize === 'medium') { 25 | this.testData.data.length = 20; 26 | for (const run of this.testData.data) { 27 | if (run.d) { 28 | run.d.length = 200; 29 | } 30 | } 31 | } else if (problemSize === 'small') { 32 | this.testData.data.length = 10; 33 | for (const run of this.testData.data) { 34 | if (run.d) { 35 | run.d.length = 15; 36 | } 37 | } 38 | } else { 39 | throw new Error('Unsupported problem size given: ' + problemSize); 40 | } 41 | (this).testData = convertToCurrentApi(this.testData); 42 | 43 | this.testData.experimentName = 'Benchmark 1'; 44 | this.testData.source.commitId = 'commit-1'; 45 | await this.db.recordAllData(this.testData, true); 46 | 47 | this.testData.experimentName = 'Benchmark 2'; 48 | this.testData.source.commitId = 'commit-2'; 49 | await this.db.recordAllData(this.testData, true); 50 | 51 | this.updater = this.db.getTimelineUpdater(); 52 | this.jobs = this.updater!.consumeUpdateJobsForBenchmarking(); 53 | } 54 | 55 | public async benchmark(): Promise { 56 | if (!this.updater || !this.jobs) { 57 | throw new Error('Timeline updater not initialized'); 58 | } 59 | if (!this.db) { 60 | throw new Error('Database is not initialized'); 61 | } 62 | 63 | // the processStart being 1 is just here for a consistent start time 64 | // that's not zero 65 | const numJobs = await this.updater.processUpdateJobsForBenchmarking( 66 | this.jobs, 67 | 1 68 | ); 69 | 70 | const result = await this.db.query({ 71 | text: `SELECT count(*) FROM Timeline` 72 | }); 73 | 74 | await this.db.query({ text: `TRUNCATE Timeline` }); 75 | 76 | return { 77 | timelineEntries: result?.rows[0], 78 | numJobs 79 | }; 80 | } 81 | 82 | public verifyResult(result: any): boolean { 83 | if (this.problemSize === 'small') { 84 | return result.numJobs === 20 && result.timelineEntries.count === '20'; 85 | } 86 | 87 | if (this.problemSize === 'medium') { 88 | return result.numJobs === 40 && result.timelineEntries.count === '40'; 89 | } 90 | 91 | if (this.problemSize === 'large') { 92 | return result.numJobs === 100 && result.timelineEntries.count === '100'; 93 | } 94 | 95 | if (this.problemSize === 'full') { 96 | return result.numJobs === 632 && result.timelineEntries.count === '632'; 97 | } 98 | 99 | throw new Error('not yet supported problem size ' + this.problemSize); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/shared/data-format.ts: -------------------------------------------------------------------------------- 1 | import type { BenchmarkId } from '../shared/api.js'; 2 | import type { Environment } from '../backend/db/types.js'; 3 | 4 | /** 5 | * Round to 0 decimal places. 6 | */ 7 | export function r0(val: number): string { 8 | const result = val.toFixed(0); 9 | if (result === '-0') { 10 | return '0'; 11 | } 12 | return result; 13 | } 14 | 15 | /** 16 | * Round to 2 decimal places. 17 | */ 18 | export function r2(val: number): string { 19 | const result = val.toFixed(2); 20 | if (result === '-0.00') { 21 | return '0.00'; 22 | } 23 | return result; 24 | } 25 | 26 | /** 27 | * As percentage. 28 | */ 29 | export function per(val: number): string { 30 | return r0(val * 100); 31 | } 32 | 33 | /** 34 | * As memory value rounded to an appropriate unit. 35 | */ 36 | export function asHumanMem(val: number, digits = 0): string { 37 | if (isNaN(val)) { 38 | return ''; 39 | } 40 | 41 | let m = val; 42 | const mem = ['b', 'kb', 'MB', 'GB']; 43 | let i = 0; 44 | while (i < 3 && m >= 1024) { 45 | m = m / 1024; 46 | i += 1; 47 | } 48 | 49 | return `${m.toFixed(digits)}${mem[i]}`; 50 | } 51 | 52 | /** 53 | * As frequency value rounded to an appropriate unit. 54 | */ 55 | export function asHumanHz(val: number, digits = 0): string { 56 | if (isNaN(val) || typeof val !== 'number') { 57 | return ''; 58 | } 59 | 60 | let h = val; 61 | const hz = ['Hz', 'kHz', 'MHz', 'GHz']; 62 | let i = 0; 63 | while (i < 3 && h >= 1000) { 64 | h = h / 1000; 65 | i += 1; 66 | } 67 | 68 | return `${h.toFixed(digits)}${hz[i]}`; 69 | } 70 | 71 | /** 72 | * Return a string with the environment information for display. 73 | */ 74 | export function formatEnvironment( 75 | envId: number, 76 | environments: Environment[] 77 | ): string | undefined { 78 | const env = environments.find((e) => e.id === envId); 79 | if (env === undefined) { 80 | return undefined; 81 | } 82 | return `${env.hostname} | ${env.ostype} | ${asHumanMem(env.memory)} | ${ 83 | env.cpu 84 | } | ${asHumanHz(env.clockspeed)}`; 85 | } 86 | 87 | /** 88 | * Return a minimal object identifying the run id. 89 | */ 90 | export function benchmarkId( 91 | benchmarkName: string, 92 | exeName: string, 93 | suiteName: string, 94 | varValue: string, 95 | numVarValues: number, 96 | cores: string, 97 | numCores: number, 98 | inputSize: string, 99 | numInputSizes: number, 100 | extraArgs: string, 101 | numExtraArgs: number 102 | ): BenchmarkId { 103 | const obj: BenchmarkId = { b: benchmarkName, e: exeName, s: suiteName }; 104 | 105 | if (numVarValues > 1) { 106 | obj.v = varValue; 107 | } 108 | if (numCores > 1) { 109 | obj.c = cores; 110 | } 111 | if (numInputSizes > 1) { 112 | obj.i = inputSize; 113 | } 114 | if (numExtraArgs > 1) { 115 | obj.ea = extraArgs; 116 | } 117 | return obj; 118 | } 119 | 120 | /** 121 | * Turn the text into lower case, but keep abbreviations in upper case. 122 | */ 123 | export function smartLower(text: string): string { 124 | const words = text.split(' '); 125 | const result: string[] = []; 126 | for (const word of words) { 127 | if (word.length > 1 && word === word.toUpperCase()) { 128 | result.push(word); 129 | } else { 130 | result.push(word.toLowerCase()); 131 | } 132 | } 133 | return result.join(' '); 134 | } 135 | 136 | export function shortenCriteria(text: string): string { 137 | const words = text.split(' '); 138 | const result: string[] = []; 139 | for (const word of words) { 140 | // skip the word 'time' 141 | if (word.toLowerCase() === 'time') { 142 | continue; 143 | } 144 | result.push(word); 145 | } 146 | return result.join(' '); 147 | } 148 | -------------------------------------------------------------------------------- /tests/backend/rebench/api.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, beforeAll, it } from '@jest/globals'; 2 | import { readFileSync } from 'node:fs'; 3 | import { ValidateFunction } from 'ajv'; 4 | 5 | import { createValidator } from '../../../src/backend/rebench/api-validator.js'; 6 | import { robustPath } from '../../../src/backend/util.js'; 7 | import { assert, log } from '../../../src/backend/logging.js'; 8 | import { 9 | loadLargePayload, 10 | loadLargePayloadApiV1, 11 | loadSmallPayload 12 | } from '../../payload.js'; 13 | import type { BenchmarkData } from '../../../src/shared/api.js'; 14 | import type { DataPointV1 } from '../../../src/backend/common/api-v1.js'; 15 | 16 | describe('Ensure Test Payloads conform to API', () => { 17 | let validateFn: ValidateFunction; 18 | 19 | beforeAll(() => { 20 | validateFn = createValidator(); 21 | }); 22 | 23 | it('should validate small-payload.json', () => { 24 | const testData = loadSmallPayload(); 25 | 26 | const result = validateFn(testData); 27 | if (!result) { 28 | log.error(validateFn.errors); 29 | } 30 | expect(result).toBeTruthy(); 31 | }); 32 | 33 | it('should validate large-payload.json', () => { 34 | const testData = loadLargePayload(); 35 | 36 | const result = validateFn(testData); 37 | if (!result) { 38 | log.error(validateFn.errors); 39 | } 40 | expect(result).toBeTruthy(); 41 | }); 42 | 43 | it('should validate profile-payload.json', () => { 44 | const testData = JSON.parse( 45 | readFileSync(robustPath('../tests/data/profile-payload.json')).toString() 46 | ); 47 | 48 | const result = validateFn(testData); 49 | if (!result) { 50 | log.error(validateFn.errors); 51 | } 52 | expect(result).toBeTruthy(); 53 | }); 54 | 55 | function countValues(data: BenchmarkData): number { 56 | let count = 0; 57 | for (const run of data.data) { 58 | if (!run.d) { 59 | continue; 60 | } 61 | for (const dp of run.d) { 62 | for (const criterion of dp.m) { 63 | if (criterion !== null) { 64 | assert(criterion !== undefined); 65 | for (const value of criterion) { 66 | if (value !== null) { 67 | assert(typeof value === 'number'); 68 | count += 1; 69 | } 70 | } 71 | } 72 | } 73 | } 74 | } 75 | return count; 76 | } 77 | 78 | function countValuesApiV1(data: any): number { 79 | let count = 0; 80 | for (const run of data.data) { 81 | if (!run.d) { 82 | continue; 83 | } 84 | for (const dp of run.d) { 85 | const dpV1 = dp as DataPointV1; 86 | for (const m of dpV1.m) { 87 | assert(m != undefined); 88 | assert(typeof m.v === 'number'); 89 | count += 1; 90 | } 91 | } 92 | } 93 | return count; 94 | } 95 | 96 | it('expected number of values in large-payload.json: 1st run', () => { 97 | const testData = loadLargePayload(); 98 | testData.data.splice(1); 99 | 100 | expect(countValues(testData)).toBe(2999); 101 | }); 102 | 103 | it('expected number of values in raw large-payload.json: 1st run', () => { 104 | const testData = loadLargePayloadApiV1(); 105 | testData.data.splice(1); 106 | expect(countValuesApiV1(testData)).toBe(2999); 107 | }); 108 | 109 | it('should give the expected number of values in large-payload.json', () => { 110 | const testData = loadLargePayload(); 111 | expect(testData.data).toHaveLength(316); 112 | expect(countValues(testData)).toBe(459928); 113 | }); 114 | 115 | it('should give expected number of values in raw large-payload.json', () => { 116 | const testData = loadLargePayloadApiV1(); 117 | expect(testData.data).toHaveLength(316); 118 | expect(countValuesApiV1(testData)).toBe(459928); 119 | }); 120 | }); 121 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ReBenchDB 2 | 3 | [![Build Status](https://travis-ci.com/smarr/ReBenchDB.svg?branch=master)](https://travis-ci.com/smarr/ReBenchDB) 4 | [![Documentation](https://readthedocs.org/projects/rebench/badge/?version=latest)](https://rebench.readthedocs.io/) 5 | [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1311762.svg)](https://doi.org/10.5281/zenodo.1311762) 6 | 7 | ReBenchDB records benchmark results and provides customizable reporting 8 | to track and analyze run-time performance of software programs. 9 | It is used for example to track the performance of multiple language implementations, 10 | including [SOMns](https://github.com/smarr/SOMns) and other implementations 11 | from the [family of Simple Object Machines](https://som-st.github.io/). 12 | [ReBench](https://github.com/smarr/ReBench) is currently the main benchmark 13 | runner to record data for ReBenchDB. 14 | 15 | ## Goals and Features 16 | 17 | ReBenchDB is designed to 18 | 19 | - record and store benchmark measurements 20 | - track information about the environment in which the benchmarks were executed 21 | - enable performance tracking over time 22 | - enable performance comparison of experiments 23 | 24 | ### Features 25 | 26 | - compare performance between specific commits 27 | - show aggregated results on an overview plot 28 | - give summary statistics 29 | - show per-benchmark details 30 | - a plot that allows to judge noise 31 | - a plot with per-iteration data 32 | - a plot of previous results 33 | - various metrics and the command line 34 | - profiling information 35 | - compare performance across different executors 36 | - per-project timeline view 37 | - per-project data inventory and data export 38 | 39 | ## Non-Goals 40 | 41 | ReBenchDB isn't 42 | 43 | - a benchmark runner or benchmarking framework: 44 | Check [ReBench](https://github.com/smarr/ReBench) if you need one 45 | - a statistics library: 46 | We currently use R for our statistic needs, but anything outputting HTML would be suitable. 47 | 48 | ## Docker/Podman Setup 49 | 50 | The repository contains a `Dockerfile` and a `docker-compose.yml`, which will 51 | install all dependencies and setup the required PostgreSQL database. 52 | 53 | With Docker, this should be usable with: 54 | 55 | ```bash 56 | docker compose -f ./docker-compose.yml up 57 | ``` 58 | 59 | For Podman users, podman-compose is needed: 60 | 61 | ```bash 62 | pip3 install podman-compose # if not already available 63 | podman-compose up 64 | ``` 65 | 66 | ## Installation and Usage 67 | 68 | 69 | 70 | ReBenchDB is implemented in TypeScript on top of Node.js. 71 | Data is stored in a PostgreSQL database. 72 | Custom reports are executed via a shell script, though, we use R to create 73 | reports. 74 | 75 | To ensure that the SVG for plots is generated using a font available in most browsers, 76 | the server needs them when generating plots. We currently use Arial as default font. 77 | 78 | On Ubuntu, these fonts can be installed with: 79 | 80 | ```bash 81 | apt install ttf-mscorefonts-installer 82 | ``` 83 | 84 | TODO: write a detailed description on what is required to set things up. 85 | 86 | ## Support and Contributions 87 | 88 | In case you encounter issues, 89 | please feel free to [open an issue](https://github.com/smarr/ReBenchDB/issues/new) 90 | so that we can help. 91 | 92 | For contributions, we use the [GitHub flow](https://guides.github.com/introduction/flow/) 93 | of pull requests, discussion, and revisions. For larger contributions, 94 | it is likely useful to discuss them upfront in an issue first. 95 | 96 | ## Similar Projects 97 | 98 | [Codespeed](https://github.com/tobami/codespeed/) "is a web application to monitor 99 | and analyze the performance of your code." We used it for almost 10 years before 100 | starting ReBenchDB. As such it provided a lot of inspiration and at this point, 101 | is a more mature and proven alternative. 102 | -------------------------------------------------------------------------------- /src/backend/project/project.ts: -------------------------------------------------------------------------------- 1 | import { ParameterizedContext } from 'koa'; 2 | import { prepareTemplate } from '../templates.js'; 3 | import { 4 | respondExpIdNotFound, 5 | respondProjectAndSourceNotFound, 6 | respondProjectIdNotFound, 7 | respondProjectNotFound 8 | } from '../common/standard-responses.js'; 9 | import { 10 | completeRequestAndHandlePromise, 11 | startRequest 12 | } from '../perf-tracker.js'; 13 | import { getExpData } from './data-export.js'; 14 | import { Database } from '../db/db.js'; 15 | import { rebenchVersion, robustPath } from '../../backend/util.js'; 16 | 17 | const projectHtml = prepareTemplate(robustPath('backend/project/project.html')); 18 | 19 | export async function renderProjectPage( 20 | ctx: ParameterizedContext, 21 | db: Database 22 | ): Promise { 23 | const project = await db.getProjectBySlug(ctx.params.projectSlug); 24 | if (project) { 25 | ctx.body = projectHtml({ ...project, rebenchVersion }); 26 | ctx.type = 'html'; 27 | } else { 28 | respondProjectNotFound(ctx, ctx.params.projectSlug); 29 | } 30 | } 31 | 32 | export async function getSourceAsJson( 33 | ctx: ParameterizedContext, 34 | db: Database 35 | ): Promise { 36 | const result = await db.getSourceById( 37 | ctx.params.projectSlug, 38 | ctx.params.sourceId 39 | ); 40 | 41 | if (result !== null) { 42 | ctx.body = result; 43 | ctx.type = 'application/json'; 44 | } else { 45 | respondProjectAndSourceNotFound( 46 | ctx, 47 | ctx.params.projectSlug, 48 | ctx.params.sourceId 49 | ); 50 | } 51 | } 52 | 53 | /** 54 | * @deprecated remove for 1.0 55 | */ 56 | export async function redirectToNewProjectDataUrl( 57 | ctx: ParameterizedContext, 58 | db: Database 59 | ): Promise { 60 | const project = await db.getProject(Number(ctx.params.projectId)); 61 | if (project) { 62 | ctx.redirect(`/${project.slug}/data`); 63 | } else { 64 | respondProjectIdNotFound(ctx, Number(ctx.params.projectId)); 65 | } 66 | ctx.type = 'html'; 67 | } 68 | 69 | const projectDataTpl = prepareTemplate( 70 | robustPath('backend/project/project-data.html'), 71 | false 72 | ); 73 | 74 | export async function renderProjectDataPage( 75 | ctx: ParameterizedContext, 76 | db: Database 77 | ): Promise { 78 | const project = await db.getProjectBySlug(ctx.params.projectSlug); 79 | if (project) { 80 | ctx.body = projectDataTpl({ project, rebenchVersion }); 81 | ctx.type = 'html'; 82 | } else { 83 | respondProjectNotFound(ctx, ctx.params.projectSlug); 84 | } 85 | } 86 | 87 | /** 88 | * @deprecated remove for 1.0 89 | */ 90 | export async function redirectToNewProjectDataExportUrl( 91 | ctx: ParameterizedContext, 92 | db: Database 93 | ): Promise { 94 | const project = await db.getProjectByExpId(Number(ctx.params.expId)); 95 | if (project) { 96 | ctx.redirect(`/${project.slug}/data/${ctx.params.expId}`); 97 | } else { 98 | respondExpIdNotFound(ctx, ctx.params.expId); 99 | } 100 | } 101 | 102 | const expDataTpl = prepareTemplate( 103 | robustPath('backend/project/get-exp-data.html'), 104 | false 105 | ); 106 | 107 | export async function renderDataExport( 108 | ctx: ParameterizedContext, 109 | db: Database 110 | ): Promise { 111 | const start = startRequest(); 112 | const format = ctx.params.expIdAndExtension.endsWith('.json.gz') 113 | ? 'json' 114 | : 'csv'; 115 | const expId = ctx.params.expIdAndExtension.replace(`.${format}.gz`, ''); 116 | 117 | const data = await getExpData( 118 | ctx.params.projectSlug, 119 | Number(expId), 120 | db, 121 | format 122 | ); 123 | 124 | if (data.preparingData) { 125 | ctx.body = expDataTpl({ ...data, rebenchVersion }); 126 | ctx.type = 'html'; 127 | ctx.set('Cache-Control', 'no-cache'); 128 | } else { 129 | ctx.redirect(data.downloadUrl); 130 | } 131 | 132 | completeRequestAndHandlePromise(start, db, 'get-exp-data'); 133 | } 134 | -------------------------------------------------------------------------------- /tests/data/expected-results/project/get-exp-data.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ReBenchDB Test Project: Preparing Data For Download 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 |

ReBenchDB for Test Project

20 |

Preparing Data for download of Test Experiment

21 |
22 |
23 |
24 | 33 |
34 |
35 |
36 | 37 | 51 | 52 | 53 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /tests/data/expected-results/project/project-data.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ReBench: Test Project 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 |
20 |
21 |

Test Project

22 | 23 |

desc

24 | 25 |
26 |
27 |
28 | 37 |
38 |
39 |
40 |
41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 65 | 66 |
ExperimentDescriptionStart/EndUserCommitMachine#runs#measurements
67 |
68 | 69 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | env: 6 | RDB_DB: postgres 7 | RDB_USER: postgres 8 | RDB_PASS: postgres 9 | DEBUG: true 10 | DB_HOST: localhost 11 | DB_PORT: 5432 # defined below in the services.postgres.ports (the first one) 12 | LANG: en_US.UTF-8 13 | LANGUAGE: en_US.UTF-8 14 | LC_ALL: en_US.UTF-8 15 | 16 | jobs: 17 | build-and-test: 18 | runs-on: ubuntu-latest 19 | 20 | services: 21 | # Label used to access the service container 22 | postgres: 23 | # Docker Hub image 24 | image: postgres 25 | env: 26 | POSTGRES_PASSWORD: ${{ env.RDB_PASS }} 27 | # Set health checks to wait until postgres has started 28 | options: >- 29 | --health-cmd pg_isready 30 | --health-interval 10s 31 | --health-timeout 5s 32 | --health-retries 5 33 | -v ${{ github.workspace }}:/postgres-export:rw 34 | ports: 35 | # Maps tcp port 5432 on service container to the host 36 | - 5432:5432 37 | 38 | steps: 39 | - name: Checkout ReBench 40 | uses: actions/checkout@v5 41 | 42 | - name: Setup Node.js 43 | uses: actions/setup-node@v5 44 | with: 45 | node-version: '24' 46 | 47 | - name: Cache node modules 48 | uses: actions/cache@v4 49 | env: 50 | cache-name: node-modules 51 | with: 52 | # npm cache files are stored in `~/.npm` on Linux/macOS 53 | path: ~/.npm 54 | key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('package.json', 'package-lock.json') }} 55 | restore-keys: | 56 | ${{ runner.os }}-${{ env.cache-name }}- 57 | ${{ runner.os }}- 58 | 59 | - name: NPM CI 60 | run: | 61 | npm ci 62 | 63 | - name: Run Tests 64 | run: | 65 | npm test 66 | echo "## Test Coverage Summary" >> $GITHUB_STEP_SUMMARY 67 | echo "\`\`\`\n" >> $GITHUB_STEP_SUMMARY 68 | cat coverage/coverage-summary.txt | grep -v "===" | tail -n4 >> $GITHUB_STEP_SUMMARY 69 | 70 | - name: Upload image difference, if there are any 71 | if: failure() 72 | uses: actions/upload-artifact@v4 73 | with: 74 | name: Image Differences of charts.test.ts 75 | path: | 76 | diff-*.png 77 | *.svg 78 | if-no-files-found: ignore 79 | 80 | - name: Lint and Formatting 81 | run: | 82 | npm run verify 83 | 84 | - name: Install ReBench 85 | run: | 86 | git clone --depth 1 https://github.com/smarr/rebench.git 87 | pushd rebench 88 | pip install . 89 | popd 90 | 91 | - name: Run ReBench Integration Tests 92 | run: | 93 | # make workspace writable for postgres container 94 | chmod a+wx ${{ github.workspace }} 95 | 96 | # start ReBenchDB server 97 | NODE_DATA_EXPORT_PATH=${{ github.workspace }} RDB_DATA_EXPORT_PATH=/postgres-export DATA_URL_BASE=/static DEV=true npm run start & 98 | sleep 5 99 | 100 | # run integration tests 101 | pushd tests/rebench-integration 102 | rebench --experiment IntegrationTest rebench.conf 103 | 104 | sleep 1 105 | PROJID=$(curl -s http://localhost:33333/ReBenchDB-integration-test/data | grep project-id | grep -o -E '[0-9]+') 106 | EXPID=$(curl -s http://localhost:33333/rebenchdb/dash/$PROJID/data-overview | jq '.data[0].expid') 107 | 108 | # Trigger data generation 109 | curl -s http://localhost:33333/ReBenchDB-integration-test/data/$EXPID.json.gz > /dev/null 110 | curl -s http://localhost:33333/ReBenchDB-integration-test/data/$EXPID.csv.gz > /dev/null 111 | 112 | sleep 10 # give the server some time to generate the files 113 | # reposses the files to be able to read them 114 | sudo chown $(whoami):$(id -g -n) ${{ github.workspace }}/*.gz 115 | 116 | # fetch the generated files via node and check them 117 | curl -sL http://localhost:33333/ReBenchDB-integration-test/data/$EXPID.json.gz -o actual.json.gz 118 | curl -sL http://localhost:33333/ReBenchDB-integration-test/data/$EXPID.csv.gz -o actual.csv.gz 119 | gzip -k -d actual.json.gz 120 | gzip -k -d actual.csv.gz 121 | node check-data.js 122 | -------------------------------------------------------------------------------- /src/backend/timeline/timeline.ts: -------------------------------------------------------------------------------- 1 | import { ParameterizedContext } from 'koa'; 2 | import { 3 | respondProjectIdNotFound, 4 | respondProjectNotFound 5 | } from '../common/standard-responses.js'; 6 | import { prepareTemplate } from '../templates.js'; 7 | import { TimelineSuite } from '../../shared/api.js'; 8 | import { Database } from '../db/db.js'; 9 | import { rebenchVersion, robustPath } from '../util.js'; 10 | import { getNumberOrError } from '../request-check.js'; 11 | import { log } from '../logging.js'; 12 | 13 | const timelineTpl = prepareTemplate( 14 | robustPath('backend/timeline/timeline.html'), 15 | false 16 | ); 17 | 18 | export async function getTimelineAsJson( 19 | ctx: ParameterizedContext, 20 | db: Database 21 | ): Promise { 22 | ctx.type = 'application/json'; 23 | 24 | const projectId = getNumberOrError(ctx, 'projectId'); 25 | if (projectId === null) { 26 | log.error((ctx.body as any).error); 27 | return; 28 | } 29 | 30 | const runId = getNumberOrError(ctx, 'runId'); 31 | if (runId === null) { 32 | log.error((ctx.body as any).error); 33 | return; 34 | } 35 | 36 | ctx.body = await db.getTimelineForRun(projectId, runId); 37 | if (ctx.body === null) { 38 | ctx.status = 500; 39 | } 40 | } 41 | 42 | /** 43 | * @deprecated remove for 1.0 44 | */ 45 | export async function redirectToNewTimelineUrl( 46 | ctx: ParameterizedContext, 47 | db: Database 48 | ): Promise { 49 | const project = await db.getProject(Number(ctx.params.projectId)); 50 | if (project) { 51 | ctx.redirect(`/${project.slug}/timeline`); 52 | } else { 53 | respondProjectIdNotFound(ctx, Number(ctx.params.projectId)); 54 | } 55 | } 56 | 57 | export async function renderTimeline( 58 | ctx: ParameterizedContext, 59 | db: Database 60 | ): Promise { 61 | const project = await db.getProjectBySlug(ctx.params.projectSlug); 62 | 63 | if (project) { 64 | ctx.body = timelineTpl({ 65 | rebenchVersion, 66 | project, 67 | benchmarks: await getLatestBenchmarksForTimelineView(project.id, db) 68 | }); 69 | ctx.type = 'html'; 70 | } else { 71 | respondProjectNotFound(ctx, ctx.params.projectSlug); 72 | } 73 | } 74 | 75 | export async function getLatestBenchmarksForTimelineView( 76 | projectId: number, 77 | db: Database 78 | ): Promise { 79 | const results = await db.getLatestBenchmarksForTimelineView(projectId); 80 | if (results === null) { 81 | return null; 82 | } 83 | 84 | // filter out things we do not want to show 85 | // per grouping and the same benchmark: 86 | // - remove cores, varValue, inputSize, or extraArgs when always the same 87 | for (const t of results) { 88 | for (const e of t.exec) { 89 | const allTheSame = new Map(); 90 | 91 | for (const b of e.benchmarks) { 92 | let sameDesc = allTheSame.get(b.benchName); 93 | if (!sameDesc) { 94 | sameDesc = { 95 | varValue: true, 96 | varValueValue: b.varValue, 97 | cores: true, 98 | coresValue: b.cores, 99 | inputSize: true, 100 | inputSizeValue: b.inputSize, 101 | extraArgs: true, 102 | extraArgsValue: b.extraArgs 103 | }; 104 | allTheSame.set(b.benchName, sameDesc); 105 | } else { 106 | if (sameDesc.varValue && sameDesc.varValueValue != b.varValue) { 107 | sameDesc.varValue = false; 108 | } 109 | if (sameDesc.cores && sameDesc.coresValue != b.cores) { 110 | sameDesc.cores = false; 111 | } 112 | if (sameDesc.inputSize && sameDesc.inputSizeValue != b.inputSize) { 113 | sameDesc.inputSize = false; 114 | } 115 | if (sameDesc.extraArgs && sameDesc.extraArgsValue != b.extraArgs) { 116 | sameDesc.extraArgs = false; 117 | } 118 | } 119 | } 120 | 121 | for (const b of e.benchmarks) { 122 | const sameDesc = allTheSame.get(b.benchName); 123 | if (sameDesc.varValue) { 124 | b.varValue = undefined; 125 | } 126 | if (sameDesc.cores) { 127 | b.cores = undefined; 128 | } 129 | if (sameDesc.inputSize) { 130 | b.inputSize = undefined; 131 | } 132 | if (sameDesc.extraArgs) { 133 | b.extraArgs = undefined; 134 | } 135 | } 136 | } 137 | } 138 | 139 | return results; 140 | } 141 | -------------------------------------------------------------------------------- /src/backend/util.ts: -------------------------------------------------------------------------------- 1 | import { dirname, resolve } from 'path'; 2 | import { fileURLToPath } from 'url'; 3 | 4 | import { promisify } from 'node:util'; 5 | import { gzip as gzipCallback } from 'node:zlib'; 6 | import { readFileSync } from 'node:fs'; 7 | import { writeFile } from 'node:fs/promises'; 8 | import type { ValuesPossiblyMissing } from '../shared/api.js'; 9 | 10 | const gzip = promisify(gzipCallback); 11 | 12 | export async function storeJsonGzip( 13 | data: any[], 14 | filePath: string 15 | ): Promise { 16 | const str = JSON.stringify(data); 17 | const compressedData = await gzip(str); 18 | await writeFile(filePath, compressedData); 19 | } 20 | 21 | export function getDirname(importMetaUrl: string): string { 22 | return dirname(fileURLToPath(importMetaUrl)); 23 | } 24 | 25 | const __dirname = getDirname(import.meta.url); 26 | 27 | /** 28 | * Get a robust path, relative to the source directory. 29 | */ 30 | export const robustPath = __dirname.includes('dist/') 31 | ? function (path) { 32 | return resolve(`${__dirname}/../../../src/${path}`); 33 | } 34 | : function (path) { 35 | return resolve(`${__dirname}/../${path}`); 36 | }; 37 | 38 | /** 39 | * Get a robust path in the compiled source directory. 40 | */ 41 | export const robustSrcPath = __dirname.includes('dist/') 42 | ? function (path) { 43 | return `${__dirname}/../${path}`; 44 | } 45 | : function (path) { 46 | return `${__dirname}/../../dist/src/${path}`; 47 | }; 48 | 49 | const port: number = process.env.RDB_PORT 50 | ? parseInt(process.env.RDB_PORT) 51 | : 5432; 52 | 53 | const _rebench_dev = 'https://rebench.dev'; 54 | const reportsUrl = process.env.REPORTS_URL || '/static/reports'; 55 | const staticUrl = process.env.STATIC_URL || '/static'; 56 | const publicUrl = process.env.PUBLIC_URL || _rebench_dev; 57 | 58 | // configuration for data export is a little more involved, 59 | // because the database might run elsewhere, but may produce 60 | // data files, which we need to be able to serve, at least in the dev mode. 61 | const dbDataExportPath = 62 | process.env.RDB_DATA_EXPORT_PATH || robustPath('../resources/exp-data'); 63 | 64 | // I assume that Node has access to files produced by itself and PostgreSQL. 65 | const nodeDataExportPath = 66 | process.env.NODE_DATA_EXPORT_PATH || dbDataExportPath; 67 | 68 | const dataExportUrlBase = process.env.DATA_URL_BASE || `${staticUrl}/exp-data`; 69 | 70 | export const dbConfig = { 71 | user: process.env.RDB_USER || '', 72 | password: process.env.RDB_PASS || '', 73 | host: process.env.RDB_HOST || 'localhost', 74 | database: process.env.RDB_DB || 'rdb_smde3', 75 | 76 | /** The path where PostgreSQL writes data files to. */ 77 | dataExportPath: dbDataExportPath, 78 | port 79 | }; 80 | 81 | export const refreshSecret = 82 | 'REFRESH_SECRET' in process.env ? process.env.REFRESH_SECRET : undefined; 83 | 84 | /** How long to still hold on to the cache after it became invalid. In ms. */ 85 | export const cacheInvalidationDelay = 1000 * 60 * 5; /* 5 minutes */ 86 | 87 | export function isReBenchDotDev(): boolean { 88 | return siteConfig.publicUrl === _rebench_dev; 89 | } 90 | 91 | export const isRunningTests = 92 | ('NODE_ENV' in process.env && process.env.NODE_ENV === 'test') || 93 | ('JEST_WORKER_ID' in process.env && process.env.JEST_WORKER_ID !== undefined); 94 | 95 | export const DEBUG = 96 | 'DEBUG' in process.env ? process.env.DEBUG === 'true' : false; 97 | export const DEV = 'DEV' in process.env ? process.env.DEV === 'true' : false; 98 | 99 | export const statsConfig = { 100 | numberOfBootstrapSamples: 50 101 | }; 102 | 103 | export const siteConfig = { 104 | port: process.env.PORT || 33333, 105 | reportsUrl, 106 | staticUrl, 107 | publicUrl, 108 | dataExportUrlBase, 109 | 110 | /** 111 | * The path where Node.js writes data files to, 112 | * and Postgres generated files are accessible. 113 | */ 114 | dataExportPath: nodeDataExportPath, 115 | appId: parseInt(process.env.GITHUB_APP_ID || '') || 76497, 116 | githubPrivateKey: 117 | process.env.GITHUB_PK || 'rebenchdb.2020-08-11.private-key.pem', 118 | 119 | canShowWarmup: (data: ValuesPossiblyMissing[]): boolean => { 120 | return data.some((ms) => ms != null && ms.length >= 5); 121 | }, 122 | inlinePlotCriterion: 'total' 123 | }; 124 | 125 | export const TotalCriterion = 'total'; 126 | 127 | export const rebenchVersion = JSON.parse( 128 | readFileSync(robustPath('../package.json'), 'utf-8') 129 | ).version; 130 | -------------------------------------------------------------------------------- /src/backend/timeline/timeline.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ReBench: Timeline {%= it.project.name %} 8 | {%- include('header.html', { rebenchVersion: it.rebenchVersion }) %} 9 | 10 | 11 | 12 | 13 | 14 |
15 |
16 |

ReBench: Timeline {%= it.project.name %}

17 | {% if (it.project.description) { %} 18 |

{%= it.project.description %}

19 | {% } 20 | if (it.project.basebranch) { %} 21 |

Timeline is based on data for the {%= it.project.basebranch %} branch.

22 | {% } %} 23 |
24 | 25 |
26 |
27 | 34 | {%- include('theme-switcher-btn.html') %} 35 | 36 | 42 |
43 |
44 |
45 | 46 |
47 |
48 | 49 | 50 |
51 |
52 |
53 | 54 | 55 |
56 | {% if (it.benchmarks) { %} 57 | 65 | 66 |
67 | {% for (const b of it.benchmarks) { 68 | for (const e of b.exec) { %} 69 |
70 |

{%= b.suiteName %}

71 |
Executor: {%= e.execName %}
72 | 73 | {% for (const bb of e.benchmarks) { %} 74 | 80 | {% } %} 81 |
82 | 83 | {% } } %} 84 |
85 |
86 | {% } else { %} 87 |
88 |
89 |

No Data Available

90 | {% if (it.project.basebranch) { %} 91 |

There are no benchmarks available for this project.

92 | {% } else { %} 93 |

The branch to show on the timeline has not been configured. Please ask the ReBenchDB administrator to set the branch for the timeline view.

94 | {% } %} 95 |
96 |
97 | {% } %} 98 | 99 | 100 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rebenchdb", 3 | "version": "0.5.1", 4 | "description": "A Web-Based Database for ReBench Results", 5 | "main": "index.js", 6 | "author": { 7 | "name": "Stefan Marr", 8 | "email": "git@stefan-marr.de" 9 | }, 10 | "license": "MIT", 11 | "type": "module", 12 | "dependencies": { 13 | "@octokit/auth-app": "8.1.2", 14 | "@octokit/rest": "22.0.1", 15 | "@sgratzl/chartjs-chart-boxplot": "4.4.5", 16 | "canvas": "3.2.0", 17 | "chart.js": "4.5.1", 18 | "chartjs-plugin-annotation": "3.1.0", 19 | "decimal.js": "10.6.0", 20 | "ejs": "3.1.10", 21 | "join-images": "1.1.5", 22 | "koa": "3.1.1", 23 | "koa-body": "7.0.0", 24 | "@koa/router": "14.0.0", 25 | "pg": "8.16.3", 26 | "promisify-child-process": "4.1.2", 27 | "sharp": "0.34.5", 28 | "tslog": "4.10.2", 29 | "uplot": "1.6.32" 30 | }, 31 | "overrides": { 32 | "join-images": { 33 | "sharp": "$sharp" 34 | }, 35 | "test-exclude": "^7.0.1" 36 | }, 37 | "engines": { 38 | "node": ">=21.0.0" 39 | }, 40 | "devDependencies": { 41 | "@octokit/types": "16.0.0", 42 | "@types/ejs": "3.1.5", 43 | "@types/jquery": "3.5.33", 44 | "@types/koa": "3.0.1", 45 | "@types/koa__router": "12.0.5", 46 | "@types/pg": "8.15.6", 47 | "@types/pixelmatch": "5.2.6", 48 | "eslint": "9.39.1", 49 | "@types/pngjs": "6.0.5", 50 | "typescript-eslint": "8.47.0", 51 | "ajv": "8.17.1", 52 | "eslint-config-prettier": "10.1.8", 53 | "eslint-plugin-jest": "29.1.0", 54 | "eslint-plugin-prettier": "5.5.4", 55 | "jest": "30.2.0", 56 | "nodemon": "3.1.11", 57 | "patch-package": "8.0.1", 58 | "pixelmatch": "7.1.0", 59 | "prettier": "3.6.2", 60 | "source-map-support": "0.5.21", 61 | "terser": "5.44.1", 62 | "ts-jest": "29.4.5", 63 | "typescript": "5.9.3", 64 | "typescript-json-schema": "0.65.1" 65 | }, 66 | "jest": { 67 | "collectCoverage": true, 68 | "coverageReporters": [ 69 | "json", 70 | "text", 71 | [ 72 | "text-summary", 73 | { 74 | "file": "coverage-summary.txt" 75 | } 76 | ] 77 | ], 78 | "preset": "ts-jest/presets/default-esm", 79 | "testEnvironment": "node", 80 | "transform": { 81 | "^.+\\.ts$": [ 82 | "ts-jest", 83 | { 84 | "useESM": true 85 | } 86 | ] 87 | }, 88 | "testPathIgnorePatterns": [ 89 | "/dist/", 90 | "/node_modules/" 91 | ], 92 | "modulePathIgnorePatterns": [ 93 | "/dist/" 94 | ], 95 | "extensionsToTreatAsEsm": [ 96 | ".ts" 97 | ], 98 | "moduleNameMapper": { 99 | "^(\\.{1,2}/.*)\\.js$": "$1", 100 | "/static/uPlot.esm.min.js": "/resources/uPlot.esm.min.js" 101 | }, 102 | "roots": [ 103 | "tests/" 104 | ] 105 | }, 106 | "scripts": { 107 | "postinstall": "if [ -z \"${SKIP_COMPILE}\" ]; then npm run compile; fi; patch-package", 108 | "precompile": "npm run prep-folders && npm run compile-uplot", 109 | "prep-folders": "mkdir -p tmp/interm tmp/knit resources/reports resources/exp-data", 110 | "compile-uplot": "terser --module --ecma 2018 --compress --mangle -o ./resources/uPlot.esm.min.js -- node_modules/uplot/dist/uPlot.esm.js", 111 | "compile": "tsc", 112 | "postcompile": "npm run prep-static && npm run download-font", 113 | "prep-static": "cp dist/src/frontend/*.js dist/src/shared/*.js ./resources/", 114 | "download-font": "if ! [ -f dist/roboto-hinted/Roboto-Black.ttf ]; then node dist/src/download.js https://github.com/googlefonts/roboto/releases/download/v2.136/roboto-hinted.zip tmp/roboto-hinted.zip && unzip -o -d dist tmp/roboto-hinted.zip; fi", 115 | "start": "node --enable-source-maps --experimental-json-modules ./dist/src/index.js", 116 | "nodemon": "DEV=true nodemon --enable-source-maps --experimental-json-modules ./dist/src/index.js --watch ./dist/src --watch ./package.json --watch ./src --ext js,json,html", 117 | "format": "prettier --config .prettierrc '{src,tests}/**/*.ts' --write", 118 | "verify": "npm run lint", 119 | "lint": "eslint . --ext .ts,.tsx", 120 | "update": "git pull && npm install . && pm2 restart 0", 121 | "watch": "tsc -w", 122 | "pretest": "(cd tests/data; bzip2 -d -f -k large-payload.json.bz2; mkdir -p actual-results/charts; mkdir -p actual-results/stats-data-prep; mkdir -p actual-results/compare-view)", 123 | "test": "node --no-warnings --experimental-vm-modules ./node_modules/jest/bin/jest.js", 124 | "update-expected-results": "UPDATE_EXPECTED_DATA=true npm test" 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /tests/views/helpers.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, it } from '@jest/globals'; 2 | import { 3 | commonStringStart, 4 | PerIterationOutput, 5 | sortTotalToFront, 6 | withoutStart 7 | } from '../../src/shared/helpers'; 8 | 9 | describe('Helper functions for the views', () => { 10 | describe('commonStringStart()', () => { 11 | it('should return the common start of a list of strings', () => { 12 | expect(commonStringStart(['foo', 'foo', 'foo'])).toBe('foo'); 13 | expect(commonStringStart(['foo', 'foo', 'foobar'])).toBe('foo'); 14 | expect(commonStringStart(['foo', 'foo', 'foobar', 'foobaz'])).toBe('foo'); 15 | }); 16 | 17 | it('should return an empty string if there is no common start', () => { 18 | expect(commonStringStart(['foo', 'bar', 'baz'])).toBe(''); 19 | }); 20 | 21 | it('should return an empty string if the list is empty', () => { 22 | expect(commonStringStart([])).toBe(''); 23 | }); 24 | 25 | it('should return empty string if list contains only empty strings', () => { 26 | expect(commonStringStart(['', '', ''])).toBe(''); 27 | }); 28 | 29 | it('should return empty string if list contains also empty strings', () => { 30 | expect(commonStringStart(['', '', 'foo'])).toBe(''); 31 | }); 32 | 33 | it('should work as expected VM name examples', () => { 34 | expect( 35 | commonStringStart([ 36 | 'SomSom-native-interp-ast', 37 | 'SomSom-native-interp-bc' 38 | ]) 39 | ).toBe('SomSom-native-interp-'); 40 | 41 | expect( 42 | commonStringStart(['TruffleSOM-graal', 'TruffleSOM-graal-bc']) 43 | ).toBe('TruffleSOM-graal'); 44 | 45 | expect( 46 | commonStringStart([ 47 | 'TruffleSOM-native-interp-ast', 48 | 'TruffleSOM-native-interp-bc' 49 | ]) 50 | ).toBe('TruffleSOM-native-interp-'); 51 | }); 52 | }); 53 | 54 | describe('withoutStart()', () => { 55 | it('should remove a prefix from a string', () => { 56 | expect(withoutStart('foo', 'foobar')).toBe('bar'); 57 | }); 58 | 59 | it('should return the string if the prefix is not present', () => { 60 | expect(withoutStart('foo', 'bar')).toBe('bar'); 61 | }); 62 | 63 | it('should return the string if the prefix is empty', () => { 64 | expect(withoutStart('', 'bar')).toBe('bar'); 65 | }); 66 | 67 | it('should return an empty string if the string is empty', () => { 68 | expect(withoutStart('foo', '')).toBe(''); 69 | }); 70 | 71 | it('should return string if prefix is longer than the it', () => { 72 | expect(withoutStart('foobar', 'foo')).toBe('foo'); 73 | }); 74 | 75 | it('should return empty string if prefix is equal to the string', () => { 76 | expect(withoutStart('foo', 'foo')).toBe(''); 77 | }); 78 | 79 | it('should work as expected for VM name examples', () => { 80 | expect( 81 | withoutStart('SomSom-native-interp-', 'SomSom-native-interp-ast') 82 | ).toBe('ast'); 83 | 84 | expect(withoutStart('TruffleSOM-graal', 'TruffleSOM-graal-bc')).toBe( 85 | '-bc' 86 | ); 87 | 88 | expect( 89 | withoutStart( 90 | 'TruffleSOM-native-interp-', 91 | 'TruffleSOM-native-interp-ast' 92 | ) 93 | ).toBe('ast'); 94 | }); 95 | }); 96 | 97 | describe('PerIterationOutput', () => { 98 | it('should return the first string on the first call', () => { 99 | const output = new PerIterationOutput('first', 'second'); 100 | expect(output.next()).toBe('first'); 101 | }); 102 | 103 | it('should return the second string on the second call', () => { 104 | const output = new PerIterationOutput('first', 'second'); 105 | output.next(); 106 | expect(output.next()).toBe('second'); 107 | }); 108 | 109 | it('should return the second string on the third call', () => { 110 | const output = new PerIterationOutput('first', 'second'); 111 | output.next(); 112 | output.next(); 113 | expect(output.next()).toBe('second'); 114 | }); 115 | }); 116 | 117 | describe('sortTotalToFront()', () => { 118 | it('should sort the list of strings with total to the front', () => { 119 | expect(sortTotalToFront(['foo', 'bar', 'total', 'baz'])).toEqual([ 120 | 'total', 121 | 'foo', 122 | 'bar', 123 | 'baz' 124 | ]); 125 | }); 126 | 127 | it('should not change array if total is already at the front', () => { 128 | expect(sortTotalToFront(['total', 'foo', 'bar', 'baz'])).toEqual([ 129 | 'total', 130 | 'foo', 131 | 'bar', 132 | 'baz' 133 | ]); 134 | }); 135 | 136 | it('should not change array if total is not present', () => { 137 | expect(sortTotalToFront(['foo', 'bar', 'baz'])).toEqual([ 138 | 'foo', 139 | 'bar', 140 | 'baz' 141 | ]); 142 | }); 143 | }); 144 | }); 145 | -------------------------------------------------------------------------------- /src/backend/project/data-export.ts: -------------------------------------------------------------------------------- 1 | import { existsSync } from 'node:fs'; 2 | import { 3 | completeRequestAndHandlePromise, 4 | startRequest 5 | } from '../perf-tracker.js'; 6 | import { dbConfig, siteConfig, storeJsonGzip } from '../util.js'; 7 | import { log } from '../logging.js'; 8 | import { Database } from '../db/db.js'; 9 | import { ParameterizedContext } from 'koa'; 10 | import { getNumberOrError } from '../request-check.js'; 11 | 12 | const expDataPreparation = new Map(); 13 | 14 | export async function getExpData( 15 | projectSlug: string, 16 | expId: number, 17 | db: Database, 18 | format: 'json' | 'csv' 19 | ): Promise { 20 | const result = await db.getExperimentDetails(expId, projectSlug); 21 | 22 | let data: any; 23 | if (!result) { 24 | data = { 25 | project: '', 26 | projectSlug, 27 | generationFailed: true, 28 | stdout: 'Experiment was not found' 29 | }; 30 | } else { 31 | data = result; 32 | } 33 | 34 | const expFilePrefix = `${data.projectSlug}-${expId}`; 35 | const expFileName = `${expFilePrefix}.${format}.gz`; 36 | 37 | if (existsSync(`${siteConfig.dataExportPath}/${expFileName}`)) { 38 | data.preparingData = false; 39 | data.downloadUrl = `${siteConfig.dataExportUrlBase}/${expFileName}`; 40 | } else { 41 | const expRequestId = `${expFilePrefix}-${format}`; 42 | data.currentTime = new Date().toISOString(); 43 | 44 | const prevPrepDetails = expDataPreparation.get(expRequestId); 45 | 46 | // no previous attempt to prepare data 47 | if (!prevPrepDetails) { 48 | const start = startRequest(); 49 | 50 | data.preparingData = true; 51 | 52 | const resultP = 53 | format === 'json' 54 | ? db.getExperimentMeasurements(expId) 55 | : db.storeExperimentMeasurements( 56 | expId, 57 | `${dbConfig.dataExportPath}/${expFileName}` 58 | ); 59 | 60 | expDataPreparation.set(expRequestId, { 61 | inProgress: true 62 | }); 63 | 64 | resultP 65 | .then(async (data: any[]) => { 66 | if (format === 'json') { 67 | await storeJsonGzip( 68 | data, 69 | `${siteConfig.dataExportPath}/${expFileName}` 70 | ); 71 | } 72 | expDataPreparation.set(expRequestId, { 73 | inProgress: false 74 | }); 75 | }) 76 | .catch((error) => { 77 | log.error('Data preparation failed', error); 78 | expDataPreparation.set(expRequestId, { 79 | error, 80 | inProgress: false 81 | }); 82 | }) 83 | .finally(() => 84 | completeRequestAndHandlePromise(start, db, 'prep-exp-data') 85 | ); 86 | } else if (prevPrepDetails.error) { 87 | // if previous attempt failed 88 | data.generationFailed = true; 89 | data.preparingData = false; 90 | } else { 91 | data.preparingData = true; 92 | } 93 | } 94 | 95 | return data; 96 | } 97 | 98 | export async function getAvailableDataAsJson( 99 | ctx: ParameterizedContext, 100 | db: Database 101 | ): Promise { 102 | ctx.type = 'application/json'; 103 | 104 | const projectId = getNumberOrError(ctx, 'projectId'); 105 | if (projectId === null) { 106 | log.error((ctx.body as any).error); 107 | return; 108 | } 109 | 110 | ctx.body = await getDataOverview(projectId, db); 111 | } 112 | 113 | export async function getDataOverview( 114 | projectId: number, 115 | db: Database 116 | ): Promise<{ data: any[] }> { 117 | const result = await db.query({ 118 | name: 'fetchDataOverview', 119 | text: ` 120 | SELECT 121 | exp.id as expId, exp.name, exp.description, 122 | min(t.startTime) as minStartTime, 123 | max(t.endTime) as maxEndTime, 124 | ARRAY_TO_STRING(ARRAY_AGG(DISTINCT t.username), ', ') as users, 125 | ARRAY_TO_STRING(ARRAY_AGG(DISTINCT src.commitId), ' ') as commitIds, 126 | ARRAY_TO_STRING(ARRAY_AGG(DISTINCT src.commitMessage), '\n\n') 127 | as commitMsgs, 128 | ARRAY_TO_STRING(ARRAY_AGG(DISTINCT env.hostName), ', ') as hostNames, 129 | 130 | -- Accessing measurements and timeline should give the same results, 131 | -- but the counting in measurements is of course a lot slower 132 | -- count(m.*) as measurements, 133 | -- count(DISTINCT m.runId) as runs 134 | SUM(tl.numSamples) as measurements, 135 | count(DISTINCT tl.runId) as runs 136 | FROM experiment exp 137 | JOIN Trial t ON exp.id = t.expId 138 | JOIN Source src ON t.sourceId = src.id 139 | JOIN Environment env ON env.id = t.envId 140 | 141 | --JOIN Measurement m ON m.trialId = t.id 142 | JOIN Timeline tl ON tl.trialId = t.id 143 | 144 | WHERE exp.projectId = $1 145 | 146 | GROUP BY exp.name, exp.description, exp.id 147 | ORDER BY minStartTime DESC;`, 148 | values: [projectId] 149 | }); 150 | return { data: result.rows }; 151 | } 152 | -------------------------------------------------------------------------------- /src/backend/perf-tracker.ts: -------------------------------------------------------------------------------- 1 | import { performance } from 'perf_hooks'; 2 | import { Database } from './db/db.js'; 3 | import type { BenchmarkData } from '../shared/api.js'; 4 | import { TotalCriterion, isRunningTests } from './util.js'; 5 | import type { Run, Trial } from './db/types.js'; 6 | import { assert, log } from './logging.js'; 7 | 8 | // Performance tracking design 9 | // - when ReBenchDB starts up, this marks a new trial with a single invocation 10 | // - at startup, we eagerly create the measurement records 11 | // - and then for the performance tracking, we append to the values array of 12 | // the specific row 13 | 14 | let startTime: string; 15 | 16 | interface TrialDetails { 17 | trial: Trial; 18 | run: Run; 19 | criterionId: number; 20 | } 21 | 22 | const trialDetails: { [key: string]: TrialDetails } = {}; 23 | 24 | const descriptions = { 25 | 'get-results': 'Time of GET /rebenchdb/dash/:projectId/results', 26 | 'put-results': 'Time of PUT /rebenchdb/results', 27 | change: 'Time of GET /compare/:project/:baseline/:change', 28 | 'change-new': 'Time of GET /compare-new/:project/:baseline/:change', 29 | 'generate-report': 'Time of Running R Reporting for /compare/*', 30 | 'generate-timeline': 'Time of Running R stats to generate timeline data', 31 | 'prep-exp-data': 'Prepare experiment data for download', 32 | 'get-exp-data': 'Starting to prepare experiment data', 33 | 'project-benchmarks': 'Time of GET /rebenchdb/dash/:projectId/benchmarks', 34 | 'get-profiles': 35 | // this url was changed to use the commitId instead of the trialId 36 | // I'll leave this unchanged here to avoid issues 37 | // with the performance tracking 38 | 'Time of GET /rebenchdb/dash/:projectId/profiles/:runId/:trialId', 39 | 'get-measurements': 'Time of GET /rebenchdb/dash/:projectId/measurements/...' 40 | }; 41 | 42 | export async function initPerfTracker(db: Database): Promise { 43 | startTime = new Date().toISOString(); 44 | 45 | const benchmarkNames = Object.keys(descriptions); 46 | const initializationData = constructInitialization(benchmarkNames); 47 | const { metadata, runs } = await db.recordMetaDataAndRuns(initializationData); 48 | 49 | const criterion = [...metadata.criteria.values()][0]; 50 | 51 | for (const run of runs) { 52 | trialDetails[run.cmdline] = { 53 | trial: metadata.trial, 54 | run, 55 | criterionId: criterion.id 56 | }; 57 | } 58 | } 59 | 60 | function constructInitialization(benchmarkNames: string[]) { 61 | const data: BenchmarkData = { 62 | experimentName: 'monitoring', 63 | data: [], 64 | criteria: [{ i: 0, c: TotalCriterion, u: 'ms' }], 65 | env: { 66 | hostName: 'self', 67 | cpu: '', 68 | memory: 0, 69 | clockSpeed: 0, 70 | osType: 'nodejs', 71 | userName: 'rebench-perf-tracking', 72 | software: [], 73 | manualRun: false, 74 | denoise: {} 75 | }, 76 | source: { 77 | repoURL: 'https://github.com/smarr/ReBenchDB', 78 | branchOrTag: 'master', 79 | commitId: '', 80 | commitMsg: '', 81 | authorEmail: '', 82 | authorName: '', 83 | committerEmail: '', 84 | committerName: '' 85 | }, 86 | startTime, 87 | endTime: null, 88 | projectName: 'ReBenchDB Self-Tracking' 89 | }; 90 | 91 | for (const name of benchmarkNames) { 92 | data.data.push({ 93 | d: [], 94 | runId: { 95 | benchmark: { 96 | name: name, 97 | suite: { 98 | name: 'ReBenchDB API', 99 | desc: 'Performance tracking of the ReBenchDB API', 100 | executor: { 101 | name: 'Node.js', 102 | desc: null 103 | } 104 | }, 105 | runDetails: { 106 | maxInvocationTime: 0, 107 | minIterationTime: 0, 108 | warmup: null 109 | }, 110 | desc: descriptions[name] 111 | }, 112 | cmdline: name, 113 | location: '', 114 | varValue: null, 115 | cores: null, 116 | inputSize: null, 117 | extraArgs: null 118 | } 119 | }); 120 | } 121 | 122 | return data; 123 | } 124 | 125 | export function startRequest(): number { 126 | return performance.now(); 127 | } 128 | 129 | /** private, exported for testing. */ 130 | export async function _completeRequest( 131 | reqStart: number, 132 | db: Database, 133 | request: string 134 | ): Promise<[number, number] | void> { 135 | const time = performance.now() - reqStart; 136 | 137 | assert( 138 | trialDetails[request] !== undefined, 139 | 'Performance tracking not initialized' 140 | ); 141 | 142 | const details = trialDetails[request]; 143 | 144 | return db.recordAdditionalMeasurementValue( 145 | details.run, 146 | details.trial, 147 | details.criterionId, 148 | time 149 | ); 150 | } 151 | 152 | export function completeRequestAndHandlePromise( 153 | reqStart: number, 154 | db: Database, 155 | request: string 156 | ): void { 157 | if (isRunningTests) { 158 | return; 159 | } 160 | 161 | _completeRequest(reqStart, db, request).catch((e) => { 162 | log.error('Error while recording performance data:', e); 163 | }); 164 | } 165 | -------------------------------------------------------------------------------- /tests/data/expected-results/main/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ReBench 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 |

ReBench

20 |

Execute and document benchmarks reproducibly.

21 |

ReBench is a tool to run and document benchmark experiments. Currently, it is mostly used for benchmarking language implementations, but it can be used to monitor the performance of all kinds of other applications and programs, too.

22 |
23 |

ReBenchDB is a project started in late 2019 to provide convenient access to data recorded with ReBench. 24 | Our focus is to facilitate the software engineering process with useful performance statistics. 25 |

26 | DOI 27 | 28 | 29 | 30 |
31 |
32 |
33 | 42 |
43 |
44 |
45 |
46 | 47 |
48 |
Small Example Project
50 |
51 | 52 |
Changes
53 |
54 |
55 |
57 |
58 |
59 |
60 |
61 |
62 | Compare 63 | 64 | Timeline 65 |
66 |
67 | 68 |
69 | 70 | 71 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | "target": "es2022", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017','ES2018' or 'ESNEXT'. */ 5 | "module": "es2022", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 6 | // "lib": [], /* Specify library files to be included in the compilation. */ 7 | // "allowJs": true, /* Allow javascript files to be compiled. */ 8 | // "checkJs": true, /* Report errors in .js files. */ 9 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 10 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 11 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 12 | "sourceMap": true, /* Generates corresponding '.map' file. */ 13 | // "outFile": "./", /* Concatenate and emit output to single file. */ 14 | "outDir": "dist", /* Redirect output structure to the directory. */ 15 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 16 | // "composite": true, /* Enable project compilation */ 17 | // "removeComments": true, /* Do not emit comments to output. */ 18 | // "noEmit": true, /* Do not emit outputs. */ 19 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 20 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 21 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 22 | 23 | /* Strict Type-Checking Options */ 24 | "strict": true, /* Enable all strict type-checking options. */ 25 | "noImplicitAny": false, /* Raise error on expressions and declarations with an implied 'any' type. */ 26 | // "strictNullChecks": true, /* Enable strict null checks. */ 27 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 28 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 29 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 30 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 31 | 32 | /* Additional Checks */ 33 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 34 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 35 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 36 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 37 | 38 | /* Module Resolution Options */ 39 | "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 40 | "baseUrl": "./src", /* Base directory to resolve non-absolute module names. */ 41 | "paths": { /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 42 | "/static/*": ["../resources/*"] 43 | }, 44 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 45 | // "typeRoots": [], /* List of folders to include type definitions from. */ 46 | // "types": [], /* Type declaration files to be included in compilation. */ 47 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 48 | "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 49 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 50 | 51 | /* Source Map Options */ 52 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 53 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 54 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 55 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 56 | 57 | /* Experimental Options */ 58 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 59 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 60 | 61 | "resolveJsonModule": true, 62 | "skipLibCheck": true 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/backend/main/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ReBench 6 | {%- include('header.html', { rebenchVersion: it.rebenchVersion }) %} 7 | 8 | 9 | 10 | 11 |
12 |
13 |

ReBench

14 |

Execute and document benchmarks reproducibly.

15 | 16 |

ReBench is a tool to run and document benchmark experiments. Currently, it is mostly used for benchmarking language implementations, but it can be used to monitor the performance of all kinds of other applications and programs, too.

17 |
18 |

ReBenchDB is a project started in late 2019 to provide convenient access to data recorded with ReBench. 19 | Our focus is to facilitate the software engineering process with useful performance statistics. 20 |

21 | 22 | DOI 23 | 24 | 25 | 26 |
27 | {%- include('common-menu.html', it) %} 28 |
29 | 30 |
31 | {% for (const p of it.projects) { %} 32 |
33 |
{%= p.name %}
35 |
36 | {% if (p.showchanges) { %} 37 |
Changes
38 |
39 |
40 |
42 |
43 |
44 |
45 |
46 |
47 | 48 | Compare 49 | {% } 50 | if (p.allresults) { %} 51 |
52 | {% } %} 53 | Timeline 54 |
55 |
56 | {% } 57 | if (!it.projects || it.projects.length === 0) { %} 58 |
59 |
60 | Welcome to your ReBenchDB Instance
61 |
62 |

Currently, there are no projects available.

63 | 64 |

To get started, run your benchmarks with 65 | ReBench 66 | and add the following to your project's ReBench configuration file:

67 | 
reporting:
 68 |   rebenchdb:
 69 |     db_url: rebenchdb
 70 |     repo_url: https://url-to-your-project-repository
 71 |     record_all: true # make sure everything is recorded
 72 |     project_name: Your-Project-Name
 73 |
74 | 77 | {% } %} 78 |
79 | 80 | {% if (it.isReBenchDotDev) { %} 81 | 105 | {% } %} 106 | 107 | 108 | --------------------------------------------------------------------------------