├── .gitignore ├── .github └── workflows │ └── ci.yml ├── examples ├── fib.zig ├── fib_build.zig └── fib2.zig ├── LICENSE ├── src ├── time.zig ├── bench_runner.zig ├── statistics.zig └── bench.zig └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | zig-out/ 2 | .zig-cache/ 3 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | workflow_dispatch: 10 | 11 | schedule: 12 | - cron: '17 15 * * 4' 13 | 14 | jobs: 15 | build: 16 | strategy: 17 | matrix: 18 | os: [ubuntu-latest, macos-latest, windows-latest] 19 | runs-on: ${{ matrix.os }} 20 | steps: 21 | - name: Checkout 22 | uses: actions/checkout@v4 23 | 24 | - name: Setup Zig 25 | uses: mlugg/setup-zig@v2 26 | with: 27 | version: master 28 | 29 | - name: Check formatting 30 | if: ${{ runner.os != 'Windows' }} 31 | run: zig fmt --check . 32 | 33 | - name: Run tests 34 | run: zig build test 35 | -------------------------------------------------------------------------------- /examples/fib.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | 3 | // pray to the stack-overflow gods 🙏 4 | pub fn fib(n: u32) u32 { 5 | return if (n == 0) 6 | 0 7 | else if (n == 1) 8 | 1 9 | else 10 | fib(n - 1) + fib(n - 2); 11 | } 12 | 13 | pub fn fibFast(n: u32) u32 { 14 | const phi = (1.0 + @sqrt(5.0)) / 2.0; 15 | const psi = (1.0 - @sqrt(5.0)) / 2.0; 16 | const float_n: f32 = @floatFromInt(n); 17 | const phi_n = std.math.pow(f32, phi, float_n); 18 | const psi_n = std.math.pow(f32, psi, float_n); 19 | return @intFromFloat((phi_n - psi_n) / @sqrt(5.0)); 20 | } 21 | 22 | test "bench fibFast" { 23 | _ = fibFast(35); 24 | } 25 | 26 | test "bench fib" { 27 | _ = fib(35); 28 | } 29 | 30 | test "regular test" { 31 | // this is not a benchmark 32 | return error.NotABenchmark; 33 | } 34 | -------------------------------------------------------------------------------- /examples/fib_build.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const zubench = @import("zubench"); 3 | const fib = @import("fib.zig"); 4 | 5 | // uncomment this to explicitly specify which clocks to use 6 | // pub const sample_spec = [_]zubench.Clock{ .real, .process, .thread }; 7 | 8 | fn fib10() u32 { 9 | return fib.fibFast(10); 10 | } 11 | 12 | pub const benchmarks = .{ 13 | .@"fib()" = zubench.Spec(fib.fib){ 14 | .args = .{35}, 15 | .max_samples = 20, 16 | .opts = .{ .outlier_detection = .none }, // disable MAD-base outlier detection 17 | }, 18 | // by default use MAD-based outlier detection 19 | .@"fibFast()" = zubench.Spec(fib.fibFast){ .args = .{35}, .max_samples = 1_000_000 }, 20 | // 0-ary functions do not need .args field 21 | .@"fib10()" = zubench.Spec(fib10){ .max_samples = 1_000 }, 22 | }; 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Dominic Weiller 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /examples/fib2.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const zubench = @import("zubench"); 3 | const fib = @import("fib.zig"); 4 | 5 | pub const sample_spec = [_]zubench.Clock{ .real, .process, .thread }; 6 | 7 | pub fn main() !void { 8 | var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 9 | defer arena.deinit(); 10 | const allocator = arena.allocator(); 11 | 12 | const progress = std.Progress.start(.{}); 13 | 14 | var bm = try zubench.Benchmark(@TypeOf(fib.fib)).init( 15 | allocator, 16 | "fib()", 17 | &fib.fib, 18 | .{35}, 19 | .{ .outlier_detection = .none }, //disable MAD-base outlier detection 20 | 20, 21 | progress, 22 | ); 23 | const report = try bm.run(); 24 | bm.deinit(); 25 | 26 | var bm_fast = try zubench.Benchmark(@TypeOf(fib.fibFast)).init( 27 | allocator, 28 | "fibFast()", 29 | &fib.fibFast, 30 | .{35}, 31 | .{}, 32 | 1_000_000, 33 | progress, 34 | ); 35 | const report_fast = try bm_fast.run(); 36 | bm_fast.deinit(); 37 | 38 | const stdout = std.io.getStdOut().writer(); 39 | // write human-readable summary 40 | try stdout.print("{}", .{report}); 41 | // write report as json 42 | try std.json.stringify(report_fast, .{}, stdout); 43 | try stdout.writeByte('\n'); 44 | } 45 | -------------------------------------------------------------------------------- /src/time.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const builtin = @import("builtin"); 3 | const posix = std.posix; 4 | 5 | pub const Clock = enum { 6 | real, 7 | process, 8 | thread, 9 | 10 | pub fn now(self: Clock) error{Unsupported}!std.time.Instant { 11 | if (posix.CLOCK == void) return .now(); 12 | const clock_id = switch (self) { 13 | .real => switch (builtin.os.tag) { 14 | .macos, .ios, .tvos, .watchos => posix.CLOCK.UPTIME_RAW, 15 | .freebsd, .dragonfly => posix.CLOCK.MONOTONIC_FAST, 16 | .linux => posix.CLOCK.BOOTTIME, 17 | else => posix.CLOCK.MONOTONIC, 18 | }, 19 | .process => posix.CLOCK.PROCESS_CPUTIME_ID, 20 | .thread => posix.CLOCK.THREAD_CPUTIME_ID, 21 | }; 22 | return .{ .timestamp = posix.clock_gettime(clock_id) catch return error.Unsupported }; 23 | } 24 | }; 25 | 26 | // this is adapted from std.time.Timer 27 | pub const Timer = struct { 28 | clock: Clock, 29 | started: std.time.Instant, 30 | previous: std.time.Instant, 31 | 32 | pub const Error = error{TimerUnsupported}; 33 | 34 | /// Initialize the timer by querying for a supported clock. 35 | /// Returns `error.TimerUnsupported` when such a clock is unavailable. 36 | /// This should only fail in hostile environments such as linux seccomp misuse. 37 | pub fn start(clock: Clock) Error!Timer { 38 | const current = clock.now() catch return error.TimerUnsupported; 39 | return Timer{ .clock = clock, .started = current, .previous = current }; 40 | } 41 | 42 | /// Reads the timer value since start or the last reset in nanoseconds. 43 | pub fn read(self: *Timer) u64 { 44 | const current = self.sample(); 45 | return current.since(self.started); 46 | } 47 | 48 | /// Resets the timer value to 0/now. 49 | pub fn reset(self: *Timer) void { 50 | const current = self.sample(); 51 | self.started = current; 52 | } 53 | 54 | /// Returns the current value of the timer in nanoseconds, then resets it. 55 | pub fn lap(self: *Timer) u64 { 56 | const current = self.sample(); 57 | defer self.started = current; 58 | return current.since(self.started); 59 | } 60 | 61 | /// Returns an Instant sampled at the callsite that is 62 | /// guaranteed to be monotonic with respect to the timer's starting point. 63 | fn sample(self: *Timer) std.time.Instant { 64 | const current = self.clock.now() catch unreachable; 65 | if (current.order(self.previous) == .gt) { 66 | self.previous = current; 67 | } 68 | return self.previous; 69 | } 70 | }; 71 | -------------------------------------------------------------------------------- /src/bench_runner.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const bench = @import("zubench"); 3 | const root = @import("@bench"); 4 | const builtin = @import("builtin"); 5 | 6 | pub const sample_spec = if (!builtin.is_test and @hasDecl(root, "sample_spec")) 7 | root.sample_spec 8 | else 9 | bench.default_sample_spec; 10 | 11 | pub fn main() !void { 12 | if (builtin.is_test) 13 | try testRunner() 14 | else 15 | try standalone(); 16 | } 17 | 18 | fn testRunner() !void { 19 | var gpa = std.heap.GeneralPurposeAllocator(.{}){}; 20 | defer _ = gpa.deinit(); 21 | const allocator = gpa.allocator(); 22 | 23 | const options = bench.Options{}; 24 | const max_samples = 100; 25 | 26 | const stderr = std.io.getStdErr().writer(); 27 | try stderr.print("Running benchmarks ({s} mode)\n", .{@tagName(builtin.mode)}); 28 | 29 | const progress = std.Progress.start(.{}); 30 | var results = try std.ArrayList(bench.Report).initCapacity(allocator, builtin.test_functions.len); 31 | defer results.deinit(); 32 | 33 | for (builtin.test_functions) |test_fn| { 34 | const bench_name = if (std.mem.indexOfScalar(u8, test_fn.name, '.')) |index| 35 | test_fn.name[index + 1 ..] 36 | else 37 | test_fn.name; 38 | var benchmark = try bench.Benchmark(std.meta.Child(@TypeOf(test_fn.func))).init( 39 | allocator, 40 | bench_name, 41 | test_fn.func, 42 | .{}, 43 | options, 44 | max_samples, 45 | progress, 46 | ); 47 | defer benchmark.deinit(); 48 | results.appendAssumeCapacity(try benchmark.run()); 49 | } 50 | 51 | const stdout = std.io.getStdOut().writer(); 52 | for (results.items) |report| { 53 | // write human-readable summary 54 | try stdout.print("{}", .{report}); 55 | } 56 | } 57 | 58 | fn standalone() !void { 59 | var gpa = std.heap.GeneralPurposeAllocator(.{}){}; 60 | defer _ = gpa.deinit(); 61 | const allocator = gpa.allocator(); 62 | 63 | const stderr = std.io.getStdErr().writer(); 64 | try stderr.print("Running benchmarks ({s} mode)\n", .{@tagName(builtin.mode)}); 65 | 66 | const progress = std.Progress.start(.{}); 67 | 68 | const benchmarks = @typeInfo(@TypeOf(root.benchmarks)).@"struct".fields; 69 | var results = std.BoundedArray(bench.Report, benchmarks.len).init(0) catch unreachable; 70 | 71 | inline for (benchmarks) |field| { 72 | const spec = @field(root.benchmarks, field.name); 73 | var benchmark = try bench.Benchmark(@TypeOf(spec.func)).init( 74 | allocator, 75 | field.name, 76 | spec.func, 77 | spec.args, 78 | spec.opts, 79 | spec.max_samples, 80 | progress, 81 | ); 82 | defer benchmark.deinit(); 83 | results.appendAssumeCapacity(try benchmark.run()); 84 | } 85 | 86 | const stdout = std.io.getStdOut().writer(); 87 | for (results.slice()) |report| { 88 | // write human-readable summary 89 | try stdout.print("{}", .{report}); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/statistics.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | 3 | /// Sample mean 4 | pub fn mean(samples: []const u64) f32 { 5 | std.debug.assert(samples.len > 0); 6 | var acc: u128 = 0; 7 | for (samples) |sample| { 8 | acc += sample; 9 | } 10 | return @as(f32, @floatFromInt(acc)) / @as(f32, @floatFromInt(samples.len)); 11 | } 12 | 13 | fn totalSquaredError(samples: []const u64, avg: f32) f32 { 14 | var acc: f32 = 0.0; 15 | for (samples) |sample| { 16 | const diff = avg - @as(f32, @floatFromInt(sample)); 17 | acc += diff * diff; 18 | } 19 | return acc; 20 | } 21 | /// sample variance 22 | pub fn variance(samples: []const u64, avg: f32) f32 { 23 | return totalSquaredError(samples, avg) / @as(f32, @floatFromInt(samples.len - 1)); 24 | } 25 | 26 | /// sample standard deviation 27 | pub fn correctedSampleStdDev(samples: []const u64, avg: f32) f32 { 28 | return std.math.sqrt(variance(samples, avg)); 29 | } 30 | 31 | /// returns the median 32 | /// modifies `samples`, make a copy if you need to keep the original data 33 | // PERF: better to use randomize quick select? 34 | pub fn median(samples: []u64) f32 { 35 | std.sort.heap(u64, samples, {}, comptime std.sort.asc(u64)); 36 | return if (samples.len % 2 == 0) 37 | @floatFromInt((samples[samples.len / 2 - 1] + samples[samples.len / 2]) / 2) 38 | else 39 | @floatFromInt(samples[samples.len / 2]); 40 | } 41 | 42 | /// median absolute deviation central tendency 43 | /// modifies `samples`, make a copy if you need to keep the original data 44 | pub fn medianAbsDev(samples: []u64, centre: f32) f32 { 45 | for (samples) |*sample| { 46 | const val: f32 = @floatFromInt(sample.*); 47 | // WARNING: cast will bias result 48 | sample.* = if (val > centre) 49 | @intFromFloat(val - centre) 50 | else 51 | @intFromFloat(centre - val); 52 | } 53 | return median(samples); 54 | } 55 | 56 | /// calculate the z-score 57 | /// For the actual z-score, call as zScore(stddev, mean, val). 58 | pub fn zScore(dispersion: f32, centre: f32, val: u64) f32 { 59 | const diff = @as(f32, @floatFromInt(val)) - centre; 60 | return diff / dispersion; 61 | } 62 | 63 | const SortContext = struct { 64 | centre: f32, 65 | dispersion: f32, 66 | }; 67 | 68 | fn ascByZScore(context: SortContext, a: u64, b: u64) bool { 69 | const zscore_a = zScore(context.dispersion, context.centre, a); 70 | const zscore_b = zScore(context.dispersion, context.centre, b); 71 | return std.sort.asc(f32, zscore_a, zscore_b); 72 | } 73 | 74 | const IndexSortContext = struct { 75 | samples: []const u64, 76 | centre: f32, 77 | dispersion: f32, 78 | }; 79 | 80 | fn ascIndexByZScore(context: SortContext, a: u16, b: u16) bool { 81 | const zscore_a = zScore(context.dispersion, context.centre, context.samples[a]); 82 | const zscore_b = zScore(context.dispersion, context.centre, context.samples[b]); 83 | return std.sort.asc(f32, zscore_a, zscore_b); 84 | } 85 | 86 | /// removes outliers from `samples`, copying data to `buf` 87 | /// and returning a subslice of `buf` containing non-outliers 88 | /// 89 | /// `cutoff` is the cutoff of MAD (median absolute deviation from the median) 90 | /// to use. For examples, if you want to remove everything outside on `n` standard 91 | /// deviations (assuming a normal distribution), `cutoff` should be set to 92 | /// approximately `1.4826 * n`. 93 | pub fn removeOutliers(buf: []u64, samples: []const u64, cutoff: f32) []u64 { 94 | std.mem.copy(u64, buf, samples); 95 | const centre = median(buf); 96 | const mad = medianAbsDev(buf, centre); 97 | const ctx = SortContext{ .centre = centre, .dispersion = mad }; 98 | 99 | std.mem.copy(u64, buf, samples); 100 | std.sort.sort(u64, buf, ctx, ascByZScore); 101 | 102 | var i: usize = buf.len; 103 | while (zScore(mad, centre, buf[i - 1]) > cutoff) : (i -= 1) {} 104 | return buf[0..i]; 105 | } 106 | 107 | pub fn removeOutliersIndices(buf: []u64, indices: []u16, samples: []const u64, cutoff: f32) []u64 { 108 | std.debug.assert(indices.len == samples.len and buf.len == samples.len); 109 | std.mem.copy(u64, buf, samples); 110 | const centre = median(buf); 111 | const mad = medianAbsDev(buf, centre); 112 | const ctx = SortContext{ .samples = samples, .centre = centre, .dispersion = mad }; 113 | 114 | std.sort.sort(u64, indices, ctx, ascIndexByZScore); 115 | 116 | var i: usize = buf.len; 117 | while (zScore(mad, centre, samples[indices[i - 1]]) > cutoff * mad) : (i -= 1) {} 118 | return buf[0..i]; 119 | } 120 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # zubench 2 | 3 | A micro-benchmarking package for [Zig](https://ziglang.org). 4 | 5 | ## goals 6 | 7 | The primary goals of **zubench** are to: 8 | 9 | - be simple to use - there should be no need to wrap a function just to benchmark it 10 | - provide standard machine-readable output for archiving or post-processing 11 | - given the user the choice of which system clock(s) to use 12 | - provide statistically relevant and accurate results 13 | - integrate with the Zig build system 14 | 15 | Not all these goals are currently met, and its always possible to debate how well they are met; feel free to open an issue (if one doesn't exist) or pull request if you would like to see improvement in one of these areas. 16 | 17 | ## features 18 | 19 | - [x] human-readable terminal-style output 20 | - [x] machine-readable JSON output 21 | - [x] wall, process, and thread time 22 | - [ ] kernel/user mode times 23 | - [x] declarative `zig test` style benchmark runner 24 | - [x] option to define benchmarks as Zig tests 25 | - [ ] adaptive sample sizes 26 | - [x] [MAD](https://en.wikipedia.org/wiki/Median_absolute_deviation)-based outlier rejection 27 | 28 | ## platforms 29 | 30 | Some attempt has been made to work on the below platforms; those with a '️️️️️⚠️' in the table below haven't been tested, but _should_ work for all implemented clocks. Windows currently only has the wall time clock implemented. If you find a non-Linux platform either works or has issues please raise an issue. 31 | 32 | | Platform | Status | 33 | | :------: | :----: | 34 | | Linux | ✅ | 35 | | Windows | ❗ | 36 | | Darwin | ⚠️ | 37 | | BSD | ⚠️ | 38 | | WASI | ⚠️ | 39 | 40 | ## usage 41 | 42 | The *main* branch follows Zig's master branch, for Zig 0.12 use the *zig-0.12* branch. 43 | 44 | The simplest way to create and run benchmarks is using one of the Zig build system integrations. There are currently two integrations, one utilising the Zig test system, and one utilising public declarations. 45 | 46 | Both integrations will compile an executable that takes a collection of functions to benchmark, runs them repeatedly and reports timing statistics. The differences between the two integrations are how you define benchmarks in your source files, and how benchmarking options are determined. 47 | 48 | ### test integration 49 | The simplest way to define and run benchmarks is utilising the Zig test system. **zubench** will use a custom test runner to run the benchmarks. This means that benchmarks are simply Zig tests, i.e. `test { // code to benchmark }`. In order to avoid benchmarking regular tests when using this system, you should consider the way that Zig analyses test declarations and either give the names of benchmark tests a unique substring that can be used as a test filter or organise your tests so that when the compiller analyses the root file for benchmark tests, it will not analyse regular tests. 50 | 51 | The following snippets show how you can use this integration. 52 | ```zig 53 | const addTestBench = @import("zubench/build.zig").addTestBench; 54 | 55 | pub fn build(b: *std.build.Builder) void { 56 | // existing build function 57 | // ... 58 | 59 | // benchmark all tests analysed by the compiler rooted in "src/file.zig", compiled in ReleaseSafe mode 60 | const benchmark_exe = zubench.addTestBench(b, "src/file.zig", .ReleaseSafe); 61 | // use a test filter to only benchmark tests whose name include the substring "bench" 62 | // note that this is not required if the compiler will not analyse tests that you don't want to benchmark 63 | benchmark_exe.setTestFilter("bench"); 64 | 65 | const bench_step = b.step("bench", "Run the benchmarks"); 66 | bench_step.dependOn(&benchmark_exe.run().step); 67 | } 68 | ``` 69 | This will make `zig build bench` benchmark tests the compiler analyses by starting at `src/file.zig`. `addTestBench()` returns a `*LibExeObjStep` for an executable that runs the benchmarks; you can integrate it into your `build.zig` however you wish. Benchmarks are `test` declarations the compiler analyses staring from `src/file.zig`: 70 | ```zig 71 | // src/file.zig 72 | 73 | test "bench 1" { 74 | // this will be benchmarked 75 | // ... 76 | } 77 | 78 | test "also a benchmark" { 79 | // this will be benchmarked 80 | // ... 81 | } 82 | 83 | test { 84 | // this will be benchmarked 85 | // the test filter is ignored for unnamed tests 86 | // ... 87 | } 88 | 89 | test "regular test" { 90 | // this will not be benchmarked 91 | return error.NotABenchmark; 92 | } 93 | ``` 94 | 95 | ### public decl integration 96 | This integration allows for fine-grained control over the execution of benchmarks, allowing you to specify various options as well as benchmark functions that take parameters. 97 | 98 | The following snippets shows how you can use this integration. 99 | ```zig 100 | // build.zig 101 | 102 | const addBench = @import("zubench/build.zig").addBench; 103 | 104 | pub fn build(b: *std.build.Builder) void { 105 | // existing build function 106 | // ... 107 | 108 | // benchmarks in "src/file.zig", compiled in ReleaseSafe mode 109 | const benchmark_exe = addBench(b, "src/file.zig", .ReleaseSafe); 110 | 111 | const bench_step = b.step("bench", "Run the benchmarks"); 112 | bench_step.dependOn(&benchmark_exe.run().step); 113 | } 114 | ``` 115 | This will make `zig build bench` run the benchmarks in `src/file.zig`, and print the results. `addBench()` returns a `*LibExeObjStep` for an executable that runs the benchmarks; you can integrate it into your `build.zig` however you wish. Benchmarks are specified in `src/file.zig` by creating a `pub const benchmarks` declaration: 116 | ```zig 117 | // add to src/file.zig 118 | 119 | // the zubench package 120 | const bench = @import("src/bench.zig"); 121 | pub const benchmarks = .{ 122 | .@"benchmark func1" = bench.Spec(func1){ .args = .{ arg1, arg2 }, .max_samples = 100 }, 123 | .@"benchmark func2" = bench.Spec(func2){ 124 | .args = .{ arg1, arg2, arg3 }, 125 | .max_samples = 1000, 126 | .opts = .{ .outlier_detection = .none }}, // disable outlier detection 127 | } 128 | ``` 129 | 130 | The above snippet would cause two benchmarks to be run called "benchmark func1" and "benchmark func2" for functions `func1` and `func2` respectively. The `.args` field of a `Spec` is a `std.meta.ArgsTuple` for the corresponding function, and `.max_samples` determines the maximum number of times the function is run during benchmarking. A complete example can be found in `examples/fib_build.zig`. 131 | 132 | It is also relatively straightforward to write a standalone executable to perform benchmarks without using the build system integration. To create a benchmark for a function `func`, run it (measuring process and wall time) and obtain a report, all that is needed is 133 | 134 | ```zig 135 | var progress = std.Progress.start(.{}); 136 | var bm = try Benchmark(func).init(allocator, "benchmark name", .{ func_arg_1, … }, .{}, max_samples, progress); 137 | const report = bm.run(); 138 | bm.deinit(); 139 | ``` 140 | 141 | The `report` then holds a statistical summary of the benchmark and can used with `std.io.Writer.print` (for terminal-style readable output) or `std.json.stringify` (for JSON output). See `examples/` for complete examples. 142 | 143 | ### Custom exectuable 144 | 145 | It is also possible to write a custom benchmarking executable using **zubench** as a dependency. There is a simple example of this in `examples/fib2.zig` or, you can examine `src/bench_runner.zig`. 146 | 147 | ## examples 148 | 149 | Examples showing some ways of producing and running benchmarks can be found the `examples/` directory. Each of these files are built using the root `build.zig` file. All examples can be built using `zig build examples` and they can be run using `zig build run`. 150 | 151 | ## status 152 | 153 | **zubench** is in early development—the API is not stable at the moment and experiments with the API are planned, so feel free to make suggestions for the API or features you would find useful. 154 | -------------------------------------------------------------------------------- /src/bench.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const builtin = @import("builtin"); 3 | 4 | const time = @import("time.zig"); 5 | const Timer = time.Timer; 6 | 7 | const Allocator = std.mem.Allocator; 8 | 9 | const stats = @import("statistics.zig"); 10 | 11 | pub const Clock = time.Clock; 12 | 13 | pub const default_sample_spec = if (builtin.os.tag != .windows) 14 | [_]Clock{ 15 | .real, 16 | .process, 17 | } 18 | else 19 | [_]Clock{.real}; 20 | 21 | pub const sample_spec = if (@hasDecl(@import("root"), "sample_spec")) 22 | @import("root").sample_spec 23 | else 24 | default_sample_spec; 25 | 26 | fn StructArray(comptime T: type) type { 27 | var fields: [sample_spec.len]std.builtin.Type.StructField = undefined; 28 | inline for (&fields, sample_spec) |*field, spec_elt| { 29 | field.* = .{ 30 | .name = @tagName(spec_elt), 31 | .type = T, 32 | .default_value_ptr = null, 33 | .is_comptime = false, 34 | .alignment = @alignOf(T), 35 | }; 36 | } 37 | return @Type(.{ 38 | .@"struct" = .{ 39 | .layout = .auto, 40 | .fields = &fields, 41 | .decls = &.{}, 42 | .is_tuple = false, 43 | }, 44 | }); 45 | } 46 | 47 | pub const Sample = StructArray(u64); 48 | 49 | const sample_fields = std.meta.fields(Sample); 50 | 51 | pub const Samples = struct { 52 | multi_array_list: std.MultiArrayList(Sample), 53 | 54 | pub fn init(allocator: Allocator, max_samples: usize) !Samples { 55 | var multi_array_list = std.MultiArrayList(Sample){}; 56 | try multi_array_list.setCapacity(allocator, max_samples); 57 | return Samples{ 58 | .multi_array_list = multi_array_list, 59 | }; 60 | } 61 | 62 | pub fn deinit(self: *Samples, allocator: Allocator) void { 63 | self.multi_array_list.deinit(allocator); 64 | } 65 | 66 | pub fn append(self: *Samples, sample: Sample) void { 67 | @setRuntimeSafety(true); 68 | self.multi_array_list.appendAssumeCapacity(sample); 69 | } 70 | 71 | /// Get statistics from benchmarking context 72 | pub fn generateStatistics(self: Samples) RunStats { 73 | const slice = self.multi_array_list.slice(); 74 | var result: RunStats = undefined; 75 | inline for (sample_fields, 0..) |field, i| { 76 | const values = slice.items(@enumFromInt(i)); 77 | const avg = stats.mean(values); 78 | const std_dev = stats.correctedSampleStdDev(values, avg); 79 | @field(result, field.name) = Statistics{ 80 | .n_samples = values.len, 81 | .mean = avg, 82 | .std_dev = std_dev, 83 | }; 84 | } 85 | return result; 86 | } 87 | }; 88 | 89 | fn startCounters() !Counters { 90 | var counters: Counters = undefined; 91 | for (&counters, sample_spec) |*counter, spec_elt| { 92 | counter.* = try time.Timer.start(spec_elt); 93 | } 94 | return counters; 95 | } 96 | 97 | fn resetCounters(counters: *Counters) void { 98 | for (counters) |*counter| { 99 | counter.reset(); 100 | } 101 | } 102 | 103 | pub const Counters = [sample_spec.len]Timer; 104 | 105 | pub const Statistics = struct { 106 | n_samples: usize, 107 | mean: f32, 108 | std_dev: f32, 109 | }; 110 | 111 | pub const RunStats = StructArray(Statistics); 112 | 113 | pub const Context = struct { 114 | counters: Counters, 115 | samples: Samples, 116 | 117 | // Call `deinit()` to free allocated samples 118 | pub fn init(allocator: Allocator, max_samples: usize) !Context { 119 | return Context{ 120 | .counters = try startCounters(), 121 | .samples = try Samples.init(allocator, max_samples), 122 | }; 123 | } 124 | 125 | pub fn deinit(self: *Context, allocator: Allocator) void { 126 | self.samples.deinit(allocator); 127 | } 128 | }; 129 | 130 | pub const Report = struct { 131 | // Thie field should probably be declared as comptime, but the compiler 132 | // doesn't seem to allow initialising a struct with normal and comptime fields 133 | // in the interrupt handler 134 | mode: []const u8 = @tagName(builtin.mode), 135 | name: []const u8, 136 | total_runs: usize, 137 | discarded_runs: u64 = 0, 138 | results: RunStats, 139 | 140 | pub fn format(value: Report, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { 141 | _ = options; 142 | _ = fmt; 143 | 144 | const header_fmt = 145 | \\ » {s} 146 | \\ discarded {d} outliers from {d} runs ({d:.2}% of total runs) 147 | \\ 148 | ; 149 | const pct = pct: { 150 | const numerator: f32 = @floatFromInt(value.discarded_runs); 151 | const denominator: f32 = @floatFromInt(value.total_runs); 152 | break :pct numerator / denominator * 100; 153 | }; 154 | try writer.print(header_fmt, .{ value.name, value.discarded_runs, value.total_runs, pct }); 155 | const counter_fmt = 156 | \\ {s}: 157 | \\ mean: {d: >6.2} 158 | \\ σ: {d: >6.2} 159 | \\ num: {d} 160 | \\ 161 | ; 162 | inline for (sample_fields) |field| { 163 | const counter_stats = @field(value.results, field.name); 164 | try writer.print(counter_fmt, .{ 165 | field.name, 166 | std.fmt.fmtDuration(@intFromFloat(counter_stats.mean)), 167 | std.fmt.fmtDuration(@intFromFloat(counter_stats.std_dev)), 168 | counter_stats.n_samples, 169 | }); 170 | } 171 | } 172 | }; 173 | 174 | pub const Options = struct { 175 | outlier_detection: Outlier = .{ .MAD = 1.4286 * 10.0 }, 176 | 177 | pub const Outlier = union(enum) { 178 | none: void, 179 | MAD: f32, 180 | }; 181 | }; 182 | 183 | pub fn Benchmark(comptime Func: type) type { 184 | return struct { 185 | allocator: Allocator, 186 | name: []const u8, 187 | func: *const Func, 188 | args: Args, 189 | options: Options, 190 | progress: std.Progress.Node, 191 | ctx: Context, 192 | 193 | pub const Args = std.meta.ArgsTuple(Func); 194 | 195 | /// borrows `name`; `name` should remain valid while the Benchmark (or its Report) is in use. 196 | pub fn init( 197 | allocator: Allocator, 198 | name: []const u8, 199 | func: *const Func, 200 | args: Args, 201 | options: Options, 202 | max_samples: usize, 203 | progress: std.Progress.Node, 204 | ) error{ TimerUnsupported, OutOfMemory }!@This() { 205 | const ctx = try Context.init(allocator, max_samples); 206 | return @This(){ 207 | .allocator = allocator, 208 | .name = name, 209 | .func = func, 210 | .args = args, 211 | .options = options, 212 | .progress = progress, 213 | .ctx = ctx, 214 | }; 215 | } 216 | 217 | pub fn deinit(self: *@This()) void { 218 | self.ctx.deinit(self.allocator); 219 | self.name = undefined; 220 | } 221 | 222 | // returns a new `Samples` that contains the samples that 223 | // have 'MAD z-score' for each eounter less than or equal to `cutoff`. 224 | // 225 | // The 'MAD z-score' is the z-score ((x-μ)/σ) with mean replaced by median and 226 | // standard deviation replaced by median absolute deviation from the median. 227 | // 228 | // Assuming a normal distribution, it would make sense to set `cutoff` to `n * 1.4286`, 229 | // where `n` is the cutoff for z-scores you want to keep. 230 | pub fn cleanSamples(self: *@This(), cutoff: f32) !Samples { 231 | const mul_ar = self.ctx.samples.multi_array_list; 232 | const max_len = mul_ar.len; 233 | var result = try Samples.init(self.allocator, max_len); 234 | // assuming that the length of `self` is less than `maxInt(u16)` 235 | var centre: [sample_fields.len]f32 = undefined; 236 | var dispersion: [sample_fields.len]f32 = undefined; 237 | const slice = mul_ar.slice(); 238 | { 239 | const buf = try self.allocator.alloc(u64, max_len); 240 | defer self.allocator.free(buf); 241 | inline for (sample_fields, 0..) |_, i| { 242 | const data = slice.items(@enumFromInt(i)); 243 | @memcpy(buf, data); 244 | centre[i] = stats.median(buf); 245 | dispersion[i] = stats.medianAbsDev(buf, centre[i]); 246 | } 247 | } 248 | 249 | for (0..max_len) |i| { 250 | const sample = mul_ar.get(i); 251 | var outlier = false; 252 | inline for (sample_fields, dispersion, centre) |field, dispersion_elt, centre_elt| { 253 | const zscore = stats.zScore( 254 | dispersion_elt, 255 | centre_elt, 256 | @field(sample, field.name), 257 | ); 258 | outlier = outlier or zscore > cutoff; 259 | } 260 | if (!outlier) { 261 | result.append(sample); 262 | } 263 | } 264 | return result; 265 | } 266 | 267 | pub fn run(self: *@This()) !Report { 268 | const mul_ar = &self.ctx.samples.multi_array_list; 269 | const max_iterations = mul_ar.capacity; 270 | const node = self.progress.start(self.name, max_iterations); 271 | 272 | while (mul_ar.len < mul_ar.capacity) { 273 | resetCounters(&self.ctx.counters); 274 | 275 | switch (@typeInfo(@typeInfo(Func).@"fn".return_type.?)) { 276 | .error_union => { 277 | _ = @call(.never_inline, self.func, self.args) catch |err| { 278 | std.debug.panic("Benchmark {s} returned error {s}", .{ self.name, @errorName(err) }); 279 | }; 280 | }, 281 | else => _ = @call(.never_inline, self.func, self.args), 282 | } 283 | 284 | var sample: Sample = undefined; 285 | inline for (sample_fields, &self.ctx.counters) |field, *timer| { 286 | @field(sample, field.name) = timer.read(); 287 | } 288 | 289 | // WARNING: append() increments samples.len BEFORE adding the data. 290 | // This may mean an asynchronous singal handler will think 291 | // there is one more sample than has actually been stored. 292 | self.ctx.samples.append(sample); 293 | node.completeOne(); 294 | } 295 | node.end(); 296 | 297 | var cleaned_samples = switch (self.options.outlier_detection) { 298 | .none => self.ctx.samples, 299 | .MAD => |cutoff| try self.cleanSamples(cutoff), 300 | }; 301 | defer if (self.options.outlier_detection != .none) cleaned_samples.deinit(self.allocator); 302 | 303 | return Report{ 304 | .name = self.name, 305 | .results = cleaned_samples.generateStatistics(), 306 | .discarded_runs = cleaned_samples.multi_array_list.capacity - cleaned_samples.multi_array_list.len, 307 | .total_runs = cleaned_samples.multi_array_list.capacity, 308 | }; 309 | } 310 | }; 311 | } 312 | 313 | pub fn Spec(comptime func: anytype) type { 314 | if (@typeInfo(@TypeOf(func)).@"fn".params.len == 0) 315 | return struct { 316 | args: std.meta.ArgsTuple(@TypeOf(func)) = .{}, 317 | max_samples: usize, 318 | func: @TypeOf(func) = func, 319 | opts: Options = .{}, 320 | }; 321 | return struct { 322 | args: std.meta.ArgsTuple(@TypeOf(func)), 323 | max_samples: usize, 324 | func: @TypeOf(func) = func, 325 | opts: Options = .{}, 326 | }; 327 | } 328 | 329 | test { 330 | std.testing.refAllDeclsRecursive(@This()); 331 | } 332 | --------------------------------------------------------------------------------