├── .github ├── FUNDING.yml └── workflows │ └── ci.yml ├── .gitignore ├── COPYING ├── Cargo.lock ├── Cargo.toml ├── LICENSE-MIT ├── README.md ├── UNLICENSE ├── rustfmt.toml └── src ├── app.rs ├── data.rs ├── main.rs └── output.rs /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [BurntSushi] 2 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | on: 3 | pull_request: 4 | push: 5 | branches: 6 | - master 7 | schedule: 8 | - cron: '00 01 * * *' 9 | jobs: 10 | test: 11 | name: test 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | matrix: 15 | build: 16 | - pinned 17 | - stable 18 | - beta 19 | - nightly 20 | - macos 21 | - win-msvc 22 | - win-gnu 23 | include: 24 | - build: pinned 25 | os: ubuntu-latest 26 | rust: 1.70.0 27 | - build: stable 28 | os: ubuntu-latest 29 | rust: stable 30 | - build: beta 31 | os: ubuntu-latest 32 | rust: beta 33 | - build: nightly 34 | os: ubuntu-latest 35 | rust: nightly 36 | - build: macos 37 | os: macos-latest 38 | rust: stable 39 | - build: win-msvc 40 | os: windows-latest 41 | rust: stable 42 | - build: win-gnu 43 | os: windows-latest 44 | rust: stable-x86_64-gnu 45 | steps: 46 | - name: Checkout repository 47 | uses: actions/checkout@v3 48 | 49 | - name: Install Rust 50 | uses: dtolnay/rust-toolchain@master 51 | with: 52 | toolchain: ${{ matrix.rust }} 53 | 54 | - name: Build 55 | run: cargo build --verbose 56 | 57 | - name: Run tests 58 | run: cargo test --verbose --all 59 | 60 | rustfmt: 61 | name: rustfmt 62 | runs-on: ubuntu-latest 63 | steps: 64 | - name: Checkout repository 65 | uses: actions/checkout@v3 66 | - name: Install Rust 67 | uses: dtolnay/rust-toolchain@master 68 | with: 69 | toolchain: stable 70 | components: rustfmt 71 | - name: Check formatting 72 | run: | 73 | cargo fmt --all -- --check 74 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | tags 3 | target 4 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | This project is dual-licensed under the Unlicense and MIT licenses. 2 | 3 | You may use this code under the terms of either license. 4 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "1.0.2" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "bitflags" 16 | version = "1.3.2" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 19 | 20 | [[package]] 21 | name = "bstr" 22 | version = "1.6.0" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" 25 | dependencies = [ 26 | "memchr", 27 | "regex-automata", 28 | "serde", 29 | ] 30 | 31 | [[package]] 32 | name = "clap" 33 | version = "2.34.0" 34 | source = "registry+https://github.com/rust-lang/crates.io-index" 35 | checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" 36 | dependencies = [ 37 | "bitflags", 38 | "strsim", 39 | "textwrap", 40 | "unicode-width", 41 | ] 42 | 43 | [[package]] 44 | name = "critcmp" 45 | version = "0.1.8" 46 | dependencies = [ 47 | "clap", 48 | "grep-cli", 49 | "lazy_static", 50 | "regex", 51 | "serde", 52 | "serde_json", 53 | "tabwriter", 54 | "termcolor", 55 | "unicode-width", 56 | "walkdir", 57 | ] 58 | 59 | [[package]] 60 | name = "fnv" 61 | version = "1.0.7" 62 | source = "registry+https://github.com/rust-lang/crates.io-index" 63 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 64 | 65 | [[package]] 66 | name = "globset" 67 | version = "0.4.11" 68 | source = "registry+https://github.com/rust-lang/crates.io-index" 69 | checksum = "1391ab1f92ffcc08911957149833e682aa3fe252b9f45f966d2ef972274c97df" 70 | dependencies = [ 71 | "aho-corasick", 72 | "bstr", 73 | "fnv", 74 | "log", 75 | "regex", 76 | ] 77 | 78 | [[package]] 79 | name = "grep-cli" 80 | version = "0.1.8" 81 | source = "registry+https://github.com/rust-lang/crates.io-index" 82 | checksum = "a174e093eaf510f24dae4b4dd996787d6e299069f05af7437996e91b029a8f8d" 83 | dependencies = [ 84 | "bstr", 85 | "globset", 86 | "lazy_static", 87 | "log", 88 | "regex", 89 | "same-file", 90 | "termcolor", 91 | "winapi-util", 92 | ] 93 | 94 | [[package]] 95 | name = "itoa" 96 | version = "1.0.8" 97 | source = "registry+https://github.com/rust-lang/crates.io-index" 98 | checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" 99 | 100 | [[package]] 101 | name = "lazy_static" 102 | version = "1.4.0" 103 | source = "registry+https://github.com/rust-lang/crates.io-index" 104 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 105 | 106 | [[package]] 107 | name = "log" 108 | version = "0.4.19" 109 | source = "registry+https://github.com/rust-lang/crates.io-index" 110 | checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" 111 | 112 | [[package]] 113 | name = "memchr" 114 | version = "2.5.0" 115 | source = "registry+https://github.com/rust-lang/crates.io-index" 116 | checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" 117 | 118 | [[package]] 119 | name = "proc-macro2" 120 | version = "1.0.64" 121 | source = "registry+https://github.com/rust-lang/crates.io-index" 122 | checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" 123 | dependencies = [ 124 | "unicode-ident", 125 | ] 126 | 127 | [[package]] 128 | name = "quote" 129 | version = "1.0.29" 130 | source = "registry+https://github.com/rust-lang/crates.io-index" 131 | checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" 132 | dependencies = [ 133 | "proc-macro2", 134 | ] 135 | 136 | [[package]] 137 | name = "regex" 138 | version = "1.9.1" 139 | source = "registry+https://github.com/rust-lang/crates.io-index" 140 | checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" 141 | dependencies = [ 142 | "aho-corasick", 143 | "memchr", 144 | "regex-automata", 145 | "regex-syntax", 146 | ] 147 | 148 | [[package]] 149 | name = "regex-automata" 150 | version = "0.3.3" 151 | source = "registry+https://github.com/rust-lang/crates.io-index" 152 | checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" 153 | dependencies = [ 154 | "aho-corasick", 155 | "memchr", 156 | "regex-syntax", 157 | ] 158 | 159 | [[package]] 160 | name = "regex-syntax" 161 | version = "0.7.4" 162 | source = "registry+https://github.com/rust-lang/crates.io-index" 163 | checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" 164 | 165 | [[package]] 166 | name = "ryu" 167 | version = "1.0.14" 168 | source = "registry+https://github.com/rust-lang/crates.io-index" 169 | checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" 170 | 171 | [[package]] 172 | name = "same-file" 173 | version = "1.0.6" 174 | source = "registry+https://github.com/rust-lang/crates.io-index" 175 | checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" 176 | dependencies = [ 177 | "winapi-util", 178 | ] 179 | 180 | [[package]] 181 | name = "serde" 182 | version = "1.0.171" 183 | source = "registry+https://github.com/rust-lang/crates.io-index" 184 | checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" 185 | dependencies = [ 186 | "serde_derive", 187 | ] 188 | 189 | [[package]] 190 | name = "serde_derive" 191 | version = "1.0.171" 192 | source = "registry+https://github.com/rust-lang/crates.io-index" 193 | checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" 194 | dependencies = [ 195 | "proc-macro2", 196 | "quote", 197 | "syn", 198 | ] 199 | 200 | [[package]] 201 | name = "serde_json" 202 | version = "1.0.102" 203 | source = "registry+https://github.com/rust-lang/crates.io-index" 204 | checksum = "b5062a995d481b2308b6064e9af76011f2921c35f97b0468811ed9f6cd91dfed" 205 | dependencies = [ 206 | "itoa", 207 | "ryu", 208 | "serde", 209 | ] 210 | 211 | [[package]] 212 | name = "strsim" 213 | version = "0.8.0" 214 | source = "registry+https://github.com/rust-lang/crates.io-index" 215 | checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" 216 | 217 | [[package]] 218 | name = "syn" 219 | version = "2.0.25" 220 | source = "registry+https://github.com/rust-lang/crates.io-index" 221 | checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" 222 | dependencies = [ 223 | "proc-macro2", 224 | "quote", 225 | "unicode-ident", 226 | ] 227 | 228 | [[package]] 229 | name = "tabwriter" 230 | version = "1.2.1" 231 | source = "registry+https://github.com/rust-lang/crates.io-index" 232 | checksum = "36205cfc997faadcc4b0b87aaef3fbedafe20d38d4959a7ca6ff803564051111" 233 | dependencies = [ 234 | "lazy_static", 235 | "regex", 236 | "unicode-width", 237 | ] 238 | 239 | [[package]] 240 | name = "termcolor" 241 | version = "1.2.0" 242 | source = "registry+https://github.com/rust-lang/crates.io-index" 243 | checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" 244 | dependencies = [ 245 | "winapi-util", 246 | ] 247 | 248 | [[package]] 249 | name = "textwrap" 250 | version = "0.11.0" 251 | source = "registry+https://github.com/rust-lang/crates.io-index" 252 | checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" 253 | dependencies = [ 254 | "unicode-width", 255 | ] 256 | 257 | [[package]] 258 | name = "unicode-ident" 259 | version = "1.0.10" 260 | source = "registry+https://github.com/rust-lang/crates.io-index" 261 | checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" 262 | 263 | [[package]] 264 | name = "unicode-width" 265 | version = "0.1.10" 266 | source = "registry+https://github.com/rust-lang/crates.io-index" 267 | checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" 268 | 269 | [[package]] 270 | name = "walkdir" 271 | version = "2.3.3" 272 | source = "registry+https://github.com/rust-lang/crates.io-index" 273 | checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" 274 | dependencies = [ 275 | "same-file", 276 | "winapi-util", 277 | ] 278 | 279 | [[package]] 280 | name = "winapi" 281 | version = "0.3.9" 282 | source = "registry+https://github.com/rust-lang/crates.io-index" 283 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 284 | dependencies = [ 285 | "winapi-i686-pc-windows-gnu", 286 | "winapi-x86_64-pc-windows-gnu", 287 | ] 288 | 289 | [[package]] 290 | name = "winapi-i686-pc-windows-gnu" 291 | version = "0.4.0" 292 | source = "registry+https://github.com/rust-lang/crates.io-index" 293 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 294 | 295 | [[package]] 296 | name = "winapi-util" 297 | version = "0.1.5" 298 | source = "registry+https://github.com/rust-lang/crates.io-index" 299 | checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" 300 | dependencies = [ 301 | "winapi", 302 | ] 303 | 304 | [[package]] 305 | name = "winapi-x86_64-pc-windows-gnu" 306 | version = "0.4.0" 307 | source = "registry+https://github.com/rust-lang/crates.io-index" 308 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 309 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "critcmp" 3 | version = "0.1.8" #:version 4 | authors = ["Andrew Gallant "] 5 | description = """ 6 | A command line utility for comparing benchmark data generated by Criterion. 7 | """ 8 | documentation = "https://github.com/BurntSushi/critcmp" 9 | homepage = "https://github.com/BurntSushi/critcmp" 10 | repository = "https://github.com/BurntSushi/critcmp" 11 | readme = "README.md" 12 | keywords = ["benchmark", "benchcmp", "compare", "cmp"] 13 | license = "Unlicense/MIT" 14 | edition = "2018" 15 | 16 | [[bin]] 17 | bench = false 18 | path = "src/main.rs" 19 | name = "critcmp" 20 | 21 | [dependencies] 22 | grep-cli = "0.1" 23 | lazy_static = "1.1" 24 | regex = { version = "1.4.5", default-features = false, features = ["std", "unicode"] } 25 | serde = { version = "1", features = ["derive"] } 26 | serde_json = "1" 27 | tabwriter = { version = "1", features = ["ansi_formatting"] } 28 | termcolor = "1" 29 | unicode-width = "0.1" 30 | walkdir = "2.2.5" 31 | 32 | [dependencies.clap] 33 | version = "2.32.0" 34 | default-features = false 35 | features = ["suggestions"] 36 | 37 | [profile.release] 38 | debug = true 39 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Andrew Gallant 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | critcmp 2 | ======= 3 | A command line tool to for comparing benchmarks run by Criterion. This supports 4 | comparing benchmarks both across and inside baselines, where a "baseline" is 5 | a collection of benchmark data produced by Criterion for a single run. 6 | 7 | [![](https://meritbadge.herokuapp.com/critcmp)](https://crates.io/crates/critcmp) 8 | [![Build status](https://github.com/BurntSushi/critcmp/workflows/ci/badge.svg)](https://github.com/BurntSushi/critcmp/actions) 9 | 10 | Dual-licensed under MIT or the [UNLICENSE](https://unlicense.org/). 11 | 12 | 13 | ### Installation 14 | 15 | Since this tool is primarily for use with the Criterion benchmark harness for 16 | Rust, you should install it with Cargo: 17 | 18 | ``` 19 | $ cargo install critcmp 20 | ``` 21 | 22 | critcmp's minimum supported Rust version is the current stable release. 23 | 24 | **WARNING**: This tool explicitly reads undocumented internal data emitted by 25 | Criterion, which means this tool can break at any point if Criterion's internal 26 | data format changes. 27 | 28 | critcmp is known to work with **Criterion 0.3.3**. This project will 29 | track the latest release of Criterion if breaking changes to Criterion's 30 | internal format occur, but will also attempt to keep working on older versions 31 | within reason. 32 | 33 | 34 | ### Example 35 | 36 | [![A screenshot of a critcmp example](https://burntsushi.net/stuff/critcmp.png)](https://burntsushi.net/stuff/critcmp.png) 37 | 38 | 39 | ### Usage 40 | 41 | critcmp works by slurping up all benchmark data from Criterion's target 42 | directory, in addition to extra data supplied as positional parameters. The 43 | primary unit that critcmp works with is Criterion's baselines. That is, the 44 | simplest way to use critcmp is to save two baselines with Criterion's benchmark 45 | harness and then compare them. For example: 46 | 47 | $ cargo bench -- --save-baseline before 48 | $ cargo bench -- --save-baseline change 49 | $ critcmp before change 50 | 51 | Filtering can be done with the -f/--filter flag to limit comparisons based on 52 | a regex: 53 | 54 | $ critcmp before change -f 'foo.*bar' 55 | 56 | Comparisons with very small differences can also be filtered out. For example, 57 | this hides comparisons with differences of 5% or less 58 | 59 | $ critcmp before change -t 5 60 | 61 | Comparisons are not limited to only two baselines. Many can be used: 62 | 63 | $ critcmp before change1 change2 64 | 65 | The list of available baselines known to critcmp can be printed: 66 | 67 | $ critcmp --baselines 68 | 69 | A baseline can exported to one JSON file for more permanent storage outside 70 | of Criterion's target directory: 71 | 72 | $ critcmp --export before > before.json 73 | $ critcmp --export change > change.json 74 | 75 | Baselines saved this way can be used by simply using their file path instead 76 | of just the name: 77 | 78 | $ critcmp before.json change.json 79 | 80 | Benchmarks within the same baseline can be compared as well. Normally, 81 | benchmarks are compared based on their name. That is, given two baselines, the 82 | correspondence between benchmarks is established by their name. Sometimes, 83 | however, you'll want to compare benchmarks that don't have the same name. This 84 | can be done by expressing the matching criteria via a regex. For example, given 85 | benchmarks 'optimized/input1' and 'naive/input1' in the baseline 'benches', the 86 | following will show a comparison between the two benchmarks despite the fact 87 | that they have different names: 88 | 89 | $ critcmp benches -g '\w+/(input1)' 90 | 91 | That is, the matching criteria is determined by the values matched by all of 92 | the capturing groups in the regex. All benchmarks with equivalent capturing 93 | groups will be included in one comparison. There is no limit on the number of 94 | benchmarks that can appear in a single comparison. 95 | 96 | Finally, if comparisons grow too large to see in the default column oriented 97 | display, then the results can be flattened into lists: 98 | 99 | $ critcmp before change1 change2 change3 change4 change5 --list 100 | 101 | 102 | ### Motivation 103 | 104 | This tool is similar to 105 | [cargo-benchcmp](https://github.com/BurntSushi/cargo-benchcmp), 106 | but it works on data gathered by Criterion. 107 | 108 | In particular, Criterion emits loads of useful data, but its facilities for 109 | interactively comparing benchmarks and analyzing benchmarks in the aggregate 110 | are exceedingly limited. Criterion does provide the ability to save benchmark 111 | results as a "baseline," and this is primarily the data with which critcmp 112 | works with. In particular, while Criterion will show changes between a saved 113 | baseline and the current benchmark, there is no way to do further comparative 114 | analysis by looking at benchmark results in different views. 115 | -------------------------------------------------------------------------------- /UNLICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | max_width = 79 2 | use_small_heuristics = "max" 3 | -------------------------------------------------------------------------------- /src/app.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeSet; 2 | use std::fs; 3 | use std::io; 4 | use std::path::{Path, PathBuf}; 5 | 6 | use clap::{crate_authors, crate_version, App, AppSettings, Arg, ArgMatches}; 7 | use grep_cli as cli; 8 | use regex::Regex; 9 | use tabwriter::TabWriter; 10 | use termcolor::{self, WriteColor}; 11 | 12 | use crate::data::{BaseBenchmarks, Benchmarks}; 13 | use crate::Result; 14 | 15 | const TEMPLATE: &'static str = "\ 16 | {bin} {version} 17 | {author} 18 | {about} 19 | 20 | USAGE: 21 | {usage} 22 | 23 | SUBCOMMANDS: 24 | {subcommands} 25 | 26 | OPTIONS: 27 | {unified}"; 28 | 29 | const ABOUT: &'static str = " 30 | critcmp is a tool for comparing benchmark results produced by Criterion. 31 | 32 | critcmp works by slurping up all benchmark data from Criterion's target 33 | directory, in addition to extra data supplied as positional parameters. The 34 | primary unit that critcmp works with is Criterion's baselines. That is, the 35 | simplest way to use critcmp is to save two baselines with Criterion's benchmark 36 | harness and then compare them. For example: 37 | 38 | $ cargo bench -- --save-baseline before 39 | $ cargo bench -- --save-baseline change 40 | $ critcmp before change 41 | 42 | Filtering can be done with the -f/--filter flag to limit comparisons based on 43 | a regex: 44 | 45 | $ critcmp before change -f 'foo.*bar' 46 | 47 | Comparisons with very small differences can also be filtered out. For example, 48 | this hides comparisons with differences of 5% or less 49 | 50 | $ critcmp before change -t 5 51 | 52 | Comparisons are not limited to only two baselines. Many can be used: 53 | 54 | $ critcmp before change1 change2 55 | 56 | The list of available baselines known to critcmp can be printed: 57 | 58 | $ critcmp --baselines 59 | 60 | A baseline can exported to one JSON file for more permanent storage outside 61 | of Criterion's target directory: 62 | 63 | $ critcmp --export before > before.json 64 | $ critcmp --export change > change.json 65 | 66 | Baselines saved this way can be used by simply using their file path instead 67 | of just the name: 68 | 69 | $ critcmp before.json change.json 70 | 71 | Benchmarks within the same baseline can be compared as well. Normally, 72 | benchmarks are compared based on their name. That is, given two baselines, the 73 | correspondence between benchmarks is established by their name. Sometimes, 74 | however, you'll want to compare benchmarks that don't have the same name. This 75 | can be done by expressing the matching criteria via a regex. For example, given 76 | benchmarks 'optimized/input1' and 'naive/input1' in the baseline 'benches', the 77 | following will show a comparison between the two benchmarks despite the fact 78 | that they have different names: 79 | 80 | $ critcmp benches -g '\\w+/(input1)' 81 | 82 | That is, the matching criteria is determined by the values matched by all of 83 | the capturing groups in the regex. All benchmarks with equivalent capturing 84 | groups will be included in one comparison. There is no limit on the number of 85 | benchmarks that can appear in a single comparison. 86 | 87 | Finally, if comparisons grow too large to see in the default column oriented 88 | display, then the results can be flattened into lists: 89 | 90 | $ critcmp before change1 change2 change3 change4 change5 --list 91 | 92 | Project home page: https://github.com/BurntSushi/critcmp 93 | Criterion home page: https://github.com/japaric/criterion.rs"; 94 | 95 | #[derive(Clone, Debug)] 96 | pub struct Args(ArgMatches<'static>); 97 | 98 | impl Args { 99 | pub fn parse() -> Args { 100 | Args(app().get_matches()) 101 | } 102 | 103 | pub fn benchmarks(&self) -> Result { 104 | // First, load benchmark data from command line parameters. If a 105 | // baseline name is given and is not a file path, then it is added to 106 | // our whitelist of baselines. 107 | let mut from_cli: Vec = vec![]; 108 | let mut whitelist = BTreeSet::new(); 109 | if let Some(args) = self.0.values_of_os("args") { 110 | for arg in args { 111 | let p = Path::new(arg); 112 | if p.is_file() { 113 | let baseb = BaseBenchmarks::from_path(p) 114 | .map_err(|err| format!("{}: {}", p.display(), err))?; 115 | whitelist.insert(baseb.name.clone()); 116 | from_cli.push(baseb); 117 | } else { 118 | whitelist.insert(arg.to_string_lossy().into_owned()); 119 | } 120 | } 121 | } 122 | 123 | let mut from_crit: Vec = vec![]; 124 | match self.criterion_dir() { 125 | Err(err) => { 126 | // If we've loaded specific benchmarks from arguments, then it 127 | // shouldn't matter whether we can find a Criterion directory. 128 | // If we haven't loaded anything explicitly though, and if 129 | // Criterion detection fails, then we won't have loaded 130 | // anything and so we should return an error. 131 | if from_cli.is_empty() { 132 | return Err(err); 133 | } 134 | } 135 | Ok(critdir) => { 136 | let data = Benchmarks::gather(critdir)?; 137 | from_crit.extend(data.by_baseline.into_iter().map(|(_, v)| v)); 138 | } 139 | } 140 | if from_cli.is_empty() && from_crit.is_empty() { 141 | fail!("could not find any benchmark data"); 142 | } 143 | 144 | let mut data = Benchmarks::default(); 145 | for basebench in from_crit.into_iter().chain(from_cli) { 146 | if !whitelist.is_empty() && !whitelist.contains(&basebench.name) { 147 | continue; 148 | } 149 | data.by_baseline.insert(basebench.name.clone(), basebench); 150 | } 151 | Ok(data) 152 | } 153 | 154 | pub fn filter(&self) -> Result> { 155 | let pattern_os = match self.0.value_of_os("filter") { 156 | None => return Ok(None), 157 | Some(pattern) => pattern, 158 | }; 159 | let pattern = cli::pattern_from_os(pattern_os)?; 160 | Ok(Some(Regex::new(pattern)?)) 161 | } 162 | 163 | pub fn group(&self) -> Result> { 164 | let pattern_os = match self.0.value_of_os("group") { 165 | None => return Ok(None), 166 | Some(pattern) => pattern, 167 | }; 168 | let pattern = cli::pattern_from_os(pattern_os)?; 169 | let re = Regex::new(pattern)?; 170 | if re.captures_len() <= 1 { 171 | fail!( 172 | "pattern '{}' has no capturing groups, by grouping \ 173 | benchmarks by a regex requires the use of at least \ 174 | one capturing group", 175 | pattern 176 | ); 177 | } 178 | Ok(Some(re)) 179 | } 180 | 181 | pub fn threshold(&self) -> Result> { 182 | let percent = match self.0.value_of_lossy("threshold") { 183 | None => return Ok(None), 184 | Some(percent) => percent, 185 | }; 186 | Ok(Some(percent.parse()?)) 187 | } 188 | 189 | pub fn baselines(&self) -> bool { 190 | self.0.is_present("baselines") 191 | } 192 | 193 | pub fn list(&self) -> bool { 194 | self.0.is_present("list") 195 | } 196 | 197 | pub fn export(&self) -> Option { 198 | self.0.value_of_lossy("export").map(|v| v.into_owned()) 199 | } 200 | 201 | pub fn criterion_dir(&self) -> Result { 202 | let target_dir = self.target_dir()?; 203 | let crit_dir = target_dir.join("criterion"); 204 | if !crit_dir.exists() { 205 | fail!( 206 | "\ 207 | no criterion data exists at {}\n\ 208 | set a different target directory with --target-dir or \ 209 | set CARGO_TARGET_DIR\ 210 | ", 211 | crit_dir.display() 212 | ); 213 | } 214 | Ok(crit_dir) 215 | } 216 | 217 | pub fn stdout(&self) -> Box { 218 | let choice = self.0.value_of("color").unwrap(); 219 | if choice == "always" || (choice == "auto" && cli::is_tty_stdout()) { 220 | Box::new(termcolor::Ansi::new(TabWriter::new(io::stdout()))) 221 | } else { 222 | Box::new(termcolor::NoColor::new(TabWriter::new(io::stdout()))) 223 | } 224 | } 225 | 226 | fn target_dir(&self) -> Result { 227 | if let Some(given) = self.0.value_of_os("target-dir") { 228 | return Ok(PathBuf::from(given)); 229 | } 230 | 231 | let mut cwd = fs::canonicalize(".")?; 232 | loop { 233 | let candidate = cwd.join("target"); 234 | if candidate.exists() { 235 | return Ok(candidate); 236 | } 237 | cwd = match cwd.parent() { 238 | Some(p) => p.to_path_buf(), 239 | None => { 240 | fail!( 241 | "\ 242 | could not find Criterion output directory\n\ 243 | try using --target-dir or set CARGO_TARGET_DIR\ 244 | " 245 | ); 246 | } 247 | } 248 | } 249 | } 250 | } 251 | 252 | fn app() -> App<'static, 'static> { 253 | // The actual App. 254 | App::new("critcmp") 255 | .author(crate_authors!()) 256 | .version(crate_version!()) 257 | .about(ABOUT) 258 | .template(TEMPLATE) 259 | .max_term_width(100) 260 | .setting(AppSettings::UnifiedHelpMessage) 261 | .arg(Arg::with_name("target-dir") 262 | .long("target-dir") 263 | .takes_value(true) 264 | .env("CARGO_TARGET_DIR") 265 | .help("The path to the target directory where Criterion's \ 266 | benchmark data is stored.")) 267 | .arg(Arg::with_name("baselines") 268 | .long("baselines") 269 | .help("List all available baselines.")) 270 | .arg(Arg::with_name("export") 271 | .long("export") 272 | .takes_value(true) 273 | .help("Export all of the benchmark data for a specific baseline \ 274 | as JSON data printed to stdout. A file containing the data \ 275 | written can be passed as a positional argument to critcmp \ 276 | in order to load the baseline data.")) 277 | .arg(Arg::with_name("list") 278 | .long("list") 279 | .help("Show each benchmark comparison as a list. This is useful \ 280 | when there are many comparisons for each benchmark such \ 281 | that they no longer fit in a column view.")) 282 | .arg(Arg::with_name("filter") 283 | .long("filter") 284 | .short("f") 285 | .takes_value(true) 286 | .help("Filter benchmarks by a regex. Benchmark names are given to \ 287 | this regex. Matches are shown while non-matches are not.")) 288 | .arg(Arg::with_name("group") 289 | .long("group") 290 | .short("g") 291 | .takes_value(true) 292 | .help("Group benchmarks by a regex. This requires at least one \ 293 | capturing group. All benchmarks whose capturing group \ 294 | values match are compared with one another.")) 295 | .arg(Arg::with_name("threshold") 296 | .long("threshold") 297 | .short("t") 298 | .takes_value(true) 299 | .help("A threshold where by comparisons with differences below \ 300 | this percentage are not shown. By default, all comparisons \ 301 | are shown. Example use: '-t 5' hides any comparisons with \ 302 | differences under 5%.")) 303 | .arg(Arg::with_name("color") 304 | .long("color") 305 | .takes_value(true) 306 | .possible_values(&["never", "always", "auto"]) 307 | .default_value("auto") 308 | .help("Set whether color should or should not be shown. When \ 309 | 'auto' is used (the default), then color will only be used \ 310 | when printing to a tty.")) 311 | .arg(Arg::with_name("args") 312 | .multiple(true) 313 | .help("A baseline name, file path to a baseline or a regex pattern 314 | for selecting benchmarks.")) 315 | } 316 | -------------------------------------------------------------------------------- /src/data.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | use std::fs::File; 3 | use std::io; 4 | use std::path::Path; 5 | 6 | use serde::de::DeserializeOwned; 7 | use serde::{Deserialize, Serialize}; 8 | use serde_json as json; 9 | use walkdir::WalkDir; 10 | 11 | use crate::Result; 12 | 13 | #[derive(Clone, Debug, Default)] 14 | pub struct Benchmarks { 15 | pub by_baseline: BTreeMap, 16 | } 17 | 18 | #[derive(Clone, Debug, Deserialize, Serialize)] 19 | pub struct BaseBenchmarks { 20 | pub name: String, 21 | pub benchmarks: BTreeMap, 22 | } 23 | 24 | #[derive(Clone, Debug, Deserialize, Serialize)] 25 | pub struct Benchmark { 26 | pub baseline: String, 27 | pub fullname: String, 28 | #[serde(rename = "criterion_benchmark_v1")] 29 | pub info: CBenchmark, 30 | #[serde(rename = "criterion_estimates_v1")] 31 | pub estimates: CEstimates, 32 | } 33 | 34 | #[derive(Clone, Debug, Deserialize, Serialize)] 35 | pub struct CBenchmark { 36 | pub group_id: String, 37 | pub function_id: Option, 38 | pub value_str: Option, 39 | pub throughput: Option, 40 | pub full_id: String, 41 | pub directory_name: String, 42 | } 43 | 44 | #[derive(Clone, Debug, Deserialize, Serialize)] 45 | #[serde(rename_all = "PascalCase")] 46 | pub struct CThroughput { 47 | pub bytes: Option, 48 | pub elements: Option, 49 | } 50 | 51 | #[derive(Clone, Debug, Deserialize, Serialize)] 52 | pub struct CEstimates { 53 | pub mean: CStats, 54 | pub median: CStats, 55 | pub median_abs_dev: CStats, 56 | pub slope: Option, 57 | pub std_dev: CStats, 58 | } 59 | 60 | #[derive(Clone, Debug, Deserialize, Serialize)] 61 | pub struct CStats { 62 | pub confidence_interval: CConfidenceInterval, 63 | pub point_estimate: f64, 64 | pub standard_error: f64, 65 | } 66 | 67 | #[derive(Clone, Debug, Deserialize, Serialize)] 68 | pub struct CConfidenceInterval { 69 | pub confidence_level: f64, 70 | pub lower_bound: f64, 71 | pub upper_bound: f64, 72 | } 73 | 74 | impl Benchmarks { 75 | pub fn gather>(criterion_dir: P) -> Result { 76 | let mut benchmarks = Benchmarks::default(); 77 | for result in WalkDir::new(criterion_dir) { 78 | let dent = result?; 79 | let b = match Benchmark::from_path(dent.path())? { 80 | None => continue, 81 | Some(b) => b, 82 | }; 83 | benchmarks 84 | .by_baseline 85 | .entry(b.baseline.clone()) 86 | .or_insert_with(|| BaseBenchmarks { 87 | name: b.baseline.clone(), 88 | benchmarks: BTreeMap::new(), 89 | }) 90 | .benchmarks 91 | .insert(b.benchmark_name().to_string(), b); 92 | } 93 | Ok(benchmarks) 94 | } 95 | } 96 | 97 | impl Benchmark { 98 | fn from_path>(path: P) -> Result> { 99 | let path = path.as_ref(); 100 | Benchmark::from_path_imp(path).map_err(|err| { 101 | if let Some(parent) = path.parent() { 102 | err!("{}: {}", parent.display(), err) 103 | } else { 104 | err!("unknown path: {}", err) 105 | } 106 | }) 107 | } 108 | 109 | fn from_path_imp(path: &Path) -> Result> { 110 | match path.file_name() { 111 | None => return Ok(None), 112 | Some(filename) => { 113 | if filename != "estimates.json" { 114 | return Ok(None); 115 | } 116 | } 117 | } 118 | // Criterion's directory structure looks like this: 119 | // 120 | // criterion/{group}/{name}/{baseline}/estimates.json 121 | // 122 | // In the same directory as `estimates.json`, there is also a 123 | // `benchmark.json` which contains most of the info we need about 124 | // a benchmark, including its name. From the path, we only extract the 125 | // baseline name. 126 | let parent = path.parent().ok_or_else(|| { 127 | err!("{}: could not find parent dir", path.display()) 128 | })?; 129 | let baseline = parent 130 | .file_name() 131 | .map(|p| p.to_string_lossy().into_owned()) 132 | .ok_or_else(|| { 133 | err!("{}: could not find baseline name", path.display()) 134 | })?; 135 | if baseline == "change" { 136 | // This isn't really a baseline, but special state emitted by 137 | // Criterion to reflect its own comparison between baselines. We 138 | // don't use it. 139 | return Ok(None); 140 | } 141 | 142 | let info = CBenchmark::from_path(parent.join("benchmark.json"))?; 143 | let estimates = CEstimates::from_path(path)?; 144 | let fullname = format!("{}/{}", baseline, info.full_id); 145 | Ok(Some(Benchmark { baseline, fullname, info, estimates })) 146 | } 147 | 148 | pub fn nanoseconds(&self) -> f64 { 149 | self.estimates.mean.point_estimate 150 | } 151 | 152 | pub fn stddev(&self) -> f64 { 153 | self.estimates.std_dev.point_estimate 154 | } 155 | 156 | pub fn fullname(&self) -> &str { 157 | &self.fullname 158 | } 159 | 160 | pub fn baseline(&self) -> &str { 161 | &self.baseline 162 | } 163 | 164 | pub fn benchmark_name(&self) -> &str { 165 | &self.info.full_id 166 | } 167 | 168 | pub fn throughput(&self) -> Option { 169 | const NANOS_PER_SECOND: f64 = 1_000_000_000.0; 170 | 171 | let scale = NANOS_PER_SECOND / self.nanoseconds(); 172 | 173 | self.info.throughput.as_ref().and_then(|t| { 174 | if let Some(num) = t.bytes { 175 | Some(Throughput::Bytes(num as f64 * scale)) 176 | } else if let Some(num) = t.elements { 177 | Some(Throughput::Elements(num as f64 * scale)) 178 | } else { 179 | None 180 | } 181 | }) 182 | } 183 | } 184 | 185 | #[derive(Clone, Copy, Debug)] 186 | pub enum Throughput { 187 | Bytes(f64), 188 | Elements(f64), 189 | } 190 | 191 | impl BaseBenchmarks { 192 | pub fn from_path>(path: P) -> Result { 193 | deserialize_json_path(path.as_ref()) 194 | } 195 | } 196 | 197 | impl CBenchmark { 198 | fn from_path>(path: P) -> Result { 199 | deserialize_json_path(path.as_ref()) 200 | } 201 | } 202 | 203 | impl CEstimates { 204 | fn from_path>(path: P) -> Result { 205 | deserialize_json_path(path.as_ref()) 206 | } 207 | } 208 | 209 | fn deserialize_json_path(path: &Path) -> Result { 210 | let file = File::open(path).map_err(|err| { 211 | if let Some(name) = path.file_name().and_then(|n| n.to_str()) { 212 | err!("{}: {}", name, err) 213 | } else { 214 | err!("{}: {}", path.display(), err) 215 | } 216 | })?; 217 | let buf = io::BufReader::new(file); 218 | let b = json::from_reader(buf).map_err(|err| { 219 | if let Some(name) = path.file_name().and_then(|n| n.to_str()) { 220 | err!("{}: {}", name, err) 221 | } else { 222 | err!("{}: {}", path.display(), err) 223 | } 224 | })?; 225 | Ok(b) 226 | } 227 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | use std::error::Error; 3 | use std::io::{self, Write}; 4 | use std::process; 5 | use std::result; 6 | 7 | use regex::Regex; 8 | 9 | use crate::app::Args; 10 | use crate::data::{Benchmark, Benchmarks}; 11 | 12 | macro_rules! err { 13 | ($($tt:tt)*) => { Box::::from(format!($($tt)*)) } 14 | } 15 | 16 | macro_rules! fail { 17 | ($($tt:tt)*) => { return Err(err!($($tt)*)) } 18 | } 19 | 20 | mod app; 21 | mod data; 22 | mod output; 23 | 24 | type Result = result::Result>; 25 | 26 | fn main() { 27 | if let Err(err) = try_main() { 28 | eprintln!("{}", err); 29 | process::exit(1); 30 | } 31 | } 32 | 33 | fn try_main() -> Result<()> { 34 | let args = Args::parse(); 35 | let benchmarks = args.benchmarks()?; 36 | 37 | if args.baselines() { 38 | let mut stdout = io::stdout(); 39 | for baseline in benchmarks.by_baseline.keys() { 40 | writeln!(stdout, "{}", baseline)?; 41 | } 42 | return Ok(()); 43 | } 44 | if let Some(baseline) = args.export() { 45 | let mut stdout = io::stdout(); 46 | let basedata = match benchmarks.by_baseline.get(&baseline) { 47 | Some(basedata) => basedata, 48 | None => fail!("failed to find baseline '{}'", baseline), 49 | }; 50 | serde_json::to_writer_pretty(&mut stdout, basedata)?; 51 | writeln!(stdout, "")?; 52 | return Ok(()); 53 | } 54 | 55 | let filter = args.filter()?; 56 | let mut comps = match args.group()? { 57 | None => group_by_baseline(&benchmarks, filter.as_ref()), 58 | Some(re) => group_by_regex(&benchmarks, &re, filter.as_ref()), 59 | }; 60 | if let Some(threshold) = args.threshold()? { 61 | comps.retain(|comp| comp.biggest_difference() > threshold); 62 | } 63 | if comps.is_empty() { 64 | fail!("no benchmark comparisons to show"); 65 | } 66 | 67 | let mut wtr = args.stdout(); 68 | if args.list() { 69 | output::rows(&mut wtr, &comps)?; 70 | } else { 71 | output::columns(&mut wtr, &comps)?; 72 | } 73 | wtr.flush()?; 74 | Ok(()) 75 | } 76 | 77 | fn group_by_baseline( 78 | benchmarks: &Benchmarks, 79 | filter: Option<&Regex>, 80 | ) -> Vec { 81 | let mut byname: BTreeMap> = BTreeMap::new(); 82 | for base_benchmarks in benchmarks.by_baseline.values() { 83 | for (name, benchmark) in base_benchmarks.benchmarks.iter() { 84 | if filter.map_or(false, |re| !re.is_match(name)) { 85 | continue; 86 | } 87 | let output_benchmark = output::Benchmark::from_data(benchmark) 88 | .name(benchmark.baseline()); 89 | byname 90 | .entry(name.to_string()) 91 | .or_insert(vec![]) 92 | .push(output_benchmark); 93 | } 94 | } 95 | byname 96 | .into_iter() 97 | .map(|(name, benchmarks)| output::Comparison::new(&name, benchmarks)) 98 | .collect() 99 | } 100 | 101 | fn group_by_regex( 102 | benchmarks: &Benchmarks, 103 | group_by: &Regex, 104 | filter: Option<&Regex>, 105 | ) -> Vec { 106 | let mut byname: BTreeMap> = BTreeMap::new(); 107 | for base_benchmarks in benchmarks.by_baseline.values() { 108 | for (name, benchmark) in base_benchmarks.benchmarks.iter() { 109 | if filter.map_or(false, |re| !re.is_match(name)) { 110 | continue; 111 | } 112 | let (bench, cmp) = match benchmark_names(&benchmark, group_by) { 113 | None => continue, 114 | Some((bench, cmp)) => (bench, cmp), 115 | }; 116 | let output_benchmark = 117 | output::Benchmark::from_data(benchmark).name(&bench); 118 | byname.entry(cmp).or_insert(vec![]).push(output_benchmark); 119 | } 120 | } 121 | byname 122 | .into_iter() 123 | .map(|(name, benchmarks)| output::Comparison::new(&name, benchmarks)) 124 | .collect() 125 | } 126 | 127 | fn benchmark_names( 128 | benchmark: &Benchmark, 129 | group_by: &Regex, 130 | ) -> Option<(String, String)> { 131 | assert!(group_by.captures_len() > 1); 132 | 133 | let caps = match group_by.captures(benchmark.benchmark_name()) { 134 | None => return None, 135 | Some(caps) => caps, 136 | }; 137 | 138 | let mut bench_name = benchmark.benchmark_name().to_string(); 139 | let mut cmp_name = String::new(); 140 | let mut offset = 0; 141 | for option in caps.iter().skip(1) { 142 | let m = match option { 143 | None => continue, 144 | Some(m) => m, 145 | }; 146 | cmp_name.push_str(m.as_str()); 147 | // Strip everything that doesn't match capturing groups. The leftovers 148 | // are our benchmark name. 149 | bench_name.drain((m.start() - offset)..(m.end() - offset)); 150 | offset += m.end() - m.start(); 151 | } 152 | // Add the baseline name to the benchmark to disambiguate it from 153 | // benchmarks with the same name in other baselines. 154 | bench_name.insert_str(0, &format!("{}/", benchmark.baseline())); 155 | 156 | Some((bench_name, cmp_name)) 157 | } 158 | -------------------------------------------------------------------------------- /src/output.rs: -------------------------------------------------------------------------------- 1 | use std::collections::{BTreeMap, BTreeSet}; 2 | use std::iter; 3 | 4 | use termcolor::{Color, ColorSpec, WriteColor}; 5 | use unicode_width::UnicodeWidthStr; 6 | 7 | use crate::data; 8 | use crate::Result; 9 | 10 | #[derive(Clone, Debug)] 11 | pub struct Comparison { 12 | name: String, 13 | benchmarks: Vec, 14 | name_to_index: BTreeMap, 15 | } 16 | 17 | #[derive(Clone, Debug)] 18 | pub struct Benchmark { 19 | name: String, 20 | nanoseconds: f64, 21 | stddev: Option, 22 | throughput: Option, 23 | /// Whether this is the best benchmark in a group. This is only populated 24 | /// when a `Comparison` is built. 25 | best: bool, 26 | /// The rank of this benchmark in a group. The best is always `1.0`. This 27 | /// is only populated when a `Comparison` is built. 28 | rank: f64, 29 | } 30 | 31 | impl Comparison { 32 | pub fn new(name: &str, benchmarks: Vec) -> Comparison { 33 | let mut comp = Comparison { 34 | name: name.to_string(), 35 | benchmarks: benchmarks, 36 | name_to_index: BTreeMap::new(), 37 | }; 38 | if comp.benchmarks.is_empty() { 39 | return comp; 40 | } 41 | 42 | comp.benchmarks.sort_by(|a, b| { 43 | a.nanoseconds.partial_cmp(&b.nanoseconds).unwrap() 44 | }); 45 | comp.benchmarks[0].best = true; 46 | 47 | let top = comp.benchmarks[0].nanoseconds; 48 | for (i, b) in comp.benchmarks.iter_mut().enumerate() { 49 | comp.name_to_index.insert(b.name.to_string(), i); 50 | b.rank = b.nanoseconds / top; 51 | } 52 | comp 53 | } 54 | 55 | /// Return the biggest difference, percentage wise, between benchmarks 56 | /// in this comparison. 57 | /// 58 | /// If this comparison has fewer than two benchmarks, then 0 is returned. 59 | pub fn biggest_difference(&self) -> f64 { 60 | if self.benchmarks.len() < 2 { 61 | return 0.0; 62 | } 63 | let best = self.benchmarks[0].nanoseconds; 64 | let worst = self.benchmarks.last().unwrap().nanoseconds; 65 | ((worst - best) / best) * 100.0 66 | } 67 | 68 | fn get(&self, name: &str) -> Option<&Benchmark> { 69 | self.name_to_index.get(name).and_then(|&i| self.benchmarks.get(i)) 70 | } 71 | } 72 | 73 | impl Benchmark { 74 | pub fn from_data(b: &data::Benchmark) -> Benchmark { 75 | Benchmark { 76 | name: b.fullname().to_string(), 77 | nanoseconds: b.nanoseconds(), 78 | stddev: Some(b.stddev()), 79 | throughput: b.throughput(), 80 | best: false, 81 | rank: 0.0, 82 | } 83 | } 84 | 85 | pub fn name(self, name: &str) -> Benchmark { 86 | Benchmark { name: name.to_string(), ..self } 87 | } 88 | } 89 | 90 | pub fn columns( 91 | mut wtr: W, 92 | groups: &[Comparison], 93 | ) -> Result<()> { 94 | let mut columns = BTreeSet::new(); 95 | for group in groups { 96 | for b in &group.benchmarks { 97 | columns.insert(b.name.to_string()); 98 | } 99 | } 100 | 101 | write!(wtr, "group")?; 102 | for column in &columns { 103 | write!(wtr, "\t {}", column)?; 104 | } 105 | writeln!(wtr, "")?; 106 | 107 | write_divider(&mut wtr, '-', "group".width())?; 108 | for column in &columns { 109 | write!(wtr, "\t ")?; 110 | write_divider(&mut wtr, '-', column.width())?; 111 | } 112 | writeln!(wtr, "")?; 113 | 114 | for group in groups { 115 | if group.benchmarks.is_empty() { 116 | continue; 117 | } 118 | 119 | write!(wtr, "{}", group.name)?; 120 | for column_name in &columns { 121 | let b = match group.get(column_name) { 122 | Some(b) => b, 123 | None => { 124 | write!(wtr, "\t")?; 125 | continue; 126 | } 127 | }; 128 | 129 | if b.best { 130 | let mut spec = ColorSpec::new(); 131 | spec.set_fg(Some(Color::Green)).set_bold(true); 132 | wtr.set_color(&spec)?; 133 | } 134 | write!( 135 | wtr, 136 | "\t {:<5.2} {:>14} {:>14}", 137 | b.rank, 138 | time(b.nanoseconds, b.stddev), 139 | throughput(b.throughput), 140 | )?; 141 | if b.best { 142 | wtr.reset()?; 143 | } 144 | } 145 | writeln!(wtr, "")?; 146 | } 147 | Ok(()) 148 | } 149 | 150 | pub fn rows(mut wtr: W, groups: &[Comparison]) -> Result<()> { 151 | for (i, group) in groups.iter().enumerate() { 152 | if i > 0 { 153 | writeln!(wtr, "")?; 154 | } 155 | rows_one(&mut wtr, group)?; 156 | } 157 | Ok(()) 158 | } 159 | 160 | fn rows_one(mut wtr: W, group: &Comparison) -> Result<()> { 161 | writeln!(wtr, "{}", group.name)?; 162 | write_divider(&mut wtr, '-', group.name.width())?; 163 | writeln!(wtr, "")?; 164 | 165 | if group.benchmarks.is_empty() { 166 | writeln!(wtr, "NOTHING TO SHOW")?; 167 | return Ok(()); 168 | } 169 | 170 | for b in &group.benchmarks { 171 | writeln!( 172 | wtr, 173 | "{}\t{:>7.2}\t{:>15}\t{:>12}", 174 | b.name, 175 | b.rank, 176 | time(b.nanoseconds, b.stddev), 177 | throughput(b.throughput), 178 | )?; 179 | } 180 | Ok(()) 181 | } 182 | 183 | fn write_divider( 184 | mut wtr: W, 185 | divider: char, 186 | width: usize, 187 | ) -> Result<()> { 188 | let div: String = iter::repeat(divider).take(width).collect(); 189 | write!(wtr, "{}", div)?; 190 | Ok(()) 191 | } 192 | 193 | fn time(nanos: f64, stddev: Option) -> String { 194 | const MIN_MICRO: f64 = 2_000.0; 195 | const MIN_MILLI: f64 = 2_000_000.0; 196 | const MIN_SEC: f64 = 2_000_000_000.0; 197 | 198 | let (div, label) = if nanos < MIN_MICRO { 199 | (1.0, "ns") 200 | } else if nanos < MIN_MILLI { 201 | (1_000.0, "µs") 202 | } else if nanos < MIN_SEC { 203 | (1_000_000.0, "ms") 204 | } else { 205 | (1_000_000_000.0, "s") 206 | }; 207 | if let Some(stddev) = stddev { 208 | format!("{:.1}±{:.2}{}", nanos / div, stddev / div, label) 209 | } else { 210 | format!("{:.1}{}", nanos / div, label) 211 | } 212 | } 213 | 214 | fn throughput(throughput: Option) -> String { 215 | use data::Throughput::*; 216 | match throughput { 217 | Some(Bytes(num)) => throughput_per(num, "B"), 218 | Some(Elements(num)) => throughput_per(num, "Elem"), 219 | _ => "? ?/sec".to_string(), 220 | } 221 | } 222 | 223 | fn throughput_per(per: f64, unit: &str) -> String { 224 | const MIN_K: f64 = (2 * (1 << 10) as u64) as f64; 225 | const MIN_M: f64 = (2 * (1 << 20) as u64) as f64; 226 | const MIN_G: f64 = (2 * (1 << 30) as u64) as f64; 227 | 228 | if per < MIN_K { 229 | format!("{} {}/sec", per as u64, unit) 230 | } else if per < MIN_M { 231 | format!("{:.1} K{}/sec", (per / (1 << 10) as f64), unit) 232 | } else if per < MIN_G { 233 | format!("{:.1} M{}/sec", (per / (1 << 20) as f64), unit) 234 | } else { 235 | format!("{:.1} G{}/sec", (per / (1 << 30) as f64), unit) 236 | } 237 | } 238 | --------------------------------------------------------------------------------