├── tests ├── fixtures │ ├── empty_results.expected │ ├── zero_improvements.expected │ ├── zero_regressions.expected │ ├── 4_cmp_5_within_threshold.expected │ ├── 6_cmp_7_within_threshold.expected │ ├── 6_cmp_7_within_threshold_regressions.expected │ ├── 6_cmp_7_within_threshold_improvements.expected │ ├── invalid_arguments.expected │ ├── bench_output_4.txt │ ├── bench_output_5.txt │ ├── bench_output_8.txt │ ├── bench_output_fraction.txt │ ├── bench_output_6.txt │ ├── bench_output_7.txt │ ├── same_input_fractional.expected │ ├── bench_output_3.txt │ ├── bench_output_2.txt │ ├── different_input_selections.expected │ ├── different_input.expected │ ├── different_input_colored.expected │ ├── non_overlapping_input.expected │ ├── bench_output_1.txt │ └── same_input.expected └── integration.rs ├── Makefile ├── coloured_output_example.png ├── COPYING ├── .travis.yml ├── .gitignore ├── appveyor.yml ├── Cargo.toml ├── LICENSE-MIT ├── UNLICENSE ├── src ├── error.rs ├── benchmark.rs └── main.rs ├── README.md └── Cargo.lock /tests/fixtures/empty_results.expected: -------------------------------------------------------------------------------- 1 | WARNING: nothing to output 2 | -------------------------------------------------------------------------------- /tests/fixtures/zero_improvements.expected: -------------------------------------------------------------------------------- 1 | 0/2 benchmarks improved 2 | -------------------------------------------------------------------------------- /tests/fixtures/zero_regressions.expected: -------------------------------------------------------------------------------- 1 | 0/2 benchmarks regressed 2 | -------------------------------------------------------------------------------- /tests/fixtures/4_cmp_5_within_threshold.expected: -------------------------------------------------------------------------------- 1 | All (2) benchmarks are within a 1% threshold 2 | -------------------------------------------------------------------------------- /tests/fixtures/6_cmp_7_within_threshold.expected: -------------------------------------------------------------------------------- 1 | All (5) benchmarks are within a 12% threshold 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | echo Nothing to do... 3 | 4 | push: 5 | git push home master 6 | git push origin master 7 | -------------------------------------------------------------------------------- /tests/fixtures/6_cmp_7_within_threshold_regressions.expected: -------------------------------------------------------------------------------- 1 | All (2/5) regressions are within a 4% threshold 2 | -------------------------------------------------------------------------------- /coloured_output_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BurntSushi/cargo-benchcmp/HEAD/coloured_output_example.png -------------------------------------------------------------------------------- /tests/fixtures/6_cmp_7_within_threshold_improvements.expected: -------------------------------------------------------------------------------- 1 | All (3/5) improvements are within a 3% threshold 2 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | This project is dual-licensed under the Unlicense and MIT licenses. 2 | 3 | You may use this code under the terms of either license. 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | rust: 3 | - stable 4 | - beta 5 | - nightly 6 | script: 7 | - cargo build --verbose 8 | - cargo test --verbose 9 | - cargo doc 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | tags 3 | target 4 | tmp 5 | *.csv 6 | *.fst 7 | *-got 8 | *.csv.idx 9 | words 10 | 98m* 11 | dict 12 | test 13 | months 14 | *.rs.bk 15 | *.iml 16 | .idea/ 17 | -------------------------------------------------------------------------------- /tests/fixtures/invalid_arguments.expected: -------------------------------------------------------------------------------- 1 | Invalid arguments. 2 | 3 | Usage: 4 | cargo benchcmp [options] 5 | cargo benchcmp [options] 6 | cargo benchcmp -h | --help 7 | cargo benchcmp --version 8 | -------------------------------------------------------------------------------- /tests/fixtures/bench_output_4.txt: -------------------------------------------------------------------------------- 1 | running 84 tests 2 | test dense::ac_one_byte ... bench: 349 ns/iter (+/- 5) = 28653 MB/s 3 | test dense::ac_one_prefix_byte_every_match ... bench: 112,957 ns/iter (+/- 1,480) = 88 MB/s 4 | 5 | test result: ok. 0 passed; 0 failed; 0 ignored; 2 measured 6 | -------------------------------------------------------------------------------- /tests/fixtures/bench_output_5.txt: -------------------------------------------------------------------------------- 1 | running 84 tests 2 | test dense::ac_one_byte ... bench: 349 ns/iter (+/- 4) = 28653 MB/s 3 | test dense::ac_one_prefix_byte_every_match ... bench: 112,957 ns/iter (+/- 1,490) = 88 MB/s 4 | 5 | test result: ok. 0 passed; 0 failed; 0 ignored; 2 measured 6 | -------------------------------------------------------------------------------- /tests/fixtures/bench_output_8.txt: -------------------------------------------------------------------------------- 1 | running 84 tests 2 | test dense::ac_one_byte ... bench: 350 ns/iter (+/- 4) = 28653 MB/s 3 | test dense::ac_one_prefix_byte_every_match ... bench: 112,960 ns/iter (+/- 1,490) = 88 MB/s 4 | 5 | test result: ok. 0 passed; 0 failed; 0 ignored; 2 measured 6 | -------------------------------------------------------------------------------- /tests/fixtures/bench_output_fraction.txt: -------------------------------------------------------------------------------- 1 | running 4 tests 2 | test add ... bench: 1.24 ns/iter (+/- 0.00) 3 | test add2 ... bench: 1.48 ns/iter (+/- 0.01) 4 | test add3 ... bench: 1.72 ns/iter (+/- 0.01) 5 | test add4 ... bench: 1.96 ns/iter (+/- 0.01) 6 | test empty ... bench: 1.24 ns/iter (+/- 0.00) -------------------------------------------------------------------------------- /tests/fixtures/bench_output_6.txt: -------------------------------------------------------------------------------- 1 | 2 | running 84 tests 3 | test dense::ac_one_byte ... bench: 349 ns/iter (+/- 5) = 28653 MB/s 4 | test dense::ac_one_prefix_byte_every_match ... bench: 112,957 ns/iter (+/- 1,480) = 88 MB/s 5 | test dense::ac_one_prefix_byte_no_match ... bench: 350 ns/iter (+/- 15) = 28571 MB/s 6 | test dense::ac_one_prefix_byte_random ... bench: 16,096 ns/iter (+/- 292) = 621 MB/s 7 | test dense::ac_ten_bytes ... bench: 58,588 ns/iter (+/- 218) = 170 MB/s 8 | 9 | test result: ok. 0 passed; 0 failed; 0 ignored; 5 measured 10 | -------------------------------------------------------------------------------- /tests/fixtures/bench_output_7.txt: -------------------------------------------------------------------------------- 1 | 2 | running 84 tests 3 | test dense::ac_one_byte ... bench: 351 ns/iter (+/- 6) = 28653 MB/s 4 | test dense::ac_one_prefix_byte_every_match ... bench: 112,960 ns/iter (+/- 1,482) = 88 MB/s 5 | test dense::ac_one_prefix_byte_no_match ... bench: 350 ns/iter (+/- 14) = 28571 MB/s 6 | test dense::ac_one_prefix_byte_random ... bench: 16,090 ns/iter (+/- 291) = 621 MB/s 7 | test dense::ac_ten_bytes ... bench: 58,580 ns/iter (+/- 215) = 170 MB/s 8 | 9 | test result: ok. 0 passed; 0 failed; 0 ignored; 5 measured 10 | -------------------------------------------------------------------------------- /tests/fixtures/same_input_fractional.expected: -------------------------------------------------------------------------------- 1 | name bench_output_fraction.txt ns/iter bench_output_fraction.txt ns/iter diff ns/iter diff % speedup 2 | add 1 1 0 0.00% x 1.00 3 | add2 1 1 0 0.00% x 1.00 4 | add3 1 1 0 0.00% x 1.00 5 | add4 1 1 0 0.00% x 1.00 6 | empty 1 1 0 0.00% x 1.00 7 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | matrix: 3 | - TARGET: x86_64-pc-windows-msvc 4 | - TARGET: i686-pc-windows-msvc 5 | - TARGET: i686-pc-windows-gnu 6 | install: 7 | - ps: Start-FileDownload "https://static.rust-lang.org/dist/rust-nightly-${env:TARGET}.exe" 8 | - rust-nightly-%TARGET%.exe /VERYSILENT /NORESTART /DIR="C:\Program Files (x86)\Rust" 9 | - SET PATH=%PATH%;C:\Program Files (x86)\Rust\bin 10 | - SET PATH=%PATH%;C:\MinGW\bin 11 | - rustc -V 12 | - cargo -V 13 | 14 | build: false 15 | 16 | test_script: 17 | # Workaround https://github.com/nathanross/second_law/issues/14 18 | - set OUT_DIR=.\target\debug\null\null\null 19 | - cargo build --verbose 20 | - cargo test --verbose 21 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = [ 3 | "Jeff Smits ", 4 | "Andrew Gallant ", 5 | ] 6 | name = "cargo-benchcmp" 7 | version = "0.4.5" #:version 8 | description = "A utility for comparing Rust micro-benchmark output." 9 | homepage = "https://github.com/BurntSushi/cargo-benchcmp" 10 | repository = "https://github.com/BurntSushi/cargo-benchcmp" 11 | readme = "README.md" 12 | keywords = ["benchmark", "compare", "bench", "micro"] 13 | license = "Unlicense/MIT" 14 | 15 | [[bin]] 16 | bench = false 17 | path = "src/main.rs" 18 | name = "cargo-benchcmp" 19 | 20 | [dependencies] 21 | docopt = "1" 22 | lazy_static = "1" 23 | regex = "1" 24 | serde = "1.0" 25 | serde_derive = "1.0" 26 | 27 | [dependencies.prettytable-rs] 28 | version = "0.10.0" 29 | default-features = false # Don't use crlf on windows 30 | 31 | [dev-dependencies] 32 | quickcheck = "0.9" 33 | rand = "0.7" 34 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Andrew Gallant 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /UNLICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /tests/fixtures/bench_output_3.txt: -------------------------------------------------------------------------------- 1 | 2 | running 14 tests 3 | test ac_one_byte ... bench: 354 ns/iter (+/- 9) = 28248 MB/s 4 | test ac_one_prefix_byte_every_match ... bench: 150,581 ns/iter (+/- 814) = 66 MB/s 5 | test ac_one_prefix_byte_no_match ... bench: 354 ns/iter (+/- 4) = 28248 MB/s 6 | test ac_one_prefix_byte_random ... bench: 20,273 ns/iter (+/- 60) = 493 MB/s 7 | test ac_ten_bytes ... bench: 108,092 ns/iter (+/- 683) = 92 MB/s 8 | test ac_ten_diff_prefix ... bench: 108,082 ns/iter (+/- 712) = 92 MB/s 9 | test ac_ten_one_prefix_byte_every_match ... bench: 150,561 ns/iter (+/- 824) = 66 MB/s 10 | test ac_ten_one_prefix_byte_no_match ... bench: 354 ns/iter (+/- 2) = 28248 MB/s 11 | test ac_ten_one_prefix_byte_random ... bench: 23,684 ns/iter (+/- 427) = 422 MB/s 12 | test ac_two_bytes ... bench: 3,138 ns/iter (+/- 11) = 3186 MB/s 13 | test ac_two_diff_prefix ... bench: 3,138 ns/iter (+/- 57) = 3186 MB/s 14 | test ac_two_one_prefix_byte_every_match ... bench: 150,571 ns/iter (+/- 1,618) = 66 MB/s 15 | test ac_two_one_prefix_byte_no_match ... bench: 354 ns/iter (+/- 2) = 28248 MB/s 16 | test ac_two_one_prefix_byte_random ... bench: 21,009 ns/iter (+/- 94) = 476 MB/s 17 | 18 | test result: ok. 0 passed; 0 failed; 0 ignored; 14 measured 19 | 20 | -------------------------------------------------------------------------------- /tests/fixtures/bench_output_2.txt: -------------------------------------------------------------------------------- 1 | 2 | running 14 tests 3 | test ac_one_byte ... bench: 349 ns/iter (+/- 5) = 28653 MB/s 4 | test ac_one_prefix_byte_every_match ... bench: 112,957 ns/iter (+/- 1,480) = 88 MB/s 5 | test ac_one_prefix_byte_no_match ... bench: 350 ns/iter (+/- 15) = 28571 MB/s 6 | test ac_one_prefix_byte_random ... bench: 16,096 ns/iter (+/- 292) = 621 MB/s 7 | test ac_ten_bytes ... bench: 58,588 ns/iter (+/- 218) = 170 MB/s 8 | test ac_ten_diff_prefix ... bench: 58,601 ns/iter (+/- 215) = 170 MB/s 9 | test ac_ten_one_prefix_byte_every_match ... bench: 112,920 ns/iter (+/- 1,454) = 88 MB/s 10 | test ac_ten_one_prefix_byte_no_match ... bench: 350 ns/iter (+/- 9) = 28571 MB/s 11 | test ac_ten_one_prefix_byte_random ... bench: 19,181 ns/iter (+/- 251) = 521 MB/s 12 | test ac_two_bytes ... bench: 3,125 ns/iter (+/- 13) = 3200 MB/s 13 | test ac_two_diff_prefix ... bench: 3,124 ns/iter (+/- 32) = 3201 MB/s 14 | test ac_two_one_prefix_byte_every_match ... bench: 112,934 ns/iter (+/- 2,037) = 88 MB/s 15 | test ac_two_one_prefix_byte_no_match ... bench: 350 ns/iter (+/- 4) = 28571 MB/s 16 | test ac_two_one_prefix_byte_random ... bench: 16,511 ns/iter (+/- 142) = 605 MB/s 17 | 18 | test result: ok. 0 passed; 0 failed; 0 ignored; 14 measured 19 | 20 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | use std::error; 3 | use std::fmt; 4 | use std::path::PathBuf; 5 | use std::result; 6 | 7 | use regex; 8 | 9 | pub type Result = result::Result; 10 | 11 | #[derive(Debug)] 12 | pub enum Error { 13 | Regex(regex::Error), 14 | Io(io::Error), 15 | OpenFile { path: PathBuf, err: io::Error }, 16 | } 17 | 18 | impl error::Error for Error { 19 | fn description(&self) -> &str { 20 | match *self { 21 | Error::Regex(ref err) => err.description(), 22 | Error::Io(ref err) => err.description(), 23 | Error::OpenFile { ref err, .. } => err.description(), 24 | } 25 | } 26 | 27 | fn cause(&self) -> Option<&error::Error> { 28 | Some(match *self { 29 | Error::Regex(ref err) => err, 30 | Error::Io(ref err) => err, 31 | Error::OpenFile { ref err, .. } => err, 32 | }) 33 | } 34 | } 35 | 36 | impl fmt::Display for Error { 37 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 38 | match *self { 39 | Error::Regex(ref err) => err.fmt(f), 40 | Error::Io(ref err) => err.fmt(f), 41 | Error::OpenFile { ref path, ref err } => write!(f, "{}: {}", err, path.display()), 42 | } 43 | } 44 | } 45 | 46 | impl From for Error { 47 | fn from(err: regex::Error) -> Error { 48 | Error::Regex(err) 49 | } 50 | } 51 | 52 | impl From for Error { 53 | fn from(err: io::Error) -> Error { 54 | Error::Io(err) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /tests/fixtures/different_input_selections.expected: -------------------------------------------------------------------------------- 1 | name dense:: ns/iter dense_boxed:: ns/iter diff ns/iter diff % speedup 2 | ac_one_byte 349 (28653 MB/s) 354 (28248 MB/s) 5 1.43% x 0.99 3 | ac_one_prefix_byte_every_match 112,957 (88 MB/s) 150,581 (66 MB/s) 37,624 33.31% x 0.75 4 | ac_one_prefix_byte_no_match 350 (28571 MB/s) 354 (28248 MB/s) 4 1.14% x 0.99 5 | ac_one_prefix_byte_random 16,096 (621 MB/s) 20,273 (493 MB/s) 4,177 25.95% x 0.79 6 | ac_ten_bytes 58,588 (170 MB/s) 108,092 (92 MB/s) 49,504 84.50% x 0.54 7 | ac_ten_diff_prefix 58,601 (170 MB/s) 108,082 (92 MB/s) 49,481 84.44% x 0.54 8 | ac_ten_one_prefix_byte_every_match 112,920 (88 MB/s) 150,561 (66 MB/s) 37,641 33.33% x 0.75 9 | ac_ten_one_prefix_byte_no_match 350 (28571 MB/s) 354 (28248 MB/s) 4 1.14% x 0.99 10 | ac_ten_one_prefix_byte_random 19,181 (521 MB/s) 23,684 (422 MB/s) 4,503 23.48% x 0.81 11 | ac_two_bytes 3,125 (3200 MB/s) 3,138 (3186 MB/s) 13 0.42% x 1.00 12 | ac_two_diff_prefix 3,124 (3201 MB/s) 3,138 (3186 MB/s) 14 0.45% x 1.00 13 | ac_two_one_prefix_byte_every_match 112,934 (88 MB/s) 150,571 (66 MB/s) 37,637 33.33% x 0.75 14 | ac_two_one_prefix_byte_no_match 350 (28571 MB/s) 354 (28248 MB/s) 4 1.14% x 0.99 15 | ac_two_one_prefix_byte_random 16,511 (605 MB/s) 21,009 (476 MB/s) 4,498 27.24% x 0.79 16 | -------------------------------------------------------------------------------- /tests/fixtures/different_input.expected: -------------------------------------------------------------------------------- 1 | name bench_output_2.txt ns/iter bench_output_3.txt ns/iter diff ns/iter diff % speedup 2 | ac_one_byte 349 (28653 MB/s) 354 (28248 MB/s) 5 1.43% x 0.99 3 | ac_one_prefix_byte_every_match 112,957 (88 MB/s) 150,581 (66 MB/s) 37,624 33.31% x 0.75 4 | ac_one_prefix_byte_no_match 350 (28571 MB/s) 354 (28248 MB/s) 4 1.14% x 0.99 5 | ac_one_prefix_byte_random 16,096 (621 MB/s) 20,273 (493 MB/s) 4,177 25.95% x 0.79 6 | ac_ten_bytes 58,588 (170 MB/s) 108,092 (92 MB/s) 49,504 84.50% x 0.54 7 | ac_ten_diff_prefix 58,601 (170 MB/s) 108,082 (92 MB/s) 49,481 84.44% x 0.54 8 | ac_ten_one_prefix_byte_every_match 112,920 (88 MB/s) 150,561 (66 MB/s) 37,641 33.33% x 0.75 9 | ac_ten_one_prefix_byte_no_match 350 (28571 MB/s) 354 (28248 MB/s) 4 1.14% x 0.99 10 | ac_ten_one_prefix_byte_random 19,181 (521 MB/s) 23,684 (422 MB/s) 4,503 23.48% x 0.81 11 | ac_two_bytes 3,125 (3200 MB/s) 3,138 (3186 MB/s) 13 0.42% x 1.00 12 | ac_two_diff_prefix 3,124 (3201 MB/s) 3,138 (3186 MB/s) 14 0.45% x 1.00 13 | ac_two_one_prefix_byte_every_match 112,934 (88 MB/s) 150,571 (66 MB/s) 37,637 33.33% x 0.75 14 | ac_two_one_prefix_byte_no_match 350 (28571 MB/s) 354 (28248 MB/s) 4 1.14% x 0.99 15 | ac_two_one_prefix_byte_random 16,511 (605 MB/s) 21,009 (476 MB/s) 4,498 27.24% x 0.79 16 | -------------------------------------------------------------------------------- /tests/fixtures/different_input_colored.expected: -------------------------------------------------------------------------------- 1 | name (B bench_output_2.txt ns/iter(B bench_output_3.txt ns/iter(B diff ns/iter(B diff %(B speedup(B 2 | ac_one_byte (B 349 (28653 MB/s) (B 354 (28248 MB/s) (B  5(B  1.43%(B  x 0.99(B 3 | ac_one_prefix_byte_every_match (B 112,957 (88 MB/s) (B 150,581 (66 MB/s) (B  37,624(B 33.31%(B  x 0.75(B 4 | ac_one_prefix_byte_no_match (B 350 (28571 MB/s) (B 354 (28248 MB/s) (B  4(B  1.14%(B  x 0.99(B 5 | ac_one_prefix_byte_random (B 16,096 (621 MB/s) (B 20,273 (493 MB/s) (B  4,177(B 25.95%(B  x 0.79(B 6 | ac_ten_bytes (B 58,588 (170 MB/s) (B 108,092 (92 MB/s) (B  49,504(B 84.50%(B  x 0.54(B 7 | ac_ten_diff_prefix (B 58,601 (170 MB/s) (B 108,082 (92 MB/s) (B  49,481(B 84.44%(B  x 0.54(B 8 | ac_ten_one_prefix_byte_every_match(B 112,920 (88 MB/s) (B 150,561 (66 MB/s) (B  37,641(B 33.33%(B  x 0.75(B 9 | ac_ten_one_prefix_byte_no_match (B 350 (28571 MB/s) (B 354 (28248 MB/s) (B  4(B  1.14%(B  x 0.99(B 10 | ac_ten_one_prefix_byte_random (B 19,181 (521 MB/s) (B 23,684 (422 MB/s) (B  4,503(B 23.48%(B  x 0.81(B 11 | ac_two_bytes (B 3,125 (3200 MB/s) (B 3,138 (3186 MB/s) (B  13(B  0.42%(B  x 1.00(B 12 | ac_two_diff_prefix (B 3,124 (3201 MB/s) (B 3,138 (3186 MB/s) (B  14(B  0.45%(B  x 1.00(B 13 | ac_two_one_prefix_byte_every_match(B 112,934 (88 MB/s) (B 150,571 (66 MB/s) (B  37,637(B 33.33%(B  x 0.75(B 14 | ac_two_one_prefix_byte_no_match (B 350 (28571 MB/s) (B 354 (28248 MB/s) (B  4(B  1.14%(B  x 0.99(B 15 | ac_two_one_prefix_byte_random (B 16,511 (605 MB/s) (B 21,009 (476 MB/s) (B  4,498(B 27.24%(B  x 0.79(B 16 | -------------------------------------------------------------------------------- /tests/fixtures/non_overlapping_input.expected: -------------------------------------------------------------------------------- 1 | WARNING: benchmarks in old but not in new: dense::ac_one_byte, dense::ac_one_prefix_byte_every_match, dense::ac_one_prefix_byte_no_match, dense::ac_one_prefix_byte_random, dense::ac_ten_bytes, dense::ac_ten_diff_prefix, dense::ac_ten_one_prefix_byte_every_match, dense::ac_ten_one_prefix_byte_no_match, dense::ac_ten_one_prefix_byte_random, dense::ac_two_bytes, dense::ac_two_diff_prefix, dense::ac_two_one_prefix_byte_every_match, dense::ac_two_one_prefix_byte_no_match, dense::ac_two_one_prefix_byte_random, dense_boxed::ac_one_byte, dense_boxed::ac_one_prefix_byte_every_match, dense_boxed::ac_one_prefix_byte_no_match, dense_boxed::ac_one_prefix_byte_random, dense_boxed::ac_ten_bytes, dense_boxed::ac_ten_diff_prefix, dense_boxed::ac_ten_one_prefix_byte_every_match, dense_boxed::ac_ten_one_prefix_byte_no_match, dense_boxed::ac_ten_one_prefix_byte_random, dense_boxed::ac_two_bytes, dense_boxed::ac_two_diff_prefix, dense_boxed::ac_two_one_prefix_byte_every_match, dense_boxed::ac_two_one_prefix_byte_no_match, dense_boxed::ac_two_one_prefix_byte_random, full::ac_one_byte, full::ac_one_prefix_byte_every_match, full::ac_one_prefix_byte_no_match, full::ac_one_prefix_byte_random, full::ac_ten_bytes, full::ac_ten_diff_prefix, full::ac_ten_one_prefix_byte_every_match, full::ac_ten_one_prefix_byte_no_match, full::ac_ten_one_prefix_byte_random, full::ac_two_bytes, full::ac_two_diff_prefix, full::ac_two_one_prefix_byte_every_match, full::ac_two_one_prefix_byte_no_match, full::ac_two_one_prefix_byte_random, full_overlap::ac_one_byte, full_overlap::ac_one_prefix_byte_every_match, full_overlap::ac_one_prefix_byte_no_match, full_overlap::ac_one_prefix_byte_random, full_overlap::ac_ten_bytes, full_overlap::ac_ten_diff_prefix, full_overlap::ac_ten_one_prefix_byte_every_match, full_overlap::ac_ten_one_prefix_byte_no_match, full_overlap::ac_ten_one_prefix_byte_random, full_overlap::ac_two_bytes, full_overlap::ac_two_diff_prefix, full_overlap::ac_two_one_prefix_byte_every_match, full_overlap::ac_two_one_prefix_byte_no_match, full_overlap::ac_two_one_prefix_byte_random, naive_one_byte, naive_one_prefix_byte_every_match, naive_one_prefix_byte_no_match, naive_one_prefix_byte_random, naive_ten_bytes, naive_ten_diff_prefix, naive_ten_one_prefix_byte_every_match, naive_ten_one_prefix_byte_no_match, naive_ten_one_prefix_byte_random, naive_two_bytes, naive_two_diff_prefix, naive_two_one_prefix_byte_every_match, naive_two_one_prefix_byte_no_match, naive_two_one_prefix_byte_random, sparse::ac_one_byte, sparse::ac_one_prefix_byte_every_match, sparse::ac_one_prefix_byte_no_match, sparse::ac_one_prefix_byte_random, sparse::ac_ten_bytes, sparse::ac_ten_diff_prefix, sparse::ac_ten_one_prefix_byte_every_match, sparse::ac_ten_one_prefix_byte_no_match, sparse::ac_ten_one_prefix_byte_random, sparse::ac_two_bytes, sparse::ac_two_diff_prefix, sparse::ac_two_one_prefix_byte_every_match, sparse::ac_two_one_prefix_byte_no_match, sparse::ac_two_one_prefix_byte_random 2 | WARNING: benchmarks in new but not in old: ac_one_byte, ac_one_prefix_byte_every_match, ac_one_prefix_byte_no_match, ac_one_prefix_byte_random, ac_ten_bytes, ac_ten_diff_prefix, ac_ten_one_prefix_byte_every_match, ac_ten_one_prefix_byte_no_match, ac_ten_one_prefix_byte_random, ac_two_bytes, ac_two_diff_prefix, ac_two_one_prefix_byte_every_match, ac_two_one_prefix_byte_no_match, ac_two_one_prefix_byte_random 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | cargo benchcmp 2 | ============== 3 | A small utility for comparing micro-benchmarks produced by `cargo bench`. The 4 | utility takes as input two sets of micro-benchmarks (one "old" and the other 5 | "new") and shows as output a comparison between each benchmark. 6 | 7 | [![Linux build status](https://api.travis-ci.org/BurntSushi/cargo-benchcmp.svg)](https://travis-ci.org/BurntSushi/cargo-benchcmp) 8 | [![Windows build status](https://ci.appveyor.com/api/projects/status/github/BurntSushi/cargo-benchcmp?svg=true)](https://ci.appveyor.com/project/BurntSushi/cargo-benchcmp) 9 | [![crates.io page](http://meritbadge.herokuapp.com/cargo-benchcmp)](https://crates.io/crates/cargo-benchcmp) 10 | 11 | Dual-licensed under MIT or the [UNLICENSE](http://unlicense.org). 12 | 13 | ### Installation 14 | 15 | `cargo benchcmp` can be installed with `cargo install`: 16 | 17 | ``` 18 | $ cargo install cargo-benchcmp 19 | ``` 20 | 21 | The resulting binary should then be in `$HOME/.cargo/bin`. 22 | 23 | ### Criterion support 24 | 25 | This tool only supports the standard benchmark output emitted by `cargo bench`. 26 | For Criterion support, a different tool was developed called 27 | [critcmp](https://github.com/BurntSushi/critcmp). 28 | 29 | ### Example output 30 | 31 | ![Coloured example output on aho-corasick benchmarks](coloured_output_example.png) 32 | 33 | ### Usage 34 | 35 | First, run your benchmarks and save them to a file: 36 | 37 | ``` 38 | $ cargo bench > control 39 | ``` 40 | 41 | Next, apply the changes you'd like to test out, and then run the benchmarks and 42 | save them to a file again. 43 | 44 | ``` 45 | $ cargo bench > variable 46 | ``` 47 | 48 | Finally, use `cargo benchcmp` to compare the benchmark results! 49 | 50 | ``` 51 | $ cargo benchcmp control variable 52 | name bench_output_3.txt ns/iter bench_output_2.txt ns/iter diff ns/iter diff % speedup 53 | ac_one_byte 354 (28248 MB/s) 349 (28653 MB/s) -5 -1.41% x 1.01 54 | ac_one_prefix_byte_every_match 150,581 (66 MB/s) 112,957 (88 MB/s) -37,624 -24.99% x 1.33 55 | ac_one_prefix_byte_no_match 354 (28248 MB/s) 350 (28571 MB/s) -4 -1.13% x 1.01 56 | ac_one_prefix_byte_random 20,273 (493 MB/s) 16,096 (621 MB/s) -4,177 -20.60% x 1.26 57 | ac_ten_bytes 108,092 (92 MB/s) 58,588 (170 MB/s) -49,504 -45.80% x 1.84 58 | ac_ten_diff_prefix 108,082 (92 MB/s) 58,601 (170 MB/s) -49,481 -45.78% x 1.84 59 | ... 60 | ``` 61 | 62 | If you want to compare the same benchmark run in multiple ways, reuse the names 63 | in different modules. Then your benchmark output will look like: 64 | 65 | ``` 66 | module1::ac_two_one_prefix_byte_random ... 67 | module2::ac_two_one_prefix_byte_random ... 68 | ``` 69 | 70 | You can then instruct benchcmp to compare the two modules by providing the two 71 | prefixes, followed by the file containing the output: 72 | 73 | ``` 74 | $ cargo benchcmp module1:: module2:: benchmark-output 75 | name dense_boxed:: ns/iter dense:: ns/iter diff ns/iter diff % speedup 76 | ac_one_byte 354 (28248 MB/s) 349 (28653 MB/s) -5 -1.41% x 1.01 77 | ac_one_prefix_byte_every_match 150,581 (66 MB/s) 112,957 (88 MB/s) -37,624 -24.99% x 1.33 78 | ac_one_prefix_byte_no_match 354 (28248 MB/s) 350 (28571 MB/s) -4 -1.13% x 1.01 79 | ac_one_prefix_byte_random 20,273 (493 MB/s) 16,096 (621 MB/s) -4,177 -20.60% x 1.26 80 | ac_ten_bytes 108,092 (92 MB/s) 58,588 (170 MB/s) -49,504 -45.80% x 1.84 81 | ac_ten_diff_prefix 108,082 (92 MB/s) 58,601 (170 MB/s) -49,481 -45.78% x 1.84 82 | ac_ten_one_prefix_byte_every_match 150,561 (66 MB/s) 112,920 (88 MB/s) -37,641 -25.00% x 1.33 83 | ac_ten_one_prefix_byte_no_match 354 (28248 MB/s) 350 (28571 MB/s) -4 -1.13% x 1.01 84 | ac_ten_one_prefix_byte_random 23,684 (422 MB/s) 19,181 (521 MB/s) -4,503 -19.01% x 1.23 85 | ac_two_bytes 3,138 (3186 MB/s) 3,125 (3200 MB/s) -13 -0.41% x 1.00 86 | ac_two_diff_prefix 3,138 (3186 MB/s) 3,124 (3201 MB/s) -14 -0.45% x 1.00 87 | ac_two_one_prefix_byte_every_match 150,571 (66 MB/s) 112,934 (88 MB/s) -37,637 -25.00% x 1.33 88 | ac_two_one_prefix_byte_no_match 354 (28248 MB/s) 350 (28571 MB/s) -4 -1.13% x 1.01 89 | ac_two_one_prefix_byte_random 21,009 (476 MB/s) 16,511 (605 MB/s) -4,498 -21.41% x 1.27 90 | ``` 91 | 92 | The tool supports basic filtering. For example, it's easy to see only 93 | improvements: 94 | 95 | ``` 96 | $ cargo benchcmp old new --improvements 97 | name full:: ns/iter full_overlap:: ns/iter diff ns/iter diff % speedup 98 | ac_one_byte 367 (27247 MB/s) 367 (27247 MB/s) 0 0.00% x 1.00 99 | ac_two_one_prefix_byte_no_match 371 (26954 MB/s) 368 (27173 MB/s) -3 -0.81% x 1.01 100 | ac_two_one_prefix_byte_random 11,530 (867 MB/s) 11,514 (868 MB/s) -16 -0.14% x 1.00 101 | ``` 102 | 103 | Or only see regressions: 104 | 105 | ``` 106 | $ cargo benchcmp old new --regressions 107 | name full:: ns/iter full_overlap:: ns/iter diff ns/iter diff % speedup 108 | ac_one_prefix_byte_every_match 27,425 (364 MB/s) 27,972 (357 MB/s) 547 1.99% x 0.98 109 | ac_one_prefix_byte_no_match 367 (27247 MB/s) 373 (26809 MB/s) 6 1.63% x 0.98 110 | ac_one_prefix_byte_random 11,076 (902 MB/s) 11,243 (889 MB/s) 167 1.51% x 0.99 111 | ac_ten_bytes 25,474 (392 MB/s) 25,754 (388 MB/s) 280 1.10% x 0.99 112 | ac_ten_diff_prefix 25,466 (392 MB/s) 25,800 (387 MB/s) 334 1.31% x 0.99 113 | ac_ten_one_prefix_byte_every_match 27,424 (364 MB/s) 28,046 (356 MB/s) 622 2.27% x 0.98 114 | ac_ten_one_prefix_byte_no_match 367 (27247 MB/s) 369 (27100 MB/s) 2 0.54% x 0.99 115 | ac_ten_one_prefix_byte_random 13,661 (732 MB/s) 13,742 (727 MB/s) 81 0.59% x 0.99 116 | ac_two_bytes 3,141 (3183 MB/s) 3,164 (3160 MB/s) 23 0.73% x 0.99 117 | ac_two_diff_prefix 3,141 (3183 MB/s) 3,174 (3150 MB/s) 33 1.05% x 0.99 118 | ac_two_one_prefix_byte_every_match 27,638 (361 MB/s) 27,953 (357 MB/s) 315 1.14% x 0.99 119 | 120 | ``` 121 | 122 | Many times, the difference in micro-benchmarks is just noise, so you can filter 123 | by percent difference: 124 | 125 | ``` 126 | $ cargo benchcmp old new --regressions --threshold 2 127 | name full:: ns/iter full_overlap:: ns/iter diff ns/iter diff % speedup 128 | ac_ten_one_prefix_byte_every_match 27,424 (364 MB/s) 28,046 (356 MB/s) 622 2.27% x 0.98 129 | ``` 130 | -------------------------------------------------------------------------------- /tests/fixtures/bench_output_1.txt: -------------------------------------------------------------------------------- 1 | 2 | running 84 tests 3 | test dense::ac_one_byte ... bench: 349 ns/iter (+/- 5) = 28653 MB/s 4 | test dense::ac_one_prefix_byte_every_match ... bench: 112,957 ns/iter (+/- 1,480) = 88 MB/s 5 | test dense::ac_one_prefix_byte_no_match ... bench: 350 ns/iter (+/- 15) = 28571 MB/s 6 | test dense::ac_one_prefix_byte_random ... bench: 16,096 ns/iter (+/- 292) = 621 MB/s 7 | test dense::ac_ten_bytes ... bench: 58,588 ns/iter (+/- 218) = 170 MB/s 8 | test dense::ac_ten_diff_prefix ... bench: 58,601 ns/iter (+/- 215) = 170 MB/s 9 | test dense::ac_ten_one_prefix_byte_every_match ... bench: 112,920 ns/iter (+/- 1,454) = 88 MB/s 10 | test dense::ac_ten_one_prefix_byte_no_match ... bench: 350 ns/iter (+/- 9) = 28571 MB/s 11 | test dense::ac_ten_one_prefix_byte_random ... bench: 19,181 ns/iter (+/- 251) = 521 MB/s 12 | test dense::ac_two_bytes ... bench: 3,125 ns/iter (+/- 13) = 3200 MB/s 13 | test dense::ac_two_diff_prefix ... bench: 3,124 ns/iter (+/- 32) = 3201 MB/s 14 | test dense::ac_two_one_prefix_byte_every_match ... bench: 112,934 ns/iter (+/- 2,037) = 88 MB/s 15 | test dense::ac_two_one_prefix_byte_no_match ... bench: 350 ns/iter (+/- 4) = 28571 MB/s 16 | test dense::ac_two_one_prefix_byte_random ... bench: 16,511 ns/iter (+/- 142) = 605 MB/s 17 | test dense_boxed::ac_one_byte ... bench: 354 ns/iter (+/- 9) = 28248 MB/s 18 | test dense_boxed::ac_one_prefix_byte_every_match ... bench: 150,581 ns/iter (+/- 814) = 66 MB/s 19 | test dense_boxed::ac_one_prefix_byte_no_match ... bench: 354 ns/iter (+/- 4) = 28248 MB/s 20 | test dense_boxed::ac_one_prefix_byte_random ... bench: 20,273 ns/iter (+/- 60) = 493 MB/s 21 | test dense_boxed::ac_ten_bytes ... bench: 108,092 ns/iter (+/- 683) = 92 MB/s 22 | test dense_boxed::ac_ten_diff_prefix ... bench: 108,082 ns/iter (+/- 712) = 92 MB/s 23 | test dense_boxed::ac_ten_one_prefix_byte_every_match ... bench: 150,561 ns/iter (+/- 824) = 66 MB/s 24 | test dense_boxed::ac_ten_one_prefix_byte_no_match ... bench: 354 ns/iter (+/- 2) = 28248 MB/s 25 | test dense_boxed::ac_ten_one_prefix_byte_random ... bench: 23,684 ns/iter (+/- 427) = 422 MB/s 26 | test dense_boxed::ac_two_bytes ... bench: 3,138 ns/iter (+/- 11) = 3186 MB/s 27 | test dense_boxed::ac_two_diff_prefix ... bench: 3,138 ns/iter (+/- 57) = 3186 MB/s 28 | test dense_boxed::ac_two_one_prefix_byte_every_match ... bench: 150,571 ns/iter (+/- 1,618) = 66 MB/s 29 | test dense_boxed::ac_two_one_prefix_byte_no_match ... bench: 354 ns/iter (+/- 2) = 28248 MB/s 30 | test dense_boxed::ac_two_one_prefix_byte_random ... bench: 21,009 ns/iter (+/- 94) = 476 MB/s 31 | test full::ac_one_byte ... bench: 367 ns/iter (+/- 2) = 27247 MB/s 32 | test full::ac_one_prefix_byte_every_match ... bench: 27,425 ns/iter (+/- 382) = 364 MB/s 33 | test full::ac_one_prefix_byte_no_match ... bench: 367 ns/iter (+/- 11) = 27247 MB/s 34 | test full::ac_one_prefix_byte_random ... bench: 11,076 ns/iter (+/- 177) = 902 MB/s 35 | test full::ac_ten_bytes ... bench: 25,474 ns/iter (+/- 278) = 392 MB/s 36 | test full::ac_ten_diff_prefix ... bench: 25,466 ns/iter (+/- 194) = 392 MB/s 37 | test full::ac_ten_one_prefix_byte_every_match ... bench: 27,424 ns/iter (+/- 103) = 364 MB/s 38 | test full::ac_ten_one_prefix_byte_no_match ... bench: 367 ns/iter (+/- 14) = 27247 MB/s 39 | test full::ac_ten_one_prefix_byte_random ... bench: 13,661 ns/iter (+/- 198) = 732 MB/s 40 | test full::ac_two_bytes ... bench: 3,141 ns/iter (+/- 32) = 3183 MB/s 41 | test full::ac_two_diff_prefix ... bench: 3,141 ns/iter (+/- 28) = 3183 MB/s 42 | test full::ac_two_one_prefix_byte_every_match ... bench: 27,638 ns/iter (+/- 616) = 361 MB/s 43 | test full::ac_two_one_prefix_byte_no_match ... bench: 371 ns/iter (+/- 16) = 26954 MB/s 44 | test full::ac_two_one_prefix_byte_random ... bench: 11,530 ns/iter (+/- 1,179) = 867 MB/s 45 | test full_overlap::ac_one_byte ... bench: 367 ns/iter (+/- 17) = 27247 MB/s 46 | test full_overlap::ac_one_prefix_byte_every_match ... bench: 27,972 ns/iter (+/- 613) = 357 MB/s 47 | test full_overlap::ac_one_prefix_byte_no_match ... bench: 373 ns/iter (+/- 18) = 26809 MB/s 48 | test full_overlap::ac_one_prefix_byte_random ... bench: 11,243 ns/iter (+/- 688) = 889 MB/s 49 | test full_overlap::ac_ten_bytes ... bench: 25,754 ns/iter (+/- 511) = 388 MB/s 50 | test full_overlap::ac_ten_diff_prefix ... bench: 25,800 ns/iter (+/- 2,568) = 387 MB/s 51 | test full_overlap::ac_ten_one_prefix_byte_every_match ... bench: 28,046 ns/iter (+/- 7,724) = 356 MB/s 52 | test full_overlap::ac_ten_one_prefix_byte_no_match ... bench: 369 ns/iter (+/- 13) = 27100 MB/s 53 | test full_overlap::ac_ten_one_prefix_byte_random ... bench: 13,742 ns/iter (+/- 240) = 727 MB/s 54 | test full_overlap::ac_two_bytes ... bench: 3,164 ns/iter (+/- 77) = 3160 MB/s 55 | test full_overlap::ac_two_diff_prefix ... bench: 3,174 ns/iter (+/- 591) = 3150 MB/s 56 | test full_overlap::ac_two_one_prefix_byte_every_match ... bench: 27,953 ns/iter (+/- 915) = 357 MB/s 57 | test full_overlap::ac_two_one_prefix_byte_no_match ... bench: 368 ns/iter (+/- 21) = 27173 MB/s 58 | test full_overlap::ac_two_one_prefix_byte_random ... bench: 11,514 ns/iter (+/- 366) = 868 MB/s 59 | test naive_one_byte ... bench: 54,401 ns/iter (+/- 1,108) = 183 MB/s 60 | test naive_one_prefix_byte_every_match ... bench: 67,674 ns/iter (+/- 9,114) = 147 MB/s 61 | test naive_one_prefix_byte_no_match ... bench: 71,770 ns/iter (+/- 12,511) = 139 MB/s 62 | test naive_one_prefix_byte_random ... bench: 84,605 ns/iter (+/- 10,741) = 118 MB/s 63 | test naive_ten_bytes ... bench: 469,473 ns/iter (+/- 11,343) = 21 MB/s 64 | test naive_ten_diff_prefix ... bench: 765,305 ns/iter (+/- 19,072) = 13 MB/s 65 | test naive_ten_one_prefix_byte_every_match ... bench: 725,095 ns/iter (+/- 64,497) = 13 MB/s 66 | test naive_ten_one_prefix_byte_no_match ... bench: 766,842 ns/iter (+/- 248,996) = 13 MB/s 67 | test naive_ten_one_prefix_byte_random ... bench: 773,612 ns/iter (+/- 138,022) = 12 MB/s 68 | test naive_two_bytes ... bench: 97,427 ns/iter (+/- 11,489) = 102 MB/s 69 | test naive_two_diff_prefix ... bench: 156,699 ns/iter (+/- 5,002) = 63 MB/s 70 | test naive_two_one_prefix_byte_every_match ... bench: 148,126 ns/iter (+/- 9,013) = 67 MB/s 71 | test naive_two_one_prefix_byte_no_match ... bench: 155,087 ns/iter (+/- 34,420) = 64 MB/s 72 | test naive_two_one_prefix_byte_random ... bench: 158,576 ns/iter (+/- 2,595) = 63 MB/s 73 | test sparse::ac_one_byte ... bench: 351 ns/iter (+/- 254) = 28490 MB/s 74 | test sparse::ac_one_prefix_byte_every_match ... bench: 80,074 ns/iter (+/- 1,296) = 124 MB/s 75 | test sparse::ac_one_prefix_byte_no_match ... bench: 355 ns/iter (+/- 271) = 28169 MB/s 76 | test sparse::ac_one_prefix_byte_random ... bench: 12,747 ns/iter (+/- 2,274) = 784 MB/s 77 | test sparse::ac_ten_bytes ... bench: 48,807 ns/iter (+/- 7,964) = 204 MB/s 78 | test sparse::ac_ten_diff_prefix ... bench: 48,121 ns/iter (+/- 1,060) = 207 MB/s 79 | test sparse::ac_ten_one_prefix_byte_every_match ... bench: 80,429 ns/iter (+/- 8,953) = 124 MB/s 80 | test sparse::ac_ten_one_prefix_byte_no_match ... bench: 351 ns/iter (+/- 92) = 28490 MB/s 81 | test sparse::ac_ten_one_prefix_byte_random ... bench: 15,244 ns/iter (+/- 185) = 656 MB/s 82 | test sparse::ac_two_bytes ... bench: 3,158 ns/iter (+/- 460) = 3166 MB/s 83 | test sparse::ac_two_diff_prefix ... bench: 3,158 ns/iter (+/- 67) = 3166 MB/s 84 | test sparse::ac_two_one_prefix_byte_every_match ... bench: 80,318 ns/iter (+/- 1,960) = 124 MB/s 85 | test sparse::ac_two_one_prefix_byte_no_match ... bench: 355 ns/iter (+/- 35) = 28169 MB/s 86 | test sparse::ac_two_one_prefix_byte_random ... bench: 13,234 ns/iter (+/- 849) = 755 MB/s 87 | 88 | test result: ok. 0 passed; 0 failed; 0 ignored; 84 measured 89 | 90 | -------------------------------------------------------------------------------- /tests/integration.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | use std::ffi::OsStr; 3 | use std::io::Write; 4 | use std::path::PathBuf; 5 | use std::process::{Command, ExitStatus, Stdio}; 6 | 7 | struct CommandUnderTest { 8 | raw: Command, 9 | stdin: Vec, 10 | run: bool, 11 | stdout: String, 12 | stderr: String, 13 | } 14 | 15 | impl CommandUnderTest { 16 | fn new() -> CommandUnderTest { 17 | // To find the directory where the built binary is, we walk up the directory tree of the test binary until the 18 | // parent is "target/". 19 | let mut binary_path = env::current_exe().expect("need current binary path to find binary to test"); 20 | loop { 21 | { 22 | let parent = binary_path.parent(); 23 | if parent.is_none() { 24 | panic!("Failed to locate binary path from original path: {:?}", env::current_exe()); 25 | } 26 | let parent = parent.unwrap(); 27 | if parent.is_dir() && parent.file_name().unwrap() == "target" { 28 | break; 29 | } 30 | } 31 | binary_path.pop(); 32 | } 33 | 34 | binary_path.push( 35 | if cfg!(target_os = "windows") { 36 | format!("{}.exe", env!("CARGO_PKG_NAME")) 37 | } else { 38 | env!("CARGO_PKG_NAME").to_string() 39 | }); 40 | 41 | let mut cmd = Command::new(binary_path); 42 | 43 | let mut work_dir = PathBuf::new(); 44 | work_dir.push(env!("CARGO_MANIFEST_DIR")); 45 | work_dir.push("tests"); 46 | work_dir.push("fixtures"); 47 | 48 | cmd.stdout(Stdio::piped()) 49 | .stderr(Stdio::piped()) 50 | .current_dir(work_dir); 51 | 52 | CommandUnderTest { 53 | raw: cmd, 54 | run: false, 55 | stdin: Vec::new(), 56 | stdout: String::new(), 57 | stderr: String::new(), 58 | } 59 | } 60 | 61 | fn keep_env(&mut self) -> &mut Self { 62 | self.raw.envs(env::vars()); 63 | self 64 | } 65 | 66 | fn arg>(&mut self, arg: S) -> &mut Self { 67 | self.raw.arg(arg); 68 | self 69 | } 70 | 71 | fn args(&mut self, args: I) -> &mut Self 72 | where I: IntoIterator, 73 | S: AsRef 74 | { 75 | self.raw.args(args); 76 | self 77 | } 78 | 79 | fn pipe_in(&mut self, fixture: &str) -> &mut Self { 80 | self.stdin = Vec::from(fixture); 81 | self.raw.stdin(Stdio::piped()); 82 | self 83 | } 84 | 85 | fn run(&mut self) -> ExitStatus { 86 | let mut child = self.raw.spawn().expect("failed to run command"); 87 | 88 | if self.stdin.len() > 0 { 89 | let stdin = child.stdin.as_mut().expect("failed to open stdin"); 90 | stdin.write_all(&self.stdin).expect("failed to write to stdin") 91 | } 92 | 93 | let output = child.wait_with_output().expect("failed waiting for command to complete"); 94 | self.stdout = String::from_utf8(output.stdout).unwrap(); 95 | self.stderr = String::from_utf8(output.stderr).unwrap(); 96 | self.run = true; 97 | output.status 98 | } 99 | 100 | fn fails(&mut self) -> &mut Self { 101 | assert!(!self.run().success(), "expected command to fail"); 102 | self 103 | } 104 | 105 | fn succeeds(&mut self) -> &mut Self { 106 | let status = self.run(); 107 | assert!(status.success(), format!( 108 | "expected command to succeed, but it failed.\nexit code: {}\nstdout: {}\nstderr:{}\n", 109 | status.code().unwrap(), 110 | self.stdout, 111 | self.stderr, 112 | )); 113 | self 114 | } 115 | 116 | fn no_stdout(&mut self) -> &mut Self { 117 | assert!(self.run, "command has not yet been run, use succeeds()/fails()"); 118 | assert!(self.stdout.is_empty(), format!("expected no stdout, got {}", self.stdout)); 119 | self 120 | } 121 | 122 | fn no_stderr(&mut self) -> &mut Self { 123 | assert!(self.run, "command has not yet been run, use succeeds()/fails()"); 124 | assert!(self.stderr.is_empty(), format!("expected no stderr, got {}", self.stderr)); 125 | self 126 | } 127 | 128 | fn stdout_is(&mut self, expected: &str) -> &mut Self { 129 | assert!(self.run, "command has not yet been run, use succeeds()/fails()"); 130 | assert_eq!(&self.stdout[..], expected, "stdout does not match expected"); 131 | self 132 | } 133 | 134 | fn stderr_is(&mut self, expected: &str) -> &mut Self { 135 | assert!(self.run, "command has not yet been run, use succeeds()/fails()"); 136 | assert_eq!(&self.stderr[..], expected, "stderr does not match expected"); 137 | self 138 | } 139 | } 140 | 141 | fn new_cmd() -> CommandUnderTest { 142 | let mut cmd = CommandUnderTest::new(); 143 | cmd.arg("benchcmp"); 144 | cmd 145 | } 146 | 147 | #[test] 148 | fn invalid_arguments() { 149 | let mut cmd = CommandUnderTest::new(); 150 | cmd.fails().no_stdout().stderr_is(include_str!("fixtures/invalid_arguments.expected")); 151 | } 152 | 153 | #[test] 154 | fn version() { 155 | new_cmd().arg("--version").succeeds().no_stderr().stdout_is(&format!("{}\n", env!("CARGO_PKG_VERSION"))); 156 | } 157 | 158 | #[test] 159 | fn same_input() { 160 | new_cmd() 161 | .args(&["bench_output_1.txt", "bench_output_1.txt"]) 162 | .succeeds() 163 | .stdout_is(include_str!("fixtures/same_input.expected")); 164 | } 165 | 166 | #[test] 167 | fn same_input_fractional() { 168 | new_cmd() 169 | .args(&["bench_output_fraction.txt", "bench_output_fraction.txt"]) 170 | .succeeds() 171 | .stdout_is(include_str!("fixtures/same_input_fractional.expected")); 172 | } 173 | 174 | #[test] 175 | fn different_input() { 176 | new_cmd() 177 | .args(&["bench_output_2.txt", "bench_output_3.txt"]) 178 | .succeeds() 179 | .no_stderr() 180 | .stdout_is(include_str!("fixtures/different_input.expected")); 181 | } 182 | 183 | #[test] 184 | fn non_overlapping_input() { 185 | new_cmd() 186 | .args(&["bench_output_1.txt", "bench_output_2.txt"]) 187 | .succeeds() 188 | .stderr_is(include_str!("fixtures/non_overlapping_input.expected")) 189 | .no_stdout(); 190 | } 191 | 192 | #[cfg(unix)] 193 | #[test] 194 | fn different_input_colored() { 195 | new_cmd() 196 | // NOTE: keeping the environment here so that terminfo is available, 197 | // which is required to get colour code in the output 198 | .keep_env() 199 | .args(&["--color", "always", "bench_output_2.txt", "bench_output_3.txt"]) 200 | .succeeds() 201 | .no_stderr() 202 | .stdout_is(include_str!("fixtures/different_input_colored.expected")); 203 | } 204 | 205 | #[test] 206 | fn different_input_selections() { 207 | new_cmd() 208 | .args(&["dense::", "dense_boxed::", "bench_output_1.txt"]) 209 | .succeeds() 210 | .no_stderr() 211 | .stdout_is(include_str!("fixtures/different_input_selections.expected")); 212 | } 213 | 214 | #[test] 215 | fn stdin() { 216 | new_cmd() 217 | .args(&["dense::", "dense_boxed::", "-"]) 218 | .pipe_in(include_str!("fixtures/bench_output_1.txt")) 219 | .succeeds() 220 | .no_stderr() 221 | .stdout_is(include_str!("fixtures/different_input_selections.expected")); 222 | } 223 | 224 | #[test] 225 | fn empty_results() { 226 | new_cmd() 227 | .args(&["bench_output_4.txt", "bench_output_5.txt", "--regressions", "--improvements"]) 228 | .succeeds() 229 | .no_stdout() 230 | .stderr_is(include_str!("fixtures/empty_results.expected")); 231 | } 232 | 233 | #[test] 234 | fn within_threshold_1_comparing_4_5() { 235 | new_cmd() 236 | .args(&["bench_output_4.txt", "bench_output_5.txt", "--threshold", "1"]) 237 | .succeeds() 238 | .no_stdout() 239 | .stderr_is(include_str!("fixtures/4_cmp_5_within_threshold.expected")); 240 | } 241 | 242 | #[test] 243 | fn within_threshold_12_comparing_6_7() { 244 | new_cmd() 245 | .args(&["bench_output_6.txt", "bench_output_7.txt", "--threshold", "12"]) 246 | .succeeds() 247 | .no_stdout() 248 | .stderr_is(include_str!("fixtures/6_cmp_7_within_threshold.expected")); 249 | } 250 | 251 | #[test] 252 | fn within_threshold_3_comparing_6_7_improvements() { 253 | new_cmd() 254 | .args(&["bench_output_6.txt", "bench_output_7.txt", "--threshold", "3", "--improvements"]) 255 | .succeeds() 256 | .no_stdout() 257 | .stderr_is(include_str!("fixtures/6_cmp_7_within_threshold_improvements.expected")); 258 | } 259 | 260 | #[test] 261 | fn within_threshold_4_comparing_6_7_regressions() { 262 | new_cmd() 263 | .args(&["bench_output_6.txt", "bench_output_7.txt", "--threshold", "4", "--regressions"]) 264 | .succeeds() 265 | .no_stdout() 266 | .stderr_is(include_str!("fixtures/6_cmp_7_within_threshold_regressions.expected")); 267 | } 268 | 269 | #[test] 270 | fn zero_regressions() { 271 | new_cmd() 272 | .args(&["bench_output_4.txt", "bench_output_5.txt", "--regressions"]) 273 | .succeeds() 274 | .no_stdout() 275 | .stderr_is(include_str!("fixtures/zero_regressions.expected")); 276 | } 277 | 278 | #[test] 279 | fn zero_regressions_threshold() { 280 | new_cmd() 281 | .args(&["bench_output_4.txt", "bench_output_5.txt", "--threshold", "2", "--regressions"]) 282 | .succeeds() 283 | .no_stdout() 284 | .stderr_is(include_str!("fixtures/zero_regressions.expected")); 285 | } 286 | 287 | #[test] 288 | fn zero_improvements() { 289 | new_cmd() 290 | .args(&["bench_output_4.txt", "bench_output_8.txt", "--improvements"]) 291 | .succeeds() 292 | .no_stdout() 293 | .stderr_is(include_str!("fixtures/zero_improvements.expected")); 294 | } 295 | 296 | #[test] 297 | fn zero_improvements_threshold() { 298 | new_cmd() 299 | .args(&["bench_output_4.txt", "bench_output_8.txt", "--threshold", "2", "--improvements"]) 300 | .succeeds() 301 | .no_stdout() 302 | .stderr_is(include_str!("fixtures/zero_improvements.expected")); 303 | } 304 | -------------------------------------------------------------------------------- /tests/fixtures/same_input.expected: -------------------------------------------------------------------------------- 1 | name bench_output_1.txt ns/iter bench_output_1.txt ns/iter diff ns/iter diff % speedup 2 | dense::ac_one_byte 349 (28653 MB/s) 349 (28653 MB/s) 0 0.00% x 1.00 3 | dense::ac_one_prefix_byte_every_match 112,957 (88 MB/s) 112,957 (88 MB/s) 0 0.00% x 1.00 4 | dense::ac_one_prefix_byte_no_match 350 (28571 MB/s) 350 (28571 MB/s) 0 0.00% x 1.00 5 | dense::ac_one_prefix_byte_random 16,096 (621 MB/s) 16,096 (621 MB/s) 0 0.00% x 1.00 6 | dense::ac_ten_bytes 58,588 (170 MB/s) 58,588 (170 MB/s) 0 0.00% x 1.00 7 | dense::ac_ten_diff_prefix 58,601 (170 MB/s) 58,601 (170 MB/s) 0 0.00% x 1.00 8 | dense::ac_ten_one_prefix_byte_every_match 112,920 (88 MB/s) 112,920 (88 MB/s) 0 0.00% x 1.00 9 | dense::ac_ten_one_prefix_byte_no_match 350 (28571 MB/s) 350 (28571 MB/s) 0 0.00% x 1.00 10 | dense::ac_ten_one_prefix_byte_random 19,181 (521 MB/s) 19,181 (521 MB/s) 0 0.00% x 1.00 11 | dense::ac_two_bytes 3,125 (3200 MB/s) 3,125 (3200 MB/s) 0 0.00% x 1.00 12 | dense::ac_two_diff_prefix 3,124 (3201 MB/s) 3,124 (3201 MB/s) 0 0.00% x 1.00 13 | dense::ac_two_one_prefix_byte_every_match 112,934 (88 MB/s) 112,934 (88 MB/s) 0 0.00% x 1.00 14 | dense::ac_two_one_prefix_byte_no_match 350 (28571 MB/s) 350 (28571 MB/s) 0 0.00% x 1.00 15 | dense::ac_two_one_prefix_byte_random 16,511 (605 MB/s) 16,511 (605 MB/s) 0 0.00% x 1.00 16 | dense_boxed::ac_one_byte 354 (28248 MB/s) 354 (28248 MB/s) 0 0.00% x 1.00 17 | dense_boxed::ac_one_prefix_byte_every_match 150,581 (66 MB/s) 150,581 (66 MB/s) 0 0.00% x 1.00 18 | dense_boxed::ac_one_prefix_byte_no_match 354 (28248 MB/s) 354 (28248 MB/s) 0 0.00% x 1.00 19 | dense_boxed::ac_one_prefix_byte_random 20,273 (493 MB/s) 20,273 (493 MB/s) 0 0.00% x 1.00 20 | dense_boxed::ac_ten_bytes 108,092 (92 MB/s) 108,092 (92 MB/s) 0 0.00% x 1.00 21 | dense_boxed::ac_ten_diff_prefix 108,082 (92 MB/s) 108,082 (92 MB/s) 0 0.00% x 1.00 22 | dense_boxed::ac_ten_one_prefix_byte_every_match 150,561 (66 MB/s) 150,561 (66 MB/s) 0 0.00% x 1.00 23 | dense_boxed::ac_ten_one_prefix_byte_no_match 354 (28248 MB/s) 354 (28248 MB/s) 0 0.00% x 1.00 24 | dense_boxed::ac_ten_one_prefix_byte_random 23,684 (422 MB/s) 23,684 (422 MB/s) 0 0.00% x 1.00 25 | dense_boxed::ac_two_bytes 3,138 (3186 MB/s) 3,138 (3186 MB/s) 0 0.00% x 1.00 26 | dense_boxed::ac_two_diff_prefix 3,138 (3186 MB/s) 3,138 (3186 MB/s) 0 0.00% x 1.00 27 | dense_boxed::ac_two_one_prefix_byte_every_match 150,571 (66 MB/s) 150,571 (66 MB/s) 0 0.00% x 1.00 28 | dense_boxed::ac_two_one_prefix_byte_no_match 354 (28248 MB/s) 354 (28248 MB/s) 0 0.00% x 1.00 29 | dense_boxed::ac_two_one_prefix_byte_random 21,009 (476 MB/s) 21,009 (476 MB/s) 0 0.00% x 1.00 30 | full::ac_one_byte 367 (27247 MB/s) 367 (27247 MB/s) 0 0.00% x 1.00 31 | full::ac_one_prefix_byte_every_match 27,425 (364 MB/s) 27,425 (364 MB/s) 0 0.00% x 1.00 32 | full::ac_one_prefix_byte_no_match 367 (27247 MB/s) 367 (27247 MB/s) 0 0.00% x 1.00 33 | full::ac_one_prefix_byte_random 11,076 (902 MB/s) 11,076 (902 MB/s) 0 0.00% x 1.00 34 | full::ac_ten_bytes 25,474 (392 MB/s) 25,474 (392 MB/s) 0 0.00% x 1.00 35 | full::ac_ten_diff_prefix 25,466 (392 MB/s) 25,466 (392 MB/s) 0 0.00% x 1.00 36 | full::ac_ten_one_prefix_byte_every_match 27,424 (364 MB/s) 27,424 (364 MB/s) 0 0.00% x 1.00 37 | full::ac_ten_one_prefix_byte_no_match 367 (27247 MB/s) 367 (27247 MB/s) 0 0.00% x 1.00 38 | full::ac_ten_one_prefix_byte_random 13,661 (732 MB/s) 13,661 (732 MB/s) 0 0.00% x 1.00 39 | full::ac_two_bytes 3,141 (3183 MB/s) 3,141 (3183 MB/s) 0 0.00% x 1.00 40 | full::ac_two_diff_prefix 3,141 (3183 MB/s) 3,141 (3183 MB/s) 0 0.00% x 1.00 41 | full::ac_two_one_prefix_byte_every_match 27,638 (361 MB/s) 27,638 (361 MB/s) 0 0.00% x 1.00 42 | full::ac_two_one_prefix_byte_no_match 371 (26954 MB/s) 371 (26954 MB/s) 0 0.00% x 1.00 43 | full::ac_two_one_prefix_byte_random 11,530 (867 MB/s) 11,530 (867 MB/s) 0 0.00% x 1.00 44 | full_overlap::ac_one_byte 367 (27247 MB/s) 367 (27247 MB/s) 0 0.00% x 1.00 45 | full_overlap::ac_one_prefix_byte_every_match 27,972 (357 MB/s) 27,972 (357 MB/s) 0 0.00% x 1.00 46 | full_overlap::ac_one_prefix_byte_no_match 373 (26809 MB/s) 373 (26809 MB/s) 0 0.00% x 1.00 47 | full_overlap::ac_one_prefix_byte_random 11,243 (889 MB/s) 11,243 (889 MB/s) 0 0.00% x 1.00 48 | full_overlap::ac_ten_bytes 25,754 (388 MB/s) 25,754 (388 MB/s) 0 0.00% x 1.00 49 | full_overlap::ac_ten_diff_prefix 25,800 (387 MB/s) 25,800 (387 MB/s) 0 0.00% x 1.00 50 | full_overlap::ac_ten_one_prefix_byte_every_match 28,046 (356 MB/s) 28,046 (356 MB/s) 0 0.00% x 1.00 51 | full_overlap::ac_ten_one_prefix_byte_no_match 369 (27100 MB/s) 369 (27100 MB/s) 0 0.00% x 1.00 52 | full_overlap::ac_ten_one_prefix_byte_random 13,742 (727 MB/s) 13,742 (727 MB/s) 0 0.00% x 1.00 53 | full_overlap::ac_two_bytes 3,164 (3160 MB/s) 3,164 (3160 MB/s) 0 0.00% x 1.00 54 | full_overlap::ac_two_diff_prefix 3,174 (3150 MB/s) 3,174 (3150 MB/s) 0 0.00% x 1.00 55 | full_overlap::ac_two_one_prefix_byte_every_match 27,953 (357 MB/s) 27,953 (357 MB/s) 0 0.00% x 1.00 56 | full_overlap::ac_two_one_prefix_byte_no_match 368 (27173 MB/s) 368 (27173 MB/s) 0 0.00% x 1.00 57 | full_overlap::ac_two_one_prefix_byte_random 11,514 (868 MB/s) 11,514 (868 MB/s) 0 0.00% x 1.00 58 | naive_one_byte 54,401 (183 MB/s) 54,401 (183 MB/s) 0 0.00% x 1.00 59 | naive_one_prefix_byte_every_match 67,674 (147 MB/s) 67,674 (147 MB/s) 0 0.00% x 1.00 60 | naive_one_prefix_byte_no_match 71,770 (139 MB/s) 71,770 (139 MB/s) 0 0.00% x 1.00 61 | naive_one_prefix_byte_random 84,605 (118 MB/s) 84,605 (118 MB/s) 0 0.00% x 1.00 62 | naive_ten_bytes 469,473 (21 MB/s) 469,473 (21 MB/s) 0 0.00% x 1.00 63 | naive_ten_diff_prefix 765,305 (13 MB/s) 765,305 (13 MB/s) 0 0.00% x 1.00 64 | naive_ten_one_prefix_byte_every_match 725,095 (13 MB/s) 725,095 (13 MB/s) 0 0.00% x 1.00 65 | naive_ten_one_prefix_byte_no_match 766,842 (13 MB/s) 766,842 (13 MB/s) 0 0.00% x 1.00 66 | naive_ten_one_prefix_byte_random 773,612 (12 MB/s) 773,612 (12 MB/s) 0 0.00% x 1.00 67 | naive_two_bytes 97,427 (102 MB/s) 97,427 (102 MB/s) 0 0.00% x 1.00 68 | naive_two_diff_prefix 156,699 (63 MB/s) 156,699 (63 MB/s) 0 0.00% x 1.00 69 | naive_two_one_prefix_byte_every_match 148,126 (67 MB/s) 148,126 (67 MB/s) 0 0.00% x 1.00 70 | naive_two_one_prefix_byte_no_match 155,087 (64 MB/s) 155,087 (64 MB/s) 0 0.00% x 1.00 71 | naive_two_one_prefix_byte_random 158,576 (63 MB/s) 158,576 (63 MB/s) 0 0.00% x 1.00 72 | sparse::ac_one_byte 351 (28490 MB/s) 351 (28490 MB/s) 0 0.00% x 1.00 73 | sparse::ac_one_prefix_byte_every_match 80,074 (124 MB/s) 80,074 (124 MB/s) 0 0.00% x 1.00 74 | sparse::ac_one_prefix_byte_no_match 355 (28169 MB/s) 355 (28169 MB/s) 0 0.00% x 1.00 75 | sparse::ac_one_prefix_byte_random 12,747 (784 MB/s) 12,747 (784 MB/s) 0 0.00% x 1.00 76 | sparse::ac_ten_bytes 48,807 (204 MB/s) 48,807 (204 MB/s) 0 0.00% x 1.00 77 | sparse::ac_ten_diff_prefix 48,121 (207 MB/s) 48,121 (207 MB/s) 0 0.00% x 1.00 78 | sparse::ac_ten_one_prefix_byte_every_match 80,429 (124 MB/s) 80,429 (124 MB/s) 0 0.00% x 1.00 79 | sparse::ac_ten_one_prefix_byte_no_match 351 (28490 MB/s) 351 (28490 MB/s) 0 0.00% x 1.00 80 | sparse::ac_ten_one_prefix_byte_random 15,244 (656 MB/s) 15,244 (656 MB/s) 0 0.00% x 1.00 81 | sparse::ac_two_bytes 3,158 (3166 MB/s) 3,158 (3166 MB/s) 0 0.00% x 1.00 82 | sparse::ac_two_diff_prefix 3,158 (3166 MB/s) 3,158 (3166 MB/s) 0 0.00% x 1.00 83 | sparse::ac_two_one_prefix_byte_every_match 80,318 (124 MB/s) 80,318 (124 MB/s) 0 0.00% x 1.00 84 | sparse::ac_two_one_prefix_byte_no_match 355 (28169 MB/s) 355 (28169 MB/s) 0 0.00% x 1.00 85 | sparse::ac_two_one_prefix_byte_random 13,234 (755 MB/s) 13,234 (755 MB/s) 0 0.00% x 1.00 86 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "1.1.3" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "bitflags" 16 | version = "2.6.0" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" 19 | 20 | [[package]] 21 | name = "byteorder" 22 | version = "1.5.0" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" 25 | 26 | [[package]] 27 | name = "cargo-benchcmp" 28 | version = "0.4.5" 29 | dependencies = [ 30 | "docopt", 31 | "lazy_static", 32 | "prettytable-rs", 33 | "quickcheck", 34 | "rand", 35 | "regex", 36 | "serde", 37 | "serde_derive", 38 | ] 39 | 40 | [[package]] 41 | name = "cfg-if" 42 | version = "1.0.0" 43 | source = "registry+https://github.com/rust-lang/crates.io-index" 44 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 45 | 46 | [[package]] 47 | name = "dirs-next" 48 | version = "2.0.0" 49 | source = "registry+https://github.com/rust-lang/crates.io-index" 50 | checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" 51 | dependencies = [ 52 | "cfg-if", 53 | "dirs-sys-next", 54 | ] 55 | 56 | [[package]] 57 | name = "dirs-sys-next" 58 | version = "0.1.2" 59 | source = "registry+https://github.com/rust-lang/crates.io-index" 60 | checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" 61 | dependencies = [ 62 | "libc", 63 | "redox_users", 64 | "winapi", 65 | ] 66 | 67 | [[package]] 68 | name = "docopt" 69 | version = "1.1.1" 70 | source = "registry+https://github.com/rust-lang/crates.io-index" 71 | checksum = "7f3f119846c823f9eafcf953a8f6ffb6ed69bf6240883261a7f13b634579a51f" 72 | dependencies = [ 73 | "lazy_static", 74 | "regex", 75 | "serde", 76 | "strsim", 77 | ] 78 | 79 | [[package]] 80 | name = "encode_unicode" 81 | version = "1.0.0" 82 | source = "registry+https://github.com/rust-lang/crates.io-index" 83 | checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" 84 | 85 | [[package]] 86 | name = "env_logger" 87 | version = "0.7.1" 88 | source = "registry+https://github.com/rust-lang/crates.io-index" 89 | checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" 90 | dependencies = [ 91 | "log", 92 | "regex", 93 | ] 94 | 95 | [[package]] 96 | name = "getrandom" 97 | version = "0.1.16" 98 | source = "registry+https://github.com/rust-lang/crates.io-index" 99 | checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" 100 | dependencies = [ 101 | "cfg-if", 102 | "libc", 103 | "wasi 0.9.0+wasi-snapshot-preview1", 104 | ] 105 | 106 | [[package]] 107 | name = "getrandom" 108 | version = "0.2.15" 109 | source = "registry+https://github.com/rust-lang/crates.io-index" 110 | checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" 111 | dependencies = [ 112 | "cfg-if", 113 | "libc", 114 | "wasi 0.11.0+wasi-snapshot-preview1", 115 | ] 116 | 117 | [[package]] 118 | name = "hermit-abi" 119 | version = "0.4.0" 120 | source = "registry+https://github.com/rust-lang/crates.io-index" 121 | checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" 122 | 123 | [[package]] 124 | name = "is-terminal" 125 | version = "0.4.13" 126 | source = "registry+https://github.com/rust-lang/crates.io-index" 127 | checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" 128 | dependencies = [ 129 | "hermit-abi", 130 | "libc", 131 | "windows-sys", 132 | ] 133 | 134 | [[package]] 135 | name = "lazy_static" 136 | version = "1.5.0" 137 | source = "registry+https://github.com/rust-lang/crates.io-index" 138 | checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" 139 | 140 | [[package]] 141 | name = "libc" 142 | version = "0.2.158" 143 | source = "registry+https://github.com/rust-lang/crates.io-index" 144 | checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" 145 | 146 | [[package]] 147 | name = "libredox" 148 | version = "0.1.3" 149 | source = "registry+https://github.com/rust-lang/crates.io-index" 150 | checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" 151 | dependencies = [ 152 | "bitflags", 153 | "libc", 154 | ] 155 | 156 | [[package]] 157 | name = "log" 158 | version = "0.4.22" 159 | source = "registry+https://github.com/rust-lang/crates.io-index" 160 | checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" 161 | 162 | [[package]] 163 | name = "memchr" 164 | version = "2.7.4" 165 | source = "registry+https://github.com/rust-lang/crates.io-index" 166 | checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" 167 | 168 | [[package]] 169 | name = "ppv-lite86" 170 | version = "0.2.20" 171 | source = "registry+https://github.com/rust-lang/crates.io-index" 172 | checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" 173 | dependencies = [ 174 | "zerocopy", 175 | ] 176 | 177 | [[package]] 178 | name = "prettytable-rs" 179 | version = "0.10.0" 180 | source = "registry+https://github.com/rust-lang/crates.io-index" 181 | checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a" 182 | dependencies = [ 183 | "encode_unicode", 184 | "is-terminal", 185 | "lazy_static", 186 | "term", 187 | "unicode-width", 188 | ] 189 | 190 | [[package]] 191 | name = "proc-macro2" 192 | version = "1.0.86" 193 | source = "registry+https://github.com/rust-lang/crates.io-index" 194 | checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" 195 | dependencies = [ 196 | "unicode-ident", 197 | ] 198 | 199 | [[package]] 200 | name = "quickcheck" 201 | version = "0.9.2" 202 | source = "registry+https://github.com/rust-lang/crates.io-index" 203 | checksum = "a44883e74aa97ad63db83c4bf8ca490f02b2fc02f92575e720c8551e843c945f" 204 | dependencies = [ 205 | "env_logger", 206 | "log", 207 | "rand", 208 | "rand_core", 209 | ] 210 | 211 | [[package]] 212 | name = "quote" 213 | version = "1.0.37" 214 | source = "registry+https://github.com/rust-lang/crates.io-index" 215 | checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" 216 | dependencies = [ 217 | "proc-macro2", 218 | ] 219 | 220 | [[package]] 221 | name = "rand" 222 | version = "0.7.3" 223 | source = "registry+https://github.com/rust-lang/crates.io-index" 224 | checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" 225 | dependencies = [ 226 | "getrandom 0.1.16", 227 | "libc", 228 | "rand_chacha", 229 | "rand_core", 230 | "rand_hc", 231 | ] 232 | 233 | [[package]] 234 | name = "rand_chacha" 235 | version = "0.2.2" 236 | source = "registry+https://github.com/rust-lang/crates.io-index" 237 | checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" 238 | dependencies = [ 239 | "ppv-lite86", 240 | "rand_core", 241 | ] 242 | 243 | [[package]] 244 | name = "rand_core" 245 | version = "0.5.1" 246 | source = "registry+https://github.com/rust-lang/crates.io-index" 247 | checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" 248 | dependencies = [ 249 | "getrandom 0.1.16", 250 | ] 251 | 252 | [[package]] 253 | name = "rand_hc" 254 | version = "0.2.0" 255 | source = "registry+https://github.com/rust-lang/crates.io-index" 256 | checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" 257 | dependencies = [ 258 | "rand_core", 259 | ] 260 | 261 | [[package]] 262 | name = "redox_users" 263 | version = "0.4.6" 264 | source = "registry+https://github.com/rust-lang/crates.io-index" 265 | checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" 266 | dependencies = [ 267 | "getrandom 0.2.15", 268 | "libredox", 269 | "thiserror", 270 | ] 271 | 272 | [[package]] 273 | name = "regex" 274 | version = "1.10.6" 275 | source = "registry+https://github.com/rust-lang/crates.io-index" 276 | checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" 277 | dependencies = [ 278 | "aho-corasick", 279 | "memchr", 280 | "regex-automata", 281 | "regex-syntax", 282 | ] 283 | 284 | [[package]] 285 | name = "regex-automata" 286 | version = "0.4.7" 287 | source = "registry+https://github.com/rust-lang/crates.io-index" 288 | checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" 289 | dependencies = [ 290 | "aho-corasick", 291 | "memchr", 292 | "regex-syntax", 293 | ] 294 | 295 | [[package]] 296 | name = "regex-syntax" 297 | version = "0.8.4" 298 | source = "registry+https://github.com/rust-lang/crates.io-index" 299 | checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" 300 | 301 | [[package]] 302 | name = "rustversion" 303 | version = "1.0.17" 304 | source = "registry+https://github.com/rust-lang/crates.io-index" 305 | checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" 306 | 307 | [[package]] 308 | name = "serde" 309 | version = "1.0.209" 310 | source = "registry+https://github.com/rust-lang/crates.io-index" 311 | checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" 312 | dependencies = [ 313 | "serde_derive", 314 | ] 315 | 316 | [[package]] 317 | name = "serde_derive" 318 | version = "1.0.209" 319 | source = "registry+https://github.com/rust-lang/crates.io-index" 320 | checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" 321 | dependencies = [ 322 | "proc-macro2", 323 | "quote", 324 | "syn", 325 | ] 326 | 327 | [[package]] 328 | name = "strsim" 329 | version = "0.10.0" 330 | source = "registry+https://github.com/rust-lang/crates.io-index" 331 | checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" 332 | 333 | [[package]] 334 | name = "syn" 335 | version = "2.0.76" 336 | source = "registry+https://github.com/rust-lang/crates.io-index" 337 | checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" 338 | dependencies = [ 339 | "proc-macro2", 340 | "quote", 341 | "unicode-ident", 342 | ] 343 | 344 | [[package]] 345 | name = "term" 346 | version = "0.7.0" 347 | source = "registry+https://github.com/rust-lang/crates.io-index" 348 | checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" 349 | dependencies = [ 350 | "dirs-next", 351 | "rustversion", 352 | "winapi", 353 | ] 354 | 355 | [[package]] 356 | name = "thiserror" 357 | version = "1.0.63" 358 | source = "registry+https://github.com/rust-lang/crates.io-index" 359 | checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" 360 | dependencies = [ 361 | "thiserror-impl", 362 | ] 363 | 364 | [[package]] 365 | name = "thiserror-impl" 366 | version = "1.0.63" 367 | source = "registry+https://github.com/rust-lang/crates.io-index" 368 | checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" 369 | dependencies = [ 370 | "proc-macro2", 371 | "quote", 372 | "syn", 373 | ] 374 | 375 | [[package]] 376 | name = "unicode-ident" 377 | version = "1.0.12" 378 | source = "registry+https://github.com/rust-lang/crates.io-index" 379 | checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" 380 | 381 | [[package]] 382 | name = "unicode-width" 383 | version = "0.1.13" 384 | source = "registry+https://github.com/rust-lang/crates.io-index" 385 | checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" 386 | 387 | [[package]] 388 | name = "wasi" 389 | version = "0.9.0+wasi-snapshot-preview1" 390 | source = "registry+https://github.com/rust-lang/crates.io-index" 391 | checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" 392 | 393 | [[package]] 394 | name = "wasi" 395 | version = "0.11.0+wasi-snapshot-preview1" 396 | source = "registry+https://github.com/rust-lang/crates.io-index" 397 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 398 | 399 | [[package]] 400 | name = "winapi" 401 | version = "0.3.9" 402 | source = "registry+https://github.com/rust-lang/crates.io-index" 403 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 404 | dependencies = [ 405 | "winapi-i686-pc-windows-gnu", 406 | "winapi-x86_64-pc-windows-gnu", 407 | ] 408 | 409 | [[package]] 410 | name = "winapi-i686-pc-windows-gnu" 411 | version = "0.4.0" 412 | source = "registry+https://github.com/rust-lang/crates.io-index" 413 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 414 | 415 | [[package]] 416 | name = "winapi-x86_64-pc-windows-gnu" 417 | version = "0.4.0" 418 | source = "registry+https://github.com/rust-lang/crates.io-index" 419 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 420 | 421 | [[package]] 422 | name = "windows-sys" 423 | version = "0.52.0" 424 | source = "registry+https://github.com/rust-lang/crates.io-index" 425 | checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" 426 | dependencies = [ 427 | "windows-targets", 428 | ] 429 | 430 | [[package]] 431 | name = "windows-targets" 432 | version = "0.52.6" 433 | source = "registry+https://github.com/rust-lang/crates.io-index" 434 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" 435 | dependencies = [ 436 | "windows_aarch64_gnullvm", 437 | "windows_aarch64_msvc", 438 | "windows_i686_gnu", 439 | "windows_i686_gnullvm", 440 | "windows_i686_msvc", 441 | "windows_x86_64_gnu", 442 | "windows_x86_64_gnullvm", 443 | "windows_x86_64_msvc", 444 | ] 445 | 446 | [[package]] 447 | name = "windows_aarch64_gnullvm" 448 | version = "0.52.6" 449 | source = "registry+https://github.com/rust-lang/crates.io-index" 450 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" 451 | 452 | [[package]] 453 | name = "windows_aarch64_msvc" 454 | version = "0.52.6" 455 | source = "registry+https://github.com/rust-lang/crates.io-index" 456 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" 457 | 458 | [[package]] 459 | name = "windows_i686_gnu" 460 | version = "0.52.6" 461 | source = "registry+https://github.com/rust-lang/crates.io-index" 462 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" 463 | 464 | [[package]] 465 | name = "windows_i686_gnullvm" 466 | version = "0.52.6" 467 | source = "registry+https://github.com/rust-lang/crates.io-index" 468 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" 469 | 470 | [[package]] 471 | name = "windows_i686_msvc" 472 | version = "0.52.6" 473 | source = "registry+https://github.com/rust-lang/crates.io-index" 474 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" 475 | 476 | [[package]] 477 | name = "windows_x86_64_gnu" 478 | version = "0.52.6" 479 | source = "registry+https://github.com/rust-lang/crates.io-index" 480 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" 481 | 482 | [[package]] 483 | name = "windows_x86_64_gnullvm" 484 | version = "0.52.6" 485 | source = "registry+https://github.com/rust-lang/crates.io-index" 486 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" 487 | 488 | [[package]] 489 | name = "windows_x86_64_msvc" 490 | version = "0.52.6" 491 | source = "registry+https://github.com/rust-lang/crates.io-index" 492 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" 493 | 494 | [[package]] 495 | name = "zerocopy" 496 | version = "0.7.35" 497 | source = "registry+https://github.com/rust-lang/crates.io-index" 498 | checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" 499 | dependencies = [ 500 | "byteorder", 501 | "zerocopy-derive", 502 | ] 503 | 504 | [[package]] 505 | name = "zerocopy-derive" 506 | version = "0.7.35" 507 | source = "registry+https://github.com/rust-lang/crates.io-index" 508 | checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" 509 | dependencies = [ 510 | "proc-macro2", 511 | "quote", 512 | "syn", 513 | ] 514 | -------------------------------------------------------------------------------- /src/benchmark.rs: -------------------------------------------------------------------------------- 1 | use std::cmp; 2 | use std::str::FromStr; 3 | 4 | use prettytable::Row; 5 | use regex::Regex; 6 | 7 | /// Two sets of benchmarks that are comparable but haven't been paired up yet. 8 | #[derive(Clone, Debug)] 9 | pub struct Benchmarks { 10 | old: Vec, 11 | new: Vec, 12 | } 13 | 14 | impl Benchmarks { 15 | /// Create a new empty set of comparable benchmarks. 16 | pub fn from(pair: (Vec, Vec)) -> Benchmarks { 17 | Benchmarks { 18 | old: pair.0, 19 | new: pair.1, 20 | } 21 | } 22 | 23 | /// Create a set of pairwise comparisons between benchmarks. 24 | /// 25 | /// The old and new benchmarks are paired based on whether they have 26 | /// equivalent names. Benchmarks without a pair are marked as unpaired. 27 | pub fn paired(self) -> PairedBenchmarks { 28 | PairedBenchmarks::from(self) 29 | } 30 | } 31 | 32 | /// `PairedBenchmarks` is a set of paired benchmarks. 33 | /// 34 | /// This also provides access to unpaired benchmarks. 35 | #[derive(Clone, Debug)] 36 | pub struct PairedBenchmarks { 37 | cmps: Vec, 38 | unpaired_old: Vec, 39 | unpaired_new: Vec, 40 | } 41 | 42 | impl From for PairedBenchmarks { 43 | fn from(mut benches: Benchmarks) -> PairedBenchmarks { 44 | benches.old.sort(); 45 | benches.new.sort(); 46 | let ov = Overlap::find(benches.old, benches.new, Benchmark::cmp); 47 | let cmps = ov.overlap.into_iter().map(|(a, b)| a.compare(b)).collect(); 48 | PairedBenchmarks { 49 | cmps: cmps, 50 | unpaired_old: ov.left, 51 | unpaired_new: ov.right, 52 | } 53 | } 54 | } 55 | 56 | impl PairedBenchmarks { 57 | /// Returns all pairwise benchmark comparisons. 58 | /// 59 | /// Each comparison provides access to the old and new benchmarks. 60 | pub fn comparisons(&self) -> &[Comparison] { 61 | &self.cmps 62 | } 63 | 64 | /// Returns all benchmarks that were in the old set that were not found 65 | /// in the new set. 66 | pub fn missing_old(&self) -> &[Benchmark] { 67 | &self.unpaired_old 68 | } 69 | 70 | /// Returns all benchmarks that were in the new set that were not found 71 | /// in the old set. 72 | pub fn missing_new(&self) -> &[Benchmark] { 73 | &self.unpaired_new 74 | } 75 | } 76 | 77 | /// All extractable data from a single micro-benchmark. 78 | #[derive(Clone, Debug)] 79 | pub struct Benchmark { 80 | pub name: String, 81 | pub ns: u64, 82 | pub variance: u64, 83 | pub throughput: Option, 84 | } 85 | 86 | impl Eq for Benchmark {} 87 | 88 | impl PartialEq for Benchmark { 89 | fn eq(&self, other: &Benchmark) -> bool { 90 | self.name == other.name 91 | } 92 | } 93 | 94 | impl Ord for Benchmark { 95 | fn cmp(&self, other: &Benchmark) -> cmp::Ordering { 96 | self.partial_cmp(other).unwrap() 97 | } 98 | } 99 | 100 | impl PartialOrd for Benchmark { 101 | fn partial_cmp(&self, other: &Benchmark) -> Option { 102 | self.name.partial_cmp(&other.name) 103 | } 104 | } 105 | 106 | lazy_static! { 107 | static ref BENCHMARK_REGEX: Regex = Regex::new(r##"(?x) 108 | test\s+(?P\S+) # test mod::test_name 109 | \s+...\sbench:\s+(?P[0-9,.]+)\s+ns/iter # ... bench: 1234 ns/iter 110 | \s+\(\+/-\s+(?P[0-9,.]+)\) # (+/- 4321) 111 | (?:\s+=\s+(?P[0-9,.]+)\sMB/s)? # = 2314 MB/s 112 | "##).unwrap(); 113 | } 114 | 115 | impl FromStr for Benchmark { 116 | type Err = (); 117 | 118 | /// Parses a single benchmark line into a Benchmark. 119 | fn from_str(line: &str) -> Result { 120 | let caps = match BENCHMARK_REGEX.captures(line) { 121 | None => return Err(()), 122 | Some(caps) => caps, 123 | }; 124 | let ns = match parse_commas(&caps["ns"]) { 125 | None => return Err(()), 126 | Some(ns) => ns, 127 | }; 128 | let variance = match parse_commas(&caps["variance"]) { 129 | None => return Err(()), 130 | Some(variance) => variance, 131 | }; 132 | let throughput = caps.name("throughput").and_then(|m| parse_commas(m.as_str())); 133 | Ok(Benchmark { 134 | name: caps["name"].to_string(), 135 | ns: ns, 136 | variance: variance, 137 | throughput: throughput, 138 | }) 139 | } 140 | } 141 | 142 | impl Benchmark { 143 | /// Compares an old benchmark (self) with a new benchmark. 144 | pub fn compare(self, new: Benchmark) -> Comparison { 145 | let diff_ns = new.ns as i64 - self.ns as i64; 146 | let diff_ratio = diff_ns as f64 / self.ns as f64; 147 | let speedup = 1.0 / (1.0 + diff_ratio); 148 | Comparison { 149 | old: self, 150 | new: new, 151 | diff_ns: diff_ns, 152 | diff_ratio: diff_ratio, 153 | speedup: speedup, 154 | } 155 | } 156 | 157 | pub fn fmt_ns(&self, variance: bool) -> String { 158 | let mut res = commafy(self.ns); 159 | if variance { 160 | res = format!("{} (+/- {})", res, self.variance); 161 | } 162 | if let Some(throughput) = self.throughput { 163 | res = format!("{} ({} MB/s)", res, throughput); 164 | } 165 | res 166 | } 167 | } 168 | 169 | /// A comparison between an old and a new benchmark. 170 | /// All differences are reported in terms of measuring improvements 171 | /// (negative) or regressions (positive). That is, if an old benchmark 172 | /// is slower than a new benchmark, then the difference is negative. 173 | /// Conversely, if an old benchmark is faster than a new benchmark, 174 | /// then the difference is positive. 175 | #[derive(Clone, Debug)] 176 | pub struct Comparison { 177 | pub old: Benchmark, 178 | pub new: Benchmark, 179 | pub diff_ns: i64, 180 | pub diff_ratio: f64, 181 | pub speedup: f64, 182 | } 183 | 184 | impl Comparison { 185 | /// Convert this comparison to a formatted row useful for printing. 186 | /// 187 | /// The columns of the row are as follows: the name of the benchmark being 188 | /// compared, the old measurement, the new measurement, the measurement 189 | /// difference and the percent measurement difference. Negative differences 190 | /// imply an improvement in performance from old to new. 191 | pub fn to_row(&self, variance: bool, regression: bool) -> Row { 192 | let name = &self.old.name; 193 | let fst_ns = self.old.fmt_ns(variance); 194 | let snd_ns = self.new.fmt_ns(variance); 195 | let diff_ratio = format!("{:.2}%", self.diff_ratio * 100f64); 196 | let speedup = format!("x {:.2}", self.speedup); 197 | let diff_ns = { 198 | let diff_ns = commafy(self.diff_ns.abs() as u64); 199 | if self.diff_ns < 0 { 200 | format!("-{}", diff_ns) 201 | } else { 202 | diff_ns 203 | } 204 | }; 205 | if regression { 206 | row![Fr->name, Fr->fst_ns, Fr->snd_ns, rFr->diff_ns, rFr->diff_ratio, rFr->speedup] 207 | } else { 208 | row![Fg->name, Fg->fst_ns, Fg->snd_ns, rFg->diff_ns, rFg->diff_ratio, rFg->speedup] 209 | } 210 | } 211 | } 212 | 213 | /// Returns what's left of the left vector and right vector that doesn't 214 | /// overlap, and the overlap as a vector of pairs 215 | #[derive(Debug)] 216 | struct Overlap { 217 | left: Vec, 218 | overlap: Vec<(T, T)>, 219 | right: Vec, 220 | } 221 | 222 | impl Overlap { 223 | /// Takes two *sorted* vectors in *ascending* order and a comparison function. 224 | /// 225 | /// Gives back a tuple of vectors, preserving the original sort order: 226 | /// - one for the elements unique to the first vector 227 | /// - one for the pairs of elements found equal 228 | /// - one of the elements unique to the second vector 229 | fn find(mut left: Vec, mut right: Vec, mut fun: F) -> Overlap 230 | where F: FnMut(&T, &T) -> cmp::Ordering 231 | { 232 | use std::cmp::Ordering::*; 233 | 234 | let (mut rleft, mut rright, mut overlap) = (vec![], vec![], vec![]); 235 | loop { 236 | match (left.pop(), right.pop()) { 237 | (None, None) => break, 238 | (None, Some(right_item)) => rright.push(right_item), 239 | (Some(left_item), None) => rleft.push(left_item), 240 | (Some(left_item), Some(right_item)) => { 241 | // sorted from small to large but pop takes from the end! 242 | match fun(&right_item, &left_item) { 243 | Less => { 244 | rleft.push(left_item); 245 | right.push(right_item); 246 | } 247 | Equal => overlap.push((left_item, right_item)), 248 | Greater => { 249 | rright.push(right_item); 250 | left.push(left_item); 251 | } 252 | } 253 | } 254 | } 255 | } 256 | 257 | // We built these in reverse, so reverse them to get original order. 258 | rleft.reverse(); 259 | rright.reverse(); 260 | overlap.reverse(); 261 | Overlap { 262 | left: rleft, 263 | overlap: overlap, 264 | right: rright, 265 | } 266 | } 267 | } 268 | 269 | /// Drops all commas in a string and parses it as a unsigned integer 270 | fn parse_commas(s: &str) -> Option { 271 | drop_commas(s).parse().ok() 272 | } 273 | 274 | /// Drops all commas in a string 275 | fn drop_commas(s: &str) -> String { 276 | s.chars().take_while(|&c| c != '.').filter(|&c| c != ',').collect() 277 | } 278 | 279 | /// Commafy a number as a string. 280 | fn commafy(n: u64) -> String { 281 | let mut with_commas = vec![]; 282 | let dits: Vec = n.to_string().into_bytes().into_iter().rev().collect(); 283 | let mut dits = &*dits; 284 | loop { 285 | if dits.len() < 3 { 286 | with_commas.extend_from_slice(dits); 287 | break; 288 | } 289 | let piece = &dits[0..3]; 290 | dits = &dits[3..]; 291 | with_commas.extend_from_slice(piece); 292 | if piece.len() == 3 && !dits.is_empty() && dits[0] != b'-' { 293 | with_commas.push(b','); 294 | } 295 | } 296 | with_commas.reverse(); 297 | String::from_utf8(with_commas).unwrap() 298 | } 299 | 300 | #[cfg(test)] 301 | mod tests { 302 | mod overlap { 303 | use super::super::Overlap; 304 | 305 | quickcheck! { 306 | fn overlap_correct(left: Vec, right: Vec) -> bool { 307 | let mut left = left; 308 | let mut right = right; 309 | left.sort(); 310 | right.sort(); 311 | 312 | let overlap = Overlap::find(left.clone(), right.clone(), usize::cmp); 313 | 314 | for (l,r) in overlap.overlap { 315 | if l != r { 316 | return false; 317 | } 318 | } 319 | true 320 | } 321 | 322 | fn result_from_vecs(left: Vec, right: Vec) -> bool { 323 | let mut left = left; 324 | let mut right = right; 325 | left.sort(); 326 | right.sort(); 327 | 328 | let overlap = Overlap::find(left.clone(), right.clone(), usize::cmp); 329 | 330 | let (ov_left, ov_right): (Vec, Vec) = 331 | overlap.overlap.into_iter().unzip(); 332 | 333 | let mut left_reconstructed: Vec = overlap.left; 334 | left_reconstructed.extend(ov_left); 335 | left_reconstructed.sort(); 336 | 337 | let mut right_reconstructed: Vec = overlap.right; 338 | right_reconstructed.extend(ov_right); 339 | right_reconstructed.sort(); 340 | 341 | left == left_reconstructed && right == right_reconstructed 342 | } 343 | 344 | fn missing_correct(left: Vec, right: Vec) -> bool { 345 | let mut left = left; 346 | let mut right = right; 347 | left.sort(); 348 | right.sort(); 349 | 350 | // duplicates in either vec would make this check more complicated 351 | left.dedup(); 352 | right.dedup(); 353 | 354 | let overlap = Overlap::find(left.clone(), right.clone(), usize::cmp); 355 | 356 | for l in overlap.left { 357 | if right.iter().find(|&&n| n == l).is_some() { 358 | return false; 359 | } 360 | } 361 | 362 | for r in overlap.right { 363 | if left.iter().find(|&&n| n == r).is_some() { 364 | return false; 365 | } 366 | } 367 | 368 | true 369 | } 370 | } 371 | } 372 | 373 | mod commafy { 374 | use super::super::commafy; 375 | 376 | quickcheck! { 377 | fn comma_every_three(n: u64) -> bool { 378 | let commafied = commafy(n); 379 | let mut commafied = commafied.split(','); 380 | let s = commafied.next().unwrap(); 381 | if s.len() == 0 || s.len() > 3 { 382 | return false; 383 | } 384 | for s in commafied { 385 | if s.len() != 3 { 386 | return false; 387 | } 388 | } 389 | true 390 | } 391 | 392 | fn number_matches(n: u64) -> bool { 393 | let commafied = commafy(n); 394 | let formatted = format!("{}", n); 395 | let stripped: String = commafied.chars().filter(|&b| b != ',').collect(); 396 | formatted == stripped 397 | } 398 | } 399 | } 400 | 401 | mod benchmark { 402 | use super::super::Benchmark; 403 | use quickcheck::Arbitrary; 404 | use quickcheck::Gen; 405 | use rand::Rng; 406 | use rand::distributions::Alphanumeric; 407 | use std::iter; 408 | 409 | impl Arbitrary for Benchmark { 410 | fn arbitrary(g: &mut G) -> Self { 411 | let (ns, variance, throughput): (u64, u64, Option) = Arbitrary::arbitrary(g); 412 | let name = { 413 | let size = g.size(); 414 | let size = g.gen_range(1, size); 415 | iter::repeat(()).map(|()| g.sample(Alphanumeric)).take(size).collect() 416 | }; 417 | Benchmark { 418 | name: name, 419 | ns: ns, 420 | variance: variance, 421 | throughput: throughput, 422 | } 423 | } 424 | } 425 | 426 | fn deep_eq(b1: &Benchmark, b2: &Benchmark) -> bool { 427 | b1.name == b2.name && b1.variance == b2.variance && b1.ns == b2.ns && 428 | b1.throughput == b2.throughput 429 | } 430 | 431 | fn as_string(b: &Benchmark) -> String { 432 | let res = format!("test {} ... bench: {} ns/iter (+/- {})", 433 | b.name, 434 | b.ns, 435 | b.variance); 436 | if let Some(throughput) = b.throughput { 437 | format!("{} = {} MB/s", res, throughput) 438 | } else { 439 | res 440 | } 441 | } 442 | 443 | quickcheck! { 444 | fn reparse(b1: Benchmark) -> bool { 445 | if let Ok(b2) = as_string(&b1).parse() { 446 | deep_eq(&b1, &b2) 447 | } else { 448 | false 449 | } 450 | } 451 | } 452 | } 453 | } 454 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | extern crate docopt; 2 | #[macro_use] 3 | extern crate lazy_static; 4 | extern crate regex; 5 | #[macro_use] 6 | extern crate prettytable; 7 | #[macro_use] 8 | extern crate serde_derive; 9 | #[cfg(test)] 10 | #[macro_use] 11 | extern crate quickcheck; 12 | #[cfg(test)] 13 | extern crate rand; 14 | 15 | use std::io::{self, BufRead}; 16 | use std::fs::File; 17 | use std::path::{Path, PathBuf}; 18 | use std::process; 19 | 20 | use docopt::Docopt; 21 | use prettytable::Table; 22 | use prettytable::format; 23 | 24 | use benchmark::{Benchmarks, Benchmark}; 25 | use error::{Result, Error}; 26 | 27 | mod benchmark; 28 | mod error; 29 | 30 | macro_rules! eprintln { 31 | ($($tt:tt)*) => {{ 32 | use std::io::{Write, stderr}; 33 | writeln!(&mut stderr(), $($tt)*).unwrap(); 34 | }} 35 | } 36 | 37 | const USAGE: &'static str = r#" 38 | Compares Rust micro-benchmark results. 39 | 40 | Usage: 41 | cargo benchcmp [options] 42 | cargo benchcmp [options] 43 | cargo benchcmp -h | --help 44 | cargo benchcmp --version 45 | 46 | The first version takes two files and compares the common benchmarks. 47 | 48 | The second version takes two benchmark name prefixes and one benchmark output 49 | file, and compares the common benchmarks (as determined by comparing the 50 | benchmark names with their prefixes stripped). Benchmarks not matching either 51 | prefix are ignored completely. 52 | 53 | If benchmark output is sent on stdin, then the second version is used and the 54 | third file parameter is not needed. 55 | 56 | Options: 57 | -h, --help Show this help message and exit. 58 | --version Show the version. 59 | --include-missing Show all benchmarks even if they were not in both files. 60 | Produces a WARNING otherwise to let you what was missing. 61 | --threshold Show only comparisons with a percentage change greater 62 | than this threshold. 63 | --variance Show the variance of each benchmark. 64 | --improvements Show only improvements. 65 | --regressions Show only regressions. 66 | --color Show colored rows: never, always or auto [default: auto] 67 | "#; 68 | 69 | #[derive(Debug, Deserialize)] 70 | struct Args { 71 | arg_old: String, 72 | arg_new: String, 73 | arg_file: Option, 74 | flag_threshold: Option, 75 | flag_include_missing: bool, 76 | flag_variance: bool, 77 | flag_improvements: bool, 78 | flag_regressions: bool, 79 | flag_color: When, 80 | } 81 | 82 | #[derive(Debug, Deserialize)] 83 | enum When { 84 | Never, 85 | Always, 86 | Auto, 87 | } 88 | 89 | fn main() { 90 | let args: Args = Docopt::new(USAGE) 91 | .and_then(|d| d.version(Some(version())).deserialize()) 92 | .unwrap_or_else(|e| e.exit()); 93 | if let Err(e) = args.run() { 94 | eprintln!("{}", e); 95 | process::exit(1); 96 | } 97 | } 98 | 99 | impl Args { 100 | fn run(&self) -> Result<()> { 101 | let (name_old, name_new) = Args::names(&self.arg_old, &self.arg_new); 102 | let benches = try!(self.parse_benchmarks()).paired(); 103 | if benches.comparisons().len() > 0 { 104 | let mut output = Table::new(); 105 | output.set_format(*format::consts::FORMAT_CLEAN); 106 | output.add_row(row![ 107 | b->"name", 108 | b->format!("{} ns/iter", name_old), 109 | b->format!("{} ns/iter", name_new), 110 | br->"diff ns/iter", 111 | br->"diff %", 112 | br->"speedup" 113 | ]); 114 | for c in benches.comparisons() { 115 | let abs_per = (c.diff_ratio * 100f64).abs().trunc() as u8; 116 | let regression = c.diff_ns > 0; 117 | if self.flag_threshold.map_or(false, |t| abs_per < t) || 118 | self.flag_regressions && !regression || 119 | self.flag_improvements && regression { 120 | continue; 121 | } 122 | output.add_row(c.to_row(self.flag_variance, regression)); 123 | } 124 | 125 | if self.flag_include_missing { 126 | for b in benches.missing_old() { 127 | output.add_row(row![b.name, b.fmt_ns(self.flag_variance), "n/a", r->"n/a", r->"n/a"]); 128 | } 129 | 130 | for b in benches.missing_new() { 131 | output.add_row(row![b.name, "n/a", b.fmt_ns(self.flag_variance), r->"n/a", r->"n/a"]); 132 | } 133 | } 134 | 135 | if output.len() > 1 { 136 | match self.flag_color { 137 | When::Auto => output.printstd(), 138 | When::Never => { 139 | try!(output.print(&mut io::stdout())); 140 | } 141 | When::Always => { 142 | try!(output.print_tty(true)); 143 | } 144 | }; 145 | } else { 146 | let comparisions = benches.comparisons().len(); 147 | let improvements = benches.comparisons().iter().filter(|c| c.diff_ns <= 0).count(); 148 | let regressions = comparisions - improvements; 149 | 150 | match (self.flag_threshold, self.flag_improvements, self.flag_regressions) { 151 | (Some(threshold), false, false) => { 152 | eprintln!( 153 | "All ({}) benchmarks are within a {}% threshold", 154 | comparisions, 155 | threshold, 156 | ) 157 | } 158 | (Some(threshold), true, false) if improvements > 0 => { 159 | eprintln!( 160 | "All ({}/{}) improvements are within a {}% threshold", 161 | improvements, 162 | comparisions, 163 | threshold, 164 | ) 165 | } 166 | (_, true, false) => { 167 | eprintln!("{}/{} benchmarks improved", improvements, comparisions) 168 | } 169 | (Some(threshold), false, true) if regressions > 0 => { 170 | eprintln!( 171 | "All ({}/{}) regressions are within a {}% threshold", 172 | regressions, 173 | comparisions, 174 | threshold, 175 | ) 176 | } 177 | (_, false, true) => { 178 | eprintln!("{}/{} benchmarks regressed", regressions, comparisions) 179 | } 180 | _ => eprintln!("WARNING: nothing to output"), 181 | } 182 | } 183 | } 184 | 185 | // If there were any unpaired benchmarks, show them now. 186 | if !self.flag_include_missing && !benches.missing_old().is_empty() { 187 | let missed = benches.missing_old() 188 | .iter() 189 | .map(|b| b.name.to_string()) 190 | .collect::>() 191 | .join(", "); 192 | eprintln!("WARNING: benchmarks in old but not in new: {}", missed); 193 | } 194 | if !self.flag_include_missing && !benches.missing_new().is_empty() { 195 | let missed = benches.missing_new() 196 | .iter() 197 | .map(|b| b.name.to_string()) 198 | .collect::>() 199 | .join(", "); 200 | eprintln!("WARNING: benchmarks in new but not in old: {}", missed); 201 | } 202 | Ok(()) 203 | } 204 | 205 | /// Parse benchmarks from the command line invocation given. 206 | fn parse_benchmarks(&self) -> Result { 207 | if let Some(ref one_file) = self.arg_file { 208 | if one_file == "-" { 209 | let stdin = io::stdin(); 210 | let stdin_lock = stdin.lock(); 211 | let benches = try!(Args::parse_buffer(stdin_lock)); 212 | Ok(Benchmarks::from(Args::split_benchmarks(benches, &self.arg_old, &self.arg_new))) 213 | } else { 214 | self.parse_file_benchmarks(one_file) 215 | } 216 | } else { 217 | self.parse_old_new_benchmarks() 218 | } 219 | } 220 | 221 | /// Parses benchmarks from two files: one containing old benchmark output 222 | /// and another containing new benchmark output. 223 | fn parse_old_new_benchmarks(&self) -> Result { 224 | let b_old = try!(Args::parse_buffer(io::BufReader::new(try!(open_file(&self.arg_old))))); 225 | let b_new = try!(Args::parse_buffer(io::BufReader::new(try!(open_file(&self.arg_new))))); 226 | 227 | Ok(Benchmarks::from((b_old, b_new))) 228 | } 229 | 230 | /// Parses benchmarks from one file, then splits on the two prefixes. 231 | /// See also: Args::split_benchmarks 232 | fn parse_file_benchmarks

(&self, file: P) -> Result 233 | where P: AsRef 234 | { 235 | let benches = try!(Args::parse_buffer(io::BufReader::new(try!(File::open(file))))); 236 | Ok(Benchmarks::from(Args::split_benchmarks(benches, &self.arg_old, &self.arg_new))) 237 | } 238 | 239 | /// Parse benchmarks from a buffered reader. 240 | fn parse_buffer(buffer: B) -> Result> { 241 | let iter = buffer.lines(); 242 | let mut vec = Vec::with_capacity(iter.size_hint().0); 243 | for result in iter { 244 | if let Ok(bench) = try!(result).parse() { 245 | vec.push(bench) 246 | } 247 | } 248 | Ok(vec) 249 | } 250 | 251 | /// Splits benchmarks from one source with two prefixes. The first prefix 252 | /// identifies benchmarks in the old set and the second prefix identifies 253 | /// benchmarks in the new set where all benchmarks are found in one file. 254 | fn split_benchmarks(vec: Vec, 255 | arg_old: &str, 256 | arg_new: &str) 257 | -> (Vec, Vec) { 258 | let mut b_old = Vec::new(); 259 | let mut b_new = Vec::new(); 260 | for mut bench in vec { 261 | if bench.name.starts_with(arg_old) { 262 | bench.name = bench.name[arg_old.len()..].to_string(); 263 | b_old.push(bench); 264 | } else if bench.name.starts_with(arg_new) { 265 | bench.name = bench.name[arg_new.len()..].to_string(); 266 | b_new.push(bench); 267 | } 268 | } 269 | (b_old, b_new) 270 | } 271 | 272 | /// Returns the names that should be used in the column header. 273 | fn names(arg_old: &str, arg_new: &str) -> (String, String) { 274 | // If either of the names are empty, substitute them with defaults. 275 | let arg_old = if arg_old.is_empty() { 276 | "old".to_string() 277 | } else { 278 | arg_old.to_string() 279 | }; 280 | let arg_new = if arg_new.is_empty() { 281 | "new".to_string() 282 | } else { 283 | arg_new.to_string() 284 | }; 285 | // The names could be either in the prefixes or in the file paths. 286 | let (old, new) = (Path::new(&arg_old), Path::new(&arg_new)); 287 | // No files paths? Don't do anything smart. 288 | if old.iter().count() <= 1 || new.iter().count() <= 1 { 289 | return (arg_old.clone(), arg_new.clone()); 290 | } 291 | // If we have file paths, try to find the shortest string that 292 | // differentiates them. 293 | let (mut uold, mut unew) = (vec![], vec![]); 294 | for (o, n) in old.iter().rev().zip(new.iter().rev()) { 295 | uold.push(o.to_string_lossy().into_owned()); 296 | unew.push(n.to_string_lossy().into_owned()); 297 | if o != n { 298 | break; 299 | } 300 | } 301 | // If for some reason one of these is empty, just fall back to the 302 | // names given. 303 | if uold.is_empty() || unew.is_empty() { 304 | return (arg_old.clone(), arg_new.clone()); 305 | } 306 | uold.reverse(); 307 | unew.reverse(); 308 | let pold: PathBuf = uold.into_iter().collect(); 309 | let pnew: PathBuf = unew.into_iter().collect(); 310 | (pold.display().to_string(), pnew.display().to_string()) 311 | } 312 | } 313 | 314 | fn version() -> String { 315 | let (maj, min, pat) = (option_env!("CARGO_PKG_VERSION_MAJOR"), 316 | option_env!("CARGO_PKG_VERSION_MINOR"), 317 | option_env!("CARGO_PKG_VERSION_PATCH")); 318 | match (maj, min, pat) { 319 | (Some(maj), Some(min), Some(pat)) => format!("{}.{}.{}", maj, min, pat), 320 | _ => "".to_owned(), 321 | } 322 | } 323 | 324 | /// `open_file` is like `File::open`, except it gives a better error message 325 | /// when it fails (i.e., it includes the file path). 326 | fn open_file>(path: P) -> Result { 327 | File::open(&path).map_err(|err| { 328 | Error::OpenFile { 329 | path: path.as_ref().to_path_buf(), 330 | err: err, 331 | } 332 | }) 333 | } 334 | 335 | #[cfg(test)] 336 | mod tests { 337 | use quickcheck::Arbitrary; 338 | use quickcheck::Gen; 339 | use rand::Rng; 340 | use rand::distributions::Alphanumeric; 341 | use std::iter; 342 | 343 | #[derive(Clone, Debug)] 344 | struct AlphaString(String); 345 | 346 | impl Arbitrary for AlphaString { 347 | fn arbitrary(g: &mut G) -> Self { 348 | let size = g.size(); 349 | let size = g.gen_range(1, size); 350 | AlphaString(iter::repeat(()).map(|()| g.sample(Alphanumeric)).take(size).collect()) 351 | } 352 | } 353 | 354 | mod names { 355 | use super::super::Args; 356 | use super::AlphaString; 357 | use std::path::{Path, PathBuf}; 358 | use std::ffi::OsStr; 359 | use quickcheck::Arbitrary; 360 | use quickcheck::Gen; 361 | use rand::Rng; 362 | 363 | #[derive(Clone, Debug)] 364 | struct ArbitraryPathBuf(PathBuf); 365 | 366 | impl Arbitrary for ArbitraryPathBuf { 367 | fn arbitrary(g: &mut G) -> Self { 368 | let components = g.size(); 369 | let components = g.gen_range(1, components); 370 | let mut path_buf = PathBuf::new(); 371 | for _ in 0..components { 372 | let AlphaString(component) = AlphaString::arbitrary(g); 373 | path_buf.push(component); 374 | } 375 | ArbitraryPathBuf(path_buf) 376 | } 377 | } 378 | 379 | #[derive(Clone, Debug)] 380 | struct ArbitraryPathBufPair(PathBuf, PathBuf); 381 | 382 | impl Arbitrary for ArbitraryPathBufPair { 383 | fn arbitrary(g: &mut G) -> Self { 384 | let components = g.size(); 385 | let components = g.gen_range(1, components); 386 | let mut path_buf1 = PathBuf::new(); 387 | let mut path_buf2 = PathBuf::new(); 388 | for component_no in 0..components { 389 | let AlphaString(component) = AlphaString::arbitrary(g); 390 | // further along in the path, the components are less likely to be different 391 | if g.gen_bool(1.0 / (2.0 + (component_no / 2) as f64)) { 392 | path_buf1.push(component); 393 | let AlphaString(component) = AlphaString::arbitrary(g); 394 | path_buf2.push(component); 395 | } else { 396 | path_buf1.push(component.clone()); 397 | path_buf2.push(component); 398 | } 399 | } 400 | ArbitraryPathBufPair(path_buf1, path_buf2) 401 | } 402 | } 403 | 404 | quickcheck! { 405 | fn empty_gives_old(new_name: AlphaString) -> bool { 406 | let AlphaString(new_name) = new_name; 407 | let empty = String::from(""); 408 | let result = Args::names(&empty, &new_name); 409 | 410 | ("old".to_string(), new_name) == result 411 | } 412 | 413 | fn empty_gives_new(old_name: AlphaString) -> bool { 414 | let AlphaString(old_name) = old_name; 415 | let empty = String::from(""); 416 | let result = Args::names(&old_name, &empty); 417 | 418 | (old_name, "new".to_string()) == result 419 | } 420 | 421 | fn non_path_gives_originals(old_name: AlphaString, new_name: AlphaString) -> bool { 422 | let AlphaString(old_name) = old_name; 423 | let AlphaString(new_name) = new_name; 424 | let result = Args::names(&old_name, &new_name); 425 | 426 | (old_name, new_name) == result 427 | } 428 | 429 | fn same_path_gives_originals(path: ArbitraryPathBuf) -> bool { 430 | let ArbitraryPathBuf(path) = path; 431 | let path = path.to_string_lossy().into_owned(); 432 | let result = Args::names(&path, &path); 433 | 434 | (path.clone(), path) == result 435 | } 436 | 437 | fn symmetric_operation(pair: ArbitraryPathBufPair) -> bool { 438 | let ArbitraryPathBufPair(old, new) = pair; 439 | let old = old.to_string_lossy().into_owned(); 440 | let new = new.to_string_lossy().into_owned(); 441 | let result = Args::names(&old, &new); 442 | 443 | (result.1, result.0) == Args::names(&new, &old) 444 | } 445 | 446 | fn difference_preserving(pair: ArbitraryPathBufPair) -> bool { 447 | let ArbitraryPathBufPair(old, new) = pair; 448 | let old = old.to_string_lossy().into_owned(); 449 | let new = new.to_string_lossy().into_owned(); 450 | let result = Args::names(&old, &new); 451 | 452 | (old == new) == (result.0 == result.1) 453 | } 454 | 455 | fn gives_suffixes(pair: ArbitraryPathBufPair) -> bool { 456 | let ArbitraryPathBufPair(old, new) = pair; 457 | let old = old.to_string_lossy().into_owned(); 458 | let new = new.to_string_lossy().into_owned(); 459 | let result = Args::names(&old, &new); 460 | 461 | old.ends_with(&result.0) && new.ends_with(&result.1) 462 | } 463 | 464 | fn shortest_difference(pair: ArbitraryPathBufPair) -> bool { 465 | let ArbitraryPathBufPair(old, new) = pair; 466 | let old = old.to_string_lossy().into_owned(); 467 | let new = new.to_string_lossy().into_owned(); 468 | let result = Args::names(&old, &new); 469 | 470 | let path_0 = Path::new(&result.0); 471 | let path_1 = Path::new(&result.1); 472 | let path_0: Vec<&OsStr> = path_0.iter().collect(); 473 | let path_1: Vec<&OsStr> = path_1.iter().collect(); 474 | let mut zipped = path_0.iter().rev().zip(path_1.iter().rev()).rev(); 475 | let shortest_difference = zipped.next().map(|(o, n)| o != n).unwrap_or(false); 476 | let shortest_difference = shortest_difference && zipped.all(|(o, n)| o == n); 477 | 478 | old == new || path_0.iter().count() <= 1 || path_1.iter().count() <= 1 || 479 | shortest_difference 480 | } 481 | } 482 | } 483 | 484 | mod split_benchmarks { 485 | use super::super::Args; 486 | use super::AlphaString; 487 | use benchmark::Benchmark; 488 | 489 | quickcheck! { 490 | fn from_original(benches: Vec, old: AlphaString, new: AlphaString) -> bool { 491 | let AlphaString(old) = old; 492 | let AlphaString(new) = new; 493 | let result = Args::split_benchmarks(benches.clone(), &old, &new); 494 | 495 | result.0.into_iter().all(|mut b| { 496 | b.name = old.clone() + &b.name; 497 | benches.contains(&b) 498 | }) && 499 | result.1.into_iter().all(|mut b| { 500 | b.name = new.clone() + &b.name; 501 | benches.contains(&b) 502 | }) 503 | } 504 | 505 | fn non_overlapping(benches: Vec, 506 | old: AlphaString, 507 | new: AlphaString) 508 | -> bool { 509 | let AlphaString(old) = old; 510 | let AlphaString(new) = new; 511 | let result = Args::split_benchmarks(benches.clone(), &old, &new); 512 | let mut benches = benches; 513 | 514 | let results: Vec = result.0 515 | .into_iter() 516 | .map(|mut b| { 517 | b.name = old.clone() + &b.name; 518 | b 519 | }) 520 | .chain(result.1.into_iter().map(|mut b| { 521 | b.name = new.clone() + &b.name; 522 | b 523 | })) 524 | .collect(); 525 | 526 | for result in results { 527 | if let Some(index) = benches.iter().position(|b| b == &result) { 528 | benches.swap_remove(index); 529 | } else { 530 | return false; 531 | } 532 | } 533 | 534 | true 535 | } 536 | 537 | fn dropped_non_prefix(benches: Vec, 538 | old: AlphaString, 539 | new: AlphaString) 540 | -> bool { 541 | let AlphaString(old) = old; 542 | let AlphaString(new) = new; 543 | let result = Args::split_benchmarks(benches.clone(), &old, &new); 544 | let mut benches = benches; 545 | 546 | let results: Vec = result.0 547 | .into_iter() 548 | .map(|mut b| { 549 | b.name = old.clone() + &b.name; 550 | b 551 | }) 552 | .chain(result.1.into_iter().map(|mut b| { 553 | b.name = new.clone() + &b.name; 554 | b 555 | })) 556 | .collect(); 557 | 558 | for result in results { 559 | if let Some(index) = benches.iter().position(|b| b == &result) { 560 | benches.swap_remove(index); 561 | } 562 | } 563 | 564 | benches.into_iter().all(|b| !(b.name.starts_with(&old) || b.name.starts_with(&new))) 565 | } 566 | } 567 | } 568 | } 569 | --------------------------------------------------------------------------------