├── .editorconfig ├── .github └── workflows │ └── ci.yaml ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Cargo.lock ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── ci ├── install.sh └── script.sh ├── integration_tests ├── Cargo.lock ├── Cargo.toml ├── benches │ ├── test_benchmark_external_c_lib.rs │ ├── test_benchmark_non_criterion.rs │ └── test_benchmarks.rs ├── criterion.toml ├── src │ └── lib.rs └── tests │ └── integration_test.rs └── src ├── analysis.rs ├── bench_target.rs ├── compile.rs ├── config.rs ├── connection.rs ├── estimate.rs ├── format.rs ├── html ├── benchmark_report.html.tt ├── common.css ├── history_report.html.tt ├── index.html.tt ├── mod.rs ├── report_link.html.tt └── summary_report.html.tt ├── kde.rs ├── macros_private.rs ├── main.rs ├── message_formats ├── json.rs ├── mod.rs └── openmetrics.rs ├── model.rs ├── plot ├── gnuplot_backend │ ├── distributions.rs │ ├── history.rs │ ├── iteration_times.rs │ ├── mod.rs │ ├── pdf.rs │ ├── regression.rs │ ├── summary.rs │ └── t_test.rs ├── mod.rs └── plotters_backend │ ├── distributions.rs │ ├── history.rs │ ├── iteration_times.rs │ ├── mod.rs │ ├── pdf.rs │ ├── regression.rs │ ├── summary.rs │ └── t_test.rs ├── report.rs ├── stats ├── bivariate │ ├── bootstrap.rs │ ├── mod.rs │ ├── regression.rs │ └── resamples.rs ├── float.rs ├── mod.rs ├── rand_util.rs ├── test.rs ├── tuple.rs └── univariate │ ├── bootstrap.rs │ ├── kde │ ├── kernel.rs │ └── mod.rs │ ├── mixed.rs │ ├── mod.rs │ ├── outliers │ ├── mod.rs │ └── tukey.rs │ ├── percentiles.rs │ ├── resamples.rs │ └── sample.rs └── value_formatter.rs /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | trim_trailing_whitespace = true 7 | charset = utf-8 8 | indent_style = space 9 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - main 5 | pull_request: 6 | branches: 7 | - main 8 | 9 | name: tests 10 | 11 | jobs: 12 | ci: 13 | runs-on: ${{ matrix.os }} 14 | strategy: 15 | matrix: 16 | os: [macos-latest, ubuntu-latest] 17 | rust: 18 | - 1.87 # stable release as of 2025-05-17 19 | - 1.80.0 # MSRV 20 | 21 | steps: 22 | - uses: actions/checkout@v2 23 | 24 | - uses: actions-rs/toolchain@v1 25 | name: Setup rust toolchain 26 | with: 27 | profile: minimal 28 | toolchain: ${{ matrix.rust }} 29 | override: true 30 | components: rustfmt, clippy 31 | 32 | - uses: Swatinem/rust-cache@v1 33 | name: Load dependencies from cache 34 | 35 | - uses: actions-rs/cargo@v1 36 | name: Build 37 | with: 38 | command: build 39 | 40 | - uses: actions-rs/cargo@v1 41 | name: Test 42 | with: 43 | command: test 44 | 45 | - name: Integration tests 46 | run: | 47 | cd integration_tests 48 | cargo test -- --format=pretty --nocapture 49 | 50 | - uses: actions-rs/cargo@v1 51 | name: Check for non-standard formatting 52 | if: ${{ matrix.rust == 'stable' }} 53 | with: 54 | command: fmt 55 | args: --all -- --check 56 | 57 | - uses: actions-rs/cargo@v1 58 | name: Check for clippy hints 59 | with: 60 | command: clippy 61 | args: -- -D warnings 62 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .criterion 2 | target 3 | 4 | **/.*.sw* 5 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) 5 | and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). 6 | 7 | # [Unreleased] 8 | ### Added 9 | - New message output format for [OpenMetrics](https://openmetrics.io). 10 | 11 | ## [1.1.0] - 2021-07-28 12 | ### Fixed 13 | - Fixed wrong exit code being returned when a panic occurs outside of the function being benchmarked. 14 | - MacOS/Windows: Fix connection issue that manifested itself in a few different ways. 15 | - Use new version of plotters. No new features but it fixes a bug that caused criterion to 16 | hang indefinitely. 17 | 18 | ### Added 19 | - Load configuration options 'criterion.toml' if 'Criterion.toml' isn't available. 20 | 21 | ## [1.0.1] - 2021-01-24 22 | ### Fixed 23 | - Changed opacity of the violin plots to full. 24 | - Fixed violin chart X axis not starting at zero in the plotters backend. 25 | - Fixed panic in the history report code. 26 | 27 | ## [1.0.0] - 2020-07-18 28 | ### Fixed 29 | - Fixed potential panic if a benchmark took zero time. 30 | - cargo-criterion now calls `cargo metadata` to find the path to the target directory. This fixes 31 | the location of the target directory in workspaces. 32 | 33 | ## Added 34 | - Added a report showing the historical performance of a benchmark. 35 | 36 | ## [1.0.0-alpha3] - 2020-07-06 37 | ### Added 38 | - The criterion.toml file can now be used to configure the colors used for the generated plots. 39 | 40 | ## [1.0.0-alpha2] - 2020-07-05 41 | ### Added 42 | - Initial version of cargo-criterion 43 | ### Fixed 44 | - Fixed problem where benchmarks that relied on dynamically linked libraries would fail 45 | in cargo-criterion but not in cargo bench. 46 | - Sort the benchmark targets before running them. This should ensure a stable execution order 47 | for all benchmarks. 48 | 49 | ### Added 50 | - Added `--message-format=json` option, which prints JSON messages about the benchmarks to 51 | stdout, similar to other Cargo commands. 52 | 53 | ### Changed 54 | - In order to accommodate the machine-readable output, all of cargo-criterion's other output 55 | is now printed to stderr. This matches Cargo's normal behavior. If benchmark targets print 56 | anything to stdout, it will be redirected to stderr if `--message-format` is set, or will be 57 | left on stderr if not. 58 | - Heavy internal refactoring of plot generation code. There may be some bugs. 59 | 60 | ## [1.0.0-alpha1] - 2020-06-29 61 | ### Added 62 | - Initial version of cargo-criterion 63 | 64 | 65 | [1.0.0-alpha1]: https://github.com/bheisler/cargo-criterion/compare/e5fa23b...1.0.0-alpha1 66 | [1.0.0-alpha2]: https://github.com/bheisler/cargo-criterion/compare/1.0.0-alpha1...1.0.0-alpha2 67 | [1.0.0-alpha3]: https://github.com/bheisler/cargo-criterion/compare/1.0.0-alpha2...1.0.0-alpha3 68 | [1.0.0]: https://github.com/bheisler/cargo-criterion/compare/1.0.0-alpha3...1.0.0 69 | [1.0.1]: https://github.com/bheisler/cargo-criterion/compare/1.0.0-alpha3...1.0.1 70 | [1.0.1]: https://github.com/bheisler/cargo-criterion/compare/1.0.1...1.1.0 71 | [Unreleased]: https://github.com/bheisler/cargo-criterion/compare/1.1.0...HEAD 72 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to cargo-criterion 2 | 3 | ## Ideas, Experiences and Questions 4 | 5 | The easiest way to contribute to cargo-criterion is to use it and report your experiences, ask questions and contribute ideas. We'd love to hear your thoughts on how to make cargo-criterion better, or your comments on why you are or are not currently using it. 6 | 7 | Issues, ideas, requests and questions should be posted on the issue tracker at: 8 | 9 | https://github.com/bheisler/cargo-criterion/issues 10 | 11 | ## A Note on Dependency Updates 12 | 13 | cargo-criterion does not accept pull requests to update dependencies unless specifically 14 | requested by the maintaner(s). Dependencies are updated manually by the maintainer(s) before each 15 | new release. 16 | 17 | ## Code 18 | 19 | Pull requests are welcome, though please raise an issue for discussion first if none exists. We're happy to assist new contributors. 20 | 21 | If you're not sure what to work on, try checking the [Beginner label](https://github.com/bheisler/cargo-criterion/issues?q=is%3Aissue+is%3Aopen+label%3ABeginner) 22 | 23 | To make changes to the code, fork the repo and clone it: 24 | 25 | `git clone git@github.com:your-username/cargo-criterion.git` 26 | 27 | You'll probably want to install [gnuplot](http://www.gnuplot.info/) as well. See the gnuplot website for installation instructions. 28 | 29 | Then make your changes to the code. When you're done, run the tests: 30 | 31 | ``` 32 | cargo test --all 33 | cargo bench 34 | ``` 35 | 36 | It's a good idea to run clippy and fix any warnings as well: 37 | 38 | ``` 39 | cargo clippy --all 40 | ``` 41 | 42 | Finally, run Rustfmt to maintain a common code style: 43 | 44 | ``` 45 | cargo fmt --all 46 | ``` 47 | 48 | Don't forget to update the CHANGELOG.md file and any appropriate documentation. Once you're finished, push to your fork and submit a pull request. We try to respond to new issues and pull requests quickly, so if there hasn't been any response for more than a few days feel free to ping @bheisler. 49 | 50 | Some things that will increase the chance that your pull request is accepted: 51 | 52 | * Write tests 53 | * Clearly document public methods 54 | * Write a good commit message 55 | 56 | ## Code of Conduct 57 | 58 | We follow the [Rust Code of Conduct](http://www.rust-lang.org/conduct.html). 59 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cargo-criterion" 3 | version = "1.1.0" 4 | authors = ["Brook Heisler "] 5 | edition = "2018" 6 | 7 | description = "Cargo extension for running Criterion.rs benchmarks and reporting the results." 8 | repository = "https://github.com/bheisler/cargo-criterion" 9 | readme = "README.md" 10 | keywords = ["criterion", "benchmark"] 11 | categories = ["development-tools::profiling", "development-tools::cargo-plugins"] 12 | license = "Apache-2.0/MIT" 13 | 14 | exclude = ["integration_tests/*", "ci/*"] 15 | 16 | [dependencies] 17 | serde = "1.0" 18 | serde_json = "1.0" 19 | serde_derive = "1.0" 20 | serde_cbor = "0.11" 21 | toml = { version = "0.5", features = ["preserve_order"] } 22 | clap = "2.33" 23 | oorandom = "11.1" 24 | cast = "0.2" 25 | num-traits = "0.2" 26 | rayon = "1.3" 27 | chrono = { version = "0.4", features = ["serde"] } 28 | anyhow = "1.0" 29 | log = "0.4" 30 | simplelog = "0.10" 31 | atty = "0.2" 32 | lazy_static = "1.4" 33 | criterion-plot = { version = "0.4.3", optional = true } 34 | tinytemplate = "1.1" 35 | linked-hash-map = "0.5" 36 | linked_hash_set = "0.1" 37 | walkdir = "2.3" 38 | 39 | [dependencies.plotters] 40 | version = "0.3.1" 41 | default-features = false 42 | features = ["svg_backend", "area_series", "line_series"] 43 | optional = true 44 | 45 | [features] 46 | default = ["gnuplot_backend", "plotters_backend"] 47 | 48 | # Enable the gnuplot plotting backend. 49 | gnuplot_backend = ["criterion-plot"] 50 | 51 | # Enable the plotters plotting backend. 52 | plotters_backend = ["plotters"] 53 | 54 | [dev-dependencies] 55 | approx = "0.3" 56 | quickcheck = { version = "0.9", default-features = false } 57 | rand = "0.7" 58 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Jorge Aparicio 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

cargo-criterion

2 | 3 |
Criterion-rs Cargo Extension
4 | 5 |
6 | Changelog 7 |
8 | 9 |
10 | 11 | Crates.io 12 | 13 |
14 | 15 | cargo-criterion is a Plugin for Cargo which handles much of the heavy lifting for analyzing and 16 | reporting on [Criterion-rs](https://github.com/bheisler/criterion.rs) benchmarks. 17 | 18 | ## Table of Contents 19 | - [Table of Contents](#table-of-contents) 20 | - [Features](#features) 21 | - [Quickstart](#quickstart) 22 | - [Goals](#goals) 23 | - [Contributing](#contributing) 24 | - [Compatibility Policy](#compatibility-policy) 25 | - [Maintenance](#maintenance) 26 | - [License](#license) 27 | 28 | ### Features 29 | 30 | - __Charts__: Uses [gnuplot](http://www.gnuplot.info/) or [plotters](https://crates.io/crates/plotters) to generate detailed graphs of benchmark results 31 | - __Reports__: In addition to the reports generated by Criterion.rs, cargo-criterion generates a historical report showing the performance of a function over time. 32 | - __Configurable__: cargo-criterion's plot generation can be configured using a criterion.toml file. 33 | 34 | ### Quickstart 35 | 36 | This assumes that you already have benchmarks which use Criterion-rs. If not, see [the Criterion-rs Quickstart Guide](https://github.com/bheisler/criterion.rs#quickstart). 37 | Note that Criterion.rs version 0.3.3 or later is recommended. Benchmarks which do not use Criterion.rs, or which use an earlier version, will run correctly but will not 38 | benefit from some of cargo-criterion's features. 39 | 40 | First install cargo-criterion: 41 | 42 | `cargo install cargo-criterion` 43 | 44 | Then you can use it to run your Criterion-rs benchmarks: 45 | 46 | `cargo criterion` 47 | 48 | Generated reports will appear in `target/criterion/reports`. 49 | 50 | ### Goals 51 | 52 | - cargo-criterion seeks to improve iteration time for Criterion-rs benchmarks. By moving functionality into a separate executable which can be installed once and reused, Criterion-rs can shrink - meaning less code to compile and link into the benchmarks themselves. 53 | - Because cargo-criterion can oversee the whole benchmark process from beginning to end, it's better placed to deliver features that would be difficult to implement in Criterion-rs. These include: 54 | - Machine-readable output 55 | - Tracking benchmarked performance over time 56 | 57 | ### Contributing 58 | 59 | First, thank you for contributing. 60 | 61 | One great way to contribute to cargo-criterion is to use it for your own benchmarking needs and report your experiences, file and comment on issues, etc. 62 | 63 | Code or documentation improvements in the form of pull requests are also welcome. If you're not 64 | sure what to work on, try checking the 65 | [Beginner label](https://github.com/bheisler/cargo-criterion/issues?q=is%3Aissue+is%3Aopen+label%3ABeginner). 66 | 67 | If your issues or pull requests have no response after a few days, feel free to ping me (@bheisler). 68 | 69 | For more details, see the [CONTRIBUTING.md file](https://github.com/bheisler/cargo-criterion/blob/master/CONTRIBUTING.md). 70 | 71 | ### Compatibility Policy 72 | 73 | cargo-criterion supports the last three stable minor releases of Rust. At time of 74 | writing, this means Rust 1.85 or later. Older versions may work, but are not tested or guaranteed. 75 | 76 | Currently, the oldest version of Rust believed to work is 1.80. Future versions of cargo-criterion may 77 | break support for such old versions, and this will not be considered a breaking change. If you 78 | require cargo-criterion to work on old versions of Rust, you will need to stick to a 79 | specific patch version of cargo-criterion. 80 | 81 | ### Maintenance 82 | 83 | cargo-criterion was originally developed and is currently maintained by Brook Heisler (@bheisler). 84 | 85 | ### License 86 | 87 | cargo-criterion is dual licensed under the Apache 2.0 license and the MIT license. 88 | -------------------------------------------------------------------------------- /ci/install.sh: -------------------------------------------------------------------------------- 1 | set -ex 2 | 3 | if [ "$CLIPPY" = "yes" ]; then 4 | rustup component add clippy-preview 5 | fi 6 | 7 | if [ "$RUSTFMT" = "yes" ]; then 8 | rustup component add rustfmt 9 | fi 10 | 11 | if [ "$TRAVIS_OS_NAME" = "osx" ] && [ "$GNUPLOT" = "yes" ]; then 12 | brew unlink python@2 # because we're installing python3 and they both want to install stuff under /usr/local/Frameworks/Python.framework/ 13 | brew install gnuplot 14 | fi 15 | -------------------------------------------------------------------------------- /ci/script.sh: -------------------------------------------------------------------------------- 1 | set -ex 2 | 3 | export CARGO_INCREMENTAL=0 4 | 5 | if [ "$CLIPPY" = "yes" ]; then 6 | cargo clippy --all -- -D warnings 7 | elif [ "$DOCS" = "yes" ]; then 8 | cargo clean 9 | cargo doc --all --no-deps 10 | cd book 11 | mdbook build 12 | cd .. 13 | cp -r book/book/html/ target/doc/book/ 14 | travis-cargo doc-upload || true 15 | elif [ "$RUSTFMT" = "yes" ]; then 16 | cargo fmt --all -- --check 17 | elif [ "$INTEGRATION_TESTS" = "yes" ]; then 18 | cargo build 19 | cd integration_tests 20 | if [ "$GNUPLOT" = "yes" ]; then 21 | cargo test -- --format=pretty --nocapture --ignored 22 | fi 23 | cargo test -- --format=pretty --nocapture 24 | 25 | else 26 | export RUSTFLAGS="-D warnings" 27 | 28 | cargo check --no-default-features --features gnuplot_backend 29 | cargo check --no-default-features --features plotters_backend 30 | 31 | cargo check --all-features 32 | 33 | cargo test 34 | fi 35 | -------------------------------------------------------------------------------- /integration_tests/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "integration_tests" 3 | version = "0.1.0" 4 | authors = ["Brook Heisler "] 5 | edition = "2018" 6 | autobenches = false 7 | 8 | [dependencies] 9 | 10 | # These are benchmarks used to test cargo-criterion itself. 11 | 12 | # This benchmark requires tch which doesn't build on arm. 13 | #[[bench]] 14 | #name = "test_benchmark_external_c_lib" 15 | #harness = false 16 | 17 | [[bench]] 18 | name = "test_benchmarks" 19 | harness = false 20 | 21 | [[bench]] 22 | name = "test_benchmark_non_criterion" 23 | harness = false 24 | 25 | [dev-dependencies] 26 | #tch = "0.1.7" 27 | bencher = "0.1.5" 28 | criterion = "0.3.4" 29 | tempfile = "3.1" 30 | walkdir = "2" 31 | bstr = "0.2" 32 | serde_json = "1.0" 33 | -------------------------------------------------------------------------------- /integration_tests/benches/test_benchmark_external_c_lib.rs: -------------------------------------------------------------------------------- 1 | //! This benchmark binds to an external C library, so it's useful for verifying that cargo-criterion 2 | //! handles library paths and similar correctly. 3 | 4 | use criterion::{black_box, criterion_group, criterion_main, Criterion}; 5 | use std::time::Duration; 6 | use tch::Tensor; 7 | 8 | fn bench_norm(c: &mut Criterion) { 9 | let t = &Tensor::of_slice(&[0_f32, 0.1f32, 0.5f32, 0.9f32]); 10 | c.bench_function("norm", |b| b.iter(|| black_box(t).norm())); 11 | } 12 | 13 | // These benchmarks are used for testing cargo-criterion, so to make the tests faster we configure 14 | // them to run quickly. This is not recommended for real benchmarks. 15 | criterion_group! { 16 | name = benches; 17 | config = Criterion::default() 18 | .warm_up_time(Duration::from_millis(250)) 19 | .measurement_time(Duration::from_millis(500)) 20 | .nresamples(2000); 21 | targets = bench_norm 22 | } 23 | 24 | criterion_main!(benches); 25 | -------------------------------------------------------------------------------- /integration_tests/benches/test_benchmark_non_criterion.rs: -------------------------------------------------------------------------------- 1 | //! This is a Bencher benchmark, used to verify that cargo-criterion can at least run non-criterion 2 | //! benchmarks, even if the more fancy features aren't available. 3 | 4 | #[macro_use] 5 | extern crate bencher; 6 | 7 | use bencher::Bencher; 8 | 9 | fn bencher_test(bench: &mut Bencher) { 10 | bench.iter(|| (0..1000).fold(0, |x, y| x + y)) 11 | } 12 | 13 | benchmark_group!(benches, bencher_test); 14 | benchmark_main!(benches); 15 | -------------------------------------------------------------------------------- /integration_tests/benches/test_benchmarks.rs: -------------------------------------------------------------------------------- 1 | //! This benchmark defines some test benchmarks that exercise various parts of cargo-criterion. 2 | 3 | use criterion::{ 4 | criterion_group, criterion_main, AxisScale, BenchmarkId, Criterion, PlotConfiguration, 5 | SamplingMode, Throughput, 6 | }; 7 | use std::thread::sleep; 8 | use std::time::Duration; 9 | 10 | fn special_characters(c: &mut Criterion) { 11 | let mut group = c.benchmark_group("\"*group/\""); 12 | group.bench_function("\"*benchmark/\" '", |b| b.iter(|| 1 + 1)); 13 | group.finish(); 14 | } 15 | 16 | fn sampling_mode_tests(c: &mut Criterion) { 17 | let mut group = c.benchmark_group("sampling_mode"); 18 | 19 | group.sampling_mode(SamplingMode::Auto); 20 | group.bench_function("Auto (short)", |bencher| { 21 | bencher.iter(|| sleep(Duration::from_millis(0))) 22 | }); 23 | group.bench_function("Auto (long)", |bencher| { 24 | bencher.iter(|| sleep(Duration::from_millis(10))) 25 | }); 26 | 27 | group.sampling_mode(SamplingMode::Linear); 28 | group.bench_function("Linear", |bencher| { 29 | bencher.iter(|| sleep(Duration::from_millis(0))) 30 | }); 31 | 32 | group.sampling_mode(SamplingMode::Flat); 33 | group.bench_function("Flat", |bencher| { 34 | bencher.iter(|| sleep(Duration::from_millis(10))) 35 | }); 36 | 37 | group.finish(); 38 | } 39 | 40 | const SIZE: usize = 1024; 41 | 42 | fn throughput_tests(c: &mut Criterion) { 43 | let mut group = c.benchmark_group("throughput"); 44 | 45 | group.throughput(Throughput::Bytes(SIZE as u64)); 46 | group.bench_function("Bytes", |bencher| { 47 | bencher.iter(|| (0..SIZE).map(|i| i as u8).collect::>()) 48 | }); 49 | 50 | group.throughput(Throughput::Elements(SIZE as u64)); 51 | group.bench_function("Elem", |bencher| { 52 | bencher.iter(|| (0..SIZE).map(|i| i as u64).collect::>()) 53 | }); 54 | 55 | group.finish(); 56 | } 57 | 58 | fn log_scale_tests(c: &mut Criterion) { 59 | let plot_config = PlotConfiguration::default().summary_scale(AxisScale::Logarithmic); 60 | 61 | let mut group = c.benchmark_group("log_scale"); 62 | group.plot_config(plot_config); 63 | 64 | for time in &[1, 100, 10000] { 65 | group.bench_with_input( 66 | BenchmarkId::new("sleep (micros)", time), 67 | time, 68 | |bencher, input| bencher.iter(|| sleep(Duration::from_micros(*input))), 69 | ); 70 | } 71 | group.finish() 72 | } 73 | 74 | fn linear_scale_tests(c: &mut Criterion) { 75 | let plot_config = PlotConfiguration::default().summary_scale(AxisScale::Linear); 76 | 77 | let mut group = c.benchmark_group("linear_scale"); 78 | group.plot_config(plot_config); 79 | 80 | for time in &[1, 2, 3] { 81 | group.bench_with_input( 82 | BenchmarkId::new("sleep (millis)", time), 83 | time, 84 | |bencher, input| bencher.iter(|| sleep(Duration::from_millis(*input))), 85 | ); 86 | } 87 | group.finish() 88 | } 89 | 90 | fn zero_test(c: &mut Criterion) { 91 | // Neither Criterion.rs nor cargo-criterion should panic if the benchmark takes zero time. 92 | // That should never happen, but in rare cases poor benchmarks can cause that. 93 | c.bench_function("forced_zero_time_test", |b| { 94 | b.iter_custom(|_iters| Duration::new(0, 0)) 95 | }); 96 | } 97 | 98 | // These benchmarks are used for testing cargo-criterion, so to make the tests faster we configure 99 | // them to run quickly. This is not recommended for real benchmarks. 100 | criterion_group! { 101 | name = benches; 102 | config = Criterion::default() 103 | .warm_up_time(Duration::from_millis(250)) 104 | .measurement_time(Duration::from_millis(500)) 105 | .nresamples(2000); 106 | targets = special_characters, sampling_mode_tests, throughput_tests, log_scale_tests, 107 | linear_scale_tests, zero_test 108 | } 109 | criterion_main!(benches); 110 | -------------------------------------------------------------------------------- /integration_tests/criterion.toml: -------------------------------------------------------------------------------- 1 | [colors] 2 | current_sample = {r = 0, g = 100, b = 0} 3 | previous_sample = {r = 255, g = 20, b = 147} -------------------------------------------------------------------------------- /integration_tests/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Nothing here 2 | -------------------------------------------------------------------------------- /src/analysis.rs: -------------------------------------------------------------------------------- 1 | use crate::connection::{SamplingMethod, Throughput}; 2 | use crate::estimate::{build_change_estimates, build_estimates, ConfidenceInterval, Estimate}; 3 | use crate::estimate::{ 4 | ChangeDistributions, ChangeEstimates, ChangePointEstimates, Distributions, Estimates, 5 | PointEstimates, 6 | }; 7 | use crate::report::MeasurementData; 8 | use crate::stats::bivariate::regression::Slope; 9 | use crate::stats::bivariate::Data; 10 | use crate::stats::univariate::outliers::tukey; 11 | use crate::stats::univariate::Sample; 12 | use crate::stats::{Distribution, Tails}; 13 | use std::time::Duration; 14 | 15 | #[derive(Debug, Clone)] 16 | pub struct BenchmarkConfig { 17 | pub confidence_level: f64, 18 | pub _measurement_time: Duration, 19 | pub noise_threshold: f64, 20 | pub nresamples: usize, 21 | pub _sample_size: usize, 22 | pub significance_level: f64, 23 | pub _warm_up_time: Duration, 24 | } 25 | 26 | pub struct MeasuredValues<'a> { 27 | pub iteration_count: &'a [f64], 28 | pub sample_values: &'a [f64], 29 | pub avg_values: &'a [f64], 30 | } 31 | 32 | // Common analysis procedure 33 | pub(crate) fn analysis<'a>( 34 | config: &BenchmarkConfig, 35 | throughput: Option, 36 | new_sample: MeasuredValues<'a>, 37 | old_sample: Option<(MeasuredValues<'a>, &'a Estimates)>, 38 | sampling_method: SamplingMethod, 39 | ) -> MeasurementData<'a> { 40 | let iters = new_sample.iteration_count; 41 | let values = new_sample.sample_values; 42 | 43 | let avg_values = Sample::new(new_sample.avg_values); 44 | 45 | let data = Data::new(iters, values); 46 | let labeled_sample = tukey::classify(avg_values); 47 | let (mut distributions, mut estimates) = estimates(avg_values, config); 48 | 49 | if sampling_method.is_linear() { 50 | let (distribution, slope) = regression(&data, config); 51 | estimates.slope = Some(slope); 52 | distributions.slope = Some(distribution); 53 | } 54 | 55 | let compare_data = if let Some((old_sample, old_estimates)) = old_sample { 56 | let (t_value, t_distribution, relative_estimates, relative_distributions, base_avg_times) = 57 | compare(avg_values, &old_sample, config); 58 | let p_value = t_distribution.p_value(t_value, &Tails::Two); 59 | Some(crate::report::ComparisonData { 60 | p_value, 61 | t_distribution, 62 | t_value, 63 | relative_estimates, 64 | relative_distributions, 65 | significance_threshold: config.significance_level, 66 | noise_threshold: config.noise_threshold, 67 | base_iter_counts: old_sample.iteration_count.to_vec(), 68 | base_sample_times: old_sample.sample_values.to_vec(), 69 | base_avg_times, 70 | base_estimates: old_estimates.clone(), 71 | }) 72 | } else { 73 | None 74 | }; 75 | 76 | MeasurementData { 77 | data: Data::new(iters, values), 78 | avg_times: labeled_sample, 79 | absolute_estimates: estimates, 80 | distributions, 81 | comparison: compare_data, 82 | throughput, 83 | } 84 | } 85 | 86 | // Performs a simple linear regression on the sample 87 | fn regression( 88 | data: &Data<'_, f64, f64>, 89 | config: &BenchmarkConfig, 90 | ) -> (Distribution, Estimate) { 91 | let cl = config.confidence_level; 92 | 93 | let distribution = elapsed!( 94 | "Bootstrapped linear regression", 95 | data.bootstrap(config.nresamples, |d| (Slope::fit(&d).0,)) 96 | ) 97 | .0; 98 | 99 | let point = Slope::fit(data); 100 | let (lb, ub) = distribution.confidence_interval(config.confidence_level); 101 | let se = distribution.std_dev(None); 102 | 103 | ( 104 | distribution, 105 | Estimate { 106 | confidence_interval: ConfidenceInterval { 107 | confidence_level: cl, 108 | lower_bound: lb, 109 | upper_bound: ub, 110 | }, 111 | point_estimate: point.0, 112 | standard_error: se, 113 | }, 114 | ) 115 | } 116 | 117 | // Estimates the statistics of the population from the sample 118 | fn estimates(avg_times: &Sample, config: &BenchmarkConfig) -> (Distributions, Estimates) { 119 | fn stats(sample: &Sample) -> (f64, f64, f64, f64) { 120 | let mean = sample.mean(); 121 | let std_dev = sample.std_dev(Some(mean)); 122 | let median = sample.percentiles().median(); 123 | let mad = sample.median_abs_dev(Some(median)); 124 | 125 | (mean, std_dev, median, mad) 126 | } 127 | 128 | let cl = config.confidence_level; 129 | let nresamples = config.nresamples; 130 | 131 | let (mean, std_dev, median, mad) = stats(avg_times); 132 | let points = PointEstimates { 133 | mean, 134 | median, 135 | std_dev, 136 | median_abs_dev: mad, 137 | }; 138 | 139 | let (dist_mean, dist_stddev, dist_median, dist_mad) = elapsed!( 140 | "Bootstrapping the absolute statistics.", 141 | avg_times.bootstrap(nresamples, stats) 142 | ); 143 | 144 | let distributions = Distributions { 145 | mean: dist_mean, 146 | slope: None, 147 | median: dist_median, 148 | median_abs_dev: dist_mad, 149 | std_dev: dist_stddev, 150 | }; 151 | 152 | let estimates = build_estimates(&distributions, &points, cl); 153 | 154 | (distributions, estimates) 155 | } 156 | 157 | // Common comparison procedure 158 | #[allow(clippy::type_complexity)] 159 | pub(crate) fn compare( 160 | new_avg_times: &Sample, 161 | old_values: &MeasuredValues, 162 | config: &BenchmarkConfig, 163 | ) -> ( 164 | f64, 165 | Distribution, 166 | ChangeEstimates, 167 | ChangeDistributions, 168 | Vec, 169 | ) { 170 | let iters = old_values.iteration_count; 171 | let values = old_values.sample_values; 172 | let base_avg_values: Vec = iters 173 | .iter() 174 | .zip(values.iter()) 175 | .map(|(iters, elapsed)| elapsed / iters) 176 | .collect(); 177 | let base_avg_value_sample = Sample::new(&base_avg_values); 178 | 179 | let (t_statistic, t_distribution) = t_test(new_avg_times, base_avg_value_sample, config); 180 | 181 | let (estimates, relative_distributions) = 182 | difference_estimates(new_avg_times, base_avg_value_sample, config); 183 | 184 | ( 185 | t_statistic, 186 | t_distribution, 187 | estimates, 188 | relative_distributions, 189 | base_avg_values, 190 | ) 191 | } 192 | 193 | // Performs a two sample t-test 194 | fn t_test( 195 | avg_times: &Sample, 196 | base_avg_times: &Sample, 197 | config: &BenchmarkConfig, 198 | ) -> (f64, Distribution) { 199 | let nresamples = config.nresamples; 200 | 201 | let t_statistic = avg_times.t(base_avg_times); 202 | let t_distribution = elapsed!( 203 | "Bootstrapping the T distribution", 204 | crate::stats::univariate::mixed::bootstrap( 205 | avg_times, 206 | base_avg_times, 207 | nresamples, 208 | |a, b| (a.t(b),) 209 | ) 210 | ) 211 | .0; 212 | 213 | // HACK: Filter out non-finite numbers, which can happen sometimes when sample size is very small. 214 | // Downstream code doesn't like non-finite values here. 215 | let t_distribution = Distribution::from( 216 | t_distribution 217 | .iter() 218 | .filter(|a| a.is_finite()) 219 | .cloned() 220 | .collect::>() 221 | .into_boxed_slice(), 222 | ); 223 | 224 | (t_statistic, t_distribution) 225 | } 226 | 227 | // Estimates the relative change in the statistics of the population 228 | fn difference_estimates( 229 | avg_times: &Sample, 230 | base_avg_times: &Sample, 231 | config: &BenchmarkConfig, 232 | ) -> (ChangeEstimates, ChangeDistributions) { 233 | fn stats(a: &Sample, b: &Sample) -> (f64, f64) { 234 | ( 235 | a.mean() / b.mean() - 1., 236 | a.percentiles().median() / b.percentiles().median() - 1., 237 | ) 238 | } 239 | 240 | let cl = config.confidence_level; 241 | let nresamples = config.nresamples; 242 | 243 | let (dist_mean, dist_median) = elapsed!( 244 | "Bootstrapping the relative statistics", 245 | crate::stats::univariate::bootstrap(avg_times, base_avg_times, nresamples, stats) 246 | ); 247 | 248 | let distributions = ChangeDistributions { 249 | mean: dist_mean, 250 | median: dist_median, 251 | }; 252 | 253 | let (mean, median) = stats(avg_times, base_avg_times); 254 | let points = ChangePointEstimates { mean, median }; 255 | 256 | let estimates = build_change_estimates(&distributions, &points, cl); 257 | 258 | (estimates, distributions) 259 | } 260 | -------------------------------------------------------------------------------- /src/compile.rs: -------------------------------------------------------------------------------- 1 | //! Module that handles calling out to `cargo bench` and parsing the machine-readable messages 2 | //! to compile the benchmarks and collect the information on the benchmark executables that it 3 | //! emits. 4 | 5 | use crate::bench_target::BenchTarget; 6 | use anyhow::{Context, Result}; 7 | use std::path::PathBuf; 8 | use std::process::{Command, ExitStatus, Stdio}; 9 | 10 | #[derive(Debug)] 11 | /// Enum representing the different ways calling Cargo might fail 12 | pub enum CompileError { 13 | CompileFailed(ExitStatus), 14 | } 15 | impl std::fmt::Display for CompileError { 16 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 17 | match self { 18 | CompileError::CompileFailed(exit_status) => write!( 19 | f, 20 | "'cargo bench' returned an error ({}); unable to continue.", 21 | exit_status 22 | ), 23 | } 24 | } 25 | } 26 | impl std::error::Error for CompileError { 27 | fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { 28 | match self { 29 | CompileError::CompileFailed(_) => None, 30 | } 31 | } 32 | } 33 | 34 | // These structs match the parts of Cargo's message format that we care about. 35 | #[derive(Serialize, Deserialize, Debug)] 36 | struct Target { 37 | name: String, 38 | kind: Vec, 39 | } 40 | 41 | /// Enum listing out the different types of messages that Cargo can send. We only care about the 42 | /// compiler-artifact message. 43 | #[derive(Serialize, Deserialize, Debug)] 44 | #[serde(tag = "reason")] 45 | #[allow(clippy::enum_variant_names)] 46 | enum Message { 47 | #[serde(rename = "compiler-artifact")] 48 | CompilerArtifact { 49 | target: Target, 50 | executable: Option, 51 | }, 52 | 53 | // TODO: Delete these and replace with a #[serde(other)] variant 54 | // See https://github.com/serde-rs/serde/issues/912 55 | #[serde(rename = "compiler-message")] 56 | CompilerMessage {}, 57 | 58 | #[serde(rename = "build-script-executed")] 59 | BuildScriptExecuted { linked_paths: Vec }, 60 | 61 | #[serde(rename = "build-finished")] 62 | BuildFinished {}, 63 | } 64 | 65 | #[derive(Debug)] 66 | pub struct CompiledBenchmarks { 67 | pub targets: Vec, 68 | pub library_paths: Vec, 69 | } 70 | 71 | /// Launches `cargo bench` with the given additional arguments, with some additional arguments to 72 | /// list out the benchmarks and their executables and parses that information. This compiles the 73 | /// benchmarks but doesn't run them. Returns information on the compiled benchmarks that we can use 74 | /// to run them directly. 75 | pub fn compile(debug_build: bool, cargo_args: &[std::ffi::OsString]) -> Result { 76 | let subcommand: &[&'static str] = if debug_build { 77 | &["test", "--benches"] 78 | } else { 79 | &["bench"] 80 | }; 81 | 82 | let mut cargo = Command::new("cargo") 83 | .args(subcommand) 84 | .args(cargo_args) 85 | .args(["--no-run", "--message-format", "json-render-diagnostics"]) 86 | .stdin(Stdio::null()) 87 | .stderr(Stdio::inherit()) // Cargo writes its normal compile output to stderr 88 | .stdout(Stdio::piped()) // Capture the JSON messages on stdout 89 | .spawn()?; 90 | 91 | // Build a message stream reading from the child process 92 | let cargo_stdout = cargo 93 | .stdout 94 | .take() 95 | .expect("Child process doesn't have a stdout handle"); 96 | let stream = serde_json::Deserializer::from_reader(cargo_stdout).into_iter::(); 97 | 98 | // Collect the benchmark artifacts from the message stream 99 | let mut targets = vec![]; 100 | let mut library_paths = vec![]; 101 | for message in stream { 102 | let message = message.context("Failed to parse message from cargo")?; 103 | match message { 104 | Message::CompilerArtifact { target, executable } => { 105 | if target 106 | .kind 107 | .iter() 108 | // Benchmarks and tests have executables. Libraries might, if they expose tests. 109 | .any(|kind| kind == "bench" || kind == "test" || kind == "lib") 110 | { 111 | if let Some(executable) = executable { 112 | targets.push(BenchTarget { 113 | name: target.name, 114 | executable, 115 | }); 116 | } 117 | } 118 | } 119 | Message::BuildScriptExecuted { linked_paths } => { 120 | for path in linked_paths { 121 | let path = path 122 | .replace("dependency=", "") 123 | .replace("crate=", "") 124 | .replace("native=", "") 125 | .replace("framework=", "") 126 | .replace("all=", ""); 127 | let path = PathBuf::from(path); 128 | library_paths.push(path); 129 | } 130 | } 131 | _ => (), 132 | } 133 | } 134 | 135 | targets.sort_by(|target1, target2| (target1.name).cmp(&target2.name)); 136 | 137 | let exit_status = cargo 138 | .wait() 139 | .context("Cargo compilation failed in an unexpected way")?; 140 | if !(exit_status.success()) { 141 | Err(CompileError::CompileFailed(exit_status).into()) 142 | } else { 143 | Ok(CompiledBenchmarks { 144 | targets, 145 | library_paths, 146 | }) 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/estimate.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use crate::stats::Distribution; 4 | 5 | #[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Deserialize, Serialize, Debug)] 6 | pub enum Statistic { 7 | Mean, 8 | Median, 9 | MedianAbsDev, 10 | Slope, 11 | StdDev, 12 | Typical, 13 | } 14 | 15 | impl fmt::Display for Statistic { 16 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 17 | match *self { 18 | Statistic::Mean => f.pad("mean"), 19 | Statistic::Median => f.pad("median"), 20 | Statistic::MedianAbsDev => f.pad("MAD"), 21 | Statistic::Slope => f.pad("slope"), 22 | Statistic::StdDev => f.pad("SD"), 23 | Statistic::Typical => f.pad("typical"), 24 | } 25 | } 26 | } 27 | 28 | #[derive(Clone, PartialEq, Deserialize, Serialize, Debug)] 29 | pub struct ConfidenceInterval { 30 | pub confidence_level: f64, 31 | pub lower_bound: f64, 32 | pub upper_bound: f64, 33 | } 34 | 35 | #[derive(Clone, PartialEq, Deserialize, Serialize, Debug)] 36 | pub struct Estimate { 37 | /// The confidence interval for this estimate 38 | pub confidence_interval: ConfidenceInterval, 39 | pub point_estimate: f64, 40 | /// The standard error of this estimate 41 | pub standard_error: f64, 42 | } 43 | 44 | pub fn build_estimates( 45 | distributions: &Distributions, 46 | points: &PointEstimates, 47 | cl: f64, 48 | ) -> Estimates { 49 | let to_estimate = |point_estimate, distribution: &Distribution| { 50 | let (lb, ub) = distribution.confidence_interval(cl); 51 | 52 | Estimate { 53 | confidence_interval: ConfidenceInterval { 54 | confidence_level: cl, 55 | lower_bound: lb, 56 | upper_bound: ub, 57 | }, 58 | point_estimate, 59 | standard_error: distribution.std_dev(None), 60 | } 61 | }; 62 | 63 | Estimates { 64 | mean: to_estimate(points.mean, &distributions.mean), 65 | median: to_estimate(points.median, &distributions.median), 66 | median_abs_dev: to_estimate(points.median_abs_dev, &distributions.median_abs_dev), 67 | slope: None, 68 | std_dev: to_estimate(points.std_dev, &distributions.std_dev), 69 | } 70 | } 71 | 72 | pub fn build_change_estimates( 73 | distributions: &ChangeDistributions, 74 | points: &ChangePointEstimates, 75 | cl: f64, 76 | ) -> ChangeEstimates { 77 | let to_estimate = |point_estimate, distribution: &Distribution| { 78 | let (lb, ub) = distribution.confidence_interval(cl); 79 | 80 | Estimate { 81 | confidence_interval: ConfidenceInterval { 82 | confidence_level: cl, 83 | lower_bound: lb, 84 | upper_bound: ub, 85 | }, 86 | point_estimate, 87 | standard_error: distribution.std_dev(None), 88 | } 89 | }; 90 | 91 | ChangeEstimates { 92 | mean: to_estimate(points.mean, &distributions.mean), 93 | median: to_estimate(points.median, &distributions.median), 94 | } 95 | } 96 | 97 | pub struct PointEstimates { 98 | pub mean: f64, 99 | pub median: f64, 100 | pub median_abs_dev: f64, 101 | pub std_dev: f64, 102 | } 103 | 104 | #[derive(Debug, Serialize, Deserialize, Clone)] 105 | pub struct Estimates { 106 | pub mean: Estimate, 107 | pub median: Estimate, 108 | pub median_abs_dev: Estimate, 109 | pub slope: Option, 110 | pub std_dev: Estimate, 111 | } 112 | impl Estimates { 113 | pub fn typical(&self) -> &Estimate { 114 | self.slope.as_ref().unwrap_or(&self.mean) 115 | } 116 | pub fn get(&self, stat: Statistic) -> Option<&Estimate> { 117 | match stat { 118 | Statistic::Mean => Some(&self.mean), 119 | Statistic::Median => Some(&self.median), 120 | Statistic::MedianAbsDev => Some(&self.median_abs_dev), 121 | Statistic::Slope => self.slope.as_ref(), 122 | Statistic::StdDev => Some(&self.std_dev), 123 | Statistic::Typical => Some(self.typical()), 124 | } 125 | } 126 | } 127 | 128 | pub struct Distributions { 129 | pub mean: Distribution, 130 | pub median: Distribution, 131 | pub median_abs_dev: Distribution, 132 | pub slope: Option>, 133 | pub std_dev: Distribution, 134 | } 135 | impl Distributions { 136 | pub fn typical(&self) -> &Distribution { 137 | self.slope.as_ref().unwrap_or(&self.mean) 138 | } 139 | pub fn get(&self, stat: Statistic) -> Option<&Distribution> { 140 | match stat { 141 | Statistic::Mean => Some(&self.mean), 142 | Statistic::Median => Some(&self.median), 143 | Statistic::MedianAbsDev => Some(&self.median_abs_dev), 144 | Statistic::Slope => self.slope.as_ref(), 145 | Statistic::StdDev => Some(&self.std_dev), 146 | Statistic::Typical => Some(self.typical()), 147 | } 148 | } 149 | } 150 | 151 | pub struct ChangePointEstimates { 152 | pub mean: f64, 153 | pub median: f64, 154 | } 155 | 156 | #[derive(Debug, Serialize, Deserialize, Clone)] 157 | pub struct ChangeEstimates { 158 | pub mean: Estimate, 159 | pub median: Estimate, 160 | } 161 | impl ChangeEstimates { 162 | pub fn get(&self, stat: Statistic) -> &Estimate { 163 | match stat { 164 | Statistic::Mean => &self.mean, 165 | Statistic::Median => &self.median, 166 | _ => panic!("Unexpected statistic"), 167 | } 168 | } 169 | } 170 | 171 | pub struct ChangeDistributions { 172 | pub mean: Distribution, 173 | pub median: Distribution, 174 | } 175 | impl ChangeDistributions { 176 | pub fn get(&self, stat: Statistic) -> &Distribution { 177 | match stat { 178 | Statistic::Mean => &self.mean, 179 | Statistic::Median => &self.median, 180 | _ => panic!("Unexpected statistic"), 181 | } 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /src/format.rs: -------------------------------------------------------------------------------- 1 | pub fn change(pct: f64, signed: bool) -> String { 2 | if signed { 3 | format!("{:>+6}%", signed_short(pct * 1e2)) 4 | } else { 5 | format!("{:>6}%", short(pct * 1e2)) 6 | } 7 | } 8 | 9 | pub fn time(ns: f64) -> String { 10 | if ns < 1.0 { 11 | format!("{:>6} ps", short(ns * 1e3)) 12 | } else if ns < 10f64.powi(3) { 13 | format!("{:>6} ns", short(ns)) 14 | } else if ns < 10f64.powi(6) { 15 | format!("{:>6} us", short(ns / 1e3)) 16 | } else if ns < 10f64.powi(9) { 17 | format!("{:>6} ms", short(ns / 1e6)) 18 | } else { 19 | format!("{:>6} s", short(ns / 1e9)) 20 | } 21 | } 22 | 23 | pub fn short(n: f64) -> String { 24 | if n < 10.0 { 25 | format!("{:.4}", n) 26 | } else if n < 100.0 { 27 | format!("{:.3}", n) 28 | } else if n < 1000.0 { 29 | format!("{:.2}", n) 30 | } else if n < 10000.0 { 31 | format!("{:.1}", n) 32 | } else { 33 | format!("{:.0}", n) 34 | } 35 | } 36 | 37 | fn signed_short(n: f64) -> String { 38 | let n_abs = n.abs(); 39 | 40 | if n_abs < 10.0 { 41 | format!("{:+.4}", n) 42 | } else if n_abs < 100.0 { 43 | format!("{:+.3}", n) 44 | } else if n_abs < 1000.0 { 45 | format!("{:+.2}", n) 46 | } else if n_abs < 10000.0 { 47 | format!("{:+.1}", n) 48 | } else { 49 | format!("{:+.0}", n) 50 | } 51 | } 52 | 53 | pub fn iter_count(iterations: u64) -> String { 54 | if iterations < 10_000 { 55 | format!("{} iterations", iterations) 56 | } else if iterations < 1_000_000 { 57 | format!("{:.0}k iterations", (iterations as f64) / 1000.0) 58 | } else if iterations < 10_000_000 { 59 | format!("{:.1}M iterations", (iterations as f64) / (1000.0 * 1000.0)) 60 | } else if iterations < 1_000_000_000 { 61 | format!("{:.0}M iterations", (iterations as f64) / (1000.0 * 1000.0)) 62 | } else if iterations < 10_000_000_000 { 63 | format!( 64 | "{:.1}B iterations", 65 | (iterations as f64) / (1000.0 * 1000.0 * 1000.0) 66 | ) 67 | } else { 68 | format!( 69 | "{:.0}B iterations", 70 | (iterations as f64) / (1000.0 * 1000.0 * 1000.0) 71 | ) 72 | } 73 | } 74 | 75 | pub fn integer(n: f64) -> String { 76 | format!("{}", n as u64) 77 | } 78 | 79 | #[cfg(test)] 80 | mod test { 81 | use super::*; 82 | 83 | #[test] 84 | fn short_max_len() { 85 | let mut float = 1.0; 86 | while float < 999_999.9 { 87 | let string = short(float); 88 | println!("{}", string); 89 | assert!(string.len() <= 6); 90 | float *= 2.0; 91 | } 92 | } 93 | 94 | #[test] 95 | fn signed_short_max_len() { 96 | let mut float = -1.0; 97 | while float > -999_999.9 { 98 | let string = signed_short(float); 99 | println!("{}", string); 100 | assert!(string.len() <= 7); 101 | float *= 2.0; 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/html/common.css: -------------------------------------------------------------------------------- 1 | body { 2 | font: 14px Helvetica Neue; 3 | text-rendering: optimizelegibility; 4 | } 5 | 6 | .body { 7 | width: 960px; 8 | margin: auto; 9 | } 10 | 11 | a:link { 12 | color: #1F78B4; 13 | text-decoration: none; 14 | } 15 | 16 | h2 { 17 | font-size: 36px; 18 | font-weight: 300; 19 | } 20 | 21 | h3 { 22 | font-size: 24px; 23 | font-weight: 300; 24 | } 25 | 26 | #footer { 27 | height: 40px; 28 | background: #888; 29 | color: white; 30 | font-size: larger; 31 | font-weight: 300; 32 | } 33 | 34 | #footer a { 35 | color: white; 36 | text-decoration: underline; 37 | } 38 | 39 | #footer p { 40 | text-align: center 41 | } -------------------------------------------------------------------------------- /src/html/history_report.html.tt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | {title} History - Criterion.rs 7 | 32 | 33 | 34 | 35 |
36 | History of { title } 37 | {{- for entry in history }} 38 |

# { entry.number }{{ if entry.id }} - {entry.id}{{ endif }}{ entry.datetime }

39 |
40 | {{- if entry.description }} 41 |

42 | { entry.description } 43 |

44 | {{ endif }} 45 |
46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | {{- if entry.throughput }} 60 | 61 | 62 | 63 | 64 | 65 | 66 | {{- endif }} 67 | {{- if entry.change_value }} 68 | 69 | 70 | 71 | 72 | 73 | 74 | {{- endif }} 75 | {{- if entry.change_throughput }} 76 | 77 | 78 | 79 | 80 | 81 | 82 | {{- endif }} 83 |
Lower BoundEstimateUpper Bound
Value:{entry.value.lower}{entry.value.point}{entry.value.upper}
Throughput:{entry.throughput.lower}{entry.throughput.point}{entry.throughput.upper}
Change in Value:{entry.change_value.lower}{entry.change_value.point}{entry.change_value.upper}
Change in Throughput:{entry.change_throughput.lower}{entry.change_throughput.point}{entry.change_throughput.upper}
84 | 85 | {{- if entry.has_improved }} 86 | Performance has improved 87 | {{- endif }} 88 | {{- if entry.has_regressed }} 89 | Performance has regressed 90 | {{- endif }} 91 | {{- if entry.is_not_significant }} 92 | Change within noise threshold. 93 | {{- endif }} 94 | {{- if entry.is_no_change }} 95 | No change in performance detected. 96 | {{- endif }} 97 | {{- if @last }}{{ else }}
{{ endif }} 98 | {{- endfor }} 99 |
100 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /src/html/index.html.tt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Index - Criterion.rs 7 | 20 | 21 | 22 | 23 |
24 |

Criterion.rs Benchmark Index

25 | See individual benchmark pages below for more details. 26 |
    27 | {{- for group in groups }} 28 |
  • {{ call report_link with group.group_report }}
    • 29 | {{- if group.function_ids }} 30 | {{- if group.values }} 31 | {# Function ids and values #} 32 |
      33 |
    • 34 | 35 | 36 | 37 | {{- for func in group.function_ids }} 38 | 39 | {{- endfor }} 40 | 41 | {{- for row in group.individual_links }} 42 | 43 | 44 | {{- for bench in row.benchmarks }} 45 | 46 | {{- endfor }} 47 | 48 | {{- endfor }} 49 |
      {{ call report_link with func }}
      {{ call report_link with row.value }}{{ call report_link with bench }}
      50 |
      • 51 |
    52 | {{- else }} 53 | {# Function IDs but not values #} 54 |
      55 | {{- for func in group.function_ids }} 56 |
    • {{ call report_link with func }}
      • 57 | {{- endfor }} 58 |
    59 | {{- endif }} 60 | {{- else }} 61 | {{- if group.values }} 62 | {# Values but not function ids #} 63 |
      64 | {{- for val in group.values }} 65 |
    • {{ call report_link with val }}
      • 66 | {{- endfor }} 67 |
    68 | {{- endif }} 69 | {{- endif }} 70 | {{- endfor }} 71 |
72 |
73 | 76 | 77 | -------------------------------------------------------------------------------- /src/html/report_link.html.tt: -------------------------------------------------------------------------------- 1 | {{ if path -}} 2 | {name} 3 | {{- else -}} 4 | {name} 5 | {{- endif}} -------------------------------------------------------------------------------- /src/html/summary_report.html.tt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | {group_id} Summary - Criterion.rs 7 | 10 | 11 | 12 | 13 |
14 |

{group_id}

15 | {{- if violin_plot }} 16 |

Violin Plot

17 | 18 | Violin Plot 19 | 20 |

This chart shows the relationship between function/parameter and iteration time. The thickness of the shaded 21 | region indicates the probability that a measurement of the given function/parameter would take a particular 22 | length of time.

23 | {{- endif }} 24 | {{- if line_chart }} 25 |

Line Chart

26 | Line Chart 27 |

This chart shows the mean measured time for each function as the input (or the size of the input) increases.

28 | {{- endif }} 29 | {{- for bench in benchmarks }} 30 |
31 | 32 |

{bench.name}

33 | 34 | 35 | 36 | 37 | 43 | 56 | 57 | 58 |
38 | 39 | PDF of Slope 41 | 42 | 44 | {{- if bench.regression_exists }} 45 | 46 | Regression 48 | 49 | {{- else }} 50 | 51 | Iteration Times 53 | 54 | {{- endif }} 55 |
59 |
60 | {{- endfor }} 61 |
62 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /src/kde.rs: -------------------------------------------------------------------------------- 1 | use crate::stats::univariate::kde::kernel::Gaussian; 2 | use crate::stats::univariate::kde::{Bandwidth, Kde}; 3 | use crate::stats::univariate::Sample; 4 | 5 | pub fn sweep( 6 | sample: &Sample, 7 | npoints: usize, 8 | range: Option<(f64, f64)>, 9 | ) -> (Box<[f64]>, Box<[f64]>) { 10 | let (xs, ys, _) = sweep_and_estimate(sample, npoints, range, sample[0]); 11 | (xs, ys) 12 | } 13 | 14 | pub fn sweep_and_estimate( 15 | sample: &Sample, 16 | npoints: usize, 17 | range: Option<(f64, f64)>, 18 | point_to_estimate: f64, 19 | ) -> (Box<[f64]>, Box<[f64]>, f64) { 20 | let x_min = sample.min(); 21 | let x_max = sample.max(); 22 | 23 | let kde = Kde::new(sample, Gaussian, Bandwidth::Silverman); 24 | let h = kde.bandwidth(); 25 | 26 | let (start, end) = match range { 27 | Some((start, end)) => (start, end), 28 | None => (x_min - 3. * h, x_max + 3. * h), 29 | }; 30 | 31 | let mut xs: Vec = Vec::with_capacity(npoints); 32 | let step_size = (end - start) / (npoints - 1) as f64; 33 | for n in 0..npoints { 34 | xs.push(start + (step_size * n as f64)); 35 | } 36 | 37 | let ys = kde.map(&xs); 38 | let point_estimate = kde.estimate(point_to_estimate); 39 | 40 | (xs.into_boxed_slice(), ys, point_estimate) 41 | } 42 | -------------------------------------------------------------------------------- /src/macros_private.rs: -------------------------------------------------------------------------------- 1 | /// Matches a result, returning the `Ok` value in case of success, 2 | /// exits the calling function otherwise. 3 | /// A closure which returns the return value for the function can 4 | /// be passed as second parameter. 5 | macro_rules! try_else_return { 6 | ($x:expr) => { 7 | try_else_return!($x, || {}); 8 | }; 9 | ($x:expr, $el:expr) => { 10 | match $x { 11 | Ok(x) => x, 12 | Err(e) => { 13 | error!("Error: {:?}", &e); 14 | let closure = $el; 15 | return closure(); 16 | } 17 | } 18 | }; 19 | } 20 | 21 | /// vec! but for pathbufs 22 | macro_rules! path { 23 | (PUSH $to:expr, $x:expr, $($y:expr),+) => { 24 | $to.push($x); 25 | path!(PUSH $to, $($y),+); 26 | }; 27 | (PUSH $to:expr, $x:expr) => { 28 | $to.push($x); 29 | }; 30 | ($x:expr, $($y:expr),+) => {{ 31 | let mut path_buffer = std::path::PathBuf::new(); 32 | path!(PUSH path_buffer, $x, $($y),+); 33 | path_buffer 34 | }}; 35 | } 36 | 37 | macro_rules! elapsed { 38 | ($msg:expr, $block:expr) => {{ 39 | let start = ::std::time::Instant::now(); 40 | let out = $block; 41 | let elapsed = &start.elapsed(); 42 | 43 | info!( 44 | "{} took {}", 45 | $msg, 46 | crate::format::time(crate::DurationExt::to_nanos(elapsed) as f64) 47 | ); 48 | 49 | out 50 | }}; 51 | } 52 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | //! A Cargo extension for running [Criterion.rs] benchmarks and reporting the results. 2 | //! 3 | //! This crate is a Cargo extension which can be used as a replacement for `cargo bench` when 4 | //! running [Criterion.rs] benchmarks. 5 | 6 | #![ 7 | allow( 8 | clippy::just_underscores_and_digits, // Used in the stats code 9 | clippy::transmute_ptr_to_ptr, // Used in the stats code 10 | ) 11 | ] 12 | 13 | #[macro_use] 14 | extern crate serde_derive; 15 | 16 | #[macro_use] 17 | extern crate log; 18 | 19 | #[macro_use] 20 | mod macros_private; 21 | 22 | #[allow(clippy::too_many_arguments)] 23 | #[macro_use] 24 | mod plot; 25 | 26 | mod analysis; 27 | mod bench_target; 28 | mod compile; 29 | mod config; 30 | mod connection; 31 | mod estimate; 32 | mod format; 33 | mod html; 34 | mod kde; 35 | mod message_formats; 36 | mod model; 37 | mod report; 38 | mod stats; 39 | mod value_formatter; 40 | 41 | use crate::config::{OutputFormat, PlottingBackend, SelfConfig, TextColor}; 42 | use crate::connection::{AxisScale, PlotConfiguration}; 43 | use crate::plot::Plotter; 44 | use crate::report::{Report, ReportContext}; 45 | use anyhow::Error; 46 | use lazy_static::lazy_static; 47 | 48 | lazy_static! { 49 | static ref DEBUG_ENABLED: bool = std::env::var_os("CRITERION_DEBUG").is_some(); 50 | } 51 | 52 | fn debug_enabled() -> bool { 53 | *DEBUG_ENABLED 54 | } 55 | 56 | /// Configures the logger based on the debug environment variable. 57 | fn configure_log() { 58 | use simplelog::*; 59 | let filter = if debug_enabled() { 60 | LevelFilter::max() 61 | } else { 62 | LevelFilter::Warn 63 | }; 64 | TermLogger::init( 65 | filter, 66 | Default::default(), 67 | TerminalMode::Stderr, 68 | ColorChoice::Never, 69 | ) 70 | .unwrap(); 71 | } 72 | 73 | // TODO: Write unit tests for serialization. 74 | 75 | /// Main entry point for cargo-criterion. 76 | fn main() -> Result<(), Box> { 77 | configure_log(); 78 | 79 | // First, load the config file and parse the command-line args. 80 | let configuration = config::configure()?; 81 | let self_config = &configuration.self_config; 82 | 83 | // Launch cargo to compile the crate and produce a list of the benchmark targets to run. 84 | let compile::CompiledBenchmarks { 85 | targets, 86 | library_paths, 87 | } = compile::compile(self_config.debug_build, &configuration.cargo_args)?; 88 | 89 | // Load the saved measurements from the last run. 90 | let mut run_model = model::Model::load( 91 | self_config.criterion_home.clone(), 92 | "main".into(), 93 | self_config.history_id.clone(), 94 | self_config.history_description.clone(), 95 | ); 96 | 97 | // Set up the reports. These receive notifications as the benchmarks proceed and generate output for the user. 98 | let cli_report = configure_cli_output(self_config); 99 | let bencher_report = crate::report::BencherReport; 100 | let html_report = get_plotter(self_config)?.map(crate::html::Html::new); 101 | let machine_report = message_formats::create_machine_report(self_config); 102 | 103 | let mut reports: Vec<&dyn crate::report::Report> = Vec::new(); 104 | match self_config.output_format { 105 | OutputFormat::Bencher => reports.push(&bencher_report), 106 | OutputFormat::Criterion | OutputFormat::Quiet | OutputFormat::Verbose => { 107 | reports.push(&cli_report) 108 | } 109 | } 110 | if let Some(html_report) = &html_report { 111 | reports.push(html_report); 112 | } 113 | if let Some(machine_report) = &machine_report { 114 | reports.push(machine_report); 115 | } 116 | let reports = crate::report::Reports::new(reports); 117 | 118 | if self_config.do_run { 119 | // Execute each benchmark target, updating the model as we go. 120 | for bench in targets { 121 | info!("Executing {} - {:?}", bench.name, bench.executable); 122 | let err = bench.execute( 123 | &self_config.criterion_home, 124 | &configuration.additional_args, 125 | &library_paths, 126 | &reports, 127 | &mut run_model, 128 | self_config.message_format.is_some(), 129 | ); 130 | 131 | if let Err(err) = err { 132 | if self_config.do_fail_fast { 133 | return Err(err.into()); 134 | } else { 135 | error!( 136 | "Failed to execute benchmark target {}:\n{}", 137 | bench.name, err 138 | ); 139 | } 140 | } 141 | } 142 | 143 | // Generate the overall summary report using all of the records in the model. 144 | let final_context = ReportContext { 145 | output_directory: self_config.criterion_home.join("reports"), 146 | plot_config: PlotConfiguration { 147 | summary_scale: AxisScale::Linear, 148 | }, 149 | }; 150 | 151 | reports.final_summary(&final_context, &run_model); 152 | } 153 | Ok(()) 154 | } 155 | 156 | /// Configure and return a Report object that prints benchmark information to the command-line. 157 | fn configure_cli_output(self_config: &crate::config::SelfConfig) -> crate::report::CliReport { 158 | let stderr_isatty = atty::is(atty::Stream::Stderr); 159 | let mut enable_text_overwrite = stderr_isatty && !debug_enabled(); 160 | let enable_text_coloring = match self_config.text_color { 161 | TextColor::Auto => stderr_isatty, 162 | TextColor::Never => { 163 | enable_text_overwrite = false; 164 | false 165 | } 166 | TextColor::Always => true, 167 | }; 168 | 169 | let show_differences = match self_config.output_format { 170 | OutputFormat::Criterion | OutputFormat::Verbose => true, 171 | OutputFormat::Quiet | OutputFormat::Bencher => false, 172 | }; 173 | let verbose = match self_config.output_format { 174 | OutputFormat::Verbose => true, 175 | OutputFormat::Criterion | OutputFormat::Quiet | OutputFormat::Bencher => debug_enabled(), 176 | }; 177 | 178 | crate::report::CliReport::new( 179 | enable_text_overwrite, 180 | enable_text_coloring, 181 | show_differences, 182 | verbose, 183 | ) 184 | } 185 | 186 | /// Configure and return a Gnuplot plotting backend, if available. 187 | #[cfg(feature = "gnuplot_backend")] 188 | fn gnuplot_plotter(config: &SelfConfig) -> Result, Error> { 189 | match criterion_plot::version() { 190 | Ok(_) => { 191 | let generator = crate::plot::PlotGenerator { 192 | backend: crate::plot::Gnuplot::new(&config.colors), 193 | }; 194 | Ok(Box::new(generator)) 195 | }, 196 | Err(_) => Err(anyhow::anyhow!("Gnuplot is not available. To continue, either install Gnuplot or allow cargo-criterion to fall back to using plotters.")), 197 | } 198 | } 199 | 200 | /// Gnuplot support was not compiled in, so the gnuplot backend is not available. 201 | #[cfg(not(feature = "gnuplot_backend"))] 202 | fn gnuplot_plotter(_: &SelfConfig) -> Result, Error> { 203 | anyhow::bail!("Gnuplot backend is disabled. To use gnuplot backend, install cargo-criterion with the 'gnuplot_backend' feature enabled") 204 | } 205 | 206 | /// Configure and return a Plotters plotting backend. 207 | #[cfg(feature = "plotters_backend")] 208 | fn plotters_plotter(config: &SelfConfig) -> Result, Error> { 209 | let generator = crate::plot::PlotGenerator { 210 | backend: crate::plot::PlottersBackend::new(&config.colors), 211 | }; 212 | Ok(Box::new(generator)) 213 | } 214 | 215 | /// Plotters support was not compiled in, so the plotters backend is not available. 216 | #[cfg(not(feature = "plotters_backend"))] 217 | fn plotters_plotter(_: &SelfConfig) -> Result, Error> { 218 | anyhow::bail!("Plotters backend is disabled. To use plotters backend, install cargo-criterion with the 'plotters_backend' feature enabled") 219 | } 220 | 221 | /// Configure and return a plotting backend. 222 | #[cfg(any(feature = "gnuplot_backend", feature = "plotters_backend"))] 223 | fn get_plotter(config: &SelfConfig) -> Result>, Error> { 224 | match config.plotting_backend { 225 | PlottingBackend::Gnuplot => gnuplot_plotter(config).map(Some), 226 | PlottingBackend::Plotters => plotters_plotter(config).map(Some), 227 | PlottingBackend::Auto => gnuplot_plotter(config) 228 | .or_else(|_| plotters_plotter(config)) 229 | .map(Some), 230 | PlottingBackend::Disabled => Ok(None), 231 | } 232 | } 233 | 234 | /// No plotting backend was compiled in. Plotting is disabled. 235 | #[cfg(not(any(feature = "gnuplot_backend", feature = "plotters_backend")))] 236 | fn get_plotter(config: &SelfConfig) -> Result>, Error> { 237 | match config.plotting_backend { 238 | PlottingBackend::Disabled => Ok(None), 239 | _ => anyhow::bail!("No plotting backend is available. At least one of the 'gnuplot_backend' or 'plotters_backend' features must be included.") 240 | } 241 | } 242 | 243 | /// Helper trait which adds a function for converting Duration to nanoseconds. 244 | trait DurationExt { 245 | fn to_nanos(&self) -> u64; 246 | } 247 | 248 | const NANOS_PER_SEC: u64 = 1_000_000_000; 249 | 250 | impl DurationExt for std::time::Duration { 251 | fn to_nanos(&self) -> u64 { 252 | self.as_secs() * NANOS_PER_SEC + u64::from(self.subsec_nanos()) 253 | } 254 | } 255 | -------------------------------------------------------------------------------- /src/message_formats/json.rs: -------------------------------------------------------------------------------- 1 | use crate::connection::Throughput as ThroughputEnum; 2 | use crate::model::BenchmarkGroup; 3 | use crate::report::{ 4 | compare_to_threshold, BenchmarkId, ComparisonResult, MeasurementData, Report, ReportContext, 5 | }; 6 | use crate::value_formatter::ValueFormatter; 7 | use anyhow::Result; 8 | use serde_derive::Serialize; 9 | use serde_json::json; 10 | use std::io::{stdout, Write}; 11 | 12 | use super::ConfidenceInterval; 13 | 14 | trait Message: serde::ser::Serialize { 15 | fn reason() -> &'static str; 16 | } 17 | 18 | #[derive(Serialize)] 19 | struct Throughput { 20 | per_iteration: u64, 21 | unit: String, 22 | } 23 | impl From<&ThroughputEnum> for Throughput { 24 | fn from(other: &ThroughputEnum) -> Self { 25 | match other { 26 | ThroughputEnum::Bytes(bytes) | ThroughputEnum::BytesDecimal(bytes) => Throughput { 27 | per_iteration: *bytes, 28 | unit: "bytes".to_owned(), 29 | }, 30 | ThroughputEnum::Elements(elements) => Throughput { 31 | per_iteration: *elements, 32 | unit: "elements".to_owned(), 33 | }, 34 | } 35 | } 36 | } 37 | 38 | #[derive(Serialize)] 39 | enum ChangeType { 40 | NoChange, 41 | Improved, 42 | Regressed, 43 | } 44 | 45 | #[derive(Serialize)] 46 | struct ChangeDetails { 47 | mean: ConfidenceInterval, 48 | median: ConfidenceInterval, 49 | 50 | change: ChangeType, 51 | } 52 | 53 | #[derive(Serialize)] 54 | struct BenchmarkComplete { 55 | id: String, 56 | report_directory: String, 57 | iteration_count: Vec, 58 | measured_values: Vec, 59 | unit: String, 60 | 61 | throughput: Vec, 62 | 63 | typical: ConfidenceInterval, 64 | mean: ConfidenceInterval, 65 | median: ConfidenceInterval, 66 | median_abs_dev: ConfidenceInterval, 67 | slope: Option, 68 | 69 | change: Option, 70 | } 71 | impl Message for BenchmarkComplete { 72 | fn reason() -> &'static str { 73 | "benchmark-complete" 74 | } 75 | } 76 | 77 | #[derive(Serialize)] 78 | struct BenchmarkGroupComplete { 79 | group_name: String, 80 | benchmarks: Vec, 81 | report_directory: String, 82 | } 83 | impl Message for BenchmarkGroupComplete { 84 | fn reason() -> &'static str { 85 | "group-complete" 86 | } 87 | } 88 | 89 | pub struct JsonMessageReport; 90 | impl JsonMessageReport { 91 | fn send_message(&self, message: M) { 92 | fn do_send(message: M) -> Result<()> { 93 | // Format the message to string 94 | let message_text = serde_json::to_string(&message)?; 95 | assert!(message_text.starts_with('{')); 96 | 97 | let reason = json!(M::reason()); 98 | 99 | // Concatenate that into the message 100 | writeln!(stdout(), "{{\"reason\":{},{}", reason, &message_text[1..])?; 101 | Ok(()) 102 | } 103 | if let Err(e) = do_send(message) { 104 | error!("Unexpected error writing JSON message: {:?}", e) 105 | } 106 | } 107 | } 108 | impl Report for JsonMessageReport { 109 | fn measurement_complete( 110 | &self, 111 | id: &BenchmarkId, 112 | context: &ReportContext, 113 | measurements: &MeasurementData<'_>, 114 | formatter: &ValueFormatter, 115 | ) { 116 | let mut measured_values = measurements.sample_times().to_vec(); 117 | let unit = formatter.scale_for_machines(&mut measured_values); 118 | 119 | let iteration_count: Vec = measurements 120 | .iter_counts() 121 | .iter() 122 | .map(|count| *count as u64) 123 | .collect(); 124 | 125 | let message = BenchmarkComplete { 126 | id: id.as_title().to_owned(), 127 | report_directory: path!(&context.output_directory, id.as_directory_name()) 128 | .display() 129 | .to_string(), 130 | iteration_count, 131 | measured_values, 132 | unit, 133 | 134 | throughput: measurements 135 | .throughput 136 | .iter() 137 | .map(Throughput::from) 138 | .collect(), 139 | 140 | typical: ConfidenceInterval::from_estimate( 141 | measurements.absolute_estimates.typical(), 142 | formatter, 143 | ), 144 | mean: ConfidenceInterval::from_estimate( 145 | &measurements.absolute_estimates.mean, 146 | formatter, 147 | ), 148 | median: ConfidenceInterval::from_estimate( 149 | &measurements.absolute_estimates.median, 150 | formatter, 151 | ), 152 | median_abs_dev: ConfidenceInterval::from_estimate( 153 | &measurements.absolute_estimates.median_abs_dev, 154 | formatter, 155 | ), 156 | slope: measurements 157 | .absolute_estimates 158 | .slope 159 | .as_ref() 160 | .map(|slope| ConfidenceInterval::from_estimate(slope, formatter)), 161 | change: measurements.comparison.as_ref().map(|comparison| { 162 | let different_mean = comparison.p_value < comparison.significance_threshold; 163 | let mean_est = &comparison.relative_estimates.mean; 164 | 165 | let change = if !different_mean { 166 | ChangeType::NoChange 167 | } else { 168 | let comparison = compare_to_threshold(mean_est, comparison.noise_threshold); 169 | match comparison { 170 | ComparisonResult::Improved => ChangeType::Improved, 171 | ComparisonResult::Regressed => ChangeType::Regressed, 172 | ComparisonResult::NonSignificant => ChangeType::NoChange, 173 | } 174 | }; 175 | 176 | ChangeDetails { 177 | mean: ConfidenceInterval::from_percent(&comparison.relative_estimates.mean), 178 | median: ConfidenceInterval::from_percent(&comparison.relative_estimates.median), 179 | change, 180 | } 181 | }), 182 | }; 183 | 184 | self.send_message(message); 185 | } 186 | 187 | fn summarize( 188 | &self, 189 | context: &ReportContext, 190 | group_id: &str, 191 | benchmark_group: &BenchmarkGroup, 192 | _formatter: &ValueFormatter, 193 | ) { 194 | let message = BenchmarkGroupComplete { 195 | group_name: group_id.to_owned(), 196 | benchmarks: benchmark_group 197 | .benchmarks 198 | .keys() 199 | .map(|id| id.as_title().to_owned()) 200 | .collect(), 201 | report_directory: path!( 202 | &context.output_directory, 203 | BenchmarkId::new(group_id.to_owned(), None, None, None).as_directory_name() 204 | ) 205 | .display() 206 | .to_string(), 207 | }; 208 | 209 | self.send_message(message); 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /src/message_formats/mod.rs: -------------------------------------------------------------------------------- 1 | mod json; 2 | mod openmetrics; 3 | 4 | use crate::config::{MessageFormat, SelfConfig}; 5 | use crate::estimate::Estimate; 6 | use crate::report::Report; 7 | use crate::value_formatter::ValueFormatter; 8 | 9 | use self::json::JsonMessageReport; 10 | use self::openmetrics::OpenMetricsMessageReport; 11 | 12 | #[derive(Serialize)] 13 | struct ConfidenceInterval { 14 | estimate: f64, 15 | lower_bound: f64, 16 | upper_bound: f64, 17 | unit: String, 18 | } 19 | impl ConfidenceInterval { 20 | fn from_estimate(estimate: &Estimate, value_formatter: &ValueFormatter) -> ConfidenceInterval { 21 | let mut array = [ 22 | estimate.point_estimate, 23 | estimate.confidence_interval.lower_bound, 24 | estimate.confidence_interval.upper_bound, 25 | ]; 26 | let unit = value_formatter.scale_for_machines(&mut array); 27 | let [estimate, lower_bound, upper_bound] = array; 28 | ConfidenceInterval { 29 | estimate, 30 | lower_bound, 31 | upper_bound, 32 | unit, 33 | } 34 | } 35 | fn from_percent(estimate: &Estimate) -> ConfidenceInterval { 36 | ConfidenceInterval { 37 | estimate: estimate.point_estimate, 38 | lower_bound: estimate.confidence_interval.lower_bound, 39 | upper_bound: estimate.confidence_interval.upper_bound, 40 | unit: "%".to_owned(), 41 | } 42 | } 43 | } 44 | 45 | pub enum MessageReport { 46 | Json(JsonMessageReport), 47 | OpenMetrics(OpenMetricsMessageReport), 48 | } 49 | impl Report for MessageReport { 50 | fn measurement_complete( 51 | &self, 52 | id: &crate::report::BenchmarkId, 53 | context: &crate::report::ReportContext, 54 | measurements: &crate::report::MeasurementData<'_>, 55 | formatter: &crate::value_formatter::ValueFormatter, 56 | ) { 57 | match self { 58 | Self::Json(report) => report.measurement_complete(id, context, measurements, formatter), 59 | Self::OpenMetrics(report) => { 60 | report.measurement_complete(id, context, measurements, formatter) 61 | } 62 | } 63 | } 64 | 65 | fn summarize( 66 | &self, 67 | context: &crate::report::ReportContext, 68 | group_id: &str, 69 | benchmark_group: &crate::model::BenchmarkGroup, 70 | formatter: &crate::value_formatter::ValueFormatter, 71 | ) { 72 | match self { 73 | Self::Json(report) => report.summarize(context, group_id, benchmark_group, formatter), 74 | Self::OpenMetrics(report) => { 75 | report.summarize(context, group_id, benchmark_group, formatter) 76 | } 77 | } 78 | } 79 | } 80 | 81 | pub fn create_machine_report(self_config: &SelfConfig) -> Option { 82 | match self_config.message_format { 83 | Some(MessageFormat::Json) => Some(MessageReport::Json(JsonMessageReport)), 84 | Some(MessageFormat::OpenMetrics) => { 85 | Some(MessageReport::OpenMetrics(OpenMetricsMessageReport)) 86 | } 87 | None => None, 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/message_formats/openmetrics.rs: -------------------------------------------------------------------------------- 1 | use crate::report::{BenchmarkId, MeasurementData, Report, ReportContext}; 2 | use crate::value_formatter::ValueFormatter; 3 | 4 | use super::ConfidenceInterval; 5 | 6 | pub struct OpenMetricsMessageReport; 7 | 8 | impl OpenMetricsMessageReport { 9 | fn print_confidence_interval(id: &BenchmarkId, metric: &ConfidenceInterval, name: &str) { 10 | let mut labels = vec![]; 11 | 12 | if let Some(func) = &id.function_id { 13 | labels.push(("function", func.clone())); 14 | } 15 | 16 | if let Some(value) = &id.value_str { 17 | labels.push(("input_size", value.clone())); 18 | } 19 | 20 | labels.push(("aggregation", name.to_owned())); 21 | 22 | let labels = labels 23 | .into_iter() 24 | .map(|(key, value)| format!("{}=\"{}\"", key, value)) 25 | .collect::>() 26 | .join(","); 27 | 28 | println!( 29 | "criterion_benchmark_result_{}{{id=\"{}\",confidence=\"estimate\",{}}} {}", 30 | metric.unit, id.group_id, labels, metric.estimate 31 | ); 32 | println!( 33 | "criterion_benchmark_result_{}{{id=\"{}\",confidence=\"upper_bound\",{}}} {}", 34 | metric.unit, id.group_id, labels, metric.upper_bound 35 | ); 36 | println!( 37 | "criterion_benchmark_result_{}{{id=\"{}\",confidence=\"lower_bound\",{}}} {}", 38 | metric.unit, id.group_id, labels, metric.lower_bound 39 | ); 40 | } 41 | } 42 | 43 | impl Report for OpenMetricsMessageReport { 44 | fn measurement_complete( 45 | &self, 46 | id: &BenchmarkId, 47 | context: &ReportContext, 48 | measurements: &MeasurementData<'_>, 49 | formatter: &ValueFormatter, 50 | ) { 51 | Self::print_confidence_interval( 52 | id, 53 | &ConfidenceInterval::from_estimate( 54 | measurements.absolute_estimates.typical(), 55 | formatter, 56 | ), 57 | "typical", 58 | ); 59 | Self::print_confidence_interval( 60 | id, 61 | &ConfidenceInterval::from_estimate(&measurements.absolute_estimates.mean, formatter), 62 | "mean", 63 | ); 64 | Self::print_confidence_interval( 65 | id, 66 | &ConfidenceInterval::from_estimate(&measurements.absolute_estimates.median, formatter), 67 | "median", 68 | ); 69 | Self::print_confidence_interval( 70 | id, 71 | &ConfidenceInterval::from_estimate( 72 | &measurements.absolute_estimates.median_abs_dev, 73 | formatter, 74 | ), 75 | "median_abs_dev", 76 | ); 77 | 78 | if let Some(slope) = measurements 79 | .absolute_estimates 80 | .slope 81 | .as_ref() 82 | .map(|slope| ConfidenceInterval::from_estimate(slope, formatter)) 83 | { 84 | Self::print_confidence_interval(id, &slope, "slope"); 85 | } 86 | 87 | let input_size = if let Some(input_size) = &id.value_str { 88 | format!("input_size=\"{}\",", input_size) 89 | } else { 90 | "".into() 91 | }; 92 | 93 | let function = if let Some(function) = &id.function_id { 94 | format!("function=\"{}\",", function) 95 | } else { 96 | "".into() 97 | }; 98 | 99 | println!( 100 | "criterion_benchmark_info{{id=\"{}\",{}{}report_directory=\"{}\"}} 1", 101 | id.group_id, 102 | input_size, 103 | function, 104 | path!(&context.output_directory, id.as_directory_name()).display() 105 | ); 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/plot/gnuplot_backend/distributions.rs: -------------------------------------------------------------------------------- 1 | use crate::estimate::Statistic; 2 | use crate::plot::gnuplot_backend::{gnuplot_escape, Colors, DEFAULT_FONT, LINEWIDTH, SIZE}; 3 | use crate::plot::Size; 4 | use crate::plot::{FilledCurve as FilledArea, Line, LineCurve, Rectangle}; 5 | use crate::report::BenchmarkId; 6 | use crate::stats::univariate::Sample; 7 | use criterion_plot::prelude::*; 8 | 9 | pub fn abs_distribution( 10 | colors: &Colors, 11 | id: &BenchmarkId, 12 | statistic: Statistic, 13 | size: Option, 14 | 15 | x_unit: &str, 16 | distribution_curve: LineCurve, 17 | bootstrap_area: FilledArea, 18 | point_estimate: Line, 19 | ) -> Figure { 20 | let xs_sample = Sample::new(distribution_curve.xs); 21 | 22 | let mut figure = Figure::new(); 23 | figure 24 | .set(Font(DEFAULT_FONT)) 25 | .set(criterion_plot::Size::from(size.unwrap_or(SIZE))) 26 | .set(Title(format!( 27 | "{}: {}", 28 | gnuplot_escape(id.as_title()), 29 | statistic 30 | ))) 31 | .configure(Axis::BottomX, |a| { 32 | a.set(Label(format!("Average time ({})", x_unit))) 33 | .set(Range::Limits(xs_sample.min(), xs_sample.max())) 34 | }) 35 | .configure(Axis::LeftY, |a| a.set(Label("Density (a.u.)"))) 36 | .configure(Key, |k| { 37 | k.set(Justification::Left) 38 | .set(Order::SampleText) 39 | .set(Position::Outside(Vertical::Top, Horizontal::Right)) 40 | }) 41 | .plot( 42 | Lines { 43 | x: distribution_curve.xs, 44 | y: distribution_curve.ys, 45 | }, 46 | |c| { 47 | c.set(colors.current_sample) 48 | .set(LINEWIDTH) 49 | .set(Label("Bootstrap distribution")) 50 | .set(LineType::Solid) 51 | }, 52 | ) 53 | .plot( 54 | FilledCurve { 55 | x: bootstrap_area.xs, 56 | y1: bootstrap_area.ys_1, 57 | y2: bootstrap_area.ys_2, 58 | }, 59 | |c| { 60 | c.set(colors.current_sample) 61 | .set(Label("Confidence interval")) 62 | .set(Opacity(0.25)) 63 | }, 64 | ) 65 | .plot( 66 | Lines { 67 | x: &[point_estimate.start.x, point_estimate.end.x], 68 | y: &[point_estimate.start.y, point_estimate.end.y], 69 | }, 70 | |c| { 71 | c.set(colors.current_sample) 72 | .set(LINEWIDTH) 73 | .set(Label("Point estimate")) 74 | .set(LineType::Dash) 75 | }, 76 | ); 77 | figure 78 | } 79 | 80 | pub fn rel_distribution( 81 | colors: &Colors, 82 | id: &BenchmarkId, 83 | statistic: Statistic, 84 | size: Option, 85 | 86 | distribution_curve: LineCurve, 87 | confidence_interval: FilledArea, 88 | point_estimate: Line, 89 | noise_threshold: Rectangle, 90 | ) -> Figure { 91 | let xs_ = Sample::new(distribution_curve.xs); 92 | let x_min = xs_.min(); 93 | let x_max = xs_.max(); 94 | 95 | let mut figure = Figure::new(); 96 | 97 | figure 98 | .set(Font(DEFAULT_FONT)) 99 | .set(criterion_plot::Size::from(size.unwrap_or(SIZE))) 100 | .configure(Axis::LeftY, |a| a.set(Label("Density (a.u.)"))) 101 | .configure(Key, |k| { 102 | k.set(Justification::Left) 103 | .set(Order::SampleText) 104 | .set(Position::Outside(Vertical::Top, Horizontal::Right)) 105 | }) 106 | .set(Title(format!( 107 | "{}: {}", 108 | gnuplot_escape(id.as_title()), 109 | statistic 110 | ))) 111 | .configure(Axis::BottomX, |a| { 112 | a.set(Label("Relative change (%)")) 113 | .set(Range::Limits(x_min * 100., x_max * 100.)) 114 | .set(ScaleFactor(100.)) 115 | }) 116 | .plot( 117 | Lines { 118 | x: distribution_curve.xs, 119 | y: distribution_curve.ys, 120 | }, 121 | |c| { 122 | c.set(colors.current_sample) 123 | .set(LINEWIDTH) 124 | .set(Label("Bootstrap distribution")) 125 | .set(LineType::Solid) 126 | }, 127 | ) 128 | .plot( 129 | FilledCurve { 130 | x: confidence_interval.xs, 131 | y1: confidence_interval.ys_1, 132 | y2: confidence_interval.ys_2, 133 | }, 134 | |c| { 135 | c.set(colors.current_sample) 136 | .set(Label("Confidence interval")) 137 | .set(Opacity(0.25)) 138 | }, 139 | ) 140 | .plot(to_lines!(point_estimate), |c| { 141 | c.set(colors.current_sample) 142 | .set(LINEWIDTH) 143 | .set(Label("Point estimate")) 144 | .set(LineType::Dash) 145 | }) 146 | .plot( 147 | FilledCurve { 148 | x: &[noise_threshold.left, noise_threshold.right], 149 | y1: &[noise_threshold.bottom, noise_threshold.bottom], 150 | y2: &[noise_threshold.top, noise_threshold.top], 151 | }, 152 | |c| { 153 | c.set(Axes::BottomXRightY) 154 | .set(colors.severe_outlier) 155 | .set(Label("Noise threshold")) 156 | .set(Opacity(0.1)) 157 | }, 158 | ); 159 | figure 160 | } 161 | -------------------------------------------------------------------------------- /src/plot/gnuplot_backend/history.rs: -------------------------------------------------------------------------------- 1 | use crate::plot::gnuplot_backend::{gnuplot_escape, Colors, DEFAULT_FONT, LINEWIDTH}; 2 | use crate::plot::Size; 3 | use crate::plot::{FilledCurve as FilledArea, LineCurve}; 4 | use criterion_plot::prelude::*; 5 | 6 | pub fn history_plot( 7 | colors: &Colors, 8 | title: &str, 9 | size: Size, 10 | point_estimate: LineCurve, 11 | confidence_interval: FilledArea, 12 | ids: &[String], 13 | unit: &str, 14 | ) -> Figure { 15 | let mut figure = Figure::new(); 16 | figure 17 | .set(Font(DEFAULT_FONT)) 18 | .set(criterion_plot::Size::from(size)) 19 | .configure(Key, |k| { 20 | k.set(Justification::Left) 21 | .set(Order::SampleText) 22 | .set(Position::Outside(Vertical::Top, Horizontal::Right)) 23 | }) 24 | .set(Title(format!("{}: History", gnuplot_escape(title)))) 25 | .configure(Axis::BottomX, |a| { 26 | a.set(Label("Benchmark")).set(TicLabels { 27 | labels: ids, 28 | positions: point_estimate.xs, 29 | }) 30 | }) 31 | .configure(Axis::LeftY, |a| { 32 | a.set(Label(format!("Average time ({})", unit))) 33 | }); 34 | 35 | figure.plot( 36 | Lines { 37 | x: point_estimate.xs, 38 | y: point_estimate.ys, 39 | }, 40 | |c| { 41 | c.set(colors.current_sample) 42 | .set(LINEWIDTH) 43 | .set(Label("Point estimate")) 44 | }, 45 | ); 46 | figure.plot( 47 | FilledCurve { 48 | x: confidence_interval.xs, 49 | y1: confidence_interval.ys_1, 50 | y2: confidence_interval.ys_2, 51 | }, 52 | |c| { 53 | c.set(colors.current_sample) 54 | .set(Opacity(0.5)) 55 | .set(Label("Confidence Interval")) 56 | }, 57 | ); 58 | figure 59 | } 60 | -------------------------------------------------------------------------------- /src/plot/gnuplot_backend/iteration_times.rs: -------------------------------------------------------------------------------- 1 | use crate::plot::gnuplot_backend::{gnuplot_escape, Colors, DEFAULT_FONT, SIZE}; 2 | use crate::plot::Points as PointPlot; 3 | use crate::plot::Size; 4 | use crate::report::BenchmarkId; 5 | use criterion_plot::prelude::*; 6 | 7 | pub fn iteration_times( 8 | colors: &Colors, 9 | id: &BenchmarkId, 10 | size: Option, 11 | 12 | unit: &str, 13 | is_thumbnail: bool, 14 | current_times: PointPlot, 15 | base_times: Option, 16 | ) -> Figure { 17 | let mut figure = Figure::new(); 18 | figure 19 | .set(Font(DEFAULT_FONT)) 20 | .set(criterion_plot::Size::from(size.unwrap_or(SIZE))) 21 | .configure(Axis::BottomX, |a| { 22 | a.configure(Grid::Major, |g| g.show()).set(Label("Sample")) 23 | }) 24 | .configure(Axis::LeftY, |a| { 25 | a.configure(Grid::Major, |g| g.show()) 26 | .set(Label(format!("Average Iteration Time ({})", unit))) 27 | }) 28 | .plot( 29 | Points { 30 | x: current_times.xs, 31 | y: current_times.ys, 32 | }, 33 | |c| { 34 | c.set(colors.current_sample) 35 | .set(Label("Current")) 36 | .set(PointSize(0.5)) 37 | .set(PointType::FilledCircle) 38 | }, 39 | ); 40 | 41 | if let Some(base_times) = base_times { 42 | figure.plot( 43 | Points { 44 | x: base_times.xs, 45 | y: base_times.ys, 46 | }, 47 | |c| { 48 | c.set(colors.previous_sample) 49 | .set(Label("Base")) 50 | .set(PointSize(0.5)) 51 | .set(PointType::FilledCircle) 52 | }, 53 | ); 54 | } 55 | 56 | if !is_thumbnail { 57 | figure.set(Title(gnuplot_escape(id.as_title()))); 58 | figure.configure(Key, |k| { 59 | k.set(Justification::Left) 60 | .set(Order::SampleText) 61 | .set(Position::Inside(Vertical::Top, Horizontal::Left)) 62 | }); 63 | } else { 64 | figure.configure(Key, |k| k.hide()); 65 | } 66 | 67 | figure 68 | } 69 | -------------------------------------------------------------------------------- /src/plot/gnuplot_backend/pdf.rs: -------------------------------------------------------------------------------- 1 | use crate::plot::gnuplot_backend::{ 2 | gnuplot_escape, Colors, DEFAULT_FONT, LINEWIDTH, POINT_SIZE, SIZE, 3 | }; 4 | use crate::plot::Size; 5 | use crate::plot::{FilledCurve as FilledArea, Line, Points as PointPlot, VerticalLine}; 6 | use crate::report::BenchmarkId; 7 | use crate::stats::univariate::Sample; 8 | use criterion_plot::prelude::*; 9 | 10 | pub fn pdf_full( 11 | colors: &Colors, 12 | id: &BenchmarkId, 13 | size: Option, 14 | unit: &str, 15 | y_label: &str, 16 | y_scale: f64, 17 | max_iters: f64, 18 | pdf: FilledArea, 19 | mean: VerticalLine, 20 | fences: (VerticalLine, VerticalLine, VerticalLine, VerticalLine), 21 | points: (PointPlot, PointPlot, PointPlot), 22 | ) -> Figure { 23 | let (low_severe, low_mild, high_mild, high_severe) = fences; 24 | let (not_outlier, mild, severe) = points; 25 | 26 | let mut figure = Figure::new(); 27 | figure 28 | .set(Font(DEFAULT_FONT)) 29 | .set(criterion_plot::Size::from(size.unwrap_or(SIZE))) 30 | .configure(Axis::BottomX, |a| { 31 | let xs_ = Sample::new(pdf.xs); 32 | a.set(Label(format!("Average time ({})", unit))) 33 | .set(Range::Limits(xs_.min(), xs_.max())) 34 | }) 35 | .configure(Axis::LeftY, |a| { 36 | a.set(Label(y_label.to_owned())) 37 | .set(Range::Limits(0., max_iters * y_scale)) 38 | .set(ScaleFactor(y_scale)) 39 | }) 40 | .configure(Axis::RightY, |a| a.set(Label("Density (a.u.)"))) 41 | .configure(Key, |k| { 42 | k.set(Justification::Left) 43 | .set(Order::SampleText) 44 | .set(Position::Outside(Vertical::Top, Horizontal::Right)) 45 | }) 46 | .plot( 47 | FilledCurve { 48 | x: pdf.xs, 49 | y1: pdf.ys_1, 50 | y2: pdf.ys_2, 51 | }, 52 | |c| { 53 | c.set(Axes::BottomXRightY) 54 | .set(colors.current_sample) 55 | .set(Label("PDF")) 56 | .set(Opacity(0.25)) 57 | }, 58 | ) 59 | .plot(to_lines!(mean, max_iters), |c| { 60 | c.set(colors.not_an_outlier) 61 | .set(LINEWIDTH) 62 | .set(LineType::Dash) 63 | .set(Label("Mean")) 64 | }) 65 | .plot( 66 | Points { 67 | x: not_outlier.xs, 68 | y: not_outlier.ys, 69 | }, 70 | |c| { 71 | c.set(colors.not_an_outlier) 72 | .set(Label("\"Clean\" sample")) 73 | .set(PointType::FilledCircle) 74 | .set(POINT_SIZE) 75 | }, 76 | ) 77 | .plot( 78 | Points { 79 | x: mild.xs, 80 | y: mild.ys, 81 | }, 82 | |c| { 83 | c.set(colors.mild_outlier) 84 | .set(Label("Mild outliers")) 85 | .set(POINT_SIZE) 86 | .set(PointType::FilledCircle) 87 | }, 88 | ) 89 | .plot( 90 | Points { 91 | x: severe.xs, 92 | y: severe.ys, 93 | }, 94 | |c| { 95 | c.set(colors.severe_outlier) 96 | .set(Label("Severe outliers")) 97 | .set(POINT_SIZE) 98 | .set(PointType::FilledCircle) 99 | }, 100 | ) 101 | .plot(to_lines!(low_mild, max_iters), |c| { 102 | c.set(colors.mild_outlier) 103 | .set(LINEWIDTH) 104 | .set(LineType::Dash) 105 | }) 106 | .plot(to_lines!(high_mild, max_iters), |c| { 107 | c.set(colors.mild_outlier) 108 | .set(LINEWIDTH) 109 | .set(LineType::Dash) 110 | }) 111 | .plot(to_lines!(low_severe, max_iters), |c| { 112 | c.set(colors.severe_outlier) 113 | .set(LINEWIDTH) 114 | .set(LineType::Dash) 115 | }) 116 | .plot(to_lines!(high_severe, max_iters), |c| { 117 | c.set(colors.severe_outlier) 118 | .set(LINEWIDTH) 119 | .set(LineType::Dash) 120 | }); 121 | figure.set(Title(gnuplot_escape(id.as_title()))); 122 | figure 123 | } 124 | 125 | pub fn pdf_thumbnail( 126 | colors: &Colors, 127 | size: Option, 128 | unit: &str, 129 | mean: Line, 130 | pdf: FilledArea, 131 | ) -> Figure { 132 | let xs_ = Sample::new(pdf.xs); 133 | let ys_ = Sample::new(pdf.ys_1); 134 | let y_limit = ys_.max() * 1.1; 135 | 136 | let mut figure = Figure::new(); 137 | figure 138 | .set(Font(DEFAULT_FONT)) 139 | .set(criterion_plot::Size::from(size.unwrap_or(SIZE))) 140 | .configure(Axis::BottomX, |a| { 141 | a.set(Label(format!("Average time ({})", unit))) 142 | .set(Range::Limits(xs_.min(), xs_.max())) 143 | }) 144 | .configure(Axis::LeftY, |a| { 145 | a.set(Label("Density (a.u.)")) 146 | .set(Range::Limits(0., y_limit)) 147 | }) 148 | .configure(Axis::RightY, |a| a.hide()) 149 | .configure(Key, |k| k.hide()) 150 | .plot( 151 | FilledCurve { 152 | x: pdf.xs, 153 | y1: pdf.ys_1, 154 | y2: pdf.ys_2, 155 | }, 156 | |c| { 157 | c.set(Axes::BottomXRightY) 158 | .set(colors.current_sample) 159 | .set(Label("PDF")) 160 | .set(Opacity(0.25)) 161 | }, 162 | ) 163 | .plot(to_lines!(mean), |c| { 164 | c.set(colors.current_sample) 165 | .set(LINEWIDTH) 166 | .set(Label("Mean")) 167 | }); 168 | 169 | figure 170 | } 171 | 172 | pub fn pdf_comparison( 173 | colors: &Colors, 174 | id: &BenchmarkId, 175 | size: Option, 176 | is_thumbnail: bool, 177 | unit: &str, 178 | current_mean: Line, 179 | current_pdf: FilledArea, 180 | base_mean: Line, 181 | base_pdf: FilledArea, 182 | ) -> Figure { 183 | let mut figure = Figure::new(); 184 | figure 185 | .set(Font(DEFAULT_FONT)) 186 | .set(criterion_plot::Size::from(size.unwrap_or(SIZE))) 187 | .configure(Axis::BottomX, |a| { 188 | a.set(Label(format!("Average time ({})", unit))) 189 | }) 190 | .configure(Axis::LeftY, |a| a.set(Label("Density (a.u.)"))) 191 | .configure(Axis::RightY, |a| a.hide()) 192 | .configure(Key, |k| { 193 | k.set(Justification::Left) 194 | .set(Order::SampleText) 195 | .set(Position::Outside(Vertical::Top, Horizontal::Right)) 196 | }) 197 | .plot( 198 | FilledCurve { 199 | x: base_pdf.xs, 200 | y1: base_pdf.ys_1, 201 | y2: base_pdf.ys_2, 202 | }, 203 | |c| { 204 | c.set(colors.previous_sample) 205 | .set(Label("Base PDF")) 206 | .set(Opacity(0.5)) 207 | }, 208 | ) 209 | .plot(to_lines!(base_mean), |c| { 210 | c.set(colors.previous_sample) 211 | .set(Label("Base Mean")) 212 | .set(LINEWIDTH) 213 | }) 214 | .plot( 215 | FilledCurve { 216 | x: current_pdf.xs, 217 | y1: current_pdf.ys_1, 218 | y2: current_pdf.ys_2, 219 | }, 220 | |c| { 221 | c.set(colors.current_sample) 222 | .set(Label("New PDF")) 223 | .set(Opacity(0.5)) 224 | }, 225 | ) 226 | .plot(to_lines!(current_mean), |c| { 227 | c.set(colors.current_sample) 228 | .set(Label("New Mean")) 229 | .set(LINEWIDTH) 230 | }); 231 | 232 | if is_thumbnail { 233 | figure.configure(Key, |k| k.hide()); 234 | } else { 235 | figure.set(Title(gnuplot_escape(id.as_title()))); 236 | } 237 | figure 238 | } 239 | -------------------------------------------------------------------------------- /src/plot/gnuplot_backend/regression.rs: -------------------------------------------------------------------------------- 1 | use crate::plot::gnuplot_backend::{gnuplot_escape, Colors, DEFAULT_FONT, LINEWIDTH, SIZE}; 2 | use crate::plot::Points as PointPlot; 3 | use crate::plot::Size; 4 | use crate::plot::{FilledCurve as FilledArea, Line}; 5 | use crate::report::BenchmarkId; 6 | use criterion_plot::prelude::*; 7 | 8 | pub fn regression( 9 | colors: &Colors, 10 | id: &BenchmarkId, 11 | size: Option, 12 | is_thumbnail: bool, 13 | x_label: &str, 14 | x_scale: f64, 15 | unit: &str, 16 | sample: PointPlot, 17 | regression: Line, 18 | confidence_interval: FilledArea, 19 | ) -> Figure { 20 | let mut figure = Figure::new(); 21 | figure 22 | .set(Font(DEFAULT_FONT)) 23 | .set(criterion_plot::Size::from(size.unwrap_or(SIZE))) 24 | .configure(Axis::BottomX, |a| { 25 | a.configure(Grid::Major, |g| g.show()) 26 | .set(Label(x_label.to_owned())) 27 | .set(ScaleFactor(x_scale)) 28 | }) 29 | .configure(Axis::LeftY, |a| { 30 | a.configure(Grid::Major, |g| g.show()) 31 | .set(Label(format!("Total sample time ({})", unit))) 32 | }) 33 | .plot( 34 | Points { 35 | x: sample.xs, 36 | y: sample.ys, 37 | }, 38 | |c| { 39 | c.set(colors.current_sample) 40 | .set(Label("Sample")) 41 | .set(PointSize(0.5)) 42 | .set(PointType::FilledCircle) 43 | }, 44 | ) 45 | .plot(to_lines!(regression), |c| { 46 | c.set(colors.current_sample) 47 | .set(LINEWIDTH) 48 | .set(Label("Linear regression")) 49 | .set(LineType::Solid) 50 | }) 51 | .plot( 52 | FilledCurve { 53 | x: confidence_interval.xs, 54 | y1: confidence_interval.ys_1, 55 | y2: confidence_interval.ys_2, 56 | }, 57 | |c| { 58 | c.set(colors.current_sample) 59 | .set(Label("Confidence interval")) 60 | .set(Opacity(0.25)) 61 | }, 62 | ); 63 | 64 | if !is_thumbnail { 65 | figure.set(Title(gnuplot_escape(id.as_title()))); 66 | figure.configure(Key, |k| { 67 | k.set(Justification::Left) 68 | .set(Order::SampleText) 69 | .set(Position::Inside(Vertical::Top, Horizontal::Left)) 70 | }); 71 | } else { 72 | figure.configure(Key, |k| k.hide()); 73 | } 74 | 75 | figure 76 | } 77 | 78 | pub fn regression_comparison( 79 | colors: &Colors, 80 | id: &BenchmarkId, 81 | size: Option, 82 | is_thumbnail: bool, 83 | x_label: &str, 84 | x_scale: f64, 85 | unit: &str, 86 | current_regression: Line, 87 | current_confidence_interval: FilledArea, 88 | base_regression: Line, 89 | base_confidence_interval: FilledArea, 90 | ) -> Figure { 91 | let mut figure = Figure::new(); 92 | figure 93 | .set(Font(DEFAULT_FONT)) 94 | .set(criterion_plot::Size::from(size.unwrap_or(SIZE))) 95 | .configure(Axis::BottomX, |a| { 96 | a.configure(Grid::Major, |g| g.show()) 97 | .set(Label(x_label.to_owned())) 98 | .set(ScaleFactor(x_scale)) 99 | }) 100 | .configure(Axis::LeftY, |a| { 101 | a.configure(Grid::Major, |g| g.show()) 102 | .set(Label(format!("Total sample time ({})", unit))) 103 | }) 104 | .configure(Key, |k| { 105 | k.set(Justification::Left) 106 | .set(Order::SampleText) 107 | .set(Position::Inside(Vertical::Top, Horizontal::Left)) 108 | }) 109 | .plot( 110 | FilledCurve { 111 | x: base_confidence_interval.xs, 112 | y1: base_confidence_interval.ys_1, 113 | y2: base_confidence_interval.ys_2, 114 | }, 115 | |c| c.set(colors.previous_sample).set(Opacity(0.25)), 116 | ) 117 | .plot( 118 | FilledCurve { 119 | x: current_confidence_interval.xs, 120 | y1: current_confidence_interval.ys_1, 121 | y2: current_confidence_interval.ys_2, 122 | }, 123 | |c| c.set(colors.current_sample).set(Opacity(0.25)), 124 | ) 125 | .plot(to_lines!(base_regression), |c| { 126 | c.set(colors.previous_sample) 127 | .set(LINEWIDTH) 128 | .set(Label("Base sample")) 129 | .set(LineType::Solid) 130 | }) 131 | .plot(to_lines!(current_regression), |c| { 132 | c.set(colors.current_sample) 133 | .set(LINEWIDTH) 134 | .set(Label("New sample")) 135 | .set(LineType::Solid) 136 | }); 137 | 138 | if is_thumbnail { 139 | figure.configure(Key, |k| k.hide()); 140 | } else { 141 | figure.set(Title(gnuplot_escape(id.as_title()))); 142 | } 143 | 144 | figure 145 | } 146 | -------------------------------------------------------------------------------- /src/plot/gnuplot_backend/summary.rs: -------------------------------------------------------------------------------- 1 | use crate::connection::AxisScale; 2 | use crate::plot::gnuplot_backend::{ 3 | gnuplot_escape, Colors, DEFAULT_FONT, LINEWIDTH, POINT_SIZE, SIZE, 4 | }; 5 | use crate::plot::LineCurve; 6 | use crate::plot::Size; 7 | use crate::report::ValueType; 8 | use criterion_plot::prelude::*; 9 | 10 | pub fn line_comparison( 11 | colors: &Colors, 12 | title: &str, 13 | unit: &str, 14 | value_type: ValueType, 15 | axis_scale: AxisScale, 16 | lines: &[(Option<&String>, LineCurve)], 17 | ) -> Figure { 18 | let mut figure = Figure::new(); 19 | 20 | let input_suffix = match value_type { 21 | ValueType::Bytes => " Size (Bytes)", 22 | ValueType::Elements => " Size (Elements)", 23 | ValueType::Value => "", 24 | }; 25 | 26 | figure 27 | .set(Font(DEFAULT_FONT)) 28 | .set(criterion_plot::Size::from(SIZE)) 29 | .configure(Key, |k| { 30 | k.set(Justification::Left) 31 | .set(Order::SampleText) 32 | .set(Position::Outside(Vertical::Top, Horizontal::Right)) 33 | }) 34 | .set(Title(format!("{}: Comparison", gnuplot_escape(title)))) 35 | .configure(Axis::BottomX, |a| { 36 | a.set(Label(format!("Input{}", input_suffix))) 37 | .set(axis_scale.to_gnuplot()) 38 | }); 39 | 40 | figure.configure(Axis::LeftY, |a| { 41 | a.configure(Grid::Major, |g| g.show()) 42 | .configure(Grid::Minor, |g| g.hide()) 43 | .set(Label(format!("Average time ({})", unit))) 44 | .set(axis_scale.to_gnuplot()) 45 | }); 46 | 47 | for (i, (name, curve)) in lines.iter().enumerate() { 48 | let function_name = name.map(|string| gnuplot_escape(string)); 49 | 50 | figure 51 | .plot( 52 | Lines { 53 | x: curve.xs, 54 | y: curve.ys, 55 | }, 56 | |c| { 57 | if let Some(name) = function_name { 58 | c.set(Label(name)); 59 | } 60 | c.set(LINEWIDTH) 61 | .set(LineType::Solid) 62 | .set(colors.comparison_colors[i % colors.comparison_colors.len()]) 63 | }, 64 | ) 65 | .plot( 66 | Points { 67 | x: curve.xs, 68 | y: curve.ys, 69 | }, 70 | |p| { 71 | p.set(PointType::FilledCircle) 72 | .set(POINT_SIZE) 73 | .set(colors.comparison_colors[i % colors.comparison_colors.len()]) 74 | }, 75 | ); 76 | } 77 | 78 | figure 79 | } 80 | 81 | pub fn violin( 82 | colors: &Colors, 83 | title: &str, 84 | unit: &str, 85 | axis_scale: AxisScale, 86 | lines: &[(&str, LineCurve)], 87 | ) -> Figure { 88 | let tics = || (0..).map(|x| (f64::from(x)) + 0.5); 89 | let size: criterion_plot::Size = Size(1280, 200 + (25 * lines.len())).into(); 90 | let mut figure = Figure::new(); 91 | figure 92 | .set(Font(DEFAULT_FONT)) 93 | .set(size) 94 | .set(Title(format!("{}: Violin plot", gnuplot_escape(title)))) 95 | .configure(Axis::BottomX, |a| { 96 | a.configure(Grid::Major, |g| g.show()) 97 | .configure(Grid::Minor, |g| g.hide()) 98 | .set(Label(format!("Average time ({})", unit))) 99 | .set(axis_scale.to_gnuplot()) 100 | }) 101 | .configure(Axis::LeftY, |a| { 102 | a.set(Label("Input")) 103 | .set(Range::Limits(0., lines.len() as f64)) 104 | .set(TicLabels { 105 | positions: tics(), 106 | labels: lines.iter().map(|(id, _)| gnuplot_escape(id)), 107 | }) 108 | }); 109 | 110 | let mut is_first = true; 111 | for (i, (_, line)) in lines.iter().enumerate() { 112 | let i = i as f64 + 0.5; 113 | let y1: Vec<_> = line.ys.iter().map(|&y| i + y * 0.45).collect(); 114 | let y2: Vec<_> = line.ys.iter().map(|&y| i - y * 0.45).collect(); 115 | 116 | figure.plot(FilledCurve { x: line.xs, y1, y2 }, |c| { 117 | if is_first { 118 | is_first = false; 119 | 120 | c.set(colors.current_sample).set(Label("PDF")) 121 | } else { 122 | c.set(colors.current_sample) 123 | } 124 | }); 125 | } 126 | figure 127 | } 128 | -------------------------------------------------------------------------------- /src/plot/gnuplot_backend/t_test.rs: -------------------------------------------------------------------------------- 1 | use crate::plot::gnuplot_backend::{gnuplot_escape, Colors, DEFAULT_FONT, LINEWIDTH, SIZE}; 2 | use crate::plot::Size; 3 | use crate::plot::{FilledCurve as FilledArea, VerticalLine}; 4 | use crate::report::BenchmarkId; 5 | use criterion_plot::prelude::*; 6 | 7 | pub fn t_test( 8 | colors: &Colors, 9 | id: &BenchmarkId, 10 | size: Option, 11 | t: VerticalLine, 12 | t_distribution: FilledArea, 13 | ) -> Figure { 14 | let mut figure = Figure::new(); 15 | figure 16 | .set(Font(DEFAULT_FONT)) 17 | .set(criterion_plot::Size::from(size.unwrap_or(SIZE))) 18 | .set(Title(format!( 19 | "{}: Welch t test", 20 | gnuplot_escape(id.as_title()) 21 | ))) 22 | .configure(Axis::BottomX, |a| a.set(Label("t score"))) 23 | .configure(Axis::LeftY, |a| a.set(Label("Density"))) 24 | .configure(Key, |k| { 25 | k.set(Justification::Left) 26 | .set(Order::SampleText) 27 | .set(Position::Outside(Vertical::Top, Horizontal::Right)) 28 | }) 29 | .plot( 30 | FilledCurve { 31 | x: t_distribution.xs, 32 | y1: t_distribution.ys_1, 33 | y2: t_distribution.ys_2, 34 | }, 35 | |c| { 36 | c.set(colors.current_sample) 37 | .set(Label("t distribution")) 38 | .set(Opacity(0.25)) 39 | }, 40 | ) 41 | .plot(to_lines!(t, 1.0), |c| { 42 | c.set(Axes::BottomXRightY) 43 | .set(colors.current_sample) 44 | .set(LINEWIDTH) 45 | .set(Label("t statistic")) 46 | .set(LineType::Solid) 47 | }); 48 | 49 | figure 50 | } 51 | -------------------------------------------------------------------------------- /src/plot/plotters_backend/distributions.rs: -------------------------------------------------------------------------------- 1 | use crate::estimate::Statistic; 2 | use crate::plot::plotters_backend::{Colors, DEFAULT_FONT, SIZE}; 3 | use crate::plot::{FilledCurve, Line, LineCurve, Rectangle as RectangleArea, Size}; 4 | use crate::report::BenchmarkId; 5 | use crate::stats::univariate::Sample; 6 | use plotters::data::float::pretty_print_float; 7 | use plotters::prelude::*; 8 | use std::path::PathBuf; 9 | 10 | pub fn abs_distribution( 11 | colors: &Colors, 12 | id: &BenchmarkId, 13 | statistic: Statistic, 14 | size: Option, 15 | path: PathBuf, 16 | 17 | x_unit: &str, 18 | distribution_curve: LineCurve, 19 | bootstrap_area: FilledCurve, 20 | point_estimate: Line, 21 | ) { 22 | let root_area = SVGBackend::new(&path, size.unwrap_or(SIZE).into()).into_drawing_area(); 23 | 24 | let x_range = plotters::data::fitting_range(distribution_curve.xs.iter()); 25 | let mut y_range = plotters::data::fitting_range(distribution_curve.ys.iter()); 26 | 27 | y_range.end *= 1.1; 28 | 29 | let mut chart = ChartBuilder::on(&root_area) 30 | .margin((5).percent()) 31 | .caption( 32 | format!("{}:{}", id.as_title(), statistic), 33 | (DEFAULT_FONT, 20), 34 | ) 35 | .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) 36 | .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) 37 | .build_cartesian_2d(x_range, y_range) 38 | .unwrap(); 39 | 40 | chart 41 | .configure_mesh() 42 | .disable_mesh() 43 | .x_desc(format!("Average time ({})", x_unit)) 44 | .y_desc("Density (a.u.)") 45 | .x_label_formatter(&|&v| pretty_print_float(v, true)) 46 | .y_label_formatter(&|&v| pretty_print_float(v, true)) 47 | .draw() 48 | .unwrap(); 49 | 50 | chart 51 | .draw_series(LineSeries::new( 52 | distribution_curve.to_points(), 53 | &colors.current_sample, 54 | )) 55 | .unwrap() 56 | .label("Bootstrap distribution") 57 | .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], colors.current_sample)); 58 | 59 | chart 60 | .draw_series(AreaSeries::new( 61 | bootstrap_area.to_points(), 62 | 0.0, 63 | colors.current_sample.mix(0.25).filled().stroke_width(3), 64 | )) 65 | .unwrap() 66 | .label("Confidence interval") 67 | .legend(|(x, y)| { 68 | Rectangle::new( 69 | [(x, y - 5), (x + 20, y + 5)], 70 | colors.current_sample.mix(0.25).filled(), 71 | ) 72 | }); 73 | 74 | chart 75 | .draw_series(std::iter::once(PathElement::new( 76 | point_estimate.to_line_vec(), 77 | colors.current_sample.filled().stroke_width(3), 78 | ))) 79 | .unwrap() 80 | .label("Point estimate") 81 | .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], colors.current_sample)); 82 | 83 | chart 84 | .configure_series_labels() 85 | .position(SeriesLabelPosition::UpperRight) 86 | .draw() 87 | .unwrap(); 88 | } 89 | 90 | pub fn rel_distribution( 91 | colors: &Colors, 92 | id: &BenchmarkId, 93 | statistic: Statistic, 94 | size: Option, 95 | path: PathBuf, 96 | 97 | distribution_curve: LineCurve, 98 | confidence_interval: FilledCurve, 99 | point_estimate: Line, 100 | noise_threshold: RectangleArea, 101 | ) { 102 | let xs_ = Sample::new(distribution_curve.xs); 103 | let x_min = xs_.min(); 104 | let x_max = xs_.max(); 105 | 106 | let y_range = plotters::data::fitting_range(distribution_curve.ys); 107 | let root_area = SVGBackend::new(&path, size.unwrap_or(SIZE).into()).into_drawing_area(); 108 | 109 | let mut chart = ChartBuilder::on(&root_area) 110 | .margin((5).percent()) 111 | .caption( 112 | format!("{}:{}", id.as_title(), statistic), 113 | (DEFAULT_FONT, 20), 114 | ) 115 | .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) 116 | .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) 117 | .build_cartesian_2d(x_min..x_max, y_range.clone()) 118 | .unwrap(); 119 | 120 | chart 121 | .configure_mesh() 122 | .disable_mesh() 123 | .x_desc("Relative change (%)") 124 | .y_desc("Density (a.u.)") 125 | .x_label_formatter(&|&v| pretty_print_float(v, true)) 126 | .y_label_formatter(&|&v| pretty_print_float(v, true)) 127 | .draw() 128 | .unwrap(); 129 | 130 | chart 131 | .draw_series(LineSeries::new( 132 | distribution_curve.to_points(), 133 | &colors.current_sample, 134 | )) 135 | .unwrap() 136 | .label("Bootstrap distribution") 137 | .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], colors.current_sample)); 138 | 139 | chart 140 | .draw_series(AreaSeries::new( 141 | confidence_interval.to_points(), 142 | 0.0, 143 | colors.current_sample.mix(0.25).filled().stroke_width(3), 144 | )) 145 | .unwrap() 146 | .label("Confidence interval") 147 | .legend(|(x, y)| { 148 | Rectangle::new( 149 | [(x, y - 5), (x + 20, y + 5)], 150 | colors.current_sample.mix(0.25).filled(), 151 | ) 152 | }); 153 | 154 | chart 155 | .draw_series(std::iter::once(PathElement::new( 156 | point_estimate.to_line_vec(), 157 | colors.current_sample.filled().stroke_width(3), 158 | ))) 159 | .unwrap() 160 | .label("Point estimate") 161 | .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], colors.current_sample)); 162 | 163 | chart 164 | .draw_series(std::iter::once(Rectangle::new( 165 | [ 166 | (noise_threshold.left, y_range.start), 167 | (noise_threshold.right, y_range.end), 168 | ], 169 | colors.previous_sample.mix(0.1).filled(), 170 | ))) 171 | .unwrap() 172 | .label("Noise threshold") 173 | .legend(|(x, y)| { 174 | Rectangle::new( 175 | [(x, y - 5), (x + 20, y + 5)], 176 | colors.previous_sample.mix(0.25).filled(), 177 | ) 178 | }); 179 | chart 180 | .configure_series_labels() 181 | .position(SeriesLabelPosition::UpperRight) 182 | .draw() 183 | .unwrap(); 184 | } 185 | -------------------------------------------------------------------------------- /src/plot/plotters_backend/history.rs: -------------------------------------------------------------------------------- 1 | use crate::plot::plotters_backend::{Colors, DEFAULT_FONT}; 2 | use crate::plot::{FilledCurve, LineCurve, Size}; 3 | use plotters::data::float::pretty_print_float; 4 | use plotters::prelude::*; 5 | use std::path::PathBuf; 6 | 7 | pub fn history( 8 | colors: &Colors, 9 | title: &str, 10 | size: Size, 11 | path: PathBuf, 12 | point_estimate: LineCurve, 13 | confidence_interval: FilledCurve, 14 | ids: &[String], 15 | unit: &str, 16 | ) { 17 | let root_area = SVGBackend::new(&path, size.into()).into_drawing_area(); 18 | 19 | let x_range = plotters::data::fitting_range(point_estimate.xs.iter()); 20 | let mut y_range = plotters::data::fitting_range( 21 | confidence_interval 22 | .ys_1 23 | .iter() 24 | .chain(confidence_interval.ys_2.iter()), 25 | ); 26 | 27 | y_range.end *= 1.1; 28 | y_range.start /= 1.1; 29 | 30 | let mut chart = ChartBuilder::on(&root_area) 31 | .margin((5).percent()) 32 | .caption(format!("{} History", title), (DEFAULT_FONT, 20)) 33 | .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) 34 | .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) 35 | .build_cartesian_2d(x_range, y_range) 36 | .unwrap(); 37 | 38 | chart 39 | .configure_mesh() 40 | .disable_mesh() 41 | .y_desc(format!("Average time ({})", unit)) 42 | .x_desc("History") 43 | .x_label_formatter(&|&v| ids[v as usize].clone()) 44 | .y_label_formatter(&|&v| pretty_print_float(v, true)) 45 | .x_labels(ids.len()) 46 | .draw() 47 | .unwrap(); 48 | 49 | chart 50 | .draw_series(LineSeries::new( 51 | point_estimate.to_points(), 52 | &colors.current_sample, 53 | )) 54 | .unwrap() 55 | .label("Point estimate") 56 | .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], colors.current_sample)); 57 | 58 | let polygon_points: Vec<(f64, f64)> = confidence_interval 59 | .xs 60 | .iter() 61 | .copied() 62 | .zip(confidence_interval.ys_1.iter().copied()) 63 | .chain( 64 | confidence_interval 65 | .xs 66 | .iter() 67 | .rev() 68 | .copied() 69 | .zip(confidence_interval.ys_2.iter().rev().copied()), 70 | ) 71 | .collect(); 72 | 73 | chart 74 | .draw_series(std::iter::once(Polygon::new( 75 | polygon_points, 76 | colors.current_sample.mix(0.25).filled(), 77 | ))) 78 | .unwrap() 79 | .label("Confidence interval") 80 | .legend(|(x, y)| { 81 | Rectangle::new( 82 | [(x, y - 5), (x + 20, y + 5)], 83 | colors.current_sample.mix(0.25).filled(), 84 | ) 85 | }); 86 | 87 | chart 88 | .configure_series_labels() 89 | .position(SeriesLabelPosition::UpperRight) 90 | .draw() 91 | .unwrap(); 92 | } 93 | -------------------------------------------------------------------------------- /src/plot/plotters_backend/iteration_times.rs: -------------------------------------------------------------------------------- 1 | use crate::plot::plotters_backend::{Colors, DEFAULT_FONT, POINT_SIZE, SIZE}; 2 | use crate::plot::{Points, Size}; 3 | use crate::report::BenchmarkId; 4 | use crate::stats::univariate::Sample; 5 | use plotters::data::float::pretty_print_float; 6 | use plotters::prelude::*; 7 | use std::path::PathBuf; 8 | 9 | pub fn iteration_times( 10 | colors: &Colors, 11 | id: &BenchmarkId, 12 | size: Option, 13 | path: PathBuf, 14 | unit: &str, 15 | is_thumbnail: bool, 16 | current_times: Points, 17 | base_times: Option, 18 | ) { 19 | let size = size.unwrap_or(SIZE); 20 | let root_area = SVGBackend::new(&path, size.into()).into_drawing_area(); 21 | 22 | let mut cb = ChartBuilder::on(&root_area); 23 | 24 | let (x_range, y_range) = if let Some(base) = &base_times { 25 | let max_x = Sample::new(current_times.xs) 26 | .max() 27 | .max(Sample::new(base.xs).max()); 28 | let x_range = (1.0)..(max_x); 29 | let y_range = plotters::data::fitting_range(current_times.ys.iter().chain(base.ys.iter())); 30 | (x_range, y_range) 31 | } else { 32 | let max_x = Sample::new(current_times.xs).max(); 33 | let x_range = (1.0)..(max_x); 34 | let y_range = plotters::data::fitting_range(current_times.ys.iter()); 35 | (x_range, y_range) 36 | }; 37 | 38 | let mut chart = cb 39 | .margin((5).percent()) 40 | .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) 41 | .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) 42 | .build_cartesian_2d(x_range, y_range) 43 | .unwrap(); 44 | 45 | chart 46 | .configure_mesh() 47 | .y_desc(format!("Average Iteration Time ({})", unit)) 48 | .x_label_formatter(&|x| pretty_print_float(*x, true)) 49 | .light_line_style(TRANSPARENT) 50 | .draw() 51 | .unwrap(); 52 | 53 | chart 54 | .draw_series( 55 | (current_times.to_points()) 56 | .map(|(x, y)| Circle::new((x, y), POINT_SIZE, colors.current_sample.filled())), 57 | ) 58 | .unwrap() 59 | .label("Current") 60 | .legend(|(x, y)| Circle::new((x + 10, y), POINT_SIZE, colors.current_sample.filled())); 61 | 62 | if let Some(base_times) = base_times { 63 | chart 64 | .draw_series( 65 | (base_times.to_points()) 66 | .map(|(x, y)| Circle::new((x, y), POINT_SIZE, colors.previous_sample.filled())), 67 | ) 68 | .unwrap() 69 | .label("Base") 70 | .legend(|(x, y)| Circle::new((x + 10, y), POINT_SIZE, colors.previous_sample.filled())); 71 | } 72 | 73 | if !is_thumbnail { 74 | cb.caption(id.as_title(), (DEFAULT_FONT, 20)); 75 | chart 76 | .configure_series_labels() 77 | .position(SeriesLabelPosition::UpperLeft) 78 | .draw() 79 | .unwrap(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/plot/plotters_backend/pdf.rs: -------------------------------------------------------------------------------- 1 | use crate::plot::plotters_backend::{Colors, DEFAULT_FONT, POINT_SIZE, SIZE}; 2 | use crate::plot::{FilledCurve, Line, Points, Size, VerticalLine}; 3 | use crate::report::BenchmarkId; 4 | use crate::stats::univariate::Sample; 5 | use plotters::data::float::pretty_print_float; 6 | use plotters::prelude::*; 7 | use plotters::style::RGBAColor; 8 | use std::path::PathBuf; 9 | 10 | pub fn pdf_full( 11 | colors: &Colors, 12 | id: &BenchmarkId, 13 | size: Option, 14 | path: PathBuf, 15 | unit: &str, 16 | y_label: &str, 17 | y_scale: f64, 18 | max_iters: f64, 19 | pdf: FilledCurve, 20 | mean: VerticalLine, 21 | fences: (VerticalLine, VerticalLine, VerticalLine, VerticalLine), 22 | points: (Points, Points, Points), 23 | ) { 24 | let (low_severe, low_mild, high_mild, high_severe) = fences; 25 | let (not_outlier, mild, severe) = points; 26 | let xs_ = Sample::new(pdf.xs); 27 | 28 | let size = size.unwrap_or(SIZE); 29 | let root_area = SVGBackend::new(&path, size.into()).into_drawing_area(); 30 | 31 | let range = plotters::data::fitting_range(pdf.ys_1.iter()); 32 | 33 | let mut chart = ChartBuilder::on(&root_area) 34 | .margin((5).percent()) 35 | .caption(id.as_title(), (DEFAULT_FONT, 20)) 36 | .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) 37 | .set_label_area_size(LabelAreaPosition::Right, (5).percent_width().min(60)) 38 | .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) 39 | .build_cartesian_2d(xs_.min()..xs_.max(), 0.0..max_iters) 40 | .unwrap() 41 | .set_secondary_coord(xs_.min()..xs_.max(), 0.0..range.end); 42 | 43 | chart 44 | .configure_mesh() 45 | .disable_mesh() 46 | .y_desc(y_label) 47 | .x_desc(format!("Average Time ({})", unit)) 48 | .x_label_formatter(&|&x| pretty_print_float(x, true)) 49 | .y_label_formatter(&|&y| pretty_print_float(y * y_scale, true)) 50 | .draw() 51 | .unwrap(); 52 | 53 | chart 54 | .configure_secondary_axes() 55 | .y_desc("Density (a.u.)") 56 | .x_label_formatter(&|&x| pretty_print_float(x, true)) 57 | .y_label_formatter(&|&y| pretty_print_float(y, true)) 58 | .draw() 59 | .unwrap(); 60 | 61 | chart 62 | .draw_secondary_series(AreaSeries::new( 63 | pdf.to_points(), 64 | 0.0, 65 | colors.current_sample.mix(0.5).filled(), 66 | )) 67 | .unwrap() 68 | .label("PDF") 69 | .legend(|(x, y)| { 70 | Rectangle::new( 71 | [(x, y - 5), (x + 20, y + 5)], 72 | colors.current_sample.mix(0.5).filled(), 73 | ) 74 | }); 75 | 76 | chart 77 | .draw_series(std::iter::once(PathElement::new( 78 | mean.to_line_vec(max_iters), 79 | colors.not_an_outlier, 80 | ))) 81 | .unwrap() 82 | .label("Mean") 83 | .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], colors.not_an_outlier)); 84 | 85 | chart 86 | .draw_series(vec![ 87 | PathElement::new(low_mild.to_line_vec(max_iters), colors.mild_outlier), 88 | PathElement::new(high_mild.to_line_vec(max_iters), colors.mild_outlier), 89 | PathElement::new(low_severe.to_line_vec(max_iters), colors.severe_outlier), 90 | PathElement::new(high_severe.to_line_vec(max_iters), colors.severe_outlier), 91 | ]) 92 | .unwrap(); 93 | 94 | let mut draw_data_point_series = |points: Points, color: RGBAColor, name: &str| { 95 | chart 96 | .draw_series( 97 | (points.to_points()).map(|(x, y)| Circle::new((x, y), POINT_SIZE, color.filled())), 98 | ) 99 | .unwrap() 100 | .label(name) 101 | .legend(move |(x, y)| Circle::new((x + 10, y), POINT_SIZE, color.filled())); 102 | }; 103 | 104 | draw_data_point_series( 105 | not_outlier, 106 | colors.not_an_outlier.to_rgba(), 107 | "\"Clean\" sample", 108 | ); 109 | draw_data_point_series(mild, colors.mild_outlier.to_rgba(), "Mild outliers"); 110 | draw_data_point_series(severe, colors.severe_outlier.to_rgba(), "Severe outliers"); 111 | chart.configure_series_labels().draw().unwrap(); 112 | } 113 | 114 | pub fn pdf_thumbnail( 115 | colors: &Colors, 116 | size: Option, 117 | path: PathBuf, 118 | unit: &str, 119 | mean: Line, 120 | pdf: FilledCurve, 121 | ) { 122 | let xs_ = Sample::new(pdf.xs); 123 | let ys_ = Sample::new(pdf.ys_1); 124 | 125 | let y_limit = ys_.max() * 1.1; 126 | 127 | let size = size.unwrap_or(SIZE); 128 | let root_area = SVGBackend::new(&path, size.into()).into_drawing_area(); 129 | 130 | let mut chart = ChartBuilder::on(&root_area) 131 | .margin((5).percent()) 132 | .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) 133 | .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) 134 | .build_cartesian_2d(xs_.min()..xs_.max(), 0.0..y_limit) 135 | .unwrap(); 136 | 137 | chart 138 | .configure_mesh() 139 | .disable_mesh() 140 | .y_desc("Density (a.u.)") 141 | .x_desc(format!("Average Time ({})", unit)) 142 | .x_label_formatter(&|&x| pretty_print_float(x, true)) 143 | .y_label_formatter(&|&y| pretty_print_float(y, true)) 144 | .x_labels(5) 145 | .draw() 146 | .unwrap(); 147 | 148 | chart 149 | .draw_series(AreaSeries::new( 150 | pdf.to_points(), 151 | 0.0, 152 | colors.current_sample.mix(0.25).filled(), 153 | )) 154 | .unwrap(); 155 | 156 | chart 157 | .draw_series(std::iter::once(PathElement::new( 158 | mean.to_line_vec(), 159 | colors.current_sample.filled().stroke_width(2), 160 | ))) 161 | .unwrap(); 162 | } 163 | 164 | pub fn pdf_comparison( 165 | colors: &Colors, 166 | id: &BenchmarkId, 167 | size: Option, 168 | path: PathBuf, 169 | is_thumbnail: bool, 170 | unit: &str, 171 | current_mean: Line, 172 | current_pdf: FilledCurve, 173 | base_mean: Line, 174 | base_pdf: FilledCurve, 175 | ) { 176 | let x_range = plotters::data::fitting_range(base_pdf.xs.iter().chain(current_pdf.xs.iter())); 177 | let y_range = 178 | plotters::data::fitting_range(base_pdf.ys_1.iter().chain(current_pdf.ys_1.iter())); 179 | 180 | let size = size.unwrap_or(SIZE); 181 | let root_area = SVGBackend::new(&path, size.into()).into_drawing_area(); 182 | 183 | let mut cb = ChartBuilder::on(&root_area); 184 | 185 | if !is_thumbnail { 186 | cb.caption(id.as_title(), (DEFAULT_FONT, 20)); 187 | } 188 | 189 | let mut chart = cb 190 | .margin((5).percent()) 191 | .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) 192 | .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) 193 | .build_cartesian_2d(x_range, y_range.clone()) 194 | .unwrap(); 195 | 196 | chart 197 | .configure_mesh() 198 | .disable_mesh() 199 | .y_desc("Density (a.u.)") 200 | .x_desc(format!("Average Time ({})", unit)) 201 | .x_label_formatter(&|&x| pretty_print_float(x, true)) 202 | .y_label_formatter(&|&y| pretty_print_float(y, true)) 203 | .x_labels(5) 204 | .draw() 205 | .unwrap(); 206 | 207 | chart 208 | .draw_series(AreaSeries::new( 209 | base_pdf.to_points(), 210 | y_range.start, 211 | colors.previous_sample.mix(0.5).filled(), 212 | )) 213 | .unwrap() 214 | .label("Base PDF") 215 | .legend(|(x, y)| { 216 | Rectangle::new( 217 | [(x, y - 5), (x + 20, y + 5)], 218 | colors.previous_sample.mix(0.5).filled(), 219 | ) 220 | }); 221 | 222 | chart 223 | .draw_series(AreaSeries::new( 224 | current_pdf.to_points(), 225 | y_range.start, 226 | colors.current_sample.mix(0.5).filled(), 227 | )) 228 | .unwrap() 229 | .label("New PDF") 230 | .legend(|(x, y)| { 231 | Rectangle::new( 232 | [(x, y - 5), (x + 20, y + 5)], 233 | colors.current_sample.mix(0.5).filled(), 234 | ) 235 | }); 236 | 237 | chart 238 | .draw_series(std::iter::once(PathElement::new( 239 | base_mean.to_line_vec(), 240 | colors.previous_sample.filled().stroke_width(2), 241 | ))) 242 | .unwrap() 243 | .label("Base Mean") 244 | .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], colors.previous_sample)); 245 | 246 | chart 247 | .draw_series(std::iter::once(PathElement::new( 248 | current_mean.to_line_vec(), 249 | colors.current_sample.filled().stroke_width(2), 250 | ))) 251 | .unwrap() 252 | .label("New Mean") 253 | .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], colors.current_sample)); 254 | 255 | if !is_thumbnail { 256 | chart.configure_series_labels().draw().unwrap(); 257 | } 258 | } 259 | -------------------------------------------------------------------------------- /src/plot/plotters_backend/regression.rs: -------------------------------------------------------------------------------- 1 | use crate::plot::plotters_backend::{Colors, DEFAULT_FONT, POINT_SIZE, SIZE}; 2 | use crate::plot::{FilledCurve, Line, Points, Size}; 3 | use crate::report::BenchmarkId; 4 | use plotters::data::float::pretty_print_float; 5 | use plotters::prelude::*; 6 | use std::path::PathBuf; 7 | 8 | pub fn regression( 9 | colors: &Colors, 10 | id: &BenchmarkId, 11 | size: Option, 12 | path: PathBuf, 13 | is_thumbnail: bool, 14 | x_label: &str, 15 | x_scale: f64, 16 | unit: &str, 17 | sample: Points, 18 | regression: Line, 19 | confidence_interval: FilledCurve, 20 | ) { 21 | let size = size.unwrap_or(SIZE); 22 | let root_area = SVGBackend::new(&path, size.into()).into_drawing_area(); 23 | 24 | let mut cb = ChartBuilder::on(&root_area); 25 | if !is_thumbnail { 26 | cb.caption(id.as_title(), (DEFAULT_FONT, 20)); 27 | } 28 | 29 | let x_range = plotters::data::fitting_range(sample.xs.iter()); 30 | let y_range = plotters::data::fitting_range(sample.ys.iter()); 31 | 32 | let mut chart = cb 33 | .margin((5).percent()) 34 | .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) 35 | .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) 36 | .build_cartesian_2d(x_range, y_range) 37 | .unwrap(); 38 | 39 | chart 40 | .configure_mesh() 41 | .x_desc(x_label) 42 | .y_desc(format!("Total sample time ({})", unit)) 43 | .x_label_formatter(&|x| pretty_print_float(x * x_scale, true)) 44 | .light_line_style(TRANSPARENT) 45 | .draw() 46 | .unwrap(); 47 | 48 | chart 49 | .draw_series( 50 | (sample.to_points()) 51 | .map(|(x, y)| Circle::new((x, y), POINT_SIZE, colors.current_sample.filled())), 52 | ) 53 | .unwrap() 54 | .label("Sample") 55 | .legend(|(x, y)| Circle::new((x + 10, y), POINT_SIZE, colors.current_sample.filled())); 56 | 57 | chart 58 | .draw_series(std::iter::once(PathElement::new( 59 | regression.to_line_vec(), 60 | colors.current_sample, 61 | ))) 62 | .unwrap() 63 | .label("Linear regression") 64 | .legend(|(x, y)| { 65 | PathElement::new( 66 | vec![(x, y), (x + 20, y)], 67 | colors.current_sample.filled().stroke_width(2), 68 | ) 69 | }); 70 | 71 | chart 72 | .draw_series(std::iter::once(Polygon::new( 73 | vec![ 74 | (confidence_interval.xs[0], confidence_interval.ys_2[0]), 75 | (confidence_interval.xs[1], confidence_interval.ys_1[1]), 76 | (confidence_interval.xs[1], confidence_interval.ys_2[1]), 77 | ], 78 | colors.current_sample.mix(0.25).filled(), 79 | ))) 80 | .unwrap() 81 | .label("Confidence interval") 82 | .legend(|(x, y)| { 83 | Rectangle::new( 84 | [(x, y - 5), (x + 20, y + 5)], 85 | colors.current_sample.mix(0.25).filled(), 86 | ) 87 | }); 88 | 89 | if !is_thumbnail { 90 | chart 91 | .configure_series_labels() 92 | .position(SeriesLabelPosition::UpperLeft) 93 | .draw() 94 | .unwrap(); 95 | } 96 | } 97 | 98 | pub fn regression_comparison( 99 | colors: &Colors, 100 | id: &BenchmarkId, 101 | size: Option, 102 | path: PathBuf, 103 | is_thumbnail: bool, 104 | x_label: &str, 105 | x_scale: f64, 106 | unit: &str, 107 | current_regression: Line, 108 | current_confidence_interval: FilledCurve, 109 | base_regression: Line, 110 | base_confidence_interval: FilledCurve, 111 | ) { 112 | let y_max = current_regression.end.y.max(base_regression.end.y); 113 | let size = size.unwrap_or(SIZE); 114 | let root_area = SVGBackend::new(&path, size.into()).into_drawing_area(); 115 | 116 | let mut cb = ChartBuilder::on(&root_area); 117 | if !is_thumbnail { 118 | cb.caption(id.as_title(), (DEFAULT_FONT, 20)); 119 | } 120 | 121 | let mut chart = cb 122 | .margin((5).percent()) 123 | .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) 124 | .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) 125 | .build_cartesian_2d(0.0..current_regression.end.x, 0.0..y_max) 126 | .unwrap(); 127 | 128 | chart 129 | .configure_mesh() 130 | .x_desc(x_label) 131 | .y_desc(format!("Total sample time ({})", unit)) 132 | .x_label_formatter(&|x| pretty_print_float(x * x_scale, true)) 133 | .light_line_style(TRANSPARENT) 134 | .draw() 135 | .unwrap(); 136 | 137 | chart 138 | .draw_series(vec![ 139 | PathElement::new(base_regression.to_line_vec(), colors.previous_sample).into_dyn(), 140 | Polygon::new( 141 | vec![ 142 | ( 143 | base_confidence_interval.xs[0], 144 | base_confidence_interval.ys_2[0], 145 | ), 146 | ( 147 | base_confidence_interval.xs[1], 148 | base_confidence_interval.ys_1[1], 149 | ), 150 | ( 151 | base_confidence_interval.xs[1], 152 | base_confidence_interval.ys_2[1], 153 | ), 154 | ], 155 | colors.previous_sample.mix(0.25).filled(), 156 | ) 157 | .into_dyn(), 158 | ]) 159 | .unwrap() 160 | .label("Base Sample") 161 | .legend(|(x, y)| { 162 | PathElement::new( 163 | vec![(x, y), (x + 20, y)], 164 | colors.previous_sample.filled().stroke_width(2), 165 | ) 166 | }); 167 | 168 | chart 169 | .draw_series(vec![ 170 | PathElement::new(current_regression.to_line_vec(), colors.current_sample).into_dyn(), 171 | Polygon::new( 172 | vec![ 173 | ( 174 | current_confidence_interval.xs[0], 175 | current_confidence_interval.ys_2[0], 176 | ), 177 | ( 178 | current_confidence_interval.xs[1], 179 | current_confidence_interval.ys_1[1], 180 | ), 181 | ( 182 | current_confidence_interval.xs[1], 183 | current_confidence_interval.ys_2[1], 184 | ), 185 | ], 186 | colors.current_sample.mix(0.25).filled(), 187 | ) 188 | .into_dyn(), 189 | ]) 190 | .unwrap() 191 | .label("New Sample") 192 | .legend(|(x, y)| { 193 | PathElement::new( 194 | vec![(x, y), (x + 20, y)], 195 | colors.current_sample.filled().stroke_width(2), 196 | ) 197 | }); 198 | 199 | if !is_thumbnail { 200 | chart 201 | .configure_series_labels() 202 | .position(SeriesLabelPosition::UpperLeft) 203 | .draw() 204 | .unwrap(); 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /src/plot/plotters_backend/summary.rs: -------------------------------------------------------------------------------- 1 | use crate::connection::AxisScale; 2 | use crate::plot::plotters_backend::{Colors, DEFAULT_FONT, POINT_SIZE, SIZE}; 3 | use crate::plot::LineCurve; 4 | use crate::report::ValueType; 5 | use plotters::coord::{ 6 | ranged1d::{AsRangedCoord, ValueFormatter as PlottersValueFormatter}, 7 | Shift, 8 | }; 9 | use plotters::prelude::*; 10 | use std::path::PathBuf; 11 | 12 | pub fn line_comparison( 13 | colors: &Colors, 14 | path: PathBuf, 15 | title: &str, 16 | unit: &str, 17 | value_type: ValueType, 18 | axis_scale: AxisScale, 19 | lines: &[(Option<&String>, LineCurve)], 20 | ) { 21 | let x_range = 22 | plotters::data::fitting_range(lines.iter().flat_map(|(_, curve)| curve.xs.iter())); 23 | let y_range = 24 | plotters::data::fitting_range(lines.iter().flat_map(|(_, curve)| curve.ys.iter())); 25 | let root_area = SVGBackend::new(&path, SIZE.into()) 26 | .into_drawing_area() 27 | .titled(&format!("{}: Comparison", title), (DEFAULT_FONT, 20)) 28 | .unwrap(); 29 | 30 | match axis_scale { 31 | AxisScale::Linear => draw_line_comparison_figure( 32 | colors, root_area, unit, x_range, y_range, value_type, lines, 33 | ), 34 | AxisScale::Logarithmic => draw_line_comparison_figure( 35 | colors, 36 | root_area, 37 | unit, 38 | x_range.log_scale(), 39 | y_range.log_scale(), 40 | value_type, 41 | lines, 42 | ), 43 | } 44 | } 45 | 46 | fn draw_line_comparison_figure, YR: AsRangedCoord>( 47 | colors: &Colors, 48 | root_area: DrawingArea, 49 | y_unit: &str, 50 | x_range: XR, 51 | y_range: YR, 52 | value_type: ValueType, 53 | data: &[(Option<&String>, LineCurve)], 54 | ) where 55 | XR::CoordDescType: PlottersValueFormatter, 56 | YR::CoordDescType: PlottersValueFormatter, 57 | { 58 | let input_suffix = match value_type { 59 | ValueType::Bytes => " Size (Bytes)", 60 | ValueType::Elements => " Size (Elements)", 61 | ValueType::Value => "", 62 | }; 63 | 64 | let mut chart = ChartBuilder::on(&root_area) 65 | .margin((5).percent()) 66 | .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) 67 | .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) 68 | .build_cartesian_2d(x_range, y_range) 69 | .unwrap(); 70 | 71 | chart 72 | .configure_mesh() 73 | .disable_mesh() 74 | .x_desc(format!("Input{}", input_suffix)) 75 | .y_desc(format!("Average time ({})", y_unit)) 76 | .draw() 77 | .unwrap(); 78 | 79 | for (id, (name, curve)) in data.iter().enumerate() { 80 | let series = chart 81 | .draw_series( 82 | LineSeries::new( 83 | curve.to_points(), 84 | colors.comparison_colors[id % colors.comparison_colors.len()].filled(), 85 | ) 86 | .point_size(POINT_SIZE), 87 | ) 88 | .unwrap(); 89 | if let Some(name) = name { 90 | let name: &str = name; 91 | series.label(name).legend(move |(x, y)| { 92 | Rectangle::new( 93 | [(x, y - 5), (x + 20, y + 5)], 94 | colors.comparison_colors[id % colors.comparison_colors.len()].filled(), 95 | ) 96 | }); 97 | } 98 | } 99 | 100 | chart 101 | .configure_series_labels() 102 | .position(SeriesLabelPosition::UpperLeft) 103 | .draw() 104 | .unwrap(); 105 | } 106 | 107 | pub fn violin( 108 | colors: &Colors, 109 | path: PathBuf, 110 | title: &str, 111 | unit: &str, 112 | axis_scale: AxisScale, 113 | lines: &[(&str, LineCurve)], 114 | ) { 115 | let mut x_range = 116 | plotters::data::fitting_range(lines.iter().flat_map(|(_, curve)| curve.xs.iter())); 117 | x_range.start = 0.0; 118 | let y_range = -0.5..lines.len() as f64 - 0.5; 119 | 120 | let size = (960, 150 + (18 * lines.len() as u32)); 121 | 122 | let root_area = SVGBackend::new(&path, size) 123 | .into_drawing_area() 124 | .titled(&format!("{}: Violin plot", title), (DEFAULT_FONT, 20)) 125 | .unwrap(); 126 | 127 | match axis_scale { 128 | AxisScale::Linear => draw_violin_figure(colors, root_area, unit, x_range, y_range, lines), 129 | AxisScale::Logarithmic => { 130 | draw_violin_figure(colors, root_area, unit, x_range.log_scale(), y_range, lines) 131 | } 132 | } 133 | } 134 | 135 | fn draw_violin_figure, YR: AsRangedCoord>( 136 | colors: &Colors, 137 | root_area: DrawingArea, 138 | unit: &str, 139 | x_range: XR, 140 | y_range: YR, 141 | data: &[(&str, LineCurve)], 142 | ) where 143 | XR::CoordDescType: PlottersValueFormatter, 144 | YR::CoordDescType: PlottersValueFormatter, 145 | { 146 | let mut chart = ChartBuilder::on(&root_area) 147 | .margin((5).percent()) 148 | .set_label_area_size(LabelAreaPosition::Left, (10).percent_width().min(60)) 149 | .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_width().min(40)) 150 | .build_cartesian_2d(x_range, y_range) 151 | .unwrap(); 152 | 153 | chart 154 | .configure_mesh() 155 | .disable_mesh() 156 | .y_desc("Input") 157 | .x_desc(format!("Average time ({})", unit)) 158 | .y_label_style((DEFAULT_FONT, 10)) 159 | .y_label_formatter(&|v: &f64| data[v.round() as usize].0.to_string()) 160 | .y_labels(data.len()) 161 | .draw() 162 | .unwrap(); 163 | 164 | for (i, (_, curve)) in data.iter().enumerate() { 165 | let base = i as f64; 166 | 167 | chart 168 | .draw_series(AreaSeries::new( 169 | curve.to_points().map(|(x, y)| (x, base + y / 2.0)), 170 | base, 171 | colors.current_sample, 172 | )) 173 | .unwrap(); 174 | 175 | chart 176 | .draw_series(AreaSeries::new( 177 | curve.to_points().map(|(x, y)| (x, base - y / 2.0)), 178 | base, 179 | colors.current_sample, 180 | )) 181 | .unwrap(); 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /src/plot/plotters_backend/t_test.rs: -------------------------------------------------------------------------------- 1 | use crate::plot::plotters_backend::{Colors, DEFAULT_FONT, SIZE}; 2 | use crate::plot::{FilledCurve, Size, VerticalLine}; 3 | use crate::report::BenchmarkId; 4 | use plotters::prelude::*; 5 | use std::path::PathBuf; 6 | 7 | pub fn t_test( 8 | colors: &Colors, 9 | id: &BenchmarkId, 10 | size: Option, 11 | path: PathBuf, 12 | t: VerticalLine, 13 | t_distribution: FilledCurve, 14 | ) { 15 | let x_range = plotters::data::fitting_range(t_distribution.xs.iter()); 16 | let mut y_range = plotters::data::fitting_range(t_distribution.ys_1.iter()); 17 | y_range.start = 0.0; 18 | y_range.end *= 1.1; 19 | 20 | let root_area = SVGBackend::new(&path, size.unwrap_or(SIZE).into()).into_drawing_area(); 21 | 22 | let mut chart = ChartBuilder::on(&root_area) 23 | .margin((5).percent()) 24 | .caption( 25 | format!("{}: Welch t test", id.as_title()), 26 | (DEFAULT_FONT, 20), 27 | ) 28 | .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) 29 | .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) 30 | .build_cartesian_2d(x_range, y_range.clone()) 31 | .unwrap(); 32 | 33 | chart 34 | .configure_mesh() 35 | .disable_mesh() 36 | .y_desc("Density") 37 | .x_desc("t score") 38 | .draw() 39 | .unwrap(); 40 | 41 | chart 42 | .draw_series(AreaSeries::new( 43 | t_distribution.to_points(), 44 | 0.0, 45 | colors.current_sample.mix(0.25), 46 | )) 47 | .unwrap() 48 | .label("t distribution") 49 | .legend(|(x, y)| { 50 | Rectangle::new( 51 | [(x, y - 5), (x + 20, y + 5)], 52 | colors.current_sample.mix(0.25).filled(), 53 | ) 54 | }); 55 | 56 | chart 57 | .draw_series(std::iter::once(PathElement::new( 58 | t.to_line_vec(y_range.end), 59 | colors.current_sample.filled().stroke_width(2), 60 | ))) 61 | .unwrap() 62 | .label("t statistic") 63 | .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], colors.current_sample)); 64 | 65 | chart.configure_series_labels().draw().unwrap(); 66 | } 67 | -------------------------------------------------------------------------------- /src/stats/bivariate/bootstrap.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | macro_rules! test { 3 | ($ty:ident) => { 4 | mod $ty { 5 | use quickcheck::TestResult; 6 | use quickcheck::quickcheck; 7 | use approx::relative_eq; 8 | 9 | use crate::stats::bivariate::regression::Slope; 10 | use crate::stats::bivariate::Data; 11 | 12 | quickcheck! { 13 | fn means(size: usize, start: usize, 14 | offset: usize, nresamples: usize) -> TestResult { 15 | if let Some(x) = crate::stats::test::vec::<$ty>(size, start) { 16 | let y = crate::stats::test::vec::<$ty>(size + offset, start + offset).unwrap(); 17 | let data = Data::new(&x[start..], &y[start+offset..]); 18 | 19 | let (x_means, y_means) = if nresamples > 0 { 20 | data.bootstrap(nresamples, |d| (d.x().mean(), d.y().mean())) 21 | } else { 22 | return TestResult::discard(); 23 | }; 24 | 25 | let x_min = data.x().min(); 26 | let x_max = data.x().max(); 27 | let y_min = data.y().min(); 28 | let y_max = data.y().max(); 29 | 30 | TestResult::from_bool( 31 | // Computed the correct number of resamples 32 | x_means.len() == nresamples && 33 | y_means.len() == nresamples && 34 | // No uninitialized values 35 | x_means.iter().all(|&x| { 36 | (x > x_min || relative_eq!(x, x_min)) && 37 | (x < x_max || relative_eq!(x, x_max)) 38 | }) && 39 | y_means.iter().all(|&y| { 40 | (y > y_min || relative_eq!(y, y_min)) && 41 | (y < y_max || relative_eq!(y, y_max)) 42 | }) 43 | ) 44 | } else { 45 | TestResult::discard() 46 | } 47 | } 48 | } 49 | 50 | quickcheck! { 51 | fn slope(size: usize, start: usize, 52 | offset: usize, nresamples: usize) -> TestResult { 53 | if let Some(x) = crate::stats::test::vec::<$ty>(size, start) { 54 | let y = crate::stats::test::vec::<$ty>(size + offset, start + offset).unwrap(); 55 | let data = Data::new(&x[start..], &y[start+offset..]); 56 | 57 | let slopes = if nresamples > 0 { 58 | data.bootstrap(nresamples, |d| (Slope::fit(&d),)).0 59 | } else { 60 | return TestResult::discard(); 61 | }; 62 | 63 | TestResult::from_bool( 64 | // Computed the correct number of resamples 65 | slopes.len() == nresamples && 66 | // No uninitialized values 67 | slopes.iter().all(|s| s.0 > 0.) 68 | ) 69 | } else { 70 | TestResult::discard() 71 | } 72 | } 73 | } 74 | 75 | } 76 | }; 77 | } 78 | 79 | #[cfg(test)] 80 | mod test { 81 | test!(f32); 82 | test!(f64); 83 | } 84 | -------------------------------------------------------------------------------- /src/stats/bivariate/mod.rs: -------------------------------------------------------------------------------- 1 | //! Bivariate analysis 2 | 3 | mod bootstrap; 4 | pub mod regression; 5 | mod resamples; 6 | 7 | use crate::stats::bivariate::resamples::Resamples; 8 | use crate::stats::float::Float; 9 | use crate::stats::tuple::{Tuple, TupledDistributionsBuilder}; 10 | use crate::stats::univariate::Sample; 11 | use rayon::iter::{IntoParallelIterator, ParallelIterator}; 12 | 13 | /// Bivariate `(X, Y)` data 14 | /// 15 | /// Invariants: 16 | /// 17 | /// - No `NaN`s in the data 18 | /// - At least two data points in the set 19 | pub struct Data<'a, X, Y>(&'a [X], &'a [Y]); 20 | 21 | impl Copy for Data<'_, X, Y> {} 22 | 23 | #[allow(clippy::expl_impl_clone_on_copy)] 24 | impl<'a, X, Y> Clone for Data<'a, X, Y> { 25 | fn clone(&self) -> Data<'a, X, Y> { 26 | *self 27 | } 28 | } 29 | 30 | impl<'a, X, Y> Data<'a, X, Y> { 31 | /// Returns the length of the data set 32 | pub fn len(&self) -> usize { 33 | self.0.len() 34 | } 35 | 36 | /// Iterate over the data set 37 | pub fn iter(&self) -> Pairs<'a, X, Y> { 38 | Pairs { 39 | data: *self, 40 | state: 0, 41 | } 42 | } 43 | } 44 | 45 | impl<'a, X, Y> Data<'a, X, Y> 46 | where 47 | X: Float, 48 | Y: Float, 49 | { 50 | /// Creates a new data set from two existing slices 51 | pub fn new(xs: &'a [X], ys: &'a [Y]) -> Data<'a, X, Y> { 52 | assert!( 53 | xs.len() == ys.len() 54 | && xs.len() > 1 55 | && xs.iter().all(|x| !x.is_nan()) 56 | && ys.iter().all(|y| !y.is_nan()) 57 | ); 58 | 59 | Data(xs, ys) 60 | } 61 | 62 | // TODO Remove the `T` parameter in favor of `S::Output` 63 | /// Returns the bootstrap distributions of the parameters estimated by the `statistic` 64 | /// 65 | /// - Multi-threaded 66 | /// - Time: `O(nresamples)` 67 | /// - Memory: `O(nresamples)` 68 | pub fn bootstrap(&self, nresamples: usize, statistic: S) -> T::Distributions 69 | where 70 | S: Fn(Data) -> T + Sync, 71 | T: Tuple + Send, 72 | T::Distributions: Send, 73 | T::Builder: Send, 74 | { 75 | (0..nresamples) 76 | .into_par_iter() 77 | .map_init( 78 | || Resamples::new(*self), 79 | |resamples, _| statistic(resamples.next()), 80 | ) 81 | .fold( 82 | || T::Builder::new(0), 83 | |mut sub_distributions, sample| { 84 | sub_distributions.push(sample); 85 | sub_distributions 86 | }, 87 | ) 88 | .reduce( 89 | || T::Builder::new(0), 90 | |mut a, mut b| { 91 | a.extend(&mut b); 92 | a 93 | }, 94 | ) 95 | .complete() 96 | } 97 | 98 | /// Returns a view into the `X` data 99 | pub fn x(&self) -> &'a Sample { 100 | Sample::new(self.0) 101 | } 102 | 103 | /// Returns a view into the `Y` data 104 | pub fn y(&self) -> &'a Sample { 105 | Sample::new(self.1) 106 | } 107 | } 108 | 109 | /// Iterator over `Data` 110 | pub struct Pairs<'a, X: 'a, Y: 'a> { 111 | data: Data<'a, X, Y>, 112 | state: usize, 113 | } 114 | 115 | impl<'a, X, Y> Iterator for Pairs<'a, X, Y> { 116 | type Item = (&'a X, &'a Y); 117 | 118 | fn next(&mut self) -> Option<(&'a X, &'a Y)> { 119 | if self.state < self.data.len() { 120 | let i = self.state; 121 | self.state += 1; 122 | 123 | // This is safe because i will always be < self.data.{0,1}.len() 124 | debug_assert!(i < self.data.0.len()); 125 | debug_assert!(i < self.data.1.len()); 126 | unsafe { Some((self.data.0.get_unchecked(i), self.data.1.get_unchecked(i))) } 127 | } else { 128 | None 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/stats/bivariate/regression.rs: -------------------------------------------------------------------------------- 1 | //! Regression analysis 2 | 3 | use crate::stats::bivariate::Data; 4 | use crate::stats::float::Float; 5 | 6 | /// A straight line that passes through the origin `y = m * x` 7 | #[derive(Clone, Copy)] 8 | pub struct Slope(pub A) 9 | where 10 | A: Float; 11 | 12 | impl Slope 13 | where 14 | A: Float, 15 | { 16 | /// Fits the data to a straight line that passes through the origin using ordinary least 17 | /// squares 18 | /// 19 | /// - Time: `O(length)` 20 | pub fn fit(data: &Data<'_, A, A>) -> Slope { 21 | let xs = data.0; 22 | let ys = data.1; 23 | 24 | let xy = crate::stats::dot(xs, ys); 25 | let x2 = crate::stats::dot(xs, xs); 26 | 27 | Slope(xy / x2) 28 | } 29 | 30 | /// Computes the goodness of fit (coefficient of determination) for this data set 31 | /// 32 | /// - Time: `O(length)` 33 | pub fn r_squared(&self, data: &Data<'_, A, A>) -> A { 34 | let _0 = A::cast(0); 35 | let _1 = A::cast(1); 36 | let m = self.0; 37 | let xs = data.0; 38 | let ys = data.1; 39 | 40 | let n = A::cast(xs.len()); 41 | let y_bar = crate::stats::sum(ys) / n; 42 | 43 | let mut ss_res = _0; 44 | let mut ss_tot = _0; 45 | 46 | for (&x, &y) in data.iter() { 47 | ss_res = ss_res + (y - m * x).powi(2); 48 | ss_tot = ss_res + (y - y_bar).powi(2); 49 | } 50 | 51 | _1 - ss_res / ss_tot 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/stats/bivariate/resamples.rs: -------------------------------------------------------------------------------- 1 | use crate::stats::bivariate::Data; 2 | use crate::stats::float::Float; 3 | use crate::stats::rand_util::{new_rng, Rng}; 4 | 5 | pub struct Resamples<'a, X, Y> 6 | where 7 | X: 'a + Float, 8 | Y: 'a + Float, 9 | { 10 | rng: Rng, 11 | data: (&'a [X], &'a [Y]), 12 | stage: Option<(Vec, Vec)>, 13 | } 14 | 15 | #[allow(clippy::should_implement_trait)] 16 | impl<'a, X, Y> Resamples<'a, X, Y> 17 | where 18 | X: 'a + Float, 19 | Y: 'a + Float, 20 | { 21 | pub fn new(data: Data<'a, X, Y>) -> Resamples<'a, X, Y> { 22 | Resamples { 23 | rng: new_rng(), 24 | data: (data.x(), data.y()), 25 | stage: None, 26 | } 27 | } 28 | 29 | pub fn next(&mut self) -> Data<'_, X, Y> { 30 | let n = self.data.0.len(); 31 | 32 | match self.stage { 33 | None => { 34 | let mut stage = (Vec::with_capacity(n), Vec::with_capacity(n)); 35 | 36 | for _ in 0..n { 37 | let i = self.rng.rand_range(0u64..(self.data.0.len() as u64)) as usize; 38 | 39 | stage.0.push(self.data.0[i]); 40 | stage.1.push(self.data.1[i]); 41 | } 42 | 43 | self.stage = Some(stage); 44 | } 45 | Some(ref mut stage) => { 46 | for i in 0..n { 47 | let j = self.rng.rand_range(0u64..(self.data.0.len() as u64)) as usize; 48 | 49 | stage.0[i] = self.data.0[j]; 50 | stage.1[i] = self.data.1[j]; 51 | } 52 | } 53 | } 54 | 55 | if let Some((ref x, ref y)) = self.stage { 56 | Data(x, y) 57 | } else { 58 | unreachable!(); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/stats/float.rs: -------------------------------------------------------------------------------- 1 | //! Float trait 2 | 3 | use cast::From; 4 | use num_traits::float; 5 | 6 | /// This is an extension of `num_traits::float::Float` that adds safe 7 | /// casting and Sync + Send. Once `num_traits` has these features this 8 | /// can be removed. 9 | pub trait Float: 10 | float::Float + From + From + Sync + Send 11 | { 12 | } 13 | 14 | impl Float for f32 {} 15 | impl Float for f64 {} 16 | -------------------------------------------------------------------------------- /src/stats/mod.rs: -------------------------------------------------------------------------------- 1 | //! [Criterion]'s statistics library. 2 | //! 3 | //! [Criterion]: https://github.com/bheisler/criterion.rs 4 | //! 5 | //! **WARNING** This library is criterion's implementation detail and there no plans to stabilize 6 | //! it. In other words, the API may break at any time without notice. 7 | 8 | #[cfg(test)] 9 | mod test; 10 | 11 | pub mod bivariate; 12 | pub mod tuple; 13 | pub mod univariate; 14 | 15 | mod float; 16 | mod rand_util; 17 | 18 | use std::mem; 19 | use std::ops::Deref; 20 | 21 | use crate::stats::float::Float; 22 | use crate::stats::univariate::Sample; 23 | 24 | /// The bootstrap distribution of some parameter 25 | #[derive(Clone)] 26 | pub struct Distribution(Box<[A]>); 27 | 28 | impl Distribution 29 | where 30 | A: Float, 31 | { 32 | /// Create a distribution from the given values 33 | pub fn from(values: Box<[A]>) -> Distribution { 34 | Distribution(values) 35 | } 36 | 37 | /// Computes the confidence interval of the population parameter using percentiles 38 | /// 39 | /// # Panics 40 | /// 41 | /// Panics if the `confidence_level` is not in the `(0, 1)` range. 42 | pub fn confidence_interval(&self, confidence_level: A) -> (A, A) 43 | where 44 | usize: cast::From>, 45 | { 46 | let _0 = A::cast(0); 47 | let _1 = A::cast(1); 48 | let _50 = A::cast(50); 49 | 50 | assert!(confidence_level > _0 && confidence_level < _1); 51 | 52 | let percentiles = self.percentiles(); 53 | 54 | // FIXME(privacy) this should use the `at_unchecked()` method 55 | ( 56 | percentiles.at(_50 * (_1 - confidence_level)), 57 | percentiles.at(_50 * (_1 + confidence_level)), 58 | ) 59 | } 60 | 61 | /// Computes the "likelihood" of seeing the value `t` or "more extreme" values in the 62 | /// distribution. 63 | pub fn p_value(&self, t: A, tails: &Tails) -> A { 64 | use std::cmp; 65 | 66 | let n = self.0.len(); 67 | let hits = self.0.iter().filter(|&&x| x < t).count(); 68 | 69 | let tails = A::cast(match *tails { 70 | Tails::One => 1, 71 | Tails::Two => 2, 72 | }); 73 | 74 | A::cast(cmp::min(hits, n - hits)) / A::cast(n) * tails 75 | } 76 | } 77 | 78 | impl Deref for Distribution { 79 | type Target = Sample; 80 | 81 | fn deref(&self) -> &Sample { 82 | let slice: &[_] = &self.0; 83 | 84 | unsafe { mem::transmute(slice) } 85 | } 86 | } 87 | 88 | /// Number of tails for significance testing 89 | pub enum Tails { 90 | /// One tailed test 91 | One, 92 | /// Two tailed test 93 | Two, 94 | } 95 | 96 | fn dot(xs: &[A], ys: &[A]) -> A 97 | where 98 | A: Float, 99 | { 100 | xs.iter() 101 | .zip(ys) 102 | .fold(A::cast(0), |acc, (&x, &y)| acc + x * y) 103 | } 104 | 105 | fn sum(xs: &[A]) -> A 106 | where 107 | A: Float, 108 | { 109 | use std::ops::Add; 110 | 111 | xs.iter().cloned().fold(A::cast(0), Add::add) 112 | } 113 | -------------------------------------------------------------------------------- /src/stats/rand_util.rs: -------------------------------------------------------------------------------- 1 | use oorandom::Rand64; 2 | use std::cell::RefCell; 3 | use std::time::{SystemTime, UNIX_EPOCH}; 4 | 5 | pub type Rng = Rand64; 6 | 7 | thread_local! { 8 | static SEED_RAND: RefCell = RefCell::new(Rand64::new( 9 | SystemTime::now().duration_since(UNIX_EPOCH) 10 | .expect("Time went backwards") 11 | .as_millis() 12 | )); 13 | } 14 | 15 | pub fn new_rng() -> Rng { 16 | SEED_RAND.with(|r| { 17 | let mut r = r.borrow_mut(); 18 | let seed = ((r.rand_u64() as u128) << 64) | (r.rand_u64() as u128); 19 | Rand64::new(seed) 20 | }) 21 | } 22 | -------------------------------------------------------------------------------- /src/stats/test.rs: -------------------------------------------------------------------------------- 1 | use rand::distributions::{Distribution, Standard}; 2 | use rand::prelude::*; 3 | use rand::rngs::StdRng; 4 | 5 | pub fn vec(size: usize, start: usize) -> Option> 6 | where 7 | Standard: Distribution, 8 | { 9 | if size > start + 2 { 10 | let mut rng = StdRng::from_entropy(); 11 | 12 | Some((0..size).map(|_| rng.gen()).collect()) 13 | } else { 14 | None 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/stats/tuple.rs: -------------------------------------------------------------------------------- 1 | //! Helper traits for tupling/untupling 2 | 3 | use crate::stats::Distribution; 4 | 5 | /// Any tuple: `(A, B, ..)` 6 | pub trait Tuple: Sized { 7 | /// A tuple of distributions associated with this tuple 8 | type Distributions: TupledDistributions; 9 | 10 | /// A tuple of vectors associated with this tuple 11 | type Builder: TupledDistributionsBuilder; 12 | } 13 | 14 | /// A tuple of distributions: `(Distribution, Distribution, ..)` 15 | pub trait TupledDistributions: Sized { 16 | /// A tuple that can be pushed/inserted into the tupled distributions 17 | type Item: Tuple; 18 | } 19 | 20 | /// A tuple of vecs used to build distributions. 21 | pub trait TupledDistributionsBuilder: Sized { 22 | /// A tuple that can be pushed/inserted into the tupled distributions 23 | type Item: Tuple; 24 | 25 | /// Creates a new tuple of vecs 26 | fn new(size: usize) -> Self; 27 | 28 | /// Push one element into each of the vecs 29 | fn push(&mut self, tuple: Self::Item); 30 | 31 | /// Append one tuple of vecs to this one, leaving the vecs in the other tuple empty 32 | fn extend(&mut self, other: &mut Self); 33 | 34 | /// Convert the tuple of vectors into a tuple of distributions 35 | fn complete(self) -> ::Distributions; 36 | } 37 | 38 | impl Tuple for (A,) 39 | where 40 | A: Copy, 41 | { 42 | type Distributions = (Distribution,); 43 | type Builder = (Vec,); 44 | } 45 | 46 | impl TupledDistributions for (Distribution,) 47 | where 48 | A: Copy, 49 | { 50 | type Item = (A,); 51 | } 52 | impl TupledDistributionsBuilder for (Vec,) 53 | where 54 | A: Copy, 55 | { 56 | type Item = (A,); 57 | 58 | fn new(size: usize) -> (Vec,) { 59 | (Vec::with_capacity(size),) 60 | } 61 | 62 | fn push(&mut self, tuple: (A,)) { 63 | (self.0).push(tuple.0); 64 | } 65 | 66 | fn extend(&mut self, other: &mut (Vec,)) { 67 | (self.0).append(&mut other.0); 68 | } 69 | 70 | fn complete(self) -> (Distribution,) { 71 | (Distribution(self.0.into_boxed_slice()),) 72 | } 73 | } 74 | 75 | impl Tuple for (A, B) 76 | where 77 | A: Copy, 78 | B: Copy, 79 | { 80 | type Distributions = (Distribution, Distribution); 81 | type Builder = (Vec, Vec); 82 | } 83 | 84 | impl TupledDistributions for (Distribution, Distribution) 85 | where 86 | A: Copy, 87 | B: Copy, 88 | { 89 | type Item = (A, B); 90 | } 91 | impl TupledDistributionsBuilder for (Vec, Vec) 92 | where 93 | A: Copy, 94 | B: Copy, 95 | { 96 | type Item = (A, B); 97 | 98 | fn new(size: usize) -> (Vec, Vec) { 99 | (Vec::with_capacity(size), Vec::with_capacity(size)) 100 | } 101 | 102 | fn push(&mut self, tuple: (A, B)) { 103 | (self.0).push(tuple.0); 104 | (self.1).push(tuple.1); 105 | } 106 | 107 | fn extend(&mut self, other: &mut (Vec, Vec)) { 108 | (self.0).append(&mut other.0); 109 | (self.1).append(&mut other.1); 110 | } 111 | 112 | fn complete(self) -> (Distribution, Distribution) { 113 | ( 114 | Distribution(self.0.into_boxed_slice()), 115 | Distribution(self.1.into_boxed_slice()), 116 | ) 117 | } 118 | } 119 | 120 | impl Tuple for (A, B, C) 121 | where 122 | A: Copy, 123 | B: Copy, 124 | C: Copy, 125 | { 126 | type Distributions = (Distribution, Distribution, Distribution); 127 | type Builder = (Vec, Vec, Vec); 128 | } 129 | 130 | impl TupledDistributions for (Distribution, Distribution, Distribution) 131 | where 132 | A: Copy, 133 | B: Copy, 134 | C: Copy, 135 | { 136 | type Item = (A, B, C); 137 | } 138 | impl TupledDistributionsBuilder for (Vec, Vec, Vec) 139 | where 140 | A: Copy, 141 | B: Copy, 142 | C: Copy, 143 | { 144 | type Item = (A, B, C); 145 | 146 | fn new(size: usize) -> (Vec, Vec, Vec) { 147 | ( 148 | Vec::with_capacity(size), 149 | Vec::with_capacity(size), 150 | Vec::with_capacity(size), 151 | ) 152 | } 153 | 154 | fn push(&mut self, tuple: (A, B, C)) { 155 | (self.0).push(tuple.0); 156 | (self.1).push(tuple.1); 157 | (self.2).push(tuple.2); 158 | } 159 | 160 | fn extend(&mut self, other: &mut (Vec, Vec, Vec)) { 161 | (self.0).append(&mut other.0); 162 | (self.1).append(&mut other.1); 163 | (self.2).append(&mut other.2); 164 | } 165 | 166 | fn complete(self) -> (Distribution, Distribution, Distribution) { 167 | ( 168 | Distribution(self.0.into_boxed_slice()), 169 | Distribution(self.1.into_boxed_slice()), 170 | Distribution(self.2.into_boxed_slice()), 171 | ) 172 | } 173 | } 174 | 175 | impl Tuple for (A, B, C, D) 176 | where 177 | A: Copy, 178 | B: Copy, 179 | C: Copy, 180 | D: Copy, 181 | { 182 | type Distributions = ( 183 | Distribution, 184 | Distribution, 185 | Distribution, 186 | Distribution, 187 | ); 188 | type Builder = (Vec, Vec, Vec, Vec); 189 | } 190 | 191 | impl TupledDistributions 192 | for ( 193 | Distribution, 194 | Distribution, 195 | Distribution, 196 | Distribution, 197 | ) 198 | where 199 | A: Copy, 200 | B: Copy, 201 | C: Copy, 202 | D: Copy, 203 | { 204 | type Item = (A, B, C, D); 205 | } 206 | impl TupledDistributionsBuilder for (Vec, Vec, Vec, Vec) 207 | where 208 | A: Copy, 209 | B: Copy, 210 | C: Copy, 211 | D: Copy, 212 | { 213 | type Item = (A, B, C, D); 214 | 215 | fn new(size: usize) -> (Vec, Vec, Vec, Vec) { 216 | ( 217 | Vec::with_capacity(size), 218 | Vec::with_capacity(size), 219 | Vec::with_capacity(size), 220 | Vec::with_capacity(size), 221 | ) 222 | } 223 | 224 | fn push(&mut self, tuple: (A, B, C, D)) { 225 | (self.0).push(tuple.0); 226 | (self.1).push(tuple.1); 227 | (self.2).push(tuple.2); 228 | (self.3).push(tuple.3); 229 | } 230 | 231 | fn extend(&mut self, other: &mut (Vec, Vec, Vec, Vec)) { 232 | (self.0).append(&mut other.0); 233 | (self.1).append(&mut other.1); 234 | (self.2).append(&mut other.2); 235 | (self.3).append(&mut other.3); 236 | } 237 | 238 | fn complete( 239 | self, 240 | ) -> ( 241 | Distribution, 242 | Distribution, 243 | Distribution, 244 | Distribution, 245 | ) { 246 | ( 247 | Distribution(self.0.into_boxed_slice()), 248 | Distribution(self.1.into_boxed_slice()), 249 | Distribution(self.2.into_boxed_slice()), 250 | Distribution(self.3.into_boxed_slice()), 251 | ) 252 | } 253 | } 254 | -------------------------------------------------------------------------------- /src/stats/univariate/bootstrap.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | macro_rules! test { 3 | ($ty:ident) => { 4 | mod $ty { 5 | use approx::relative_eq; 6 | use quickcheck::quickcheck; 7 | use quickcheck::TestResult; 8 | 9 | use crate::stats::univariate::{Sample, mixed, self}; 10 | 11 | quickcheck!{ 12 | fn mean(size: usize, start: usize, nresamples: usize) -> TestResult { 13 | if let Some(v) = crate::stats::test::vec::<$ty>(size, start) { 14 | let sample = Sample::new(&v[start..]); 15 | 16 | let means = if nresamples > 0 { 17 | sample.bootstrap(nresamples, |s| (s.mean(),)).0 18 | } else { 19 | return TestResult::discard(); 20 | }; 21 | 22 | let min = sample.min(); 23 | let max = sample.max(); 24 | 25 | TestResult::from_bool( 26 | // Computed the correct number of resamples 27 | means.len() == nresamples && 28 | // No uninitialized values 29 | means.iter().all(|&x| { 30 | (x > min || relative_eq!(x, min)) && 31 | (x < max || relative_eq!(x, max)) 32 | }) 33 | ) 34 | } else { 35 | TestResult::discard() 36 | } 37 | } 38 | } 39 | 40 | quickcheck!{ 41 | fn mean_median(size: usize, start: usize, nresamples: usize) -> TestResult { 42 | if let Some(v) = crate::stats::test::vec::<$ty>(size, start) { 43 | let sample = Sample::new(&v[start..]); 44 | 45 | let (means, medians) = if nresamples > 0 { 46 | sample.bootstrap(nresamples, |s| (s.mean(), s.median())) 47 | } else { 48 | return TestResult::discard(); 49 | }; 50 | 51 | let min = sample.min(); 52 | let max = sample.max(); 53 | 54 | TestResult::from_bool( 55 | // Computed the correct number of resamples 56 | means.len() == nresamples && 57 | medians.len() == nresamples && 58 | // No uninitialized values 59 | means.iter().all(|&x| { 60 | (x > min || relative_eq!(x, min)) && 61 | (x < max || relative_eq!(x, max)) 62 | }) && 63 | medians.iter().all(|&x| { 64 | (x > min || relative_eq!(x, min)) && 65 | (x < max || relative_eq!(x, max)) 66 | }) 67 | ) 68 | } else { 69 | TestResult::discard() 70 | } 71 | } 72 | } 73 | 74 | quickcheck!{ 75 | fn mixed_two_sample( 76 | a_size: usize, a_start: usize, 77 | b_size: usize, b_start: usize, 78 | nresamples: usize 79 | ) -> TestResult { 80 | if let (Some(a), Some(b)) = 81 | (crate::stats::test::vec::<$ty>(a_size, a_start), crate::stats::test::vec::<$ty>(b_size, b_start)) 82 | { 83 | let a = Sample::new(&a); 84 | let b = Sample::new(&b); 85 | 86 | let distribution = if nresamples > 0 { 87 | mixed::bootstrap(a, b, nresamples, |a, b| (a.mean() - b.mean(),)).0 88 | } else { 89 | return TestResult::discard(); 90 | }; 91 | 92 | let min = <$ty>::min(a.min() - b.max(), b.min() - a.max()); 93 | let max = <$ty>::max(a.max() - b.min(), b.max() - a.min()); 94 | 95 | TestResult::from_bool( 96 | // Computed the correct number of resamples 97 | distribution.len() == nresamples && 98 | // No uninitialized values 99 | distribution.iter().all(|&x| { 100 | (x > min || relative_eq!(x, min)) && 101 | (x < max || relative_eq!(x, max)) 102 | }) 103 | ) 104 | } else { 105 | TestResult::discard() 106 | } 107 | } 108 | } 109 | 110 | quickcheck!{ 111 | fn two_sample( 112 | a_size: usize, a_start: usize, 113 | b_size: usize, b_start: usize, 114 | nresamples: usize 115 | ) -> TestResult { 116 | if let (Some(a), Some(b)) = 117 | (crate::stats::test::vec::<$ty>(a_size, a_start), crate::stats::test::vec::<$ty>(b_size, b_start)) 118 | { 119 | let a = Sample::new(&a[a_start..]); 120 | let b = Sample::new(&b[b_start..]); 121 | 122 | let distribution = if nresamples > 0 { 123 | univariate::bootstrap(a, b, nresamples, |a, b| (a.mean() - b.mean(),)).0 124 | } else { 125 | return TestResult::discard(); 126 | }; 127 | 128 | let min = <$ty>::min(a.min() - b.max(), b.min() - a.max()); 129 | let max = <$ty>::max(a.max() - b.min(), b.max() - a.min()); 130 | 131 | // Computed the correct number of resamples 132 | let pass = distribution.len() == nresamples && 133 | // No uninitialized values 134 | distribution.iter().all(|&x| { 135 | (x > min || relative_eq!(x, min)) && 136 | (x < max || relative_eq!(x, max)) 137 | }); 138 | 139 | if !pass { 140 | println!("A: {:?} (len={})", a.as_ref(), a.len()); 141 | println!("B: {:?} (len={})", b.as_ref(), b.len()); 142 | println!("Dist: {:?} (len={})", distribution.as_ref(), distribution.len()); 143 | println!("Min: {}, Max: {}, nresamples: {}", 144 | min, max, nresamples); 145 | } 146 | 147 | TestResult::from_bool(pass) 148 | } else { 149 | TestResult::discard() 150 | } 151 | } 152 | } 153 | } 154 | } 155 | } 156 | 157 | #[cfg(test)] 158 | mod test { 159 | test!(f32); 160 | test!(f64); 161 | } 162 | -------------------------------------------------------------------------------- /src/stats/univariate/kde/kernel.rs: -------------------------------------------------------------------------------- 1 | //! Kernels 2 | 3 | use crate::stats::float::Float; 4 | 5 | /// Kernel function 6 | pub trait Kernel: Copy + Sync 7 | where 8 | A: Float, 9 | { 10 | /// Apply the kernel function to the given x-value. 11 | fn evaluate(&self, x: A) -> A; 12 | } 13 | 14 | /// Gaussian kernel 15 | #[derive(Clone, Copy)] 16 | pub struct Gaussian; 17 | 18 | impl Kernel for Gaussian 19 | where 20 | A: Float, 21 | { 22 | fn evaluate(&self, x: A) -> A { 23 | use std::f32::consts::PI; 24 | 25 | (x.powi(2).exp() * A::cast(2. * PI)).sqrt().recip() 26 | } 27 | } 28 | 29 | #[cfg(test)] 30 | macro_rules! test { 31 | ($ty:ident) => { 32 | mod $ty { 33 | mod gaussian { 34 | use approx::relative_eq; 35 | use quickcheck::quickcheck; 36 | use quickcheck::TestResult; 37 | 38 | use crate::stats::univariate::kde::kernel::{Gaussian, Kernel}; 39 | 40 | quickcheck! { 41 | fn symmetric(x: $ty) -> bool { 42 | relative_eq!(Gaussian.evaluate(-x), Gaussian.evaluate(x)) 43 | } 44 | } 45 | 46 | // Any [a b] integral should be in the range [0 1] 47 | quickcheck! { 48 | fn integral(a: $ty, b: $ty) -> TestResult { 49 | const DX: $ty = 1e-3; 50 | 51 | if a > b { 52 | TestResult::discard() 53 | } else { 54 | let mut acc = 0.; 55 | let mut x = a; 56 | let mut y = Gaussian.evaluate(a); 57 | 58 | while x < b { 59 | acc += DX * y / 2.; 60 | 61 | x += DX; 62 | y = Gaussian.evaluate(x); 63 | 64 | acc += DX * y / 2.; 65 | } 66 | 67 | TestResult::from_bool( 68 | (acc > 0. || relative_eq!(acc, 0.)) && 69 | (acc < 1. || relative_eq!(acc, 1.))) 70 | } 71 | } 72 | } 73 | } 74 | } 75 | }; 76 | } 77 | 78 | #[cfg(test)] 79 | mod test { 80 | test!(f32); 81 | test!(f64); 82 | } 83 | -------------------------------------------------------------------------------- /src/stats/univariate/kde/mod.rs: -------------------------------------------------------------------------------- 1 | //! Kernel density estimation 2 | 3 | pub mod kernel; 4 | 5 | use self::kernel::Kernel; 6 | use crate::stats::float::Float; 7 | use crate::stats::univariate::Sample; 8 | use rayon::prelude::*; 9 | 10 | /// Univariate kernel density estimator 11 | pub struct Kde<'a, A, K> 12 | where 13 | A: Float, 14 | K: Kernel, 15 | { 16 | bandwidth: A, 17 | kernel: K, 18 | sample: &'a Sample, 19 | } 20 | 21 | impl<'a, A, K> Kde<'a, A, K> 22 | where 23 | A: 'a + Float, 24 | K: Kernel, 25 | { 26 | /// Creates a new kernel density estimator from the `sample`, using a kernel and estimating 27 | /// the bandwidth using the method `bw` 28 | pub fn new(sample: &'a Sample, kernel: K, bw: Bandwidth) -> Kde<'a, A, K> { 29 | Kde { 30 | bandwidth: bw.estimate(sample), 31 | kernel, 32 | sample, 33 | } 34 | } 35 | 36 | /// Returns the bandwidth used by the estimator 37 | pub fn bandwidth(&self) -> A { 38 | self.bandwidth 39 | } 40 | 41 | /// Maps the KDE over `xs` 42 | /// 43 | /// - Multihreaded 44 | pub fn map(&self, xs: &[A]) -> Box<[A]> { 45 | xs.par_iter() 46 | .map(|&x| self.estimate(x)) 47 | .collect::>() 48 | .into_boxed_slice() 49 | } 50 | 51 | /// Estimates the probability density of `x` 52 | pub fn estimate(&self, x: A) -> A { 53 | let _0 = A::cast(0); 54 | let slice = self.sample; 55 | let h = self.bandwidth; 56 | let n = A::cast(slice.len()); 57 | 58 | let sum = slice 59 | .iter() 60 | .fold(_0, |acc, &x_i| acc + self.kernel.evaluate((x - x_i) / h)); 61 | 62 | sum / (h * n) 63 | } 64 | } 65 | 66 | /// Method to estimate the bandwidth 67 | pub enum Bandwidth { 68 | /// Use Silverman's rule of thumb to estimate the bandwidth from the sample 69 | Silverman, 70 | } 71 | 72 | impl Bandwidth { 73 | fn estimate(self, sample: &Sample) -> A { 74 | match self { 75 | Bandwidth::Silverman => { 76 | let factor = A::cast(4. / 3.); 77 | let exponent = A::cast(1. / 5.); 78 | let n = A::cast(sample.len()); 79 | let sigma = sample.std_dev(None); 80 | 81 | sigma * (factor / n).powf(exponent) 82 | } 83 | } 84 | } 85 | } 86 | 87 | #[cfg(test)] 88 | macro_rules! test { 89 | ($ty:ident) => { 90 | mod $ty { 91 | use approx::relative_eq; 92 | use quickcheck::quickcheck; 93 | use quickcheck::TestResult; 94 | 95 | use crate::stats::univariate::kde::kernel::Gaussian; 96 | use crate::stats::univariate::kde::{Bandwidth, Kde}; 97 | use crate::stats::univariate::Sample; 98 | 99 | // The [-inf inf] integral of the estimated PDF should be one 100 | quickcheck! { 101 | fn integral(size: usize, start: usize) -> TestResult { 102 | const DX: $ty = 1e-3; 103 | 104 | if let Some(v) = crate::stats::test::vec::<$ty>(size, start) { 105 | let slice = &v[start..]; 106 | let data = Sample::new(slice); 107 | let kde = Kde::new(data, Gaussian, Bandwidth::Silverman); 108 | let h = kde.bandwidth(); 109 | // NB Obviously a [-inf inf] integral is not feasible, but this range works 110 | // quite well 111 | let (a, b) = (data.min() - 5. * h, data.max() + 5. * h); 112 | 113 | let mut acc = 0.; 114 | let mut x = a; 115 | let mut y = kde.estimate(a); 116 | 117 | while x < b { 118 | acc += DX * y / 2.; 119 | 120 | x += DX; 121 | y = kde.estimate(x); 122 | 123 | acc += DX * y / 2.; 124 | } 125 | 126 | TestResult::from_bool(relative_eq!(acc, 1., epsilon = 2e-5)) 127 | } else { 128 | TestResult::discard() 129 | } 130 | } 131 | } 132 | } 133 | }; 134 | } 135 | 136 | #[cfg(test)] 137 | mod test { 138 | test!(f32); 139 | test!(f64); 140 | } 141 | -------------------------------------------------------------------------------- /src/stats/univariate/mixed.rs: -------------------------------------------------------------------------------- 1 | //! Mixed bootstrap 2 | 3 | use crate::stats::float::Float; 4 | use crate::stats::tuple::{Tuple, TupledDistributionsBuilder}; 5 | use crate::stats::univariate::Resamples; 6 | use crate::stats::univariate::Sample; 7 | use rayon::prelude::*; 8 | 9 | /// Performs a *mixed* two-sample bootstrap 10 | pub fn bootstrap( 11 | a: &Sample, 12 | b: &Sample, 13 | nresamples: usize, 14 | statistic: S, 15 | ) -> T::Distributions 16 | where 17 | A: Float, 18 | S: Fn(&Sample, &Sample) -> T + Sync, 19 | T: Tuple + Send, 20 | T::Distributions: Send, 21 | T::Builder: Send, 22 | { 23 | let n_a = a.len(); 24 | let n_b = b.len(); 25 | let mut c = Vec::with_capacity(n_a + n_b); 26 | c.extend_from_slice(a); 27 | c.extend_from_slice(b); 28 | let c = Sample::new(&c); 29 | 30 | (0..nresamples) 31 | .into_par_iter() 32 | .map_init( 33 | || Resamples::new(c), 34 | |resamples, _| { 35 | let resample = resamples.next(); 36 | let a: &Sample = Sample::new(&resample[..n_a]); 37 | let b: &Sample = Sample::new(&resample[n_a..]); 38 | 39 | statistic(a, b) 40 | }, 41 | ) 42 | .fold( 43 | || T::Builder::new(0), 44 | |mut sub_distributions, sample| { 45 | sub_distributions.push(sample); 46 | sub_distributions 47 | }, 48 | ) 49 | .reduce( 50 | || T::Builder::new(0), 51 | |mut a, mut b| { 52 | a.extend(&mut b); 53 | a 54 | }, 55 | ) 56 | .complete() 57 | } 58 | -------------------------------------------------------------------------------- /src/stats/univariate/mod.rs: -------------------------------------------------------------------------------- 1 | //! Univariate analysis 2 | 3 | mod bootstrap; 4 | mod percentiles; 5 | mod resamples; 6 | mod sample; 7 | 8 | pub mod kde; 9 | pub mod mixed; 10 | pub mod outliers; 11 | 12 | use crate::stats::float::Float; 13 | use crate::stats::tuple::{Tuple, TupledDistributionsBuilder}; 14 | use rayon::prelude::*; 15 | use std::cmp; 16 | 17 | use self::resamples::Resamples; 18 | 19 | pub use self::percentiles::Percentiles; 20 | pub use self::sample::Sample; 21 | 22 | /// Performs a two-sample bootstrap 23 | /// 24 | /// - Multithreaded 25 | /// - Time: `O(nresamples)` 26 | /// - Memory: `O(nresamples)` 27 | #[allow(clippy::cast_lossless)] 28 | pub fn bootstrap( 29 | a: &Sample, 30 | b: &Sample, 31 | nresamples: usize, 32 | statistic: S, 33 | ) -> T::Distributions 34 | where 35 | A: Float, 36 | B: Float, 37 | S: Fn(&Sample, &Sample) -> T + Sync, 38 | T: Tuple + Send, 39 | T::Distributions: Send, 40 | T::Builder: Send, 41 | { 42 | let nresamples_sqrt = (nresamples as f64).sqrt().ceil() as usize; 43 | let per_chunk = nresamples.div_ceil(nresamples_sqrt); 44 | 45 | (0..nresamples_sqrt) 46 | .into_par_iter() 47 | .map_init( 48 | || (Resamples::new(a), Resamples::new(b)), 49 | |(a_resamples, b_resamples), i| { 50 | let start = i * per_chunk; 51 | let end = cmp::min((i + 1) * per_chunk, nresamples); 52 | let a_resample = a_resamples.next(); 53 | 54 | let mut sub_distributions: T::Builder = 55 | TupledDistributionsBuilder::new(end - start); 56 | 57 | for _ in start..end { 58 | let b_resample = b_resamples.next(); 59 | sub_distributions.push(statistic(a_resample, b_resample)); 60 | } 61 | sub_distributions 62 | }, 63 | ) 64 | .reduce( 65 | || T::Builder::new(0), 66 | |mut a, mut b| { 67 | a.extend(&mut b); 68 | a 69 | }, 70 | ) 71 | .complete() 72 | } 73 | -------------------------------------------------------------------------------- /src/stats/univariate/outliers/mod.rs: -------------------------------------------------------------------------------- 1 | //! Classification of outliers 2 | //! 3 | //! WARNING: There's no formal/mathematical definition of what an outlier actually is. Therefore, 4 | //! all outlier classifiers are *subjective*, however some classifiers that have become *de facto* 5 | //! standard are provided here. 6 | 7 | pub mod tukey; 8 | -------------------------------------------------------------------------------- /src/stats/univariate/outliers/tukey.rs: -------------------------------------------------------------------------------- 1 | //! Tukey's method 2 | //! 3 | //! The original method uses two "fences" to classify the data. All the observations "inside" the 4 | //! fences are considered "normal", and the rest are considered outliers. 5 | //! 6 | //! The fences are computed from the quartiles of the sample, according to the following formula: 7 | //! 8 | //! ``` ignore 9 | //! // q1, q3 are the first and third quartiles 10 | //! let iqr = q3 - q1; // The interquartile range 11 | //! let (f1, f2) = (q1 - 1.5 * iqr, q3 + 1.5 * iqr); // the "fences" 12 | //! 13 | //! let is_outlier = |x| if x > f1 && x < f2 { true } else { false }; 14 | //! ``` 15 | //! 16 | //! The classifier provided here adds two extra outer fences: 17 | //! 18 | //! ``` ignore 19 | //! let (f3, f4) = (q1 - 3 * iqr, q3 + 3 * iqr); // the outer "fences" 20 | //! ``` 21 | //! 22 | //! The extra fences add a sense of "severity" to the classification. Data points outside of the 23 | //! outer fences are considered "severe" outliers, whereas points outside the inner fences are just 24 | //! "mild" outliers, and, as the original method, everything inside the inner fences is considered 25 | //! "normal" data. 26 | //! 27 | //! Some ASCII art for the visually oriented people: 28 | //! 29 | //! ``` ignore 30 | //! LOW-ish NORMAL-ish HIGH-ish 31 | //! x | + | o o o o o o o | + | x 32 | //! f3 f1 f2 f4 33 | //! 34 | //! Legend: 35 | //! o: "normal" data (not an outlier) 36 | //! +: "mild" outlier 37 | //! x: "severe" outlier 38 | //! ``` 39 | 40 | use std::iter::IntoIterator; 41 | use std::ops::{Deref, Index}; 42 | use std::slice; 43 | 44 | use crate::stats::float::Float; 45 | use crate::stats::univariate::Sample; 46 | 47 | use self::Label::*; 48 | 49 | /// A classified/labeled sample. 50 | /// 51 | /// The labeled data can be accessed using the indexing operator. The order of the data points is 52 | /// retained. 53 | /// 54 | /// NOTE: Due to limitations in the indexing traits, only the label is returned. Once the 55 | /// `IndexGet` trait lands in stdlib, the indexing operation will return a `(data_point, label)` 56 | /// pair. 57 | #[derive(Clone, Copy)] 58 | pub struct LabeledSample<'a, A> 59 | where 60 | A: Float, 61 | { 62 | fences: (A, A, A, A), 63 | sample: &'a Sample, 64 | } 65 | 66 | impl<'a, A> LabeledSample<'a, A> 67 | where 68 | A: Float, 69 | { 70 | /// Returns the number of data points per label 71 | /// 72 | /// - Time: `O(length)` 73 | #[allow(clippy::similar_names)] 74 | pub fn count(&self) -> (usize, usize, usize, usize, usize) { 75 | let (mut los, mut lom, mut noa, mut him, mut his) = (0, 0, 0, 0, 0); 76 | 77 | for (_, label) in self { 78 | match label { 79 | LowSevere => { 80 | los += 1; 81 | } 82 | LowMild => { 83 | lom += 1; 84 | } 85 | NotAnOutlier => { 86 | noa += 1; 87 | } 88 | HighMild => { 89 | him += 1; 90 | } 91 | HighSevere => { 92 | his += 1; 93 | } 94 | } 95 | } 96 | 97 | (los, lom, noa, him, his) 98 | } 99 | 100 | /// Returns the fences used to classify the outliers 101 | pub fn fences(&self) -> (A, A, A, A) { 102 | self.fences 103 | } 104 | 105 | /// Returns an iterator over the labeled data 106 | pub fn iter(&self) -> Iter<'a, A> { 107 | Iter { 108 | fences: self.fences, 109 | iter: self.sample.iter(), 110 | } 111 | } 112 | } 113 | 114 | impl Deref for LabeledSample<'_, A> 115 | where 116 | A: Float, 117 | { 118 | type Target = Sample; 119 | 120 | fn deref(&self) -> &Sample { 121 | self.sample 122 | } 123 | } 124 | 125 | // FIXME Use the `IndexGet` trait 126 | impl Index for LabeledSample<'_, A> 127 | where 128 | A: Float, 129 | { 130 | type Output = Label; 131 | 132 | #[allow(clippy::similar_names)] 133 | fn index(&self, i: usize) -> &Label { 134 | static LOW_SEVERE: Label = LowSevere; 135 | static LOW_MILD: Label = LowMild; 136 | static HIGH_MILD: Label = HighMild; 137 | static HIGH_SEVERE: Label = HighSevere; 138 | static NOT_AN_OUTLIER: Label = NotAnOutlier; 139 | 140 | let x = self.sample[i]; 141 | let (lost, lomt, himt, hist) = self.fences; 142 | 143 | if x < lost { 144 | &LOW_SEVERE 145 | } else if x > hist { 146 | &HIGH_SEVERE 147 | } else if x < lomt { 148 | &LOW_MILD 149 | } else if x > himt { 150 | &HIGH_MILD 151 | } else { 152 | &NOT_AN_OUTLIER 153 | } 154 | } 155 | } 156 | 157 | impl<'a, A> IntoIterator for &LabeledSample<'a, A> 158 | where 159 | A: Float, 160 | { 161 | type Item = (A, Label); 162 | type IntoIter = Iter<'a, A>; 163 | 164 | fn into_iter(self) -> Iter<'a, A> { 165 | self.iter() 166 | } 167 | } 168 | 169 | /// Iterator over the labeled data 170 | pub struct Iter<'a, A> 171 | where 172 | A: Float, 173 | { 174 | fences: (A, A, A, A), 175 | iter: slice::Iter<'a, A>, 176 | } 177 | 178 | impl Iterator for Iter<'_, A> 179 | where 180 | A: Float, 181 | { 182 | type Item = (A, Label); 183 | 184 | #[allow(clippy::similar_names)] 185 | fn next(&mut self) -> Option<(A, Label)> { 186 | self.iter.next().map(|&x| { 187 | let (lost, lomt, himt, hist) = self.fences; 188 | 189 | let label = if x < lost { 190 | LowSevere 191 | } else if x > hist { 192 | HighSevere 193 | } else if x < lomt { 194 | LowMild 195 | } else if x > himt { 196 | HighMild 197 | } else { 198 | NotAnOutlier 199 | }; 200 | 201 | (x, label) 202 | }) 203 | } 204 | 205 | fn size_hint(&self) -> (usize, Option) { 206 | self.iter.size_hint() 207 | } 208 | } 209 | 210 | #[derive(Debug, Clone, Copy, Eq, PartialEq)] 211 | /// Labels used to classify outliers 212 | pub enum Label { 213 | /// A "mild" outlier in the "high" spectrum 214 | HighMild, 215 | /// A "severe" outlier in the "high" spectrum 216 | HighSevere, 217 | /// A "mild" outlier in the "low" spectrum 218 | LowMild, 219 | /// A "severe" outlier in the "low" spectrum 220 | LowSevere, 221 | /// A normal data point 222 | NotAnOutlier, 223 | } 224 | 225 | impl Label { 226 | /// Checks if the data point has an "unusually" high value 227 | pub fn is_high(&self) -> bool { 228 | matches!(*self, HighMild | HighSevere) 229 | } 230 | 231 | /// Checks if the data point is labeled as a "mild" outlier 232 | pub fn is_mild(&self) -> bool { 233 | matches!(*self, HighMild | LowMild) 234 | } 235 | 236 | /// Checks if the data point has an "unusually" low value 237 | pub fn is_low(&self) -> bool { 238 | matches!(*self, LowMild | LowSevere) 239 | } 240 | 241 | /// Checks if the data point is labeled as an outlier 242 | pub fn is_outlier(&self) -> bool { 243 | !matches!(*self, NotAnOutlier) 244 | } 245 | 246 | /// Checks if the data point is labeled as a "severe" outlier 247 | pub fn is_severe(&self) -> bool { 248 | matches!(*self, HighSevere | LowSevere) 249 | } 250 | } 251 | 252 | /// Classifies the sample, and returns a labeled sample. 253 | /// 254 | /// - Time: `O(N log N) where N = length` 255 | pub fn classify(sample: &Sample) -> LabeledSample<'_, A> 256 | where 257 | A: Float, 258 | usize: cast::From>, 259 | { 260 | let (q1, _, q3) = sample.percentiles().quartiles(); 261 | let iqr = q3 - q1; 262 | 263 | // Mild 264 | let k_m = A::cast(1.5_f32); 265 | // Severe 266 | let k_s = A::cast(3); 267 | 268 | LabeledSample { 269 | fences: ( 270 | q1 - k_s * iqr, 271 | q1 - k_m * iqr, 272 | q3 + k_m * iqr, 273 | q3 + k_s * iqr, 274 | ), 275 | sample, 276 | } 277 | } 278 | -------------------------------------------------------------------------------- /src/stats/univariate/percentiles.rs: -------------------------------------------------------------------------------- 1 | use crate::stats::float::Float; 2 | use cast::{self, usize}; 3 | 4 | /// A "view" into the percentiles of a sample 5 | pub struct Percentiles(Box<[A]>) 6 | where 7 | A: Float; 8 | 9 | // TODO(rust-lang/rfcs#735) move this `impl` into a private percentiles module 10 | impl Percentiles 11 | where 12 | A: Float, 13 | usize: cast::From>, 14 | { 15 | /// Returns the percentile at `p`% 16 | /// 17 | /// Safety: 18 | /// 19 | /// - Make sure that `p` is in the range `[0, 100]` 20 | unsafe fn at_unchecked(&self, p: A) -> A { 21 | let _100 = A::cast(100); 22 | debug_assert!(p >= A::cast(0) && p <= _100); 23 | debug_assert!(!self.0.is_empty()); 24 | let len = self.0.len() - 1; 25 | 26 | if p == _100 { 27 | self.0[len] 28 | } else { 29 | let rank = (p / _100) * A::cast(len); 30 | let integer = rank.floor(); 31 | let fraction = rank - integer; 32 | let n = usize(integer).unwrap(); 33 | let &floor = self.0.get_unchecked(n); 34 | let &ceiling = self.0.get_unchecked(n + 1); 35 | 36 | floor + (ceiling - floor) * fraction 37 | } 38 | } 39 | 40 | /// Returns the percentile at `p`% 41 | /// 42 | /// # Panics 43 | /// 44 | /// Panics if `p` is outside the closed `[0, 100]` range 45 | pub fn at(&self, p: A) -> A { 46 | let _0 = A::cast(0); 47 | let _100 = A::cast(100); 48 | 49 | assert!(p >= _0 && p <= _100); 50 | assert!(!self.0.is_empty()); 51 | 52 | unsafe { self.at_unchecked(p) } 53 | } 54 | 55 | /// Returns the interquartile range 56 | pub fn iqr(&self) -> A { 57 | unsafe { 58 | let q1 = self.at_unchecked(A::cast(25)); 59 | let q3 = self.at_unchecked(A::cast(75)); 60 | 61 | q3 - q1 62 | } 63 | } 64 | 65 | /// Returns the 50th percentile 66 | pub fn median(&self) -> A { 67 | unsafe { self.at_unchecked(A::cast(50)) } 68 | } 69 | 70 | /// Returns the 25th, 50th and 75th percentiles 71 | pub fn quartiles(&self) -> (A, A, A) { 72 | unsafe { 73 | ( 74 | self.at_unchecked(A::cast(25)), 75 | self.at_unchecked(A::cast(50)), 76 | self.at_unchecked(A::cast(75)), 77 | ) 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/stats/univariate/resamples.rs: -------------------------------------------------------------------------------- 1 | use std::mem; 2 | 3 | use crate::stats::float::Float; 4 | use crate::stats::rand_util::{new_rng, Rng}; 5 | use crate::stats::univariate::Sample; 6 | 7 | pub struct Resamples<'a, A> 8 | where 9 | A: Float, 10 | { 11 | rng: Rng, 12 | sample: &'a [A], 13 | stage: Option>, 14 | } 15 | 16 | #[allow(clippy::should_implement_trait)] 17 | impl<'a, A> Resamples<'a, A> 18 | where 19 | A: 'a + Float, 20 | { 21 | pub fn new(sample: &'a Sample) -> Resamples<'a, A> { 22 | let slice = sample; 23 | 24 | Resamples { 25 | rng: new_rng(), 26 | sample: slice, 27 | stage: None, 28 | } 29 | } 30 | 31 | pub fn next(&mut self) -> &Sample { 32 | let n = self.sample.len(); 33 | let rng = &mut self.rng; 34 | 35 | match self.stage { 36 | None => { 37 | let mut stage = Vec::with_capacity(n); 38 | 39 | for _ in 0..n { 40 | let idx = rng.rand_range(0u64..(self.sample.len() as u64)); 41 | stage.push(self.sample[idx as usize]) 42 | } 43 | 44 | self.stage = Some(stage); 45 | } 46 | Some(ref mut stage) => { 47 | for elem in stage.iter_mut() { 48 | let idx = rng.rand_range(0u64..(self.sample.len() as u64)); 49 | *elem = self.sample[idx as usize] 50 | } 51 | } 52 | } 53 | 54 | if let Some(ref v) = self.stage { 55 | unsafe { mem::transmute::<&[A], &Sample>(v) } 56 | } else { 57 | unreachable!(); 58 | } 59 | } 60 | } 61 | 62 | #[cfg(test)] 63 | mod test { 64 | use quickcheck::quickcheck; 65 | use quickcheck::TestResult; 66 | use std::collections::HashSet; 67 | 68 | use crate::stats::univariate::resamples::Resamples; 69 | use crate::stats::univariate::Sample; 70 | 71 | // Check that the resample is a subset of the sample 72 | quickcheck! { 73 | fn subset(size: usize, nresamples: usize) -> TestResult { 74 | if size > 1 { 75 | let v: Vec<_> = (0..size).map(|i| i as f32).collect(); 76 | let sample = Sample::new(&v); 77 | let mut resamples = Resamples::new(sample); 78 | let sample = v.iter().map(|&x| x as i64).collect::>(); 79 | 80 | TestResult::from_bool((0..nresamples).all(|_| { 81 | let resample = resamples.next() 82 | 83 | .iter() 84 | .map(|&x| x as i64) 85 | .collect::>(); 86 | 87 | resample.is_subset(&sample) 88 | })) 89 | } else { 90 | TestResult::discard() 91 | } 92 | } 93 | } 94 | 95 | #[test] 96 | fn different_subsets() { 97 | let size = 1000; 98 | let v: Vec<_> = (0..size).map(|i| i as f32).collect(); 99 | let sample = Sample::new(&v); 100 | let mut resamples = Resamples::new(sample); 101 | 102 | // Hypothetically, we might see one duplicate, but more than one is likely to be a bug. 103 | let mut num_duplicated = 0; 104 | for _ in 0..1000 { 105 | let sample_1 = resamples.next().iter().cloned().collect::>(); 106 | let sample_2 = resamples.next().iter().cloned().collect::>(); 107 | 108 | if sample_1 == sample_2 { 109 | num_duplicated += 1; 110 | } 111 | } 112 | 113 | if num_duplicated > 1 { 114 | panic!("Found {} duplicate samples", num_duplicated); 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/stats/univariate/sample.rs: -------------------------------------------------------------------------------- 1 | use std::{mem, ops}; 2 | 3 | use crate::stats::float::Float; 4 | use crate::stats::tuple::{Tuple, TupledDistributionsBuilder}; 5 | use crate::stats::univariate::Percentiles; 6 | use crate::stats::univariate::Resamples; 7 | use rayon::prelude::*; 8 | 9 | /// A collection of data points drawn from a population 10 | /// 11 | /// Invariants: 12 | /// 13 | /// - The sample contains at least 2 data points 14 | /// - The sample contains no `NaN`s 15 | pub struct Sample([A]); 16 | 17 | // TODO(rust-lang/rfcs#735) move this `impl` into a private percentiles module 18 | impl Sample 19 | where 20 | A: Float, 21 | { 22 | /// Creates a new sample from an existing slice 23 | /// 24 | /// # Panics 25 | /// 26 | /// Panics if `slice` contains any `NaN` or if `slice` has less than two elements 27 | #[allow(clippy::new_ret_no_self)] 28 | pub fn new(slice: &[A]) -> &Sample { 29 | assert!(slice.len() > 1 && slice.iter().all(|x| !x.is_nan())); 30 | 31 | unsafe { mem::transmute(slice) } 32 | } 33 | 34 | /// Returns the biggest element in the sample 35 | /// 36 | /// - Time: `O(length)` 37 | pub fn max(&self) -> A { 38 | let mut elems = self.iter(); 39 | 40 | match elems.next() { 41 | Some(&head) => elems.fold(head, |a, &b| a.max(b)), 42 | // NB `unreachable!` because `Sample` is guaranteed to have at least one data point 43 | None => unreachable!(), 44 | } 45 | } 46 | 47 | /// Returns the arithmetic average of the sample 48 | /// 49 | /// - Time: `O(length)` 50 | pub fn mean(&self) -> A { 51 | let n = self.len(); 52 | 53 | self.sum() / A::cast(n) 54 | } 55 | 56 | /// Returns the median absolute deviation 57 | /// 58 | /// The `median` can be optionally passed along to speed up (2X) the computation 59 | /// 60 | /// - Time: `O(length)` 61 | /// - Memory: `O(length)` 62 | pub fn median_abs_dev(&self, median: Option) -> A 63 | where 64 | usize: cast::From>, 65 | { 66 | let median = median.unwrap_or_else(|| self.percentiles().median()); 67 | 68 | // NB Although this operation can be SIMD accelerated, the gain is negligible because the 69 | // bottle neck is the sorting operation which is part of the computation of the median 70 | let abs_devs = self.iter().map(|&x| (x - median).abs()).collect::>(); 71 | 72 | let abs_devs: &Self = Self::new(&abs_devs); 73 | 74 | abs_devs.percentiles().median() * A::cast(1.4826) 75 | } 76 | 77 | /// Returns the median absolute deviation as a percentage of the median 78 | /// 79 | /// - Time: `O(length)` 80 | /// - Memory: `O(length)` 81 | pub fn median_abs_dev_pct(&self) -> A 82 | where 83 | usize: cast::From>, 84 | { 85 | let _100 = A::cast(100); 86 | let median = self.percentiles().median(); 87 | let mad = self.median_abs_dev(Some(median)); 88 | 89 | (mad / median) * _100 90 | } 91 | 92 | /// Returns the smallest element in the sample 93 | /// 94 | /// - Time: `O(length)` 95 | pub fn min(&self) -> A { 96 | let mut elems = self.iter(); 97 | 98 | match elems.next() { 99 | Some(&elem) => elems.fold(elem, |a, &b| a.min(b)), 100 | // NB `unreachable!` because `Sample` is guaranteed to have at least one data point 101 | None => unreachable!(), 102 | } 103 | } 104 | 105 | /// Returns a "view" into the percentiles of the sample 106 | /// 107 | /// This "view" makes consecutive computations of percentiles much faster (`O(1)`) 108 | /// 109 | /// - Time: `O(N log N) where N = length` 110 | /// - Memory: `O(length)` 111 | pub fn percentiles(&self) -> Percentiles 112 | where 113 | usize: cast::From>, 114 | { 115 | use std::cmp::Ordering; 116 | 117 | // NB This function assumes that there are no `NaN`s in the sample 118 | fn cmp(a: &T, b: &T) -> Ordering 119 | where 120 | T: PartialOrd, 121 | { 122 | match a.partial_cmp(b) { 123 | Some(o) => o, 124 | // Arbitrary way to handle NaNs that should never happen 125 | None => Ordering::Equal, 126 | } 127 | } 128 | 129 | let mut v = self.to_vec().into_boxed_slice(); 130 | v.par_sort_unstable_by(cmp); 131 | 132 | // NB :-1: to intra-crate privacy rules 133 | unsafe { mem::transmute(v) } 134 | } 135 | 136 | /// Returns the standard deviation of the sample 137 | /// 138 | /// The `mean` can be optionally passed along to speed up (2X) the computation 139 | /// 140 | /// - Time: `O(length)` 141 | pub fn std_dev(&self, mean: Option) -> A { 142 | self.var(mean).sqrt() 143 | } 144 | 145 | /// Returns the standard deviation as a percentage of the mean 146 | /// 147 | /// - Time: `O(length)` 148 | pub fn std_dev_pct(&self) -> A { 149 | let _100 = A::cast(100); 150 | let mean = self.mean(); 151 | let std_dev = self.std_dev(Some(mean)); 152 | 153 | (std_dev / mean) * _100 154 | } 155 | 156 | /// Returns the sum of all the elements of the sample 157 | /// 158 | /// - Time: `O(length)` 159 | pub fn sum(&self) -> A { 160 | crate::stats::sum(self) 161 | } 162 | 163 | /// Returns the t score between these two samples 164 | /// 165 | /// - Time: `O(length)` 166 | pub fn t(&self, other: &Sample) -> A { 167 | let (x_bar, y_bar) = (self.mean(), other.mean()); 168 | let (s2_x, s2_y) = (self.var(Some(x_bar)), other.var(Some(y_bar))); 169 | let n_x = A::cast(self.len()); 170 | let n_y = A::cast(other.len()); 171 | let num = x_bar - y_bar; 172 | let den = (s2_x / n_x + s2_y / n_y).sqrt(); 173 | 174 | num / den 175 | } 176 | 177 | /// Returns the variance of the sample 178 | /// 179 | /// The `mean` can be optionally passed along to speed up (2X) the computation 180 | /// 181 | /// - Time: `O(length)` 182 | pub fn var(&self, mean: Option) -> A { 183 | use std::ops::Add; 184 | 185 | let mean = mean.unwrap_or_else(|| self.mean()); 186 | let slice = self; 187 | 188 | let sum = slice 189 | .iter() 190 | .map(|&x| (x - mean).powi(2)) 191 | .fold(A::cast(0), Add::add); 192 | 193 | sum / A::cast(slice.len() - 1) 194 | } 195 | 196 | // TODO Remove the `T` parameter in favor of `S::Output` 197 | /// Returns the bootstrap distributions of the parameters estimated by the 1-sample statistic 198 | /// 199 | /// - Multi-threaded 200 | /// - Time: `O(nresamples)` 201 | /// - Memory: `O(nresamples)` 202 | pub fn bootstrap(&self, nresamples: usize, statistic: S) -> T::Distributions 203 | where 204 | S: Fn(&Sample) -> T + Sync, 205 | T: Tuple + Send, 206 | T::Distributions: Send, 207 | T::Builder: Send, 208 | { 209 | (0..nresamples) 210 | .into_par_iter() 211 | .map_init( 212 | || Resamples::new(self), 213 | |resamples, _| statistic(resamples.next()), 214 | ) 215 | .fold( 216 | || T::Builder::new(0), 217 | |mut sub_distributions, sample| { 218 | sub_distributions.push(sample); 219 | sub_distributions 220 | }, 221 | ) 222 | .reduce( 223 | || T::Builder::new(0), 224 | |mut a, mut b| { 225 | a.extend(&mut b); 226 | a 227 | }, 228 | ) 229 | .complete() 230 | } 231 | 232 | #[cfg(test)] 233 | pub fn iqr(&self) -> A 234 | where 235 | usize: cast::From>, 236 | { 237 | self.percentiles().iqr() 238 | } 239 | 240 | #[cfg(test)] 241 | pub fn median(&self) -> A 242 | where 243 | usize: cast::From>, 244 | { 245 | self.percentiles().median() 246 | } 247 | } 248 | 249 | impl ops::Deref for Sample { 250 | type Target = [A]; 251 | 252 | fn deref(&self) -> &[A] { 253 | &self.0 254 | } 255 | } 256 | -------------------------------------------------------------------------------- /src/value_formatter.rs: -------------------------------------------------------------------------------- 1 | use crate::connection::{Connection, IncomingMessage, OutgoingMessage, Throughput}; 2 | use std::cell::RefCell; 3 | 4 | pub struct ValueFormatter<'a> { 5 | connection: RefCell<&'a mut Connection>, 6 | } 7 | impl ValueFormatter<'_> { 8 | pub fn new(conn: &mut Connection) -> ValueFormatter { 9 | ValueFormatter { 10 | connection: RefCell::new(conn), 11 | } 12 | } 13 | } 14 | impl ValueFormatter<'_> { 15 | pub fn format_value(&self, value: f64) -> String { 16 | self.connection 17 | .borrow_mut() 18 | .send(&OutgoingMessage::FormatValue { value }) 19 | .unwrap(); 20 | match self.connection.borrow_mut().recv().unwrap().unwrap() { 21 | IncomingMessage::FormattedValue { value } => value, 22 | other => panic!("Unexpected message {:?}", other), 23 | } 24 | } 25 | 26 | pub fn format_throughput(&self, throughput: &Throughput, value: f64) -> String { 27 | self.connection 28 | .borrow_mut() 29 | .send(&OutgoingMessage::FormatThroughput { 30 | value, 31 | throughput: throughput.clone(), 32 | }) 33 | .unwrap(); 34 | match self.connection.borrow_mut().recv().unwrap().unwrap() { 35 | IncomingMessage::FormattedValue { value } => value, 36 | other => panic!("Unexpected message {:?}", other), 37 | } 38 | } 39 | 40 | pub fn scale_values(&self, typical_value: f64, values: &mut [f64]) -> String { 41 | self.connection 42 | .borrow_mut() 43 | .send(&OutgoingMessage::ScaleValues { 44 | typical_value, 45 | values, 46 | }) 47 | .unwrap(); 48 | match self.connection.borrow_mut().recv().unwrap().unwrap() { 49 | IncomingMessage::ScaledValues { 50 | scaled_values, 51 | unit, 52 | } => { 53 | values.copy_from_slice(&scaled_values); 54 | unit 55 | } 56 | other => panic!("Unexpected message {:?}", other), 57 | } 58 | } 59 | 60 | // This will be needed when we add the throughput plots. 61 | #[allow(dead_code)] 62 | pub fn scale_throughputs( 63 | &self, 64 | typical_value: f64, 65 | throughput: &Throughput, 66 | values: &mut [f64], 67 | ) -> String { 68 | self.connection 69 | .borrow_mut() 70 | .send(&OutgoingMessage::ScaleThroughputs { 71 | typical_value, 72 | values, 73 | throughput: throughput.clone(), 74 | }) 75 | .unwrap(); 76 | match self.connection.borrow_mut().recv().unwrap().unwrap() { 77 | IncomingMessage::ScaledValues { 78 | scaled_values, 79 | unit, 80 | } => { 81 | values.copy_from_slice(&scaled_values); 82 | unit 83 | } 84 | other => panic!("Unexpected message {:?}", other), 85 | } 86 | } 87 | 88 | pub fn scale_for_machines(&self, values: &mut [f64]) -> String { 89 | self.connection 90 | .borrow_mut() 91 | .send(&OutgoingMessage::ScaleForMachines { values }) 92 | .unwrap(); 93 | match self.connection.borrow_mut().recv().unwrap().unwrap() { 94 | IncomingMessage::ScaledValues { 95 | scaled_values, 96 | unit, 97 | } => { 98 | values.copy_from_slice(&scaled_values); 99 | unit 100 | } 101 | other => panic!("Unexpected message {:?}", other), 102 | } 103 | } 104 | } 105 | impl Drop for ValueFormatter<'_> { 106 | fn drop(&mut self) { 107 | let _ = self 108 | .connection 109 | .borrow_mut() 110 | .send(&OutgoingMessage::Continue); 111 | } 112 | } 113 | --------------------------------------------------------------------------------