├── .gitignore ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── examples └── ctbench-foo.rs └── src ├── ctbench.rs ├── lib.rs ├── macros.rs └── stats.rs /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | *.lock 3 | *.csv 4 | *.txt 5 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | Entries are listed in reverse chronological order. 4 | 5 | # 0.6.x series 6 | 7 | ## 0.6.0 8 | 9 | * Added default `core-hint-black-box` feature, which enables Rust's built-in best-effort black box abstraction for versions >= 1.66. 10 | * Fixed macro issue that required the end user to pull in `clap` as a dependency 11 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = ["Michael Rosenberg "] 3 | name = "dudect-bencher" 4 | version = "0.6.0" 5 | edition = "2021" 6 | 7 | license = "MIT OR Apache-2.0" 8 | 9 | repository = "https://github.com/rozbb/dudect-bencher/" 10 | documentation = "https://docs.rs/dudect-bencher/" 11 | readme = "README.md" 12 | 13 | description = "An implementation of the DudeCT constant-time function tester" 14 | 15 | keywords = ["constant", "constant-time", "crypto", "benchmark"] 16 | 17 | [dependencies] 18 | clap = "2" 19 | ctrlc = "3" 20 | rand = "0.8" 21 | rand_chacha = "0.3" 22 | 23 | [features] 24 | default = ["core-hint-black-box"] 25 | core-hint-black-box = [] 26 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Copyright 2022 Michael Rosenberg 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | 6 | https://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software distributed under the License is 9 | distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 10 | implied. See the License for the specific language governing permissions and limitations under the 11 | License. 12 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2022 Michael Rosenberg 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and 6 | associated documentation files (the "Software"), to deal in the Software without restriction, 7 | including without limitation the rights to use, copy, modify, merge, publish, distribute, 8 | sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all copies or substantial 12 | portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT 15 | NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 16 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES 17 | OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dudect-bencher 2 | [![Version](https://img.shields.io/crates/v/dudect-bencher.svg)](https://crates.io/crates/dudect-bencher) 3 | [![Docs](https://docs.rs/dudect-bencher/badge.svg)](https://docs.rs/dudect-bencher) 4 | 5 | This crate implements the [DudeCT](https://eprint.iacr.org/2016/1123.pdf) statistical methods for testing whether functions are constant-time. It is based loosely off of the [`bencher`](https://github.com/bluss/bencher) benchmarking framework. 6 | 7 | In general, it is not possible to prove that a function always runs in constant time. The purpose of this tool is to find non-constant-timeness when it exists. This is not easy, and it requires the user to think very hard about where the non-constant-timeness might be. 8 | 9 | # Import and features 10 | 11 | To import this crate, put the following line in your `Cargo.toml`: 12 | ```toml 13 | dudect-bencher = "0.6" 14 | ``` 15 | 16 | Feature flags exposed by this crate: 17 | 18 | * `core-hint-black-box` (default) — Enables a new best-effort optimization barrier (`core::hint::black_box`). **This will not compile if you're using a Rust version <1.66.** 19 | 20 | # Usage 21 | 22 | This framework builds a standalone binary. So you must define a `main.rs`, or a file in your `src/bin` directory, or a separate binary crate that pulls in the library you want to test. 23 | 24 | At a high, level you test a function `f` by first defining two sets inputs to `f`, called Right and Left. The way you pick these is highly subjective. You need to already have an idea of what might cause non-constant-time behavior. You then fill in the Left and Right sets such that (you think) `f(l)` and `f(r)` will take a different amount of time to run, on average, where `l` comes from Left and `r` from Right. Finally, you run the benchmarks and label which set is which. 25 | 26 | Here is an example of testing the equality function `v == u` where `v` and `u` are `Vec` of the same length. This is clearly not a constant time function. We define the left distribution to be a set of `(v, u)` where `v == u`, and the right distribution to be the set of `(v, u)` where `v[6] != u[6]`. 27 | 28 | ```rust 29 | use dudect_bencher::{ctbench_main, BenchRng, Class, CtRunner}; 30 | use rand::{Rng, RngCore}; 31 | 32 | // Return a random vector of length len 33 | fn rand_vec(len: usize, rng: &mut BenchRng) -> Vec { 34 | let mut arr = vec![0u8; len]; 35 | rng.fill(arr.as_mut_slice()); 36 | arr 37 | } 38 | 39 | // Benchmark for equality of vectors. This does an early return when it finds an 40 | // inequality, so it should be very much not constant-time 41 | fn vec_eq(runner: &mut CtRunner, rng: &mut BenchRng) { 42 | // Make vectors of size 100 43 | let vlen = 100; 44 | let mut inputs: Vec<(Vec, Vec)> = Vec::new(); 45 | let mut classes = Vec::new(); 46 | 47 | // Make 100,000 random pairs of vectors 48 | for _ in 0..100_000 { 49 | // Flip a coin. If true, make a pair of vectors that are equal to each 50 | // other and put it in the Left distribution 51 | if rng.gen::() { 52 | let v1 = rand_vec(vlen, rng); 53 | let v2 = v1.clone(); 54 | inputs.push((v1, v2)); 55 | classes.push(Class::Left); 56 | } 57 | // Otherwise, make a pair of vectors that differ at the 6th element and 58 | // put it in the right distribution 59 | else { 60 | let v1 = rand_vec(vlen, rng); 61 | let mut v2 = v1.clone(); 62 | v2[5] = 7; 63 | inputs.push((v1, v2)); 64 | classes.push(Class::Right); 65 | } 66 | } 67 | 68 | for (class, (u, v)) in classes.into_iter().zip(inputs.into_iter()) { 69 | // Now time how long it takes to do a vector comparison 70 | runner.run_one(class, || u == v); 71 | } 72 | } 73 | 74 | // Crate the main function to include the bench for vec_eq 75 | ctbench_main!(vec_eq); 76 | ``` 77 | 78 | This is a portion of the example code in [`examples/ctbench-foo.rs`](examples/). To run the example, run 79 | 80 | ```shell 81 | cargo run --release --example ctbench-foo 82 | ``` 83 | 84 | See more command line arguments [below](#command-line-arguments) 85 | 86 | ## Bencher output 87 | 88 | The program output looks like 89 | 90 | ```ignore 91 | bench array_eq ... : n == +0.046M, max t = +61.61472, max tau = +0.28863, (5/tau)^2 = 300 92 | ``` 93 | 94 | It is interpreted as follows. Firstly note that the runtime distributions are cropped at different percentiles and about 100 t-tests are performed. Of these t-tests, the one that produces the largest absolute t-value is printed as `max_t`. The other values printed are 95 | 96 | * `n`, indicating the number of samples used in computing this t-value 97 | * `max_tau`, which is the t-value scaled for the samples size (formally, `max_tau = max_t / sqrt(n)`) 98 | * `(5/tau)^2`, which indicates the number of measurements that would be needed to distinguish the two distributions with t > 5 99 | 100 | t-values greater than 5 are generally considered a good indication that the function is not constant time. t-values less than 5 does not necessarily imply that the function is constant-time, since there may be other input distributions under which the function behaves significantly differently. 101 | 102 | ## Command line arguments 103 | 104 | * `--filter` runs a subset of the benchmarks whose name contains a specific string. Example: 105 | ```shell 106 | cargo run --release --example ctbench-foo -- --filter ar 107 | ``` 108 | will run only the benchmarks with the substring `ar` in it, i.e., `arith`, and not `vec_eq`. 109 | * `--continuous` run a benchmark continuously, collecting more samples as it goes along. Example: 110 | ```shell 111 | cargo run --release --example ctbench-foo -- --continuous vec_eq 112 | ``` 113 | will run the `vec_eq` benchmark continuously. 114 | 115 | * `--out` outputs raw runtimes in CSV format. Example: 116 | ```shell 117 | cargo run --release --example ctbench-foo -- --out data.csv 118 | ``` 119 | will output all the benchmarks in `ctbench-foo.rs` to `data.csv`. 120 | 121 | # License 122 | 123 | Licensed under either of 124 | 125 | * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 126 | * MIT license ([LICENSE-MIT](LICENSE-MIT)) 127 | 128 | at your option. 129 | -------------------------------------------------------------------------------- /examples/ctbench-foo.rs: -------------------------------------------------------------------------------- 1 | use dudect_bencher::{ctbench_main_with_seeds, BenchRng, Class, CtRunner}; 2 | use rand::Rng; 3 | 4 | // Return a random vector of length len 5 | fn rand_vec(len: usize, rng: &mut BenchRng) -> Vec { 6 | let mut arr = vec![0u8; len]; 7 | rng.fill(arr.as_mut_slice()); 8 | arr 9 | } 10 | 11 | // Benchmark for some random arithmetic operations. This should produce small t-values 12 | fn arith(runner: &mut CtRunner, rng: &mut BenchRng) { 13 | let mut inputs = Vec::new(); 14 | let mut classes = Vec::new(); 15 | 16 | // Make 100,000 inputs on each run 17 | for _ in 0..100_000 { 18 | inputs.push(rng.gen::()); 19 | // Randomly pick which distribution this example belongs to 20 | if rng.gen::() { 21 | classes.push(Class::Left); 22 | } else { 23 | classes.push(Class::Right); 24 | } 25 | } 26 | 27 | for (u, class) in inputs.into_iter().zip(classes.into_iter()) { 28 | // Time some random arithmetic operations 29 | runner.run_one(class, || ((u + 10) / 6) << 5); 30 | } 31 | } 32 | 33 | // Benchmark for equality of vectors. This does an early return when it finds an inequality, so it 34 | // should be very much not constant-time 35 | fn vec_eq(runner: &mut CtRunner, rng: &mut BenchRng) { 36 | // Make vectors of size 100 37 | let vlen = 100; 38 | let mut inputs: Vec<(Vec, Vec)> = Vec::new(); 39 | let mut classes = Vec::new(); 40 | 41 | // Make 100,000 random pairs of vectors 42 | for _ in 0..100_000 { 43 | // Flip a coin. If true, make a pair of vectors that are equal to each other and put it 44 | // in the Left distribution 45 | if rng.gen::() { 46 | let v1 = rand_vec(vlen, rng); 47 | let v2 = v1.clone(); 48 | inputs.push((v1, v2)); 49 | classes.push(Class::Left); 50 | } 51 | // Otherwise, make a pair of vectors that differ at the 6th element and put it in the 52 | // right distribution 53 | else { 54 | let v1 = rand_vec(vlen, rng); 55 | let mut v2 = v1.clone(); 56 | v2[5] = 7; 57 | inputs.push((v1, v2)); 58 | classes.push(Class::Right); 59 | } 60 | } 61 | 62 | for (class, (u, v)) in classes.into_iter().zip(inputs.into_iter()) { 63 | // Now time how long it takes to do a vector comparison 64 | runner.run_one(class, || u == v); 65 | } 66 | } 67 | 68 | // Expand the main function to include benches for arith and vec_eq 69 | ctbench_main_with_seeds!((arith, Some(0x6b6c816d)), (vec_eq, None)); 70 | // Alternatively, for no explicit seeds, you can use 71 | // ctbench_main!(arith, vec_eq); 72 | -------------------------------------------------------------------------------- /src/ctbench.rs: -------------------------------------------------------------------------------- 1 | use crate::stats; 2 | 3 | use std::{ 4 | fs::{File, OpenOptions}, 5 | io::{self, Write}, 6 | iter::repeat, 7 | path::PathBuf, 8 | process, 9 | sync::{ 10 | atomic::{self, AtomicBool}, 11 | Arc, 12 | }, 13 | time::Instant, 14 | }; 15 | 16 | use ctrlc; 17 | use rand::{Rng, SeedableRng}; 18 | use rand_chacha::ChaChaRng; 19 | 20 | /// Just a static str representing the name of a function 21 | #[derive(Copy, Clone)] 22 | pub struct BenchName(pub &'static str); 23 | 24 | impl BenchName { 25 | fn padded(&self, column_count: usize) -> String { 26 | let mut name = self.0.to_string(); 27 | let fill = column_count.saturating_sub(name.len()); 28 | let pad = repeat(" ").take(fill).collect::(); 29 | name.push_str(&pad); 30 | 31 | name 32 | } 33 | } 34 | 35 | /// A random number generator implementing [`rand::SeedableRng`]. This is given to every 36 | /// benchmarking function to use as a source of randomness. 37 | pub type BenchRng = ChaChaRng; 38 | 39 | /// A function that is to be benchmarked. This crate only supports statically-defined functions. 40 | pub type BenchFn = fn(&mut CtRunner, &mut BenchRng); 41 | 42 | // TODO: Consider giving this a lifetime so we don't have to copy names and vecs into it 43 | #[derive(Clone)] 44 | enum BenchEvent { 45 | BContStart, 46 | BBegin(Vec), 47 | BWait(BenchName), 48 | BResult(MonitorMsg), 49 | BSeed(u64, BenchName), 50 | } 51 | 52 | type MonitorMsg = (BenchName, stats::CtSummary); 53 | 54 | /// CtBencher is the primary interface for benchmarking. All setup for function inputs should be 55 | /// doen within the closure supplied to the `iter` method. 56 | struct CtBencher { 57 | samples: (Vec, Vec), 58 | ctx: Option, 59 | file_out: Option, 60 | rng: BenchRng, 61 | } 62 | 63 | impl CtBencher { 64 | /// Creates and returns a new empty `CtBencher` whose `BenchRng` is zero-seeded 65 | pub fn new() -> CtBencher { 66 | CtBencher { 67 | samples: (Vec::new(), Vec::new()), 68 | ctx: None, 69 | file_out: None, 70 | rng: BenchRng::seed_from_u64(0u64), 71 | } 72 | } 73 | 74 | /// Runs the bench function and returns the CtSummary 75 | fn go(&mut self, f: BenchFn) -> stats::CtSummary { 76 | // This populates self.samples 77 | let mut runner = CtRunner::default(); 78 | f(&mut runner, &mut self.rng); 79 | self.samples = runner.runtimes; 80 | 81 | // Replace the old CtCtx with an updated one 82 | let old_self = ::std::mem::replace(self, CtBencher::new()); 83 | let (summ, new_ctx) = stats::update_ct_stats(old_self.ctx, &old_self.samples); 84 | 85 | // Copy the old stuff back in 86 | self.samples = old_self.samples; 87 | self.file_out = old_self.file_out; 88 | self.ctx = Some(new_ctx); 89 | self.rng = old_self.rng; 90 | 91 | summ 92 | } 93 | 94 | /// Returns a random seed 95 | fn rand_seed() -> u64 { 96 | rand::thread_rng().gen() 97 | } 98 | 99 | /// Reseeds the internal RNG with the given seed 100 | pub fn seed_with(&mut self, seed: u64) { 101 | self.rng = BenchRng::seed_from_u64(seed); 102 | } 103 | 104 | /// Clears out all sample and contextual data 105 | fn clear_data(&mut self) { 106 | self.samples = (Vec::new(), Vec::new()); 107 | self.ctx = None; 108 | } 109 | } 110 | 111 | /// Represents a single benchmark to conduct 112 | pub struct BenchMetadata { 113 | pub name: BenchName, 114 | pub seed: Option, 115 | pub benchfn: BenchFn, 116 | } 117 | 118 | /// Benchmarking options. 119 | /// 120 | /// When `continuous` is set, it will continuously set the first (alphabetically) of the benchmarks 121 | /// after they have been optionally filtered. 122 | /// 123 | /// When `filter` is set and `continuous` is not set, only benchmarks whose names contain the 124 | /// filter string as a substring will be executed. 125 | /// 126 | /// `file_out` is optionally the filename where CSV output of raw runtime data should be written 127 | #[derive(Default)] 128 | pub struct BenchOpts { 129 | pub continuous: bool, 130 | pub filter: Option, 131 | pub file_out: Option, 132 | } 133 | 134 | #[derive(Default)] 135 | struct ConsoleBenchState { 136 | // Number of columns to fill when aligning names 137 | max_name_len: usize, 138 | } 139 | 140 | impl ConsoleBenchState { 141 | fn write_plain(&mut self, s: &str) -> io::Result<()> { 142 | let mut stdout = io::stdout(); 143 | stdout.write_all(s.as_bytes())?; 144 | stdout.flush() 145 | } 146 | 147 | fn write_bench_start(&mut self, name: &BenchName) -> io::Result<()> { 148 | let name = name.padded(self.max_name_len); 149 | self.write_plain(&format!("bench {} ... ", name)) 150 | } 151 | 152 | fn write_seed(&mut self, seed: u64, name: &BenchName) -> io::Result<()> { 153 | let name = name.padded(self.max_name_len); 154 | self.write_plain(&format!("bench {} seeded with 0x{:016x}\n", name, seed)) 155 | } 156 | 157 | fn write_run_start(&mut self, len: usize) -> io::Result<()> { 158 | let noun = if len != 1 { "benches" } else { "bench" }; 159 | self.write_plain(&format!("\nrunning {} {}\n", len, noun)) 160 | } 161 | 162 | fn write_continuous_start(&mut self) -> io::Result<()> { 163 | self.write_plain("running 1 benchmark continuously\n") 164 | } 165 | 166 | fn write_result(&mut self, summ: &stats::CtSummary) -> io::Result<()> { 167 | self.write_plain(&format!(": {}\n", summ.fmt())) 168 | } 169 | 170 | fn write_run_finish(&mut self) -> io::Result<()> { 171 | self.write_plain("\ndudect benches complete\n\n") 172 | } 173 | } 174 | 175 | /// Runs the given benches under the given options and prints the output to the console 176 | pub fn run_benches_console(opts: BenchOpts, benches: Vec) -> io::Result<()> { 177 | // TODO: Consider making this do screen updates in continuous mode 178 | // TODO: Consider making this run in its own thread 179 | fn callback(event: &BenchEvent, st: &mut ConsoleBenchState) -> io::Result<()> { 180 | match (*event).clone() { 181 | BenchEvent::BContStart => st.write_continuous_start(), 182 | BenchEvent::BBegin(ref filtered_benches) => st.write_run_start(filtered_benches.len()), 183 | BenchEvent::BWait(ref b) => st.write_bench_start(b), 184 | BenchEvent::BResult(msg) => { 185 | let (_, summ) = msg; 186 | st.write_result(&summ) 187 | } 188 | BenchEvent::BSeed(seed, ref name) => st.write_seed(seed, name), 189 | } 190 | } 191 | 192 | let mut st = ConsoleBenchState::default(); 193 | st.max_name_len = benches.iter().map(|t| t.name.0.len()).max().unwrap_or(0); 194 | 195 | run_benches(&opts, benches, |x| callback(&x, &mut st))?; 196 | st.write_run_finish() 197 | } 198 | 199 | /// Returns an atomic bool that indicates whether Ctrl-C was pressed 200 | fn setup_kill_bit() -> Arc { 201 | let x = Arc::new(AtomicBool::new(false)); 202 | let y = x.clone(); 203 | 204 | ctrlc::set_handler(move || y.store(true, atomic::Ordering::SeqCst)) 205 | .expect("Error setting Ctrl-C handler"); 206 | 207 | x 208 | } 209 | 210 | fn run_benches(opts: &BenchOpts, benches: Vec, mut callback: F) -> io::Result<()> 211 | where 212 | F: FnMut(BenchEvent) -> io::Result<()>, 213 | { 214 | use self::BenchEvent::*; 215 | 216 | let filter = &opts.filter; 217 | let filtered_benches = filter_benches(filter, benches); 218 | let filtered_names = filtered_benches.iter().map(|b| b.name).collect(); 219 | 220 | // Write the CSV header line to the file if the file is defined 221 | let mut file_out = opts.file_out.as_ref().map(|filename| { 222 | OpenOptions::new() 223 | .write(true) 224 | .truncate(true) 225 | .create(true) 226 | .open(filename) 227 | .expect(&*format!( 228 | "Could not open file '{:?}' for writing", 229 | filename 230 | )) 231 | }); 232 | file_out.as_mut().map(|f| { 233 | f.write(b"benchname,class,runtime") 234 | .expect("Error writing CSV header to file") 235 | }); 236 | 237 | // Make a bencher with the optional file output specified 238 | let mut cb: CtBencher = { 239 | let mut d = CtBencher::new(); 240 | d.file_out = file_out; 241 | d 242 | }; 243 | 244 | if opts.continuous { 245 | callback(BContStart)?; 246 | 247 | if filtered_benches.is_empty() { 248 | match *filter { 249 | Some(ref f) => panic!("No benchmark matching '{}' was found", f), 250 | None => return Ok(()), 251 | } 252 | } 253 | 254 | // Get a bit that tells us when we've been killed 255 | let kill_bit = setup_kill_bit(); 256 | 257 | // Continuously run the first matched bench we see 258 | let mut filtered_benches = filtered_benches; 259 | let bench = filtered_benches.remove(0); 260 | 261 | // If a seed was specified for this bench, use it. Otherwise, use a random seed 262 | let seed = bench.seed.unwrap_or_else(CtBencher::rand_seed); 263 | cb.seed_with(seed); 264 | callback(BSeed(seed, bench.name))?; 265 | 266 | loop { 267 | callback(BWait(bench.name))?; 268 | let msg = run_bench_with_bencher(&bench.name, bench.benchfn, &mut cb); 269 | callback(BResult(msg))?; 270 | 271 | // Check if the program has been killed. If so, exit 272 | if kill_bit.load(atomic::Ordering::SeqCst) { 273 | process::exit(0); 274 | } 275 | } 276 | } else { 277 | callback(BBegin(filtered_names))?; 278 | 279 | // Run different benches 280 | for bench in filtered_benches { 281 | // Clear the data out from the previous bench, but keep the CSV file open 282 | cb.clear_data(); 283 | 284 | // If a seed was specified for this bench, use it. Otherwise, use a random seed 285 | let seed = bench.seed.unwrap_or_else(CtBencher::rand_seed); 286 | cb.seed_with(seed); 287 | callback(BSeed(seed, bench.name))?; 288 | 289 | callback(BWait(bench.name))?; 290 | let msg = run_bench_with_bencher(&bench.name, bench.benchfn, &mut cb); 291 | callback(BResult(msg))?; 292 | } 293 | Ok(()) 294 | } 295 | } 296 | 297 | fn run_bench_with_bencher(name: &BenchName, benchfn: BenchFn, cb: &mut CtBencher) -> MonitorMsg { 298 | let summ = cb.go(benchfn); 299 | 300 | // Write the runtime samples out 301 | let samples_iter = cb.samples.0.iter().zip(cb.samples.1.iter()); 302 | if let Some(f) = cb.file_out.as_mut() { 303 | for (x, y) in samples_iter { 304 | write!(f, "\n{},0,{}", name.0, x).expect("Error writing data to file"); 305 | write!(f, "\n{},0,{}", name.0, y).expect("Error writing data to file"); 306 | } 307 | }; 308 | 309 | (*name, summ) 310 | } 311 | 312 | fn filter_benches(filter: &Option, bs: Vec) -> Vec { 313 | let mut filtered = bs; 314 | 315 | // Remove benches that don't match the filter 316 | filtered = match *filter { 317 | None => filtered, 318 | Some(ref filter) => filtered 319 | .into_iter() 320 | .filter(|b| b.name.0.contains(&filter[..])) 321 | .collect(), 322 | }; 323 | 324 | // Sort them alphabetically 325 | filtered.sort_by(|b1, b2| b1.name.0.cmp(&b2.name.0)); 326 | 327 | filtered 328 | } 329 | 330 | // NOTE: We don't have a proper black box in stable Rust. This is a workaround implementation, 331 | // that may have a too big performance overhead, depending on operation, or it may fail to 332 | // properly avoid having code optimized out. It is good enough that it is used by default. 333 | // 334 | // A function that is opaque to the optimizer, to allow benchmarks to pretend to use outputs to 335 | // assist in avoiding dead-code elimination. 336 | #[cfg(not(feature = "core-hint-black-box"))] 337 | fn black_box(dummy: T) -> T { 338 | unsafe { 339 | let ret = ::std::ptr::read_volatile(&dummy); 340 | ::std::mem::forget(dummy); 341 | ret 342 | } 343 | } 344 | 345 | #[cfg(feature = "core-hint-black-box")] 346 | #[inline] 347 | fn black_box(dummy: T) -> T { 348 | ::core::hint::black_box(dummy) 349 | } 350 | 351 | /// Specifies the distribution that a particular run belongs to 352 | #[derive(Copy, Clone)] 353 | pub enum Class { 354 | Left, 355 | Right, 356 | } 357 | 358 | /// Used for timing single operations at a time 359 | #[derive(Default)] 360 | pub struct CtRunner { 361 | // Runtimes of left and right distributions in nanoseconds 362 | runtimes: (Vec, Vec), 363 | } 364 | 365 | impl CtRunner { 366 | /// Runs and times a single operation whose constant-timeness is in question 367 | pub fn run_one(&mut self, class: Class, f: F) 368 | where 369 | F: Fn() -> T, 370 | { 371 | let start = Instant::now(); 372 | black_box(f()); 373 | let end = Instant::now(); 374 | 375 | let runtime = { 376 | let dur = end.duration_since(start); 377 | dur.as_secs() * 1_000_000_000 + u64::from(dur.subsec_nanos()) 378 | }; 379 | 380 | match class { 381 | Class::Left => self.runtimes.0.push(runtime), 382 | Class::Right => self.runtimes.1.push(runtime), 383 | } 384 | } 385 | } 386 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2012-2016 The Rust Project Developers. See the COPYRIGHT 2 | // file at the top-level directory of this distribution and at 3 | // http://rust-lang.org/COPYRIGHT. 4 | // 5 | // Licensed under the Apache License, Version 2.0 or the MIT license 7 | // , at your 8 | // option. This file may not be copied, modified, or distributed 9 | // except according to those terms. 10 | 11 | #![doc = include_str!("../README.md")] 12 | 13 | // TODO: More comments 14 | // TODO: Do "higher order preprocessing" from the paper 15 | 16 | pub mod ctbench; 17 | #[doc(hidden)] 18 | pub mod macros; 19 | mod stats; 20 | 21 | // Re-export the rand dependency 22 | pub use rand; 23 | 24 | #[doc(inline)] 25 | pub use ctbench::{BenchRng, Class, CtRunner}; 26 | -------------------------------------------------------------------------------- /src/macros.rs: -------------------------------------------------------------------------------- 1 | /// Defines a `fn main()` that will run all benchmarks defined by listed functions `$function` and 2 | /// their associated seeds (if present). Seeds are represented as `Option`. If `None` is 3 | /// given, a random seed will be used. The seeds are used to seed the 4 | /// [`BenchRng`](crate::ctbench::BenchRng) that's passed to each function. 5 | /// 6 | /// ``` 7 | /// use dudect_bencher::{ctbench_main_with_seeds, rand::Rng, BenchRng, Class, CtRunner}; 8 | /// 9 | /// fn foo(runner: &mut CtRunner, rng: &mut BenchRng) { 10 | /// println!("first u64 is {}", rng.gen::()); 11 | /// 12 | /// // Run something so we don't get a panic 13 | /// runner.run_one(Class::Left, || 0); 14 | /// runner.run_one(Class::Right, || 0); 15 | /// } 16 | /// 17 | /// fn bar(runner: &mut CtRunner, rng: &mut BenchRng) { 18 | /// println!("first u64 is {}", rng.gen::()); 19 | /// 20 | /// // Run something so we don't get a panic 21 | /// runner.run_one(Class::Left, || 0); 22 | /// runner.run_one(Class::Right, || 0); 23 | /// } 24 | /// 25 | /// ctbench_main_with_seeds!( 26 | /// (foo, None), 27 | /// (bar, Some(0xdeadbeef)) 28 | /// ); 29 | /// ``` 30 | #[macro_export] 31 | macro_rules! ctbench_main_with_seeds { 32 | ($(($function:path, $seed:expr)),+) => { 33 | use $crate::macros::__macro_internal::{clap::App, PathBuf}; 34 | use $crate::ctbench::{run_benches_console, BenchName, BenchMetadata, BenchOpts}; 35 | fn main() { 36 | let mut benches = Vec::new(); 37 | $( 38 | benches.push(BenchMetadata { 39 | name: BenchName(stringify!($function)), 40 | seed: $seed, 41 | benchfn: $function, 42 | }); 43 | )+ 44 | let matches = App::new("dudect-bencher") 45 | .arg_from_usage( 46 | "--filter [BENCH] \ 47 | 'Only run the benchmarks whose name contains BENCH'" 48 | ) 49 | .arg_from_usage( 50 | "--continuous [BENCH] \ 51 | 'Runs a continuous benchmark on the first bench matching BENCH'" 52 | ) 53 | .arg_from_usage( 54 | "--out [FILE] \ 55 | 'Appends raw benchmarking data in CSV format to FILE'" 56 | ) 57 | .get_matches(); 58 | 59 | let mut test_opts = BenchOpts::default(); 60 | test_opts.filter = matches 61 | .value_of("continuous") 62 | .or(matches.value_of("filter")) 63 | .map(|s| s.to_string()); 64 | test_opts.continuous = matches.is_present("continuous"); 65 | test_opts.file_out = matches.value_of("out").map(PathBuf::from); 66 | 67 | run_benches_console(test_opts, benches).unwrap(); 68 | } 69 | } 70 | } 71 | 72 | /// Defines a `fn main()` that will run all benchmarks defined by listed functions `$function`. The 73 | /// [`BenchRng`](crate::ctbench::BenchRng)s given to each function are randomly seeded. Exmaple 74 | /// usage: 75 | /// 76 | /// ``` 77 | /// use dudect_bencher::{ctbench_main, rand::{Rng, RngCore}, BenchRng, Class, CtRunner}; 78 | /// 79 | /// // Return a random vector of length len 80 | /// fn rand_vec(len: usize, rng: &mut BenchRng) -> Vec { 81 | /// let mut arr = vec![0u8; len]; 82 | /// rng.fill_bytes(&mut arr); 83 | /// arr 84 | /// } 85 | /// 86 | /// // Benchmark for some random arithmetic operations. This should produce small t-values 87 | /// fn arith(runner: &mut CtRunner, rng: &mut BenchRng) { 88 | /// let mut inputs = Vec::new(); 89 | /// let mut classes = Vec::new(); 90 | /// 91 | /// // Make 100,000 inputs on each run 92 | /// for _ in 0..100_000 { 93 | /// inputs.push(rng.gen::()); 94 | /// // Randomly pick which distribution this example belongs to 95 | /// if rng.gen::() { 96 | /// classes.push(Class::Left); 97 | /// } 98 | /// else { 99 | /// classes.push(Class::Right); 100 | /// } 101 | /// } 102 | /// 103 | /// for (u, class) in inputs.into_iter().zip(classes.into_iter()) { 104 | /// // Time some random arithmetic operations 105 | /// runner.run_one(class, || ((u + 10) / 6) << 5); 106 | /// } 107 | /// } 108 | /// 109 | /// // Benchmark for equality of vectors. This does an early return when it finds an inequality, 110 | /// // so it should be very much not constant-time 111 | /// fn vec_eq(runner: &mut CtRunner, rng: &mut BenchRng) { 112 | /// // Make vectors of size 100 113 | /// let vlen = 100; 114 | /// let mut inputs: Vec<(Vec, Vec)> = Vec::new(); 115 | /// let mut classes = Vec::new(); 116 | /// 117 | /// // Make 100,000 random pairs of vectors 118 | /// for _ in 0..100_000 { 119 | /// // Flip a coin. If true, make a pair of vectors that are equal to each other and put 120 | /// // it in the Left distribution 121 | /// if rng.gen::() { 122 | /// let v1 = rand_vec(vlen, rng); 123 | /// let v2 = v1.clone(); 124 | /// inputs.push((v1, v2)); 125 | /// classes.push(Class::Left); 126 | /// } 127 | /// // Otherwise, make a pair of vectors that differ at the 6th element and put it in the 128 | /// // right distribution 129 | /// else { 130 | /// let v1 = rand_vec(vlen, rng); 131 | /// let mut v2 = v1.clone(); 132 | /// v2[5] = 7; 133 | /// inputs.push((v1, v2)); 134 | /// classes.push(Class::Right); 135 | /// } 136 | /// } 137 | /// 138 | /// for (class, (u, v)) in classes.into_iter().zip(inputs.into_iter()) { 139 | /// // Now time how long it takes to do a vector comparison 140 | /// runner.run_one(class, || u == v); 141 | /// } 142 | /// } 143 | /// 144 | /// // Expand the main function to include benches for arith and vec_eq. Use random RNG seeds 145 | /// ctbench_main!(arith, vec_eq); 146 | /// ``` 147 | #[macro_export] 148 | macro_rules! ctbench_main { 149 | ($($function:path),+) => { 150 | use $crate::macros::__macro_internal::Option; 151 | $crate::ctbench_main_with_seeds!($(($function, Option::None)),+); 152 | } 153 | } 154 | 155 | #[doc(hidden)] 156 | pub mod __macro_internal { 157 | pub use ::clap; 158 | pub use ::std::{option::Option, path::PathBuf}; 159 | } 160 | -------------------------------------------------------------------------------- /src/stats.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2012 The Rust Project Developers. See the COPYRIGHT 2 | // file at the top-level directory of this distribution and at 3 | // http://rust-lang.org/COPYRIGHT. 4 | // 5 | // Licensed under the Apache License, Version 2.0 or the MIT license 7 | // , at your 8 | // option. This file may not be copied, modified, or distributed 9 | // except according to those terms. 10 | 11 | use std::cmp; 12 | 13 | #[derive(Copy, Clone, Debug, Default, PartialEq)] 14 | pub struct CtSummary { 15 | pub max_t: f64, 16 | pub max_tau: f64, 17 | pub sample_size: usize, 18 | } 19 | 20 | impl CtSummary { 21 | pub fn fmt(&self) -> String { 22 | let &CtSummary { 23 | max_t, 24 | max_tau, 25 | sample_size, 26 | } = self; 27 | format!( 28 | "n == {:+0.3}M, max t = {:+0.5}, max tau = {:+0.5}, (5/tau)^2 = {}", 29 | (sample_size as f64) / 1_000_000f64, 30 | max_t, 31 | max_tau, 32 | (5f64 / max_tau).powi(2) as usize 33 | ) 34 | } 35 | } 36 | 37 | #[derive(Copy, Clone, Debug, Default)] 38 | struct CtTest { 39 | means: (f64, f64), 40 | sq_diffs: (f64, f64), 41 | sizes: (usize, usize), 42 | } 43 | 44 | #[derive(Default)] 45 | pub struct CtCtx { 46 | tests: Vec, 47 | percentiles: Vec, 48 | } 49 | 50 | // NaNs are smaller than everything 51 | fn local_cmp(x: f64, y: f64) -> cmp::Ordering { 52 | use std::cmp::Ordering::{Equal, Greater, Less}; 53 | if y.is_nan() { 54 | Greater 55 | } else if x.is_nan() || x < y { 56 | Less 57 | } else if x == y { 58 | Equal 59 | } else { 60 | Greater 61 | } 62 | } 63 | 64 | /// Helper function: extract a value representing the `pct` percentile of a sorted sample-set, 65 | /// using linear interpolation. If samples are not sorted, return nonsensical value. 66 | fn percentile_of_sorted(sorted_samples: &[f64], pct: f64) -> f64 { 67 | assert!(!sorted_samples.is_empty()); 68 | if sorted_samples.len() == 1 { 69 | return sorted_samples[0]; 70 | } 71 | let zero = 0f64; 72 | assert!(zero <= pct); 73 | let hundred = 100f64; 74 | assert!(pct <= hundred); 75 | let length = (sorted_samples.len() - 1) as f64; 76 | let rank = (pct / hundred) * length; 77 | let lrank = rank.floor(); 78 | let d = rank - lrank; 79 | let n = lrank as usize; 80 | let lo = sorted_samples[n]; 81 | let hi = sorted_samples[n + 1]; 82 | lo + (hi - lo) * d 83 | } 84 | 85 | /// Return the percentiles at f(1), f(2), ..., f(100) of the runtime distribution, where 86 | /// `f(k) = 1 - 0.5^(10k / 100)` 87 | pub fn prepare_percentiles(durations: &[u64]) -> Vec { 88 | let sorted: Vec = { 89 | let mut v = durations.to_vec(); 90 | v.sort(); 91 | v.into_iter().map(|d| d as f64).collect() 92 | }; 93 | 94 | // Collect all the percentile values 95 | (0..100) 96 | .map(|i| { 97 | let pct = { 98 | let exp = f64::from(10 * (i + 1)) / 100f64; 99 | 1f64 - 0.5f64.powf(exp) 100 | }; 101 | percentile_of_sorted(&sorted, 100f64 * pct) 102 | }) 103 | .collect() 104 | } 105 | 106 | pub fn update_ct_stats( 107 | ctx: Option, 108 | &(ref left_samples, ref right_samples): &(Vec, Vec), 109 | ) -> (CtSummary, CtCtx) { 110 | // Only construct the context (that is, percentiles and test structs) on the first run 111 | let (mut tests, percentiles) = match ctx { 112 | Some(c) => (c.tests, c.percentiles), 113 | None => { 114 | let all_samples = { 115 | let mut v = left_samples.clone(); 116 | v.extend_from_slice(&*right_samples); 117 | v 118 | }; 119 | let pcts = prepare_percentiles(&*all_samples); 120 | let tests = vec![CtTest::default(); 101]; 121 | 122 | (tests, pcts) 123 | } 124 | }; 125 | 126 | let left_samples: Vec = left_samples.iter().map(|&n| n as f64).collect(); 127 | let right_samples: Vec = right_samples.iter().map(|&n| n as f64).collect(); 128 | 129 | for &left_sample in left_samples.iter() { 130 | update_test_left(&mut tests[0], left_sample); 131 | } 132 | for &right_sample in right_samples.iter() { 133 | update_test_right(&mut tests[0], right_sample); 134 | } 135 | 136 | for (test, &pct) in tests.iter_mut().skip(1).zip(percentiles.iter()) { 137 | let left_cropped = left_samples.iter().filter(|&&x| x < pct); 138 | let right_cropped = right_samples.iter().filter(|&&x| x < pct); 139 | 140 | for &left_sample in left_cropped { 141 | update_test_left(test, left_sample); 142 | } 143 | for &right_sample in right_cropped { 144 | update_test_right(test, right_sample); 145 | } 146 | } 147 | 148 | let (max_t, max_tau, sample_size) = { 149 | // Get the test with the maximum t 150 | let max_test = tests 151 | .iter() 152 | .max_by(|&x, &y| local_cmp(compute_t(x).abs(), compute_t(y).abs())) 153 | .unwrap(); 154 | let sample_size = max_test.sizes.0 + max_test.sizes.1; 155 | let max_t = compute_t(&max_test); 156 | let max_tau = max_t / (sample_size as f64).sqrt(); 157 | 158 | (max_t, max_tau, sample_size) 159 | }; 160 | 161 | let new_ctx = CtCtx { tests, percentiles }; 162 | let summ = CtSummary { 163 | max_t, 164 | max_tau, 165 | sample_size, 166 | }; 167 | 168 | (summ, new_ctx) 169 | } 170 | 171 | fn compute_t(test: &CtTest) -> f64 { 172 | let &CtTest { 173 | means, 174 | sq_diffs, 175 | sizes, 176 | } = test; 177 | let num = means.0 - means.1; 178 | let n0 = sizes.0 as f64; 179 | let n1 = sizes.1 as f64; 180 | let var0 = sq_diffs.0 / (n0 - 1f64); 181 | let var1 = sq_diffs.1 / (n1 - 1f64); 182 | let den = (var0 / n0 + var1 / n1).sqrt(); 183 | 184 | num / den 185 | } 186 | 187 | fn update_test_left(test: &mut CtTest, datum: f64) { 188 | test.sizes.0 += 1; 189 | let diff = datum - test.means.0; 190 | test.means.0 += diff / (test.sizes.0 as f64); 191 | test.sq_diffs.0 += diff * (datum - test.means.0); 192 | } 193 | 194 | fn update_test_right(test: &mut CtTest, datum: f64) { 195 | test.sizes.1 += 1; 196 | let diff = datum - test.means.1; 197 | test.means.1 += diff / (test.sizes.1 as f64); 198 | test.sq_diffs.1 += diff * (datum - test.means.1); 199 | } 200 | --------------------------------------------------------------------------------