├── .gitignore ├── .github ├── codecov.yml └── workflows │ ├── msrv.yml │ ├── os-check.yml │ ├── test.yml │ ├── coverage.yml │ ├── minimal.yml │ └── style.yml ├── Cargo.toml ├── benches ├── arc_mutex_std.rs └── arc_rwlock_std.rs ├── README.md ├── LICENSE-APACHE └── src └── lib.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /.github/codecov.yml: -------------------------------------------------------------------------------- 1 | # ref: https://docs.codecov.com/docs/codecovyml-reference 2 | coverage: 3 | # Hold ourselves to a high bar 4 | range: 85..100 5 | round: down 6 | precision: 1 7 | status: 8 | # ref: https://docs.codecov.com/docs/commit-status 9 | project: 10 | default: 11 | # Avoid false negatives 12 | threshold: 1% 13 | 14 | # Test files aren't important for coverage 15 | ignore: 16 | - "tests" 17 | 18 | # Make comments less noisy 19 | comment: 20 | layout: "files" 21 | require_changes: yes 22 | -------------------------------------------------------------------------------- /.github/workflows/msrv.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: [main] 4 | pull_request: 5 | name: Minimum Supported Rust Version 6 | jobs: 7 | check: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions-rs/toolchain@v1 11 | with: 12 | profile: minimal 13 | toolchain: 1.56.1 # 2021 edition requires 1.56 14 | override: true 15 | - uses: actions/checkout@v3 16 | with: 17 | submodules: true 18 | - name: cargo +1.56.1 check 19 | uses: actions-rs/cargo@v1 20 | with: 21 | command: check 22 | -------------------------------------------------------------------------------- /.github/workflows/os-check.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: [main] 4 | pull_request: 5 | name: os check 6 | jobs: 7 | os-check: 8 | runs-on: ${{ matrix.os }} 9 | name: ${{ matrix.os }} 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | os: [macos-latest, windows-latest] 14 | steps: 15 | - uses: actions-rs/toolchain@v1 16 | with: 17 | profile: minimal 18 | toolchain: stable 19 | - uses: actions/checkout@v3 20 | - name: cargo test 21 | uses: actions-rs/cargo@v1 22 | with: 23 | command: test 24 | args: --all-features --all-targets 25 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: [main] 4 | pull_request: 5 | name: cargo test 6 | jobs: 7 | test: 8 | runs-on: ubuntu-latest 9 | name: ubuntu / ${{ matrix.toolchain }} 10 | strategy: 11 | matrix: 12 | toolchain: [stable, beta, nightly] 13 | steps: 14 | - uses: actions-rs/toolchain@v1 15 | with: 16 | profile: minimal 17 | toolchain: ${{ matrix.toolchain }} 18 | - uses: actions/checkout@v3 19 | with: 20 | submodules: true 21 | - name: cargo test 22 | uses: actions-rs/cargo@v1 23 | with: 24 | command: test 25 | args: --all-features --all-targets 26 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: [main] 4 | pull_request: 5 | name: coverage 6 | jobs: 7 | test: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v3 11 | with: 12 | submodules: true 13 | - uses: actions-rs/toolchain@v1 14 | with: 15 | profile: minimal 16 | toolchain: stable 17 | override: true 18 | components: llvm-tools-preview 19 | - name: Install cargo-llvm-cov 20 | uses: taiki-e/install-action@cargo-llvm-cov 21 | - name: Generate code coverage 22 | run: cargo llvm-cov --all-features --lcov --output-path lcov.info 23 | - name: Upload to codecov.io 24 | uses: codecov/codecov-action@v2 25 | with: 26 | fail_ci_if_error: true 27 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "bustle" 3 | version = "0.5.1" 4 | authors = ["Jon Gjengset "] 5 | edition = "2021" 6 | license = "Apache-2.0" 7 | 8 | readme = "README.md" 9 | description = "Benchmarking harness for concurrent key-value collections" 10 | repository = "https://github.com/jonhoo/bustle.git" 11 | 12 | keywords = ["concurrent","benchmark","harness"] 13 | categories = ["concurrency", "development-tools::profiling", "development-tools::testing"] 14 | 15 | [dependencies] 16 | rand = { version = "0.8", features = ["small_rng"] } 17 | scopeguard = "1" 18 | tracing = { version = "0.1", features = ["std"], default-features = false } 19 | 20 | [dev-dependencies] 21 | tracing-subscriber = "0.3.8" 22 | num_cpus = "1" 23 | 24 | [[bench]] 25 | name = "arc_mutex_std" 26 | harness = false 27 | 28 | [[bench]] 29 | name = "arc_rwlock_std" 30 | harness = false 31 | -------------------------------------------------------------------------------- /.github/workflows/minimal.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: [main] 4 | pull_request: 5 | name: With dependencies at minimal versions 6 | jobs: 7 | test: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions-rs/toolchain@v1 11 | with: 12 | profile: minimal 13 | toolchain: nightly 14 | - uses: actions-rs/toolchain@v1 15 | with: 16 | profile: minimal 17 | toolchain: stable 18 | - uses: actions/checkout@v3 19 | with: 20 | submodules: true 21 | - name: cargo update -Zminimal-versions 22 | uses: actions-rs/cargo@v1 23 | with: 24 | command: update 25 | toolchain: nightly 26 | args: -Zminimal-versions 27 | - name: cargo test 28 | uses: actions-rs/cargo@v1 29 | with: 30 | command: test 31 | args: --all-features --all-targets 32 | -------------------------------------------------------------------------------- /.github/workflows/style.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: [main] 4 | pull_request: 5 | name: lint 6 | jobs: 7 | style: 8 | runs-on: ubuntu-latest 9 | name: ${{ matrix.toolchain }} 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | toolchain: [stable, beta] 14 | steps: 15 | - uses: actions-rs/toolchain@v1 16 | with: 17 | profile: minimal 18 | toolchain: ${{ matrix.toolchain }} 19 | components: rustfmt, clippy 20 | - uses: actions/checkout@v3 21 | with: 22 | submodules: true 23 | - name: cargo fmt --check 24 | uses: actions-rs/cargo@v1 25 | with: 26 | command: fmt 27 | args: --check 28 | - name: cargo clippy 29 | uses: actions-rs/clippy-check@v1 30 | if: always() 31 | with: 32 | token: ${{ secrets.GITHUB_TOKEN }} 33 | doc: 34 | runs-on: ubuntu-latest 35 | steps: 36 | - uses: actions-rs/toolchain@v1 37 | with: 38 | profile: minimal 39 | toolchain: nightly 40 | - uses: actions/checkout@v3 41 | - name: cargo doc 42 | uses: actions-rs/cargo@v1 43 | with: 44 | toolchain: nightly 45 | command: doc 46 | args: --no-deps --all-features 47 | env: 48 | RUSTDOCFLAGS: --cfg docsrs 49 | -------------------------------------------------------------------------------- /benches/arc_mutex_std.rs: -------------------------------------------------------------------------------- 1 | use bustle::*; 2 | use std::collections::HashMap; 3 | use std::sync::Mutex; 4 | 5 | #[derive(Clone)] 6 | struct Table(std::sync::Arc>>); 7 | 8 | impl Collection for Table 9 | where 10 | K: Send + From + Copy + 'static + std::hash::Hash + Eq, 11 | { 12 | type Handle = Self; 13 | fn with_capacity(capacity: usize) -> Self { 14 | Self(std::sync::Arc::new(Mutex::new(HashMap::with_capacity( 15 | capacity, 16 | )))) 17 | } 18 | 19 | fn pin(&self) -> Self::Handle { 20 | self.clone() 21 | } 22 | } 23 | 24 | impl CollectionHandle for Table 25 | where 26 | K: Send + From + Copy + 'static + std::hash::Hash + Eq, 27 | { 28 | type Key = K; 29 | 30 | fn get(&mut self, key: &Self::Key) -> bool { 31 | self.0.lock().unwrap().get(key).is_some() 32 | } 33 | 34 | fn insert(&mut self, key: &Self::Key) -> bool { 35 | self.0.lock().unwrap().insert(*key, ()).is_none() 36 | } 37 | 38 | fn remove(&mut self, key: &Self::Key) -> bool { 39 | self.0.lock().unwrap().remove(key).is_some() 40 | } 41 | 42 | fn update(&mut self, key: &Self::Key) -> bool { 43 | use std::collections::hash_map::Entry; 44 | let mut map = self.0.lock().unwrap(); 45 | if let Entry::Occupied(mut e) = map.entry(*key) { 46 | e.insert(()); 47 | true 48 | } else { 49 | false 50 | } 51 | } 52 | } 53 | 54 | fn main() { 55 | tracing_subscriber::fmt::init(); 56 | for n in 1..=num_cpus::get() { 57 | Workload::new(n, Mix::read_heavy()).run::>(); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /benches/arc_rwlock_std.rs: -------------------------------------------------------------------------------- 1 | use bustle::*; 2 | use std::collections::HashMap; 3 | use std::sync::RwLock; 4 | 5 | #[derive(Clone)] 6 | struct Table(std::sync::Arc>>); 7 | 8 | impl Collection for Table 9 | where 10 | K: Send + Sync + From + Copy + 'static + std::hash::Hash + Eq + std::fmt::Debug, 11 | { 12 | type Handle = Self; 13 | fn with_capacity(capacity: usize) -> Self { 14 | Self(std::sync::Arc::new(RwLock::new(HashMap::with_capacity( 15 | capacity, 16 | )))) 17 | } 18 | 19 | fn pin(&self) -> Self::Handle { 20 | self.clone() 21 | } 22 | } 23 | 24 | impl CollectionHandle for Table 25 | where 26 | K: Send + Sync + From + Copy + 'static + std::hash::Hash + Eq + std::fmt::Debug, 27 | { 28 | type Key = K; 29 | 30 | fn get(&mut self, key: &Self::Key) -> bool { 31 | self.0.read().unwrap().get(key).is_some() 32 | } 33 | 34 | fn insert(&mut self, key: &Self::Key) -> bool { 35 | self.0.write().unwrap().insert(*key, ()).is_none() 36 | } 37 | 38 | fn remove(&mut self, key: &Self::Key) -> bool { 39 | self.0.write().unwrap().remove(key).is_some() 40 | } 41 | 42 | fn update(&mut self, key: &Self::Key) -> bool { 43 | use std::collections::hash_map::Entry; 44 | let mut map = self.0.write().unwrap(); 45 | if let Entry::Occupied(mut e) = map.entry(*key) { 46 | e.insert(()); 47 | true 48 | } else { 49 | false 50 | } 51 | } 52 | } 53 | 54 | fn main() { 55 | tracing_subscriber::fmt::init(); 56 | for n in 1..=num_cpus::get() { 57 | Workload::new(n, Mix::read_heavy()).run::>(); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Crates.io](https://img.shields.io/crates/v/bustle.svg)](https://crates.io/crates/bustle) 2 | [![Documentation](https://docs.rs/bustle/badge.svg)](https://docs.rs/bustle/) 3 | 4 | Bustle is a benchmarking harness for concurrent key-value collections. 5 | 6 | Say you have a concurrent collection (like a `HashMap`) and you want to measure how well it 7 | performs across different workloads. Does it collapse when there are many writes? Or when there 8 | are many threads? Or if there are concurrent removals? This crate tries to give you answers. 9 | 10 | `bustle` runs a concurrent mix of operations (a "workload") against your collection, measuring 11 | statistics as it goes, and gives you a report at the end about how you did. There are many 12 | parameters to tweak, but hopefully the documentation for each element will help you decide. You 13 | probably want to measure your collection against many different workloads, rather than just a 14 | single one. 15 | 16 | See [the documentation](https://docs.rs/bustle) for details. 17 | 18 | The crate is, at the time of writing, a pretty direct port of the [Universal Benchmark] from 19 | `libcuckoo`, though that may change over time. 20 | 21 | If you have a concurrent key-value collection you would like to run 22 | through this benchmark, feel free to send a PR to add it as a benchmark 23 | to this repository! 24 | 25 | ## License 26 | 27 | Licensed under Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 28 | 29 | ## Contribution 30 | 31 | Unless you explicitly state otherwise, any contribution intentionally submitted 32 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall be 33 | licensed as above, without any additional terms or conditions. 34 | 35 | [Universal Benchmark]: https://github.com/efficient/libcuckoo/tree/master/tests/universal-benchmark 36 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2020 Jon Gjengset 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! A benchmarking harness for concurrent key-value collections. 2 | //! 3 | //! Say you have a concurrent collection (like a `HashMap`) and you want to measure how well it 4 | //! performs across different workloads. Does it collapse when there are many writes? Or when there 5 | //! are many threads? Or if there are concurrent removals? This crate tries to give you answers. 6 | //! 7 | //! `bustle` runs a concurrent mix of operations (a "workload") against your collection, measuring 8 | //! statistics as it goes, and gives you a report at the end about how you did. There are many 9 | //! parameters to tweak, but hopefully the documentation for each element will help you decide. You 10 | //! probably want to measure your collection against many different workloads, rather than just a 11 | //! single one. 12 | //! 13 | //! To run the benchmark, just implement [`Collection`] for your collection (`CollectionHandle` may 14 | //! end up just being a call to `clone`), build a [`Workload`], and call [`Workload::run`] 15 | //! parameterized by your collection type. You may want to look at the benchmarks for 16 | //! lock-protected collections from the standard library in `benches/` for inspiration. 17 | //! 18 | //! The crate is, at the time of writing, a pretty direct port of the [Universal Benchmark] from 19 | //! `libcuckoo`, though that may change over time. 20 | //! 21 | //! [Universal Benchmark]: https://github.com/efficient/libcuckoo/tree/master/tests/universal-benchmark 22 | #![deny(missing_docs)] 23 | #![warn( 24 | rust_2018_idioms, 25 | missing_debug_implementations, 26 | unreachable_pub, 27 | rustdoc::broken_intra_doc_links 28 | )] 29 | 30 | use std::{sync::Arc, sync::Barrier, time::Duration}; 31 | 32 | use rand::prelude::*; 33 | use tracing::{debug, info, info_span}; 34 | 35 | /// A workload mix configration. 36 | /// 37 | /// The sum of the fields must add to 100. 38 | #[derive(Clone, Copy, Debug)] 39 | pub struct Mix { 40 | /// The percentage of operations in the mix that are reads. 41 | pub read: u8, 42 | /// The percentage of operations in the mix that are inserts. 43 | pub insert: u8, 44 | /// The percentage of operations in the mix that are removals. 45 | pub remove: u8, 46 | /// The percentage of operations in the mix that are updates. 47 | pub update: u8, 48 | /// The percentage of operations in the mix that are update-or-inserts. 49 | pub upsert: u8, 50 | } 51 | 52 | impl Mix { 53 | /// Constructs a very read-heavy workload (~95%), with limited concurrent modifications. 54 | pub fn read_heavy() -> Self { 55 | Self { 56 | read: 94, 57 | insert: 2, 58 | update: 3, 59 | remove: 1, 60 | upsert: 0, 61 | } 62 | } 63 | 64 | /// Constructs a very insert-heavy workload (~80%), with some reads and updates. 65 | pub fn insert_heavy() -> Self { 66 | Self { 67 | read: 10, 68 | insert: 80, 69 | update: 10, 70 | remove: 0, 71 | upsert: 0, 72 | } 73 | } 74 | 75 | /// Constructs a very update-heavy workload (~50%), with some other modifications and the rest 76 | /// reads. 77 | pub fn update_heavy() -> Self { 78 | Self { 79 | read: 35, 80 | insert: 5, 81 | update: 50, 82 | remove: 5, 83 | upsert: 5, 84 | } 85 | } 86 | 87 | /// Constructs a workload where all operations occur with equal probability. 88 | pub fn uniform() -> Self { 89 | Self { 90 | read: 20, 91 | insert: 20, 92 | update: 20, 93 | remove: 20, 94 | upsert: 20, 95 | } 96 | } 97 | } 98 | 99 | /// A benchmark workload builder. 100 | #[derive(Clone, Copy, Debug)] 101 | pub struct Workload { 102 | /// The mix of operations to run. 103 | mix: Mix, 104 | 105 | /// The initial capacity of the table, specified as a power of 2. 106 | initial_cap_log2: u8, 107 | 108 | /// The fraction of the initial table capacity should we populate before running the benchmark. 109 | prefill_f: f64, 110 | 111 | /// Total number of operations as a multiple of the initial capacity. 112 | ops_f: f64, 113 | 114 | /// Number of threads to run the benchmark with. 115 | threads: usize, 116 | 117 | /// Random seed to randomize the workload. 118 | /// 119 | /// If `None`, the seed is picked randomly. 120 | /// If `Some`, the workload is deterministic if `threads == 1`. 121 | seed: Option<[u8; 32]>, 122 | } 123 | 124 | /// A collection that can be benchmarked by bustle. 125 | /// 126 | /// Any thread that performs operations on the collection will first call `pin` and then perform 127 | /// collection operations on the `Handle` that is returned. `pin` will not be called in the hot 128 | /// loop of the benchmark. 129 | pub trait Collection: Send + Sync + 'static { 130 | /// A thread-local handle to the concurrent collection under test. 131 | type Handle: CollectionHandle; 132 | 133 | /// Allocate a new instance of the benchmark target with the given capacity. 134 | fn with_capacity(capacity: usize) -> Self; 135 | 136 | /// Pin a thread-local handle to the concurrent collection under test. 137 | fn pin(&self) -> Self::Handle; 138 | } 139 | 140 | /// A handle to a key-value collection. 141 | /// 142 | /// Note that for all these methods, the benchmarker does not dictate what the values are. Feel 143 | /// free to use the same value for all operations, or use distinct ones and check that your 144 | /// retrievals indeed return the right results. 145 | pub trait CollectionHandle { 146 | /// The `u64` seeds used to construct `Key` (through `From`) are distinct. 147 | /// The returned keys must be as well. 148 | type Key: From; 149 | 150 | /// Perform a lookup for `key`. 151 | /// 152 | /// Should return `true` if the key is found. 153 | fn get(&mut self, key: &Self::Key) -> bool; 154 | 155 | /// Insert `key` into the collection. 156 | /// 157 | /// Should return `true` if no value previously existed for the key. 158 | fn insert(&mut self, key: &Self::Key) -> bool; 159 | 160 | /// Remove `key` from the collection. 161 | /// 162 | /// Should return `true` if the key existed and was removed. 163 | fn remove(&mut self, key: &Self::Key) -> bool; 164 | 165 | /// Update the value for `key` in the collection, if it exists. 166 | /// 167 | /// Should return `true` if the key existed and was updated. 168 | /// 169 | /// Should **not** insert the key if it did not exist. 170 | fn update(&mut self, key: &Self::Key) -> bool; 171 | } 172 | 173 | /// Information about a measurement. 174 | #[derive(Debug, Clone)] 175 | pub struct Measurement { 176 | /// A seed used for the run. 177 | pub seed: [u8; 32], 178 | /// A total number of operations. 179 | pub total_ops: u64, 180 | /// Spent time. 181 | pub spent: Duration, 182 | /// A number of operations per second. 183 | pub throughput: f64, 184 | /// An average value of latency. 185 | pub latency: Duration, 186 | } 187 | 188 | impl Workload { 189 | /// Start building a new benchmark workload. 190 | pub fn new(threads: usize, mix: Mix) -> Self { 191 | Self { 192 | mix, 193 | initial_cap_log2: 25, 194 | prefill_f: 0.0, 195 | ops_f: 0.75, 196 | threads, 197 | seed: None, 198 | } 199 | } 200 | 201 | /// Set the initial capacity for the map. 202 | /// 203 | /// Note that the capacity will be `2^` the given capacity! 204 | /// 205 | /// The number of operations and the number of pre-filled keys are determined based on the 206 | /// computed initial capacity, so keep that in mind if you change this parameter. 207 | /// 208 | /// Defaults to 25 (so `2^25 ~= 34M`). 209 | pub fn initial_capacity_log2(&mut self, capacity: u8) -> &mut Self { 210 | self.initial_cap_log2 = capacity; 211 | self 212 | } 213 | 214 | /// Set the fraction of the initial table capacity we should populate before running the 215 | /// benchmark. 216 | /// 217 | /// Defaults to 0%. 218 | pub fn prefill_fraction(&mut self, fraction: f64) -> &mut Self { 219 | assert!(fraction >= 0.0); 220 | assert!(fraction <= 1.0); 221 | self.prefill_f = fraction; 222 | self 223 | } 224 | 225 | /// Set the number of operations to run as a multiple of the initial capacity. 226 | /// 227 | /// This value can exceed 1.0. 228 | /// 229 | /// Defaults to 0.75 (75%). 230 | pub fn operations(&mut self, multiple: f64) -> &mut Self { 231 | assert!(multiple >= 0.0); 232 | self.ops_f = multiple; 233 | self 234 | } 235 | 236 | /// Set the seed used to randomize the workload. 237 | /// 238 | /// The seed does _not_ dictate thread interleaving, so you will only observe the exact same 239 | /// workload if you run the benchmark with `nthreads == 1`. 240 | pub fn seed(&mut self, seed: [u8; 32]) -> &mut Self { 241 | self.seed = Some(seed); 242 | self 243 | } 244 | 245 | /// Execute this workload against the collection type given by `T`. 246 | /// 247 | /// The key type must be `Send` since we generate the keys on a different thread than the one 248 | /// we do the benchmarks on. 249 | /// 250 | /// The key type must be `Debug` so that we can print meaningful errors if an assertion is 251 | /// violated during the benchmark. 252 | /// 253 | /// Returns the seed used for the run. 254 | pub fn run(&self) -> [u8; 32] 255 | where 256 | ::Key: Send + std::fmt::Debug, 257 | { 258 | let m = self.run_silently::(); 259 | 260 | // TODO: do more with this information 261 | // TODO: collect statistics per operation type 262 | eprintln!( 263 | "{} operations across {} thread(s) in {:?}; time/op = {:?}", 264 | m.total_ops, 265 | self.threads, 266 | m.spent, 267 | m.spent / m.total_ops as u32, 268 | ); 269 | 270 | m.seed 271 | } 272 | 273 | /// Execute this workload against the collection type given by `T`. 274 | /// 275 | /// The key type must be `Send` since we generate the keys on a different thread than the one 276 | /// we do the benchmarks on. 277 | /// 278 | /// The key type must be `Debug` so that we can print meaningful errors if an assertion is 279 | /// violated during the benchmark. 280 | #[allow(clippy::cognitive_complexity)] 281 | pub fn run_silently(&self) -> Measurement 282 | where 283 | ::Key: Send + std::fmt::Debug, 284 | { 285 | assert_eq!( 286 | self.mix.read + self.mix.insert + self.mix.remove + self.mix.update + self.mix.upsert, 287 | 100, 288 | "mix fractions do not add up to 100%" 289 | ); 290 | 291 | let initial_capacity = 1 << self.initial_cap_log2; 292 | let total_ops = (initial_capacity as f64 * self.ops_f) as usize; 293 | 294 | let seed = self.seed.unwrap_or_else(rand::random); 295 | let mut rng: rand::rngs::SmallRng = rand::SeedableRng::from_seed(seed); 296 | 297 | // NOTE: it'd be nice to include std::intrinsics::type_name:: here 298 | let span = info_span!("benchmark", mix = ?self.mix, threads = self.threads); 299 | let _guard = span.enter(); 300 | debug!(initial_capacity, total_ops, ?seed, "workload parameters"); 301 | 302 | info!("generating operation mix"); 303 | let mut op_mix = Vec::with_capacity(100); 304 | op_mix.append(&mut vec![Operation::Read; usize::from(self.mix.read)]); 305 | op_mix.append(&mut vec![Operation::Insert; usize::from(self.mix.insert)]); 306 | op_mix.append(&mut vec![Operation::Remove; usize::from(self.mix.remove)]); 307 | op_mix.append(&mut vec![Operation::Update; usize::from(self.mix.update)]); 308 | op_mix.append(&mut vec![Operation::Upsert; usize::from(self.mix.upsert)]); 309 | op_mix.shuffle(&mut rng); 310 | 311 | info!("generating key space"); 312 | let prefill = (initial_capacity as f64 * self.prefill_f) as usize; 313 | // We won't be running through `op_mix` more than ceil(total_ops / 100), so calculate that 314 | // ceiling and multiply by the number of inserts and upserts to get an upper bound on how 315 | // many elements we'll be inserting. 316 | let max_insert_ops = 317 | (total_ops + 99) / 100 * usize::from(self.mix.insert + self.mix.upsert); 318 | let insert_keys = std::cmp::max(initial_capacity, max_insert_ops) + prefill; 319 | // Round this quantity up to a power of 2, so that we can use an LCG to cycle over the 320 | // array "randomly". 321 | let insert_keys_per_thread = 322 | ((insert_keys + self.threads - 1) / self.threads).next_power_of_two(); 323 | let mut generators = Vec::new(); 324 | for _ in 0..self.threads { 325 | let mut thread_seed = [0u8; 32]; 326 | rng.fill_bytes(&mut thread_seed[..]); 327 | generators.push(std::thread::spawn(move || { 328 | let mut rng: rand::rngs::SmallRng = rand::SeedableRng::from_seed(thread_seed); 329 | let mut keys: Vec<::Key> = 330 | Vec::with_capacity(insert_keys_per_thread); 331 | keys.extend((0..insert_keys_per_thread).map(|_| rng.next_u64().into())); 332 | keys 333 | })); 334 | } 335 | let keys: Vec<_> = generators 336 | .into_iter() 337 | .map(|jh| jh.join().unwrap()) 338 | .collect(); 339 | 340 | info!("constructing initial table"); 341 | let table = Arc::new(T::with_capacity(initial_capacity)); 342 | 343 | // And fill it 344 | let prefill_per_thread = prefill / self.threads; 345 | let mut prefillers = Vec::new(); 346 | for keys in keys { 347 | let table = Arc::clone(&table); 348 | prefillers.push(std::thread::spawn(move || { 349 | let mut table = table.pin(); 350 | for key in &keys[0..prefill_per_thread] { 351 | let inserted = table.insert(key); 352 | assert!(inserted); 353 | } 354 | keys 355 | })); 356 | } 357 | let keys: Vec<_> = prefillers 358 | .into_iter() 359 | .map(|jh| jh.join().unwrap()) 360 | .collect(); 361 | 362 | info!("start workload mix"); 363 | let ops_per_thread = total_ops / self.threads; 364 | let op_mix = Arc::new(op_mix.into_boxed_slice()); 365 | let barrier = Arc::new(Barrier::new(self.threads + 1)); 366 | let mut mix_threads = Vec::with_capacity(self.threads); 367 | for keys in keys { 368 | let table = Arc::clone(&table); 369 | let op_mix = Arc::clone(&op_mix); 370 | let barrier = Arc::clone(&barrier); 371 | mix_threads.push(std::thread::spawn(move || { 372 | let mut table = table.pin(); 373 | mix( 374 | &mut table, 375 | &keys, 376 | &op_mix, 377 | ops_per_thread, 378 | prefill_per_thread, 379 | barrier, 380 | ) 381 | })); 382 | } 383 | 384 | barrier.wait(); 385 | let start = std::time::Instant::now(); 386 | barrier.wait(); 387 | let spent = start.elapsed(); 388 | 389 | let _samples: Vec<_> = mix_threads 390 | .into_iter() 391 | .map(|jh| jh.join().unwrap()) 392 | .collect(); 393 | 394 | let avg = spent / total_ops as u32; 395 | info!(?spent, ops = total_ops, ?avg, "workload mix finished"); 396 | 397 | let total_ops = total_ops as u64; 398 | let threads = self.threads as u32; 399 | 400 | Measurement { 401 | seed, 402 | total_ops, 403 | spent, 404 | throughput: total_ops as f64 / spent.as_secs_f64(), 405 | latency: Duration::from_nanos((spent * threads).as_nanos() as u64 / total_ops), 406 | } 407 | } 408 | } 409 | 410 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] 411 | enum Operation { 412 | Read, 413 | Insert, 414 | Remove, 415 | Update, 416 | Upsert, 417 | } 418 | 419 | fn mix( 420 | tbl: &mut H, 421 | keys: &[H::Key], 422 | op_mix: &[Operation], 423 | ops: usize, 424 | prefilled: usize, 425 | barrier: Arc, 426 | ) where 427 | H::Key: std::fmt::Debug, 428 | { 429 | // Invariant: erase_seq <= insert_seq 430 | // Invariant: insert_seq < numkeys 431 | let nkeys = keys.len(); 432 | let mut erase_seq = 0; 433 | let mut insert_seq = prefilled; 434 | let mut find_seq = 0; 435 | 436 | // We're going to use a very simple LCG to pick random keys. 437 | // We want it to be _super_ fast so it doesn't add any overhead. 438 | assert!(nkeys.is_power_of_two()); 439 | assert!(nkeys > 4); 440 | assert_eq!(op_mix.len(), 100); 441 | let a = nkeys / 2 + 1; 442 | let c = nkeys / 4 - 1; 443 | let find_seq_mask = nkeys - 1; 444 | 445 | // The elapsed time is measured by the lifetime of `workload_scope`. 446 | let workload_scope = scopeguard::guard(barrier, |barrier| { 447 | barrier.wait(); 448 | }); 449 | workload_scope.wait(); 450 | 451 | for (i, op) in (0..((ops + op_mix.len() - 1) / op_mix.len())) 452 | .flat_map(|_| op_mix.iter()) 453 | .enumerate() 454 | { 455 | if i == ops { 456 | break; 457 | } 458 | 459 | match op { 460 | Operation::Read => { 461 | let should_find = find_seq >= erase_seq && find_seq < insert_seq; 462 | let found = tbl.get(&keys[find_seq]); 463 | if find_seq >= erase_seq { 464 | assert_eq!( 465 | should_find, found, 466 | "get({:?}) {} {} {}", 467 | &keys[find_seq], find_seq, erase_seq, insert_seq 468 | ); 469 | } else { 470 | // due to upserts, we may _or may not_ find the key 471 | } 472 | 473 | // Twist the LCG since we used find_seq 474 | find_seq = (a * find_seq + c) & find_seq_mask; 475 | } 476 | Operation::Insert => { 477 | let new_key = tbl.insert(&keys[insert_seq]); 478 | assert!( 479 | new_key, 480 | "insert({:?}) should insert a new value", 481 | &keys[insert_seq] 482 | ); 483 | insert_seq += 1; 484 | } 485 | Operation::Remove => { 486 | if erase_seq == insert_seq { 487 | // If `erase_seq` == `insert_eq`, the table should be empty. 488 | let removed = tbl.remove(&keys[find_seq]); 489 | assert!( 490 | !removed, 491 | "remove({:?}) succeeded on empty table", 492 | &keys[find_seq] 493 | ); 494 | 495 | // Twist the LCG since we used find_seq 496 | find_seq = (a * find_seq + c) & find_seq_mask; 497 | } else { 498 | let removed = tbl.remove(&keys[erase_seq]); 499 | assert!(removed, "remove({:?}) should succeed", &keys[erase_seq]); 500 | erase_seq += 1; 501 | } 502 | } 503 | Operation::Update => { 504 | // Same as find, except we update to the same default value 505 | let should_exist = find_seq >= erase_seq && find_seq < insert_seq; 506 | let updated = tbl.update(&keys[find_seq]); 507 | if find_seq >= erase_seq { 508 | assert_eq!(should_exist, updated, "update({:?})", &keys[find_seq]); 509 | } else { 510 | // due to upserts, we may or may not have updated an existing key 511 | } 512 | 513 | // Twist the LCG since we used find_seq 514 | find_seq = (a * find_seq + c) & find_seq_mask; 515 | } 516 | Operation::Upsert => { 517 | // Pick a number from the full distribution, but cap it to the insert_seq, so we 518 | // don't insert a number greater than insert_seq. 519 | let n = std::cmp::min(find_seq, insert_seq); 520 | 521 | // Twist the LCG since we used find_seq 522 | find_seq = (a * find_seq + c) & find_seq_mask; 523 | 524 | let _inserted = tbl.insert(&keys[n]); 525 | if n == insert_seq { 526 | insert_seq += 1; 527 | } 528 | } 529 | } 530 | } 531 | } 532 | --------------------------------------------------------------------------------