├── .dockerignore ├── .github └── workflows │ ├── docker.yml │ └── rust.yml ├── .gitignore ├── .rustfmt.toml ├── Cargo.toml ├── Dockerfile ├── LICENSE ├── README.md └── src └── main.rs /.dockerignore: -------------------------------------------------------------------------------- 1 | target -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Build and publish a Docker image to ghcr.io 2 | on: 3 | release: 4 | types: [ published ] 5 | 6 | jobs: 7 | docker_publish: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: 'Checkout GitHub Action' 11 | uses: actions/checkout@v4 12 | 13 | - name: Set up QEMU 14 | uses: docker/setup-qemu-action@v3 15 | 16 | - name: Set up Docker Buildx 17 | uses: docker/setup-buildx-action@v3 18 | 19 | - name: Docker meta 20 | id: meta 21 | uses: docker/metadata-action@v4 22 | with: 23 | images: ghcr.io/seddonm1/sqlite-bench 24 | flavor: latest=true 25 | tags: | 26 | type=semver,pattern={{version}} 27 | 28 | - name: Login to image repository 29 | if: github.ref_type == 'tag' 30 | uses: docker/login-action@v3 31 | with: 32 | registry: ghcr.io 33 | username: ${{ github.actor }} 34 | password: ${{ secrets.PACKAGES_GITHUB_TOKEN }} 35 | 36 | - name: Build and push 37 | uses: docker/build-push-action@v6 38 | with: 39 | context: . 40 | file: Dockerfile 41 | platforms: linux/amd64,linux/arm64 42 | push: ${{ github.ref_type == 'tag' }} 43 | tags: ${{ steps.meta.outputs.tags }} 44 | labels: ${{ steps.meta.outputs.labels }} -------------------------------------------------------------------------------- /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | test: 6 | name: Test Workspace on AMD64 Rust ${{ matrix.rust }} 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | arch: [amd64] 11 | rust: [stable] 12 | container: 13 | image: ${{ matrix.arch }}/rust 14 | env: 15 | # Disable full debug symbol generation to speed up CI build and keep memory down 16 | # "1" means line tables only, which is useful for panic tracebacks. 17 | RUSTFLAGS: "-C debuginfo=1" 18 | steps: 19 | - uses: actions/checkout@v4 20 | with: 21 | submodules: true 22 | - name: Cache Cargo 23 | uses: actions/cache@v4 24 | with: 25 | path: /home/runner/.cargo 26 | key: cargo-cache- 27 | - name: Cache Rust dependencies 28 | uses: actions/cache@v4 29 | with: 30 | path: /home/runner/target 31 | key: target-cache- 32 | - name: Setup Rust toolchain 33 | run: | 34 | rustup toolchain install ${{ matrix.rust }} 35 | rustup default ${{ matrix.rust }} 36 | rustup component add rustfmt 37 | rustup component add clippy 38 | apt update 39 | apt install -y clang 40 | - name: Run tests 41 | run: | 42 | cargo test 43 | - name: Run format 44 | run: cargo fmt --all -- --check 45 | - name: Run clippy 46 | run: cargo clippy --all-targets --workspace -- -D warnings 47 | 48 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | debug/ 4 | target/ 5 | 6 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 7 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 8 | Cargo.lock 9 | 10 | # These are backup files generated by rustfmt 11 | **/*.rs.bk 12 | 13 | # MSVC Windows builds of rustc generate these, which store debugging information 14 | *.pdb 15 | 16 | .DS_Store -------------------------------------------------------------------------------- /.rustfmt.toml: -------------------------------------------------------------------------------- 1 | max_width = 150 2 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sqlite-bench" 3 | version = "0.2.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | anyhow = "1.0.86" 8 | clap = { version = "4.5.9", features = ["derive"] } 9 | indicatif = "0.17.8" 10 | itertools = "0.13.0" 11 | rand = "0.8.5" 12 | rusqlite = { git = "https://github.com/seddonm1/rusqlite", branch = "begin-concurrent", features = [ 13 | "bundled", 14 | "buildtime_bindgen", 15 | ] } 16 | serde = { version = "1.0.204", features = ["derive"] } 17 | serde_json = "1.0.120" 18 | 19 | [profile.release] 20 | codegen-units = 1 21 | opt-level = 3 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:1.79.0 AS build-env 2 | RUN apt update \ 3 | && apt install -y \ 4 | clang 5 | WORKDIR /app 6 | COPY . /app 7 | RUN cargo build --release 8 | 9 | FROM gcr.io/distroless/cc 10 | COPY --from=build-env /app/target/release/sqlite-bench / 11 | ENTRYPOINT ["./sqlite-bench"] 12 | CMD ["--help"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Mike Seddon 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sqlite-bench 2 | 3 | A project to test SQLite Transaction behavior. 4 | 5 | Code to accompany blog post: https://reorchestrate.com/posts/sqlite-transactions 6 | 7 | ## How to use 8 | 9 | Compile by running `cargo build --release`. 10 | 11 | Run like: `cargo run --release -- --help`: 12 | 13 | ```bash 14 | Benchmarking SQLite 15 | 16 | Usage: sqlite-bench [OPTIONS] --path --output 17 | 18 | Options: 19 | -p, --path Path to the SQLite file 20 | -o, --output Path to the output result file 21 | -s, --seed Number of records to seed the into the table [default: 1000000] 22 | -t, --threads ... Number of concurrent threads to spawn [default: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16] 23 | -s, --scans ... Scan operations to perform per transaction [default: 0 10] 24 | -u, --updates ... Update operations to perform per transaction [default: 0 1 10] 25 | -h, --help Print help 26 | -V, --version Print version 27 | ``` 28 | 29 | It is a good idea to run this against an in-memory filesystem first to protect your solid-state-drive. 30 | 31 | MacOS: 32 | 33 | ```bash 34 | diskutil erasevolume apfs 'ramdisk' `hdiutil attach -nobrowse -nomount ram://33554432` 35 | ``` 36 | 37 | Linux: 38 | 39 | ```bash 40 | sudo mkdir -p /mnt/ramdisk 41 | sudo mount -t tmpfs -o size=16g tmpfs /mnt/ramdisk 42 | ``` 43 | 44 | A multiplatform Docker image is available at: https://github.com/users/seddonm1/packages/container/package/sqlite-bench -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use indicatif::{ProgressBar, ProgressStyle}; 4 | use itertools::Itertools; 5 | use rand::{distributions::Uniform, prelude::*}; 6 | use rusqlite::{Connection, ErrorCode, OpenFlags, TransactionBehavior}; 7 | use serde::Serialize; 8 | use std::{ 9 | fs, 10 | ops::Add, 11 | path::{Path, PathBuf}, 12 | sync::{ 13 | atomic::{AtomicUsize, Ordering}, 14 | Arc, 15 | }, 16 | time::{Duration, Instant}, 17 | }; 18 | 19 | /// Benchmarking SQLite 20 | #[derive(Parser, Debug)] 21 | #[command(version, about, long_about = None)] 22 | struct Args { 23 | /// Path to the SQLite file. 24 | #[arg(short, long)] 25 | path: PathBuf, 26 | 27 | /// Path to the output result file. 28 | #[arg(short, long)] 29 | output: PathBuf, 30 | 31 | /// Number of records to seed the into the table. 32 | #[arg(short, long, default_value_t = 1_000_000)] 33 | seed: usize, 34 | 35 | /// Number of concurrent threads to spawn. 36 | #[arg(short, long, num_args = 1.., value_delimiter = ' ', default_values_t = (1..=16).collect::>())] 37 | threads: Vec, 38 | 39 | /// Scan operations to perform per transaction. 40 | #[arg(short, long, num_args = 1.., value_delimiter = ' ', default_values_t = vec![0, 10])] 41 | scans: Vec, 42 | 43 | /// Update operations to perform per transaction. 44 | #[arg(short, long, num_args = 1.., value_delimiter = ' ', default_values_t = vec![0, 1, 10])] 45 | updates: Vec, 46 | } 47 | 48 | struct Hexadecimal; 49 | impl Distribution for Hexadecimal { 50 | fn sample(&self, rng: &mut R) -> char { 51 | *b"0123456789ABCDEF".choose(rng).unwrap() as char 52 | } 53 | } 54 | 55 | const SCAN: &str = "SELECT * FROM tbl WHERE substr(c, 1, 16)>=? ORDER BY substr(c, 1, 16) LIMIT 10;"; 56 | const UPDATE: &str = "UPDATE tbl SET b=?, c=? WHERE a=?;"; 57 | 58 | #[derive(Debug, Serialize)] 59 | struct Transactions { 60 | behavior: String, 61 | seed: usize, 62 | n_threads: usize, 63 | n_scans: usize, 64 | n_updates: usize, 65 | retries: usize, 66 | transactions: usize, 67 | tps: u128, 68 | } 69 | 70 | fn main() -> Result<()> { 71 | let args = Args::parse(); 72 | 73 | if args.output.exists() { 74 | return Err(anyhow::anyhow!("file already exists {:?}", args.output)); 75 | } 76 | 77 | // remove any state 78 | fs::remove_file(&args.path).ok(); 79 | fs::remove_file(args.path.join("-shm")).ok(); 80 | fs::remove_file(args.path.join("-wal")).ok(); 81 | 82 | let iterations = args 83 | .threads 84 | .iter() 85 | .cartesian_product(args.scans) 86 | .cartesian_product(args.updates) 87 | .cartesian_product([ 88 | TransactionBehavior::Deferred, 89 | TransactionBehavior::Immediate, 90 | TransactionBehavior::Concurrent, 91 | ]) 92 | .map(|(((n_threads, n_scans), n_updates), trasaction_behavior)| (*n_threads, n_scans, n_updates, trasaction_behavior)) 93 | .filter(|(_, n_scans, n_updates, _)| !(*n_scans == 0 && *n_updates == 0)) 94 | .collect::>(); 95 | 96 | // seed database 97 | seed(&args.path, args.seed)?; 98 | 99 | let pb = ProgressBar::new(iterations.len() as u64).with_style(ProgressStyle::with_template("{wide_bar} {pos}/{len} {eta_precise}")?); 100 | pb.inc(0); 101 | 102 | let mut results = Vec::with_capacity(iterations.len()); 103 | 104 | for (n_threads, n_scans, n_updates, trasaction_behavior) in iterations { 105 | results.push(begin(&args.path, args.seed, n_threads, n_scans, n_updates, trasaction_behavior)?); 106 | pb.inc(1); 107 | } 108 | 109 | pb.finish(); 110 | 111 | fs::write(args.output, serde_json::to_string_pretty(&results)?)?; 112 | 113 | // remove any state 114 | fs::remove_file(&args.path).ok(); 115 | fs::remove_file(args.path.join("-shm")).ok(); 116 | fs::remove_file(args.path.join("-wal")).ok(); 117 | 118 | Ok(()) 119 | } 120 | 121 | fn seed(path: &Path, rows: usize) -> Result<()> { 122 | let conn = Connection::open(path)?; 123 | conn.execute_batch(&format!( 124 | " 125 | PRAGMA journal_mode = WAL; 126 | PRAGMA mmap_size = 1000000000; 127 | PRAGMA synchronous = off; 128 | PRAGMA journal_size_limit = 16777216; 129 | 130 | CREATE TABLE tbl( 131 | a INTEGER PRIMARY KEY, 132 | b BLOB(200), 133 | c CHAR(64) 134 | ); 135 | 136 | -- https://www.sqlite.org/series.html 137 | WITH RECURSIVE generate_series(value) AS ( 138 | SELECT 0 139 | UNION ALL 140 | SELECT value+1 FROM generate_series 141 | WHERE value < {rows} 142 | ) 143 | INSERT INTO tbl 144 | SELECT value, randomblob(200), hex(randomblob(32)) 145 | FROM generate_series; 146 | 147 | CREATE INDEX tbl_i1 ON tbl(substr(c, 1, 16)); 148 | CREATE INDEX tbl_i2 ON tbl(substr(c, 2, 16)); 149 | " 150 | ))?; 151 | 152 | Ok(()) 153 | } 154 | 155 | fn begin( 156 | path: &Path, 157 | seed: usize, 158 | n_threads: usize, 159 | n_scans: usize, 160 | n_updates: usize, 161 | trasaction_behavior: TransactionBehavior, 162 | ) -> Result { 163 | let transactions = Arc::new(AtomicUsize::new(0)); 164 | let retries = Arc::new(AtomicUsize::new(0)); 165 | (0..n_threads) 166 | .map(|thread_id| { 167 | let path = path.to_path_buf(); 168 | let transactions = transactions.clone(); 169 | let retries = retries.clone(); 170 | 171 | std::thread::spawn(move || { 172 | let between_ids = Uniform::from(0..1_000_000); 173 | let mut rng: StdRng = SeedableRng::seed_from_u64(thread_id as u64); 174 | let mut conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE)?; 175 | conn.busy_timeout(Duration::from_millis(5000))?; 176 | 177 | let finish_time = Instant::now().add(Duration::from_secs(30)); 178 | while Instant::now() <= finish_time { 179 | let scans = (0..n_scans) 180 | .map(|_| (&mut rng).sample_iter(&Hexadecimal).take(16).map(char::from).collect::()) 181 | .collect::>(); 182 | let updates: Vec<([u8; 200], String, i32)> = (0..n_updates) 183 | .map(|_| { 184 | let mut bytes = [0; 200]; 185 | rng.fill_bytes(&mut bytes); 186 | ( 187 | bytes, 188 | (&mut rng).sample_iter(&Hexadecimal).take(64).map(char::from).collect::(), 189 | between_ids.sample(&mut rng), 190 | ) 191 | }) 192 | .collect::>(); 193 | 194 | loop { 195 | let mut transaction = || { 196 | let txn = conn.transaction_with_behavior(trasaction_behavior)?; 197 | 198 | if !scans.is_empty() { 199 | let mut scan = txn.prepare_cached(SCAN)?; 200 | for random_hex in &scans { 201 | // Consume the results 202 | scan.query_map([random_hex], |row| row.get::<_, i32>(0))?.for_each(drop); 203 | } 204 | } 205 | 206 | if !updates.is_empty() { 207 | let mut update = txn.prepare_cached(UPDATE)?; 208 | for (random_bytes, random_hex, random_id) in &updates { 209 | update.execute((random_bytes, random_hex, random_id))?; 210 | } 211 | } 212 | 213 | txn.commit() 214 | }; 215 | 216 | match transaction() { 217 | Err(rusqlite::Error::SqliteFailure(err, _)) if err.code == ErrorCode::DatabaseBusy => { 218 | retries.fetch_add(1, Ordering::Relaxed); 219 | continue; 220 | } 221 | Ok(_) => { 222 | transactions.fetch_add(1, Ordering::Relaxed); 223 | break; 224 | } 225 | err => unimplemented!("{err:?}"), 226 | } 227 | } 228 | } 229 | 230 | anyhow::Ok(()) 231 | }) 232 | }) 233 | .collect::>() 234 | .into_iter() 235 | .for_each(|thread| thread.join().expect("should not fail").expect("should not fail")); 236 | 237 | Ok(Transactions { 238 | behavior: match trasaction_behavior { 239 | TransactionBehavior::Deferred => "DEFERRED", 240 | TransactionBehavior::Immediate => "IMMEDIATE", 241 | TransactionBehavior::Exclusive => "EXCLUSIVE", 242 | TransactionBehavior::Concurrent => "CONCURRENT", 243 | _ => unreachable!(), 244 | } 245 | .to_string(), 246 | seed, 247 | n_threads, 248 | n_scans, 249 | n_updates, 250 | retries: retries.load(Ordering::Relaxed), 251 | transactions: transactions.load(Ordering::Relaxed), 252 | tps: Duration::from_secs(1).as_nanos() / Duration::from_secs(30).div_f32(transactions.load(Ordering::Relaxed) as f32).as_nanos(), 253 | }) 254 | } 255 | --------------------------------------------------------------------------------