├── rust-toolchain ├── crates ├── redb-bench │ ├── src │ │ └── lib.rs │ ├── benches │ │ ├── redb_benchmark.rs │ │ ├── multithreaded_insert_benchmark.rs │ │ ├── atomics_benchmark.rs │ │ ├── int_benchmark.rs │ │ ├── savepoint_benchmark.rs │ │ ├── large_values_benchmark.rs │ │ └── lmdb_benchmark.rs │ └── Cargo.toml ├── redb-python │ ├── build.rs │ ├── src │ │ ├── python.rs │ │ └── lib.rs │ ├── test │ │ └── __init__.py │ ├── py_publish.sh │ ├── pyproject.toml │ └── Cargo.toml └── redb-derive │ ├── Cargo.toml │ ├── tests │ └── derive_tests.rs │ └── src │ └── lib.rs ├── rustfmt.toml ├── src ├── sealed.rs ├── backends.rs ├── tree_store │ ├── page_store │ │ ├── file_backend │ │ │ ├── mod.rs │ │ │ ├── fallback.rs │ │ │ └── optimized.rs │ │ ├── mod.rs │ │ ├── fast_hash.rs │ │ ├── backends.rs │ │ ├── lru_cache.rs │ │ ├── savepoint.rs │ │ ├── region.rs │ │ ├── layout.rs │ │ └── base.rs │ └── mod.rs ├── types │ └── uuid.rs ├── complex_types.rs ├── lib.rs ├── transaction_tracker.rs ├── legacy_tuple_types.rs └── tuple_types.rs ├── fuzz ├── .gitignore ├── Cargo.toml └── fuzz_targets │ └── common.rs ├── clippy.toml ├── .dockerignore ├── .editorconfig ├── .cargo └── config.toml ├── Dockerfile.bench ├── .gitignore ├── examples ├── int_keys.rs ├── bincode_keys.rs ├── multithread.rs └── special_values.rs ├── LICENSE-MIT ├── Cargo.toml ├── justfile ├── .github └── workflows │ └── ci.yml ├── README.md ├── tests ├── multithreading_tests.rs └── backward_compatibility.rs ├── deny.toml └── LICENSE-APACHE /rust-toolchain: -------------------------------------------------------------------------------- 1 | 1.89 2 | -------------------------------------------------------------------------------- /crates/redb-bench/src/lib.rs: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | max_width = 100 2 | -------------------------------------------------------------------------------- /src/sealed.rs: -------------------------------------------------------------------------------- 1 | pub trait Sealed {} 2 | -------------------------------------------------------------------------------- /fuzz/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | corpus 3 | artifacts 4 | coverage/ 5 | -------------------------------------------------------------------------------- /clippy.toml: -------------------------------------------------------------------------------- 1 | disallowed-methods = [ 2 | "usize::to_le_bytes", 3 | ] 4 | -------------------------------------------------------------------------------- /crates/redb-python/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | pyo3_build_config::add_extension_module_link_args(); 3 | } 4 | -------------------------------------------------------------------------------- /src/backends.rs: -------------------------------------------------------------------------------- 1 | pub use crate::tree_store::InMemoryBackend; 2 | pub use crate::tree_store::file_backend::FileBackend; 3 | -------------------------------------------------------------------------------- /crates/redb-python/src/python.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | 3 | #[pymodule] 4 | pub fn redb(_m: &Bound<'_, PyModule>) -> PyResult<()> { 5 | Ok(()) 6 | } 7 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !src/ 3 | !crates/ 4 | !tests/ 5 | !examples/ 6 | !cargo/ 7 | !./*.toml 8 | !./build.rs 9 | !./rust-toolchain 10 | !./LICENSE 11 | **/target/ 12 | -------------------------------------------------------------------------------- /crates/redb-python/test/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | from unittest import TestCase 4 | 5 | 6 | class TableTestCase(TestCase): 7 | def test_import(self): 8 | import redb 9 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_size = 4 7 | indent_style = space 8 | insert_final_newline = true 9 | max_line_length = 100 10 | trim_trailing_whitespace = true 11 | -------------------------------------------------------------------------------- /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [target.wasm32-wasip1] 2 | runner = "wasmtime --dir=/tmp" 3 | 4 | [target.wasm32-wasip1-threads] 5 | runner = "wasmtime --dir=/tmp -W threads=y -S threads=y" 6 | 7 | [target.wasm32-wasip2] 8 | runner = "wasmtime --dir=/tmp" 9 | -------------------------------------------------------------------------------- /src/tree_store/page_store/file_backend/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(any(windows, unix, target_os = "wasi"))] 2 | mod optimized; 3 | #[cfg(any(windows, unix, target_os = "wasi"))] 4 | pub use optimized::FileBackend; 5 | 6 | #[cfg(not(any(windows, unix, target_os = "wasi")))] 7 | mod fallback; 8 | #[cfg(not(any(windows, unix, target_os = "wasi")))] 9 | pub use fallback::FileBackend; 10 | -------------------------------------------------------------------------------- /Dockerfile.bench: -------------------------------------------------------------------------------- 1 | FROM ubuntu:24.04 2 | 3 | ARG DEBIAN_FRONTEND=noninteractive 4 | 5 | RUN apt update && apt install -y git build-essential curl libclang-dev 6 | 7 | ENV PATH=/root/.cargo/bin:$PATH 8 | 9 | ADD rust-toolchain /code/redb/rust-toolchain 10 | 11 | RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=$(cat /code/redb/rust-toolchain) 12 | 13 | ADD . /code/redb/ 14 | 15 | RUN cd /code/redb && cargo bench --no-run -p redb-bench 16 | -------------------------------------------------------------------------------- /crates/redb-python/py_publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PYTHON3=/opt/python/cp311-cp311/bin/python3 4 | 5 | cp -r /redb-ro /redb 6 | cd /redb 7 | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=1.85.0 8 | source $HOME/.cargo/env 9 | 10 | cd /tmp 11 | $PYTHON3 -m venv venv 12 | cd /redb/crates/redb-python 13 | source /tmp/venv/bin/activate 14 | python3 -m pip install --upgrade pip 15 | python3 -m pip install maturin 16 | 17 | python3 -m maturin publish 18 | -------------------------------------------------------------------------------- /crates/redb-derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "redb-derive" 3 | description = "Derive macros for redb" 4 | version = "0.1.0" 5 | edition.workspace = true 6 | rust-version.workspace = true 7 | license.workspace = true 8 | homepage.workspace = true 9 | repository.workspace = true 10 | authors.workspace = true 11 | 12 | [lib] 13 | proc-macro = true 14 | 15 | [dependencies] 16 | proc-macro2 = "1.0" 17 | quote = "1.0" 18 | syn = { version = "2.0", features = ["full"] } 19 | 20 | [dev-dependencies] 21 | redb = { path = "../.." } 22 | tempfile = "3.5.0" 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | 12 | # IntelliJ 13 | .idea/ 14 | **/*.iml 15 | 16 | 17 | **/__pycache__ 18 | **/*.pyc 19 | 20 | # Profiling 21 | perf.data* 22 | flamegraph.svg 23 | 24 | # benchmark and test temporary files 25 | /.tmp* 26 | .vscode/settings.json 27 | 28 | **/*.redb 29 | -------------------------------------------------------------------------------- /crates/redb-python/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "redb" 3 | requires-python = ">=3.7" 4 | dynamic = ["version"] 5 | classifier = ["Development Status :: 4 - Beta", 6 | "License :: OSI Approved :: MIT License", 7 | "License :: OSI Approved :: Apache Software License", 8 | "Programming Language :: Python", 9 | "Programming Language :: Python :: 3", 10 | "Programming Language :: Python :: 3 :: Only", 11 | "Programming Language :: Rust"] 12 | 13 | [build-system] 14 | requires = ["maturin>=1.0,<2.0"] 15 | build-backend = "maturin" 16 | 17 | [tool.maturin] 18 | compatibility = "manylinux2014" 19 | -------------------------------------------------------------------------------- /crates/redb-python/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "redb-python" 3 | description = "Python bindings for redb" 4 | publish = false 5 | version = "0.5.0" 6 | edition.workspace = true 7 | rust-version.workspace = true 8 | license.workspace = true 9 | homepage.workspace = true 10 | repository.workspace = true 11 | authors.workspace = true 12 | 13 | [lib] 14 | name = "redb" 15 | doc = false 16 | crate-type = ["cdylib"] 17 | 18 | [build-dependencies] 19 | pyo3-build-config = "0.24.1" 20 | 21 | [dependencies] 22 | pyo3 = { version = "0.24.1", features=["extension-module", "abi3-py37"] } 23 | redb = { path = "../.." } 24 | 25 | [dev-dependencies] 26 | tempfile = "3.5.0" 27 | -------------------------------------------------------------------------------- /examples/int_keys.rs: -------------------------------------------------------------------------------- 1 | use redb::{Database, Error, ReadableDatabase, TableDefinition}; 2 | 3 | const TABLE: TableDefinition = TableDefinition::new("my_data"); 4 | 5 | #[allow(clippy::result_large_err)] 6 | fn main() -> Result<(), Error> { 7 | let db = Database::create("int_keys.redb")?; 8 | let write_txn = db.begin_write()?; 9 | { 10 | let mut table = write_txn.open_table(TABLE)?; 11 | table.insert(0, 0)?; 12 | } 13 | write_txn.commit()?; 14 | 15 | let read_txn = db.begin_read()?; 16 | let table = read_txn.open_table(TABLE)?; 17 | assert_eq!(table.get(0)?.unwrap().value(), 0); 18 | 19 | Ok(()) 20 | } 21 | -------------------------------------------------------------------------------- /fuzz/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "redb-fuzz" 3 | version = "0.0.0" 4 | authors = ["Automatically generated"] 5 | publish = false 6 | edition = "2018" 7 | 8 | [package.metadata] 9 | cargo-fuzz = true 10 | 11 | [dependencies] 12 | arbitrary = { version = "1.1.0", features = ["derive"] } 13 | libfuzzer-sys = { version = "0.4.0", features = ["arbitrary-derive"] } 14 | tempfile = "3.2.0" 15 | rand = "0.8.5" 16 | rand_distr = "0.4.3" 17 | 18 | [dependencies.redb] 19 | path = ".." 20 | 21 | # Prevent this from interfering with workspaces 22 | [workspace] 23 | members = ["."] 24 | 25 | [[bin]] 26 | name = "fuzz_redb" 27 | path = "fuzz_targets/fuzz_redb.rs" 28 | test = false 29 | doc = false 30 | -------------------------------------------------------------------------------- /crates/redb-python/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(clippy::all, clippy::pedantic, clippy::disallowed_methods)] 2 | // TODO: revisit this list and see if we can enable some 3 | #![allow( 4 | clippy::default_trait_access, 5 | clippy::if_not_else, 6 | clippy::iter_not_returning_iterator, 7 | clippy::missing_errors_doc, 8 | clippy::missing_panics_doc, 9 | clippy::module_name_repetitions, 10 | clippy::must_use_candidate, 11 | clippy::needless_pass_by_value, 12 | clippy::redundant_closure_for_method_calls, 13 | clippy::similar_names, 14 | clippy::too_many_lines, 15 | clippy::unnecessary_wraps, 16 | clippy::unreadable_literal 17 | )] 18 | 19 | mod python; 20 | pub use crate::python::redb; 21 | -------------------------------------------------------------------------------- /src/tree_store/page_store/mod.rs: -------------------------------------------------------------------------------- 1 | mod backends; 2 | mod base; 3 | mod bitmap; 4 | mod buddy_allocator; 5 | mod cached_file; 6 | mod fast_hash; 7 | pub mod file_backend; 8 | mod header; 9 | mod layout; 10 | mod lru_cache; 11 | mod page_manager; 12 | mod region; 13 | mod savepoint; 14 | #[allow(clippy::pedantic, dead_code)] 15 | mod xxh3; 16 | 17 | pub use backends::InMemoryBackend; 18 | pub(crate) use backends::ReadOnlyBackend; 19 | pub(crate) use base::{ 20 | MAX_PAIR_LENGTH, MAX_VALUE_LENGTH, Page, PageHint, PageNumber, PageTrackerPolicy, 21 | }; 22 | pub(crate) use header::PAGE_SIZE; 23 | pub(crate) use page_manager::{ 24 | FILE_FORMAT_VERSION3, ShrinkPolicy, TransactionalMemory, xxh3_checksum, 25 | }; 26 | pub use savepoint::Savepoint; 27 | pub(crate) use savepoint::SerializedSavepoint; 28 | 29 | pub(super) use base::{PageImpl, PageMut}; 30 | pub(super) use xxh3::hash128_with_seed; 31 | -------------------------------------------------------------------------------- /crates/redb-bench/benches/redb_benchmark.rs: -------------------------------------------------------------------------------- 1 | use std::env::current_dir; 2 | use std::{fs, process}; 3 | use tempfile::NamedTempFile; 4 | 5 | #[expect(dead_code)] 6 | mod common; 7 | use common::*; 8 | 9 | fn main() { 10 | let _ = env_logger::try_init(); 11 | let tmpdir = current_dir().unwrap().join(".benchmark"); 12 | fs::create_dir(&tmpdir).unwrap(); 13 | 14 | let tmpdir2 = tmpdir.clone(); 15 | ctrlc::set_handler(move || { 16 | fs::remove_dir_all(&tmpdir2).unwrap(); 17 | process::exit(1); 18 | }) 19 | .unwrap(); 20 | 21 | let tmpfile: NamedTempFile = NamedTempFile::new_in(&tmpdir).unwrap(); 22 | let mut db = redb::Database::builder() 23 | .set_cache_size(CACHE_SIZE) 24 | .create(tmpfile.path()) 25 | .unwrap(); 26 | let table = RedbBenchDatabase::new(&mut db); 27 | benchmark(table, tmpfile.path()); 28 | 29 | fs::remove_dir_all(&tmpdir).unwrap(); 30 | } 31 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021 Christopher Berner 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. -------------------------------------------------------------------------------- /src/tree_store/mod.rs: -------------------------------------------------------------------------------- 1 | mod btree; 2 | mod btree_base; 3 | mod btree_iters; 4 | mod btree_mutator; 5 | mod page_store; 6 | mod table_tree; 7 | mod table_tree_base; 8 | 9 | pub(crate) use btree::{ 10 | Btree, BtreeMut, BtreeStats, PagePath, RawBtree, UntypedBtree, UntypedBtreeMut, btree_stats, 11 | }; 12 | pub use btree_base::{AccessGuard, AccessGuardMut, AccessGuardMutInPlace}; 13 | pub(crate) use btree_base::{ 14 | BRANCH, BranchAccessor, BranchMutator, BtreeHeader, Checksum, DEFERRED, LEAF, LeafAccessor, 15 | LeafMutator, RawLeafBuilder, 16 | }; 17 | pub(crate) use btree_iters::{AllPageNumbersBtreeIter, BtreeExtractIf, BtreeRangeIter}; 18 | pub(crate) use page_store::ReadOnlyBackend; 19 | pub(crate) use page_store::{ 20 | FILE_FORMAT_VERSION3, MAX_PAIR_LENGTH, MAX_VALUE_LENGTH, PAGE_SIZE, Page, PageHint, PageNumber, 21 | PageTrackerPolicy, SerializedSavepoint, ShrinkPolicy, TransactionalMemory, 22 | }; 23 | pub use page_store::{InMemoryBackend, Savepoint, file_backend}; 24 | pub(crate) use table_tree::{PageListMut, TableTree, TableTreeMut}; 25 | pub(crate) use table_tree_base::{InternalTableDefinition, TableType}; 26 | -------------------------------------------------------------------------------- /src/tree_store/page_store/fast_hash.rs: -------------------------------------------------------------------------------- 1 | use crate::tree_store::PageNumber; 2 | use std::collections::{HashMap, HashSet}; 3 | use std::hash::{BuildHasherDefault, Hasher}; 4 | 5 | // See "Computationally easy, spectrally good multipliers for congruential pseudorandom number generators" by Steele & Vigna 6 | const K: u64 = 0xf135_7aea_2e62_a9c5; 7 | 8 | pub(crate) type FastHashMapU64 = HashMap>; 9 | pub(crate) type PageNumberHashSet = HashSet>; 10 | 11 | #[derive(Copy, Clone, Default, Eq, PartialEq)] 12 | pub(crate) struct FastHasher64 { 13 | hash: u64, 14 | } 15 | 16 | impl Hasher for FastHasher64 { 17 | fn finish(&self) -> u64 { 18 | #[cfg(target_pointer_width = "64")] 19 | const ROTATE: u32 = 26; 20 | #[cfg(target_pointer_width = "32")] 21 | const ROTATE: u32 = 15; 22 | 23 | self.hash.rotate_left(ROTATE) 24 | } 25 | 26 | fn write(&mut self, _bytes: &[u8]) { 27 | unreachable!("Only hashing 8 bytes is supported"); 28 | } 29 | 30 | fn write_u64(&mut self, x: u64) { 31 | debug_assert_eq!(self.hash, 0); 32 | self.hash = x.wrapping_mul(K); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /crates/redb-bench/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "redb-bench" 3 | version = "0.0.0" 4 | publish = false 5 | edition.workspace = true 6 | rust-version.workspace = true 7 | license.workspace = true 8 | homepage.workspace = true 9 | repository.workspace = true 10 | authors.workspace = true 11 | 12 | # Common test/bench dependencies 13 | [dev-dependencies] 14 | redb = { path = "../.." } 15 | rand = "0.9" 16 | tempfile = "3.5.0" 17 | walkdir = "2.5.0" 18 | byte-unit = "5.1.6" 19 | fastrand = "2.0.0" 20 | sled = "0.34.7" 21 | libc = "0.2.99" 22 | ctrlc = "3.2.3" 23 | heed = "0.22" 24 | rocksdb = { version = "0.22.0", default-features = false, features = ["lz4"] } 25 | fjall = "=2.11" 26 | rusqlite = { version = "0.37", features = ["bundled"] } 27 | comfy-table = "7.0.1" 28 | env_logger = "0.11" 29 | 30 | [target.'cfg(target_os = "linux")'.dev-dependencies] 31 | io-uring = "0.7.4" 32 | 33 | [[bench]] 34 | name = "atomics_benchmark" 35 | harness = false 36 | 37 | [[bench]] 38 | name = "multithreaded_insert_benchmark" 39 | harness = false 40 | 41 | [[bench]] 42 | name = "userspace_cache_benchmark" 43 | harness = false 44 | 45 | [[bench]] 46 | name = "savepoint_benchmark" 47 | harness = false 48 | 49 | [[bench]] 50 | name = "lmdb_benchmark" 51 | harness = false 52 | 53 | [[bench]] 54 | name = "redb_benchmark" 55 | harness = false 56 | 57 | [[bench]] 58 | name = "large_values_benchmark" 59 | harness = false 60 | 61 | [[bench]] 62 | name = "int_benchmark" 63 | harness = false 64 | 65 | [[bench]] 66 | name = "syscall_benchmark" 67 | harness = false 68 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "redb" 3 | description = "Rust Embedded DataBase" 4 | readme = "README.md" 5 | version = "3.1.0" 6 | exclude = ["fuzz/"] 7 | edition.workspace = true 8 | rust-version.workspace = true 9 | license.workspace = true 10 | homepage.workspace = true 11 | repository.workspace = true 12 | authors.workspace = true 13 | 14 | [workspace] 15 | members = [".", "crates/redb-bench", "crates/redb-derive", "crates/redb-python"] 16 | default-members = [".", "crates/redb-derive", "crates/redb-python"] 17 | 18 | [workspace.package] 19 | edition = "2024" 20 | rust-version = "1.89" 21 | license = "MIT OR Apache-2.0" 22 | homepage = "https://www.redb.org" 23 | repository = "https://github.com/cberner/redb" 24 | authors = ["Christopher Berner "] 25 | 26 | [dependencies] 27 | log = { version = "0.4.17", optional = true } 28 | chrono_v0_4 = { package = "chrono", version= "0.4.41", optional = true } 29 | uuid = { version= "1.17.0", optional = true } 30 | 31 | [target.'cfg(target_os = "wasi")'.dependencies] 32 | libc = "0.2.174" 33 | 34 | # Common test/bench dependencies 35 | [dev-dependencies] 36 | rand = "0.9" 37 | tempfile = "3.5.0" 38 | # for backwards compatibility testing - pin at 2.6.0 39 | redb2_6 = { version = "=2.6.0", package = "redb" } 40 | bincode = "2.0.1" 41 | uuid = { version= "1.17.0", features = ["v4"] } 42 | 43 | [features] 44 | # Enables log messages 45 | logging = ["dep:log"] 46 | # Enable cache hit metrics 47 | cache_metrics = [] 48 | 49 | [profile.bench] 50 | debug = true 51 | 52 | [lints.clippy] 53 | big_endian_bytes = "deny" 54 | dbg_macro = "deny" 55 | host_endian_bytes = "deny" 56 | -------------------------------------------------------------------------------- /src/tree_store/page_store/file_backend/fallback.rs: -------------------------------------------------------------------------------- 1 | use crate::{DatabaseError, Result, StorageBackend}; 2 | use std::fs::File; 3 | use std::io; 4 | use std::io::{Read, Seek, SeekFrom, Write}; 5 | use std::sync::Mutex; 6 | 7 | /// Stores a database as a file on-disk. 8 | #[derive(Debug)] 9 | pub struct FileBackend { 10 | file: Mutex, 11 | } 12 | 13 | impl FileBackend { 14 | /// Creates a new backend which stores data to the given file. 15 | pub fn new(file: File) -> Result { 16 | Self::new_internal(file, false) 17 | } 18 | 19 | pub(crate) fn new_internal(file: File, _: bool) -> Result { 20 | Ok(Self { 21 | file: Mutex::new(file), 22 | }) 23 | } 24 | } 25 | 26 | impl StorageBackend for FileBackend { 27 | fn len(&self) -> Result { 28 | Ok(self.file.lock().unwrap().metadata()?.len()) 29 | } 30 | 31 | fn read(&self, offset: u64, out: &mut [u8]) -> Result<(), io::Error> { 32 | let mut file = self.file.lock().unwrap(); 33 | file.seek(SeekFrom::Start(offset))?; 34 | file.read_exact(out)?; 35 | Ok(()) 36 | } 37 | 38 | fn set_len(&self, len: u64) -> Result<(), io::Error> { 39 | self.file.lock().unwrap().set_len(len) 40 | } 41 | 42 | fn sync_data(&self) -> Result<(), io::Error> { 43 | self.file.lock().unwrap().sync_data() 44 | } 45 | 46 | fn write(&self, offset: u64, data: &[u8]) -> Result<(), io::Error> { 47 | let mut file = self.file.lock().unwrap(); 48 | file.seek(SeekFrom::Start(offset))?; 49 | file.write_all(data) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/types/uuid.rs: -------------------------------------------------------------------------------- 1 | use crate::{Key, TypeName, Value}; 2 | use std::cmp::Ordering; 3 | use uuid::Uuid; 4 | 5 | impl Value for Uuid { 6 | type SelfType<'a> 7 | = Uuid 8 | where 9 | Self: 'a; 10 | type AsBytes<'a> 11 | = &'a uuid::Bytes 12 | where 13 | Self: 'a; 14 | 15 | fn fixed_width() -> Option { 16 | Some(16) 17 | } 18 | 19 | fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> 20 | where 21 | Self: 'a, 22 | { 23 | Uuid::from_slice(data).unwrap() 24 | } 25 | 26 | fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> 27 | where 28 | Self: 'b, 29 | { 30 | value.as_bytes() 31 | } 32 | 33 | fn type_name() -> TypeName { 34 | TypeName::new("uuid::Uuid") 35 | } 36 | } 37 | 38 | impl Key for Uuid { 39 | fn compare(data1: &[u8], data2: &[u8]) -> Ordering { 40 | data1.cmp(data2) 41 | } 42 | } 43 | 44 | #[cfg(test)] 45 | mod tests { 46 | use crate::{Database, Key, ReadableDatabase, TableDefinition, Value}; 47 | use tempfile::NamedTempFile; 48 | use uuid::Uuid; 49 | 50 | const UUID_TABLE: TableDefinition = TableDefinition::new("table"); 51 | 52 | #[test] 53 | fn test_uuid_ordering() { 54 | let uuid1 = Uuid::new_v4(); 55 | let uuid2 = Uuid::new_v4(); 56 | let bytes1 = ::as_bytes(&uuid1); 57 | let bytes2 = ::as_bytes(&uuid2); 58 | assert_eq!( 59 | uuid1.cmp(&uuid2), 60 | Uuid::compare(bytes1.as_slice(), bytes2.as_slice()) 61 | ); 62 | } 63 | 64 | #[test] 65 | fn test_uuid_table() { 66 | let uuid1 = Uuid::new_v4(); 67 | let uuid2 = Uuid::new_v4(); 68 | let db = Database::create(NamedTempFile::new().unwrap()).unwrap(); 69 | let write_txn = db.begin_write().unwrap(); 70 | { 71 | let mut table = write_txn.open_table(UUID_TABLE).unwrap(); 72 | table.insert(uuid1, uuid2).unwrap(); 73 | } 74 | write_txn.commit().unwrap(); 75 | 76 | let read_txn = db.begin_read().unwrap(); 77 | { 78 | let table = read_txn.open_table(UUID_TABLE).unwrap(); 79 | let value = table.get(&uuid1).unwrap().unwrap(); 80 | assert_eq!(value.value(), uuid2); 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/complex_types.rs: -------------------------------------------------------------------------------- 1 | use crate::types::{TypeName, Value}; 2 | 3 | // Encode len as a varint and store it at the end of output 4 | pub(super) fn encode_varint_len(len: usize, output: &mut Vec) { 5 | if len < 254 { 6 | output.push(len.try_into().unwrap()); 7 | } else if len <= u16::MAX.into() { 8 | let u16_len: u16 = len.try_into().unwrap(); 9 | output.push(254); 10 | output.extend_from_slice(&u16_len.to_le_bytes()); 11 | } else { 12 | let u32_len: u32 = len.try_into().unwrap(); 13 | output.push(255); 14 | output.extend_from_slice(&u32_len.to_le_bytes()); 15 | } 16 | } 17 | 18 | // Decode a variable length int starting at the beginning of data 19 | // Returns (decoded length, length consumed of `data`) 20 | pub(super) fn decode_varint_len(data: &[u8]) -> (usize, usize) { 21 | match data[0] { 22 | 0..=253 => (data[0] as usize, 1), 23 | 254 => ( 24 | u16::from_le_bytes(data[1..3].try_into().unwrap()) as usize, 25 | 3, 26 | ), 27 | 255 => ( 28 | u32::from_le_bytes(data[1..5].try_into().unwrap()) as usize, 29 | 5, 30 | ), 31 | } 32 | } 33 | 34 | impl Value for Vec { 35 | type SelfType<'a> 36 | = Vec> 37 | where 38 | Self: 'a; 39 | type AsBytes<'a> 40 | = Vec 41 | where 42 | Self: 'a; 43 | 44 | fn fixed_width() -> Option { 45 | None 46 | } 47 | 48 | fn from_bytes<'a>(data: &'a [u8]) -> Vec> 49 | where 50 | Self: 'a, 51 | { 52 | let (elements, mut offset) = decode_varint_len(data); 53 | let mut result = Vec::with_capacity(elements); 54 | for _ in 0..elements { 55 | let element_len = if let Some(len) = T::fixed_width() { 56 | len 57 | } else { 58 | let (len, consumed) = decode_varint_len(&data[offset..]); 59 | offset += consumed; 60 | len 61 | }; 62 | result.push(T::from_bytes(&data[offset..(offset + element_len)])); 63 | offset += element_len; 64 | } 65 | assert_eq!(offset, data.len()); 66 | result 67 | } 68 | 69 | fn as_bytes<'a, 'b: 'a>(value: &'a Vec>) -> Vec 70 | where 71 | Self: 'b, 72 | { 73 | let mut result = if let Some(width) = T::fixed_width() { 74 | Vec::with_capacity(value.len() * width + 5) 75 | } else { 76 | Vec::with_capacity(value.len() * 2 + 5) 77 | }; 78 | encode_varint_len(value.len(), &mut result); 79 | 80 | for element in value { 81 | let serialized = T::as_bytes(element); 82 | if T::fixed_width().is_none() { 83 | encode_varint_len(serialized.as_ref().len(), &mut result); 84 | } 85 | result.extend_from_slice(serialized.as_ref()); 86 | } 87 | result 88 | } 89 | 90 | fn type_name() -> TypeName { 91 | TypeName::internal(&format!("Vec<{}>", T::type_name().name())) 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /examples/bincode_keys.rs: -------------------------------------------------------------------------------- 1 | use std::any::type_name; 2 | use std::cmp::Ordering; 3 | use std::fmt::Debug; 4 | 5 | use bincode::{Decode, Encode, decode_from_slice, encode_to_vec}; 6 | use redb::{Database, Error, Key, Range, ReadableDatabase, TableDefinition, TypeName, Value}; 7 | 8 | #[derive(Debug, Decode, Encode, PartialEq, Eq, PartialOrd, Ord)] 9 | struct SomeKey { 10 | foo: String, 11 | bar: i32, 12 | } 13 | 14 | #[derive(Debug, Decode, Encode, PartialEq)] 15 | struct SomeValue { 16 | foo: [f64; 3], 17 | bar: bool, 18 | } 19 | 20 | const TABLE: TableDefinition, Bincode> = 21 | TableDefinition::new("my_data"); 22 | 23 | #[allow(clippy::result_large_err)] 24 | fn main() -> Result<(), Error> { 25 | let some_key = SomeKey { 26 | foo: "hello world".to_string(), 27 | bar: 42, 28 | }; 29 | let some_value = SomeValue { 30 | foo: [1., 2., 3.], 31 | bar: true, 32 | }; 33 | let lower = SomeKey { 34 | foo: "a".to_string(), 35 | bar: 42, 36 | }; 37 | let upper = SomeKey { 38 | foo: "z".to_string(), 39 | bar: 42, 40 | }; 41 | 42 | let db = Database::create("bincode_keys.redb")?; 43 | let write_txn = db.begin_write()?; 44 | { 45 | let mut table = write_txn.open_table(TABLE)?; 46 | 47 | table.insert(&some_key, &some_value).unwrap(); 48 | } 49 | write_txn.commit()?; 50 | 51 | let read_txn = db.begin_read()?; 52 | let table = read_txn.open_table(TABLE)?; 53 | 54 | let mut iter: Range, Bincode> = table.range(lower..upper).unwrap(); 55 | assert_eq!(iter.next().unwrap().unwrap().1.value(), some_value); 56 | assert!(iter.next().is_none()); 57 | 58 | Ok(()) 59 | } 60 | 61 | /// Wrapper type to handle keys and values using bincode serialization 62 | #[derive(Debug)] 63 | pub struct Bincode(pub T); 64 | 65 | impl Value for Bincode 66 | where 67 | T: Debug + Encode + Decode<()>, 68 | { 69 | type SelfType<'a> 70 | = T 71 | where 72 | Self: 'a; 73 | 74 | type AsBytes<'a> 75 | = Vec 76 | where 77 | Self: 'a; 78 | 79 | fn fixed_width() -> Option { 80 | None 81 | } 82 | 83 | fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> 84 | where 85 | Self: 'a, 86 | { 87 | decode_from_slice(data, bincode::config::standard()) 88 | .unwrap() 89 | .0 90 | } 91 | 92 | fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> 93 | where 94 | Self: 'a, 95 | Self: 'b, 96 | { 97 | encode_to_vec(value, bincode::config::standard()).unwrap() 98 | } 99 | 100 | fn type_name() -> TypeName { 101 | TypeName::new(&format!("Bincode<{}>", type_name::())) 102 | } 103 | } 104 | 105 | impl Key for Bincode 106 | where 107 | T: Debug + Decode<()> + Encode + Ord, 108 | { 109 | fn compare(data1: &[u8], data2: &[u8]) -> Ordering { 110 | Self::from_bytes(data1).cmp(&Self::from_bytes(data2)) 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /examples/multithread.rs: -------------------------------------------------------------------------------- 1 | use redb::{Database, Error, TableDefinition}; 2 | use redb::{ReadableDatabase, TableHandle}; 3 | use std::time::Instant; 4 | use std::{sync::Arc, time::Duration}; 5 | 6 | #[allow(clippy::result_large_err)] 7 | fn main() -> Result<(), Error> { 8 | let db = Database::create("my_db.redb")?; 9 | let definition: TableDefinition<&str, u32> = TableDefinition::new("my_data"); 10 | 11 | let db = Arc::new(db); 12 | // Seed the database with some information 13 | let write_txn = db.begin_write()?; 14 | { 15 | let mut table = write_txn.open_table(definition)?; 16 | table.insert(&0.to_string().as_str(), 0_u32)?; 17 | // The resulting table should have a different "a" value each time this example is run 18 | table.insert("a".to_string().as_str(), 0_u32)?; 19 | } 20 | write_txn.commit()?; 21 | 22 | let read_threads = 8; 23 | let write_threads = 2; 24 | let mut handles = Vec::with_capacity(read_threads + write_threads); 25 | for i in 0..read_threads { 26 | let db = db.clone(); 27 | let h = std::thread::spawn(move || -> Result<(), Error> { 28 | let start = Instant::now(); 29 | while start.elapsed() < Duration::from_millis(100) { 30 | let read_txn = db.begin_read()?; 31 | // Print every (key, value) pair in the table 32 | let table = read_txn.open_table(definition)?; 33 | for (k, v) in table.range("0"..)?.flatten() { 34 | println!("From read_thread #{}: {:?}, {:?}", i, k.value(), v.value()); 35 | } 36 | } 37 | Ok(()) 38 | }); 39 | handles.push(h); 40 | } 41 | 42 | for i in 0..write_threads { 43 | let db = db.clone(); 44 | let h = std::thread::spawn(move || -> Result<(), Error> { 45 | let start = Instant::now(); 46 | while start.elapsed() < Duration::from_millis(100) { 47 | let write_txn = db.begin_write()?; 48 | { 49 | let mut table = write_txn.open_table(definition)?; 50 | table.insert(&i.to_string().as_str(), i as u32)?; 51 | // The resulting table should have a different "a" value each time this example is run 52 | table.insert("a".to_string().as_str(), i as u32)?; 53 | println!("Inserted data from write_thread #{i}"); 54 | } 55 | write_txn.commit()?; 56 | } 57 | Ok(()) 58 | }); 59 | handles.push(h); 60 | } 61 | 62 | // See if there any errors were returned from the threads 63 | for handle in handles { 64 | if let Err(e) = handle.join() { 65 | println!("{e:?}"); 66 | } 67 | } 68 | 69 | // Check that the `Database` has the table (and only the table) that we created 70 | let read_txn = db.begin_read()?; 71 | let tables = read_txn.list_tables()?; 72 | for table in tables { 73 | println!("Table: {}", table.name()); 74 | let _d = TableDefinition::<&str, u32>::new(table.name()); 75 | } 76 | 77 | // Print every (key, value) pair in the table 78 | let table = read_txn.open_table(definition)?; 79 | for (k, v) in table.range("0"..)?.flatten() { 80 | println!("{:?}, {:?}", k.value(), v.value()); 81 | } 82 | 83 | Ok(()) 84 | } 85 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | build: pre 2 | cargo build --all-targets --all-features 3 | cargo doc 4 | 5 | build_all: pre_all 6 | cargo build --all --all-targets --all-features 7 | cargo doc --all 8 | 9 | pre: 10 | cargo deny --workspace --all-features check licenses 11 | cargo fmt --all -- --check 12 | cargo clippy --all-targets --all-features 13 | 14 | pre_all: 15 | cargo deny --workspace --all-features check licenses 16 | cargo fmt --all -- --check 17 | cargo clippy --all --all-targets --all-features 18 | 19 | release: pre 20 | cargo build --release 21 | 22 | flamegraph: 23 | cargo flamegraph -p redb-bench --bench redb_benchmark 24 | firefox ./flamegraph.svg 25 | 26 | publish_py: test_py 27 | docker pull quay.io/pypa/manylinux2014_x86_64 28 | MATURIN_PYPI_TOKEN=$(cat ~/.pypi/redb_token) docker run -it --rm -e "MATURIN_PYPI_TOKEN" -v `pwd`:/redb-ro:ro quay.io/pypa/manylinux2014_x86_64 /redb-ro/crates/redb-python/py_publish.sh 29 | 30 | test_py: install_py 31 | python3 -m unittest discover --start-directory=./crates/redb-python 32 | 33 | install_py: pre 34 | maturin develop --manifest-path=./crates/redb-python/Cargo.toml 35 | 36 | test: pre 37 | RUST_BACKTRACE=1 cargo test --all-features 38 | 39 | test_all: build_all 40 | RUST_BACKTRACE=1 cargo test --all --all-features 41 | 42 | test_wasi: 43 | rustup install nightly-2025-07-26 --target wasm32-wasip1-threads 44 | # Uses cargo pkgid because "redb" is ambiguous with the test dependency on an old version of redb 45 | cargo +nightly-2025-07-26 test -p $(cargo pkgid) --target=wasm32-wasip1-threads -- --nocapture 46 | cargo +nightly-2025-07-26 test -p redb-derive --target=wasm32-wasip1-threads -- --nocapture 47 | 48 | bench bench='redb_benchmark': pre 49 | cargo bench -p redb-bench --bench {{bench}} 50 | 51 | build_bench_container: 52 | docker build -t redb-bench:latest -f Dockerfile.bench . 53 | 54 | bench_containerized bench='lmdb_benchmark': build_bench_container 55 | # Exec the binary directly, because at low memory limits there may not be enough to invoke cargo & rustc 56 | docker run --rm -it --memory=4g redb-bench:latest bash -c "cd /code/redb && ./target/release/deps/{{bench}}-*" 57 | 58 | watch +args='test': 59 | cargo watch --clear --exec "{{args}}" 60 | 61 | fuzz: pre 62 | cargo fuzz run --sanitizer=none fuzz_redb -- -max_len=10000 63 | 64 | fuzz_cmin: 65 | cargo fuzz cmin --sanitizer=none fuzz_redb -- -max_len=10000 66 | 67 | fuzz_ci: pre_all 68 | cargo fuzz run --sanitizer=none fuzz_redb -- -max_len=10000 -max_total_time=60 69 | 70 | fuzz_coverage: pre 71 | #!/usr/bin/env bash 72 | set -euxo pipefail 73 | rustup component add llvm-tools-preview 74 | RUST_SYSROOT=`cargo rustc -- --print sysroot 2>/dev/null` 75 | LLVM_COV=`find $RUST_SYSROOT -name llvm-cov` 76 | echo $LLVM_COV 77 | cargo fuzz coverage --sanitizer=none fuzz_redb 78 | $LLVM_COV show target/*/coverage/*/release/fuzz_redb --format html \ 79 | -instr-profile=fuzz/coverage/fuzz_redb/coverage.profdata \ 80 | -ignore-filename-regex='.*(cargo/registry|redb/fuzz|rustc).*' > fuzz/coverage/coverage_report.html 81 | $LLVM_COV report target/*/coverage/*/release/fuzz_redb \ 82 | -instr-profile=fuzz/coverage/fuzz_redb/coverage.profdata \ 83 | -ignore-filename-regex='.*(cargo/registry|redb/fuzz|rustc).*' 84 | firefox ./fuzz/coverage/coverage_report.html 85 | -------------------------------------------------------------------------------- /src/tree_store/page_store/backends.rs: -------------------------------------------------------------------------------- 1 | use crate::StorageBackend; 2 | use std::io; 3 | use std::io::Error; 4 | use std::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard}; 5 | 6 | #[derive(Debug)] 7 | pub(crate) struct ReadOnlyBackend { 8 | inner: Box, 9 | } 10 | 11 | impl ReadOnlyBackend { 12 | pub fn new(inner: Box) -> Self { 13 | Self { inner } 14 | } 15 | } 16 | 17 | impl StorageBackend for ReadOnlyBackend { 18 | fn len(&self) -> Result { 19 | self.inner.len() 20 | } 21 | 22 | fn read(&self, offset: u64, out: &mut [u8]) -> Result<(), Error> { 23 | self.inner.read(offset, out) 24 | } 25 | 26 | fn set_len(&self, _len: u64) -> Result<(), Error> { 27 | unreachable!() 28 | } 29 | 30 | fn sync_data(&self) -> Result<(), Error> { 31 | unreachable!() 32 | } 33 | 34 | fn write(&self, _offset: u64, _data: &[u8]) -> Result<(), Error> { 35 | unreachable!() 36 | } 37 | 38 | fn close(&self) -> Result<(), Error> { 39 | self.inner.close() 40 | } 41 | } 42 | 43 | /// Acts as temporal in-memory database storage. 44 | #[derive(Debug, Default)] 45 | pub struct InMemoryBackend(RwLock>); 46 | 47 | impl InMemoryBackend { 48 | fn out_of_range() -> io::Error { 49 | io::Error::new(io::ErrorKind::InvalidInput, "Index out-of-range.") 50 | } 51 | } 52 | 53 | impl InMemoryBackend { 54 | /// Creates a new, empty memory backend. 55 | pub fn new() -> Self { 56 | Self::default() 57 | } 58 | 59 | /// Gets a read guard for this backend. 60 | fn read(&self) -> RwLockReadGuard<'_, Vec> { 61 | self.0.read().expect("Could not acquire read lock.") 62 | } 63 | 64 | /// Gets a write guard for this backend. 65 | fn write(&self) -> RwLockWriteGuard<'_, Vec> { 66 | self.0.write().expect("Could not acquire write lock.") 67 | } 68 | } 69 | 70 | impl StorageBackend for InMemoryBackend { 71 | fn len(&self) -> Result { 72 | Ok(self.read().len() as u64) 73 | } 74 | 75 | fn read(&self, offset: u64, out: &mut [u8]) -> Result<(), io::Error> { 76 | let guard = self.read(); 77 | let offset = usize::try_from(offset).map_err(|_| Self::out_of_range())?; 78 | if offset + out.len() <= guard.len() { 79 | out.copy_from_slice(&guard[offset..offset + out.len()]); 80 | Ok(()) 81 | } else { 82 | Err(Self::out_of_range()) 83 | } 84 | } 85 | 86 | fn set_len(&self, len: u64) -> Result<(), io::Error> { 87 | let mut guard = self.write(); 88 | let len = usize::try_from(len).map_err(|_| Self::out_of_range())?; 89 | if guard.len() < len { 90 | let additional = len - guard.len(); 91 | guard.reserve(additional); 92 | for _ in 0..additional { 93 | guard.push(0); 94 | } 95 | } else { 96 | guard.truncate(len); 97 | } 98 | 99 | Ok(()) 100 | } 101 | 102 | fn sync_data(&self) -> Result<(), io::Error> { 103 | Ok(()) 104 | } 105 | 106 | fn write(&self, offset: u64, data: &[u8]) -> Result<(), io::Error> { 107 | let mut guard = self.write(); 108 | let offset = usize::try_from(offset).map_err(|_| Self::out_of_range())?; 109 | if offset + data.len() <= guard.len() { 110 | guard[offset..offset + data.len()].copy_from_slice(data); 111 | Ok(()) 112 | } else { 113 | Err(Self::out_of_range()) 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/tree_store/page_store/lru_cache.rs: -------------------------------------------------------------------------------- 1 | use crate::tree_store::page_store::fast_hash::FastHashMapU64; 2 | use std::collections::VecDeque; 3 | use std::sync::atomic::{AtomicBool, Ordering}; 4 | 5 | #[derive(Default)] 6 | pub struct LRUCache { 7 | // AtomicBool is the second chance flag 8 | cache: FastHashMapU64<(T, AtomicBool)>, 9 | lru_queue: VecDeque, 10 | } 11 | 12 | impl LRUCache { 13 | pub(crate) fn new() -> Self { 14 | Self { 15 | cache: Default::default(), 16 | lru_queue: Default::default(), 17 | } 18 | } 19 | 20 | pub(crate) fn len(&self) -> usize { 21 | self.cache.len() 22 | } 23 | 24 | pub(crate) fn insert(&mut self, key: u64, value: T) -> Option { 25 | let result = self 26 | .cache 27 | .insert(key, (value, AtomicBool::new(false))) 28 | .map(|(x, _)| x); 29 | if result.is_none() { 30 | self.lru_queue.push_back(key); 31 | } 32 | result 33 | } 34 | 35 | pub(crate) fn remove(&mut self, key: u64) -> Option { 36 | if let Some((value, _)) = self.cache.remove(&key) { 37 | if self.lru_queue.len() > 2 * self.cache.len() { 38 | // Cycle two elements of the LRU queue to ensure it doesn't grow without bound 39 | for _ in 0..2 { 40 | if let Some(removed_key) = self.lru_queue.pop_front() 41 | && let Some((_, second_chance)) = self.cache.get(&removed_key) 42 | { 43 | second_chance.store(false, Ordering::Release); 44 | self.lru_queue.push_back(removed_key); 45 | } 46 | } 47 | } 48 | Some(value) 49 | } else { 50 | None 51 | } 52 | } 53 | 54 | pub(crate) fn get(&self, key: u64) -> Option<&T> { 55 | if let Some((value, second_chance)) = self.cache.get(&key) { 56 | second_chance.store(true, Ordering::Release); 57 | Some(value) 58 | } else { 59 | None 60 | } 61 | } 62 | 63 | pub(crate) fn get_mut(&mut self, key: u64) -> Option<&mut T> { 64 | if let Some((value, second_chance)) = self.cache.get_mut(&key) { 65 | second_chance.store(true, Ordering::Release); 66 | Some(value) 67 | } else { 68 | None 69 | } 70 | } 71 | 72 | pub(crate) fn iter(&self) -> impl ExactSizeIterator { 73 | self.cache.iter().map(|(k, (v, _))| (k, v)) 74 | } 75 | 76 | pub(crate) fn iter_mut(&mut self) -> impl ExactSizeIterator { 77 | self.cache.iter_mut().map(|(k, (v, _))| (k, v)) 78 | } 79 | 80 | pub(crate) fn pop_lowest_priority(&mut self) -> Option<(u64, T)> { 81 | while let Some(key) = self.lru_queue.pop_front() { 82 | if let Some((_, second_chance)) = self.cache.get(&key) { 83 | if second_chance 84 | .compare_exchange(true, false, Ordering::AcqRel, Ordering::Acquire) 85 | .is_ok() 86 | { 87 | self.lru_queue.push_back(key); 88 | } else { 89 | let (value, _) = self.cache.remove(&key).unwrap(); 90 | return Some((key, value)); 91 | } 92 | } 93 | } 94 | None 95 | } 96 | 97 | pub(crate) fn clear(&mut self) { 98 | self.cache.shrink_to_fit(); 99 | self.cache.clear(); 100 | self.lru_queue.shrink_to_fit(); 101 | self.lru_queue.clear(); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /crates/redb-bench/benches/multithreaded_insert_benchmark.rs: -------------------------------------------------------------------------------- 1 | use std::env::current_dir; 2 | use std::{fs, process, thread}; 3 | use tempfile::NamedTempFile; 4 | 5 | use rand::rngs::StdRng; 6 | use rand::{Rng, SeedableRng}; 7 | use redb::{Database, ReadableDatabase, ReadableTableMetadata, TableDefinition}; 8 | use std::time::Instant; 9 | 10 | const ELEMENTS: u64 = 1_000_000; 11 | const RNG_SEED: u64 = 3; 12 | 13 | const TABLE1: TableDefinition = TableDefinition::new("x"); 14 | const TABLE2: TableDefinition = TableDefinition::new("y"); 15 | 16 | #[inline(never)] 17 | fn single_threaded(values: &[u128]) { 18 | let tmpfile: NamedTempFile = NamedTempFile::new_in(current_dir().unwrap()).unwrap(); 19 | let db = Database::builder().create(tmpfile.path()).unwrap(); 20 | 21 | let start = Instant::now(); 22 | let write_txn = db.begin_write().unwrap(); 23 | { 24 | let mut table1 = write_txn.open_table(TABLE1).unwrap(); 25 | let mut table2 = write_txn.open_table(TABLE2).unwrap(); 26 | 27 | for value in values.iter() { 28 | table1.insert(value, value).unwrap(); 29 | table2.insert(value, value).unwrap(); 30 | } 31 | } 32 | write_txn.commit().unwrap(); 33 | let end = Instant::now(); 34 | let duration = end - start; 35 | println!( 36 | "single threaded load: {} pairs in {}ms", 37 | 2 * ELEMENTS, 38 | duration.as_millis() 39 | ); 40 | let read_txn = db.begin_read().unwrap(); 41 | let table = read_txn.open_table(TABLE1).unwrap(); 42 | assert_eq!(table.len().unwrap(), ELEMENTS); 43 | let table = read_txn.open_table(TABLE2).unwrap(); 44 | assert_eq!(table.len().unwrap(), ELEMENTS); 45 | } 46 | 47 | #[inline(never)] 48 | fn multi_threaded(values: &[u128]) { 49 | let tmpfile: NamedTempFile = NamedTempFile::new_in(current_dir().unwrap()).unwrap(); 50 | let db = Database::builder().create(tmpfile.path()).unwrap(); 51 | 52 | let start = Instant::now(); 53 | let write_txn = db.begin_write().unwrap(); 54 | { 55 | let mut table1 = write_txn.open_table(TABLE1).unwrap(); 56 | let mut table2 = write_txn.open_table(TABLE2).unwrap(); 57 | 58 | thread::scope(|s| { 59 | s.spawn(|| { 60 | for value in values.iter() { 61 | table1.insert(value, value).unwrap(); 62 | } 63 | }); 64 | s.spawn(|| { 65 | for value in values.iter() { 66 | table2.insert(value, value).unwrap(); 67 | } 68 | }); 69 | }); 70 | } 71 | write_txn.commit().unwrap(); 72 | let end = Instant::now(); 73 | let duration = end - start; 74 | println!( 75 | "2 threaded load: {} pairs in {}ms", 76 | 2 * ELEMENTS, 77 | duration.as_millis() 78 | ); 79 | let read_txn = db.begin_read().unwrap(); 80 | let table = read_txn.open_table(TABLE1).unwrap(); 81 | assert_eq!(table.len().unwrap(), ELEMENTS); 82 | let table = read_txn.open_table(TABLE2).unwrap(); 83 | assert_eq!(table.len().unwrap(), ELEMENTS); 84 | } 85 | 86 | // TODO: multi-threaded inserts are slower. Probably due to lock contention checking dirty pages 87 | fn main() { 88 | let mut rng = StdRng::seed_from_u64(RNG_SEED); 89 | let mut values = vec![]; 90 | for _ in 0..ELEMENTS { 91 | values.push(rng.random()); 92 | } 93 | 94 | let tmpdir = current_dir().unwrap().join(".benchmark"); 95 | fs::create_dir(&tmpdir).unwrap(); 96 | 97 | let tmpdir2 = tmpdir.clone(); 98 | ctrlc::set_handler(move || { 99 | fs::remove_dir_all(&tmpdir2).unwrap(); 100 | process::exit(1); 101 | }) 102 | .unwrap(); 103 | 104 | single_threaded(&values); 105 | 106 | multi_threaded(&values); 107 | 108 | fs::remove_dir_all(&tmpdir).unwrap(); 109 | } 110 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(clippy::all, clippy::pedantic, clippy::disallowed_methods)] 2 | // TODO: revisit this list and see if we can enable some 3 | #![allow( 4 | clippy::default_trait_access, 5 | clippy::if_not_else, 6 | clippy::iter_not_returning_iterator, 7 | clippy::missing_errors_doc, 8 | clippy::missing_panics_doc, 9 | clippy::module_name_repetitions, 10 | clippy::must_use_candidate, 11 | clippy::needless_pass_by_value, 12 | clippy::redundant_closure_for_method_calls, 13 | clippy::similar_names, 14 | clippy::too_many_lines, 15 | clippy::unnecessary_wraps, 16 | clippy::unreadable_literal 17 | )] 18 | 19 | //! # redb 20 | //! 21 | //! A simple, portable, high-performance, ACID, embedded key-value store. 22 | //! 23 | //! redb is written in pure Rust and is loosely inspired by [lmdb][lmdb]. Data is stored in a collection 24 | //! of copy-on-write B-trees. For more details, see the [design doc][design]. 25 | //! 26 | //! # Features 27 | //! 28 | //! - Zero-copy, thread-safe, `BTreeMap` based API 29 | //! - Fully ACID-compliant transactions 30 | //! - MVCC support for concurrent readers & writer, without blocking 31 | //! - Crash-safe by default 32 | //! - Savepoints and rollbacks 33 | //! 34 | //! # Example 35 | //! 36 | //! ``` 37 | //! use redb::{Database, Error, ReadableDatabase, ReadableTable, TableDefinition}; 38 | //! 39 | //! const TABLE: TableDefinition<&str, u64> = TableDefinition::new("my_data"); 40 | //! 41 | //! fn main() -> Result<(), Error> { 42 | //! # #[cfg(not(target_os = "wasi"))] 43 | //! let file = tempfile::NamedTempFile::new().unwrap(); 44 | //! # #[cfg(target_os = "wasi")] 45 | //! # let file = tempfile::NamedTempFile::new_in("/tmp").unwrap(); 46 | //! let db = Database::create(file.path())?; 47 | //! let write_txn = db.begin_write()?; 48 | //! { 49 | //! let mut table = write_txn.open_table(TABLE)?; 50 | //! table.insert("my_key", &123)?; 51 | //! } 52 | //! write_txn.commit()?; 53 | //! 54 | //! let read_txn = db.begin_read()?; 55 | //! let table = read_txn.open_table(TABLE)?; 56 | //! assert_eq!(table.get("my_key")?.unwrap().value(), 123); 57 | //! 58 | //! Ok(()) 59 | //! } 60 | //! ``` 61 | //! 62 | //! [lmdb]: https://www.lmdb.tech/doc/ 63 | //! [design]: https://github.com/cberner/redb/blob/master/docs/design.md 64 | 65 | pub use db::{ 66 | Builder, CacheStats, Database, MultimapTableDefinition, MultimapTableHandle, ReadOnlyDatabase, 67 | ReadableDatabase, RepairSession, StorageBackend, TableDefinition, TableHandle, 68 | UntypedMultimapTableHandle, UntypedTableHandle, 69 | }; 70 | pub use error::{ 71 | CommitError, CompactionError, DatabaseError, Error, SavepointError, SetDurabilityError, 72 | StorageError, TableError, TransactionError, 73 | }; 74 | pub use legacy_tuple_types::Legacy; 75 | pub use multimap_table::{ 76 | MultimapRange, MultimapTable, MultimapValue, ReadOnlyMultimapTable, 77 | ReadOnlyUntypedMultimapTable, ReadableMultimapTable, 78 | }; 79 | pub use table::{ 80 | ExtractIf, Range, ReadOnlyTable, ReadOnlyUntypedTable, ReadableTable, ReadableTableMetadata, 81 | Table, TableStats, 82 | }; 83 | pub use transactions::{DatabaseStats, Durability, ReadTransaction, WriteTransaction}; 84 | pub use tree_store::{AccessGuard, AccessGuardMut, AccessGuardMutInPlace, Savepoint}; 85 | pub use types::{Key, MutInPlaceValue, TypeName, Value}; 86 | 87 | pub type Result = std::result::Result; 88 | 89 | pub mod backends; 90 | mod complex_types; 91 | mod db; 92 | mod error; 93 | mod legacy_tuple_types; 94 | mod multimap_table; 95 | mod sealed; 96 | mod table; 97 | mod transaction_tracker; 98 | mod transactions; 99 | mod tree_store; 100 | mod tuple_types; 101 | mod types; 102 | 103 | #[cfg(test)] 104 | fn create_tempfile() -> tempfile::NamedTempFile { 105 | if cfg!(target_os = "wasi") { 106 | tempfile::NamedTempFile::new_in("/tmp").unwrap() 107 | } else { 108 | tempfile::NamedTempFile::new().unwrap() 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: 5 | push: # required for actions/cache to work 6 | branches: 7 | - master 8 | 9 | jobs: 10 | ci: 11 | strategy: 12 | matrix: 13 | os: ["ubuntu-latest", "macos-latest", "windows-latest", "ubuntu-24.04-arm"] 14 | 15 | runs-on: ${{ matrix.os }} 16 | 17 | env: 18 | RUSTFLAGS: --deny warnings 19 | 20 | steps: 21 | - uses: actions/checkout@v4 22 | - name: Cache 23 | id: rust-cache 24 | uses: actions/cache@v4 25 | with: 26 | path: | 27 | ~/.cargo/bin/ 28 | ~/.cargo/registry/index/ 29 | ~/.cargo/registry/cache/ 30 | ~/.cargo/git/db/ 31 | target/ 32 | fuzz/target/ 33 | key: ${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/Cargo.toml', '.github/workflows/*.yml', 'rust-toolchain') }} 34 | 35 | - name: Install Python 36 | uses: actions/setup-python@v5 37 | with: 38 | python-version: "3.12" 39 | 40 | - name: Install Rust 41 | run: | 42 | rustup component add rustfmt 43 | rustup component add clippy 44 | 45 | - name: OSX x86 rust 46 | if: startsWith(matrix.os, 'macos') 47 | run: | 48 | # For some reason this is required to run the fuzzer on OSX 49 | rustup target add x86_64-apple-darwin 50 | 51 | - name: Setup wasmtime 52 | uses: bytecodealliance/actions/wasmtime/setup@v1 53 | with: 54 | version: "24.0.0" 55 | 56 | - name: Install cargo-deny 57 | if: steps.rust-cache.outputs.cache-hit != 'true' 58 | run: cargo install --force --version 0.16.2 cargo-deny --locked 59 | 60 | - name: Install cargo-fuzz 61 | if: steps.rust-cache.outputs.cache-hit != 'true' 62 | run: cargo install --force --version 0.12.0 cargo-fuzz --locked 63 | 64 | - name: Install just 65 | if: steps.rust-cache.outputs.cache-hit != 'true' 66 | run: cargo install --force --version 1.36.0 just --locked 67 | 68 | - name: Format 69 | run: cargo fmt --all -- --check 70 | 71 | - name: Compile (default features) 72 | run: cargo check --all --all-targets 73 | 74 | - name: Compile (all features) 75 | run: cargo check --all --all-targets --all-features 76 | 77 | - name: Compile (no features) 78 | run: cargo check --all --all-targets --no-default-features 79 | 80 | - name: Compile (`wasm32-unknown-unknown`) 81 | if: matrix.os == 'ubuntu-latest' 82 | run: | 83 | rustup target add wasm32-unknown-unknown 84 | # Not possible to set default members for specific a target: 85 | # https://github.com/rust-lang/cargo/issues/6179 86 | command="cargo check -p $(cargo pkgid) -p redb-derive --target wasm32-unknown-unknown" 87 | $command 88 | $command --all-features 89 | $command --no-default-features 90 | rustup target remove wasm32-unknown-unknown 91 | 92 | - name: Clippy 93 | run: cargo clippy --all --all-targets -- -Dwarnings 94 | 95 | - name: Fuzzer 96 | if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos') 97 | run: just fuzz_ci 98 | 99 | - name: Run tests 100 | run: just test_all 101 | 102 | - name: Run CPython wrapper tests 103 | if: runner.os != 'Windows' 104 | run: | 105 | python3 -m venv venv 106 | source venv/bin/activate 107 | pip3 install --upgrade pip 108 | pip3 install maturin 109 | just test_py 110 | 111 | - name: Run CPython wrapper tests 112 | if: runner.os == 'Windows' 113 | run: | 114 | python3 -m venv venv 115 | venv\Scripts\activate 116 | pip3 install --upgrade pip 117 | pip3 install maturin 118 | just test_py 119 | 120 | - name: Run WASI tests 121 | if: runner.os != 'Windows' 122 | run: | 123 | RUSTFLAGS="" just test_wasi 124 | -------------------------------------------------------------------------------- /crates/redb-bench/benches/atomics_benchmark.rs: -------------------------------------------------------------------------------- 1 | use std::hint::black_box; 2 | use std::sync::atomic::{AtomicU64, Ordering}; 3 | use std::sync::{Mutex, RwLock}; 4 | use std::thread; 5 | use std::time::SystemTime; 6 | 7 | const ITERATIONS: usize = 1000 * 1000; 8 | 9 | fn baseline(num_threads: usize) { 10 | let start = SystemTime::now(); 11 | for _ in 0..num_threads { 12 | thread::scope(|s| { 13 | s.spawn(|| { 14 | let mut value = 0u64; 15 | for _ in 0..ITERATIONS { 16 | let value = black_box(&mut value); 17 | *value += 1; 18 | } 19 | for _ in 0..ITERATIONS { 20 | let value = black_box(&mut value); 21 | *value -= 1; 22 | } 23 | }); 24 | }); 25 | } 26 | 27 | let end = SystemTime::now(); 28 | let duration = end.duration_since(start).unwrap(); 29 | println!( 30 | "baseline (NOT atomic) ({} threads): {} ops in {}ms", 31 | num_threads, 32 | 2 * ITERATIONS, 33 | duration.as_millis(), 34 | ); 35 | } 36 | 37 | fn atomics(num_threads: usize) { 38 | let start = SystemTime::now(); 39 | let value = AtomicU64::new(0); 40 | for _ in 0..num_threads { 41 | thread::scope(|s| { 42 | s.spawn(|| { 43 | for _ in 0..ITERATIONS { 44 | let value = black_box(&value); 45 | value.fetch_add(1, Ordering::Release); 46 | } 47 | for _ in 0..ITERATIONS { 48 | let value = black_box(&value); 49 | value.fetch_sub(1, Ordering::Release); 50 | } 51 | }); 52 | }); 53 | } 54 | assert_eq!(0, value.load(Ordering::Acquire)); 55 | 56 | let end = SystemTime::now(); 57 | let duration = end.duration_since(start).unwrap(); 58 | println!( 59 | "atomics ({} threads): {} ops in {}ms", 60 | num_threads, 61 | 2 * ITERATIONS, 62 | duration.as_millis(), 63 | ); 64 | } 65 | 66 | fn mutex(num_threads: usize) { 67 | let start = SystemTime::now(); 68 | let value = Mutex::new(0u64); 69 | for _ in 0..num_threads { 70 | thread::scope(|s| { 71 | s.spawn(|| { 72 | for _ in 0..ITERATIONS { 73 | let value = black_box(&value); 74 | *value.lock().unwrap() += 1; 75 | } 76 | for _ in 0..ITERATIONS { 77 | let value = black_box(&value); 78 | *value.lock().unwrap() -= 1; 79 | } 80 | }); 81 | }); 82 | } 83 | assert_eq!(0u64, *value.lock().unwrap()); 84 | 85 | let end = SystemTime::now(); 86 | let duration = end.duration_since(start).unwrap(); 87 | println!( 88 | "mutex ({} threads): {} ops in {}ms", 89 | num_threads, 90 | 2 * ITERATIONS, 91 | duration.as_millis(), 92 | ); 93 | } 94 | 95 | fn rw_lock(num_threads: usize) { 96 | let start = SystemTime::now(); 97 | let value = RwLock::new(0u64); 98 | for _ in 0..num_threads { 99 | thread::scope(|s| { 100 | s.spawn(|| { 101 | for _ in 0..ITERATIONS { 102 | let value = black_box(&value); 103 | *value.write().unwrap() += 1; 104 | } 105 | for _ in 0..ITERATIONS { 106 | let value = black_box(&value); 107 | *value.write().unwrap() -= 1; 108 | } 109 | }); 110 | }); 111 | } 112 | assert_eq!(0u64, *value.read().unwrap()); 113 | 114 | let end = SystemTime::now(); 115 | let duration = end.duration_since(start).unwrap(); 116 | println!( 117 | "rwlock ({} threads): {} ops in {}ms", 118 | num_threads, 119 | 2 * ITERATIONS, 120 | duration.as_millis(), 121 | ); 122 | } 123 | 124 | fn main() { 125 | for threads in [1, 2, 4, 8, 16, 32, 64] { 126 | baseline(threads); 127 | atomics(threads); 128 | mutex(threads); 129 | rw_lock(threads); 130 | println!(); 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /crates/redb-bench/benches/int_benchmark.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | 3 | use std::env::current_dir; 4 | use tempfile::{NamedTempFile, TempDir}; 5 | 6 | mod common; 7 | use common::*; 8 | 9 | use rand::rngs::StdRng; 10 | use rand::{Rng, SeedableRng}; 11 | use std::time::{Duration, Instant}; 12 | 13 | const ELEMENTS: usize = 1_000_000; 14 | 15 | /// Returns pairs of key, value 16 | fn random_data(count: usize) -> Vec<(u32, u64)> { 17 | let mut rng = StdRng::seed_from_u64(0); 18 | let mut pairs = vec![]; 19 | for _ in 0..count { 20 | pairs.push(rng.random()); 21 | } 22 | pairs 23 | } 24 | 25 | fn benchmark(db: T) -> Vec<(&'static str, Duration)> { 26 | let mut results = Vec::new(); 27 | let pairs = random_data(1_000_000); 28 | let mut written = 0; 29 | 30 | let start = Instant::now(); 31 | let connection = db.connect(); 32 | let mut txn = connection.write_transaction(); 33 | let mut inserter = txn.get_inserter(); 34 | { 35 | for _ in 0..ELEMENTS { 36 | let len = pairs.len(); 37 | let (key, value) = pairs[written % len]; 38 | inserter 39 | .insert(&key.to_le_bytes(), &value.to_le_bytes()) 40 | .unwrap(); 41 | written += 1; 42 | } 43 | } 44 | drop(inserter); 45 | txn.commit().unwrap(); 46 | 47 | let end = Instant::now(); 48 | let duration = end - start; 49 | println!( 50 | "{}: Bulk loaded {} (u32, u64) pairs in {}ms", 51 | T::db_type_name(), 52 | ELEMENTS, 53 | duration.as_millis() 54 | ); 55 | results.push(("bulk load", duration)); 56 | 57 | results 58 | } 59 | 60 | fn main() { 61 | let _ = env_logger::try_init(); 62 | 63 | let redb_results = { 64 | let tmpfile: NamedTempFile = NamedTempFile::new_in(current_dir().unwrap()).unwrap(); 65 | let mut db = redb::Database::create(tmpfile.path()).unwrap(); 66 | let table = RedbBenchDatabase::new(&mut db); 67 | benchmark(table) 68 | }; 69 | 70 | let lmdb_results = { 71 | let tmpfile: TempDir = tempfile::tempdir_in(current_dir().unwrap()).unwrap(); 72 | let env = unsafe { 73 | heed::EnvOpenOptions::new() 74 | .map_size(10 * 4096 * 1024 * 1024) 75 | .open(tmpfile.path()) 76 | .unwrap() 77 | }; 78 | let table = HeedBenchDatabase::new(env); 79 | benchmark(table) 80 | }; 81 | 82 | let rocksdb_results = { 83 | let tmpfile: TempDir = tempfile::tempdir_in(current_dir().unwrap()).unwrap(); 84 | 85 | let mut bb = rocksdb::BlockBasedOptions::default(); 86 | bb.set_block_cache(&rocksdb::Cache::new_lru_cache(4 * 1_024 * 1_024 * 1_024)); 87 | bb.set_bloom_filter(10.0, false); 88 | 89 | let mut opts = rocksdb::Options::default(); 90 | opts.set_block_based_table_factory(&bb); 91 | opts.create_if_missing(true); 92 | opts.increase_parallelism( 93 | std::thread::available_parallelism().map_or(1, |n| n.get()) as i32 94 | ); 95 | 96 | let db = rocksdb::OptimisticTransactionDB::open(&opts, tmpfile.path()).unwrap(); 97 | let table = RocksdbBenchDatabase::new(&db); 98 | benchmark(table) 99 | }; 100 | 101 | let sled_results = { 102 | let tmpfile: TempDir = tempfile::tempdir_in(current_dir().unwrap()).unwrap(); 103 | let db = sled::Config::new().path(tmpfile.path()).open().unwrap(); 104 | let table = SledBenchDatabase::new(&db, tmpfile.path()); 105 | benchmark(table) 106 | }; 107 | 108 | let mut rows = Vec::new(); 109 | 110 | for (benchmark, _duration) in &redb_results { 111 | rows.push(vec![benchmark.to_string()]); 112 | } 113 | 114 | for results in [redb_results, lmdb_results, rocksdb_results, sled_results] { 115 | for (i, (_benchmark, duration)) in results.iter().enumerate() { 116 | rows[i].push(format!("{}ms", duration.as_millis())); 117 | } 118 | } 119 | 120 | let mut table = comfy_table::Table::new(); 121 | table.set_width(100); 122 | table.set_header(["", "redb", "lmdb", "rocksdb", "sled"]); 123 | for row in rows { 124 | table.add_row(row); 125 | } 126 | 127 | println!(); 128 | println!("{table}"); 129 | } 130 | -------------------------------------------------------------------------------- /crates/redb-bench/benches/savepoint_benchmark.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | 3 | use std::env::current_dir; 4 | use tempfile::NamedTempFile; 5 | 6 | use rand::Rng; 7 | use redb::{Database, TableDefinition}; 8 | use std::time::{Duration, Instant}; 9 | 10 | const TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("x"); 11 | 12 | const VALUE_SIZE: usize = 3_000; 13 | const SAVEPOINT_WINDOW: usize = 10; 14 | 15 | struct Timing { 16 | insert: Duration, 17 | savepoint_creation: Duration, 18 | savepoint_restore: Duration, 19 | } 20 | 21 | /// Returns pairs of key, value 22 | fn random_data(count: usize, key_size: usize, value_size: usize) -> Vec<(Vec, Vec)> { 23 | let mut pairs = vec![]; 24 | 25 | for _ in 0..count { 26 | let key: Vec = (0..key_size).map(|_| rand::rng().random()).collect(); 27 | let value: Vec = (0..value_size).map(|_| rand::rng().random()).collect(); 28 | pairs.push((key, value)); 29 | } 30 | 31 | pairs 32 | } 33 | 34 | fn benchmark(db: &Database, insertions: usize) -> Timing { 35 | let mut pairs = random_data(insertions, 24, VALUE_SIZE); 36 | let mut written = 0; 37 | 38 | let mut total_savepoint_creation = Duration::from_micros(0); 39 | let mut total_insert = Duration::from_micros(0); 40 | let mut first_savepoint = None; 41 | for _ in 0..SAVEPOINT_WINDOW { 42 | let txn = db.begin_write().unwrap(); 43 | let mut table = txn.open_table(TABLE).unwrap(); 44 | let start = Instant::now(); 45 | { 46 | for _ in 0..(insertions / SAVEPOINT_WINDOW) { 47 | let len = pairs.len(); 48 | let (key, value) = &mut pairs[written % len]; 49 | key[16..].copy_from_slice(&(written as u64).to_le_bytes()); 50 | table.insert(key.as_slice(), value.as_slice()).unwrap(); 51 | written += 1; 52 | } 53 | } 54 | let end = Instant::now(); 55 | total_insert += end - start; 56 | drop(table); 57 | txn.commit().unwrap(); 58 | 59 | let txn = db.begin_write().unwrap(); 60 | let start = Instant::now(); 61 | let savepoint_id = txn.persistent_savepoint().unwrap(); 62 | if first_savepoint.is_none() { 63 | first_savepoint = Some(savepoint_id); 64 | } 65 | let end = Instant::now(); 66 | total_savepoint_creation += end - start; 67 | txn.commit().unwrap(); 68 | } 69 | 70 | let mut txn = db.begin_write().unwrap(); 71 | let savepoint = txn 72 | .get_persistent_savepoint(first_savepoint.unwrap()) 73 | .unwrap(); 74 | let start = Instant::now(); 75 | txn.restore_savepoint(&savepoint).unwrap(); 76 | let end = Instant::now(); 77 | let restore_duration = end - start; 78 | txn.abort().unwrap(); 79 | 80 | let txn = db.begin_write().unwrap(); 81 | for id in txn.list_persistent_savepoints().unwrap() { 82 | txn.delete_persistent_savepoint(id).unwrap(); 83 | } 84 | txn.commit().unwrap(); 85 | 86 | Timing { 87 | insert: total_insert / insertions as u32, 88 | savepoint_creation: total_savepoint_creation / SAVEPOINT_WINDOW as u32, 89 | savepoint_restore: restore_duration, 90 | } 91 | } 92 | 93 | fn main() { 94 | let tmpfile: NamedTempFile = NamedTempFile::new_in(current_dir().unwrap()).unwrap(); 95 | let db = Database::builder().create(tmpfile.path()).unwrap(); 96 | 97 | let mut table = comfy_table::Table::new(); 98 | table.set_width(100); 99 | table.set_header([ 100 | "DB size", 101 | "insert()", 102 | "persistent_savepoint()", 103 | "restore_savepoint()", 104 | ]); 105 | for inserts in [ 106 | 10_000, 20_000, 40_000, 80_000, 160_000, 320_000, 640_000, 1_280_000, 2_560_000, 5_120_000, 107 | ] { 108 | let timing = benchmark(&db, inserts); 109 | let len = tmpfile.as_file().metadata().unwrap().len(); 110 | let row = vec![ 111 | format!("{}MiB", len / 1024 / 1024), 112 | format!("{}ns", timing.insert.as_nanos()), 113 | format!("{}us", timing.savepoint_creation.as_micros()), 114 | format!("{}us", timing.savepoint_restore.as_micros()), 115 | ]; 116 | table.add_row(row); 117 | } 118 | 119 | println!(); 120 | println!("{table}"); 121 | } 122 | -------------------------------------------------------------------------------- /examples/special_values.rs: -------------------------------------------------------------------------------- 1 | use redb::{ 2 | Database, Error, Key, ReadableTable, Table, TableDefinition, TableHandle, Value, 3 | WriteTransaction, 4 | }; 5 | use std::fs::{File, OpenOptions}; 6 | use std::io::{Read, Seek, SeekFrom, Write}; 7 | use std::marker::PhantomData; 8 | 9 | const TABLE: TableDefinition = TableDefinition::new("my_data"); 10 | 11 | struct SpecialValuesDb { 12 | database: Database, 13 | file: File, 14 | } 15 | 16 | impl SpecialValuesDb { 17 | fn new() -> Self { 18 | SpecialValuesDb { 19 | database: Database::create("index.redb").unwrap(), 20 | file: OpenOptions::new() 21 | .write(true) 22 | .truncate(true) 23 | .create(true) 24 | .read(true) 25 | .open("values.dat") 26 | .unwrap(), 27 | } 28 | } 29 | 30 | fn begin_txn(&mut self) -> SpecialValuesTransaction<'_> { 31 | SpecialValuesTransaction { 32 | inner: self.database.begin_write().unwrap(), 33 | file: &mut self.file, 34 | } 35 | } 36 | } 37 | 38 | struct SpecialValuesTransaction<'db> { 39 | inner: WriteTransaction, 40 | file: &'db mut File, 41 | } 42 | 43 | impl SpecialValuesTransaction<'_> { 44 | fn open_table( 45 | &mut self, 46 | table: TableDefinition, 47 | ) -> SpecialValuesTable<'_, K, V> { 48 | let def: TableDefinition = TableDefinition::new(table.name()); 49 | SpecialValuesTable { 50 | inner: self.inner.open_table(def).unwrap(), 51 | file: self.file, 52 | _value_type: Default::default(), 53 | } 54 | } 55 | 56 | fn commit(self) { 57 | self.file.sync_all().unwrap(); 58 | self.inner.commit().unwrap(); 59 | } 60 | } 61 | 62 | struct SpecialValuesTable<'txn, K: Key + 'static, V: Value + 'static> { 63 | inner: Table<'txn, K, (u64, u64)>, 64 | file: &'txn mut File, 65 | _value_type: PhantomData, 66 | } 67 | 68 | impl SpecialValuesTable<'_, K, V> { 69 | fn insert(&mut self, key: K::SelfType<'_>, value: V::SelfType<'_>) { 70 | // Append to end of file 71 | let offset = self.file.seek(SeekFrom::End(0)).unwrap(); 72 | let value = V::as_bytes(&value); 73 | self.file.write_all(value.as_ref()).unwrap(); 74 | self.inner 75 | .insert(key, (offset, value.as_ref().len() as u64)) 76 | .unwrap(); 77 | } 78 | 79 | fn get(&mut self, key: K::SelfType<'_>) -> ValueAccessor { 80 | let (offset, length) = self.inner.get(key).unwrap().unwrap().value(); 81 | self.file.seek(SeekFrom::Start(offset)).unwrap(); 82 | let mut data = vec![0u8; length as usize]; 83 | self.file.read_exact(data.as_mut_slice()).unwrap(); 84 | ValueAccessor { 85 | data, 86 | _value_type: Default::default(), 87 | } 88 | } 89 | } 90 | 91 | struct ValueAccessor { 92 | data: Vec, 93 | _value_type: PhantomData, 94 | } 95 | 96 | impl ValueAccessor { 97 | fn value(&self) -> V::SelfType<'_> { 98 | V::from_bytes(&self.data) 99 | } 100 | } 101 | 102 | /// redb is not designed to support very large values, or values with special requirements (such as alignment or mutability). 103 | /// There's a hard limit of slightly less than 4GiB per value, and performance is likely to be poor when mutating values above a few megabytes. 104 | /// Additionally, because redb is copy-on-write, mutating a value in-place is not possible, and therefore mutating large values is slow. 105 | /// Storing values with alignment requirements is also not supported. 106 | /// 107 | /// This example demonstrates one way to handle such values, via a sidecar file. 108 | #[allow(clippy::result_large_err)] 109 | fn main() -> Result<(), Error> { 110 | let mut db = SpecialValuesDb::new(); 111 | let mut txn = db.begin_txn(); 112 | { 113 | let mut table = txn.open_table(TABLE); 114 | table.insert(0, "hello world"); 115 | } 116 | txn.commit(); 117 | 118 | let mut txn = db.begin_txn(); 119 | let mut table = txn.open_table(TABLE); 120 | assert_eq!(table.get(0).value(), "hello world"); 121 | 122 | Ok(()) 123 | } 124 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # redb 2 | 3 | ![CI](https://github.com/cberner/redb/actions/workflows/ci.yml/badge.svg) 4 | [![Crates.io](https://img.shields.io/crates/v/redb.svg)](https://crates.io/crates/redb) 5 | [![Documentation](https://docs.rs/redb/badge.svg)](https://docs.rs/redb) 6 | [![License](https://img.shields.io/crates/l/redb)](https://crates.io/crates/redb) 7 | [![dependency status](https://deps.rs/repo/github/cberner/redb/status.svg)](https://deps.rs/repo/github/cberner/redb) 8 | 9 | A simple, portable, high-performance, ACID, embedded key-value store. 10 | 11 | redb is written in pure Rust and is loosely inspired by [lmdb](http://www.lmdb.tech/doc/). Data is stored in a collection 12 | of copy-on-write B-trees. For more details, see the [design doc](docs/design.md) 13 | 14 | ```rust 15 | use redb::{Database, Error, ReadableDatabase, TableDefinition}; 16 | 17 | const TABLE: TableDefinition<&str, u64> = TableDefinition::new("my_data"); 18 | 19 | fn main() -> Result<(), Error> { 20 | let db = Database::create("my_db.redb")?; 21 | let write_txn = db.begin_write()?; 22 | { 23 | let mut table = write_txn.open_table(TABLE)?; 24 | table.insert("my_key", &123)?; 25 | } 26 | write_txn.commit()?; 27 | 28 | let read_txn = db.begin_read()?; 29 | let table = read_txn.open_table(TABLE)?; 30 | assert_eq!(table.get("my_key")?.unwrap().value(), 123); 31 | 32 | Ok(()) 33 | } 34 | ``` 35 | 36 | ## Status 37 | Stable and maintained. 38 | 39 | The file format is stable, and a reasonable effort will be made to provide an upgrade path if there 40 | are any future changes to it. 41 | 42 | ## Features 43 | * Zero-copy, thread-safe, `BTreeMap` based API 44 | * Fully ACID-compliant transactions 45 | * MVCC support for concurrent readers & writer, without blocking 46 | * Crash-safe by default 47 | * Savepoints and rollbacks 48 | 49 | ## Development 50 | To run all the tests and benchmarks a few extra dependencies are required: 51 | * `cargo install cargo-deny --locked` 52 | * `cargo install cargo-fuzz --locked` 53 | * `apt install libclang-dev` 54 | 55 | ## Benchmarks 56 | redb has similar performance to other top embedded key-value stores such as lmdb and rocksdb 57 | 58 | | | redb | lmdb | rocksdb | sled | fjall | sqlite | 59 | |---------------------------|-----------|------------|----------------|----------|-------------|------------| 60 | | bulk load | 17063ms | **9232ms** | 13969ms | 24971ms | 18619ms | 15341ms | 61 | | individual writes | **920ms** | 1598ms | 2432ms | 2701ms | 3488ms | 7040ms | 62 | | batch writes | 1595ms | 942ms | 451ms | 853ms | **353ms** | 2625ms | 63 | | len() | **0ms** | **0ms** | 749ms | 1573ms | 1181ms | 30ms | 64 | | random reads | 1138ms | **637ms** | 2911ms | 1601ms | 2177ms | 4283ms | 65 | | random reads | 934ms | **631ms** | 2884ms | 1592ms | 2357ms | 4281ms | 66 | | random range reads | 1174ms | **565ms** | 2734ms | 1992ms | 2564ms | 8431ms | 67 | | random range reads | 1173ms | **565ms** | 2742ms | 1993ms | 2690ms | 8449ms | 68 | | random reads (4 threads) | 1390ms | **840ms** | 3995ms | 1913ms | 2606ms | 7000ms | 69 | | random reads (8 threads) | 757ms | **427ms** | 2147ms | 1019ms | 1352ms | 8123ms | 70 | | random reads (16 threads) | 652ms | **216ms** | 1478ms | 690ms | 963ms | 23022ms | 71 | | random reads (32 threads) | 410ms | **125ms** | 1100ms | 444ms | 576ms | 26536ms | 72 | | removals | 23297ms | 10435ms | 6900ms | 11088ms | **6004ms** | 10323ms | 73 | | uncompacted size | 4.00 GiB | 2.61 GiB | **893.18 MiB** | 2.13 GiB | 1000.95 MiB | 1.09 GiB | 74 | | compacted size | 1.69 GiB | 1.26 GiB | **454.71 MiB** | N/A | 1000.95 MiB | 556.85 MiB | 75 | 76 | Source code for benchmark [here](./crates/redb-bench/benches/lmdb_benchmark.rs). Results collected on a Ryzen 9950X3D with Samsung 9100 PRO NVMe. 77 | 78 | ## License 79 | 80 | Licensed under either of 81 | 82 | * [Apache License, Version 2.0](LICENSE-APACHE) 83 | * [MIT License](LICENSE-MIT) 84 | 85 | at your option. 86 | 87 | ### Contribution 88 | 89 | Unless you explicitly state otherwise, any contribution intentionally 90 | submitted for inclusion in the work by you, as defined in the Apache-2.0 91 | license, shall be dual licensed as above, without any additional terms or 92 | conditions. 93 | -------------------------------------------------------------------------------- /tests/multithreading_tests.rs: -------------------------------------------------------------------------------- 1 | #[cfg(not(target_os = "wasi"))] 2 | mod multithreading_test { 3 | use redb::{Database, ReadableDatabase, ReadableTable, ReadableTableMetadata, TableDefinition}; 4 | use std::sync::Arc; 5 | use std::thread; 6 | 7 | fn create_tempfile() -> tempfile::NamedTempFile { 8 | if cfg!(target_os = "wasi") { 9 | tempfile::NamedTempFile::new_in("/tmp").unwrap() 10 | } else { 11 | tempfile::NamedTempFile::new().unwrap() 12 | } 13 | } 14 | 15 | const TABLE: TableDefinition<&str, &str> = TableDefinition::new("x"); 16 | #[test] 17 | fn len() { 18 | let tmpfile = create_tempfile(); 19 | let db = Database::create(tmpfile.path()).unwrap(); 20 | let db = Arc::new(db); 21 | let write_txn = db.begin_write().unwrap(); 22 | { 23 | let mut table = write_txn.open_table(TABLE).unwrap(); 24 | table.insert("hello", "world").unwrap(); 25 | table.insert("hello2", "world2").unwrap(); 26 | table.insert("hi", "world").unwrap(); 27 | } 28 | write_txn.commit().unwrap(); 29 | 30 | let db2 = db.clone(); 31 | let t = thread::spawn(move || { 32 | let read_txn = db2.begin_read().unwrap(); 33 | let table = read_txn.open_table(TABLE).unwrap(); 34 | assert_eq!(table.len().unwrap(), 3); 35 | }); 36 | t.join().unwrap(); 37 | 38 | let read_txn = db.begin_read().unwrap(); 39 | let table = read_txn.open_table(TABLE).unwrap(); 40 | assert_eq!(table.len().unwrap(), 3); 41 | } 42 | 43 | #[test] 44 | fn multithreaded_insert() { 45 | let tmpfile = create_tempfile(); 46 | let db = Database::create(tmpfile.path()).unwrap(); 47 | 48 | const DEF1: TableDefinition<&str, &str> = TableDefinition::new("x"); 49 | const DEF2: TableDefinition<&str, &str> = TableDefinition::new("y"); 50 | let write_txn = db.begin_write().unwrap(); 51 | { 52 | let mut table1 = write_txn.open_table(DEF1).unwrap(); 53 | let mut table2 = write_txn.open_table(DEF2).unwrap(); 54 | 55 | thread::scope(|s| { 56 | s.spawn(|| { 57 | table2.insert("hello", "world").unwrap(); 58 | table2.insert("hello2", "world2").unwrap(); 59 | }); 60 | }); 61 | 62 | table1.insert("hello", "world").unwrap(); 63 | table1.insert("hello2", "world2").unwrap(); 64 | } 65 | write_txn.commit().unwrap(); 66 | 67 | let read_txn = db.begin_read().unwrap(); 68 | let table = read_txn.open_table(DEF1).unwrap(); 69 | assert_eq!(table.len().unwrap(), 2); 70 | let table = read_txn.open_table(DEF2).unwrap(); 71 | assert_eq!(table.len().unwrap(), 2); 72 | } 73 | 74 | #[test] 75 | fn multithreaded_re_read() { 76 | let tmpfile = create_tempfile(); 77 | let db = Database::create(tmpfile.path()).unwrap(); 78 | 79 | const DEF1: TableDefinition<&str, &str> = TableDefinition::new("x"); 80 | const DEF2: TableDefinition<&str, &str> = TableDefinition::new("y"); 81 | const DEF3: TableDefinition<&str, &str> = TableDefinition::new("z"); 82 | let write_txn = db.begin_write().unwrap(); 83 | { 84 | let mut table1 = write_txn.open_table(DEF1).unwrap(); 85 | let mut table2 = write_txn.open_table(DEF2).unwrap(); 86 | let mut table3 = write_txn.open_table(DEF3).unwrap(); 87 | table1.insert("hello", "world").unwrap(); 88 | 89 | thread::scope(|s| { 90 | s.spawn(|| { 91 | let value = table1.get("hello").unwrap().unwrap(); 92 | table2.insert("hello2", value.value()).unwrap(); 93 | }); 94 | }); 95 | thread::scope(|s| { 96 | s.spawn(|| { 97 | let value = table1.get("hello").unwrap().unwrap(); 98 | table3.insert("hello2", value.value()).unwrap(); 99 | }); 100 | }); 101 | 102 | assert_eq!(table2.get("hello2").unwrap().unwrap().value(), "world"); 103 | assert_eq!(table3.get("hello2").unwrap().unwrap().value(), "world"); 104 | } 105 | write_txn.commit().unwrap(); 106 | 107 | let read_txn = db.begin_read().unwrap(); 108 | let table = read_txn.open_table(DEF1).unwrap(); 109 | assert_eq!(table.len().unwrap(), 1); 110 | let table = read_txn.open_table(DEF2).unwrap(); 111 | assert_eq!(table.len().unwrap(), 1); 112 | let table = read_txn.open_table(DEF3).unwrap(); 113 | assert_eq!(table.len().unwrap(), 1); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /crates/redb-bench/benches/large_values_benchmark.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | 3 | use std::env::current_dir; 4 | use tempfile::{NamedTempFile, TempDir}; 5 | 6 | mod common; 7 | use common::*; 8 | 9 | use rand::RngCore; 10 | use std::time::{Duration, Instant}; 11 | 12 | const ELEMENTS: usize = 1_000_000; 13 | 14 | /// Returns pairs of key, value 15 | fn random_data(count: usize, key_size: usize, value_size: usize) -> Vec<(Vec, Vec)> { 16 | let mut pairs = vec![]; 17 | 18 | for _ in 0..count { 19 | let mut key = vec![0; key_size]; 20 | rand::rng().fill_bytes(&mut key); 21 | let mut value = vec![0; value_size]; 22 | rand::rng().fill_bytes(&mut value); 23 | pairs.push((key, value)); 24 | } 25 | 26 | pairs 27 | } 28 | 29 | fn benchmark(db: T) -> Vec<(&'static str, Duration)> { 30 | let mut results = Vec::new(); 31 | let mut pairs = random_data(1_000_000, 24, 150); 32 | let mut written = 0; 33 | 34 | let mut bigpairs = random_data(100, 24, 2_000_000); 35 | let bigelements = 4000; 36 | 37 | let start = Instant::now(); 38 | let connection = db.connect(); 39 | let mut txn = connection.write_transaction(); 40 | let mut inserter = txn.get_inserter(); 41 | { 42 | for _ in 0..bigelements { 43 | let len = bigpairs.len(); 44 | let (key, value) = &mut bigpairs[written % len]; 45 | key[16..].copy_from_slice(&(written as u64).to_le_bytes()); 46 | inserter.insert(key, value).unwrap(); 47 | written += 1; 48 | } 49 | for _ in 0..ELEMENTS { 50 | let len = pairs.len(); 51 | let (key, value) = &mut pairs[written % len]; 52 | key[16..].copy_from_slice(&(written as u64).to_le_bytes()); 53 | inserter.insert(key, value).unwrap(); 54 | written += 1; 55 | } 56 | } 57 | drop(inserter); 58 | txn.commit().unwrap(); 59 | 60 | let end = Instant::now(); 61 | let duration = end - start; 62 | println!( 63 | "{}: Bulk loaded {} 2MB items and {} small items in {}ms", 64 | T::db_type_name(), 65 | bigelements, 66 | ELEMENTS, 67 | duration.as_millis() 68 | ); 69 | results.push(("bulk load (2MB values)", duration)); 70 | 71 | results 72 | } 73 | 74 | fn main() { 75 | let _ = env_logger::try_init(); 76 | 77 | let redb_latency_results = { 78 | let tmpfile: NamedTempFile = NamedTempFile::new_in(current_dir().unwrap()).unwrap(); 79 | let mut db = redb::Database::builder().create(tmpfile.path()).unwrap(); 80 | let table = RedbBenchDatabase::new(&mut db); 81 | benchmark(table) 82 | }; 83 | 84 | let lmdb_results = { 85 | let tmpfile: TempDir = tempfile::tempdir_in(current_dir().unwrap()).unwrap(); 86 | let env = unsafe { 87 | heed::EnvOpenOptions::new() 88 | .map_size(10 * 4096 * 1024 * 1024) 89 | .open(tmpfile.path()) 90 | .unwrap() 91 | }; 92 | let table = HeedBenchDatabase::new(env); 93 | benchmark(table) 94 | }; 95 | 96 | let rocksdb_results = { 97 | let tmpfile: TempDir = tempfile::tempdir_in(current_dir().unwrap()).unwrap(); 98 | 99 | let mut bb = rocksdb::BlockBasedOptions::default(); 100 | bb.set_block_cache(&rocksdb::Cache::new_lru_cache(4 * 1_024 * 1_024 * 1_024)); 101 | bb.set_bloom_filter(10.0, false); 102 | 103 | let mut opts = rocksdb::Options::default(); 104 | opts.set_block_based_table_factory(&bb); 105 | opts.create_if_missing(true); 106 | opts.increase_parallelism( 107 | std::thread::available_parallelism().map_or(1, |n| n.get()) as i32 108 | ); 109 | 110 | let db = rocksdb::OptimisticTransactionDB::open(&opts, tmpfile.path()).unwrap(); 111 | let table = RocksdbBenchDatabase::new(&db); 112 | benchmark(table) 113 | }; 114 | 115 | let sled_results = { 116 | let tmpfile: TempDir = tempfile::tempdir_in(current_dir().unwrap()).unwrap(); 117 | let db = sled::Config::new().path(tmpfile.path()).open().unwrap(); 118 | let table = SledBenchDatabase::new(&db, tmpfile.path()); 119 | benchmark(table) 120 | }; 121 | 122 | let mut rows = Vec::new(); 123 | 124 | for (benchmark, _duration) in &redb_latency_results { 125 | rows.push(vec![benchmark.to_string()]); 126 | } 127 | 128 | for results in [ 129 | redb_latency_results, 130 | lmdb_results, 131 | rocksdb_results, 132 | sled_results, 133 | ] { 134 | for (i, (_benchmark, duration)) in results.iter().enumerate() { 135 | rows[i].push(format!("{}ms", duration.as_millis())); 136 | } 137 | } 138 | 139 | let mut table = comfy_table::Table::new(); 140 | table.set_width(100); 141 | table.set_header(["", "redb", "lmdb", "rocksdb", "sled"]); 142 | for row in rows { 143 | table.add_row(row); 144 | } 145 | 146 | println!(); 147 | println!("{table}"); 148 | } 149 | -------------------------------------------------------------------------------- /crates/redb-bench/benches/lmdb_benchmark.rs: -------------------------------------------------------------------------------- 1 | use std::env::current_dir; 2 | use std::{fs, process}; 3 | use tempfile::{NamedTempFile, TempDir}; 4 | 5 | mod common; 6 | use common::*; 7 | 8 | fn main() { 9 | let _ = env_logger::try_init(); 10 | let tmpdir = current_dir().unwrap().join(".benchmark"); 11 | fs::create_dir(&tmpdir).unwrap(); 12 | 13 | let tmpdir2 = tmpdir.clone(); 14 | ctrlc::set_handler(move || { 15 | fs::remove_dir_all(&tmpdir2).unwrap(); 16 | process::exit(1); 17 | }) 18 | .unwrap(); 19 | 20 | let redb_results = { 21 | let tmpfile: NamedTempFile = NamedTempFile::new_in(&tmpdir).unwrap(); 22 | let mut db = redb::Database::builder() 23 | .set_cache_size(CACHE_SIZE) 24 | .create(tmpfile.path()) 25 | .unwrap(); 26 | let table = RedbBenchDatabase::new(&mut db); 27 | benchmark(table, tmpfile.path()) 28 | }; 29 | 30 | let lmdb_results = { 31 | let tempdir: TempDir = tempfile::tempdir_in(&tmpdir).unwrap(); 32 | let env = unsafe { 33 | heed::EnvOpenOptions::new() 34 | .map_size(4096 * 1024 * 1024) 35 | .open(tempdir.path()) 36 | .unwrap() 37 | }; 38 | let table = HeedBenchDatabase::new(env); 39 | benchmark(table, tempdir.path()) 40 | }; 41 | 42 | let rocksdb_results = { 43 | let tmpfile: TempDir = tempfile::tempdir_in(&tmpdir).unwrap(); 44 | 45 | let cache = rocksdb::Cache::new_lru_cache(CACHE_SIZE); 46 | let write_buffer = rocksdb::WriteBufferManager::new_write_buffer_manager_with_cache( 47 | CACHE_SIZE / 2, 48 | false, 49 | cache.clone(), 50 | ); 51 | 52 | let mut bb = rocksdb::BlockBasedOptions::default(); 53 | bb.set_block_cache(&cache); 54 | bb.set_bloom_filter(10.0, false); 55 | bb.set_cache_index_and_filter_blocks(true); 56 | bb.set_pin_l0_filter_and_index_blocks_in_cache(false); 57 | bb.set_pin_top_level_index_and_filter(false); 58 | 59 | let mut opts = rocksdb::Options::default(); 60 | opts.set_block_based_table_factory(&bb); 61 | opts.set_write_buffer_manager(&write_buffer); 62 | opts.set_max_write_buffer_size_to_maintain((CACHE_SIZE / 2) as i64); 63 | opts.create_if_missing(true); 64 | opts.increase_parallelism( 65 | std::thread::available_parallelism().map_or(1, |n| n.get()) as i32 66 | ); 67 | 68 | let db = rocksdb::OptimisticTransactionDB::open(&opts, tmpfile.path()).unwrap(); 69 | let table = RocksdbBenchDatabase::new(&db); 70 | benchmark(table, tmpfile.path()) 71 | }; 72 | 73 | let sled_results = { 74 | let tmpfile: TempDir = tempfile::tempdir_in(&tmpdir).unwrap(); 75 | 76 | let db = sled::Config::new() 77 | .path(tmpfile.path()) 78 | .cache_capacity(CACHE_SIZE as u64) 79 | .open() 80 | .unwrap(); 81 | 82 | let table = SledBenchDatabase::new(&db, tmpfile.path()); 83 | benchmark(table, tmpfile.path()) 84 | }; 85 | 86 | let fjall_results = { 87 | let tmpfile: TempDir = tempfile::tempdir_in(&tmpdir).unwrap(); 88 | 89 | let mut db = fjall::Config::new(tmpfile.path()) 90 | .cache_size(CACHE_SIZE.try_into().unwrap()) 91 | .open_transactional() 92 | .unwrap(); 93 | 94 | let table = FjallBenchDatabase::new(&mut db); 95 | benchmark(table, tmpfile.path()) 96 | }; 97 | 98 | let sqlite_results = { 99 | let tmpfile: NamedTempFile = NamedTempFile::new_in(&tmpdir).unwrap(); 100 | let table = SqliteBenchDatabase::new(tmpfile.path()); 101 | benchmark(table, tmpfile.path()) 102 | }; 103 | 104 | fs::remove_dir_all(&tmpdir).unwrap(); 105 | 106 | let mut rows = Vec::new(); 107 | 108 | for (benchmark, _duration) in &redb_results { 109 | rows.push(vec![benchmark.to_string()]); 110 | } 111 | 112 | let results = [ 113 | redb_results, 114 | lmdb_results, 115 | rocksdb_results, 116 | sled_results, 117 | fjall_results, 118 | sqlite_results, 119 | ]; 120 | 121 | let mut identified_smallests = vec![vec![false; results.len()]; rows.len()]; 122 | for (i, identified_smallests_row) in identified_smallests.iter_mut().enumerate() { 123 | let mut smallest = None; 124 | for (j, _) in identified_smallests_row.iter().enumerate() { 125 | let (_, rt) = &results[j][i]; 126 | smallest = match smallest { 127 | Some((_, prev)) if rt < prev => Some((j, rt)), 128 | Some((pi, prev)) => Some((pi, prev)), 129 | None => Some((j, rt)), 130 | }; 131 | } 132 | let (j, _rt) = smallest.unwrap(); 133 | identified_smallests_row[j] = true; 134 | } 135 | 136 | for (j, results) in results.iter().enumerate() { 137 | for (i, (_benchmark, result_type)) in results.iter().enumerate() { 138 | rows[i].push(if identified_smallests[i][j] { 139 | format!("**{result_type}**") 140 | } else { 141 | result_type.to_string() 142 | }); 143 | } 144 | } 145 | 146 | let mut table = comfy_table::Table::new(); 147 | table.load_preset(comfy_table::presets::ASCII_MARKDOWN); 148 | table.set_width(100); 149 | table.set_header(["", "redb", "lmdb", "rocksdb", "sled", "fjall", "sqlite"]); 150 | for row in rows { 151 | table.add_row(row); 152 | } 153 | 154 | println!(); 155 | println!("{table}"); 156 | } 157 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/common.rs: -------------------------------------------------------------------------------- 1 | use arbitrary::Unstructured; 2 | use libfuzzer_sys::arbitrary::Arbitrary; 3 | use rand::rngs::StdRng; 4 | use rand::SeedableRng; 5 | use rand_distr::{Binomial, Distribution}; 6 | use std::mem::size_of; 7 | 8 | const MAX_CRASH_OPS: u64 = 20; 9 | const MAX_CACHE_SIZE: usize = 100_000_000; 10 | // Limit values to 100KiB 11 | const MAX_VALUE_SIZE: usize = 100_000; 12 | const KEY_SPACE: u64 = 1_000_000; 13 | pub const MAX_SAVEPOINTS: usize = 6; 14 | 15 | #[derive(Debug, Clone)] 16 | pub(crate) struct BoundedU64 { 17 | pub value: u64, 18 | } 19 | 20 | impl Arbitrary<'_> for BoundedU64 { 21 | fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result { 22 | let value: u64 = u.int_in_range(0..=(N - 1))?; 23 | Ok(Self { value }) 24 | } 25 | 26 | fn size_hint(_depth: usize) -> (usize, Option) { 27 | (size_of::(), Some(size_of::())) 28 | } 29 | } 30 | 31 | #[derive(Debug, Clone)] 32 | pub(crate) struct U64Between { 33 | pub value: u64, 34 | } 35 | 36 | impl Arbitrary<'_> for U64Between { 37 | fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result { 38 | let value: u64 = u.int_in_range(MIN..=MAX)?; 39 | Ok(Self { value }) 40 | } 41 | 42 | fn size_hint(_depth: usize) -> (usize, Option) { 43 | (size_of::(), Some(size_of::())) 44 | } 45 | } 46 | 47 | #[derive(Debug, Clone)] 48 | pub(crate) struct BinomialDifferenceBoundedUSize { 49 | pub value: usize, 50 | } 51 | 52 | impl Arbitrary<'_> for BinomialDifferenceBoundedUSize { 53 | fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result { 54 | let seed: u64 = u.arbitrary()?; 55 | let mut rng = StdRng::seed_from_u64(seed); 56 | // Distribution which is the difference from the median of B(N, 0.5) 57 | let distribution = Binomial::new(N as u64, 0.5).unwrap(); 58 | let value = distribution.sample(&mut rng) as isize; 59 | let value = (value - N as isize / 2).abs() as usize; 60 | Ok(Self { value }) 61 | } 62 | 63 | fn size_hint(_depth: usize) -> (usize, Option) { 64 | (size_of::(), Some(size_of::())) 65 | } 66 | } 67 | 68 | #[derive(Debug, Clone)] 69 | pub(crate) struct PowerOfTwoBetween { 70 | pub value: usize, 71 | } 72 | 73 | impl Arbitrary<'_> for PowerOfTwoBetween { 74 | fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result { 75 | let value: u32 = u.int_in_range(M..=N)?; 76 | Ok(Self { 77 | value: 2usize.pow(value), 78 | }) 79 | } 80 | 81 | fn size_hint(_depth: usize) -> (usize, Option) { 82 | (size_of::(), Some(size_of::())) 83 | } 84 | } 85 | 86 | #[derive(Debug, Clone)] 87 | pub(crate) struct BoundedUSize { 88 | pub value: usize, 89 | } 90 | 91 | impl Arbitrary<'_> for BoundedUSize { 92 | fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result { 93 | let value: usize = u.int_in_range(0..=(N - 1))?; 94 | Ok(Self { value }) 95 | } 96 | 97 | fn size_hint(_depth: usize) -> (usize, Option) { 98 | (size_of::(), Some(size_of::())) 99 | } 100 | } 101 | 102 | #[derive(Arbitrary, Debug, Clone)] 103 | pub(crate) enum FuzzOperation { 104 | Get { 105 | key: BoundedU64, 106 | }, 107 | GetMut { 108 | key: BoundedU64, 109 | value_size: BinomialDifferenceBoundedUSize, 110 | }, 111 | Insert { 112 | key: BoundedU64, 113 | value_size: BinomialDifferenceBoundedUSize, 114 | }, 115 | InsertReserve { 116 | key: BoundedU64, 117 | value_size: BinomialDifferenceBoundedUSize, 118 | }, 119 | Remove { 120 | key: BoundedU64, 121 | }, 122 | RemoveOne { 123 | key: BoundedU64, 124 | value_size: BinomialDifferenceBoundedUSize, 125 | }, 126 | Len {}, 127 | PopFirst {}, 128 | PopLast {}, 129 | Retain { 130 | modulus: U64Between<1, 8>, 131 | }, 132 | RetainIn { 133 | start_key: BoundedU64, 134 | len: BoundedU64, 135 | modulus: U64Between<1, 8>, 136 | }, 137 | ExtractIf { 138 | modulus: U64Between<1, 8>, 139 | take: BoundedUSize<10>, 140 | reversed: bool, 141 | }, 142 | ExtractFromIf { 143 | start_key: BoundedU64, 144 | range_len: BoundedU64, 145 | take: BoundedUSize<10>, 146 | modulus: U64Between<1, 8>, 147 | reversed: bool, 148 | }, 149 | Range { 150 | start_key: BoundedU64, 151 | len: BoundedU64, 152 | reversed: bool, 153 | }, 154 | } 155 | 156 | #[derive(Arbitrary, Debug, Clone)] 157 | pub(crate) struct FuzzTransaction { 158 | pub ops: Vec, 159 | pub durable: bool, 160 | pub quick_repair: bool, 161 | pub commit: bool, 162 | pub close_db: bool, 163 | pub create_ephemeral_savepoint: bool, 164 | pub create_persistent_savepoint: bool, 165 | pub restore_savepoint: Option>, 166 | } 167 | 168 | #[derive(Arbitrary, Debug, Clone)] 169 | pub(crate) struct FuzzConfig { 170 | pub multimap_table: bool, 171 | pub cache_size: BoundedUSize, 172 | pub crash_after_ops: BoundedU64, 173 | pub transactions: Vec, 174 | pub page_size: PowerOfTwoBetween<9, 14>, 175 | // Must not be too small, otherwise persistent savepoints won't fit into a region 176 | pub region_size: PowerOfTwoBetween<20, 30>, 177 | } 178 | -------------------------------------------------------------------------------- /src/tree_store/page_store/savepoint.rs: -------------------------------------------------------------------------------- 1 | use crate::transaction_tracker::{SavepointId, TransactionId, TransactionTracker}; 2 | use crate::tree_store::page_store::page_manager::FILE_FORMAT_VERSION3; 3 | use crate::tree_store::{BtreeHeader, TransactionalMemory}; 4 | use crate::{TypeName, Value}; 5 | use std::fmt::Debug; 6 | use std::mem::size_of; 7 | use std::sync::Arc; 8 | 9 | // on-disk format: 10 | // * 1 byte: version 11 | // * 8 bytes: savepoint id 12 | // * 8 bytes: transaction id 13 | // * 1 byte: user root not-null 14 | // * 8 bytes: user root page 15 | // * 8 bytes: user root checksum 16 | /// A database savepoint 17 | /// 18 | /// May be used with [`WriteTransaction::restore_savepoint`] to restore the database to the state 19 | /// when this savepoint was created 20 | /// 21 | /// [`WriteTransaction::restore_savepoint`]: crate::WriteTransaction::restore_savepoint 22 | pub struct Savepoint { 23 | version: u8, 24 | id: SavepointId, 25 | // Each savepoint has an associated read transaction id to ensure that any pages it references 26 | // are not freed 27 | transaction_id: TransactionId, 28 | user_root: Option, 29 | transaction_tracker: Arc, 30 | ephemeral: bool, 31 | } 32 | 33 | impl Savepoint { 34 | #[allow(clippy::too_many_arguments)] 35 | pub(crate) fn new_ephemeral( 36 | mem: &TransactionalMemory, 37 | transaction_tracker: Arc, 38 | id: SavepointId, 39 | transaction_id: TransactionId, 40 | user_root: Option, 41 | ) -> Self { 42 | Self { 43 | id, 44 | transaction_id, 45 | version: mem.get_version(), 46 | user_root, 47 | transaction_tracker, 48 | ephemeral: true, 49 | } 50 | } 51 | 52 | pub(crate) fn get_version(&self) -> u8 { 53 | self.version 54 | } 55 | 56 | pub(crate) fn get_id(&self) -> SavepointId { 57 | self.id 58 | } 59 | 60 | pub(crate) fn get_transaction_id(&self) -> TransactionId { 61 | self.transaction_id 62 | } 63 | 64 | pub(crate) fn get_user_root(&self) -> Option { 65 | self.user_root 66 | } 67 | 68 | pub(crate) fn db_address(&self) -> *const TransactionTracker { 69 | std::ptr::from_ref(self.transaction_tracker.as_ref()) 70 | } 71 | 72 | pub(crate) fn set_persistent(&mut self) { 73 | self.ephemeral = false; 74 | } 75 | } 76 | 77 | impl Drop for Savepoint { 78 | fn drop(&mut self) { 79 | if self.ephemeral { 80 | self.transaction_tracker 81 | .deallocate_savepoint(self.get_id(), self.get_transaction_id()); 82 | } 83 | } 84 | } 85 | 86 | #[derive(Debug)] 87 | pub(crate) enum SerializedSavepoint<'a> { 88 | Ref(&'a [u8]), 89 | Owned(Vec), 90 | } 91 | 92 | impl SerializedSavepoint<'_> { 93 | pub(crate) fn from_savepoint(savepoint: &Savepoint) -> Self { 94 | assert_eq!(savepoint.version, FILE_FORMAT_VERSION3); 95 | let mut result = vec![savepoint.version]; 96 | result.extend(savepoint.id.0.to_le_bytes()); 97 | result.extend(savepoint.transaction_id.raw_id().to_le_bytes()); 98 | 99 | if let Some(header) = savepoint.user_root { 100 | result.push(1); 101 | result.extend(header.to_le_bytes()); 102 | } else { 103 | result.push(0); 104 | result.extend([0; BtreeHeader::serialized_size()]); 105 | } 106 | 107 | Self::Owned(result) 108 | } 109 | 110 | fn data(&self) -> &[u8] { 111 | match self { 112 | SerializedSavepoint::Ref(x) => x, 113 | SerializedSavepoint::Owned(x) => x.as_slice(), 114 | } 115 | } 116 | 117 | pub(crate) fn to_savepoint(&self, transaction_tracker: Arc) -> Savepoint { 118 | let data = self.data(); 119 | let mut offset = 0; 120 | let version = data[offset]; 121 | assert_eq!(version, FILE_FORMAT_VERSION3); 122 | offset += size_of::(); 123 | 124 | let id = u64::from_le_bytes( 125 | data[offset..(offset + size_of::())] 126 | .try_into() 127 | .unwrap(), 128 | ); 129 | offset += size_of::(); 130 | 131 | let transaction_id = u64::from_le_bytes( 132 | data[offset..(offset + size_of::())] 133 | .try_into() 134 | .unwrap(), 135 | ); 136 | offset += size_of::(); 137 | 138 | let not_null = data[offset]; 139 | assert!(not_null == 0 || not_null == 1); 140 | offset += 1; 141 | let user_root = if not_null == 1 { 142 | Some(BtreeHeader::from_le_bytes( 143 | data[offset..(offset + BtreeHeader::serialized_size())] 144 | .try_into() 145 | .unwrap(), 146 | )) 147 | } else { 148 | None 149 | }; 150 | offset += BtreeHeader::serialized_size(); 151 | assert_eq!(offset, data.len()); 152 | 153 | Savepoint { 154 | version, 155 | id: SavepointId(id), 156 | transaction_id: TransactionId::new(transaction_id), 157 | user_root, 158 | transaction_tracker, 159 | ephemeral: false, 160 | } 161 | } 162 | } 163 | 164 | impl Value for SerializedSavepoint<'_> { 165 | type SelfType<'a> 166 | = SerializedSavepoint<'a> 167 | where 168 | Self: 'a; 169 | type AsBytes<'a> 170 | = &'a [u8] 171 | where 172 | Self: 'a; 173 | 174 | fn fixed_width() -> Option { 175 | None 176 | } 177 | 178 | fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> 179 | where 180 | Self: 'a, 181 | { 182 | SerializedSavepoint::Ref(data) 183 | } 184 | 185 | fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> 186 | where 187 | Self: 'b, 188 | { 189 | value.data() 190 | } 191 | 192 | fn type_name() -> TypeName { 193 | TypeName::internal("redb::SerializedSavepoint") 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /src/tree_store/page_store/file_backend/optimized.rs: -------------------------------------------------------------------------------- 1 | use crate::{DatabaseError, Result, StorageBackend}; 2 | use std::fs::{File, TryLockError}; 3 | use std::io; 4 | 5 | #[cfg(feature = "logging")] 6 | use log::warn; 7 | 8 | #[cfg(unix)] 9 | use std::os::unix::fs::FileExt; 10 | 11 | #[cfg(windows)] 12 | use std::os::windows::fs::FileExt; 13 | 14 | /// Stores a database as a file on-disk. 15 | #[derive(Debug)] 16 | pub struct FileBackend { 17 | lock_supported: bool, 18 | file: File, 19 | } 20 | 21 | impl FileBackend { 22 | /// Creates a new backend which stores data to the given file. 23 | pub fn new(file: File) -> Result { 24 | Self::new_internal(file, false) 25 | } 26 | 27 | pub(crate) fn new_internal(file: File, read_only: bool) -> Result { 28 | let result = if read_only { 29 | file.try_lock_shared() 30 | } else { 31 | file.try_lock() 32 | }; 33 | 34 | match result { 35 | Ok(()) => Ok(Self { 36 | file, 37 | lock_supported: true, 38 | }), 39 | Err(TryLockError::WouldBlock) => Err(DatabaseError::DatabaseAlreadyOpen), 40 | Err(TryLockError::Error(err)) if err.kind() == io::ErrorKind::Unsupported => { 41 | #[cfg(feature = "logging")] 42 | warn!( 43 | "File locks not supported on this platform. You must ensure that only a single process opens the database file, at a time" 44 | ); 45 | 46 | Ok(Self { 47 | file, 48 | lock_supported: false, 49 | }) 50 | } 51 | Err(TryLockError::Error(err)) => Err(err.into()), 52 | } 53 | } 54 | } 55 | 56 | impl StorageBackend for FileBackend { 57 | fn len(&self) -> Result { 58 | Ok(self.file.metadata()?.len()) 59 | } 60 | 61 | #[cfg(unix)] 62 | fn read(&self, offset: u64, out: &mut [u8]) -> Result<(), io::Error> { 63 | self.file.read_exact_at(out, offset)?; 64 | Ok(()) 65 | } 66 | 67 | #[cfg(target_os = "wasi")] 68 | fn read(&self, offset: u64, out: &mut [u8]) -> Result<(), io::Error> { 69 | read_exact_at(&self.file, out, offset)?; 70 | Ok(()) 71 | } 72 | 73 | #[cfg(windows)] 74 | fn read(&self, mut offset: u64, out: &mut [u8]) -> Result<(), io::Error> { 75 | let mut data_offset = 0; 76 | while data_offset < out.len() { 77 | let read = self.file.seek_read(&mut out[data_offset..], offset)?; 78 | offset += read as u64; 79 | data_offset += read; 80 | } 81 | Ok(()) 82 | } 83 | 84 | fn set_len(&self, len: u64) -> Result<(), io::Error> { 85 | self.file.set_len(len) 86 | } 87 | 88 | fn sync_data(&self) -> Result<(), io::Error> { 89 | self.file.sync_data() 90 | } 91 | 92 | #[cfg(unix)] 93 | fn write(&self, offset: u64, data: &[u8]) -> Result<(), io::Error> { 94 | self.file.write_all_at(data, offset) 95 | } 96 | 97 | #[cfg(target_os = "wasi")] 98 | fn write(&self, offset: u64, data: &[u8]) -> Result<(), io::Error> { 99 | write_all_at(&self.file, data, offset) 100 | } 101 | 102 | #[cfg(windows)] 103 | fn write(&self, mut offset: u64, data: &[u8]) -> Result<(), io::Error> { 104 | let mut data_offset = 0; 105 | while data_offset < data.len() { 106 | let written = self.file.seek_write(&data[data_offset..], offset)?; 107 | offset += written as u64; 108 | data_offset += written; 109 | } 110 | Ok(()) 111 | } 112 | 113 | fn close(&self) -> Result<(), io::Error> { 114 | if self.lock_supported { 115 | self.file.unlock()?; 116 | } 117 | 118 | Ok(()) 119 | } 120 | } 121 | 122 | // TODO: replace these with wasi::FileExt when https://github.com/rust-lang/rust/issues/71213 123 | // is stable 124 | #[cfg(target_os = "wasi")] 125 | fn read_exact_at(file: &File, mut buf: &mut [u8], mut offset: u64) -> io::Result<()> { 126 | use std::os::fd::AsRawFd; 127 | 128 | while !buf.is_empty() { 129 | let nbytes = unsafe { 130 | libc::pread( 131 | file.as_raw_fd(), 132 | buf.as_mut_ptr() as _, 133 | std::cmp::min(buf.len(), libc::ssize_t::MAX as _), 134 | offset as _, 135 | ) 136 | }; 137 | match nbytes { 138 | 0 => break, 139 | -1 => match io::Error::last_os_error() { 140 | err if err.kind() == io::ErrorKind::Interrupted => {} 141 | err => return Err(err), 142 | }, 143 | n => { 144 | let tmp = buf; 145 | buf = &mut tmp[n as usize..]; 146 | offset += n as u64; 147 | } 148 | } 149 | } 150 | if !buf.is_empty() { 151 | Err(io::Error::new( 152 | io::ErrorKind::UnexpectedEof, 153 | "failed to fill whole buffer", 154 | )) 155 | } else { 156 | Ok(()) 157 | } 158 | } 159 | 160 | #[cfg(target_os = "wasi")] 161 | fn write_all_at(file: &File, mut buf: &[u8], mut offset: u64) -> io::Result<()> { 162 | use std::os::fd::AsRawFd; 163 | 164 | while !buf.is_empty() { 165 | let nbytes = unsafe { 166 | libc::pwrite( 167 | file.as_raw_fd(), 168 | buf.as_ptr() as _, 169 | std::cmp::min(buf.len(), libc::ssize_t::MAX as _), 170 | offset as _, 171 | ) 172 | }; 173 | match nbytes { 174 | 0 => { 175 | return Err(io::Error::new( 176 | io::ErrorKind::WriteZero, 177 | "failed to write whole buffer", 178 | )); 179 | } 180 | -1 => match io::Error::last_os_error() { 181 | err if err.kind() == io::ErrorKind::Interrupted => {} 182 | err => return Err(err), 183 | }, 184 | n => { 185 | buf = &buf[n as usize..]; 186 | offset += n as u64 187 | } 188 | } 189 | } 190 | Ok(()) 191 | } 192 | -------------------------------------------------------------------------------- /deny.toml: -------------------------------------------------------------------------------- 1 | # This template contains all of the possible sections and their default values 2 | 3 | # Note that all fields that take a lint level have these possible values: 4 | # * deny - An error will be produced and the check will fail 5 | # * warn - A warning will be produced, but the check will not fail 6 | # * allow - No warning or error will be produced, though in some cases a note 7 | # will be 8 | 9 | # The values provided in this template are the default values that will be used 10 | # when any section or field is not specified in your own configuration 11 | 12 | # This section is considered when running `cargo deny check advisories` 13 | # More documentation for the advisories section can be found here: 14 | # https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html 15 | [advisories] 16 | # The path where the advisory database is cloned/fetched into 17 | db-path = "~/.cargo/advisory-db" 18 | # The url of the advisory database to use 19 | db-urls = ["https://github.com/rustsec/advisory-db"] 20 | # The lint level for crates that have been yanked from their source registry 21 | yanked = "warn" 22 | # A list of advisory IDs to ignore. Note that ignored advisories will still 23 | # output a note when they are encountered. 24 | ignore = [ 25 | #"RUSTSEC-0000-0000", 26 | ] 27 | # Threshold for security vulnerabilities, any vulnerability with a CVSS score 28 | # lower than the range specified will be ignored. Note that ignored advisories 29 | # will still output a note when they are encountered. 30 | # * None - CVSS Score 0.0 31 | # * Low - CVSS Score 0.1 - 3.9 32 | # * Medium - CVSS Score 4.0 - 6.9 33 | # * High - CVSS Score 7.0 - 8.9 34 | # * Critical - CVSS Score 9.0 - 10.0 35 | #severity-threshold = 36 | 37 | # This section is considered when running `cargo deny check licenses` 38 | # More documentation for the licenses section can be found here: 39 | # https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html 40 | [licenses] 41 | # List of explictly allowed licenses 42 | # See https://spdx.org/licenses/ for list of possible licenses 43 | # [possible values: any SPDX 3.7 short identifier (+ optional exception)]. 44 | allow = [ 45 | "MIT", 46 | # "BSD-2-Clause", 47 | # "BSD-3-Clause", 48 | "Apache-2.0", 49 | "Apache-2.0 WITH LLVM-exception", 50 | ] 51 | # The confidence threshold for detecting a license from license text. 52 | # The higher the value, the more closely the license text must be to the 53 | # canonical license text of a valid SPDX license file. 54 | # [possible values: any between 0.0 and 1.0]. 55 | confidence-threshold = 0.8 56 | # Allow 1 or more licenses on a per-crate basis, so that particular licenses 57 | # aren't accepted for every possible crate as with the normal allow list 58 | exceptions = [ 59 | # Each entry is the crate and version constraint, and its specific allow 60 | # list 61 | { allow = ["Unicode-3.0"], name = "unicode-ident", version = "*" }, # Used only by comfy-table in the benchmarks 62 | # { allow = ["ISC"], name = "libloading", version = "*" }, # Used only by rocksdb in the benchmarks 63 | ] 64 | 65 | # Some crates don't have (easily) machine readable licensing information, 66 | # adding a clarification entry for it allows you to manually specify the 67 | # licensing information 68 | #[[licenses.clarify]] 69 | # The name of the crate the clarification applies to 70 | #name = "ring" 71 | # THe optional version constraint for the crate 72 | #version = "*" 73 | # The SPDX expression for the license requirements of the crate 74 | #expression = "MIT AND ISC AND OpenSSL" 75 | # One or more files in the crate's source used as the "source of truth" for 76 | # the license expression. If the contents match, the clarification will be used 77 | # when running the license check, otherwise the clarification will be ignored 78 | # and the crate will be checked normally, which may produce warnings or errors 79 | # depending on the rest of your configuration 80 | #license-files = [ 81 | # Each entry is a crate relative path, and the (opaque) hash of its contents 82 | #{ path = "LICENSE", hash = 0xbd0eed23 } 83 | #] 84 | 85 | [licenses.private] 86 | # If true, ignores workspace crates that aren't published, or are only 87 | # published to private registries 88 | ignore = false 89 | # One or more private registries that you might publish crates to, if a crate 90 | # is only published to private registries, and ignore is true, the crate will 91 | # not have its license(s) checked 92 | registries = [ 93 | #"https://sekretz.com/registry 94 | ] 95 | 96 | # This section is considered when running `cargo deny check bans`. 97 | # More documentation about the 'bans' section can be found here: 98 | # https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html 99 | [bans] 100 | # Lint level for when multiple versions of the same crate are detected 101 | multiple-versions = "warn" 102 | # The graph highlighting used when creating dotgraphs for crates 103 | # with multiple versions 104 | # * lowest-version - The path to the lowest versioned duplicate is highlighted 105 | # * simplest-path - The path to the version with the fewest edges is highlighted 106 | # * all - Both lowest-version and simplest-path are used 107 | highlight = "all" 108 | # List of crates that are allowed. Use with care! 109 | allow = [ 110 | #{ name = "ansi_term", version = "=0.11.0" }, 111 | ] 112 | # List of crates to deny 113 | deny = [ 114 | # Each entry the name of a crate and a version range. If version is 115 | # not specified, all versions will be matched. 116 | #{ name = "ansi_term", version = "=0.11.0" }, 117 | ] 118 | # Certain crates/versions that will be skipped when doing duplicate detection. 119 | skip = [ 120 | #{ name = "ansi_term", version = "=0.11.0" }, 121 | ] 122 | # Similarly to `skip` allows you to skip certain crates during duplicate 123 | # detection. Unlike skip, it also includes the entire tree of transitive 124 | # dependencies starting at the specified crate, up to a certain depth, which is 125 | # by default infinite 126 | skip-tree = [ 127 | #{ name = "ansi_term", version = "=0.11.0", depth = 20 }, 128 | ] 129 | 130 | # This section is considered when running `cargo deny check sources`. 131 | # More documentation about the 'sources' section can be found here: 132 | # https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html 133 | [sources] 134 | # Lint level for what to happen when a crate from a crate registry that is not 135 | # in the allow list is encountered 136 | unknown-registry = "warn" 137 | # Lint level for what to happen when a crate from a git repository that is not 138 | # in the allow list is encountered 139 | unknown-git = "warn" 140 | # List of URLs for allowed crate registries. Defaults to the crates.io index 141 | # if not specified. If it is specified but empty, no registries are allowed. 142 | allow-registry = ["https://github.com/rust-lang/crates.io-index"] 143 | # List of URLs for allowed Git repositories 144 | allow-git = [] 145 | -------------------------------------------------------------------------------- /crates/redb-derive/tests/derive_tests.rs: -------------------------------------------------------------------------------- 1 | use redb::{Database, Key, ReadableDatabase, TableDefinition, Value}; 2 | use redb_derive::{Key, Value}; 3 | use std::fmt::Debug; 4 | use tempfile::NamedTempFile; 5 | 6 | fn create_tempfile() -> NamedTempFile { 7 | if cfg!(target_os = "wasi") { 8 | NamedTempFile::new_in("/tmp").unwrap() 9 | } else { 10 | NamedTempFile::new().unwrap() 11 | } 12 | } 13 | 14 | #[derive(Key, Value, Debug, PartialEq, Eq, PartialOrd, Ord)] 15 | struct SimpleStruct { 16 | id: u32, 17 | name: String, 18 | } 19 | 20 | #[derive(Key, Value, Debug, PartialEq, Eq, PartialOrd, Ord)] 21 | struct TupleStruct0(); 22 | 23 | #[derive(Key, Value, Debug, PartialEq, Eq, PartialOrd, Ord)] 24 | struct TupleStruct1(u64); 25 | 26 | #[derive(Key, Value, Debug, PartialEq, Eq, PartialOrd, Ord)] 27 | struct TupleStruct2(u64, bool); 28 | 29 | #[derive(Key, Value, Debug, PartialEq, Eq, PartialOrd, Ord)] 30 | struct ZeroField {} 31 | 32 | #[derive(Key, Value, Debug, PartialEq, Eq, PartialOrd, Ord)] 33 | struct SingleField { 34 | value: i32, 35 | } 36 | 37 | #[derive(Key, Value, Debug, PartialEq, Eq, PartialOrd, Ord)] 38 | struct ComplexStruct<'inner, 'inner2> { 39 | tuple_field: (u8, u16, u32), 40 | array_field: [(u8, Option); 2], 41 | reference: &'inner str, 42 | reference2: &'inner2 str, 43 | } 44 | 45 | #[derive(Value, Debug, PartialEq)] 46 | struct UnitStruct; 47 | 48 | fn test_key_helper(key: &::SelfType<'_>) { 49 | let file = create_tempfile(); 50 | let db = Database::create(file.path()).unwrap(); 51 | let table_def: TableDefinition = TableDefinition::new("test"); 52 | 53 | let write_txn = db.begin_write().unwrap(); 54 | { 55 | let mut table = write_txn.open_table(table_def).unwrap(); 56 | table.insert(key, 1).unwrap(); 57 | } 58 | write_txn.commit().unwrap(); 59 | 60 | let read_txn = db.begin_read().unwrap(); 61 | let table = read_txn.open_table(table_def).unwrap(); 62 | let retrieved = table.get(key).unwrap().unwrap(); 63 | let retrieved_value = retrieved.value(); 64 | assert_eq!(retrieved_value, 1); 65 | } 66 | 67 | fn test_value_helper( 68 | value: ::SelfType<'_>, 69 | expected_type_name: &str, 70 | ) where 71 | for<'x> ::SelfType<'x>: PartialEq, 72 | { 73 | let type_name = V::type_name(); 74 | assert_eq!(type_name.name(), expected_type_name); 75 | 76 | let file = create_tempfile(); 77 | let db = Database::create(file.path()).unwrap(); 78 | let table_def: TableDefinition = TableDefinition::new("test"); 79 | 80 | let write_txn = db.begin_write().unwrap(); 81 | { 82 | let mut table = write_txn.open_table(table_def).unwrap(); 83 | table.insert(1, &value).unwrap(); 84 | } 85 | write_txn.commit().unwrap(); 86 | 87 | let read_txn = db.begin_read().unwrap(); 88 | let table = read_txn.open_table(table_def).unwrap(); 89 | let retrieved = table.get(1).unwrap().unwrap(); 90 | // Due to the lifetimes of SelfType we can't compare the values themselves, so instead compare 91 | // the serialized representation 92 | let retrieved_value = retrieved.value(); 93 | let expected_bytes = V::as_bytes(&value); 94 | let bytes = V::as_bytes(&retrieved_value); 95 | assert_eq!(expected_bytes.as_ref(), bytes.as_ref()); 96 | } 97 | 98 | #[test] 99 | fn test_key_ordering() { 100 | let first = SimpleStruct { 101 | id: 1, 102 | name: "a".to_string(), 103 | }; 104 | let second = SimpleStruct { 105 | id: 2, 106 | name: "a".to_string(), 107 | }; 108 | let third = SimpleStruct { 109 | id: 2, 110 | name: "b".to_string(), 111 | }; 112 | let fourth = SimpleStruct { 113 | id: 3, 114 | name: "a".to_string(), 115 | }; 116 | 117 | let first_bytes = SimpleStruct::as_bytes(&first); 118 | let second_bytes = SimpleStruct::as_bytes(&second); 119 | let third_bytes = SimpleStruct::as_bytes(&third); 120 | let fourth_bytes = SimpleStruct::as_bytes(&fourth); 121 | 122 | assert_eq!( 123 | SimpleStruct::compare(&first_bytes, &second_bytes), 124 | first.cmp(&second) 125 | ); 126 | assert_eq!( 127 | SimpleStruct::compare(&second_bytes, &third_bytes), 128 | second.cmp(&third) 129 | ); 130 | assert_eq!( 131 | SimpleStruct::compare(&third_bytes, &fourth_bytes), 132 | third.cmp(&fourth) 133 | ); 134 | } 135 | 136 | #[test] 137 | fn test_simple_struct() { 138 | let original = SimpleStruct { 139 | id: 42, 140 | name: "test".to_string(), 141 | }; 142 | let bytes = SimpleStruct::as_bytes(&original); 143 | let (id, name) = <(u32, String)>::from_bytes(&bytes); 144 | assert_eq!(id, original.id); 145 | assert_eq!(name, original.name); 146 | 147 | test_key_helper::(&original); 148 | test_value_helper::(original, "SimpleStruct {id: u32, name: String}"); 149 | } 150 | 151 | #[test] 152 | fn test_unit_struct() { 153 | let original = UnitStruct; 154 | let bytes = UnitStruct::as_bytes(&original); 155 | <()>::from_bytes(&bytes); 156 | test_value_helper::(original, "UnitStruct"); 157 | } 158 | 159 | #[test] 160 | fn test_tuple_struct0() { 161 | let original = TupleStruct0(); 162 | let bytes = TupleStruct0::as_bytes(&original); 163 | <()>::from_bytes(&bytes); 164 | test_key_helper::(&original); 165 | test_value_helper::(original, "TupleStruct0()"); 166 | } 167 | 168 | #[test] 169 | fn test_tuple_struct1() { 170 | let original = TupleStruct1(123456789); 171 | let bytes = TupleStruct1::as_bytes(&original); 172 | let (x,) = <(u64,)>::from_bytes(&bytes); 173 | assert_eq!(x, original.0); 174 | test_key_helper::(&original); 175 | test_value_helper::(original, "TupleStruct1(u64)"); 176 | } 177 | 178 | #[test] 179 | fn test_tuple_struct2() { 180 | let original = TupleStruct2(123456789, true); 181 | let bytes = TupleStruct2::as_bytes(&original); 182 | let (x, y) = <(u64, bool)>::from_bytes(&bytes); 183 | assert_eq!(x, original.0); 184 | assert_eq!(y, original.1); 185 | test_key_helper::(&original); 186 | test_value_helper::(original, "TupleStruct2(u64, bool)"); 187 | } 188 | 189 | #[test] 190 | fn test_zero_fields() { 191 | let original = ZeroField {}; 192 | let bytes = ZeroField::as_bytes(&original); 193 | <()>::from_bytes(&bytes); 194 | test_key_helper::(&original); 195 | test_value_helper::(original, "ZeroField {}"); 196 | } 197 | 198 | #[test] 199 | fn test_single_field() { 200 | let original = SingleField { value: -42 }; 201 | let bytes = SingleField::as_bytes(&original); 202 | let value = ::from_bytes(&bytes); 203 | assert_eq!(value, original.value); 204 | test_key_helper::(&original); 205 | test_value_helper::(original, "SingleField {value: i32}"); 206 | } 207 | 208 | #[test] 209 | fn test_complex_struct() { 210 | let original = ComplexStruct { 211 | tuple_field: (1, 2, 3), 212 | array_field: [(4, Some(5)), (6, None)], 213 | reference: "hello", 214 | reference2: "world", 215 | }; 216 | let bytes = ComplexStruct::as_bytes(&original); 217 | let (tuple_field, array_field, reference, reference2) = 218 | <((u8, u16, u32), [(u8, Option); 2], &str, &str)>::from_bytes(&bytes); 219 | assert_eq!(tuple_field, original.tuple_field); 220 | assert_eq!(array_field, original.array_field); 221 | assert_eq!(reference, original.reference); 222 | assert_eq!(reference2, original.reference2); 223 | 224 | let expected_name = "ComplexStruct {tuple_field: (u8,u16,u32), array_field: [(u8,Option);2], reference: &str, reference2: &str}"; 225 | test_key_helper::(&original); 226 | test_value_helper::(original, expected_name); 227 | } 228 | -------------------------------------------------------------------------------- /src/tree_store/page_store/region.rs: -------------------------------------------------------------------------------- 1 | use crate::tree_store::page_store::base::MAX_REGIONS; 2 | use crate::tree_store::page_store::bitmap::BtreeBitmap; 3 | use crate::tree_store::page_store::buddy_allocator::BuddyAllocator; 4 | use crate::tree_store::page_store::layout::DatabaseLayout; 5 | use crate::tree_store::page_store::page_manager::{INITIAL_REGIONS, MAX_MAX_PAGE_ORDER}; 6 | use std::cmp::{self, max}; 7 | use std::mem::size_of; 8 | 9 | // Tracks the page orders that MAY BE free in each region. This data structure is optimistic, so 10 | // a region may not actually have a page free for a given order 11 | pub(crate) struct RegionTracker { 12 | order_trackers: Vec, 13 | } 14 | 15 | impl RegionTracker { 16 | pub(crate) fn new(regions: u32, orders: u8) -> Self { 17 | let mut data = vec![]; 18 | for _ in 0..orders { 19 | data.push(BtreeBitmap::new(regions, MAX_REGIONS)); 20 | } 21 | Self { 22 | order_trackers: data, 23 | } 24 | } 25 | 26 | // Format: 27 | // num_orders: u32 number of order allocators 28 | // allocator_lens: u32 length of each allocator 29 | // data: BtreeBitmap data for each order 30 | pub(super) fn to_vec(&self) -> Vec { 31 | let mut result = vec![]; 32 | let orders: u32 = self.order_trackers.len().try_into().unwrap(); 33 | let allocator_lens: Vec = self 34 | .order_trackers 35 | .iter() 36 | .map(|x| x.to_vec().len().try_into().unwrap()) 37 | .collect(); 38 | result.extend(orders.to_le_bytes()); 39 | for allocator_len in allocator_lens { 40 | result.extend(allocator_len.to_le_bytes()); 41 | } 42 | for order in &self.order_trackers { 43 | result.extend(&order.to_vec()); 44 | } 45 | result 46 | } 47 | 48 | pub(super) fn from_bytes(page: &[u8]) -> Self { 49 | let orders = u32::from_le_bytes(page[..size_of::()].try_into().unwrap()); 50 | let mut start = size_of::(); 51 | let mut allocator_lens = vec![]; 52 | for _ in 0..orders { 53 | let allocator_len = 54 | u32::from_le_bytes(page[start..start + size_of::()].try_into().unwrap()) 55 | as usize; 56 | allocator_lens.push(allocator_len); 57 | start += size_of::(); 58 | } 59 | let mut data = vec![]; 60 | for allocator_len in allocator_lens { 61 | data.push(BtreeBitmap::from_bytes( 62 | &page[start..(start + allocator_len)], 63 | )); 64 | start += allocator_len; 65 | } 66 | 67 | Self { 68 | order_trackers: data, 69 | } 70 | } 71 | 72 | pub(crate) fn find_free(&self, order: u8) -> Option { 73 | self.order_trackers[order as usize].find_first_unset() 74 | } 75 | 76 | pub(crate) fn mark_free(&mut self, order: u8, region: u32) { 77 | let order: usize = order.into(); 78 | for i in 0..=order { 79 | self.order_trackers[i].clear(region); 80 | } 81 | } 82 | 83 | pub(crate) fn mark_full(&mut self, order: u8, region: u32) { 84 | let order: usize = order.into(); 85 | assert!(order < self.order_trackers.len()); 86 | for i in order..self.order_trackers.len() { 87 | self.order_trackers[i].set(region); 88 | } 89 | } 90 | 91 | fn resize(&mut self, new_capacity: u32) { 92 | for order in &mut self.order_trackers { 93 | order.resize(new_capacity, true); 94 | } 95 | } 96 | 97 | fn len(&self) -> u32 { 98 | self.order_trackers[0].len() 99 | } 100 | } 101 | 102 | pub(super) struct Allocators { 103 | pub(super) region_tracker: RegionTracker, 104 | pub(super) region_allocators: Vec, 105 | } 106 | 107 | impl Allocators { 108 | pub(super) fn new(layout: DatabaseLayout) -> Self { 109 | let mut region_allocators = vec![]; 110 | let initial_regions = max(INITIAL_REGIONS, layout.num_regions()); 111 | let mut region_tracker = RegionTracker::new(initial_regions, MAX_MAX_PAGE_ORDER + 1); 112 | for i in 0..layout.num_regions() { 113 | let region_layout = layout.region_layout(i); 114 | let allocator = BuddyAllocator::new( 115 | region_layout.num_pages(), 116 | layout.full_region_layout().num_pages(), 117 | ); 118 | let max_order = allocator.get_max_order(); 119 | region_tracker.mark_free(max_order, i); 120 | region_allocators.push(allocator); 121 | } 122 | 123 | Self { 124 | region_tracker, 125 | region_allocators, 126 | } 127 | } 128 | 129 | pub(crate) fn xxh3_hash(&self) -> u128 { 130 | // Ignore the region tracker because it is an optimistic cache, and so may not match 131 | // between repairs of the allocators 132 | let mut result = 0; 133 | for allocator in &self.region_allocators { 134 | result ^= allocator.xxh3_hash(); 135 | } 136 | result 137 | } 138 | 139 | pub(super) fn resize_to(&mut self, new_layout: DatabaseLayout) { 140 | let shrink = match (new_layout.num_regions() as usize).cmp(&self.region_allocators.len()) { 141 | cmp::Ordering::Less => true, 142 | cmp::Ordering::Equal => { 143 | let allocator = self.region_allocators.last().unwrap(); 144 | let last_region = new_layout 145 | .trailing_region_layout() 146 | .unwrap_or_else(|| new_layout.full_region_layout()); 147 | match last_region.num_pages().cmp(&allocator.len()) { 148 | cmp::Ordering::Less => true, 149 | cmp::Ordering::Equal => { 150 | // No-op 151 | return; 152 | } 153 | cmp::Ordering::Greater => false, 154 | } 155 | } 156 | cmp::Ordering::Greater => false, 157 | }; 158 | 159 | if shrink { 160 | // Drop all regions that were removed 161 | for i in new_layout.num_regions()..(self.region_allocators.len().try_into().unwrap()) { 162 | self.region_tracker.mark_full(0, i); 163 | } 164 | self.region_allocators 165 | .drain((new_layout.num_regions() as usize)..); 166 | 167 | // Resize the last region 168 | let last_region = new_layout 169 | .trailing_region_layout() 170 | .unwrap_or_else(|| new_layout.full_region_layout()); 171 | let allocator = self.region_allocators.last_mut().unwrap(); 172 | if allocator.len() > last_region.num_pages() { 173 | allocator.resize(last_region.num_pages()); 174 | } 175 | } else { 176 | let old_num_regions = self.region_allocators.len(); 177 | for i in 0..new_layout.num_regions() { 178 | let new_region = new_layout.region_layout(i); 179 | if (i as usize) < old_num_regions { 180 | let allocator = &mut self.region_allocators[i as usize]; 181 | assert!(new_region.num_pages() >= allocator.len()); 182 | if new_region.num_pages() != allocator.len() { 183 | allocator.resize(new_region.num_pages()); 184 | let highest_free = allocator.highest_free_order().unwrap(); 185 | self.region_tracker.mark_free(highest_free, i); 186 | } 187 | } else { 188 | // brand new region 189 | let allocator = BuddyAllocator::new( 190 | new_region.num_pages(), 191 | new_layout.full_region_layout().num_pages(), 192 | ); 193 | let highest_free = allocator.highest_free_order().unwrap(); 194 | if i >= self.region_tracker.len() { 195 | self.region_tracker.resize(i + 1); 196 | } 197 | self.region_tracker.mark_free(highest_free, i); 198 | self.region_allocators.push(allocator); 199 | } 200 | } 201 | } 202 | } 203 | } 204 | -------------------------------------------------------------------------------- /src/tree_store/page_store/layout.rs: -------------------------------------------------------------------------------- 1 | use std::ops::Range; 2 | 3 | fn round_up_to_multiple_of(value: u64, multiple: u64) -> u64 { 4 | if value % multiple == 0 { 5 | value 6 | } else { 7 | value + multiple - value % multiple 8 | } 9 | } 10 | 11 | // Regions are laid out starting with the allocator state header, followed by the pages aligned 12 | // to the next page 13 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] 14 | pub(super) struct RegionLayout { 15 | num_pages: u32, 16 | // Offset where data pages start 17 | header_pages: u32, 18 | page_size: u32, 19 | } 20 | 21 | impl RegionLayout { 22 | pub(super) fn new(num_pages: u32, header_pages: u32, page_size: u32) -> Self { 23 | assert!(num_pages > 0); 24 | Self { 25 | num_pages, 26 | header_pages, 27 | page_size, 28 | } 29 | } 30 | 31 | pub(super) fn calculate( 32 | desired_usable_bytes: u64, 33 | page_capacity: u32, 34 | region_header_pages: u32, 35 | page_size: u32, 36 | ) -> RegionLayout { 37 | assert!(desired_usable_bytes <= u64::from(page_capacity) * u64::from(page_size)); 38 | let num_pages = 39 | round_up_to_multiple_of(desired_usable_bytes, page_size.into()) / u64::from(page_size); 40 | 41 | Self { 42 | num_pages: num_pages.try_into().unwrap(), 43 | header_pages: region_header_pages, 44 | page_size, 45 | } 46 | } 47 | 48 | pub(super) fn data_section(&self) -> Range { 49 | let header_bytes = u64::from(self.header_pages) * u64::from(self.page_size); 50 | header_bytes..(header_bytes + self.usable_bytes()) 51 | } 52 | 53 | pub(super) fn get_header_pages(&self) -> u32 { 54 | self.header_pages 55 | } 56 | 57 | pub(super) fn num_pages(&self) -> u32 { 58 | self.num_pages 59 | } 60 | 61 | pub(super) fn page_size(&self) -> u32 { 62 | self.page_size 63 | } 64 | 65 | pub(super) fn len(&self) -> u64 { 66 | u64::from(self.header_pages) * u64::from(self.page_size) + self.usable_bytes() 67 | } 68 | 69 | pub(super) fn usable_bytes(&self) -> u64 { 70 | u64::from(self.page_size) * u64::from(self.num_pages) 71 | } 72 | } 73 | 74 | #[derive(Clone, Copy, Debug)] 75 | pub(crate) struct DatabaseLayout { 76 | full_region_layout: RegionLayout, 77 | num_full_regions: u32, 78 | trailing_partial_region: Option, 79 | } 80 | 81 | impl DatabaseLayout { 82 | pub(super) fn new( 83 | full_regions: u32, 84 | full_region: RegionLayout, 85 | trailing_region: Option, 86 | ) -> Self { 87 | Self { 88 | full_region_layout: full_region, 89 | num_full_regions: full_regions, 90 | trailing_partial_region: trailing_region, 91 | } 92 | } 93 | 94 | pub(super) fn reduce_last_region(&mut self, pages: u32) { 95 | if let Some(ref mut trailing) = self.trailing_partial_region { 96 | assert!(pages <= trailing.num_pages); 97 | trailing.num_pages -= pages; 98 | if trailing.num_pages == 0 { 99 | self.trailing_partial_region = None; 100 | } 101 | } else { 102 | self.num_full_regions -= 1; 103 | let full_layout = self.full_region_layout; 104 | if full_layout.num_pages > pages { 105 | self.trailing_partial_region = Some(RegionLayout::new( 106 | full_layout.num_pages - pages, 107 | full_layout.header_pages, 108 | full_layout.page_size, 109 | )); 110 | } 111 | } 112 | } 113 | 114 | pub(super) fn recalculate( 115 | file_len: u64, 116 | region_header_pages_u32: u32, 117 | region_max_data_pages_u32: u32, 118 | page_size_u32: u32, 119 | ) -> Self { 120 | let page_size = u64::from(page_size_u32); 121 | let region_header_pages = u64::from(region_header_pages_u32); 122 | let region_max_data_pages = u64::from(region_max_data_pages_u32); 123 | // Super-header 124 | let mut remaining = file_len - page_size; 125 | let full_region_size = (region_header_pages + region_max_data_pages) * page_size; 126 | let full_regions = remaining / full_region_size; 127 | remaining -= full_regions * full_region_size; 128 | let trailing = if remaining >= (region_header_pages + 1) * page_size { 129 | remaining -= region_header_pages * page_size; 130 | let remaining: u32 = remaining.try_into().unwrap(); 131 | let data_pages = remaining / page_size_u32; 132 | assert!(data_pages < region_max_data_pages_u32); 133 | Some(RegionLayout::new( 134 | data_pages, 135 | region_header_pages_u32, 136 | page_size_u32, 137 | )) 138 | } else { 139 | None 140 | }; 141 | let full_layout = RegionLayout::new( 142 | region_max_data_pages_u32, 143 | region_header_pages_u32, 144 | page_size_u32, 145 | ); 146 | 147 | Self { 148 | full_region_layout: full_layout, 149 | num_full_regions: full_regions.try_into().unwrap(), 150 | trailing_partial_region: trailing, 151 | } 152 | } 153 | 154 | pub(super) fn calculate( 155 | desired_usable_bytes: u64, 156 | page_capacity: u32, 157 | region_header_pages: u32, 158 | page_size: u32, 159 | ) -> Self { 160 | let full_region_layout = RegionLayout::new(page_capacity, region_header_pages, page_size); 161 | if desired_usable_bytes <= full_region_layout.usable_bytes() { 162 | // Single region layout 163 | let region_layout = RegionLayout::calculate( 164 | desired_usable_bytes, 165 | page_capacity, 166 | region_header_pages, 167 | page_size, 168 | ); 169 | DatabaseLayout { 170 | full_region_layout, 171 | num_full_regions: 0, 172 | trailing_partial_region: Some(region_layout), 173 | } 174 | } else { 175 | // Multi region layout 176 | let full_regions = desired_usable_bytes / full_region_layout.usable_bytes(); 177 | let remaining_desired = 178 | desired_usable_bytes - full_regions * full_region_layout.usable_bytes(); 179 | assert!(full_regions > 0); 180 | let trailing_region = if remaining_desired > 0 { 181 | Some(RegionLayout::calculate( 182 | remaining_desired, 183 | page_capacity, 184 | region_header_pages, 185 | page_size, 186 | )) 187 | } else { 188 | None 189 | }; 190 | if let Some(ref region) = trailing_region { 191 | // All regions must have the same header size 192 | assert_eq!(region.header_pages, full_region_layout.header_pages); 193 | } 194 | DatabaseLayout { 195 | full_region_layout, 196 | num_full_regions: full_regions.try_into().unwrap(), 197 | trailing_partial_region: trailing_region, 198 | } 199 | } 200 | } 201 | 202 | pub(super) fn full_region_layout(&self) -> &RegionLayout { 203 | &self.full_region_layout 204 | } 205 | 206 | pub(super) fn trailing_region_layout(&self) -> Option<&RegionLayout> { 207 | self.trailing_partial_region.as_ref() 208 | } 209 | 210 | pub(super) fn num_full_regions(&self) -> u32 { 211 | self.num_full_regions 212 | } 213 | 214 | pub(super) fn num_regions(&self) -> u32 { 215 | if self.trailing_partial_region.is_some() { 216 | self.num_full_regions + 1 217 | } else { 218 | self.num_full_regions 219 | } 220 | } 221 | 222 | pub(super) fn len(&self) -> u64 { 223 | let last = self.num_regions() - 1; 224 | self.region_base_address(last) + self.region_layout(last).len() 225 | } 226 | 227 | pub(super) fn usable_bytes(&self) -> u64 { 228 | let trailing = self 229 | .trailing_partial_region 230 | .as_ref() 231 | .map(RegionLayout::usable_bytes) 232 | .unwrap_or_default(); 233 | u64::from(self.num_full_regions) * self.full_region_layout.usable_bytes() + trailing 234 | } 235 | 236 | pub(super) fn region_base_address(&self, region: u32) -> u64 { 237 | assert!(region < self.num_regions()); 238 | u64::from(self.full_region_layout.page_size()) 239 | + u64::from(region) * self.full_region_layout.len() 240 | } 241 | 242 | pub(super) fn region_layout(&self, region: u32) -> RegionLayout { 243 | assert!(region < self.num_regions()); 244 | if region == self.num_full_regions { 245 | self.trailing_partial_region.unwrap() 246 | } else { 247 | self.full_region_layout 248 | } 249 | } 250 | } 251 | -------------------------------------------------------------------------------- /src/transaction_tracker.rs: -------------------------------------------------------------------------------- 1 | use crate::tree_store::TransactionalMemory; 2 | use crate::{Key, Result, Savepoint, TypeName, Value}; 3 | #[cfg(feature = "logging")] 4 | use log::debug; 5 | use std::cmp::Ordering; 6 | use std::collections::btree_map::BTreeMap; 7 | use std::collections::{BTreeSet, HashMap}; 8 | use std::mem; 9 | use std::mem::size_of; 10 | use std::sync::{Condvar, Mutex}; 11 | 12 | #[derive(Copy, Clone, Hash, Ord, PartialOrd, Eq, PartialEq, Debug)] 13 | pub(crate) struct TransactionId(u64); 14 | 15 | impl TransactionId { 16 | pub(crate) fn new(value: u64) -> TransactionId { 17 | Self(value) 18 | } 19 | 20 | pub(crate) fn raw_id(self) -> u64 { 21 | self.0 22 | } 23 | 24 | pub(crate) fn next(self) -> TransactionId { 25 | TransactionId(self.0 + 1) 26 | } 27 | 28 | pub(crate) fn increment(&mut self) -> TransactionId { 29 | let next = self.next(); 30 | *self = next; 31 | next 32 | } 33 | } 34 | 35 | #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Hash, Debug)] 36 | pub(crate) struct SavepointId(pub u64); 37 | 38 | impl SavepointId { 39 | pub(crate) fn next(self) -> SavepointId { 40 | SavepointId(self.0 + 1) 41 | } 42 | } 43 | 44 | impl Value for SavepointId { 45 | type SelfType<'a> = SavepointId; 46 | type AsBytes<'a> = [u8; size_of::()]; 47 | 48 | fn fixed_width() -> Option { 49 | Some(size_of::()) 50 | } 51 | 52 | fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> 53 | where 54 | Self: 'a, 55 | { 56 | SavepointId(u64::from_le_bytes(data.try_into().unwrap())) 57 | } 58 | 59 | fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> 60 | where 61 | Self: 'b, 62 | { 63 | value.0.to_le_bytes() 64 | } 65 | 66 | fn type_name() -> TypeName { 67 | TypeName::internal("redb::SavepointId") 68 | } 69 | } 70 | 71 | impl Key for SavepointId { 72 | fn compare(data1: &[u8], data2: &[u8]) -> Ordering { 73 | Self::from_bytes(data1).0.cmp(&Self::from_bytes(data2).0) 74 | } 75 | } 76 | 77 | struct State { 78 | next_savepoint_id: SavepointId, 79 | // reference count of read transactions per transaction id 80 | live_read_transactions: BTreeMap, 81 | next_transaction_id: TransactionId, 82 | live_write_transaction: Option, 83 | valid_savepoints: BTreeMap, 84 | // Non-durable commits that are still in-memory, and waiting for a durable commit to get flushed 85 | // We need to make sure that the freed-table does not get processed for these, since they are not durable yet 86 | // Therefore, we hold a read transaction on their nearest durable ancestor 87 | // 88 | // Maps non-durable transaction id -> durable ancestor 89 | pending_non_durable_commits: HashMap, 90 | // Non-durable commits which have NOT been processed in the freed table 91 | unprocessed_freed_non_durable_commits: BTreeSet, 92 | } 93 | 94 | pub(crate) struct TransactionTracker { 95 | state: Mutex, 96 | live_write_transaction_available: Condvar, 97 | } 98 | 99 | impl TransactionTracker { 100 | pub(crate) fn new(next_transaction_id: TransactionId) -> Self { 101 | Self { 102 | state: Mutex::new(State { 103 | next_savepoint_id: SavepointId(0), 104 | live_read_transactions: Default::default(), 105 | next_transaction_id, 106 | live_write_transaction: None, 107 | valid_savepoints: Default::default(), 108 | pending_non_durable_commits: Default::default(), 109 | unprocessed_freed_non_durable_commits: Default::default(), 110 | }), 111 | live_write_transaction_available: Condvar::new(), 112 | } 113 | } 114 | 115 | pub(crate) fn start_write_transaction(&self) -> TransactionId { 116 | let mut state = self.state.lock().unwrap(); 117 | while state.live_write_transaction.is_some() { 118 | state = self.live_write_transaction_available.wait(state).unwrap(); 119 | } 120 | assert!(state.live_write_transaction.is_none()); 121 | let transaction_id = state.next_transaction_id.increment(); 122 | #[cfg(feature = "logging")] 123 | debug!("Beginning write transaction id={transaction_id:?}"); 124 | state.live_write_transaction = Some(transaction_id); 125 | 126 | transaction_id 127 | } 128 | 129 | pub(crate) fn end_write_transaction(&self, id: TransactionId) { 130 | let mut state = self.state.lock().unwrap(); 131 | assert_eq!(state.live_write_transaction.unwrap(), id); 132 | state.live_write_transaction = None; 133 | self.live_write_transaction_available.notify_one(); 134 | } 135 | 136 | pub(crate) fn clear_pending_non_durable_commits(&self) { 137 | let mut state = self.state.lock().unwrap(); 138 | let ids = mem::take(&mut state.pending_non_durable_commits); 139 | for (_, durable_ancestor) in ids { 140 | let ref_count = state 141 | .live_read_transactions 142 | .get_mut(&durable_ancestor) 143 | .unwrap(); 144 | *ref_count -= 1; 145 | if *ref_count == 0 { 146 | state.live_read_transactions.remove(&durable_ancestor); 147 | } 148 | } 149 | } 150 | 151 | pub(crate) fn is_unprocessed_non_durable_commit(&self, id: TransactionId) -> bool { 152 | let state = self.state.lock().unwrap(); 153 | state.unprocessed_freed_non_durable_commits.contains(&id) 154 | } 155 | 156 | pub(crate) fn mark_unprocessed_non_durable_commit(&self, id: TransactionId) { 157 | let mut state = self.state.lock().unwrap(); 158 | state.unprocessed_freed_non_durable_commits.remove(&id); 159 | } 160 | 161 | pub(crate) fn oldest_unprocessed_non_durable_commit(&self) -> Option { 162 | let state = self.state.lock().unwrap(); 163 | state 164 | .unprocessed_freed_non_durable_commits 165 | .iter() 166 | .next() 167 | .copied() 168 | } 169 | 170 | pub(crate) fn register_non_durable_commit( 171 | &self, 172 | id: TransactionId, 173 | durable_ancestor: TransactionId, 174 | ) { 175 | let mut state = self.state.lock().unwrap(); 176 | state 177 | .live_read_transactions 178 | .entry(durable_ancestor) 179 | .and_modify(|x| *x += 1) 180 | .or_insert(1); 181 | state 182 | .pending_non_durable_commits 183 | .insert(id, durable_ancestor); 184 | state.unprocessed_freed_non_durable_commits.insert(id); 185 | } 186 | 187 | pub(crate) fn restore_savepoint_counter_state(&self, next_savepoint: SavepointId) { 188 | let mut state = self.state.lock().unwrap(); 189 | assert!(state.valid_savepoints.is_empty()); 190 | state.next_savepoint_id = next_savepoint; 191 | } 192 | 193 | pub(crate) fn register_persistent_savepoint(&self, savepoint: &Savepoint) { 194 | let mut state = self.state.lock().unwrap(); 195 | state 196 | .live_read_transactions 197 | .entry(savepoint.get_transaction_id()) 198 | .and_modify(|x| *x += 1) 199 | .or_insert(1); 200 | state 201 | .valid_savepoints 202 | .insert(savepoint.get_id(), savepoint.get_transaction_id()); 203 | } 204 | 205 | pub(crate) fn register_read_transaction( 206 | &self, 207 | mem: &TransactionalMemory, 208 | ) -> Result { 209 | let mut state = self.state.lock()?; 210 | let id = mem.get_last_committed_transaction_id()?; 211 | state 212 | .live_read_transactions 213 | .entry(id) 214 | .and_modify(|x| *x += 1) 215 | .or_insert(1); 216 | 217 | Ok(id) 218 | } 219 | 220 | pub(crate) fn deallocate_read_transaction(&self, id: TransactionId) { 221 | let mut state = self.state.lock().unwrap(); 222 | let ref_count = state.live_read_transactions.get_mut(&id).unwrap(); 223 | *ref_count -= 1; 224 | if *ref_count == 0 { 225 | state.live_read_transactions.remove(&id); 226 | } 227 | } 228 | 229 | pub(crate) fn any_savepoint_exists(&self) -> bool { 230 | !self.state.lock().unwrap().valid_savepoints.is_empty() 231 | } 232 | 233 | pub(crate) fn allocate_savepoint(&self, transaction_id: TransactionId) -> SavepointId { 234 | let mut state = self.state.lock().unwrap(); 235 | let id = state.next_savepoint_id.next(); 236 | state.next_savepoint_id = id; 237 | state.valid_savepoints.insert(id, transaction_id); 238 | id 239 | } 240 | 241 | // Deallocates the given savepoint and its matching reference count on the transcation 242 | pub(crate) fn deallocate_savepoint(&self, savepoint: SavepointId, transaction: TransactionId) { 243 | self.state 244 | .lock() 245 | .unwrap() 246 | .valid_savepoints 247 | .remove(&savepoint); 248 | self.deallocate_read_transaction(transaction); 249 | } 250 | 251 | pub(crate) fn is_valid_savepoint(&self, id: SavepointId) -> bool { 252 | self.state 253 | .lock() 254 | .unwrap() 255 | .valid_savepoints 256 | .contains_key(&id) 257 | } 258 | 259 | pub(crate) fn invalidate_savepoints_after(&self, id: SavepointId) { 260 | self.state 261 | .lock() 262 | .unwrap() 263 | .valid_savepoints 264 | .retain(|x, _| *x <= id); 265 | } 266 | 267 | pub(crate) fn oldest_savepoint(&self) -> Option<(SavepointId, TransactionId)> { 268 | self.state 269 | .lock() 270 | .unwrap() 271 | .valid_savepoints 272 | .first_key_value() 273 | .map(|x| (*x.0, *x.1)) 274 | } 275 | 276 | pub(crate) fn oldest_live_read_transaction(&self) -> Option { 277 | self.state 278 | .lock() 279 | .unwrap() 280 | .live_read_transactions 281 | .keys() 282 | .next() 283 | .copied() 284 | } 285 | 286 | // Returns the transaction id of the oldest non-durable transaction which has not been processed 287 | // for freeing, which has live read transactions 288 | pub(crate) fn oldest_live_read_nondurable_transaction(&self) -> Option { 289 | let state = self.state.lock().unwrap(); 290 | for id in state.live_read_transactions.keys() { 291 | if state.pending_non_durable_commits.contains_key(id) { 292 | return Some(*id); 293 | } 294 | } 295 | None 296 | } 297 | } 298 | -------------------------------------------------------------------------------- /src/legacy_tuple_types.rs: -------------------------------------------------------------------------------- 1 | use crate::types::{Key, TypeName, Value}; 2 | use std::borrow::Borrow; 3 | use std::cmp::Ordering; 4 | use std::mem::size_of; 5 | 6 | #[repr(transparent)] 7 | #[derive(Debug)] 8 | /// Legacy wrapper for tuple types created with redb version 2.x 9 | /// 10 | /// See the CHANGELOG.md file for more details 11 | pub struct Legacy(T); 12 | 13 | fn serialize_tuple_elements_variable(slices: &[&[u8]]) -> Vec { 14 | let total_len: usize = slices.iter().map(|x| x.len()).sum(); 15 | let mut output = Vec::with_capacity((slices.len() - 1) * size_of::() + total_len); 16 | for len in slices.iter().map(|x| x.len()).take(slices.len() - 1) { 17 | output.extend_from_slice(&(u32::try_from(len).unwrap()).to_le_bytes()); 18 | } 19 | 20 | for slice in slices { 21 | output.extend_from_slice(slice); 22 | } 23 | 24 | output 25 | } 26 | 27 | fn serialize_tuple_elements_fixed(slices: &[&[u8]]) -> Vec { 28 | let total_len: usize = slices.iter().map(|x| x.len()).sum(); 29 | let mut output = Vec::with_capacity(total_len); 30 | for slice in slices { 31 | output.extend_from_slice(slice); 32 | } 33 | output 34 | } 35 | 36 | fn parse_lens(data: &[u8]) -> [usize; N] { 37 | let mut result = [0; N]; 38 | for i in 0..N { 39 | result[i] = u32::from_le_bytes(data[4 * i..4 * (i + 1)].try_into().unwrap()) as usize; 40 | } 41 | result 42 | } 43 | 44 | fn not_equal(data1: &[u8], data2: &[u8]) -> Option { 45 | match T::compare(data1, data2) { 46 | Ordering::Less => Some(Ordering::Less), 47 | Ordering::Equal => None, 48 | Ordering::Greater => Some(Ordering::Greater), 49 | } 50 | } 51 | 52 | macro_rules! fixed_width_impl { 53 | ( $( $t:ty ),+ ) => { 54 | { 55 | let mut sum = 0; 56 | $( 57 | sum += <$t>::fixed_width()?; 58 | )+ 59 | Some(sum) 60 | } 61 | }; 62 | } 63 | 64 | macro_rules! as_bytes_impl { 65 | ( $value:expr, $( $t:ty, $i:tt ),+ ) => {{ 66 | if Self::fixed_width().is_some() { 67 | serialize_tuple_elements_fixed(&[ 68 | $( 69 | <$t>::as_bytes($value.$i.borrow()).as_ref(), 70 | )+ 71 | ]) 72 | } else { 73 | serialize_tuple_elements_variable(&[ 74 | $( 75 | <$t>::as_bytes($value.$i.borrow()).as_ref(), 76 | )+ 77 | ]) 78 | } 79 | }}; 80 | } 81 | 82 | macro_rules! type_name_impl { 83 | ( $head:ty $(,$tail:ty)+ ) => { 84 | { 85 | let mut result = String::new(); 86 | result.push('('); 87 | result.push_str(&<$head>::type_name().name()); 88 | $( 89 | result.push(','); 90 | result.push_str(&<$tail>::type_name().name()); 91 | )+ 92 | result.push(')'); 93 | 94 | TypeName::internal(&result) 95 | } 96 | }; 97 | } 98 | 99 | macro_rules! from_bytes_variable_impl { 100 | ( $data:expr $(,$t:ty, $v:ident, $i:literal )+ | $t_last:ty, $v_last:ident, $i_last:literal ) => { 101 | #[allow(clippy::manual_bits)] 102 | { 103 | let lens: [usize; $i_last] = parse_lens($data); 104 | let mut offset = $i_last * size_of::(); 105 | $( 106 | let len = lens[$i]; 107 | let $v = <$t>::from_bytes(&$data[offset..(offset + len)]); 108 | offset += len; 109 | )+ 110 | let $v_last = <$t_last>::from_bytes(&$data[offset..]); 111 | ($( 112 | $v, 113 | )+ 114 | $v_last 115 | ) 116 | } 117 | }; 118 | } 119 | 120 | macro_rules! from_bytes_fixed_impl { 121 | ( $data:expr $(,$t:ty, $v:ident )+ ) => { 122 | { 123 | let mut offset = 0; 124 | $( 125 | let len = <$t>::fixed_width().unwrap(); 126 | let $v = <$t>::from_bytes(&$data[offset..(offset + len)]); 127 | #[allow(unused_assignments)] 128 | { 129 | offset += len; 130 | } 131 | )+ 132 | 133 | ($( 134 | $v, 135 | )+) 136 | } 137 | }; 138 | } 139 | 140 | macro_rules! compare_variable_impl { 141 | ( $data0:expr, $data1:expr $(,$t:ty, $i:literal )+ | $t_last:ty, $i_last:literal ) => { 142 | #[allow(clippy::manual_bits)] 143 | { 144 | let lens0: [usize; $i_last] = parse_lens($data0); 145 | let lens1: [usize; $i_last] = parse_lens($data1); 146 | let mut offset0 = $i_last * size_of::(); 147 | let mut offset1 = $i_last * size_of::(); 148 | $( 149 | let index = $i; 150 | let len0 = lens0[index]; 151 | let len1 = lens1[index]; 152 | if let Some(order) = not_equal::<$t>( 153 | &$data0[offset0..(offset0 + len0)], 154 | &$data1[offset1..(offset1 + len1)], 155 | ) { 156 | return order; 157 | } 158 | offset0 += len0; 159 | offset1 += len1; 160 | )+ 161 | 162 | <$t_last>::compare(&$data0[offset0..], &$data1[offset1..]) 163 | } 164 | }; 165 | } 166 | 167 | macro_rules! compare_fixed_impl { 168 | ( $data0:expr, $data1:expr, $($t:ty),+ ) => { 169 | { 170 | let mut offset0 = 0; 171 | let mut offset1 = 0; 172 | $( 173 | let len = <$t>::fixed_width().unwrap(); 174 | if let Some(order) = not_equal::<$t>( 175 | &$data0[offset0..(offset0 + len)], 176 | &$data1[offset1..(offset1 + len)], 177 | ) { 178 | return order; 179 | } 180 | #[allow(unused_assignments)] 181 | { 182 | offset0 += len; 183 | offset1 += len; 184 | } 185 | )+ 186 | 187 | Ordering::Equal 188 | } 189 | }; 190 | } 191 | 192 | macro_rules! tuple_impl { 193 | ( $($t:ident, $v:ident, $i:tt ),+ | $t_last:ident, $v_last:ident, $i_last:tt ) => { 194 | impl<$($t: Value,)+ $t_last: Value> Value for Legacy<($($t,)+ $t_last)> { 195 | type SelfType<'a> = ( 196 | $(<$t>::SelfType<'a>,)+ 197 | <$t_last>::SelfType<'a>, 198 | ) 199 | where 200 | Self: 'a; 201 | type AsBytes<'a> = Vec 202 | where 203 | Self: 'a; 204 | 205 | fn fixed_width() -> Option { 206 | fixed_width_impl!($($t,)+ $t_last) 207 | } 208 | 209 | fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> 210 | where 211 | Self: 'a, 212 | { 213 | if Self::fixed_width().is_some() { 214 | from_bytes_fixed_impl!(data $(,$t,$v)+, $t_last, $v_last) 215 | } else { 216 | from_bytes_variable_impl!(data $(,$t,$v,$i)+ | $t_last, $v_last, $i_last) 217 | } 218 | } 219 | 220 | fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Vec 221 | where 222 | Self: 'a, 223 | Self: 'b, 224 | { 225 | as_bytes_impl!(value, $($t,$i,)+ $t_last, $i_last) 226 | } 227 | 228 | fn type_name() -> TypeName { 229 | type_name_impl!($($t,)+ $t_last) 230 | } 231 | } 232 | 233 | impl<$($t: Key,)+ $t_last: Key> Key for Legacy<($($t,)+ $t_last)> { 234 | fn compare(data1: &[u8], data2: &[u8]) -> Ordering { 235 | if Self::fixed_width().is_some() { 236 | compare_fixed_impl!(data1, data2, $($t,)+ $t_last) 237 | } else { 238 | compare_variable_impl!(data1, data2 $(,$t,$i)+ | $t_last, $i_last) 239 | } 240 | } 241 | } 242 | }; 243 | } 244 | 245 | impl Value for Legacy<(T,)> { 246 | type SelfType<'a> 247 | = (T::SelfType<'a>,) 248 | where 249 | Self: 'a; 250 | type AsBytes<'a> 251 | = T::AsBytes<'a> 252 | where 253 | Self: 'a; 254 | 255 | fn fixed_width() -> Option { 256 | T::fixed_width() 257 | } 258 | 259 | fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> 260 | where 261 | Self: 'a, 262 | { 263 | (T::from_bytes(data),) 264 | } 265 | 266 | fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> 267 | where 268 | Self: 'a, 269 | Self: 'b, 270 | { 271 | T::as_bytes(&value.0) 272 | } 273 | 274 | fn type_name() -> TypeName { 275 | TypeName::internal(&format!("({},)", T::type_name().name())) 276 | } 277 | } 278 | 279 | impl Key for Legacy<(T,)> { 280 | fn compare(data1: &[u8], data2: &[u8]) -> Ordering { 281 | T::compare(data1, data2) 282 | } 283 | } 284 | 285 | tuple_impl! { 286 | T0, t0, 0 287 | | T1, t1, 1 288 | } 289 | 290 | tuple_impl! { 291 | T0, t0, 0, 292 | T1, t1, 1 293 | | T2, t2, 2 294 | } 295 | 296 | tuple_impl! { 297 | T0, t0, 0, 298 | T1, t1, 1, 299 | T2, t2, 2 300 | | T3, t3, 3 301 | } 302 | 303 | tuple_impl! { 304 | T0, t0, 0, 305 | T1, t1, 1, 306 | T2, t2, 2, 307 | T3, t3, 3 308 | | T4, t4, 4 309 | } 310 | 311 | tuple_impl! { 312 | T0, t0, 0, 313 | T1, t1, 1, 314 | T2, t2, 2, 315 | T3, t3, 3, 316 | T4, t4, 4 317 | | T5, t5, 5 318 | } 319 | 320 | tuple_impl! { 321 | T0, t0, 0, 322 | T1, t1, 1, 323 | T2, t2, 2, 324 | T3, t3, 3, 325 | T4, t4, 4, 326 | T5, t5, 5 327 | | T6, t6, 6 328 | } 329 | 330 | tuple_impl! { 331 | T0, t0, 0, 332 | T1, t1, 1, 333 | T2, t2, 2, 334 | T3, t3, 3, 335 | T4, t4, 4, 336 | T5, t5, 5, 337 | T6, t6, 6 338 | | T7, t7, 7 339 | } 340 | 341 | tuple_impl! { 342 | T0, t0, 0, 343 | T1, t1, 1, 344 | T2, t2, 2, 345 | T3, t3, 3, 346 | T4, t4, 4, 347 | T5, t5, 5, 348 | T6, t6, 6, 349 | T7, t7, 7 350 | | T8, t8, 8 351 | } 352 | 353 | tuple_impl! { 354 | T0, t0, 0, 355 | T1, t1, 1, 356 | T2, t2, 2, 357 | T3, t3, 3, 358 | T4, t4, 4, 359 | T5, t5, 5, 360 | T6, t6, 6, 361 | T7, t7, 7, 362 | T8, t8, 8 363 | | T9, t9, 9 364 | } 365 | 366 | tuple_impl! { 367 | T0, t0, 0, 368 | T1, t1, 1, 369 | T2, t2, 2, 370 | T3, t3, 3, 371 | T4, t4, 4, 372 | T5, t5, 5, 373 | T6, t6, 6, 374 | T7, t7, 7, 375 | T8, t8, 8, 376 | T9, t9, 9 377 | | T10, t10, 10 378 | } 379 | 380 | tuple_impl! { 381 | T0, t0, 0, 382 | T1, t1, 1, 383 | T2, t2, 2, 384 | T3, t3, 3, 385 | T4, t4, 4, 386 | T5, t5, 5, 387 | T6, t6, 6, 388 | T7, t7, 7, 389 | T8, t8, 8, 390 | T9, t9, 9, 391 | T10, t10, 10 392 | | T11, t11, 11 393 | } 394 | 395 | #[cfg(test)] 396 | mod test { 397 | use crate::legacy_tuple_types::Legacy; 398 | use crate::types::Value; 399 | 400 | #[test] 401 | fn width() { 402 | assert!(>::fixed_width().is_none()); 403 | assert!(>::fixed_width().is_none()); 404 | assert_eq!(>::fixed_width().unwrap(), 2); 405 | assert_eq!(>::fixed_width().unwrap(), 3); 406 | assert_eq!(>::fixed_width().unwrap(), 19); 407 | assert_eq!(>::fixed_width().unwrap(), 20); 408 | } 409 | } 410 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /src/tree_store/page_store/base.rs: -------------------------------------------------------------------------------- 1 | use crate::tree_store::page_store::cached_file::WritablePage; 2 | use crate::tree_store::page_store::page_manager::MAX_MAX_PAGE_ORDER; 3 | use std::cmp::Ordering; 4 | #[cfg(debug_assertions)] 5 | use std::collections::HashMap; 6 | use std::collections::HashSet; 7 | use std::fmt::{Debug, Formatter}; 8 | use std::hash::{Hash, Hasher}; 9 | use std::mem; 10 | use std::ops::Range; 11 | use std::sync::Arc; 12 | #[cfg(debug_assertions)] 13 | use std::sync::Mutex; 14 | 15 | pub(crate) const MAX_VALUE_LENGTH: usize = 3 * 1024 * 1024 * 1024; 16 | pub(crate) const MAX_PAIR_LENGTH: usize = 3 * 1024 * 1024 * 1024 + 768 * 1024 * 1024; 17 | pub(crate) const MAX_PAGE_INDEX: u32 = 0x000F_FFFF; 18 | pub(crate) const MAX_REGIONS: u32 = 0x0010_0000; 19 | 20 | // On-disk format is: 21 | // TODO: consider implementing an optimization in which we store the number of order-0 pages that 22 | // are actually used, in these reserved bits, so that the reads to the PagedCachedFile layer can avoid 23 | // reading all the zeros at the end of the page. 24 | // lowest 20bits: page index within the region. Only the lowest `20 - order_exponent` bits may be read. 25 | // The remaining bits are reserved for future use and must be ignored 26 | // second 20bits: region number 27 | // 19bits: reserved 28 | // highest 5bits: page order exponent 29 | // 30 | // Assuming a reasonable page size, like 4kiB, this allows for 4kiB * 2^20 * 2^20 = 4PiB of usable space 31 | #[derive(Copy, Clone, Eq, PartialEq)] 32 | pub(crate) struct PageNumber { 33 | pub(crate) region: u32, 34 | pub(crate) page_index: u32, 35 | pub(crate) page_order: u8, 36 | } 37 | 38 | impl Hash for PageNumber { 39 | fn hash(&self, state: &mut H) { 40 | // TODO: maybe we should store these fields as a single u64 in PageNumber. The field access 41 | // will be a little more expensive, but I think it's less frequent than these hashes 42 | let mut temp = 0x000F_FFFF & u64::from(self.page_index); 43 | temp |= (0x000F_FFFF & u64::from(self.region)) << 20; 44 | temp |= (0b0001_1111 & u64::from(self.page_order)) << 59; 45 | state.write_u64(temp); 46 | } 47 | } 48 | 49 | // PageNumbers are ordered as determined by their starting address in the database file 50 | impl Ord for PageNumber { 51 | fn cmp(&self, other: &Self) -> Ordering { 52 | match self.region.cmp(&other.region) { 53 | Ordering::Less => Ordering::Less, 54 | Ordering::Equal => { 55 | let self_order0 = self.page_index * 2u32.pow(self.page_order.into()); 56 | let other_order0 = other.page_index * 2u32.pow(other.page_order.into()); 57 | assert!( 58 | self_order0 != other_order0 || self.page_order == other.page_order, 59 | "{self:?} overlaps {other:?}, but is not equal" 60 | ); 61 | self_order0.cmp(&other_order0) 62 | } 63 | Ordering::Greater => Ordering::Greater, 64 | } 65 | } 66 | } 67 | 68 | impl PartialOrd for PageNumber { 69 | fn partial_cmp(&self, other: &Self) -> Option { 70 | Some(self.cmp(other)) 71 | } 72 | } 73 | 74 | impl PageNumber { 75 | pub(crate) const fn serialized_size() -> usize { 76 | 8 77 | } 78 | 79 | pub(crate) fn new(region: u32, page_index: u32, page_order: u8) -> Self { 80 | debug_assert!(region <= 0x000F_FFFF); 81 | debug_assert!(page_index <= MAX_PAGE_INDEX); 82 | debug_assert!(page_order <= MAX_MAX_PAGE_ORDER); 83 | Self { 84 | region, 85 | page_index, 86 | page_order, 87 | } 88 | } 89 | 90 | pub(crate) fn to_le_bytes(self) -> [u8; 8] { 91 | let mut temp = 0x000F_FFFF & u64::from(self.page_index); 92 | temp |= (0x000F_FFFF & u64::from(self.region)) << 20; 93 | temp |= (0b0001_1111 & u64::from(self.page_order)) << 59; 94 | temp.to_le_bytes() 95 | } 96 | 97 | pub(crate) fn from_le_bytes(bytes: [u8; 8]) -> Self { 98 | let temp = u64::from_le_bytes(bytes); 99 | let order = (temp >> 59) as u8; 100 | let index = u32::try_from(temp & (0x000F_FFFF >> order)).unwrap(); 101 | let region = ((temp >> 20) & 0x000F_FFFF) as u32; 102 | 103 | Self { 104 | region, 105 | page_index: index, 106 | page_order: order, 107 | } 108 | } 109 | 110 | #[cfg(test)] 111 | pub(crate) fn to_order0(self) -> Vec { 112 | let mut pages = vec![self]; 113 | loop { 114 | let mut progress = false; 115 | let mut new_pages = vec![]; 116 | for page in pages { 117 | if page.page_order == 0 { 118 | new_pages.push(page); 119 | } else { 120 | progress = true; 121 | new_pages.push(PageNumber::new( 122 | page.region, 123 | page.page_index * 2, 124 | page.page_order - 1, 125 | )); 126 | new_pages.push(PageNumber::new( 127 | page.region, 128 | page.page_index * 2 + 1, 129 | page.page_order - 1, 130 | )); 131 | } 132 | } 133 | pages = new_pages; 134 | if !progress { 135 | break; 136 | } 137 | } 138 | 139 | pages 140 | } 141 | 142 | pub(crate) fn address_range( 143 | &self, 144 | data_section_offset: u64, 145 | region_size: u64, 146 | region_pages_start: u64, 147 | page_size: u32, 148 | ) -> Range { 149 | let regional_start = 150 | region_pages_start + u64::from(self.page_index) * self.page_size_bytes(page_size); 151 | debug_assert!(regional_start < region_size); 152 | let region_base = u64::from(self.region) * region_size; 153 | let start = data_section_offset + region_base + regional_start; 154 | let end = start + self.page_size_bytes(page_size); 155 | start..end 156 | } 157 | 158 | pub(crate) fn page_size_bytes(&self, page_size: u32) -> u64 { 159 | let pages = 1u64 << self.page_order; 160 | pages * u64::from(page_size) 161 | } 162 | } 163 | 164 | impl Debug for PageNumber { 165 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 166 | write!( 167 | f, 168 | "r{}.{}/{}", 169 | self.region, self.page_index, self.page_order 170 | ) 171 | } 172 | } 173 | 174 | pub(crate) trait Page { 175 | fn memory(&self) -> &[u8]; 176 | 177 | fn get_page_number(&self) -> PageNumber; 178 | } 179 | 180 | pub struct PageImpl { 181 | pub(super) mem: Arc<[u8]>, 182 | pub(super) page_number: PageNumber, 183 | #[cfg(debug_assertions)] 184 | pub(super) open_pages: Arc>>, 185 | } 186 | 187 | impl PageImpl { 188 | pub(crate) fn to_arc(&self) -> Arc<[u8]> { 189 | self.mem.clone() 190 | } 191 | } 192 | 193 | impl Debug for PageImpl { 194 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 195 | f.write_fmt(format_args!("PageImpl: page_number={:?}", self.page_number)) 196 | } 197 | } 198 | 199 | #[cfg(debug_assertions)] 200 | impl Drop for PageImpl { 201 | fn drop(&mut self) { 202 | let mut open_pages = self.open_pages.lock().unwrap(); 203 | let value = open_pages.get_mut(&self.page_number).unwrap(); 204 | assert!(*value > 0); 205 | *value -= 1; 206 | if *value == 0 { 207 | open_pages.remove(&self.page_number); 208 | } 209 | } 210 | } 211 | 212 | impl Page for PageImpl { 213 | fn memory(&self) -> &[u8] { 214 | self.mem.as_ref() 215 | } 216 | 217 | fn get_page_number(&self) -> PageNumber { 218 | self.page_number 219 | } 220 | } 221 | 222 | impl Clone for PageImpl { 223 | fn clone(&self) -> Self { 224 | #[cfg(debug_assertions)] 225 | { 226 | *self 227 | .open_pages 228 | .lock() 229 | .unwrap() 230 | .get_mut(&self.page_number) 231 | .unwrap() += 1; 232 | } 233 | Self { 234 | mem: self.mem.clone(), 235 | page_number: self.page_number, 236 | #[cfg(debug_assertions)] 237 | open_pages: self.open_pages.clone(), 238 | } 239 | } 240 | } 241 | 242 | pub(crate) struct PageMut { 243 | pub(super) mem: WritablePage, 244 | pub(super) page_number: PageNumber, 245 | #[cfg(debug_assertions)] 246 | pub(super) open_pages: Arc>>, 247 | } 248 | 249 | impl PageMut { 250 | pub(crate) fn memory_mut(&mut self) -> &mut [u8] { 251 | self.mem.mem_mut() 252 | } 253 | } 254 | 255 | impl Page for PageMut { 256 | fn memory(&self) -> &[u8] { 257 | self.mem.mem() 258 | } 259 | 260 | fn get_page_number(&self) -> PageNumber { 261 | self.page_number 262 | } 263 | } 264 | 265 | #[cfg(debug_assertions)] 266 | impl Drop for PageMut { 267 | fn drop(&mut self) { 268 | assert!(self.open_pages.lock().unwrap().remove(&self.page_number)); 269 | } 270 | } 271 | 272 | #[derive(Copy, Clone)] 273 | pub(crate) enum PageHint { 274 | None, 275 | Clean, 276 | } 277 | 278 | pub(crate) enum PageTrackerPolicy { 279 | Ignore, 280 | Track(HashSet), 281 | Closed, 282 | } 283 | 284 | impl PageTrackerPolicy { 285 | pub(crate) fn new_tracking() -> Self { 286 | PageTrackerPolicy::Track(HashSet::new()) 287 | } 288 | 289 | pub(crate) fn is_empty(&self) -> bool { 290 | match self { 291 | PageTrackerPolicy::Ignore | PageTrackerPolicy::Closed => true, 292 | PageTrackerPolicy::Track(x) => x.is_empty(), 293 | } 294 | } 295 | 296 | pub(super) fn remove(&mut self, page: PageNumber) { 297 | match self { 298 | PageTrackerPolicy::Ignore => {} 299 | PageTrackerPolicy::Track(x) => { 300 | assert!(x.remove(&page)); 301 | } 302 | PageTrackerPolicy::Closed => { 303 | panic!("Page tracker is closed"); 304 | } 305 | } 306 | } 307 | 308 | pub(super) fn insert(&mut self, page: PageNumber) { 309 | match self { 310 | PageTrackerPolicy::Ignore => {} 311 | PageTrackerPolicy::Track(x) => { 312 | assert!(x.insert(page)); 313 | } 314 | PageTrackerPolicy::Closed => { 315 | panic!("Page tracker is closed"); 316 | } 317 | } 318 | } 319 | 320 | pub(crate) fn close(&mut self) -> HashSet { 321 | let old = mem::replace(self, PageTrackerPolicy::Closed); 322 | match old { 323 | PageTrackerPolicy::Ignore => HashSet::new(), 324 | PageTrackerPolicy::Track(x) => x, 325 | PageTrackerPolicy::Closed => { 326 | panic!("Page tracker is closed"); 327 | } 328 | } 329 | } 330 | } 331 | 332 | #[cfg(test)] 333 | mod test { 334 | use crate::tree_store::PageNumber; 335 | 336 | #[test] 337 | fn last_page() { 338 | let region_data_size = 2u64.pow(32); 339 | let page_size = 4096; 340 | let pages_per_region = region_data_size / page_size; 341 | let region_header_size = 2u64.pow(16); 342 | let last_page_index = pages_per_region - 1; 343 | let page_number = PageNumber::new(1, last_page_index.try_into().unwrap(), 0); 344 | page_number.address_range( 345 | 4096, 346 | region_data_size + region_header_size, 347 | region_header_size, 348 | page_size.try_into().unwrap(), 349 | ); 350 | } 351 | 352 | #[test] 353 | fn reserved_bits() { 354 | let page_number = PageNumber::new(0, 0, 12); 355 | let mut bytes = page_number.to_le_bytes(); 356 | bytes[1] = 0xFF; 357 | let page_number2 = PageNumber::from_le_bytes(bytes); 358 | assert_eq!(page_number, page_number2); 359 | } 360 | } 361 | -------------------------------------------------------------------------------- /tests/backward_compatibility.rs: -------------------------------------------------------------------------------- 1 | use redb::{Legacy, ReadableDatabase, ReadableTableMetadata, TableError}; 2 | 3 | const ELEMENTS: usize = 3; 4 | 5 | trait TestData: redb::Value + redb2_6::Value { 6 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] 7 | where 8 | Self: 'a; 9 | 10 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] 11 | where 12 | Self: 'a; 13 | } 14 | 15 | impl TestData for u8 { 16 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 17 | [0, 1, 2] 18 | } 19 | 20 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 21 | [0, 1, 2] 22 | } 23 | } 24 | 25 | impl TestData for u16 { 26 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 27 | [0, 1, 2] 28 | } 29 | 30 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 31 | [0, 1, 2] 32 | } 33 | } 34 | 35 | impl TestData for u32 { 36 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 37 | [0, 1, 2] 38 | } 39 | 40 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 41 | [0, 1, 2] 42 | } 43 | } 44 | 45 | impl TestData for u64 { 46 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 47 | [0, 1, 2] 48 | } 49 | 50 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 51 | [0, 1, 2] 52 | } 53 | } 54 | 55 | impl TestData for u128 { 56 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 57 | [0, 1, 2] 58 | } 59 | 60 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 61 | [0, 1, 2] 62 | } 63 | } 64 | 65 | impl TestData for i8 { 66 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 67 | [-1, 1, 2] 68 | } 69 | 70 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 71 | [-1, 1, 2] 72 | } 73 | } 74 | 75 | impl TestData for i16 { 76 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 77 | [-1, 1, 2] 78 | } 79 | 80 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 81 | [-1, 1, 2] 82 | } 83 | } 84 | 85 | impl TestData for i32 { 86 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 87 | [-1, 1, 2] 88 | } 89 | 90 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 91 | [-1, 1, 2] 92 | } 93 | } 94 | 95 | impl TestData for i64 { 96 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 97 | [-1, 1, 2] 98 | } 99 | 100 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 101 | [-1, 1, 2] 102 | } 103 | } 104 | 105 | impl TestData for i128 { 106 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 107 | [-1, 1, 2] 108 | } 109 | 110 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 111 | [-1, 1, 2] 112 | } 113 | } 114 | 115 | impl TestData for f32 { 116 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 117 | [f32::NAN, f32::INFINITY, f32::MIN_POSITIVE] 118 | } 119 | 120 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 121 | [f32::NAN, f32::INFINITY, f32::MIN_POSITIVE] 122 | } 123 | } 124 | 125 | impl TestData for f64 { 126 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 127 | [f64::MIN, f64::NEG_INFINITY, f64::MAX] 128 | } 129 | 130 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 131 | [f64::MIN, f64::NEG_INFINITY, f64::MAX] 132 | } 133 | } 134 | 135 | impl TestData for () { 136 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 137 | [(), (), ()] 138 | } 139 | 140 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 141 | [(), (), ()] 142 | } 143 | } 144 | 145 | impl TestData for &'static str { 146 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 147 | ["hello", "world1", "hi"] 148 | } 149 | 150 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 151 | ["hello", "world1", "hi"] 152 | } 153 | } 154 | 155 | impl TestData for &'static [u8] { 156 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 157 | [b"test", b"bytes", b"now"] 158 | } 159 | 160 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 161 | [b"test", b"bytes", b"now"] 162 | } 163 | } 164 | 165 | impl TestData for &'static [u8; 5] { 166 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 167 | [b"test1", b"bytes", b"now12"] 168 | } 169 | 170 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 171 | [b"test1", b"bytes", b"now12"] 172 | } 173 | } 174 | 175 | impl TestData for [&str; 3] { 176 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] 177 | where 178 | Self: 'a, 179 | { 180 | [ 181 | ["test1", "hi", "world"], 182 | ["test2", "hi", "world"], 183 | ["test3", "hi", "world"], 184 | ] 185 | } 186 | 187 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] 188 | where 189 | Self: 'a, 190 | { 191 | [ 192 | ["test1", "hi", "world"], 193 | ["test2", "hi", "world"], 194 | ["test3", "hi", "world"], 195 | ] 196 | } 197 | } 198 | 199 | impl TestData for [u128; 3] { 200 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 201 | [[1, 2, 3], [3, 2, 1], [300, 200, 100]] 202 | } 203 | 204 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 205 | [[1, 2, 3], [3, 2, 1], [300, 200, 100]] 206 | } 207 | } 208 | 209 | impl TestData for Vec<&str> { 210 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] 211 | where 212 | Self: 'a, 213 | { 214 | [ 215 | vec!["test1", "hi", "world"], 216 | vec!["test2", "hi", "world"], 217 | vec!["test3", "hi", "world"], 218 | ] 219 | } 220 | 221 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] 222 | where 223 | Self: 'a, 224 | { 225 | [ 226 | vec!["test1", "hi", "world"], 227 | vec!["test2", "hi", "world"], 228 | vec!["test3", "hi", "world"], 229 | ] 230 | } 231 | } 232 | 233 | impl TestData for Option { 234 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 235 | [None, Some(0), Some(7)] 236 | } 237 | 238 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 239 | [None, Some(0), Some(7)] 240 | } 241 | } 242 | 243 | impl TestData for (u64, &'static str) { 244 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 245 | [(0, "hi"), (1, "bye"), (2, "byte")] 246 | } 247 | 248 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 249 | [(0, "hi"), (1, "bye"), (2, "byte")] 250 | } 251 | } 252 | 253 | impl TestData for (u64, u32) { 254 | fn make_data_v2_6<'a>() -> [::SelfType<'a>; ELEMENTS] { 255 | [(0, 3), (1, 4), (2, 5)] 256 | } 257 | 258 | fn make_data<'a>() -> [::SelfType<'a>; ELEMENTS] { 259 | [(0, 3), (1, 4), (2, 5)] 260 | } 261 | } 262 | 263 | fn create_tempfile() -> tempfile::NamedTempFile { 264 | if cfg!(target_os = "wasi") { 265 | tempfile::NamedTempFile::new_in("/tmp").unwrap() 266 | } else { 267 | tempfile::NamedTempFile::new().unwrap() 268 | } 269 | } 270 | 271 | fn test_helper() { 272 | { 273 | let tmpfile = create_tempfile(); 274 | let db = redb2_6::Database::builder() 275 | .create_with_file_format_v3(true) 276 | .create(tmpfile.path()) 277 | .unwrap(); 278 | let table_def: redb2_6::TableDefinition = redb2_6::TableDefinition::new("table"); 279 | let write_txn = db.begin_write().unwrap(); 280 | { 281 | let mut table = write_txn.open_table(table_def).unwrap(); 282 | for i in 0..ELEMENTS { 283 | table 284 | .insert(&K::make_data_v2_6()[i], &V::make_data_v2_6()[i]) 285 | .unwrap(); 286 | } 287 | } 288 | write_txn.commit().unwrap(); 289 | drop(db); 290 | 291 | let db = redb::Database::open(tmpfile.path()).unwrap(); 292 | let read_txn = db.begin_read().unwrap(); 293 | let table_def: redb::TableDefinition = redb::TableDefinition::new("table"); 294 | let table = read_txn.open_table(table_def).unwrap(); 295 | assert_eq!(table.len().unwrap(), ELEMENTS as u64); 296 | for i in 0..ELEMENTS { 297 | let result = table.get(&K::make_data()[i]).unwrap().unwrap(); 298 | let value = result.value(); 299 | let bytes = ::as_bytes(&value); 300 | let expected = &V::make_data()[i]; 301 | let expected_bytes = ::as_bytes(expected); 302 | assert_eq!(bytes.as_ref(), expected_bytes.as_ref()); 303 | } 304 | } 305 | } 306 | 307 | #[test] 308 | fn primitive_types() { 309 | test_helper::(); 310 | test_helper::(); 311 | test_helper::(); 312 | test_helper::(); 313 | test_helper::(); 314 | test_helper::(); 315 | test_helper::(); 316 | test_helper::(); 317 | test_helper::(); 318 | test_helper::(); 319 | test_helper::(); 320 | test_helper::(); 321 | test_helper::<&str, &str>(); 322 | test_helper::(); 323 | } 324 | 325 | #[test] 326 | fn container_types() { 327 | test_helper::<&[u8], &[u8]>(); 328 | test_helper::<&[u8; 5], &[u8; 5]>(); 329 | test_helper::>(); 330 | test_helper::<(u64, u32), &str>(); 331 | test_helper::<[&str; 3], [u128; 3]>(); 332 | test_helper::>(); 333 | } 334 | 335 | #[test] 336 | fn mixed_width() { 337 | test_helper::(); 338 | test_helper::<&[u8; 5], &str>(); 339 | } 340 | 341 | #[test] 342 | fn tuple_types() { 343 | let tmpfile = create_tempfile(); 344 | let db = redb2_6::Database::builder() 345 | .create_with_file_format_v3(true) 346 | .create(tmpfile.path()) 347 | .unwrap(); 348 | let table_def: redb2_6::TableDefinition<(u64, &str), &str> = 349 | redb2_6::TableDefinition::new("table"); 350 | let write_txn = db.begin_write().unwrap(); 351 | { 352 | let mut table = write_txn.open_table(table_def).unwrap(); 353 | for i in 0..ELEMENTS { 354 | table 355 | .insert( 356 | &<(u64, &str)>::make_data_v2_6()[i], 357 | &<&str>::make_data_v2_6()[i], 358 | ) 359 | .unwrap(); 360 | } 361 | } 362 | write_txn.commit().unwrap(); 363 | drop(db); 364 | 365 | let db = redb::Database::open(tmpfile.path()).unwrap(); 366 | let read_txn = db.begin_read().unwrap(); 367 | let bad_table_def: redb::TableDefinition<(u64, &str), &str> = 368 | redb::TableDefinition::new("table"); 369 | assert!(matches!( 370 | read_txn.open_table(bad_table_def).unwrap_err(), 371 | TableError::TableTypeMismatch { .. } 372 | )); 373 | let table_def: redb::TableDefinition, &str> = 374 | redb::TableDefinition::new("table"); 375 | let table = read_txn.open_table(table_def).unwrap(); 376 | assert_eq!(table.len().unwrap(), ELEMENTS as u64); 377 | for i in 0..ELEMENTS { 378 | let result = table.get(&<(u64, &str)>::make_data()[i]).unwrap().unwrap(); 379 | let value = result.value(); 380 | let bytes = <&str as redb::Value>::as_bytes(&value); 381 | let expected = &<&str>::make_data()[i]; 382 | let expected_bytes = <&str as redb::Value>::as_bytes(expected); 383 | assert_eq!(bytes, expected_bytes); 384 | } 385 | } 386 | -------------------------------------------------------------------------------- /src/tuple_types.rs: -------------------------------------------------------------------------------- 1 | use crate::complex_types::{decode_varint_len, encode_varint_len}; 2 | use crate::types::{Key, TypeName, Value}; 3 | use std::borrow::Borrow; 4 | use std::cmp::Ordering; 5 | 6 | fn serialize_tuple_elements_variable( 7 | is_fixed_width: [bool; N], 8 | slices: [&[u8]; N], 9 | ) -> Vec { 10 | let total_len: usize = slices.iter().map(|x| x.len()).sum(); 11 | let worst_case_len_overhead: usize = 12 | is_fixed_width.iter().map(|x| if *x { 0 } else { 5 }).sum(); 13 | let mut output = Vec::with_capacity(total_len + worst_case_len_overhead); 14 | let zipped = is_fixed_width.iter().zip(slices.iter()); 15 | for len in zipped 16 | .map(|(fixed, x)| if *fixed { None } else { Some(x.len()) }) 17 | .take(slices.len() - 1) 18 | .flatten() 19 | { 20 | encode_varint_len(len, &mut output); 21 | } 22 | 23 | for slice in slices { 24 | output.extend_from_slice(slice); 25 | } 26 | 27 | debug_assert!(output.len() <= total_len + worst_case_len_overhead); 28 | 29 | output 30 | } 31 | 32 | fn serialize_tuple_elements_fixed(slices: &[&[u8]]) -> Vec { 33 | let total_len: usize = slices.iter().map(|x| x.len()).sum(); 34 | let mut output = Vec::with_capacity(total_len); 35 | for slice in slices { 36 | output.extend_from_slice(slice); 37 | } 38 | output 39 | } 40 | 41 | fn parse_lens(fixed_width: [Option; N], data: &[u8]) -> (usize, [usize; N]) { 42 | let mut result = [0; N]; 43 | let mut offset = 0; 44 | for (i, &fixed) in fixed_width.iter().enumerate() { 45 | if let Some(len) = fixed { 46 | result[i] = len; 47 | } else { 48 | let (len, bytes_read) = decode_varint_len(&data[offset..]); 49 | result[i] = len; 50 | offset += bytes_read; 51 | } 52 | } 53 | (offset, result) 54 | } 55 | 56 | fn not_equal(data1: &[u8], data2: &[u8]) -> Option { 57 | match T::compare(data1, data2) { 58 | Ordering::Less => Some(Ordering::Less), 59 | Ordering::Equal => None, 60 | Ordering::Greater => Some(Ordering::Greater), 61 | } 62 | } 63 | 64 | macro_rules! fixed_width_impl { 65 | ( $( $t:ty ),+ ) => { 66 | { 67 | let mut sum = 0; 68 | $( 69 | sum += <$t>::fixed_width()?; 70 | )+ 71 | Some(sum) 72 | } 73 | }; 74 | } 75 | 76 | macro_rules! as_bytes_impl { 77 | ( $value:expr, $( $t:ty, $i:tt ),+ ) => {{ 78 | if Self::fixed_width().is_some() { 79 | serialize_tuple_elements_fixed(&[ 80 | $( 81 | <$t>::as_bytes($value.$i.borrow()).as_ref(), 82 | )+ 83 | ]) 84 | } else { 85 | serialize_tuple_elements_variable( 86 | [ 87 | $( 88 | <$t>::fixed_width().is_some(), 89 | )+ 90 | ], 91 | [ 92 | $( 93 | <$t>::as_bytes($value.$i.borrow()).as_ref(), 94 | )+ 95 | ]) 96 | } 97 | }}; 98 | } 99 | 100 | macro_rules! type_name_impl { 101 | ( $head:ty $(,$tail:ty)+ ) => { 102 | { 103 | let mut result = String::new(); 104 | result.push('('); 105 | result.push_str(&<$head>::type_name().name()); 106 | $( 107 | result.push(','); 108 | result.push_str(&<$tail>::type_name().name()); 109 | )+ 110 | result.push(')'); 111 | 112 | if Self::fixed_width().is_some() { 113 | TypeName::internal(&result) 114 | } else { 115 | TypeName::internal2(&result) 116 | } 117 | } 118 | }; 119 | } 120 | 121 | macro_rules! from_bytes_variable_impl { 122 | ( $data:expr $(,$t:ty, $v:ident, $i:literal )+ | $t_last:ty, $v_last:ident, $i_last:literal ) => { 123 | #[allow(clippy::manual_bits)] 124 | { 125 | let (mut offset, lens) = parse_lens::<$i_last>( 126 | [ 127 | $( 128 | <$t>::fixed_width(), 129 | )+ 130 | ], 131 | $data); 132 | $( 133 | let len = lens[$i]; 134 | let $v = <$t>::from_bytes(&$data[offset..(offset + len)]); 135 | offset += len; 136 | )+ 137 | let $v_last = <$t_last>::from_bytes(&$data[offset..]); 138 | ($( 139 | $v, 140 | )+ 141 | $v_last 142 | ) 143 | } 144 | }; 145 | } 146 | 147 | macro_rules! from_bytes_fixed_impl { 148 | ( $data:expr $(,$t:ty, $v:ident )+ ) => { 149 | { 150 | let mut offset = 0; 151 | $( 152 | let len = <$t>::fixed_width().unwrap(); 153 | let $v = <$t>::from_bytes(&$data[offset..(offset + len)]); 154 | #[allow(unused_assignments)] 155 | { 156 | offset += len; 157 | } 158 | )+ 159 | 160 | ($( 161 | $v, 162 | )+) 163 | } 164 | }; 165 | } 166 | 167 | macro_rules! compare_variable_impl { 168 | ( $data0:expr, $data1:expr $(,$t:ty, $i:literal )+ | $t_last:ty, $i_last:literal ) => { 169 | #[allow(clippy::manual_bits)] 170 | { 171 | let fixed_width = [ 172 | $( 173 | <$t>::fixed_width(), 174 | )+ 175 | ]; 176 | let (mut offset0, lens0) = parse_lens::<$i_last>(fixed_width, $data0); 177 | let (mut offset1, lens1) = parse_lens::<$i_last>(fixed_width, $data1); 178 | $( 179 | let index = $i; 180 | let len0 = lens0[index]; 181 | let len1 = lens1[index]; 182 | if let Some(order) = not_equal::<$t>( 183 | &$data0[offset0..(offset0 + len0)], 184 | &$data1[offset1..(offset1 + len1)], 185 | ) { 186 | return order; 187 | } 188 | offset0 += len0; 189 | offset1 += len1; 190 | )+ 191 | 192 | <$t_last>::compare(&$data0[offset0..], &$data1[offset1..]) 193 | } 194 | }; 195 | } 196 | 197 | macro_rules! compare_fixed_impl { 198 | ( $data0:expr, $data1:expr, $($t:ty),+ ) => { 199 | { 200 | let mut offset0 = 0; 201 | let mut offset1 = 0; 202 | $( 203 | let len = <$t>::fixed_width().unwrap(); 204 | if let Some(order) = not_equal::<$t>( 205 | &$data0[offset0..(offset0 + len)], 206 | &$data1[offset1..(offset1 + len)], 207 | ) { 208 | return order; 209 | } 210 | #[allow(unused_assignments)] 211 | { 212 | offset0 += len; 213 | offset1 += len; 214 | } 215 | )+ 216 | 217 | Ordering::Equal 218 | } 219 | }; 220 | } 221 | 222 | macro_rules! tuple_impl { 223 | ( $($t:ident, $v:ident, $i:tt ),+ | $t_last:ident, $v_last:ident, $i_last:tt ) => { 224 | impl<$($t: Value,)+ $t_last: Value> Value for ($($t,)+ $t_last) { 225 | type SelfType<'a> = ( 226 | $(<$t>::SelfType<'a>,)+ 227 | <$t_last>::SelfType<'a>, 228 | ) 229 | where 230 | Self: 'a; 231 | type AsBytes<'a> = Vec 232 | where 233 | Self: 'a; 234 | 235 | fn fixed_width() -> Option { 236 | fixed_width_impl!($($t,)+ $t_last) 237 | } 238 | 239 | fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> 240 | where 241 | Self: 'a, 242 | { 243 | if Self::fixed_width().is_some() { 244 | from_bytes_fixed_impl!(data $(,$t,$v)+, $t_last, $v_last) 245 | } else { 246 | from_bytes_variable_impl!(data $(,$t,$v,$i)+ | $t_last, $v_last, $i_last) 247 | } 248 | } 249 | 250 | fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Vec 251 | where 252 | Self: 'a, 253 | Self: 'b, 254 | { 255 | as_bytes_impl!(value, $($t,$i,)+ $t_last, $i_last) 256 | } 257 | 258 | fn type_name() -> TypeName { 259 | type_name_impl!($($t,)+ $t_last) 260 | } 261 | } 262 | 263 | impl<$($t: Key,)+ $t_last: Key> Key for ($($t,)+ $t_last) { 264 | fn compare(data1: &[u8], data2: &[u8]) -> Ordering { 265 | if Self::fixed_width().is_some() { 266 | compare_fixed_impl!(data1, data2, $($t,)+ $t_last) 267 | } else { 268 | compare_variable_impl!(data1, data2 $(,$t,$i)+ | $t_last, $i_last) 269 | } 270 | } 271 | } 272 | }; 273 | } 274 | 275 | impl Value for (T,) { 276 | type SelfType<'a> 277 | = (T::SelfType<'a>,) 278 | where 279 | Self: 'a; 280 | type AsBytes<'a> 281 | = T::AsBytes<'a> 282 | where 283 | Self: 'a; 284 | 285 | fn fixed_width() -> Option { 286 | T::fixed_width() 287 | } 288 | 289 | fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> 290 | where 291 | Self: 'a, 292 | { 293 | (T::from_bytes(data),) 294 | } 295 | 296 | fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> 297 | where 298 | Self: 'a, 299 | Self: 'b, 300 | { 301 | T::as_bytes(&value.0) 302 | } 303 | 304 | fn type_name() -> TypeName { 305 | TypeName::internal(&format!("({},)", T::type_name().name())) 306 | } 307 | } 308 | 309 | impl Key for (T,) { 310 | fn compare(data1: &[u8], data2: &[u8]) -> Ordering { 311 | T::compare(data1, data2) 312 | } 313 | } 314 | 315 | tuple_impl! { 316 | T0, t0, 0 317 | | T1, t1, 1 318 | } 319 | 320 | tuple_impl! { 321 | T0, t0, 0, 322 | T1, t1, 1 323 | | T2, t2, 2 324 | } 325 | 326 | tuple_impl! { 327 | T0, t0, 0, 328 | T1, t1, 1, 329 | T2, t2, 2 330 | | T3, t3, 3 331 | } 332 | 333 | tuple_impl! { 334 | T0, t0, 0, 335 | T1, t1, 1, 336 | T2, t2, 2, 337 | T3, t3, 3 338 | | T4, t4, 4 339 | } 340 | 341 | tuple_impl! { 342 | T0, t0, 0, 343 | T1, t1, 1, 344 | T2, t2, 2, 345 | T3, t3, 3, 346 | T4, t4, 4 347 | | T5, t5, 5 348 | } 349 | 350 | tuple_impl! { 351 | T0, t0, 0, 352 | T1, t1, 1, 353 | T2, t2, 2, 354 | T3, t3, 3, 355 | T4, t4, 4, 356 | T5, t5, 5 357 | | T6, t6, 6 358 | } 359 | 360 | tuple_impl! { 361 | T0, t0, 0, 362 | T1, t1, 1, 363 | T2, t2, 2, 364 | T3, t3, 3, 365 | T4, t4, 4, 366 | T5, t5, 5, 367 | T6, t6, 6 368 | | T7, t7, 7 369 | } 370 | 371 | tuple_impl! { 372 | T0, t0, 0, 373 | T1, t1, 1, 374 | T2, t2, 2, 375 | T3, t3, 3, 376 | T4, t4, 4, 377 | T5, t5, 5, 378 | T6, t6, 6, 379 | T7, t7, 7 380 | | T8, t8, 8 381 | } 382 | 383 | tuple_impl! { 384 | T0, t0, 0, 385 | T1, t1, 1, 386 | T2, t2, 2, 387 | T3, t3, 3, 388 | T4, t4, 4, 389 | T5, t5, 5, 390 | T6, t6, 6, 391 | T7, t7, 7, 392 | T8, t8, 8 393 | | T9, t9, 9 394 | } 395 | 396 | tuple_impl! { 397 | T0, t0, 0, 398 | T1, t1, 1, 399 | T2, t2, 2, 400 | T3, t3, 3, 401 | T4, t4, 4, 402 | T5, t5, 5, 403 | T6, t6, 6, 404 | T7, t7, 7, 405 | T8, t8, 8, 406 | T9, t9, 9 407 | | T10, t10, 10 408 | } 409 | 410 | tuple_impl! { 411 | T0, t0, 0, 412 | T1, t1, 1, 413 | T2, t2, 2, 414 | T3, t3, 3, 415 | T4, t4, 4, 416 | T5, t5, 5, 417 | T6, t6, 6, 418 | T7, t7, 7, 419 | T8, t8, 8, 420 | T9, t9, 9, 421 | T10, t10, 10 422 | | T11, t11, 11 423 | } 424 | 425 | #[cfg(test)] 426 | mod test { 427 | use crate::types::Value; 428 | 429 | #[test] 430 | fn width() { 431 | assert!(<(&str, u8)>::fixed_width().is_none()); 432 | assert!(<(u16, u8, &str, u128)>::fixed_width().is_none()); 433 | assert_eq!(<(u16,)>::fixed_width().unwrap(), 2); 434 | assert_eq!(<(u16, u8)>::fixed_width().unwrap(), 3); 435 | assert_eq!(<(u16, u8, u128)>::fixed_width().unwrap(), 19); 436 | assert_eq!(<(u16, u8, i8, u128)>::fixed_width().unwrap(), 20); 437 | // Check that length of final field is elided 438 | assert_eq!( 439 | <(u8, &str)>::as_bytes(&(1, "hello")).len(), 440 | "hello".len() + size_of::() 441 | ); 442 | // Check that varint encoding uses only 1 byte for small strings 443 | assert_eq!( 444 | <(&str, u8)>::as_bytes(&("hello", 1)).len(), 445 | "hello".len() + size_of::() + size_of::() 446 | ); 447 | } 448 | } 449 | -------------------------------------------------------------------------------- /crates/redb-derive/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(clippy::all, clippy::pedantic, clippy::disallowed_methods)] 2 | #![allow( 3 | clippy::must_use_candidate, 4 | clippy::redundant_closure_for_method_calls, 5 | clippy::similar_names, 6 | clippy::too_many_lines 7 | )] 8 | 9 | use proc_macro::TokenStream; 10 | use quote::quote; 11 | use syn::{Data, DeriveInput, Fields, GenericParam, Ident, parse_macro_input}; 12 | 13 | #[proc_macro_derive(Key)] 14 | pub fn derive_key(input: TokenStream) -> TokenStream { 15 | let input = parse_macro_input!(input as DeriveInput); 16 | 17 | match generate_key_impl(&input) { 18 | Ok(tokens) => tokens.into(), 19 | Err(err) => err.to_compile_error().into(), 20 | } 21 | } 22 | 23 | fn generate_key_impl(input: &DeriveInput) -> syn::Result { 24 | let Data::Struct(_) = &input.data else { 25 | return Err(syn::Error::new_spanned( 26 | input, 27 | "Key can only be derived for structs", 28 | )); 29 | }; 30 | 31 | let name = &input.ident; 32 | let generics = &input.generics; 33 | let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); 34 | 35 | Ok(quote! { 36 | impl #impl_generics redb::Key for #name #ty_generics #where_clause { 37 | fn compare(data1: &[u8], data2: &[u8]) -> std::cmp::Ordering { 38 | let value1 = #name::from_bytes(data1); 39 | let value2 = #name::from_bytes(data2); 40 | Ord::cmp(&value1, &value2) 41 | } 42 | } 43 | }) 44 | } 45 | 46 | #[proc_macro_derive(Value)] 47 | pub fn derive_value(input: TokenStream) -> TokenStream { 48 | let input = parse_macro_input!(input as DeriveInput); 49 | 50 | match generate_value_impl(&input) { 51 | Ok(tokens) => tokens.into(), 52 | Err(err) => err.to_compile_error().into(), 53 | } 54 | } 55 | 56 | fn generate_value_impl(input: &DeriveInput) -> syn::Result { 57 | let Data::Struct(data_struct) = &input.data else { 58 | return Err(syn::Error::new_spanned( 59 | input, 60 | "Value can only be derived for structs", 61 | )); 62 | }; 63 | 64 | let name = &input.ident; 65 | let generics = &input.generics; 66 | let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); 67 | 68 | let self_type = generate_self_type(name, generics)?; 69 | 70 | let type_name_impl = generate_type_name(name, &data_struct.fields); 71 | let as_bytes_impl = generate_as_bytes(&data_struct.fields); 72 | let from_bytes_impl = generate_from_bytes(name, &data_struct.fields); 73 | let fixed_width_impl = generate_fixed_width(&data_struct.fields); 74 | 75 | Ok(quote! { 76 | impl #impl_generics redb::Value for #name #ty_generics #where_clause { 77 | type SelfType<'a> = #self_type 78 | where 79 | Self: 'a; 80 | type AsBytes<'a> = Vec 81 | where 82 | Self: 'a; 83 | 84 | fn fixed_width() -> Option { 85 | #fixed_width_impl 86 | } 87 | 88 | fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> 89 | where 90 | Self: 'a, 91 | { 92 | #from_bytes_impl 93 | } 94 | 95 | fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> 96 | where 97 | Self: 'b, 98 | { 99 | #as_bytes_impl 100 | } 101 | 102 | fn type_name() -> redb::TypeName { 103 | #type_name_impl 104 | } 105 | } 106 | }) 107 | } 108 | 109 | fn generate_self_type( 110 | name: &syn::Ident, 111 | generics: &syn::Generics, 112 | ) -> syn::Result { 113 | if generics.params.is_empty() { 114 | Ok(quote! { #name }) 115 | } else { 116 | let mut params = vec![]; 117 | for param in &generics.params { 118 | match param { 119 | GenericParam::Lifetime(_) => params.push(quote! { 'a }), 120 | GenericParam::Type(type_param) => { 121 | return Err(syn::Error::new_spanned( 122 | type_param, 123 | "Value derivation is not implemented for structs with type parameters", 124 | )); 125 | } 126 | GenericParam::Const(const_param) => { 127 | return Err(syn::Error::new_spanned( 128 | const_param, 129 | "Value derivation is not implemented for structs with const parameters", 130 | )); 131 | } 132 | } 133 | } 134 | 135 | Ok(quote! { #name<#(#params),*> }) 136 | } 137 | } 138 | 139 | fn generate_type_name(struct_name: &Ident, fields: &Fields) -> proc_macro2::TokenStream { 140 | match fields { 141 | Fields::Named(fields_named) => { 142 | let field_strings: Vec<_> = fields_named 143 | .named 144 | .iter() 145 | .map(|field| { 146 | let field_name = field.ident.as_ref().unwrap(); 147 | let field_type = &field.ty; 148 | quote! { 149 | format!("{}: {}", stringify!(#field_name), <#field_type>::type_name().name()) 150 | } 151 | }) 152 | .collect(); 153 | 154 | if field_strings.is_empty() { 155 | quote! { 156 | redb::TypeName::new(&format!("{} {{}}", 157 | stringify!(#struct_name), 158 | )) 159 | } 160 | } else { 161 | quote! { 162 | redb::TypeName::new(&format!("{} {{{}}}", 163 | stringify!(#struct_name), 164 | [#(#field_strings),*].join(", ") 165 | )) 166 | } 167 | } 168 | } 169 | Fields::Unnamed(fields_unnamed) => { 170 | let field_strings: Vec<_> = fields_unnamed 171 | .unnamed 172 | .iter() 173 | .map(|field| { 174 | let field_type = &field.ty; 175 | quote! { 176 | <#field_type>::type_name().name() 177 | } 178 | }) 179 | .collect(); 180 | 181 | if field_strings.is_empty() { 182 | quote! { 183 | redb::TypeName::new(&format!("{}()", 184 | stringify!(#struct_name), 185 | )) 186 | } 187 | } else { 188 | quote! { 189 | redb::TypeName::new(&format!("{}({})", 190 | stringify!(#struct_name), 191 | [#(#field_strings),*].join(", ") 192 | )) 193 | } 194 | } 195 | } 196 | Fields::Unit => { 197 | quote! { 198 | redb::TypeName::new(stringify!(#struct_name)) 199 | } 200 | } 201 | } 202 | } 203 | 204 | fn get_field_types(fields: &Fields) -> Vec { 205 | match fields { 206 | Fields::Named(fields_named) => fields_named 207 | .named 208 | .iter() 209 | .map(|field| &field.ty) 210 | .cloned() 211 | .collect(), 212 | Fields::Unnamed(fields_unnamed) => fields_unnamed 213 | .unnamed 214 | .iter() 215 | .map(|field| &field.ty) 216 | .cloned() 217 | .collect(), 218 | Fields::Unit => vec![], 219 | } 220 | } 221 | 222 | fn generate_fixed_width(fields: &Fields) -> proc_macro2::TokenStream { 223 | let field_types = get_field_types(fields); 224 | quote! { 225 | let mut total_width = 0usize; 226 | #( 227 | total_width += <#field_types>::fixed_width()?; 228 | )* 229 | Some(total_width) 230 | } 231 | } 232 | 233 | fn generate_as_bytes(fields: &Fields) -> proc_macro2::TokenStream { 234 | let field_types = get_field_types(fields); 235 | let field_accessors = match fields { 236 | Fields::Named(fields_named) => fields_named 237 | .named 238 | .iter() 239 | .map(|field| { 240 | let name = &field.ident; 241 | quote! { #name } 242 | }) 243 | .collect(), 244 | Fields::Unnamed(_) => (0..field_types.len()) 245 | .map(|i| { 246 | let index = syn::Index::from(i); 247 | quote! { #index } 248 | }) 249 | .collect(), 250 | Fields::Unit => Vec::new(), 251 | }; 252 | 253 | let num_fields = field_types.len(); 254 | 255 | if num_fields == 0 { 256 | quote! { Vec::new() } 257 | } else if num_fields == 1 { 258 | let field_accessor = &field_accessors[0]; 259 | let field_type = &field_types[0]; 260 | quote! { 261 | { 262 | let field_bytes = <#field_type>::as_bytes(&value.#field_accessor); 263 | field_bytes.as_ref().to_vec() 264 | } 265 | } 266 | } else { 267 | let field_types_except_last = &field_types[..num_fields - 1]; 268 | let field_accessors_except_last = &field_accessors[..num_fields - 1]; 269 | 270 | quote! { 271 | { 272 | let mut result = Vec::new(); 273 | 274 | #( 275 | if <#field_types_except_last>::fixed_width().is_none() { 276 | let field_bytes = <#field_types_except_last>::as_bytes(&value.#field_accessors_except_last); 277 | let bytes: &[u8] = field_bytes.as_ref(); 278 | let len = bytes.len(); 279 | if len < 254 { 280 | result.push(len.try_into().unwrap()); 281 | } else if let Ok(u16_len) = u16::try_from(len) { 282 | result.push(254u8); 283 | result.extend_from_slice(&u16_len.to_le_bytes()); 284 | } else { 285 | let u32_len: u32 = len.try_into().unwrap(); 286 | result.push(255u8); 287 | result.extend_from_slice(&u32_len.to_le_bytes()); 288 | } 289 | } 290 | )* 291 | 292 | #( 293 | { 294 | let field_bytes = <#field_types>::as_bytes(&value.#field_accessors); 295 | result.extend_from_slice(field_bytes.as_ref()); 296 | } 297 | )* 298 | 299 | result 300 | } 301 | } 302 | } 303 | } 304 | 305 | fn generate_from_bytes(name: &Ident, fields: &Fields) -> proc_macro2::TokenStream { 306 | let field_types = get_field_types(fields); 307 | let field_vars: Vec<_> = (0..field_types.len()) 308 | .map(|i| quote::format_ident!("field_{}", i)) 309 | .collect(); 310 | let num_fields = field_types.len(); 311 | 312 | let body = if num_fields == 0 { 313 | quote! {} 314 | } else if num_fields == 1 { 315 | let field_var = &field_vars[0]; 316 | let field_type = &field_types[0]; 317 | quote! { 318 | let #field_var = <#field_type>::from_bytes(data); 319 | } 320 | } else { 321 | let field_types_except_last = &field_types[..num_fields - 1]; 322 | let field_vars_except_last = &field_vars[..num_fields - 1]; 323 | let last_field_var = field_vars.last(); 324 | let last_field_type = field_types.last(); 325 | 326 | quote! { 327 | let mut offset = 0usize; 328 | let mut var_lengths = Vec::new(); 329 | 330 | #( 331 | if <#field_types_except_last>::fixed_width().is_none() { 332 | let (len, bytes_read) = match data[offset] { 333 | 0u8..=253u8 => (data[offset] as usize, 1usize), 334 | 254u8 => ( 335 | u16::from_le_bytes(data[offset + 1..offset + 3].try_into().unwrap()) as usize, 336 | 3usize, 337 | ), 338 | 255u8 => ( 339 | u32::from_le_bytes(data[offset + 1..offset + 5].try_into().unwrap()) as usize, 340 | 5usize, 341 | ), 342 | }; 343 | var_lengths.push(len); 344 | offset += bytes_read; 345 | } 346 | )* 347 | 348 | let mut var_index = 0; 349 | #( 350 | let #field_vars_except_last = if let Some(fixed_width) = <#field_types_except_last>::fixed_width() { 351 | let field_data = &data[offset..offset + fixed_width]; 352 | offset += fixed_width; 353 | <#field_types_except_last>::from_bytes(field_data) 354 | } else { 355 | let len = var_lengths[var_index]; 356 | let field_data = &data[offset..offset + len]; 357 | offset += len; 358 | var_index += 1; 359 | <#field_types_except_last>::from_bytes(field_data) 360 | }; 361 | )* 362 | 363 | let #last_field_var = if let Some(fixed_width) = <#last_field_type>::fixed_width() { 364 | let field_data = &data[offset..offset + fixed_width]; 365 | <#last_field_type>::from_bytes(field_data) 366 | } else { 367 | <#last_field_type>::from_bytes(&data[offset..]) 368 | }; 369 | } 370 | }; 371 | match fields { 372 | Fields::Named(fields_named) => { 373 | let field_names: Vec<_> = fields_named 374 | .named 375 | .iter() 376 | .map(|field| &field.ident) 377 | .collect(); 378 | 379 | quote! { 380 | { 381 | #body 382 | #name { 383 | #(#field_names: #field_vars),* 384 | } 385 | } 386 | } 387 | } 388 | Fields::Unnamed(_) => { 389 | quote! { 390 | { 391 | #body 392 | #name(#(#field_vars),*) 393 | } 394 | } 395 | } 396 | Fields::Unit => { 397 | quote! { #name } 398 | } 399 | } 400 | } 401 | --------------------------------------------------------------------------------