├── src ├── wal │ ├── iter.rs │ ├── query.rs │ └── pointer.rs ├── wal.rs ├── memtable │ ├── linked.rs │ ├── arena.rs │ ├── linked │ │ ├── table.rs │ │ └── multiple_version.rs │ ├── alternative │ │ ├── table.rs │ │ └── multiple_version.rs │ ├── arena │ │ ├── table.rs │ │ └── multiple_version.rs │ └── alternative.rs ├── utils.rs ├── swmr │ ├── reader.rs │ ├── wal.rs │ ├── tests │ │ ├── constructor.rs │ │ ├── multiple_version_constructor.rs │ │ └── get.rs │ ├── writer.rs │ └── tests.rs ├── lib.rs ├── batch.rs ├── types.rs ├── types │ └── base.rs ├── swmr.rs ├── options.rs ├── memtable.rs ├── builder.rs └── error.rs ├── benches └── foo.rs ├── .github ├── FUNDING.yml ├── dependabot.yml └── workflows │ └── loc.yml ├── ci ├── tsan ├── miri_sb.sh ├── sanitizer.sh └── miri_tb.sh ├── rustfmt.toml ├── .gitignore ├── CHANGELOG.md ├── examples ├── generic_not_sized.rs ├── multiple_version.rs └── zero_cost.rs ├── .codecov.yml ├── LICENSE-MIT ├── Cargo.toml ├── README.md ├── README-zh_CN.md └── LICENSE-APACHE /src/wal/iter.rs: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /benches/foo.rs: -------------------------------------------------------------------------------- 1 | fn main() {} 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: al8n 2 | patreon: al8n 3 | ko_fi: al8n9434 4 | -------------------------------------------------------------------------------- /ci/tsan: -------------------------------------------------------------------------------- 1 | # TSAN suppressions file for orderwal 2 | 3 | # The epoch-based GC uses fences. 4 | race:crossbeam_epoch 5 | -------------------------------------------------------------------------------- /src/wal.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod base; 2 | pub(crate) mod iter; 3 | pub(crate) mod multiple_version; 4 | 5 | mod query; 6 | pub(crate) use query::*; 7 | 8 | mod pointer; 9 | pub use pointer::*; 10 | -------------------------------------------------------------------------------- /src/memtable/linked.rs: -------------------------------------------------------------------------------- 1 | /// The multiple version memtable implementation. 2 | pub mod multiple_version; 3 | /// The memtable implementation. 4 | pub mod table; 5 | 6 | pub use multiple_version::MultipleVersionTable; 7 | pub use table::Table; 8 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | max_width = 100 2 | hard_tabs = false 3 | tab_spaces = 2 4 | newline_style = "Auto" 5 | use_small_heuristics = "Default" 6 | reorder_imports = true 7 | reorder_modules = true 8 | remove_nested_parens = true 9 | merge_derives = true 10 | use_try_shorthand = true 11 | use_field_init_shorthand = true 12 | force_explicit_abi = true 13 | 14 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "cargo" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | open-pull-requests-limit: 50 8 | 9 | - package-ecosystem: "github-actions" 10 | directory: "/" 11 | schedule: 12 | # Check for updates to GitHub Actions every weekday 13 | interval: "daily" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **.idea/ 2 | 3 | # Generated by Cargo 4 | # will have compiled files and executables 5 | **target/ 6 | 7 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 8 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 9 | **Cargo.lock 10 | 11 | # These are backup files generated by rustfmt 12 | **/*.rs.bk 13 | 14 | 15 | # Added by cargo 16 | 17 | /target 18 | Cargo.lock 19 | -------------------------------------------------------------------------------- /src/utils.rs: -------------------------------------------------------------------------------- 1 | pub use dbutils::leb128; 2 | 3 | /// Merge two `u32` into a `u64`. 4 | /// 5 | /// - high 32 bits: `a` 6 | /// - low 32 bits: `b` 7 | #[inline] 8 | pub(crate) const fn merge_lengths(a: u32, b: u32) -> u64 { 9 | (a as u64) << 32 | b as u64 10 | } 11 | 12 | /// Split a `u64` into two `u32`. 13 | /// 14 | /// - high 32 bits: the first `u32` 15 | /// - low 32 bits: the second `u32` 16 | #[inline] 17 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 18 | pub(crate) const fn split_lengths(len: u64) -> (u32, u32) { 19 | ((len >> 32) as u32, len as u32) 20 | } 21 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Rleases 2 | 3 | ## 0.5.0 (Oct 27th, 2024) 4 | 5 | - Refactor the project to make all of the WALs based on the generic implementation. 6 | - Support different memtables based on [`crossbeam-skiplist`](https://github.com/crossbeam-rs/crossbeam) or [`skl`](https://github.com/al8n/skl) 7 | - More user-friendly APIs 8 | - Support `no-std` environment 9 | 10 | ## 0.4.0 (Sep 30th, 2024) 11 | 12 | FEATURES 13 | 14 | - Support `K: ?Sized` and `V: ?Sized` for `OrderWal`. 15 | - Use `flush_header_and_range` instead of `flush_range` when insertion. 16 | 17 | ## 0.1.0 (Sep 14th, 2024) 18 | 19 | - Publish version `0.1.0` 20 | -------------------------------------------------------------------------------- /ci/miri_sb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # Check if TARGET and CONFIG_FLAGS are provided, otherwise panic 5 | if [ -z "$1" ]; then 6 | echo "Error: TARGET is not provided" 7 | exit 1 8 | fi 9 | 10 | if [ -z "$2" ]; then 11 | echo "Error: CONFIG_FLAGS are not provided" 12 | exit 1 13 | fi 14 | 15 | TARGET=$1 16 | CONFIG_FLAGS=$2 17 | 18 | rustup toolchain install nightly --component miri 19 | rustup override set nightly 20 | cargo miri setup 21 | 22 | export MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-symbolic-alignment-check" 23 | export RUSTFLAGS="--cfg test_$CONFIG_FLAGS" 24 | 25 | cargo miri test --tests --target $TARGET --lib 26 | -------------------------------------------------------------------------------- /examples/generic_not_sized.rs: -------------------------------------------------------------------------------- 1 | use orderwal::{ 2 | base::{OrderWal, Reader, Writer}, 3 | Builder, 4 | }; 5 | 6 | fn main() { 7 | let dir = tempfile::tempdir().unwrap(); 8 | let path = dir.path().join("not_sized.wal"); 9 | 10 | let mut wal = unsafe { 11 | Builder::new() 12 | .with_capacity(1024 * 1024) 13 | .with_create_new(true) 14 | .with_read(true) 15 | .with_write(true) 16 | .map_mut::, _>(&path) 17 | .unwrap() 18 | }; 19 | 20 | wal.insert("a", b"a1".as_slice()).unwrap(); 21 | wal.insert("c", b"c1".as_slice()).unwrap(); 22 | 23 | let a = wal.get("a").unwrap(); 24 | let c = wal.get("c").unwrap(); 25 | 26 | assert_eq!(a.value(), b"a1"); 27 | assert_eq!(c.value(), b"c1"); 28 | } 29 | -------------------------------------------------------------------------------- /.codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: false 3 | 4 | ignore: 5 | - "**/integration/" 6 | - "**/examples/" 7 | - "**/benches/" 8 | - "src/error.rs" 9 | - "src/swmr/tests.rs" 10 | - "src/swmr/tests/" 11 | 12 | coverage: 13 | status: 14 | project: # Overall project status 15 | default: 16 | target: auto 17 | if_not_found: success 18 | only_pulls: false 19 | patch: # Status for the patch in pull requests 20 | default: 21 | target: auto 22 | if_not_found: success 23 | only_pulls: true 24 | changes: false # Whether to comment on the coverage changes in pull requests 25 | 26 | comment: 27 | layout: "header, diff, files, footer" 28 | behavior: default 29 | require_changes: false 30 | -------------------------------------------------------------------------------- /ci/sanitizer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | export ASAN_OPTIONS="detect_odr_violation=0 detect_leaks=0" 6 | 7 | # Run address sanitizer 8 | RUSTFLAGS="-Z sanitizer=address --cfg all_orderwal_tests" \ 9 | cargo test -Z build-std --all --release --tests --target x86_64-unknown-linux-gnu --all-features --exclude benchmarks -- --test-threads=1 10 | 11 | # Run memory sanitizer 12 | RUSTFLAGS="-Z sanitizer=memory --cfg all_orderwal_tests" \ 13 | cargo test -Z build-std --all --release --tests --target x86_64-unknown-linux-gnu --all-features --exclude benchmarks -- --test-threads=1 14 | 15 | # Run thread sanitizer 16 | cargo clean 17 | TSAN_OPTIONS="suppressions=$(pwd)/ci/tsan" \ 18 | RUSTFLAGS="${RUSTFLAGS:-} -Z sanitizer=thread --cfg all_orderwal_tests" \ 19 | cargo test -Z build-std --all --release --target x86_64-unknown-linux-gnu --all-features --tests --exclude benchmarks -- --test-threads=1 -------------------------------------------------------------------------------- /ci/miri_tb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euxo pipefail 3 | IFS=$'\n\t' 4 | 5 | # We need 'ts' for the per-line timing 6 | sudo apt-get -y install moreutils 7 | echo 8 | 9 | # Check if TARGET and CONFIG_FLAGS are provided, otherwise panic 10 | if [ -z "$1" ]; then 11 | echo "Error: TARGET is not provided" 12 | exit 1 13 | fi 14 | 15 | if [ -z "$2" ]; then 16 | echo "Error: CONFIG_FLAGS are not provided" 17 | exit 1 18 | fi 19 | 20 | TARGET=$1 21 | CONFIG_FLAGS=$2 22 | 23 | rustup toolchain install nightly --component miri 24 | rustup override set nightly 25 | cargo miri setup 26 | 27 | # Zmiri-ignore-leaks needed because of https://github.com/crossbeam-rs/crossbeam/issues/579 28 | export MIRIFLAGS="-Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-tree-borrows -Zmiri-ignore-leaks" 29 | export RUSTFLAGS="--cfg test_$CONFIG_FLAGS" 30 | 31 | cargo miri test --tests --target $TARGET --lib 2>&1 | ts -i '%.s ' 32 | 33 | -------------------------------------------------------------------------------- /examples/multiple_version.rs: -------------------------------------------------------------------------------- 1 | use orderwal::{ 2 | multiple_version::{OrderWal, Reader, Writer}, 3 | Builder, 4 | }; 5 | 6 | fn main() { 7 | let dir = tempfile::tempdir().unwrap(); 8 | let path = dir.path().join("not_sized.wal"); 9 | 10 | let mut wal = unsafe { 11 | Builder::new() 12 | .with_capacity(1024 * 1024) 13 | .with_create_new(true) 14 | .with_read(true) 15 | .with_write(true) 16 | .map_mut::, _>(&path) 17 | .unwrap() 18 | }; 19 | 20 | wal.insert(1, "a", b"a1".as_slice()).unwrap(); 21 | wal.insert(3, "a", b"a3".as_slice()).unwrap(); 22 | wal.insert(1, "c", b"c1".as_slice()).unwrap(); 23 | wal.insert(3, "c", b"c3".as_slice()).unwrap(); 24 | 25 | let a = wal.get(2, "a").unwrap(); 26 | let c = wal.get(2, "c").unwrap(); 27 | 28 | assert_eq!(a.value(), b"a1"); 29 | assert_eq!(c.value(), b"c1"); 30 | 31 | let a = wal.get(3, "a").unwrap(); 32 | let c = wal.get(3, "c").unwrap(); 33 | 34 | assert_eq!(a.value(), b"a3"); 35 | assert_eq!(c.value(), b"c3"); 36 | } 37 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 The Rust Project Developers 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /.github/workflows/loc.yml: -------------------------------------------------------------------------------- 1 | name: loc 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths-ignore: 8 | - 'README.md' 9 | - 'COPYRIGHT' 10 | - 'LICENSE*' 11 | - '**.md' 12 | - '**.txt' 13 | - 'art' 14 | pull_request: 15 | paths-ignore: 16 | - 'README.md' 17 | - 'COPYRIGHT' 18 | - 'LICENSE*' 19 | - '**.md' 20 | - '**.txt' 21 | - 'art' 22 | workflow_dispatch: 23 | 24 | jobs: 25 | loc: 26 | runs-on: ubuntu-latest 27 | steps: 28 | - uses: actions/checkout@v4 29 | 30 | - name: Install Rust 31 | run: | 32 | rustup update stable && rustup default stable 33 | rustup component add clippy 34 | rustup component add rustfmt 35 | 36 | - name: Install tokeit 37 | run: | 38 | cargo install tokeit --force 39 | 40 | - name: Count total lines of code 41 | run: | 42 | tokeit 43 | - name: Upload total loc to GitHub Gist 44 | uses: actions/github-script@v7 45 | with: 46 | github-token: ${{ secrets.GIST_PAT }} 47 | script: | 48 | const fs = require('fs'); 49 | const output = fs.readFileSync('tokeit.json', 'utf8'); 50 | const gistId = '327b2a8aef9003246e45c6e47fe63937'; 51 | await github.rest.gists.update({ 52 | gist_id: gistId, 53 | files: { 54 | "orderwal": { 55 | content: output 56 | } 57 | } 58 | }); 59 | -------------------------------------------------------------------------------- /src/swmr/reader.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use rarena_allocator::sync::Arena; 4 | 5 | use crate::{ 6 | memtable::BaseTable, 7 | sealed::{Constructable, Immutable}, 8 | swmr::wal::OrderCore, 9 | }; 10 | 11 | use super::writer::OrderWal; 12 | 13 | /// An [`OrderWal`] reader. 14 | pub struct OrderWalReader(OrderWal); 15 | 16 | impl core::fmt::Debug for OrderWalReader 17 | where 18 | K: ?Sized, 19 | V: ?Sized, 20 | { 21 | #[inline] 22 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 23 | f.debug_tuple("OrderWalReader").field(&self.0.core).finish() 24 | } 25 | } 26 | 27 | impl Immutable for OrderWalReader {} 28 | 29 | impl OrderWalReader 30 | where 31 | K: ?Sized, 32 | V: ?Sized, 33 | { 34 | /// Creates a new read-only WAL reader. 35 | #[inline] 36 | pub(super) fn new(wal: Arc>) -> Self { 37 | Self(OrderWal::construct(wal)) 38 | } 39 | } 40 | 41 | impl Constructable for OrderWalReader 42 | where 43 | K: ?Sized + 'static, 44 | V: ?Sized + 'static, 45 | S: 'static, 46 | M: BaseTable + 'static, 47 | { 48 | type Allocator = Arena; 49 | type Wal = OrderCore; 50 | type Memtable = M; 51 | type Checksumer = S; 52 | type Reader = OrderWalReader; 53 | 54 | #[inline] 55 | fn as_wal(&self) -> &Self::Wal { 56 | self.0.as_wal() 57 | } 58 | 59 | #[inline] 60 | fn from_core(core: Self::Wal) -> Self { 61 | Self(OrderWal { 62 | core: Arc::new(core), 63 | }) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/swmr/wal.rs: -------------------------------------------------------------------------------- 1 | use core::marker::PhantomData; 2 | 3 | use rarena_allocator::sync::Arena; 4 | 5 | use crate::{memtable::BaseTable, sealed::Wal, Options}; 6 | 7 | pub struct OrderCore 8 | where 9 | K: ?Sized, 10 | V: ?Sized, 11 | { 12 | pub(super) arena: Arena, 13 | pub(super) map: M, 14 | pub(super) opts: Options, 15 | pub(super) cks: S, 16 | pub(super) _m: PhantomData<(fn() -> K, fn() -> V)>, 17 | } 18 | 19 | impl core::fmt::Debug for OrderCore 20 | where 21 | K: ?Sized, 22 | V: ?Sized, 23 | { 24 | #[inline] 25 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 26 | f.debug_struct("OrderCore") 27 | .field("arena", &self.arena) 28 | .field("options", &self.opts) 29 | .finish() 30 | } 31 | } 32 | 33 | impl Wal for OrderCore 34 | where 35 | K: ?Sized, 36 | V: ?Sized, 37 | M: BaseTable, 38 | { 39 | type Allocator = Arena; 40 | type Memtable = M; 41 | 42 | #[inline] 43 | fn memtable(&self) -> &Self::Memtable { 44 | &self.map 45 | } 46 | 47 | #[inline] 48 | fn memtable_mut(&mut self) -> &mut Self::Memtable { 49 | &mut self.map 50 | } 51 | 52 | #[inline] 53 | fn construct(arena: Self::Allocator, set: Self::Memtable, opts: Options, checksumer: S) -> Self { 54 | Self { 55 | arena, 56 | map: set, 57 | opts, 58 | cks: checksumer, 59 | _m: PhantomData, 60 | } 61 | } 62 | 63 | #[inline] 64 | fn options(&self) -> &Options { 65 | &self.opts 66 | } 67 | 68 | #[inline] 69 | fn allocator(&self) -> &Self::Allocator { 70 | &self.arena 71 | } 72 | 73 | #[inline] 74 | fn hasher(&self) -> &S { 75 | &self.cks 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! An ordered Write-Ahead Log implementation for Rust. 2 | #![doc = include_str!("../README.md")] 3 | #![cfg_attr(not(feature = "std"), no_std)] 4 | #![cfg_attr(docsrs, feature(doc_cfg))] 5 | #![cfg_attr(docsrs, allow(unused_attributes))] 6 | #![deny(missing_docs)] 7 | #![allow(clippy::type_complexity)] 8 | 9 | use core::mem; 10 | 11 | pub use among; 12 | 13 | #[cfg(feature = "std")] 14 | extern crate std; 15 | 16 | #[cfg(not(feature = "std"))] 17 | extern crate alloc as std; 18 | 19 | pub use dbutils::{ 20 | checksum::{self, Crc32}, 21 | equivalent::{Comparable, ComparableRangeBounds, Equivalent}, 22 | }; 23 | 24 | #[cfg(feature = "xxhash3")] 25 | #[cfg_attr(docsrs, doc(cfg(feature = "xxhash3")))] 26 | pub use dbutils::checksum::XxHash3; 27 | 28 | #[cfg(feature = "xxhash64")] 29 | #[cfg_attr(docsrs, doc(cfg(feature = "xxhash64")))] 30 | pub use dbutils::checksum::XxHash64; 31 | 32 | const RECORD_FLAG_SIZE: usize = mem::size_of::(); 33 | const CHECKSUM_SIZE: usize = mem::size_of::(); 34 | const CURRENT_VERSION: u16 = 0; 35 | const MAGIC_TEXT: [u8; 5] = *b"order"; 36 | const MAGIC_TEXT_SIZE: usize = MAGIC_TEXT.len(); 37 | const WAL_KIND_SIZE: usize = mem::size_of::(); 38 | const MAGIC_VERSION_SIZE: usize = mem::size_of::(); 39 | const HEADER_SIZE: usize = MAGIC_TEXT_SIZE + WAL_KIND_SIZE + MAGIC_VERSION_SIZE; 40 | /// The mvcc version size. 41 | const VERSION_SIZE: usize = mem::size_of::(); 42 | 43 | /// Error types. 44 | pub mod error; 45 | 46 | mod builder; 47 | pub use builder::Builder; 48 | 49 | /// Types 50 | pub mod types; 51 | 52 | mod options; 53 | pub use options::Options; 54 | pub use skl::KeySize; 55 | 56 | /// Batch insertions related traits and structs. 57 | pub mod batch; 58 | 59 | /// A single writer multiple readers ordered write-ahead Log implementation. 60 | mod swmr; 61 | mod wal; 62 | pub use swmr::*; 63 | 64 | /// The memory table implementation. 65 | pub mod memtable; 66 | 67 | mod sealed; 68 | pub use sealed::Immutable; 69 | 70 | /// The utilities functions. 71 | pub mod utils; 72 | 73 | bitflags::bitflags! { 74 | /// The flags for each atomic write. 75 | struct Flags: u8 { 76 | /// First bit: 1 indicates committed, 0 indicates uncommitted 77 | const COMMITTED = 0b00000001; 78 | /// Second bit: 1 indicates batching, 0 indicates single entry 79 | const BATCHING = 0b00000010; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/wal/query.rs: -------------------------------------------------------------------------------- 1 | use core::{ 2 | cmp, 3 | marker::PhantomData, 4 | ops::{Bound, RangeBounds}, 5 | }; 6 | 7 | use dbutils::{ 8 | equivalent::{Comparable, Equivalent}, 9 | types::{KeyRef, Type, TypeRef}, 10 | }; 11 | use ref_cast::RefCast; 12 | 13 | use super::KeyPointer; 14 | 15 | #[derive(ref_cast::RefCast)] 16 | #[repr(transparent)] 17 | pub struct Slice<'a, K: ?Sized> { 18 | _k: PhantomData<&'a K>, 19 | data: [u8], 20 | } 21 | 22 | impl<'a, K> Equivalent> for Slice<'a, K> 23 | where 24 | K: Type + ?Sized, 25 | K::Ref<'a>: KeyRef<'a, K>, 26 | { 27 | fn equivalent(&self, key: &KeyPointer) -> bool { 28 | self.data.eq(key.as_slice()) 29 | } 30 | } 31 | 32 | impl<'a, K> Comparable> for Slice<'a, K> 33 | where 34 | K: Type + ?Sized, 35 | K::Ref<'a>: KeyRef<'a, K>, 36 | { 37 | fn compare(&self, p: &KeyPointer) -> cmp::Ordering { 38 | unsafe { as KeyRef>::compare_binary(&self.data, p.as_slice()) } 39 | } 40 | } 41 | 42 | pub struct QueryRange<'a, K: ?Sized, Q: ?Sized, R> 43 | where 44 | R: RangeBounds, 45 | { 46 | r: R, 47 | _q: PhantomData<(&'a Q, &'a K)>, 48 | } 49 | 50 | impl QueryRange<'_, K, Q, R> 51 | where 52 | R: RangeBounds, 53 | { 54 | #[inline] 55 | pub(super) const fn new(r: R) -> Self { 56 | Self { r, _q: PhantomData } 57 | } 58 | } 59 | 60 | impl<'a, K: ?Sized, Q: ?Sized, R> RangeBounds> for QueryRange<'a, K, Q, R> 61 | where 62 | R: RangeBounds, 63 | { 64 | #[inline] 65 | fn start_bound(&self) -> Bound<&Query<'a, K, Q>> { 66 | self.r.start_bound().map(RefCast::ref_cast) 67 | } 68 | 69 | fn end_bound(&self) -> Bound<&Query<'a, K, Q>> { 70 | self.r.end_bound().map(RefCast::ref_cast) 71 | } 72 | } 73 | 74 | #[derive(ref_cast::RefCast)] 75 | #[repr(transparent)] 76 | pub struct Query<'a, K, Q> 77 | where 78 | K: ?Sized, 79 | Q: ?Sized, 80 | { 81 | _k: PhantomData<&'a K>, 82 | key: Q, 83 | } 84 | 85 | impl<'a, K, Q> Equivalent> for Query<'a, K, Q> 86 | where 87 | K: Type + ?Sized, 88 | Q: ?Sized + Equivalent>, 89 | { 90 | #[inline] 91 | fn equivalent(&self, p: &KeyPointer) -> bool { 92 | let kr = unsafe { as TypeRef<'_>>::from_slice(p.as_slice()) }; 93 | Equivalent::equivalent(&self.key, &kr) 94 | } 95 | } 96 | 97 | impl<'a, K, Q> Comparable> for Query<'a, K, Q> 98 | where 99 | K: Type + ?Sized, 100 | Q: ?Sized + Comparable>, 101 | { 102 | #[inline] 103 | fn compare(&self, p: &KeyPointer) -> cmp::Ordering { 104 | let kr = unsafe { as TypeRef<'_>>::from_slice(p.as_slice()) }; 105 | Comparable::compare(&self.key, &kr) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/memtable/arena.rs: -------------------------------------------------------------------------------- 1 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 2 | macro_rules! memmap_or_not { 3 | ($opts:ident($arena_opts:ident)) => {{ 4 | if $opts.map_anon() { 5 | $arena_opts 6 | .map_anon::, ValuePointer, _>() 7 | .map_err(skl::error::Error::IO) 8 | } else { 9 | $arena_opts.alloc::, ValuePointer, _>() 10 | } 11 | .map(|map| Self { map }) 12 | }}; 13 | } 14 | 15 | #[cfg(not(all(feature = "memmap", not(target_family = "wasm"))))] 16 | macro_rules! memmap_or_not { 17 | ($opts:ident($arena_opts:ident)) => {{ 18 | $arena_opts 19 | .alloc::, ValuePointer, _>() 20 | .map(|map| Self { map }) 21 | }}; 22 | } 23 | 24 | pub use skl::Height; 25 | 26 | /// Options to configure the [`Table`] or [`MultipleVersionTable`]. 27 | #[derive(Debug, Copy, Clone)] 28 | pub struct TableOptions { 29 | capacity: u32, 30 | map_anon: bool, 31 | max_height: Height, 32 | } 33 | 34 | impl Default for TableOptions { 35 | #[inline] 36 | fn default() -> Self { 37 | Self::new() 38 | } 39 | } 40 | 41 | impl TableOptions { 42 | /// Creates a new instance of `TableOptions` with the default options. 43 | #[inline] 44 | pub const fn new() -> Self { 45 | Self { 46 | capacity: 8192, 47 | map_anon: false, 48 | max_height: Height::new(), 49 | } 50 | } 51 | 52 | /// Sets the capacity of the table. 53 | /// 54 | /// Default is `8KB`. 55 | #[inline] 56 | pub const fn with_capacity(mut self, capacity: u32) -> Self { 57 | self.capacity = capacity; 58 | self 59 | } 60 | 61 | /// Sets the table to use anonymous memory. 62 | #[inline] 63 | pub const fn with_map_anon(mut self, map_anon: bool) -> Self { 64 | self.map_anon = map_anon; 65 | self 66 | } 67 | 68 | /// Sets the maximum height of the table. 69 | /// 70 | /// Default is `20`. 71 | #[inline] 72 | pub const fn with_max_height(mut self, max_height: Height) -> Self { 73 | self.max_height = max_height; 74 | self 75 | } 76 | 77 | /// Returns the capacity of the table. 78 | #[inline] 79 | pub const fn capacity(&self) -> u32 { 80 | self.capacity 81 | } 82 | 83 | /// Returns `true` if the table is using anonymous memory. 84 | #[inline] 85 | pub const fn map_anon(&self) -> bool { 86 | self.map_anon 87 | } 88 | 89 | /// Returns the maximum height of the table. 90 | #[inline] 91 | pub const fn max_height(&self) -> Height { 92 | self.max_height 93 | } 94 | } 95 | 96 | /// The multiple version memtable implementation. 97 | pub mod multiple_version; 98 | /// The memtable implementation. 99 | pub mod table; 100 | 101 | pub use multiple_version::MultipleVersionTable; 102 | pub use table::Table; 103 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "orderwal" 3 | version = "0.5.1" 4 | edition = "2021" 5 | repository = "https://github.com/al8n/orderwal" 6 | homepage = "https://github.com/al8n/orderwal" 7 | documentation = "https://docs.rs/orderwal" 8 | description = "A generic-purpose, atomic, ordered, zero-copy read, zero-cost (in-place) write, Write-Ahead Log implementation for Rust." 9 | license = "MIT OR Apache-2.0" 10 | rust-version = "1.81.0" 11 | categories = ["filesystem", "database-implementations", "development-tools", "data-structures", "no-std"] 12 | keywords = ["wal", "write-ahead-log", "append-only", "append-only-log", "bitcask"] 13 | 14 | [[bench]] 15 | path = "benches/foo.rs" 16 | name = "foo" 17 | harness = false 18 | 19 | [features] 20 | default = ["memmap"] 21 | alloc = ["rarena-allocator/alloc", "skl/alloc", "dbutils/alloc"] 22 | std = ["rarena-allocator/default", "crossbeam-skiplist/default", "crossbeam-skiplist-mvcc/default", "bitflags/std", "dbutils/default", "among/default", "skl/std"] 23 | memmap = ["std", "rarena-allocator/memmap", "skl/memmap"] 24 | 25 | xxhash3 = ["dbutils/xxhash3", "std"] 26 | xxhash64 = ["dbutils/xxhash64", "std"] 27 | 28 | tracing = ["dep:tracing", "dbutils/tracing"] 29 | 30 | [dependencies] 31 | among = { version = "0.1", default-features = false, features = ["either"] } 32 | bitflags = { version = "2", default-features = false } 33 | dbutils = { version = "0.9", default-features = false, features = ["crc32fast"] } 34 | derive-where = "1" 35 | ref-cast = "1" 36 | rarena-allocator = { version = "0.4", default-features = false } 37 | crossbeam-skiplist = { version = "0.1", default-features = false, package = "crossbeam-skiplist-pr1132", optional = true } 38 | crossbeam-skiplist-mvcc = { version = "0.2", optional = true } 39 | skl = { version = "0.19", default-features = false, features = ["alloc"] } 40 | paste = "1" 41 | 42 | tracing = { version = "0.1", default-features = false, optional = true } 43 | 44 | [dev-dependencies] 45 | criterion = "0.5" 46 | names = "0.14" 47 | rand = "0.8" 48 | tempfile = "3" 49 | 50 | [profile.bench] 51 | opt-level = 3 52 | debug = false 53 | codegen-units = 1 54 | lto = 'thin' 55 | incremental = false 56 | debug-assertions = false 57 | overflow-checks = false 58 | rpath = false 59 | 60 | [package.metadata.docs.rs] 61 | all-features = true 62 | rustdoc-args = ["--cfg", "docsrs"] 63 | 64 | [lints.rust] 65 | rust_2018_idioms = "warn" 66 | single_use_lifetimes = "warn" 67 | unexpected_cfgs = { level = "warn", check-cfg = [ 68 | 'cfg(all_orderwal_tests)', 69 | 'cfg(test_swmr_constructor)', 70 | 'cfg(test_swmr_insert)', 71 | 'cfg(test_swmr_iters)', 72 | 'cfg(test_swmr_get)', 73 | 'cfg(test_swmr_multiple_version_constructor)', 74 | 'cfg(test_swmr_multiple_version_insert)', 75 | 'cfg(test_swmr_multiple_version_iters)', 76 | 'cfg(test_swmr_multiple_version_get)', 77 | ] } 78 | 79 | [[example]] 80 | name = "zero_cost" 81 | path = "examples/zero_cost.rs" 82 | required-features = ["memmap"] 83 | 84 | [[example]] 85 | name = "multiple_version" 86 | path = "examples/multiple_version.rs" 87 | required-features = ["memmap"] 88 | 89 | [[example]] 90 | name = "generic_not_sized" 91 | path = "examples/generic_not_sized.rs" 92 | required-features = ["memmap"] 93 | -------------------------------------------------------------------------------- /src/swmr/tests/constructor.rs: -------------------------------------------------------------------------------- 1 | use base::{Reader, Writer}; 2 | use skl::KeySize; 3 | 4 | use crate::memtable::{ 5 | alternative::{Table, TableOptions}, 6 | Memtable, MemtableEntry, 7 | }; 8 | 9 | use super::*; 10 | 11 | fn zero_reserved(wal: &mut OrderWal) 12 | where 13 | M: Memtable + 'static, 14 | for<'a> M::Item<'a>: MemtableEntry<'a>, 15 | M::Error: std::fmt::Debug, 16 | { 17 | unsafe { 18 | assert_eq!(wal.reserved_slice(), b""); 19 | assert_eq!(wal.reserved_slice_mut(), b""); 20 | 21 | let wal = wal.reader(); 22 | assert_eq!(wal.reserved_slice(), b""); 23 | } 24 | } 25 | 26 | fn reserved(wal: &mut OrderWal) 27 | where 28 | M: Memtable + 'static, 29 | for<'a> M::Item<'a>: MemtableEntry<'a>, 30 | M::Error: std::fmt::Debug, 31 | { 32 | unsafe { 33 | let buf = wal.reserved_slice_mut(); 34 | buf.copy_from_slice(b"al8n"); 35 | assert_eq!(wal.reserved_slice(), b"al8n"); 36 | assert_eq!(wal.reserved_slice_mut(), b"al8n"); 37 | 38 | let wal = wal.reader(); 39 | assert_eq!(wal.reserved_slice(), b"al8n"); 40 | } 41 | } 42 | 43 | #[cfg(feature = "std")] 44 | expand_unit_tests!( 45 | "linked": OrderWalAlternativeTable [TableOptions::Linked]: Table<_, _> { 46 | zero_reserved, 47 | } 48 | ); 49 | 50 | #[cfg(feature = "std")] 51 | expand_unit_tests!( 52 | "linked": OrderWalAlternativeTable [TableOptions::Linked]: Table<_, _> { 53 | reserved({ 54 | crate::Builder::new() 55 | .with_capacity(MB) 56 | .with_reserved(4) 57 | }), 58 | } 59 | ); 60 | 61 | expand_unit_tests!( 62 | "arena": OrderWalAlternativeTable [TableOptions::Arena(Default::default())]: Table<_, _> { 63 | zero_reserved, 64 | } 65 | ); 66 | 67 | expand_unit_tests!( 68 | "arena": OrderWalAlternativeTable [TableOptions::Arena(Default::default())]: Table<_, _> { 69 | reserved({ 70 | crate::Builder::new() 71 | .with_capacity(MB) 72 | .with_reserved(4) 73 | }), 74 | } 75 | ); 76 | 77 | #[test] 78 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 79 | #[cfg_attr(miri, ignore)] 80 | fn reopen_wrong_kind() { 81 | use crate::Builder; 82 | 83 | let dir = tempfile::tempdir().unwrap(); 84 | let path = dir.path().join("test_reopen_wrong_kind"); 85 | let wal = unsafe { 86 | Builder::new() 87 | .with_capacity(MB) 88 | .with_maximum_key_size(KeySize::with(10)) 89 | .with_maximum_value_size(10) 90 | .with_create_new(true) 91 | .with_read(true) 92 | .with_write(true) 93 | .map_mut::, _>(path.as_path()) 94 | .unwrap() 95 | }; 96 | 97 | assert!(!wal.read_only()); 98 | assert_eq!(wal.capacity(), MB); 99 | assert!(wal.remaining() < MB); 100 | assert_eq!(wal.maximum_key_size(), 10); 101 | assert_eq!(wal.maximum_value_size(), 10); 102 | assert_eq!(wal.path().unwrap().as_path(), path.as_path()); 103 | assert_eq!(wal.options().maximum_key_size(), 10); 104 | 105 | let err = unsafe { 106 | Builder::new() 107 | .with_capacity(MB) 108 | .with_read(true) 109 | .map_mut::, _>(path.as_path()) 110 | .unwrap_err() 111 | }; 112 | assert!(matches!(err, crate::error::Error::KindMismatch { .. })); 113 | } 114 | -------------------------------------------------------------------------------- /src/swmr/tests/multiple_version_constructor.rs: -------------------------------------------------------------------------------- 1 | use multiple_version::{OrderWal, Reader, Writer}; 2 | use skl::KeySize; 3 | 4 | use crate::memtable::{ 5 | alternative::{MultipleVersionTable, TableOptions}, 6 | MultipleVersionMemtable, VersionedMemtableEntry, 7 | }; 8 | 9 | use super::*; 10 | 11 | fn zero_reserved(wal: &mut OrderWal) 12 | where 13 | M: MultipleVersionMemtable + 'static, 14 | M::Error: std::fmt::Debug, 15 | for<'a> M::Item<'a>: VersionedMemtableEntry<'a> + std::fmt::Debug, 16 | { 17 | unsafe { 18 | assert_eq!(wal.reserved_slice(), b""); 19 | assert_eq!(wal.reserved_slice_mut(), b""); 20 | 21 | let wal = wal.reader(); 22 | assert_eq!(wal.reserved_slice(), b""); 23 | } 24 | } 25 | 26 | fn reserved(wal: &mut OrderWal) 27 | where 28 | M: MultipleVersionMemtable + 'static, 29 | M::Error: std::fmt::Debug, 30 | for<'a> M::Item<'a>: VersionedMemtableEntry<'a> + std::fmt::Debug, 31 | { 32 | unsafe { 33 | let buf = wal.reserved_slice_mut(); 34 | buf.copy_from_slice(b"al8n"); 35 | assert_eq!(wal.reserved_slice(), b"al8n"); 36 | assert_eq!(wal.reserved_slice_mut(), b"al8n"); 37 | 38 | let wal = wal.reader(); 39 | assert_eq!(wal.reserved_slice(), b"al8n"); 40 | } 41 | } 42 | 43 | #[cfg(feature = "std")] 44 | expand_unit_tests!( 45 | "linked": MultipleVersionOrderWalAlternativeTable [TableOptions::Linked]: MultipleVersionTable<_, _> { 46 | zero_reserved, 47 | } 48 | ); 49 | 50 | #[cfg(feature = "std")] 51 | expand_unit_tests!( 52 | "linked": MultipleVersionOrderWalAlternativeTable [TableOptions::Linked]: MultipleVersionTable<_, _> { 53 | reserved({ 54 | crate::Builder::new() 55 | .with_capacity(MB) 56 | .with_reserved(4) 57 | }), 58 | } 59 | ); 60 | 61 | expand_unit_tests!( 62 | "arena": MultipleVersionOrderWalAlternativeTable [TableOptions::Arena(Default::default())]: MultipleVersionTable<_, _> { 63 | zero_reserved, 64 | } 65 | ); 66 | 67 | expand_unit_tests!( 68 | "arena": MultipleVersionOrderWalAlternativeTable [TableOptions::Arena(Default::default())]: MultipleVersionTable<_, _> { 69 | reserved({ 70 | crate::Builder::new() 71 | .with_capacity(MB) 72 | .with_reserved(4) 73 | }), 74 | } 75 | ); 76 | 77 | #[test] 78 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 79 | #[cfg_attr(miri, ignore)] 80 | fn reopen_wrong_kind() { 81 | use crate::Builder; 82 | 83 | let dir = tempfile::tempdir().unwrap(); 84 | let path = dir.path().join("test_reopen_wrong_kind"); 85 | let wal = unsafe { 86 | Builder::new() 87 | .with_capacity(MB) 88 | .with_maximum_key_size(KeySize::with(10)) 89 | .with_maximum_value_size(10) 90 | .with_create_new(true) 91 | .with_read(true) 92 | .with_write(true) 93 | .map_mut::, _>(path.as_path()) 94 | .unwrap() 95 | }; 96 | 97 | assert!(!wal.read_only()); 98 | assert_eq!(wal.capacity(), MB); 99 | assert!(wal.remaining() < MB); 100 | assert_eq!(wal.maximum_key_size(), 10); 101 | assert_eq!(wal.maximum_value_size(), 10); 102 | assert_eq!(wal.path().unwrap().as_path(), path.as_path()); 103 | assert_eq!(wal.options().maximum_key_size(), 10); 104 | 105 | let err = unsafe { 106 | Builder::new() 107 | .with_capacity(MB) 108 | .with_read(true) 109 | .map_mut::, _>(path.as_path()) 110 | .unwrap_err() 111 | }; 112 | assert!(matches!(err, crate::error::Error::KindMismatch { .. })); 113 | } 114 | -------------------------------------------------------------------------------- /src/swmr/writer.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | memtable::{BaseTable, Memtable, MemtableEntry, MultipleVersionMemtable, VersionedMemtableEntry}, 3 | sealed::{Constructable, WithVersion}, 4 | }; 5 | use dbutils::{checksum::Crc32, types::Type}; 6 | use rarena_allocator::sync::Arena; 7 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 8 | use rarena_allocator::Allocator; 9 | 10 | use std::sync::Arc; 11 | 12 | use super::{reader::OrderWalReader, wal::OrderCore}; 13 | 14 | /// A ordered write-ahead log implementation for concurrent thread environments. 15 | pub struct OrderWal { 16 | pub(super) core: Arc>, 17 | } 18 | 19 | impl core::fmt::Debug for OrderWal 20 | where 21 | K: ?Sized, 22 | V: ?Sized, 23 | { 24 | #[inline] 25 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 26 | f.debug_tuple("OrderWal").field(&self.core).finish() 27 | } 28 | } 29 | 30 | unsafe impl Send for OrderWal {} 31 | unsafe impl Sync for OrderWal {} 32 | 33 | impl OrderWal { 34 | #[inline] 35 | pub(super) const fn construct(core: Arc>) -> Self { 36 | Self { core } 37 | } 38 | } 39 | 40 | impl Constructable for OrderWal 41 | where 42 | K: ?Sized + 'static, 43 | V: ?Sized + 'static, 44 | S: 'static, 45 | M: BaseTable + 'static, 46 | { 47 | type Allocator = Arena; 48 | type Wal = OrderCore; 49 | type Memtable = M; 50 | type Checksumer = S; 51 | type Reader = OrderWalReader; 52 | 53 | #[inline] 54 | fn as_wal(&self) -> &Self::Wal { 55 | &self.core 56 | } 57 | 58 | #[inline] 59 | fn from_core(core: Self::Wal) -> Self { 60 | Self { 61 | core: Arc::new(core), 62 | } 63 | } 64 | } 65 | 66 | impl OrderWal 67 | where 68 | K: ?Sized + 'static, 69 | V: ?Sized + 'static, 70 | S: 'static, 71 | M: BaseTable + 'static, 72 | { 73 | /// Returns the path of the WAL if it is backed by a file. 74 | /// 75 | /// ## Example 76 | /// 77 | /// ```rust 78 | /// use orderwal::{base::OrderWal, Builder}; 79 | /// 80 | /// // A in-memory WAL 81 | /// let wal = Builder::new().with_capacity(100).alloc::>().unwrap(); 82 | /// 83 | /// assert!(wal.path_buf().is_none()); 84 | /// ``` 85 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 86 | #[cfg_attr(docsrs, doc(cfg(all(feature = "std", not(target_family = "wasm")))))] 87 | #[inline] 88 | pub fn path_buf(&self) -> Option<&std::sync::Arc> { 89 | self.as_wal().arena.path() 90 | } 91 | } 92 | 93 | impl crate::wal::base::Writer for OrderWal 94 | where 95 | K: ?Sized + Type + Ord + 'static, 96 | V: ?Sized + Type + 'static, 97 | M: Memtable + 'static, 98 | for<'a> M::Item<'a>: MemtableEntry<'a>, 99 | S: 'static, 100 | { 101 | #[inline] 102 | fn reader(&self) -> Self::Reader { 103 | OrderWalReader::new(self.core.clone()) 104 | } 105 | } 106 | 107 | impl crate::wal::multiple_version::Writer for OrderWal 108 | where 109 | K: ?Sized + Type + Ord + 'static, 110 | V: ?Sized + Type + 'static, 111 | M: MultipleVersionMemtable + 'static, 112 | for<'a> M::Item<'a>: VersionedMemtableEntry<'a>, 113 | for<'a> M::VersionedItem<'a>: WithVersion, 114 | for<'a> M::Item<'a>: WithVersion, 115 | S: 'static, 116 | { 117 | #[inline] 118 | fn reader(&self) -> Self::Reader { 119 | OrderWalReader::new(self.core.clone()) 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/memtable/linked/table.rs: -------------------------------------------------------------------------------- 1 | use core::{convert::Infallible, ops::RangeBounds}; 2 | 3 | use crossbeam_skiplist::SkipMap; 4 | use dbutils::{ 5 | equivalent::Comparable, 6 | types::{KeyRef, Type}, 7 | }; 8 | 9 | use crate::{ 10 | memtable, 11 | sealed::WithoutVersion, 12 | types::Kind, 13 | wal::{KeyPointer, ValuePointer}, 14 | }; 15 | 16 | pub use crossbeam_skiplist::map::{Entry, Iter, Range}; 17 | 18 | /// An memory table implementation based on [`crossbeam_skiplist::SkipMap`]. 19 | pub struct Table(SkipMap, ValuePointer>); 20 | 21 | impl core::fmt::Debug for Table 22 | where 23 | K: ?Sized + Type + Ord, 24 | for<'a> K::Ref<'a>: KeyRef<'a, K>, 25 | V: ?Sized, 26 | { 27 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 28 | f.debug_tuple("Table").field(&self.0).finish() 29 | } 30 | } 31 | 32 | impl Default for Table { 33 | #[inline] 34 | fn default() -> Self { 35 | Self(SkipMap::new()) 36 | } 37 | } 38 | 39 | impl<'a, K, V> memtable::BaseEntry<'a> for Entry<'a, KeyPointer, ValuePointer> 40 | where 41 | K: ?Sized + Type + Ord, 42 | K::Ref<'a>: KeyRef<'a, K>, 43 | V: ?Sized + Type, 44 | { 45 | type Key = K; 46 | type Value = V; 47 | 48 | #[inline] 49 | fn next(&mut self) -> Option { 50 | Entry::next(self) 51 | } 52 | 53 | #[inline] 54 | fn prev(&mut self) -> Option { 55 | Entry::prev(self) 56 | } 57 | 58 | #[inline] 59 | fn key(&self) -> KeyPointer { 60 | *self.key() 61 | } 62 | } 63 | 64 | impl<'a, K, V> memtable::MemtableEntry<'a> for Entry<'a, KeyPointer, ValuePointer> 65 | where 66 | K: ?Sized + Type + Ord, 67 | K::Ref<'a>: KeyRef<'a, K>, 68 | V: ?Sized + Type, 69 | { 70 | #[inline] 71 | fn value(&self) -> ValuePointer { 72 | *self.value() 73 | } 74 | } 75 | 76 | impl WithoutVersion for Entry<'_, KeyPointer, ValuePointer> 77 | where 78 | K: ?Sized, 79 | V: ?Sized, 80 | { 81 | } 82 | 83 | impl memtable::BaseTable for Table 84 | where 85 | K: ?Sized + Type + Ord, 86 | for<'a> K::Ref<'a>: KeyRef<'a, K>, 87 | V: ?Sized + Type + 'static, 88 | { 89 | type Key = K; 90 | type Value = V; 91 | type Item<'a> 92 | = Entry<'a, KeyPointer, ValuePointer> 93 | where 94 | Self: 'a; 95 | 96 | type Iterator<'a> 97 | = Iter<'a, KeyPointer, ValuePointer> 98 | where 99 | Self: 'a; 100 | 101 | type Range<'a, Q, R> 102 | = Range<'a, Q, R, KeyPointer, ValuePointer> 103 | where 104 | Self: 'a, 105 | R: RangeBounds + 'a, 106 | Q: ?Sized + Comparable>; 107 | 108 | type Options = (); 109 | type Error = Infallible; 110 | 111 | fn new(_: Self::Options) -> Result 112 | where 113 | Self: Sized, 114 | { 115 | Ok(Self(SkipMap::new())) 116 | } 117 | 118 | #[inline] 119 | fn insert( 120 | &self, 121 | _: Option, 122 | kp: KeyPointer, 123 | vp: ValuePointer, 124 | ) -> Result<(), Self::Error> 125 | where 126 | KeyPointer: Ord + 'static, 127 | { 128 | self.0.insert(kp, vp); 129 | Ok(()) 130 | } 131 | 132 | #[inline] 133 | fn remove(&self, _: Option, key: KeyPointer) -> Result<(), Self::Error> 134 | where 135 | KeyPointer: Ord + 'static, 136 | { 137 | self.0.remove(&key); 138 | Ok(()) 139 | } 140 | 141 | #[inline] 142 | fn kind() -> Kind { 143 | Kind::Plain 144 | } 145 | } 146 | 147 | impl memtable::Memtable for Table 148 | where 149 | K: ?Sized + Type + Ord + 'static, 150 | for<'a> K::Ref<'a>: KeyRef<'a, K>, 151 | V: ?Sized + Type + 'static, 152 | { 153 | #[inline] 154 | fn len(&self) -> usize { 155 | self.0.len() 156 | } 157 | 158 | #[inline] 159 | fn upper_bound(&self, bound: core::ops::Bound<&Q>) -> Option> 160 | where 161 | Q: ?Sized + Comparable>, 162 | { 163 | self.0.upper_bound(bound) 164 | } 165 | 166 | #[inline] 167 | fn lower_bound(&self, bound: core::ops::Bound<&Q>) -> Option> 168 | where 169 | Q: ?Sized + Comparable>, 170 | { 171 | self.0.lower_bound(bound) 172 | } 173 | 174 | #[inline] 175 | fn first(&self) -> Option> { 176 | self.0.front() 177 | } 178 | 179 | #[inline] 180 | fn last(&self) -> Option> { 181 | self.0.back() 182 | } 183 | 184 | #[inline] 185 | fn get(&self, key: &Q) -> Option> 186 | where 187 | Q: ?Sized + Comparable>, 188 | { 189 | self.0.get(key) 190 | } 191 | 192 | #[inline] 193 | fn contains(&self, key: &Q) -> bool 194 | where 195 | Q: ?Sized + Comparable>, 196 | { 197 | self.0.contains_key(key) 198 | } 199 | 200 | #[inline] 201 | fn iter(&self) -> Self::Iterator<'_> { 202 | self.0.iter() 203 | } 204 | 205 | #[inline] 206 | fn range<'a, Q, R>(&'a self, range: R) -> Self::Range<'a, Q, R> 207 | where 208 | R: RangeBounds + 'a, 209 | Q: ?Sized + Comparable>, 210 | { 211 | self.0.range(range) 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /src/memtable/alternative/table.rs: -------------------------------------------------------------------------------- 1 | use core::ops::{Bound, RangeBounds}; 2 | 3 | use crate::{ 4 | memtable::{ 5 | arena::{ 6 | table::{Entry as ArenaEntry, Iter as ArenaIter, Range as ArenaRange}, 7 | Table as ArenaTable, 8 | }, 9 | BaseEntry, BaseTable, Memtable, MemtableEntry, 10 | }, 11 | sealed::WithoutVersion, 12 | types::Kind, 13 | wal::{KeyPointer, ValuePointer}, 14 | }; 15 | 16 | #[cfg(feature = "std")] 17 | use crate::memtable::linked::{ 18 | table::{Entry as LinkedEntry, Iter as LinkedIter, Range as LinkedRange}, 19 | Table as LinkedTable, 20 | }; 21 | 22 | use dbutils::{ 23 | equivalent::Comparable, 24 | types::{KeyRef, Type}, 25 | }; 26 | 27 | use super::TableOptions; 28 | 29 | base_entry!( 30 | enum Entry { 31 | Arena(ArenaEntry), 32 | Linked(LinkedEntry), 33 | } 34 | ); 35 | 36 | impl<'a, K, V> MemtableEntry<'a> for Entry<'a, K, V> 37 | where 38 | K: ?Sized + Type + Ord, 39 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 40 | V: ?Sized + Type, 41 | { 42 | #[inline] 43 | fn value(&self) -> ValuePointer { 44 | *match_op!(self.value()) 45 | } 46 | } 47 | 48 | impl WithoutVersion for Entry<'_, K, V> {} 49 | 50 | iter!( 51 | enum Iter { 52 | Arena(ArenaIter), 53 | Linked(LinkedIter), 54 | } -> Entry 55 | ); 56 | 57 | range!( 58 | enum Range { 59 | Arena(ArenaRange), 60 | Linked(LinkedRange), 61 | } -> Entry 62 | ); 63 | 64 | /// A sum type for different memtable implementations. 65 | #[non_exhaustive] 66 | pub enum Table { 67 | /// Arena memtable 68 | Arena(ArenaTable), 69 | /// Linked memtable 70 | #[cfg(feature = "std")] 71 | #[cfg_attr(docsrs, doc(cfg(feature = "std")))] 72 | Linked(LinkedTable), 73 | } 74 | 75 | impl BaseTable for Table 76 | where 77 | K: ?Sized + Type + Ord + 'static, 78 | for<'a> K::Ref<'a>: KeyRef<'a, K>, 79 | for<'a> KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 80 | V: ?Sized + Type + 'static, 81 | { 82 | type Key = K; 83 | 84 | type Value = V; 85 | 86 | type Options = TableOptions; 87 | 88 | type Error = super::Error; 89 | 90 | type Item<'a> 91 | = Entry<'a, K, V> 92 | where 93 | Self: 'a; 94 | 95 | type Iterator<'a> 96 | = Iter<'a, K, V> 97 | where 98 | Self: 'a; 99 | 100 | type Range<'a, Q, R> 101 | = Range<'a, K, V, Q, R> 102 | where 103 | Self: 'a, 104 | R: RangeBounds + 'a, 105 | Q: ?Sized + Comparable>; 106 | 107 | #[inline] 108 | fn new(opts: Self::Options) -> Result 109 | where 110 | Self: Sized, 111 | { 112 | match_op!(new(opts)) 113 | } 114 | 115 | #[inline] 116 | fn insert( 117 | &self, 118 | version: Option, 119 | kp: KeyPointer, 120 | vp: ValuePointer, 121 | ) -> Result<(), Self::Error> 122 | where 123 | KeyPointer: Ord + 'static, 124 | { 125 | match_op!(update(self.insert(version, kp, vp))) 126 | } 127 | 128 | #[inline] 129 | fn remove(&self, version: Option, key: KeyPointer) -> Result<(), Self::Error> 130 | where 131 | KeyPointer: Ord + 'static, 132 | { 133 | match_op!(update(self.remove(version, key))) 134 | } 135 | 136 | #[inline] 137 | fn kind() -> Kind { 138 | Kind::Plain 139 | } 140 | } 141 | 142 | impl Memtable for Table 143 | where 144 | K: ?Sized + Type + Ord + 'static, 145 | for<'a> K::Ref<'a>: KeyRef<'a, K>, 146 | for<'a> KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 147 | V: ?Sized + Type + 'static, 148 | { 149 | #[inline] 150 | fn len(&self) -> usize { 151 | match_op!(self.len()) 152 | } 153 | 154 | #[inline] 155 | fn upper_bound(&self, bound: Bound<&Q>) -> Option> 156 | where 157 | Q: ?Sized + Comparable>, 158 | { 159 | match_op!(self.upper_bound(bound).map(Item)) 160 | } 161 | 162 | #[inline] 163 | fn lower_bound(&self, bound: Bound<&Q>) -> Option> 164 | where 165 | Q: ?Sized + Comparable>, 166 | { 167 | match_op!(self.lower_bound(bound).map(Item)) 168 | } 169 | 170 | #[inline] 171 | fn first(&self) -> Option> 172 | where 173 | KeyPointer: Ord, 174 | { 175 | match_op!(self.first().map(Item)) 176 | } 177 | 178 | #[inline] 179 | fn last(&self) -> Option> 180 | where 181 | KeyPointer: Ord, 182 | { 183 | match_op!(self.last().map(Item)) 184 | } 185 | 186 | #[inline] 187 | fn get(&self, key: &Q) -> Option> 188 | where 189 | Q: ?Sized + Comparable>, 190 | { 191 | match_op!(self.get(key).map(Item)) 192 | } 193 | 194 | #[inline] 195 | fn contains(&self, key: &Q) -> bool 196 | where 197 | Q: ?Sized + Comparable>, 198 | { 199 | match_op!(self.contains(key)) 200 | } 201 | 202 | #[inline] 203 | fn iter(&self) -> Self::Iterator<'_> { 204 | match_op!(Dispatch::Iterator(self.iter())) 205 | } 206 | 207 | #[inline] 208 | fn range<'a, Q, R>(&'a self, range: R) -> Self::Range<'a, Q, R> 209 | where 210 | R: RangeBounds + 'a, 211 | Q: ?Sized + Comparable>, 212 | { 213 | match_op!(Dispatch::Range(self.range(range))) 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /src/memtable/arena/table.rs: -------------------------------------------------------------------------------- 1 | use core::ops::{Bound, RangeBounds}; 2 | 3 | use among::Among; 4 | use dbutils::{ 5 | equivalent::Comparable, 6 | types::{KeyRef, Type}, 7 | }; 8 | use skl::{ 9 | either::Either, 10 | map::{sync::SkipMap, Map as _}, 11 | Arena as _, EntryRef, Options, 12 | }; 13 | 14 | use crate::{ 15 | memtable::{BaseEntry, BaseTable, Memtable, MemtableEntry}, 16 | sealed::WithoutVersion, 17 | types::Kind, 18 | wal::{KeyPointer, ValuePointer}, 19 | }; 20 | 21 | use super::TableOptions; 22 | 23 | pub use skl::map::sync::{Entry, Iter, Range}; 24 | 25 | impl<'a, K, V> BaseEntry<'a> for Entry<'a, KeyPointer, ValuePointer> 26 | where 27 | K: ?Sized + Type + Ord, 28 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 29 | V: ?Sized + Type, 30 | { 31 | type Key = K; 32 | type Value = V; 33 | 34 | #[inline] 35 | fn next(&mut self) -> Option { 36 | Entry::next(self) 37 | } 38 | 39 | #[inline] 40 | fn prev(&mut self) -> Option { 41 | Entry::prev(self) 42 | } 43 | 44 | #[inline] 45 | fn key(&self) -> KeyPointer { 46 | *EntryRef::key(self) 47 | } 48 | } 49 | 50 | impl<'a, K, V> MemtableEntry<'a> for Entry<'a, KeyPointer, ValuePointer> 51 | where 52 | K: ?Sized + Type + Ord, 53 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 54 | V: ?Sized + Type, 55 | { 56 | #[inline] 57 | fn value(&self) -> ValuePointer { 58 | *EntryRef::value(self) 59 | } 60 | } 61 | 62 | impl WithoutVersion for Entry<'_, KeyPointer, ValuePointer> {} 63 | 64 | /// A memory table implementation based on ARENA [`SkipMap`](skl). 65 | pub struct Table { 66 | map: SkipMap, ValuePointer>, 67 | } 68 | 69 | impl BaseTable for Table 70 | where 71 | K: ?Sized + Type + Ord + 'static, 72 | for<'a> KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 73 | V: ?Sized + Type + 'static, 74 | { 75 | type Key = K; 76 | type Value = V; 77 | type Item<'a> 78 | = Entry<'a, KeyPointer, ValuePointer> 79 | where 80 | Self: 'a; 81 | 82 | type Iterator<'a> 83 | = Iter<'a, KeyPointer, ValuePointer> 84 | where 85 | Self: 'a; 86 | 87 | type Range<'a, Q, R> 88 | = Range<'a, KeyPointer, ValuePointer, Q, R> 89 | where 90 | Self: 'a, 91 | R: RangeBounds + 'a, 92 | Q: ?Sized + Comparable>; 93 | 94 | type Options = TableOptions; 95 | type Error = skl::error::Error; 96 | 97 | #[inline] 98 | fn new(opts: Self::Options) -> Result { 99 | let arena_opts = Options::new() 100 | .with_capacity(opts.capacity()) 101 | .with_freelist(skl::Freelist::None) 102 | .with_unify(false) 103 | .with_max_height(opts.max_height()); 104 | 105 | memmap_or_not!(opts(arena_opts)) 106 | } 107 | 108 | fn insert( 109 | &self, 110 | _: Option, 111 | kp: KeyPointer, 112 | vp: ValuePointer, 113 | ) -> Result<(), Self::Error> 114 | where 115 | KeyPointer: Ord + 'static, 116 | { 117 | self.map.insert(&kp, &vp).map(|_| ()).map_err(|e| match e { 118 | Among::Right(e) => e, 119 | _ => unreachable!(), 120 | }) 121 | } 122 | 123 | fn remove(&self, _: Option, key: KeyPointer) -> Result<(), Self::Error> 124 | where 125 | KeyPointer: Ord + 'static, 126 | { 127 | match self.map.get_or_remove(&key) { 128 | Err(Either::Right(e)) => Err(e), 129 | Err(Either::Left(_)) => unreachable!(), 130 | _ => Ok(()), 131 | } 132 | } 133 | 134 | #[inline] 135 | fn kind() -> Kind { 136 | Kind::Plain 137 | } 138 | } 139 | 140 | impl Memtable for Table 141 | where 142 | K: ?Sized + Type + Ord + 'static, 143 | for<'a> KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 144 | V: ?Sized + Type + 'static, 145 | { 146 | #[inline] 147 | fn len(&self) -> usize { 148 | self.map.len() 149 | } 150 | 151 | fn upper_bound(&self, bound: Bound<&Q>) -> Option> 152 | where 153 | Q: ?Sized + Comparable>, 154 | { 155 | self.map.upper_bound(bound) 156 | } 157 | 158 | fn lower_bound(&self, bound: Bound<&Q>) -> Option> 159 | where 160 | Q: ?Sized + Comparable>, 161 | { 162 | self.map.lower_bound(bound) 163 | } 164 | 165 | fn first(&self) -> Option> 166 | where 167 | KeyPointer: Ord, 168 | { 169 | self.map.first() 170 | } 171 | 172 | fn last(&self) -> Option> 173 | where 174 | KeyPointer: Ord, 175 | { 176 | self.map.last() 177 | } 178 | 179 | fn get(&self, key: &Q) -> Option> 180 | where 181 | Q: ?Sized + Comparable>, 182 | { 183 | self.map.get(key) 184 | } 185 | 186 | fn contains(&self, key: &Q) -> bool 187 | where 188 | Q: ?Sized + Comparable>, 189 | { 190 | self.map.contains_key(key) 191 | } 192 | 193 | fn iter(&self) -> Self::Iterator<'_> { 194 | self.map.iter() 195 | } 196 | 197 | fn range<'a, Q, R>(&'a self, range: R) -> Self::Range<'a, Q, R> 198 | where 199 | R: RangeBounds + 'a, 200 | Q: ?Sized + Comparable>, 201 | { 202 | self.map.range(range) 203 | } 204 | } 205 | -------------------------------------------------------------------------------- /examples/zero_cost.rs: -------------------------------------------------------------------------------- 1 | use std::{cmp, sync::Arc, thread::spawn}; 2 | 3 | use dbutils::leb128::{decode_u64_varint, encode_u64_varint, encoded_u64_varint_len}; 4 | use orderwal::{ 5 | base::{OrderWal, Reader, Writer}, 6 | types::{KeyRef, Type, TypeRef}, 7 | Builder, Comparable, Equivalent, 8 | }; 9 | 10 | #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] 11 | struct Person { 12 | id: u64, 13 | name: String, 14 | } 15 | 16 | impl Person { 17 | fn random() -> Self { 18 | Self { 19 | id: rand::random(), 20 | name: names::Generator::default().next().unwrap(), 21 | } 22 | } 23 | } 24 | 25 | #[derive(Debug, Clone, Copy)] 26 | struct PersonRef<'a> { 27 | id: u64, 28 | name: &'a str, 29 | } 30 | 31 | impl PartialEq for PersonRef<'_> { 32 | fn eq(&self, other: &Self) -> bool { 33 | self.id == other.id && self.name == other.name 34 | } 35 | } 36 | 37 | impl Eq for PersonRef<'_> {} 38 | 39 | impl PartialOrd for PersonRef<'_> { 40 | fn partial_cmp(&self, other: &Self) -> Option { 41 | Some(self.cmp(other)) 42 | } 43 | } 44 | 45 | impl Ord for PersonRef<'_> { 46 | fn cmp(&self, other: &Self) -> cmp::Ordering { 47 | self 48 | .id 49 | .cmp(&other.id) 50 | .then_with(|| self.name.cmp(other.name)) 51 | } 52 | } 53 | 54 | impl Equivalent for PersonRef<'_> { 55 | fn equivalent(&self, key: &Person) -> bool { 56 | self.id == key.id && self.name == key.name 57 | } 58 | } 59 | 60 | impl Comparable for PersonRef<'_> { 61 | fn compare(&self, key: &Person) -> core::cmp::Ordering { 62 | self.id.cmp(&key.id).then_with(|| self.name.cmp(&key.name)) 63 | } 64 | } 65 | 66 | impl Equivalent> for Person { 67 | fn equivalent(&self, key: &PersonRef<'_>) -> bool { 68 | self.id == key.id && self.name == key.name 69 | } 70 | } 71 | 72 | impl Comparable> for Person { 73 | fn compare(&self, key: &PersonRef<'_>) -> core::cmp::Ordering { 74 | self 75 | .id 76 | .cmp(&key.id) 77 | .then_with(|| self.name.as_str().cmp(key.name)) 78 | } 79 | } 80 | 81 | impl<'a> KeyRef<'a, Person> for PersonRef<'a> { 82 | fn compare(&self, a: &Q) -> cmp::Ordering 83 | where 84 | Q: ?Sized + Comparable, 85 | { 86 | Comparable::compare(a, self).reverse() 87 | } 88 | 89 | unsafe fn compare_binary(this: &[u8], other: &[u8]) -> cmp::Ordering { 90 | let (this_id_size, this_id) = decode_u64_varint(this).unwrap(); 91 | let (other_id_size, other_id) = decode_u64_varint(other).unwrap(); 92 | 93 | PersonRef { 94 | id: this_id, 95 | name: std::str::from_utf8(&this[this_id_size..]).unwrap(), 96 | } 97 | .cmp(&PersonRef { 98 | id: other_id, 99 | name: std::str::from_utf8(&other[other_id_size..]).unwrap(), 100 | }) 101 | } 102 | } 103 | 104 | impl Type for Person { 105 | type Ref<'a> = PersonRef<'a>; 106 | type Error = dbutils::error::InsufficientBuffer; 107 | 108 | fn encoded_len(&self) -> usize { 109 | encoded_u64_varint_len(self.id) + self.name.len() 110 | } 111 | 112 | #[inline] 113 | fn encode(&self, buf: &mut [u8]) -> Result { 114 | let id_size = encode_u64_varint(self.id, buf)?; 115 | buf[id_size..].copy_from_slice(self.name.as_bytes()); 116 | Ok(id_size + self.name.len()) 117 | } 118 | 119 | #[inline] 120 | fn encode_to_buffer( 121 | &self, 122 | buf: &mut orderwal::types::VacantBuffer<'_>, 123 | ) -> Result { 124 | let id_size = buf.put_u64_varint(self.id)?; 125 | buf.put_slice_unchecked(self.name.as_bytes()); 126 | Ok(id_size + self.name.len()) 127 | } 128 | } 129 | 130 | impl<'a> TypeRef<'a> for PersonRef<'a> { 131 | unsafe fn from_slice(src: &'a [u8]) -> Self { 132 | let (id_size, id) = decode_u64_varint(src).unwrap(); 133 | let name = std::str::from_utf8(&src[id_size..]).unwrap(); 134 | PersonRef { id, name } 135 | } 136 | } 137 | 138 | fn main() { 139 | let dir = tempfile::tempdir().unwrap(); 140 | let path = dir.path().join("zero_copy.wal"); 141 | 142 | let people = (0..100) 143 | .map(|_| { 144 | let p = Person::random(); 145 | let v = std::format!("My name is {}", p.name); 146 | (p, v) 147 | }) 148 | .collect::>(); 149 | 150 | let mut wal = unsafe { 151 | Builder::new() 152 | .with_capacity(1024 * 1024) 153 | .with_create_new(true) 154 | .with_read(true) 155 | .with_write(true) 156 | .map_mut::, _>(&path) 157 | .unwrap() 158 | }; 159 | 160 | // Create 100 readers 161 | let readers = (0..100).map(|_| wal.reader()).collect::>(); 162 | 163 | let people = Arc::new(people); 164 | 165 | // Spawn 100 threads to read from the wal 166 | let handles = readers.into_iter().enumerate().map(|(i, reader)| { 167 | let people = people.clone(); 168 | spawn(move || loop { 169 | let (person, hello) = &people[i]; 170 | let person_ref = PersonRef { 171 | id: person.id, 172 | name: &person.name, 173 | }; 174 | if let Some(p) = reader.get(person) { 175 | assert_eq!(p.key().id, person.id); 176 | assert_eq!(p.key().name, person.name); 177 | assert_eq!(p.value(), hello); 178 | break; 179 | } 180 | 181 | if let Some(p) = reader.get(&person_ref) { 182 | assert_eq!(p.key().id, person.id); 183 | assert_eq!(p.key().name, person.name); 184 | assert_eq!(p.value(), hello); 185 | break; 186 | }; 187 | }) 188 | }); 189 | 190 | // Insert 100 people into the wal 191 | for (p, h) in people.iter() { 192 | wal.insert(p, h).unwrap(); 193 | } 194 | 195 | // Wait for all threads to finish 196 | for handle in handles { 197 | handle.join().unwrap(); 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /src/batch.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | memtable::BaseTable, 3 | wal::{KeyPointer, ValuePointer}, 4 | }; 5 | 6 | use super::{ 7 | sealed::{WithVersion, WithoutVersion}, 8 | types::{BufWriter, EncodedEntryMeta, EntryFlags}, 9 | }; 10 | 11 | /// An entry can be inserted into the WALs through [`Batch`]. 12 | pub struct BatchEntry { 13 | pub(crate) key: K, 14 | pub(crate) value: Option, 15 | pub(crate) flag: EntryFlags, 16 | pub(crate) meta: EncodedEntryMeta, 17 | pointers: Option<(KeyPointer, Option>)>, 18 | pub(crate) version: Option, 19 | } 20 | 21 | impl BatchEntry 22 | where 23 | M: BaseTable, 24 | for<'a> M::Item<'a>: WithoutVersion, 25 | { 26 | /// Creates a new entry. 27 | #[inline] 28 | pub const fn new(key: K, value: V) -> Self { 29 | Self { 30 | key, 31 | value: Some(value), 32 | flag: EntryFlags::empty(), 33 | meta: EncodedEntryMeta::batch_zero(false), 34 | pointers: None, 35 | version: None, 36 | } 37 | } 38 | 39 | /// Creates a tombstone entry. 40 | #[inline] 41 | pub const fn tombstone(key: K) -> Self { 42 | Self { 43 | key, 44 | value: None, 45 | flag: EntryFlags::REMOVED, 46 | meta: EncodedEntryMeta::batch_zero(false), 47 | pointers: None, 48 | version: None, 49 | } 50 | } 51 | } 52 | 53 | impl BatchEntry 54 | where 55 | M: BaseTable, 56 | for<'a> M::Item<'a>: WithVersion, 57 | { 58 | /// Creates a new entry with version. 59 | #[inline] 60 | pub fn with_version(version: u64, key: K, value: V) -> Self { 61 | Self { 62 | key, 63 | value: Some(value), 64 | flag: EntryFlags::empty() | EntryFlags::VERSIONED, 65 | meta: EncodedEntryMeta::batch_zero(true), 66 | pointers: None, 67 | version: Some(version), 68 | } 69 | } 70 | 71 | /// Creates a tombstone entry with version. 72 | #[inline] 73 | pub fn tombstone_with_version(version: u64, key: K) -> Self { 74 | Self { 75 | key, 76 | value: None, 77 | flag: EntryFlags::REMOVED | EntryFlags::VERSIONED, 78 | meta: EncodedEntryMeta::batch_zero(true), 79 | pointers: None, 80 | version: Some(version), 81 | } 82 | } 83 | 84 | /// Returns the version of the entry. 85 | #[inline] 86 | pub const fn version(&self) -> u64 { 87 | match self.version { 88 | Some(version) => version, 89 | None => unreachable!(), 90 | } 91 | } 92 | 93 | /// Set the version of the entry. 94 | #[inline] 95 | pub fn set_version(&mut self, version: u64) { 96 | self.version = Some(version); 97 | } 98 | } 99 | 100 | impl BatchEntry 101 | where 102 | M: BaseTable, 103 | { 104 | /// Returns the length of the key. 105 | #[inline] 106 | pub fn key_len(&self) -> usize 107 | where 108 | K: BufWriter, 109 | { 110 | self.key.encoded_len() 111 | } 112 | 113 | /// Returns the length of the value. 114 | #[inline] 115 | pub fn value_len(&self) -> usize 116 | where 117 | V: BufWriter, 118 | { 119 | self.value.as_ref().map_or(0, |v| v.encoded_len()) 120 | } 121 | 122 | /// Returns the key. 123 | #[inline] 124 | pub const fn key(&self) -> &K { 125 | &self.key 126 | } 127 | 128 | /// Returns the value. 129 | #[inline] 130 | pub const fn value(&self) -> Option<&V> { 131 | self.value.as_ref() 132 | } 133 | 134 | /// Consumes the entry and returns the key and value. 135 | #[inline] 136 | pub fn into_components(self) -> (K, Option) { 137 | (self.key, self.value) 138 | } 139 | 140 | #[inline] 141 | pub(crate) fn encoded_key_len(&self) -> usize 142 | where 143 | K: BufWriter, 144 | V: BufWriter, 145 | { 146 | self.key.encoded_len() 147 | } 148 | 149 | #[inline] 150 | pub(crate) const fn internal_version(&self) -> Option { 151 | self.version 152 | } 153 | 154 | #[inline] 155 | pub(crate) fn take_pointer( 156 | &mut self, 157 | ) -> Option<(KeyPointer, Option>)> { 158 | self.pointers.take() 159 | } 160 | 161 | #[inline] 162 | pub(crate) fn set_pointer(&mut self, kp: KeyPointer, vp: Option>) { 163 | self.pointers = Some((kp, vp)); 164 | } 165 | 166 | #[inline] 167 | pub(crate) fn set_encoded_meta(&mut self, meta: EncodedEntryMeta) { 168 | self.meta = meta; 169 | } 170 | 171 | #[inline] 172 | pub(crate) fn encoded_meta(&self) -> &EncodedEntryMeta { 173 | &self.meta 174 | } 175 | } 176 | 177 | /// A trait for batch insertions. 178 | pub trait Batch { 179 | /// Any type that can be converted into a key. 180 | type Key; 181 | /// Any type that can be converted into a value. 182 | type Value; 183 | 184 | /// The iterator type. 185 | type IterMut<'a>: Iterator> 186 | where 187 | Self: 'a, 188 | Self::Key: 'a, 189 | Self::Value: 'a, 190 | M: 'a; 191 | 192 | /// Returns an iterator over the keys and values. 193 | fn iter_mut<'a>(&'a mut self) -> Self::IterMut<'a> 194 | where 195 | Self: 'a, 196 | Self::Key: 'a, 197 | Self::Value: 'a, 198 | M: 'a; 199 | } 200 | 201 | impl Batch for T 202 | where 203 | M: BaseTable, 204 | for<'a> &'a mut T: IntoIterator>, 205 | { 206 | type Key = K; 207 | type Value = V; 208 | 209 | type IterMut<'a> 210 | = <&'a mut T as IntoIterator>::IntoIter 211 | where 212 | Self: 'a, 213 | Self::Key: 'a, 214 | Self::Value: 'a, 215 | M: 'a; 216 | 217 | fn iter_mut<'a>(&'a mut self) -> Self::IterMut<'a> 218 | where 219 | Self: 'a, 220 | Self::Key: 'a, 221 | Self::Value: 'a, 222 | M: 'a, 223 | { 224 | IntoIterator::into_iter(self) 225 | } 226 | } 227 | -------------------------------------------------------------------------------- /src/types.rs: -------------------------------------------------------------------------------- 1 | use dbutils::leb128::encoded_u64_varint_len; 2 | pub use dbutils::{ 3 | buffer::{BufWriter, BufWriterOnce, VacantBuffer}, 4 | types::*, 5 | }; 6 | 7 | use crate::{utils::merge_lengths, CHECKSUM_SIZE, RECORD_FLAG_SIZE, VERSION_SIZE}; 8 | 9 | pub(crate) mod base; 10 | pub(crate) mod multiple_version; 11 | 12 | const ENTRY_FLAGS_SIZE: usize = core::mem::size_of::(); 13 | 14 | /// The kind of the Write-Ahead Log. 15 | /// 16 | /// Currently, there are two kinds of Write-Ahead Log: 17 | /// 1. Plain: The Write-Ahead Log is plain, which means it does not support multiple versions. 18 | /// 2. MultipleVersion: The Write-Ahead Log supports multiple versions. 19 | #[derive(Debug, PartialEq, Eq)] 20 | #[repr(u8)] 21 | #[non_exhaustive] 22 | pub enum Kind { 23 | /// The Write-Ahead Log is plain, which means it does not support multiple versions. 24 | Plain = 0, 25 | /// The Write-Ahead Log supports multiple versions. 26 | MultipleVersion = 1, 27 | } 28 | 29 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 30 | impl TryFrom for Kind { 31 | type Error = crate::error::UnknownKind; 32 | 33 | #[inline] 34 | fn try_from(value: u8) -> Result { 35 | Ok(match value { 36 | 0 => Self::Plain, 37 | 1 => Self::MultipleVersion, 38 | _ => return Err(crate::error::UnknownKind(value)), 39 | }) 40 | } 41 | } 42 | 43 | bitflags::bitflags! { 44 | /// The flags for each entry. 45 | #[derive(Debug, Copy, Clone)] 46 | pub struct EntryFlags: u8 { 47 | /// First bit: 1 indicates removed 48 | const REMOVED = 0b00000001; 49 | /// Second bit: 1 indicates the key is pointer 50 | const POINTER = 0b00000010; 51 | /// Third bit: 1 indicates the entry contains a version 52 | const VERSIONED = 0b00000100; 53 | } 54 | } 55 | 56 | impl EntryFlags { 57 | pub(crate) const SIZE: usize = core::mem::size_of::(); 58 | } 59 | 60 | #[derive(Debug)] 61 | pub(crate) struct EncodedEntryMeta { 62 | pub(crate) packed_kvlen_size: usize, 63 | pub(crate) packed_kvlen: u64, 64 | pub(crate) entry_size: u32, 65 | pub(crate) klen: usize, 66 | pub(crate) vlen: usize, 67 | pub(crate) versioned: bool, 68 | batch: bool, 69 | } 70 | 71 | impl EncodedEntryMeta { 72 | #[inline] 73 | pub(crate) const fn new(key_len: usize, value_len: usize, versioned: bool) -> Self { 74 | // Cast to u32 is safe, because we already checked those values before calling this function. 75 | 76 | let len = merge_lengths(key_len as u32, value_len as u32); 77 | let len_size = encoded_u64_varint_len(len); 78 | let version_size = if versioned { VERSION_SIZE } else { 0 }; 79 | let elen = RECORD_FLAG_SIZE as u32 80 | + len_size as u32 81 | + ENTRY_FLAGS_SIZE as u32 82 | + version_size as u32 83 | + key_len as u32 84 | + value_len as u32 85 | + CHECKSUM_SIZE as u32; 86 | 87 | Self { 88 | packed_kvlen_size: len_size, 89 | batch: false, 90 | packed_kvlen: len, 91 | entry_size: elen, 92 | klen: key_len, 93 | vlen: value_len, 94 | versioned, 95 | } 96 | } 97 | 98 | #[inline] 99 | pub(crate) const fn batch(key_len: usize, value_len: usize, versioned: bool) -> Self { 100 | // Cast to u32 is safe, because we already checked those values before calling this function. 101 | 102 | let len = merge_lengths(key_len as u32, value_len as u32); 103 | let len_size = encoded_u64_varint_len(len); 104 | let version_size = if versioned { VERSION_SIZE } else { 0 }; 105 | let elen = len_size as u32 106 | + EntryFlags::SIZE as u32 107 | + version_size as u32 108 | + key_len as u32 109 | + value_len as u32; 110 | 111 | Self { 112 | packed_kvlen_size: len_size, 113 | packed_kvlen: len, 114 | entry_size: elen, 115 | klen: key_len, 116 | vlen: value_len, 117 | versioned, 118 | batch: true, 119 | } 120 | } 121 | 122 | #[inline] 123 | pub(crate) const fn batch_zero(versioned: bool) -> Self { 124 | Self { 125 | packed_kvlen_size: 0, 126 | packed_kvlen: 0, 127 | entry_size: 0, 128 | klen: 0, 129 | vlen: 0, 130 | versioned, 131 | batch: true, 132 | } 133 | } 134 | 135 | #[inline] 136 | pub(crate) const fn entry_flag_offset(&self) -> usize { 137 | if self.batch { 138 | return self.packed_kvlen_size; 139 | } 140 | 141 | RECORD_FLAG_SIZE + self.packed_kvlen_size 142 | } 143 | 144 | #[inline] 145 | pub(crate) const fn version_offset(&self) -> usize { 146 | self.entry_flag_offset() + ENTRY_FLAGS_SIZE 147 | } 148 | 149 | #[inline] 150 | pub(crate) const fn key_offset(&self) -> usize { 151 | if self.versioned { 152 | self.version_offset() + VERSION_SIZE 153 | } else { 154 | self.version_offset() 155 | } 156 | } 157 | 158 | #[inline] 159 | pub(crate) const fn value_offset(&self) -> usize { 160 | self.key_offset() + self.klen 161 | } 162 | 163 | #[inline] 164 | pub(crate) const fn checksum_offset(&self) -> usize { 165 | if self.batch { 166 | self.value_offset() + self.vlen 167 | } else { 168 | self.entry_size as usize - CHECKSUM_SIZE 169 | } 170 | } 171 | } 172 | 173 | macro_rules! builder_ext { 174 | ($($name:ident),+ $(,)?) => { 175 | $( 176 | paste::paste! { 177 | impl $name { 178 | #[doc = "Creates a new `" $name "` with the given size and builder closure which requires `FnOnce`."] 179 | #[inline] 180 | pub const fn once(size: usize, f: F) -> Self 181 | where 182 | F: for<'a> FnOnce(&mut dbutils::buffer::VacantBuffer<'a>) -> Result, 183 | { 184 | Self { size, f } 185 | } 186 | } 187 | } 188 | )* 189 | }; 190 | } 191 | 192 | dbutils::builder!( 193 | /// A value builder for the wal, which requires the value size for accurate allocation and a closure to build the value. 194 | pub ValueBuilder; 195 | /// A key builder for the wal, which requires the key size for accurate allocation and a closure to build the key. 196 | pub KeyBuilder; 197 | ); 198 | 199 | builder_ext!(ValueBuilder, KeyBuilder,); 200 | -------------------------------------------------------------------------------- /src/wal/pointer.rs: -------------------------------------------------------------------------------- 1 | use core::{cmp, marker::PhantomData, mem, slice}; 2 | 3 | use dbutils::{ 4 | buffer::VacantBuffer, 5 | equivalent::Comparable, 6 | types::{KeyRef, Type, TypeRef}, 7 | }; 8 | 9 | use crate::types::EntryFlags; 10 | 11 | const PTR_SIZE: usize = mem::size_of::(); 12 | const U32_SIZE: usize = mem::size_of::(); 13 | 14 | pub struct ValuePointer { 15 | ptr: *const u8, 16 | len: usize, 17 | _m: PhantomData, 18 | } 19 | 20 | unsafe impl Send for ValuePointer {} 21 | unsafe impl Sync for ValuePointer {} 22 | 23 | impl core::fmt::Debug for ValuePointer { 24 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 25 | f.debug_struct("ValuePointer") 26 | .field("ptr", &self.ptr) 27 | .field("value", &self.as_slice()) 28 | .finish() 29 | } 30 | } 31 | 32 | impl Clone for ValuePointer { 33 | #[inline] 34 | fn clone(&self) -> Self { 35 | *self 36 | } 37 | } 38 | 39 | impl Copy for ValuePointer {} 40 | 41 | impl ValuePointer { 42 | #[inline] 43 | pub(crate) fn new(len: usize, ptr: *const u8) -> Self { 44 | Self { 45 | ptr, 46 | len, 47 | _m: PhantomData, 48 | } 49 | } 50 | 51 | #[inline] 52 | pub(crate) fn as_slice<'a>(&self) -> &'a [u8] { 53 | if self.len == 0 { 54 | return &[]; 55 | } 56 | 57 | // SAFETY: `ptr` is a valid pointer to `len` bytes. 58 | unsafe { slice::from_raw_parts(self.ptr, self.len) } 59 | } 60 | } 61 | 62 | impl Type for ValuePointer 63 | where 64 | V: ?Sized, 65 | { 66 | type Ref<'a> = Self; 67 | 68 | type Error = (); 69 | 70 | #[inline] 71 | fn encoded_len(&self) -> usize { 72 | const SIZE: usize = PTR_SIZE + U32_SIZE; 73 | SIZE 74 | } 75 | 76 | #[inline] 77 | fn encode_to_buffer(&self, buf: &mut VacantBuffer<'_>) -> Result { 78 | // Safe to cast to u32 here, because the key and value length are guaranteed to be less than or equal to u32::MAX. 79 | let val_len = self.len as u32; 80 | let ptr = self.ptr as usize; 81 | 82 | buf.set_len(self.encoded_len()); 83 | buf[0..PTR_SIZE].copy_from_slice(&ptr.to_le_bytes()); 84 | 85 | buf[PTR_SIZE..PTR_SIZE + U32_SIZE].copy_from_slice(&val_len.to_le_bytes()); 86 | 87 | Ok(PTR_SIZE + U32_SIZE) 88 | } 89 | } 90 | 91 | impl<'a, V: ?Sized> TypeRef<'a> for ValuePointer { 92 | unsafe fn from_slice(src: &'a [u8]) -> Self { 93 | let ptr = usize_to_addr(usize::from_le_bytes((&src[..PTR_SIZE]).try_into().unwrap())); 94 | let len = 95 | u32::from_le_bytes((&src[PTR_SIZE..PTR_SIZE + U32_SIZE]).try_into().unwrap()) as usize; 96 | 97 | Self::new(len, ptr) 98 | } 99 | } 100 | 101 | #[doc(hidden)] 102 | pub struct KeyPointer { 103 | flag: EntryFlags, 104 | ptr: *const u8, 105 | len: usize, 106 | _m: PhantomData, 107 | } 108 | 109 | unsafe impl Send for KeyPointer {} 110 | unsafe impl Sync for KeyPointer {} 111 | 112 | impl core::fmt::Debug for KeyPointer { 113 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 114 | f.debug_struct("KeyPointer") 115 | .field("ptr", &self.ptr) 116 | .field("flag", &self.flag) 117 | .field("key", &self.as_slice()) 118 | .finish() 119 | } 120 | } 121 | 122 | impl Clone for KeyPointer { 123 | #[inline] 124 | fn clone(&self) -> Self { 125 | *self 126 | } 127 | } 128 | 129 | impl Copy for KeyPointer {} 130 | 131 | impl KeyPointer { 132 | #[inline] 133 | pub(crate) fn new(flag: EntryFlags, len: usize, ptr: *const u8) -> Self { 134 | Self { 135 | ptr, 136 | flag, 137 | len, 138 | _m: PhantomData, 139 | } 140 | } 141 | 142 | #[inline] 143 | pub(crate) fn as_slice<'a>(&self) -> &'a [u8] { 144 | if self.len == 0 { 145 | return &[]; 146 | } 147 | 148 | // SAFETY: `ptr` is a valid pointer to `len` bytes. 149 | unsafe { slice::from_raw_parts(self.ptr, self.len) } 150 | } 151 | } 152 | 153 | impl PartialEq for KeyPointer { 154 | fn eq(&self, other: &Self) -> bool { 155 | self.as_slice() == other.as_slice() 156 | } 157 | } 158 | 159 | impl Eq for KeyPointer {} 160 | 161 | impl<'a, K> PartialOrd for KeyPointer 162 | where 163 | K: Type + Ord + ?Sized, 164 | K::Ref<'a>: KeyRef<'a, K>, 165 | { 166 | fn partial_cmp(&self, other: &Self) -> Option { 167 | Some(self.cmp(other)) 168 | } 169 | } 170 | 171 | impl<'a, K> Ord for KeyPointer 172 | where 173 | K: Type + Ord + ?Sized, 174 | K::Ref<'a>: KeyRef<'a, K>, 175 | { 176 | fn cmp(&self, other: &Self) -> cmp::Ordering { 177 | // SAFETY: WALs guarantee that the self and other must be the same as the result returned by `::encode`. 178 | unsafe { as KeyRef>::compare_binary(self.as_slice(), other.as_slice()) } 179 | } 180 | } 181 | 182 | impl Type for KeyPointer 183 | where 184 | K: ?Sized, 185 | { 186 | type Ref<'a> = Self; 187 | 188 | type Error = (); 189 | 190 | #[inline] 191 | fn encoded_len(&self) -> usize { 192 | const SIZE: usize = PTR_SIZE + U32_SIZE + mem::size_of::(); 193 | SIZE 194 | } 195 | 196 | #[inline] 197 | fn encode_to_buffer(&self, buf: &mut VacantBuffer<'_>) -> Result { 198 | // Safe to cast to u32 here, because the key and value length are guaranteed to be less than or equal to u32::MAX. 199 | let key_len = self.len as u32; 200 | let ptr = self.ptr as usize; 201 | 202 | buf.set_len(self.encoded_len()); 203 | buf[0..PTR_SIZE].copy_from_slice(&ptr.to_le_bytes()); 204 | 205 | let mut offset = PTR_SIZE; 206 | buf[offset] = self.flag.bits(); 207 | offset += 1; 208 | buf[offset..offset + U32_SIZE].copy_from_slice(&key_len.to_le_bytes()); 209 | 210 | Ok(offset + U32_SIZE) 211 | } 212 | } 213 | 214 | impl<'a, K: ?Sized> TypeRef<'a> for KeyPointer { 215 | unsafe fn from_slice(src: &'a [u8]) -> Self { 216 | let ptr = usize_to_addr(usize::from_le_bytes((&src[..PTR_SIZE]).try_into().unwrap())); 217 | let mut offset = PTR_SIZE; 218 | let flag = EntryFlags::from_bits_retain(src[offset]); 219 | offset += 1; 220 | let key_len = 221 | u32::from_le_bytes((&src[offset..offset + U32_SIZE]).try_into().unwrap()) as usize; 222 | 223 | Self::new(flag, key_len, ptr) 224 | } 225 | } 226 | 227 | impl<'a, K> KeyRef<'a, Self> for KeyPointer 228 | where 229 | K: Type + Ord + ?Sized, 230 | K::Ref<'a>: KeyRef<'a, K>, 231 | { 232 | #[inline] 233 | fn compare(&self, a: &Q) -> cmp::Ordering 234 | where 235 | Q: ?Sized + Ord + Comparable, 236 | { 237 | Comparable::compare(a, self).reverse() 238 | } 239 | 240 | #[inline] 241 | unsafe fn compare_binary(a: &[u8], b: &[u8]) -> cmp::Ordering { 242 | as KeyRef>::compare_binary(a, b) 243 | } 244 | } 245 | 246 | #[inline] 247 | const fn usize_to_addr(addr: usize) -> *const T { 248 | addr as *const T 249 | } 250 | -------------------------------------------------------------------------------- /src/swmr/tests/get.rs: -------------------------------------------------------------------------------- 1 | use base::OrderWal; 2 | 3 | use dbutils::{buffer::VacantBuffer, types::MaybeStructured}; 4 | 5 | use std::collections::BTreeMap; 6 | 7 | use crate::{ 8 | memtable::{alternative::TableOptions, Memtable, MemtableEntry}, 9 | swmr::base::{Reader, Writer}, 10 | types::{KeyBuilder, ValueBuilder}, 11 | }; 12 | 13 | use super::*; 14 | 15 | fn first(wal: &mut OrderWal) 16 | where 17 | M: Memtable + 'static, 18 | for<'a> M::Item<'a>: MemtableEntry<'a>, 19 | M::Error: std::fmt::Debug, 20 | { 21 | let people = (0..10) 22 | .map(|_| { 23 | let p = Person::random(); 24 | let v = std::format!("My name is {}", p.name); 25 | wal.insert(&p, &v).unwrap(); 26 | 27 | (p, v) 28 | }) 29 | .collect::>(); 30 | 31 | let ent = wal.first().unwrap(); 32 | let (p, v) = people.first_key_value().unwrap(); 33 | assert!(ent.key().equivalent(p)); 34 | assert_eq!(ent.value(), v); 35 | 36 | let wal = wal.reader(); 37 | let ent = wal.first().unwrap(); 38 | let (p, v) = people.first_key_value().unwrap(); 39 | assert!(ent.key().equivalent(p)); 40 | assert_eq!(ent.value(), v); 41 | } 42 | 43 | fn last(wal: &mut OrderWal) 44 | where 45 | M: Memtable + 'static, 46 | for<'a> M::Item<'a>: MemtableEntry<'a>, 47 | M::Error: std::fmt::Debug, 48 | { 49 | let people = (0..10) 50 | .map(|_| { 51 | let p = Person::random(); 52 | let v = std::format!("My name is {}", p.name); 53 | wal.insert(&p, &v).unwrap(); 54 | 55 | (p, v) 56 | }) 57 | .collect::>(); 58 | 59 | let ent = wal.last().unwrap(); 60 | let (p, v) = people.last_key_value().unwrap(); 61 | assert!(ent.key().equivalent(p)); 62 | assert_eq!(ent.value(), v); 63 | 64 | let wal = wal.reader(); 65 | let ent = wal.last().unwrap(); 66 | assert!(ent.key().equivalent(p)); 67 | assert_eq!(ent.value(), v); 68 | } 69 | 70 | #[allow(clippy::needless_borrows_for_generic_args)] 71 | fn insert(wal: &mut OrderWal) 72 | where 73 | M: Memtable + 'static, 74 | for<'a> M::Item<'a>: MemtableEntry<'a>, 75 | M::Error: std::fmt::Debug, 76 | { 77 | let people = (0..100) 78 | .map(|_| { 79 | let p = Person::random(); 80 | let v = std::format!("My name is {}", p.name); 81 | wal.insert(&p, &v).unwrap(); 82 | (p, v) 83 | }) 84 | .collect::>(); 85 | 86 | assert_eq!(wal.len(), 100); 87 | 88 | for (p, pv) in &people { 89 | assert!(wal.contains_key(p)); 90 | 91 | assert_eq!(wal.get(p).unwrap().value(), pv); 92 | } 93 | 94 | for (p, _) in &people { 95 | assert!(wal.contains_key(p)); 96 | } 97 | } 98 | 99 | fn insert_with_value_builder(wal: &mut OrderWal) 100 | where 101 | M: Memtable + 'static, 102 | for<'a> M::Item<'a>: MemtableEntry<'a>, 103 | M::Error: std::fmt::Debug, 104 | { 105 | let people = (0..100) 106 | .map(|_| { 107 | let p = Person::random(); 108 | let v = std::format!("My name is {}", p.name); 109 | wal 110 | .insert_with_value_builder( 111 | &p, 112 | ValueBuilder::new(v.len(), |buf: &mut VacantBuffer<'_>| { 113 | buf.put_slice(v.as_bytes()).map(|_| v.len()) 114 | }), 115 | ) 116 | .unwrap(); 117 | (p, v) 118 | }) 119 | .collect::>(); 120 | 121 | assert_eq!(wal.len(), 100); 122 | 123 | for (p, _) in &people { 124 | assert!(wal.contains_key(p)); 125 | assert!(wal.contains_key(&p.as_ref())); 126 | } 127 | } 128 | 129 | #[allow(clippy::needless_borrows_for_generic_args)] 130 | fn insert_with_key_builder(wal: &mut OrderWal) 131 | where 132 | M: Memtable + 'static, 133 | for<'a> M::Item<'a>: MemtableEntry<'a>, 134 | M::Error: std::fmt::Debug, 135 | { 136 | let people = (0..100) 137 | .map(|_| { 138 | let p = Person::random(); 139 | let pvec = p.to_vec(); 140 | let v = std::format!("My name is {}", p.name); 141 | unsafe { 142 | wal 143 | .insert_with_key_builder( 144 | KeyBuilder::once(p.encoded_len(), |buf| p.encode_to_buffer(buf)), 145 | &v, 146 | ) 147 | .unwrap(); 148 | } 149 | (p, v) 150 | }) 151 | .collect::>(); 152 | 153 | assert_eq!(wal.len(), 100); 154 | 155 | for (p, pv) in &people { 156 | assert!(wal.contains_key(p)); 157 | assert_eq!(wal.get(p).unwrap().value(), pv); 158 | } 159 | 160 | for (p, _) in &people { 161 | assert!(wal.contains_key(p)); 162 | } 163 | } 164 | 165 | fn insert_with_bytes(wal: &mut OrderWal) 166 | where 167 | M: Memtable + 'static, 168 | for<'a> M::Item<'a>: MemtableEntry<'a>, 169 | M::Error: std::fmt::Debug, 170 | { 171 | let people = (0..100) 172 | .map(|_| { 173 | let p = Person::random(); 174 | let v = std::format!("My name is {}", p.name); 175 | unsafe { 176 | wal 177 | .insert( 178 | MaybeStructured::from_slice(p.to_vec().as_slice()), 179 | MaybeStructured::from_slice(v.as_bytes()), 180 | ) 181 | .unwrap(); 182 | } 183 | (p, v) 184 | }) 185 | .collect::>(); 186 | 187 | assert_eq!(wal.len(), 100); 188 | 189 | for (p, pv) in &people { 190 | assert!(wal.contains_key(p)); 191 | assert!(wal.contains_key(&p.as_ref())); 192 | assert_eq!(wal.get(p).unwrap().value(), pv); 193 | } 194 | } 195 | 196 | fn insert_with_builders(wal: &mut OrderWal) 197 | where 198 | M: Memtable + 'static, 199 | for<'a> M::Item<'a>: MemtableEntry<'a> + std::fmt::Debug, 200 | M::Error: std::fmt::Debug, 201 | { 202 | let people = (0..1) 203 | .map(|_| { 204 | let p = Person::random(); 205 | let pvec = p.to_vec(); 206 | let v = std::format!("My name is {}", p.name); 207 | wal 208 | .insert_with_builders( 209 | KeyBuilder::new(pvec.len(), |buf: &mut VacantBuffer<'_>| { 210 | p.encode_to_buffer(buf) 211 | }), 212 | ValueBuilder::new(v.len(), |buf: &mut VacantBuffer<'_>| { 213 | buf.put_slice(v.as_bytes()).map(|_| v.len()) 214 | }), 215 | ) 216 | .unwrap(); 217 | (p, pvec, v) 218 | }) 219 | .collect::>(); 220 | 221 | assert_eq!(wal.len(), 1); 222 | 223 | for (p, pvec, pv) in &people { 224 | assert!(wal.contains_key(p)); 225 | unsafe { 226 | assert_eq!(wal.get_by_bytes(pvec.as_ref()).unwrap().value(), pv); 227 | } 228 | } 229 | 230 | for (p, _, _) in &people { 231 | assert!(wal.contains_key(p)); 232 | } 233 | } 234 | 235 | #[cfg(feature = "std")] 236 | expand_unit_tests!("linked": OrderWalAlternativeTable [TableOptions::Linked]: crate::memtable::alternative::Table<_, _> { 237 | first, 238 | last, 239 | insert, 240 | insert_with_value_builder, 241 | insert_with_key_builder, 242 | insert_with_bytes, 243 | insert_with_builders, 244 | }); 245 | 246 | expand_unit_tests!("arena": OrderWalAlternativeTable [TableOptions::Arena(Default::default())]: crate::memtable::alternative::Table<_, _> { 247 | first, 248 | last, 249 | insert, 250 | insert_with_value_builder, 251 | insert_with_key_builder, 252 | insert_with_bytes, 253 | insert_with_builders, 254 | }); 255 | -------------------------------------------------------------------------------- /src/types/base.rs: -------------------------------------------------------------------------------- 1 | use dbutils::types::{KeyRef, Type}; 2 | use skl::LazyRef; 3 | 4 | use crate::{memtable::MemtableEntry, sealed::WithoutVersion}; 5 | 6 | /// The reference to an entry in the generic WALs. 7 | pub struct Entry<'a, E> 8 | where 9 | E: MemtableEntry<'a>, 10 | E::Key: Type, 11 | E::Value: Type, 12 | { 13 | ent: E, 14 | key: LazyRef<'a, E::Key>, 15 | value: LazyRef<'a, E::Value>, 16 | } 17 | 18 | impl<'a, E> core::fmt::Debug for Entry<'a, E> 19 | where 20 | E: MemtableEntry<'a> + core::fmt::Debug, 21 | E::Key: Type, 22 | E::Value: Type, 23 | { 24 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 25 | f.debug_struct("Entry") 26 | .field("key", &self.key()) 27 | .field("value", &self.value()) 28 | .finish() 29 | } 30 | } 31 | 32 | impl<'a, E> Clone for Entry<'a, E> 33 | where 34 | E: MemtableEntry<'a> + Clone, 35 | E::Key: Type, 36 | E::Value: Type, 37 | { 38 | #[inline] 39 | fn clone(&self) -> Self { 40 | Self { 41 | ent: self.ent.clone(), 42 | key: self.key.clone(), 43 | value: self.value.clone(), 44 | } 45 | } 46 | } 47 | 48 | impl<'a, E> Entry<'a, E> 49 | where 50 | E: MemtableEntry<'a> + WithoutVersion, 51 | E::Key: Type, 52 | E::Value: Type, 53 | { 54 | #[inline] 55 | pub(crate) fn new(ent: E) -> Self { 56 | let raw_key = ent.key().as_slice(); 57 | let raw_value = ent.value().as_slice(); 58 | unsafe { 59 | Self { 60 | key: LazyRef::from_raw(raw_key), 61 | value: LazyRef::from_raw(raw_value), 62 | ent, 63 | } 64 | } 65 | } 66 | } 67 | 68 | impl<'a, E> Entry<'a, E> 69 | where 70 | E: MemtableEntry<'a> + WithoutVersion, 71 | E::Key: Type + Ord, 72 | ::Ref<'a>: KeyRef<'a, E::Key>, 73 | E::Value: Type, 74 | { 75 | /// Returns the next entry in the generic WALs. 76 | /// 77 | /// This does not move the cursor. 78 | #[inline] 79 | #[allow(clippy::should_implement_trait)] 80 | pub fn next(&mut self) -> Option { 81 | self.ent.next().map(Self::new) 82 | } 83 | 84 | /// Returns the previous entry in the generic WALs. 85 | /// 86 | /// This does not move the cursor. 87 | #[inline] 88 | pub fn prev(&mut self) -> Option { 89 | self.ent.prev().map(Self::new) 90 | } 91 | } 92 | 93 | impl<'a, E> Entry<'a, E> 94 | where 95 | E: MemtableEntry<'a>, 96 | E::Key: Type, 97 | E::Value: Type, 98 | { 99 | /// Returns the key of the entry. 100 | #[inline] 101 | pub fn key(&self) -> &::Ref<'a> { 102 | self.key.get() 103 | } 104 | 105 | /// Returns the raw key of the entry. 106 | #[inline] 107 | pub fn raw_key(&self) -> &[u8] { 108 | self.key.raw().expect("Entry's raw key cannot be None") 109 | } 110 | 111 | /// Returns the value of the entry. 112 | #[inline] 113 | pub fn value(&self) -> &::Ref<'a> { 114 | self.value.get() 115 | } 116 | 117 | /// Returns the raw value of the entry. 118 | #[inline] 119 | pub fn raw_value(&self) -> &[u8] { 120 | self.value.raw().expect("Entry's raw value cannot be None") 121 | } 122 | } 123 | 124 | /// The reference to a key of the entry in the generic WALs. 125 | pub struct Key<'a, E> 126 | where 127 | E: MemtableEntry<'a>, 128 | E::Key: Type, 129 | { 130 | ent: E, 131 | key: LazyRef<'a, E::Key>, 132 | } 133 | 134 | impl<'a, E> core::fmt::Debug for Key<'a, E> 135 | where 136 | E: MemtableEntry<'a> + core::fmt::Debug, 137 | E::Key: Type, 138 | { 139 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 140 | f.debug_struct("Key").field("key", &self.key()).finish() 141 | } 142 | } 143 | 144 | impl<'a, E> Clone for Key<'a, E> 145 | where 146 | E: MemtableEntry<'a> + Clone, 147 | E::Key: Type, 148 | { 149 | #[inline] 150 | fn clone(&self) -> Self { 151 | Self { 152 | ent: self.ent.clone(), 153 | key: self.key.clone(), 154 | } 155 | } 156 | } 157 | 158 | impl<'a, E> Key<'a, E> 159 | where 160 | E::Key: Type + Ord, 161 | ::Ref<'a>: KeyRef<'a, E::Key>, 162 | E: MemtableEntry<'a>, 163 | { 164 | /// Returns the next entry in the generic WALs. 165 | /// 166 | /// This does not move the cursor. 167 | #[inline] 168 | #[allow(clippy::should_implement_trait)] 169 | pub fn next(&mut self) -> Option { 170 | self.ent.next().map(Self::new) 171 | } 172 | 173 | /// Returns the previous entry in the generic WALs. 174 | /// 175 | /// This does not move the cursor. 176 | #[inline] 177 | pub fn prev(&mut self) -> Option { 178 | self.ent.prev().map(Self::new) 179 | } 180 | } 181 | 182 | impl<'a, E> Key<'a, E> 183 | where 184 | E::Key: Type, 185 | E: MemtableEntry<'a>, 186 | { 187 | /// Returns the key of the entry. 188 | #[inline] 189 | pub fn key(&self) -> &::Ref<'a> { 190 | self.key.get() 191 | } 192 | 193 | /// Returns the raw key of the entry. 194 | #[inline] 195 | pub fn raw_key(&self) -> &[u8] { 196 | self.key.raw().expect("Key's raw key cannot be None") 197 | } 198 | 199 | #[inline] 200 | pub(crate) fn new(ent: E) -> Self { 201 | let raw_key = ent.key().as_slice(); 202 | unsafe { 203 | Self { 204 | key: LazyRef::from_raw(raw_key), 205 | ent, 206 | } 207 | } 208 | } 209 | } 210 | 211 | /// The reference to a value of the entry in the generic WALs. 212 | pub struct Value<'a, E> 213 | where 214 | E::Value: Type, 215 | E: MemtableEntry<'a>, 216 | { 217 | ent: E, 218 | raw_key: &'a [u8], 219 | value: LazyRef<'a, E::Value>, 220 | } 221 | 222 | impl<'a, E> core::fmt::Debug for Value<'a, E> 223 | where 224 | E: MemtableEntry<'a> + core::fmt::Debug, 225 | E::Value: Type, 226 | ::Ref<'a>: core::fmt::Debug, 227 | { 228 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 229 | f.debug_struct("Value") 230 | .field("value", &self.value()) 231 | .finish() 232 | } 233 | } 234 | 235 | impl<'a, E> Clone for Value<'a, E> 236 | where 237 | E: MemtableEntry<'a> + Clone, 238 | E::Value: Type, 239 | { 240 | #[inline] 241 | fn clone(&self) -> Self { 242 | Self { 243 | ent: self.ent.clone(), 244 | raw_key: self.raw_key, 245 | value: self.value.clone(), 246 | } 247 | } 248 | } 249 | 250 | impl<'a, E> Value<'a, E> 251 | where 252 | E: MemtableEntry<'a>, 253 | E::Value: Type, 254 | { 255 | #[inline] 256 | pub(crate) fn new(ent: E) -> Self { 257 | let raw_key = ent.key().as_slice(); 258 | let raw_value = ent.value().as_slice(); 259 | unsafe { 260 | Self { 261 | raw_key, 262 | value: LazyRef::from_raw(raw_value), 263 | ent, 264 | } 265 | } 266 | } 267 | 268 | /// Returns the next entry in the generic WALs. 269 | /// 270 | /// This does not move the cursor. 271 | #[inline] 272 | #[allow(clippy::should_implement_trait)] 273 | pub fn next(&mut self) -> Option { 274 | self.ent.next().map(Self::new) 275 | } 276 | 277 | /// Returns the previous entry in the generic WALs. 278 | /// 279 | /// This does not move the cursor. 280 | #[inline] 281 | pub fn prev(&mut self) -> Option { 282 | self.ent.prev().map(Self::new) 283 | } 284 | 285 | /// Returns the value of the entry. 286 | #[inline] 287 | pub fn value(&self) -> &::Ref<'a> { 288 | self.value.get() 289 | } 290 | 291 | /// Returns the raw value of the entry. 292 | #[inline] 293 | pub fn raw_value(&self) -> &[u8] { 294 | self.value.raw().expect("Value's raw value cannot be None") 295 | } 296 | } 297 | -------------------------------------------------------------------------------- /src/swmr.rs: -------------------------------------------------------------------------------- 1 | mod reader; 2 | mod wal; 3 | mod writer; 4 | 5 | #[cfg(all( 6 | test, 7 | any( 8 | all_orderwal_tests, 9 | test_swmr_constructor, 10 | test_swmr_insert, 11 | test_swmr_get, 12 | test_swmr_iters, 13 | ) 14 | ))] 15 | mod tests; 16 | 17 | /// The ordered write-ahead log without multiple version support. 18 | pub mod base { 19 | use dbutils::checksum::Crc32; 20 | 21 | use super::{reader, writer}; 22 | #[cfg(feature = "std")] 23 | use crate::memtable::linked::Table as BaseLinkedTable; 24 | use crate::memtable::{ 25 | alternative::Table as BaseAlternativeTable, arena::Table as BaseArenaTable, 26 | }; 27 | 28 | pub use crate::{ 29 | memtable::arena::TableOptions as ArenaTableOptions, 30 | types::base::{Entry, Key, Value}, 31 | wal::base::{Iter, Keys, RangeKeys, RangeValues, Reader, Writer}, 32 | }; 33 | 34 | /// An memory table for [`OrderWal`] or [`OrderWalReader`] based on [`linked::Table`](BaseLinkedTable). 35 | #[cfg(feature = "std")] 36 | #[cfg_attr(docsrs, doc(cfg(feature = "std")))] 37 | pub type LinkedTable = BaseLinkedTable; 38 | 39 | /// An memory table for [`OrderWal`] or [`OrderWalReader`] based on [`arena::Table`](BaseArenaTable). 40 | pub type ArenaTable = BaseArenaTable; 41 | 42 | /// An memory table for [`OrderWal`] or [`OrderWalReader`] based on [`alternative::Table`](BaseAlternativeTable). 43 | pub type AlternativeTable = BaseAlternativeTable; 44 | 45 | /// The default memory table used by [`OrderWal`] or [`OrderWalReader`]. 46 | #[cfg(feature = "std")] 47 | pub type DefaultTable = LinkedTable; 48 | 49 | /// The default memory table used by [`OrderWal`] or [`OrderWalReader`]. 50 | #[cfg(not(feature = "std"))] 51 | pub type DefaultTable = ArenaTable; 52 | 53 | /// A generic ordered write-ahead log implementation for multiple threads environments. 54 | /// 55 | /// ```text 56 | /// +----------------------+-------------------------+--------------------+ 57 | /// | magic text (6 bytes) | magic version (2 bytes) | header (8 bytes) | 58 | /// +----------------------+-------------------------+--------------------+---------------------+-----------------+--------------------+ 59 | /// | flag (1 byte) | key len (4 bytes) | key (n bytes) | value len (4 bytes) | value (n bytes) | checksum (8 bytes) | 60 | /// +----------------------+-------------------------+--------------------+---------------------+-----------------|--------------------+ 61 | /// | flag (1 byte) | key len (4 bytes) | key (n bytes) | value len (4 bytes) | value (n bytes) | checksum (8 bytes) | 62 | /// +----------------------+-------------------------+--------------------+---------------------+-----------------+--------------------+ 63 | /// | flag (1 byte) | key len (4 bytes) | key (n bytes) | value len (4 bytes) | value (n bytes) | checksum (8 bytes) | 64 | /// +----------------------+-------------------------+--------------------+---------------------+-----------------+--------------------+ 65 | /// | ... | ... | ... | ... | ... | ... | 66 | /// +----------------------+-------------------------+--------------------+---------------------+-----------------+--------------------+ 67 | /// | ... | ... | ... | ... | ... | ... | 68 | /// +----------------------+-------------------------+--------------------+---------------------+-----------------+--------------------+ 69 | /// ``` 70 | pub type OrderWal, S = Crc32> = writer::OrderWal; 71 | 72 | /// Immutable reader for the generic ordered write-ahead log [`OrderWal`]. 73 | pub type OrderWalReader, S = Crc32> = 74 | reader::OrderWalReader; 75 | } 76 | 77 | /// A multiple version ordered write-ahead log implementation for multiple threads environments. 78 | pub mod multiple_version { 79 | use dbutils::checksum::Crc32; 80 | 81 | use super::{reader, writer}; 82 | #[cfg(feature = "std")] 83 | use crate::memtable::linked::MultipleVersionTable as BaseLinkedTable; 84 | use crate::memtable::{ 85 | alternative::MultipleVersionTable as BaseAlternativeTable, 86 | arena::MultipleVersionTable as BaseArenaTable, 87 | }; 88 | 89 | pub use crate::{ 90 | memtable::arena::TableOptions as ArenaTableOptions, 91 | types::multiple_version::{Entry, Key, Value, VersionedEntry}, 92 | wal::multiple_version::{ 93 | Iter, IterAll, Keys, RangeAll, RangeKeys, RangeValues, Reader, Writer, 94 | }, 95 | }; 96 | 97 | /// An memory table for multiple version [`OrderWal`] or [`OrderWalReader`] based on [`linked::MultipleVersionTable`](BaseLinkedTable). 98 | #[cfg(feature = "std")] 99 | #[cfg_attr(docsrs, doc(cfg(feature = "std")))] 100 | pub type LinkedTable = BaseLinkedTable; 101 | 102 | /// An memory table for multiple version [`OrderWal`] or [`OrderWalReader`] based on [`arena::MultipleVersionTable`](BaseArenaTable). 103 | pub type ArenaTable = BaseArenaTable; 104 | 105 | /// An memory table for multiple version [`OrderWal`] or [`OrderWalReader`] based on [`alternative::MultipleVersionTable`](BaseAlternativeTable). 106 | pub type AlternativeTable = BaseAlternativeTable; 107 | 108 | /// The default memory table used by [`OrderWal`] or [`OrderWalReader`]. 109 | #[cfg(feature = "std")] 110 | pub type DefaultTable = LinkedTable; 111 | 112 | /// The default memory table used by [`OrderWal`] or [`OrderWalReader`]. 113 | #[cfg(not(feature = "std"))] 114 | pub type DefaultTable = ArenaTable; 115 | 116 | /// A multiple versioned generic ordered write-ahead log implementation for multiple threads environments. 117 | /// 118 | /// ```text 119 | /// +----------------------+-------------------------+--------------------+ 120 | /// | magic text (6 bytes) | magic version (2 bytes) | header (8 bytes) | 121 | /// +----------------------+-------------------------+--------------------+---------------------+---------------------+-----------------+--------------------+ 122 | /// | flag (1 byte) | version (8 bytes) | key len (4 bytes) | key (n bytes) | value len (4 bytes) | value (n bytes) | checksum (8 bytes) | 123 | /// +----------------------+-------------------------+--------------------+---------------------+---------------------+-----------------+--------------------+ 124 | /// | flag (1 byte) | version (8 bytes) | key len (4 bytes) | key (n bytes) | value len (4 bytes) | value (n bytes) | checksum (8 bytes) | 125 | /// +----------------------+-------------------------+--------------------+---------------------+---------------------+-----------------+--------------------+ 126 | /// | flag (1 byte) | version (8 bytes) | key len (4 bytes) | key (n bytes) | value len (4 bytes) | value (n bytes) | checksum (8 bytes) | 127 | /// +----------------------+-------------------------+--------------------+---------------------+---------------------+-----------------+--------------------+ 128 | /// | ... | ... | ... | ... | ... | ... | ,,, | 129 | /// +----------------------+-------------------------+--------------------+---------------------+---------------------+-----------------+--------------------+ 130 | /// ``` 131 | pub type OrderWal, S = Crc32> = writer::OrderWal; 132 | 133 | /// Immutable reader for the multiple versioned generic ordered write-ahead log [`OrderWal`]. 134 | pub type OrderWalReader, S = Crc32> = 135 | reader::OrderWalReader; 136 | } 137 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 |

OrderWAL

3 |
4 |
5 | 6 | A generic-purpose, atomic, ordered, zero-copy read, zero-cost (in-place) write, Write-Ahead Log implementation for Rust. 7 | 8 | [github][Github-url] 9 | LoC 10 | [Build][CI-url] 11 | [codecov][codecov-url] 12 | 13 | [docs.rs][doc-url] 14 | [crates.io][crates-url] 15 | [crates.io][crates-url] 16 | license 17 | 18 | English | [简体中文][zh-cn-url] 19 | 20 |
21 | 22 | ## Introduction 23 | 24 | `orderwal` is generic-purpose, atomic, ordered, zero-copy read, zero-cost (in-place) write, concurrent-safe, pre-allocate style (memory map) write-ahead-log for developing databases. 25 | 26 | `orderwal` also supports generic structured key and value types, which is not limited to just bytes like other implementations. 27 | 28 | ## Installation 29 | 30 | - Default (with on-disk support) 31 | 32 | ```toml 33 | [dependencies] 34 | orderwal = "0.5" 35 | ``` 36 | 37 | - `std` only (without on-disk support) 38 | 39 | ```toml 40 | [dependencies] 41 | orderwal = { version = "0.5", default-features = false, features = ["std"] } 42 | ``` 43 | 44 | - `no-std` (`alloc` required) 45 | 46 | ```toml 47 | [dependencies] 48 | orderwal = { version = "0.5", default-features = false, features = ["alloc"] } 49 | ``` 50 | 51 | ## Example 52 | 53 | See [examples](./examples/) for more information. 54 | 55 | ## Related projects 56 | 57 | - [`aol`](https://github.com/al8n/aol): Yet another generic purpose, append-only write-ahead log implementation based on `std::fs::File`. 58 | - [`skl`](https://github.com/al8n/skl): A lock-free, ARNEA based skiplist implementation, which supports in-memory and on-disk, suitable for frozen durable data file or memtable for LSM database. 59 | - [`valog`](https://github.com/al8n/valog): A lock-free, generic, lightweight value log for WiscKey or Bitcask architecture databases. 60 | - [`dtlog`](https://github.com/al8n/dtlog): A log for tracking discard stats of multi-files databases. 61 | 62 | #### License 63 | 64 | `orderwal` is under the terms of both the MIT license and the 65 | Apache License (Version 2.0). 66 | 67 | See [LICENSE-APACHE](LICENSE-APACHE), [LICENSE-MIT](LICENSE-MIT) for details. 68 | 69 | Copyright (c) 2024 Al Liu. 70 | 71 | [Github-url]: https://github.com/al8n/orderwal/ 72 | [CI-url]: https://github.com/al8n/orderwal/actions/workflows/ci.yml 73 | [doc-url]: https://docs.rs/orderwal 74 | [crates-url]: https://crates.io/crates/orderwal 75 | [codecov-url]: https://app.codecov.io/gh/al8n/orderwal/ 76 | [zh-cn-url]: https://github.com/al8n/orderwal/tree/main/README-zh_CN.md 77 | -------------------------------------------------------------------------------- /README-zh_CN.md: -------------------------------------------------------------------------------- 1 |
2 |

OrderWAL

3 |
4 |
5 | 6 | A generic-purpose, atomic, ordered, zero-copy read, zero-cost (in-place) write, Write-Ahead Log implementation for Rust. 7 | 8 | [github][Github-url] 9 | LoC 10 | [Build][CI-url] 11 | [codecov][codecov-url] 12 | 13 | [docs.rs][doc-url] 14 | [crates.io][crates-url] 15 | [crates.io][crates-url] 16 | license 17 | 18 | English | [简体中文][zh-cn-url] 19 | 20 |
21 | 22 | ## Introduction 23 | 24 | `orderwal` is generic-purpose, atomic, ordered, zero-copy read, zero-cost (in-place) write, concurrent-safe, pre-allocate style (memory map) write-ahead-log for developing databases. 25 | 26 | `orderwal` also supports generic structured key and value types, which is not limited to just bytes like other implementations. 27 | 28 | ## Installation 29 | 30 | - Default (with on-disk support) 31 | 32 | ```toml 33 | [dependencies] 34 | orderwal = "0.5" 35 | ``` 36 | 37 | - `std` only (without on-disk support) 38 | 39 | ```toml 40 | [dependencies] 41 | orderwal = { version = "0.5", default-features = false, features = ["std"] } 42 | ``` 43 | 44 | - `no-std` (`alloc` required) 45 | 46 | ```toml 47 | [dependencies] 48 | orderwal = { version = "0.5", default-features = false, features = ["alloc"] } 49 | ``` 50 | 51 | ## Example 52 | 53 | See [examples](./examples/) for more information. 54 | 55 | ## Related projects 56 | 57 | - [`aol`](https://github.com/al8n/aol): Yet another generic purpose, append-only write-ahead log implementation based on `std::fs::File`. 58 | - [`skl`](https://github.com/al8n/skl): A lock-free, ARNEA based skiplist implementation, which supports in-memory and on-disk, suitable for frozen durable data file or memtable for LSM database. 59 | - [`valog`](https://github.com/al8n/valog): A lock-free, generic, lightweight value log for WiscKey or Bitcask architecture databases. 60 | - [`dtlog`](https://github.com/al8n/dtlog): A log for tracking discard stats of multi-files databases. 61 | 62 | #### License 63 | 64 | `orderwal` is under the terms of both the MIT license and the 65 | Apache License (Version 2.0). 66 | 67 | See [LICENSE-APACHE](LICENSE-APACHE), [LICENSE-MIT](LICENSE-MIT) for details. 68 | 69 | Copyright (c) 2024 Al Liu. 70 | 71 | [Github-url]: https://github.com/al8n/orderwal/ 72 | [CI-url]: https://github.com/al8n/orderwal/actions/workflows/ci.yml 73 | [doc-url]: https://docs.rs/orderwal 74 | [crates-url]: https://crates.io/crates/orderwal 75 | [codecov-url]: https://app.codecov.io/gh/al8n/orderwal/ 76 | [zh-cn-url]: https://github.com/al8n/orderwal/tree/main/README-zh_CN.md 77 | -------------------------------------------------------------------------------- /src/memtable/alternative.rs: -------------------------------------------------------------------------------- 1 | pub use multiple_version::MultipleVersionTable; 2 | pub use table::Table; 3 | 4 | macro_rules! match_op { 5 | ($self:ident.$op:ident($($args:ident),*) $(.map($associated_ty:ident))?) => {{ 6 | match $self { 7 | Self::Arena(e) => e.$op($($args,)*) $(.map(Self::$associated_ty::Arena))?, 8 | #[cfg(feature = "std")] 9 | Self::Linked(e) => e.$op($($args,)*) $(.map(Self::$associated_ty::Linked))?, 10 | }} 11 | }; 12 | (Dispatch::$associated_ty:ident($self:ident.$op:ident($($args:ident),*))) => {{ 13 | match $self { 14 | Self::Arena(e) => Self::$associated_ty::Arena(e.$op($($args,)*)), 15 | #[cfg(feature = "std")] 16 | Self::Linked(e) => Self::$associated_ty::Linked(e.$op($($args,)*)), 17 | }} 18 | }; 19 | (new($opts:ident)) => {{ 20 | match $opts { 21 | Self::Options::Arena(opts) => ArenaTable::new(opts).map(Self::Arena).map_err(Self::Error::Arena), 22 | #[cfg(feature = "std")] 23 | Self::Options::Linked => LinkedTable::new(()) 24 | .map(Self::Linked) 25 | .map_err(|_| Self::Error::Linked), 26 | } 27 | }}; 28 | (update($self:ident.$op:ident($($args:ident),*))) => {{ 29 | match $self { 30 | Self::Arena(t) => t.$op($($args,)*).map_err(Self::Error::Arena), 31 | #[cfg(feature = "std")] 32 | Self::Linked(t) => t.$op($($args,)*).map_err(|_| Self::Error::Linked), 33 | } 34 | }}; 35 | } 36 | 37 | macro_rules! iter { 38 | (enum $name:ident { 39 | Arena($arena:ident), 40 | Linked($linked:ident), 41 | } -> $ent:ident) => { 42 | /// A sum type of iter for different memtable implementations. 43 | #[non_exhaustive] 44 | pub enum $name<'a, K, V> 45 | where 46 | K: ?Sized + Type + Ord, 47 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 48 | V: ?Sized + Type, 49 | { 50 | /// Arena iter 51 | Arena($arena<'a, KeyPointer, ValuePointer>), 52 | /// Linked iter 53 | #[cfg(feature = "std")] 54 | #[cfg_attr(docsrs, doc(cfg(feature = "std")))] 55 | Linked($linked<'a, KeyPointer, ValuePointer>), 56 | } 57 | 58 | impl<'a, K, V> Iterator for $name<'a, K, V> 59 | where 60 | K: ?Sized + Type + Ord + 'static, 61 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 62 | V: ?Sized + Type + 'static, 63 | { 64 | type Item = $ent<'a, K, V>; 65 | 66 | #[inline] 67 | fn next(&mut self) -> Option { 68 | match_op!(self.next().map(Item)) 69 | } 70 | } 71 | 72 | impl<'a, K, V> DoubleEndedIterator for $name<'a, K, V> 73 | where 74 | K: ?Sized + Type + Ord + 'static, 75 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 76 | V: ?Sized + Type + 'static, 77 | { 78 | #[inline] 79 | fn next_back(&mut self) -> Option { 80 | match_op!(self.next_back().map(Item)) 81 | } 82 | } 83 | }; 84 | } 85 | 86 | macro_rules! range { 87 | (enum $name:ident { 88 | Arena($arena:ident), 89 | Linked($linked:ident), 90 | } -> $ent:ident) => { 91 | /// A sum type of range for different memtable implementations. 92 | #[non_exhaustive] 93 | pub enum $name<'a, K, V, Q, R> 94 | where 95 | R: RangeBounds, 96 | Q: ?Sized + Comparable>, 97 | K: ?Sized + Type + Ord, 98 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 99 | V: ?Sized + Type, 100 | { 101 | /// Arena range 102 | Arena($arena<'a, KeyPointer, ValuePointer, Q, R>), 103 | #[cfg(feature = "std")] 104 | #[cfg_attr(docsrs, doc(cfg(feature = "std")))] 105 | /// Linked range 106 | Linked($linked<'a, Q, R, KeyPointer, ValuePointer>), 107 | } 108 | 109 | impl<'a, K, V, Q, R> Iterator for $name<'a, K, V, Q, R> 110 | where 111 | R: RangeBounds, 112 | Q: ?Sized + Comparable>, 113 | K: ?Sized + Type + Ord + 'a, 114 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 115 | V: ?Sized + Type + 'a, 116 | { 117 | type Item = $ent<'a, K, V>; 118 | 119 | #[inline] 120 | fn next(&mut self) -> Option { 121 | match_op!(self.next().map(Item)) 122 | } 123 | } 124 | 125 | impl<'a, K, V, Q, R> DoubleEndedIterator for $name<'a, K, V, Q, R> 126 | where 127 | R: RangeBounds, 128 | Q: ?Sized + Comparable>, 129 | K: ?Sized + Type + Ord + 'a, 130 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 131 | V: ?Sized + Type + 'a, 132 | { 133 | fn next_back(&mut self) -> Option { 134 | match_op!(self.next_back().map(Item)) 135 | } 136 | } 137 | }; 138 | } 139 | 140 | macro_rules! base_entry { 141 | (enum $name:ident { 142 | Arena($arena:ident), 143 | Linked($linked:ident), 144 | }) => { 145 | /// A sum type of entry for different memtable implementations. 146 | #[derive(Debug)] 147 | #[non_exhaustive] 148 | pub enum $name<'a, K, V> 149 | where 150 | K: ?Sized, 151 | V: ?Sized, 152 | { 153 | /// Arena entry 154 | Arena($arena<'a, KeyPointer, ValuePointer>), 155 | /// Linked entry 156 | #[cfg(feature = "std")] 157 | #[cfg_attr(docsrs, doc(cfg(feature = "std")))] 158 | Linked($linked<'a, KeyPointer, ValuePointer>), 159 | } 160 | 161 | impl Clone for $name<'_, K, V> { 162 | #[inline] 163 | fn clone(&self) -> Self { 164 | match self { 165 | Self::Arena(e) => Self::Arena(e.clone()), 166 | #[cfg(feature = "std")] 167 | Self::Linked(e) => Self::Linked(e.clone()), 168 | } 169 | } 170 | } 171 | 172 | impl<'a, K, V> BaseEntry<'a> for $name<'a, K, V> 173 | where 174 | K: ?Sized + Type + Ord, 175 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 176 | V: ?Sized + Type, 177 | { 178 | type Key = K; 179 | 180 | type Value = V; 181 | 182 | #[inline] 183 | fn key(&self) -> KeyPointer { 184 | *match_op!(self.key()) 185 | } 186 | 187 | fn next(&mut self) -> Option { 188 | match self { 189 | Self::Arena(e) => e.next().map(Self::Arena), 190 | #[cfg(feature = "std")] 191 | Self::Linked(e) => e.next().map(Self::Linked), 192 | } 193 | } 194 | 195 | fn prev(&mut self) -> Option { 196 | match self { 197 | Self::Arena(e) => e.prev().map(Self::Arena), 198 | #[cfg(feature = "std")] 199 | Self::Linked(e) => e.prev().map(Self::Linked), 200 | } 201 | } 202 | } 203 | }; 204 | } 205 | 206 | /// The sum type for different memtable implementations options. 207 | #[derive(Debug)] 208 | #[non_exhaustive] 209 | pub enum TableOptions { 210 | /// The options for the arena memtable. 211 | Arena(super::arena::TableOptions), 212 | /// The options for the linked memtable. 213 | #[cfg(feature = "std")] 214 | #[cfg_attr(docsrs, doc(cfg(feature = "std")))] 215 | Linked, 216 | } 217 | 218 | #[cfg(feature = "std")] 219 | impl Default for TableOptions { 220 | #[inline] 221 | fn default() -> Self { 222 | Self::linked() 223 | } 224 | } 225 | 226 | #[cfg(not(feature = "std"))] 227 | impl Default for TableOptions { 228 | #[inline] 229 | fn default() -> Self { 230 | Self::arena() 231 | } 232 | } 233 | 234 | impl From for TableOptions { 235 | #[inline] 236 | fn from(opts: super::arena::TableOptions) -> Self { 237 | Self::Arena(opts) 238 | } 239 | } 240 | 241 | impl TableOptions { 242 | /// Create a new arena memtable options with the default values. 243 | #[inline] 244 | pub const fn arena() -> Self { 245 | Self::Arena(super::arena::TableOptions::new()) 246 | } 247 | 248 | /// Create a new linked memtable options with the default values. 249 | #[inline] 250 | #[cfg(feature = "std")] 251 | #[cfg_attr(docsrs, doc(cfg(feature = "std")))] 252 | pub const fn linked() -> Self { 253 | Self::Linked 254 | } 255 | } 256 | 257 | /// The sum type of error for different memtable implementations. 258 | #[derive(Debug)] 259 | #[non_exhaustive] 260 | pub enum Error { 261 | /// The error for the arena memtable. 262 | Arena(skl::error::Error), 263 | /// The error for the linked memtable. 264 | #[cfg(feature = "std")] 265 | #[cfg_attr(docsrs, doc(cfg(feature = "std")))] 266 | Linked, 267 | } 268 | 269 | impl From for Error { 270 | #[inline] 271 | fn from(e: skl::error::Error) -> Self { 272 | Self::Arena(e) 273 | } 274 | } 275 | 276 | impl core::fmt::Display for Error { 277 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 278 | match self { 279 | Self::Arena(e) => write!(f, "{e}"), 280 | #[cfg(feature = "std")] 281 | Self::Linked => Ok(()), 282 | } 283 | } 284 | } 285 | 286 | impl core::error::Error for Error {} 287 | 288 | mod multiple_version; 289 | mod table; 290 | -------------------------------------------------------------------------------- /src/memtable/linked/multiple_version.rs: -------------------------------------------------------------------------------- 1 | use core::{ 2 | convert::Infallible, 3 | ops::{Bound, RangeBounds}, 4 | }; 5 | 6 | use crossbeam_skiplist_mvcc::nested::SkipMap; 7 | pub use crossbeam_skiplist_mvcc::nested::{Entry, Iter, IterAll, Range, RangeAll, VersionedEntry}; 8 | 9 | use dbutils::{ 10 | equivalent::Comparable, 11 | types::{KeyRef, Type}, 12 | }; 13 | 14 | use crate::{ 15 | memtable::{self, BaseEntry, VersionedMemtableEntry}, 16 | sealed::WithVersion, 17 | types::Kind, 18 | wal::{KeyPointer, ValuePointer}, 19 | }; 20 | 21 | /// An memory table implementation based on [`crossbeam_skiplist::SkipSet`]. 22 | pub struct MultipleVersionTable(SkipMap, ValuePointer>); 23 | 24 | impl Default for MultipleVersionTable 25 | where 26 | K: ?Sized, 27 | V: ?Sized, 28 | { 29 | #[inline] 30 | fn default() -> Self { 31 | Self(SkipMap::new()) 32 | } 33 | } 34 | 35 | impl<'a, K, V> BaseEntry<'a> for Entry<'a, KeyPointer, ValuePointer> 36 | where 37 | K: ?Sized + Type + Ord, 38 | K::Ref<'a>: KeyRef<'a, K>, 39 | V: ?Sized, 40 | { 41 | type Key = K; 42 | type Value = V; 43 | 44 | #[inline] 45 | fn next(&mut self) -> Option { 46 | Entry::next(self) 47 | } 48 | 49 | #[inline] 50 | fn prev(&mut self) -> Option { 51 | Entry::prev(self) 52 | } 53 | 54 | #[inline] 55 | fn key(&self) -> KeyPointer { 56 | *self.key() 57 | } 58 | } 59 | 60 | impl<'a, K, V> memtable::VersionedMemtableEntry<'a> for Entry<'a, KeyPointer, ValuePointer> 61 | where 62 | K: ?Sized + Type + Ord, 63 | K::Ref<'a>: KeyRef<'a, K>, 64 | V: ?Sized, 65 | { 66 | #[inline] 67 | fn value(&self) -> Option> { 68 | Some(*self.value()) 69 | } 70 | 71 | #[inline] 72 | fn version(&self) -> u64 { 73 | Entry::version(self) 74 | } 75 | } 76 | 77 | impl WithVersion for Entry<'_, KeyPointer, ValuePointer> 78 | where 79 | K: ?Sized, 80 | V: ?Sized, 81 | { 82 | } 83 | 84 | impl<'a, K, V> BaseEntry<'a> for VersionedEntry<'a, KeyPointer, ValuePointer> 85 | where 86 | K: ?Sized + Type + Ord, 87 | K::Ref<'a>: KeyRef<'a, K>, 88 | V: ?Sized, 89 | { 90 | type Key = K; 91 | type Value = V; 92 | 93 | #[inline] 94 | fn next(&mut self) -> Option { 95 | VersionedEntry::next(self) 96 | } 97 | 98 | #[inline] 99 | fn prev(&mut self) -> Option { 100 | VersionedEntry::prev(self) 101 | } 102 | 103 | #[inline] 104 | fn key(&self) -> KeyPointer { 105 | *self.key() 106 | } 107 | } 108 | 109 | impl<'a, K, V> VersionedMemtableEntry<'a> for VersionedEntry<'a, KeyPointer, ValuePointer> 110 | where 111 | K: ?Sized + Type + Ord, 112 | K::Ref<'a>: KeyRef<'a, K>, 113 | V: ?Sized, 114 | { 115 | #[inline] 116 | fn version(&self) -> u64 { 117 | VersionedEntry::version(self) 118 | } 119 | 120 | #[inline] 121 | fn value(&self) -> Option> { 122 | self.value().copied() 123 | } 124 | } 125 | 126 | impl WithVersion for VersionedEntry<'_, KeyPointer, ValuePointer> 127 | where 128 | K: ?Sized, 129 | V: ?Sized, 130 | { 131 | } 132 | 133 | impl memtable::BaseTable for MultipleVersionTable 134 | where 135 | K: ?Sized + Type + Ord + 'static, 136 | for<'a> K::Ref<'a>: KeyRef<'a, K>, 137 | V: ?Sized + 'static, 138 | { 139 | type Key = K; 140 | type Value = V; 141 | type Item<'a> 142 | = Entry<'a, KeyPointer, ValuePointer> 143 | where 144 | Self: 'a; 145 | 146 | type Iterator<'a> 147 | = Iter<'a, KeyPointer, ValuePointer> 148 | where 149 | Self: 'a; 150 | 151 | type Range<'a, Q, R> 152 | = Range<'a, Q, R, KeyPointer, ValuePointer> 153 | where 154 | Self: 'a, 155 | R: RangeBounds + 'a, 156 | Q: ?Sized + Comparable>; 157 | 158 | type Options = (); 159 | type Error = Infallible; 160 | 161 | fn new(_: Self::Options) -> Result 162 | where 163 | Self: Sized, 164 | { 165 | Ok(Self(SkipMap::new())) 166 | } 167 | 168 | #[inline] 169 | fn insert( 170 | &self, 171 | version: Option, 172 | kp: KeyPointer, 173 | vp: ValuePointer, 174 | ) -> Result<(), Self::Error> 175 | where 176 | KeyPointer: Ord + 'static, 177 | { 178 | self.0.insert_unchecked(version.unwrap_or(0), kp, vp); 179 | Ok(()) 180 | } 181 | 182 | #[inline] 183 | fn remove(&self, version: Option, key: KeyPointer) -> Result<(), Self::Error> 184 | where 185 | KeyPointer: Ord + 'static, 186 | { 187 | self.0.remove_unchecked(version.unwrap_or(0), key); 188 | Ok(()) 189 | } 190 | 191 | #[inline] 192 | fn kind() -> Kind { 193 | Kind::MultipleVersion 194 | } 195 | } 196 | 197 | impl memtable::MultipleVersionMemtable for MultipleVersionTable 198 | where 199 | K: ?Sized + Type + Ord + 'static, 200 | for<'a> K::Ref<'a>: KeyRef<'a, K>, 201 | V: ?Sized + 'static, 202 | { 203 | type VersionedItem<'a> 204 | = VersionedEntry<'a, KeyPointer, ValuePointer> 205 | where 206 | Self: 'a; 207 | 208 | type IterAll<'a> 209 | = IterAll<'a, KeyPointer, ValuePointer> 210 | where 211 | Self: 'a; 212 | 213 | type RangeAll<'a, Q, R> 214 | = RangeAll<'a, Q, R, KeyPointer, ValuePointer> 215 | where 216 | Self: 'a, 217 | R: RangeBounds + 'a, 218 | Q: ?Sized + Comparable>; 219 | 220 | #[inline] 221 | fn maximum_version(&self) -> u64 { 222 | self.0.maximum_version() 223 | } 224 | 225 | #[inline] 226 | fn minimum_version(&self) -> u64 { 227 | self.0.minimum_version() 228 | } 229 | 230 | #[inline] 231 | fn may_contain_version(&self, version: u64) -> bool { 232 | self.0.may_contain_version(version) 233 | } 234 | 235 | fn upper_bound(&self, version: u64, bound: Bound<&Q>) -> Option> 236 | where 237 | Q: ?Sized + Comparable>, 238 | { 239 | self.0.upper_bound(version, bound) 240 | } 241 | 242 | fn upper_bound_versioned( 243 | &self, 244 | version: u64, 245 | bound: Bound<&Q>, 246 | ) -> Option> 247 | where 248 | Q: ?Sized + Comparable>, 249 | { 250 | self.0.upper_bound_versioned(version, bound) 251 | } 252 | 253 | fn lower_bound(&self, version: u64, bound: Bound<&Q>) -> Option> 254 | where 255 | Q: ?Sized + Comparable>, 256 | { 257 | self.0.lower_bound(version, bound) 258 | } 259 | 260 | fn lower_bound_versioned( 261 | &self, 262 | version: u64, 263 | bound: Bound<&Q>, 264 | ) -> Option> 265 | where 266 | Q: ?Sized + Comparable>, 267 | { 268 | self.0.lower_bound_versioned(version, bound) 269 | } 270 | 271 | fn first(&self, version: u64) -> Option> 272 | where 273 | KeyPointer: Ord, 274 | { 275 | self.0.front(version) 276 | } 277 | 278 | fn first_versioned(&self, version: u64) -> Option> 279 | where 280 | KeyPointer: Ord, 281 | { 282 | self.0.front_versioned(version) 283 | } 284 | 285 | fn last(&self, version: u64) -> Option> 286 | where 287 | KeyPointer: Ord, 288 | { 289 | self.0.back(version) 290 | } 291 | 292 | fn last_versioned(&self, version: u64) -> Option> 293 | where 294 | KeyPointer: Ord, 295 | { 296 | self.0.back_versioned(version) 297 | } 298 | 299 | fn get(&self, version: u64, key: &Q) -> Option> 300 | where 301 | Q: ?Sized + Comparable>, 302 | { 303 | self.0.get(version, key) 304 | } 305 | 306 | fn get_versioned(&self, version: u64, key: &Q) -> Option> 307 | where 308 | Q: ?Sized + Comparable>, 309 | { 310 | self.0.get_versioned(version, key) 311 | } 312 | 313 | fn contains(&self, version: u64, key: &Q) -> bool 314 | where 315 | Q: ?Sized + Comparable>, 316 | { 317 | self.0.contains_key(version, key) 318 | } 319 | 320 | fn contains_versioned(&self, version: u64, key: &Q) -> bool 321 | where 322 | Q: ?Sized + Comparable>, 323 | { 324 | self.0.contains_key_versioned(version, key) 325 | } 326 | 327 | fn iter(&self, version: u64) -> Self::Iterator<'_> { 328 | self.0.iter(version) 329 | } 330 | 331 | fn iter_all_versions(&self, version: u64) -> Self::IterAll<'_> { 332 | self.0.iter_all_versions(version) 333 | } 334 | 335 | fn range<'a, Q, R>(&'a self, version: u64, range: R) -> Self::Range<'a, Q, R> 336 | where 337 | R: RangeBounds + 'a, 338 | Q: ?Sized + Comparable>, 339 | { 340 | self.0.range(version, range) 341 | } 342 | 343 | fn range_all_versions<'a, Q, R>(&'a self, version: u64, range: R) -> Self::RangeAll<'a, Q, R> 344 | where 345 | R: RangeBounds + 'a, 346 | Q: ?Sized + Comparable>, 347 | { 348 | self.0.range_all_versions(version, range) 349 | } 350 | } 351 | -------------------------------------------------------------------------------- /src/options.rs: -------------------------------------------------------------------------------- 1 | use rarena_allocator::{Freelist, Options as ArenaOptions}; 2 | pub use skl::KeySize; 3 | 4 | use super::{CURRENT_VERSION, HEADER_SIZE}; 5 | 6 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 7 | #[cfg_attr(docsrs, doc(cfg(all(feature = "memmap", not(target_family = "wasm")))))] 8 | mod memmap; 9 | 10 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 11 | pub(crate) use memmap::*; 12 | 13 | /// Options for the WAL. 14 | #[derive(Debug, Clone)] 15 | pub struct Options { 16 | maximum_key_size: KeySize, 17 | maximum_value_size: u32, 18 | sync: bool, 19 | magic_version: u16, 20 | cap: Option, 21 | reserved: u32, 22 | 23 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 24 | pub(crate) lock_meta: bool, 25 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 26 | pub(crate) read: bool, 27 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 28 | pub(crate) write: bool, 29 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 30 | pub(crate) create_new: bool, 31 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 32 | pub(crate) create: bool, 33 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 34 | pub(crate) truncate: bool, 35 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 36 | pub(crate) append: bool, 37 | 38 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 39 | pub(crate) stack: bool, 40 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 41 | pub(crate) populate: bool, 42 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 43 | pub(crate) huge: Option, 44 | } 45 | 46 | impl Default for Options { 47 | #[inline] 48 | fn default() -> Self { 49 | Self::new() 50 | } 51 | } 52 | 53 | impl Options { 54 | /// Create a new `Options` instance. 55 | /// 56 | /// 57 | /// ## Example 58 | /// 59 | /// **Note:** If you are creating in-memory WAL, then you must specify the capacity. 60 | /// 61 | /// ```rust 62 | /// use orderwal::Options; 63 | /// 64 | /// let options = Options::new().with_capacity(1024 * 1024 * 8); // 8MB in-memory WAL 65 | /// ``` 66 | #[inline] 67 | pub const fn new() -> Self { 68 | Self { 69 | maximum_key_size: KeySize::new(), 70 | maximum_value_size: u32::MAX, 71 | sync: true, 72 | magic_version: 0, 73 | cap: None, 74 | reserved: 0, 75 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 76 | lock_meta: false, 77 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 78 | read: false, 79 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 80 | write: false, 81 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 82 | create_new: false, 83 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 84 | create: false, 85 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 86 | truncate: false, 87 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 88 | append: false, 89 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 90 | stack: false, 91 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 92 | populate: false, 93 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 94 | huge: None, 95 | } 96 | } 97 | 98 | /// Set the reserved bytes of the WAL. 99 | /// 100 | /// The `reserved` is used to configure the start position of the WAL. This is useful 101 | /// when you want to add some bytes as your own WAL's header. 102 | /// 103 | /// The default reserved is `0`. 104 | /// 105 | /// ## Example 106 | /// 107 | /// ```rust 108 | /// use orderwal::Options; 109 | /// 110 | /// let opts = Options::new().with_reserved(8); 111 | /// ``` 112 | #[inline] 113 | pub const fn with_reserved(mut self, reserved: u32) -> Self { 114 | self.reserved = reserved; 115 | self 116 | } 117 | 118 | /// Get the reserved of the WAL. 119 | /// 120 | /// The `reserved` is used to configure the start position of the WAL. This is useful 121 | /// when you want to add some bytes as your own WAL's header. 122 | /// 123 | /// The default reserved is `0`. 124 | /// 125 | /// ## Example 126 | /// 127 | /// ```rust 128 | /// use orderwal::Options; 129 | /// 130 | /// let opts = Options::new().with_reserved(8); 131 | /// 132 | /// assert_eq!(opts.reserved(), 8); 133 | /// ``` 134 | #[inline] 135 | pub const fn reserved(&self) -> u32 { 136 | self.reserved 137 | } 138 | 139 | /// Returns the magic version. 140 | /// 141 | /// The default value is `0`. 142 | /// 143 | /// ## Example 144 | /// 145 | /// ```rust 146 | /// use orderwal::Options; 147 | /// 148 | /// let options = Options::new().with_magic_version(1); 149 | /// assert_eq!(options.magic_version(), 1); 150 | /// ``` 151 | #[inline] 152 | pub const fn magic_version(&self) -> u16 { 153 | self.magic_version 154 | } 155 | 156 | /// Returns the capacity of the WAL. 157 | /// 158 | /// The default value is `0`. 159 | /// 160 | /// ## Example 161 | /// 162 | /// ```rust 163 | /// use orderwal::Options; 164 | /// 165 | /// let options = Options::new().with_capacity(1000); 166 | /// assert_eq!(options.capacity(), 1000); 167 | /// ``` 168 | #[inline] 169 | pub const fn capacity(&self) -> u32 { 170 | match self.cap { 171 | Some(cap) => cap, 172 | None => 0, 173 | } 174 | } 175 | 176 | /// Returns the maximum key length. 177 | /// 178 | /// The default value is `u16::MAX`. 179 | /// 180 | /// ## Example 181 | /// 182 | /// ```rust 183 | /// use orderwal::{Options, KeySize}; 184 | /// 185 | /// let options = Options::new().with_maximum_key_size(KeySize::with(1024)); 186 | /// assert_eq!(options.maximum_key_size(), KeySize::with(1024)); 187 | /// ``` 188 | #[inline] 189 | pub const fn maximum_key_size(&self) -> KeySize { 190 | self.maximum_key_size 191 | } 192 | 193 | /// Returns the maximum value length. 194 | /// 195 | /// The default value is `u32::MAX`. 196 | /// 197 | /// ## Example 198 | /// 199 | /// ```rust 200 | /// use orderwal::Options; 201 | /// 202 | /// let options = Options::new().with_maximum_value_size(1024); 203 | /// assert_eq!(options.maximum_value_size(), 1024); 204 | /// ``` 205 | #[inline] 206 | pub const fn maximum_value_size(&self) -> u32 { 207 | self.maximum_value_size 208 | } 209 | 210 | /// Returns `true` if the WAL syncs on write. 211 | /// 212 | /// The default value is `true`. 213 | /// 214 | /// ## Example 215 | /// 216 | /// ```rust 217 | /// use orderwal::Options; 218 | /// 219 | /// let options = Options::new(); 220 | /// assert_eq!(options.sync(), true); 221 | /// ``` 222 | #[inline] 223 | pub const fn sync(&self) -> bool { 224 | self.sync 225 | } 226 | 227 | /// Sets the capacity of the WAL. 228 | /// 229 | /// This configuration will be ignored when using file-backed memory maps. 230 | /// 231 | /// The default value is `0`. 232 | /// 233 | /// ## Example 234 | /// 235 | /// ```rust 236 | /// use orderwal::Options; 237 | /// 238 | /// let options = Options::new().with_capacity(100); 239 | /// assert_eq!(options.capacity(), 100); 240 | /// ``` 241 | #[inline] 242 | pub const fn with_capacity(mut self, cap: u32) -> Self { 243 | self.cap = Some(cap); 244 | self 245 | } 246 | 247 | /// Sets the maximum key length. 248 | /// 249 | /// ## Example 250 | /// 251 | /// ```rust 252 | /// use orderwal::{Options, KeySize}; 253 | /// 254 | /// let options = Options::new().with_maximum_key_size(KeySize::with(1024)); 255 | /// assert_eq!(options.maximum_key_size(), KeySize::with(1024)); 256 | /// ``` 257 | #[inline] 258 | pub const fn with_maximum_key_size(mut self, size: KeySize) -> Self { 259 | self.maximum_key_size = size; 260 | self 261 | } 262 | 263 | /// Sets the maximum value length. 264 | /// 265 | /// ## Example 266 | /// 267 | /// ```rust 268 | /// use orderwal::Options; 269 | /// 270 | /// let options = Options::new().with_maximum_value_size(1024); 271 | /// assert_eq!(options.maximum_value_size(), 1024); 272 | /// ``` 273 | #[inline] 274 | pub const fn with_maximum_value_size(mut self, size: u32) -> Self { 275 | self.maximum_value_size = size; 276 | self 277 | } 278 | 279 | /// Sets the WAL to sync on write. 280 | /// 281 | /// The default value is `true`. 282 | /// 283 | /// ## Example 284 | /// 285 | /// ```rust 286 | /// use orderwal::Options; 287 | /// 288 | /// let options = Options::new().with_sync(false); 289 | /// assert_eq!(options.sync(), false); 290 | /// ``` 291 | #[inline] 292 | pub const fn with_sync(mut self, sync: bool) -> Self { 293 | self.sync = sync; 294 | self 295 | } 296 | 297 | /// Sets the magic version. 298 | /// 299 | /// The default value is `0`. 300 | /// 301 | /// ## Example 302 | /// 303 | /// ```rust 304 | /// use orderwal::Options; 305 | /// 306 | /// let options = Options::new().with_magic_version(1); 307 | /// assert_eq!(options.magic_version(), 1); 308 | /// ``` 309 | #[inline] 310 | pub const fn with_magic_version(mut self, version: u16) -> Self { 311 | self.magic_version = version; 312 | self 313 | } 314 | } 315 | 316 | #[inline] 317 | pub(crate) const fn arena_options(reserved: u32) -> ArenaOptions { 318 | ArenaOptions::new() 319 | .with_magic_version(CURRENT_VERSION) 320 | .with_freelist(Freelist::None) 321 | .with_reserved((HEADER_SIZE + reserved as usize) as u32) 322 | .with_unify(true) 323 | } 324 | -------------------------------------------------------------------------------- /src/memtable/arena/multiple_version.rs: -------------------------------------------------------------------------------- 1 | use core::ops::{Bound, RangeBounds}; 2 | 3 | use among::Among; 4 | use dbutils::{ 5 | equivalent::Comparable, 6 | types::{KeyRef, Type}, 7 | }; 8 | use skl::{ 9 | either::Either, 10 | multiple_version::{sync::SkipMap, Map as _}, 11 | Options, 12 | }; 13 | 14 | pub use skl::multiple_version::sync::{Entry, Iter, IterAll, Range, RangeAll, VersionedEntry}; 15 | 16 | use crate::{ 17 | memtable::{BaseEntry, BaseTable, MultipleVersionMemtable, VersionedMemtableEntry}, 18 | sealed::WithVersion, 19 | types::Kind, 20 | wal::{KeyPointer, ValuePointer}, 21 | }; 22 | 23 | use super::TableOptions; 24 | 25 | impl<'a, K, V> BaseEntry<'a> for Entry<'a, KeyPointer, ValuePointer> 26 | where 27 | K: ?Sized + Type + Ord, 28 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 29 | V: ?Sized + Type, 30 | { 31 | type Key = K; 32 | type Value = V; 33 | 34 | #[inline] 35 | fn next(&mut self) -> Option { 36 | Entry::next(self) 37 | } 38 | 39 | #[inline] 40 | fn prev(&mut self) -> Option { 41 | Entry::prev(self) 42 | } 43 | 44 | #[inline] 45 | fn key(&self) -> KeyPointer { 46 | *Entry::key(self) 47 | } 48 | } 49 | 50 | impl<'a, K, V> VersionedMemtableEntry<'a> for Entry<'a, KeyPointer, ValuePointer> 51 | where 52 | K: ?Sized + Type + Ord, 53 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 54 | V: ?Sized + Type, 55 | { 56 | #[inline] 57 | fn value(&self) -> Option> { 58 | Some(*Entry::value(self)) 59 | } 60 | 61 | #[inline] 62 | fn version(&self) -> u64 { 63 | Entry::version(self) 64 | } 65 | } 66 | 67 | impl WithVersion for Entry<'_, KeyPointer, ValuePointer> 68 | where 69 | K: ?Sized, 70 | V: ?Sized, 71 | { 72 | } 73 | 74 | impl<'a, K, V> BaseEntry<'a> for VersionedEntry<'a, KeyPointer, ValuePointer> 75 | where 76 | K: ?Sized + Type + Ord, 77 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 78 | V: ?Sized + Type, 79 | { 80 | type Key = K; 81 | type Value = V; 82 | 83 | #[inline] 84 | fn next(&mut self) -> Option { 85 | VersionedEntry::next(self) 86 | } 87 | 88 | #[inline] 89 | fn prev(&mut self) -> Option { 90 | VersionedEntry::prev(self) 91 | } 92 | 93 | #[inline] 94 | fn key(&self) -> KeyPointer { 95 | *VersionedEntry::key(self) 96 | } 97 | } 98 | 99 | impl<'a, K, V> VersionedMemtableEntry<'a> for VersionedEntry<'a, KeyPointer, ValuePointer> 100 | where 101 | K: ?Sized + Type + Ord, 102 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 103 | V: ?Sized + Type, 104 | { 105 | #[inline] 106 | fn version(&self) -> u64 { 107 | self.version() 108 | } 109 | 110 | #[inline] 111 | fn value(&self) -> Option> { 112 | VersionedEntry::value(self).copied() 113 | } 114 | } 115 | 116 | impl WithVersion for VersionedEntry<'_, KeyPointer, ValuePointer> 117 | where 118 | K: ?Sized, 119 | V: ?Sized, 120 | { 121 | } 122 | 123 | /// A memory table implementation based on ARENA [`SkipMap`](skl). 124 | pub struct MultipleVersionTable { 125 | map: SkipMap, ValuePointer>, 126 | } 127 | 128 | impl BaseTable for MultipleVersionTable 129 | where 130 | K: ?Sized + Type + Ord + 'static, 131 | for<'a> KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 132 | V: ?Sized + Type + 'static, 133 | { 134 | type Key = K; 135 | type Value = V; 136 | 137 | type Item<'a> 138 | = Entry<'a, KeyPointer, ValuePointer> 139 | where 140 | Self: 'a; 141 | 142 | type Iterator<'a> 143 | = Iter<'a, KeyPointer, ValuePointer> 144 | where 145 | Self: 'a; 146 | 147 | type Range<'a, Q, R> 148 | = Range<'a, KeyPointer, ValuePointer, Q, R> 149 | where 150 | Self: 'a, 151 | R: RangeBounds + 'a, 152 | Q: ?Sized + Comparable>; 153 | 154 | type Options = TableOptions; 155 | 156 | type Error = skl::error::Error; 157 | 158 | #[inline] 159 | fn new(opts: Self::Options) -> Result { 160 | let arena_opts = Options::new() 161 | .with_capacity(opts.capacity()) 162 | .with_freelist(skl::Freelist::None) 163 | .with_unify(false) 164 | .with_max_height(opts.max_height()); 165 | 166 | memmap_or_not!(opts(arena_opts)) 167 | } 168 | 169 | fn insert( 170 | &self, 171 | version: Option, 172 | kp: KeyPointer, 173 | vp: ValuePointer, 174 | ) -> Result<(), Self::Error> 175 | where 176 | KeyPointer: Ord + 'static, 177 | { 178 | self 179 | .map 180 | .insert(version.unwrap_or(0), &kp, &vp) 181 | .map(|_| ()) 182 | .map_err(|e| match e { 183 | Among::Right(e) => e, 184 | _ => unreachable!(), 185 | }) 186 | } 187 | 188 | fn remove(&self, version: Option, key: KeyPointer) -> Result<(), Self::Error> 189 | where 190 | KeyPointer: Ord + 'static, 191 | { 192 | match self.map.get_or_remove(version.unwrap_or(0), &key) { 193 | Err(Either::Right(e)) => Err(e), 194 | Err(Either::Left(_)) => unreachable!(), 195 | _ => Ok(()), 196 | } 197 | } 198 | 199 | #[inline] 200 | fn kind() -> Kind { 201 | Kind::MultipleVersion 202 | } 203 | } 204 | 205 | impl MultipleVersionMemtable for MultipleVersionTable 206 | where 207 | K: ?Sized + Type + Ord + 'static, 208 | for<'a> KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 209 | V: ?Sized + Type + 'static, 210 | { 211 | type VersionedItem<'a> 212 | = VersionedEntry<'a, KeyPointer, ValuePointer> 213 | where 214 | Self: 'a; 215 | 216 | type IterAll<'a> 217 | = IterAll<'a, KeyPointer, ValuePointer> 218 | where 219 | Self: 'a; 220 | 221 | type RangeAll<'a, Q, R> 222 | = RangeAll<'a, KeyPointer, ValuePointer, Q, R> 223 | where 224 | Self: 'a, 225 | R: RangeBounds + 'a, 226 | Q: ?Sized + Comparable>; 227 | 228 | #[inline] 229 | fn maximum_version(&self) -> u64 { 230 | self.map.maximum_version() 231 | } 232 | 233 | #[inline] 234 | fn minimum_version(&self) -> u64 { 235 | self.map.minimum_version() 236 | } 237 | 238 | #[inline] 239 | fn may_contain_version(&self, version: u64) -> bool { 240 | self.map.may_contain_version(version) 241 | } 242 | 243 | fn upper_bound(&self, version: u64, bound: Bound<&Q>) -> Option> 244 | where 245 | Q: ?Sized + Comparable>, 246 | { 247 | self.map.upper_bound(version, bound) 248 | } 249 | 250 | fn upper_bound_versioned( 251 | &self, 252 | version: u64, 253 | bound: Bound<&Q>, 254 | ) -> Option> 255 | where 256 | Q: ?Sized + Comparable>, 257 | { 258 | self.map.upper_bound_versioned(version, bound) 259 | } 260 | 261 | fn lower_bound(&self, version: u64, bound: Bound<&Q>) -> Option> 262 | where 263 | Q: ?Sized + Comparable>, 264 | { 265 | self.map.lower_bound(version, bound) 266 | } 267 | 268 | fn lower_bound_versioned( 269 | &self, 270 | version: u64, 271 | bound: Bound<&Q>, 272 | ) -> Option> 273 | where 274 | Q: ?Sized + Comparable>, 275 | { 276 | self.map.lower_bound_versioned(version, bound) 277 | } 278 | 279 | fn first(&self, version: u64) -> Option> 280 | where 281 | KeyPointer: Ord, 282 | { 283 | self.map.first(version) 284 | } 285 | 286 | fn first_versioned(&self, version: u64) -> Option> 287 | where 288 | KeyPointer: Ord, 289 | { 290 | self.map.first_versioned(version) 291 | } 292 | 293 | fn last(&self, version: u64) -> Option> 294 | where 295 | KeyPointer: Ord, 296 | { 297 | self.map.last(version) 298 | } 299 | 300 | fn last_versioned(&self, version: u64) -> Option> 301 | where 302 | KeyPointer: Ord, 303 | { 304 | self.map.last_versioned(version) 305 | } 306 | 307 | fn get(&self, version: u64, key: &Q) -> Option> 308 | where 309 | Q: ?Sized + Comparable>, 310 | { 311 | self.map.get(version, key) 312 | } 313 | 314 | fn get_versioned(&self, version: u64, key: &Q) -> Option> 315 | where 316 | Q: ?Sized + Comparable>, 317 | { 318 | self.map.get_versioned(version, key) 319 | } 320 | 321 | fn contains(&self, version: u64, key: &Q) -> bool 322 | where 323 | Q: ?Sized + Comparable>, 324 | { 325 | self.map.contains_key(version, key) 326 | } 327 | 328 | fn contains_versioned(&self, version: u64, key: &Q) -> bool 329 | where 330 | Q: ?Sized + Comparable>, 331 | { 332 | self.map.contains_key_versioned(version, key) 333 | } 334 | 335 | fn iter(&self, version: u64) -> Self::Iterator<'_> { 336 | self.map.iter(version) 337 | } 338 | 339 | fn iter_all_versions(&self, version: u64) -> Self::IterAll<'_> { 340 | self.map.iter_all_versions(version) 341 | } 342 | 343 | fn range<'a, Q, R>(&'a self, version: u64, range: R) -> Self::Range<'a, Q, R> 344 | where 345 | R: RangeBounds + 'a, 346 | Q: ?Sized + Comparable>, 347 | { 348 | self.map.range(version, range) 349 | } 350 | 351 | fn range_all_versions<'a, Q, R>(&'a self, version: u64, range: R) -> Self::RangeAll<'a, Q, R> 352 | where 353 | R: RangeBounds + 'a, 354 | Q: ?Sized + Comparable>, 355 | { 356 | self.map.range_all_versions(version, range) 357 | } 358 | } 359 | -------------------------------------------------------------------------------- /src/memtable.rs: -------------------------------------------------------------------------------- 1 | use core::ops::{Bound, RangeBounds}; 2 | use dbutils::equivalent::Comparable; 3 | 4 | use crate::{ 5 | sealed::{WithVersion, WithoutVersion}, 6 | types::Kind, 7 | wal::{KeyPointer, ValuePointer}, 8 | }; 9 | 10 | /// Memtable implementation based on linked based [`SkipMap`][`crossbeam_skiplist`]. 11 | #[cfg(feature = "std")] 12 | #[cfg_attr(docsrs, doc(cfg(feature = "std")))] 13 | pub mod linked; 14 | 15 | /// Memtable implementation based on ARNEA based [`SkipMap`](skl). 16 | pub mod arena; 17 | 18 | /// Sum type for different memtable implementations. 19 | pub mod alternative; 20 | 21 | /// An entry which is stored in the memory table. 22 | pub trait BaseEntry<'a>: Sized { 23 | /// The key type. 24 | type Key: ?Sized; 25 | /// The value type. 26 | type Value: ?Sized; 27 | 28 | /// Returns the key in the entry. 29 | fn key(&self) -> KeyPointer; 30 | 31 | /// Returns the next entry in the memory table. 32 | fn next(&mut self) -> Option; 33 | 34 | /// Returns the previous entry in the memory table. 35 | fn prev(&mut self) -> Option; 36 | } 37 | 38 | /// An entry which is stored in the memory table. 39 | pub trait MemtableEntry<'a>: BaseEntry<'a> + WithoutVersion { 40 | /// Returns the value in the entry. 41 | fn value(&self) -> ValuePointer; 42 | } 43 | 44 | /// An entry which is stored in the multiple versioned memory table. 45 | pub trait VersionedMemtableEntry<'a>: BaseEntry<'a> + WithVersion { 46 | /// Returns the value in the entry. 47 | fn value(&self) -> Option>; 48 | 49 | /// Returns the version of the entry if it is versioned. 50 | fn version(&self) -> u64; 51 | } 52 | 53 | /// A memory table which is used to store pointers to the underlying entries. 54 | pub trait BaseTable { 55 | /// The key type. 56 | type Key: ?Sized; 57 | 58 | /// The value type. 59 | type Value: ?Sized; 60 | 61 | /// The configuration options for the memtable. 62 | type Options; 63 | 64 | /// The error type may be returned when constructing the memtable. 65 | type Error; 66 | 67 | /// The item returned by the iterator or query methods. 68 | type Item<'a>: BaseEntry<'a, Key = Self::Key, Value = Self::Value> + Clone 69 | where 70 | Self: 'a; 71 | 72 | /// The iterator type. 73 | type Iterator<'a>: DoubleEndedIterator> 74 | where 75 | Self: 'a; 76 | 77 | /// The range iterator type. 78 | type Range<'a, Q, R>: DoubleEndedIterator> 79 | where 80 | Self: 'a, 81 | R: RangeBounds + 'a, 82 | Q: ?Sized + Comparable>; 83 | 84 | /// Creates a new memtable with the specified options. 85 | fn new(opts: Self::Options) -> Result 86 | where 87 | Self: Sized; 88 | 89 | /// Inserts a pointer into the memtable. 90 | fn insert( 91 | &self, 92 | version: Option, 93 | kp: KeyPointer, 94 | vp: ValuePointer, 95 | ) -> Result<(), Self::Error> 96 | where 97 | KeyPointer: Ord + 'static; 98 | 99 | /// Removes the pointer associated with the key. 100 | fn remove(&self, version: Option, key: KeyPointer) -> Result<(), Self::Error> 101 | where 102 | KeyPointer: Ord + 'static; 103 | 104 | /// Returns the kind of the memtable. 105 | fn kind() -> Kind; 106 | } 107 | 108 | /// A memory table which is used to store pointers to the underlying entries. 109 | pub trait Memtable: BaseTable 110 | where 111 | for<'a> Self::Item<'a>: MemtableEntry<'a>, 112 | { 113 | /// Returns the number of entries in the memtable. 114 | fn len(&self) -> usize; 115 | 116 | /// Returns `true` if the memtable is empty. 117 | fn is_empty(&self) -> bool { 118 | self.len() == 0 119 | } 120 | 121 | /// Returns the upper bound of the memtable. 122 | fn upper_bound(&self, bound: Bound<&Q>) -> Option> 123 | where 124 | Q: ?Sized + Comparable>; 125 | 126 | /// Returns the lower bound of the memtable. 127 | fn lower_bound(&self, bound: Bound<&Q>) -> Option> 128 | where 129 | Q: ?Sized + Comparable>; 130 | 131 | /// Returns the first pointer in the memtable. 132 | fn first(&self) -> Option> 133 | where 134 | KeyPointer: Ord; 135 | 136 | /// Returns the last pointer in the memtable. 137 | fn last(&self) -> Option> 138 | where 139 | KeyPointer: Ord; 140 | 141 | /// Returns the pointer associated with the key. 142 | fn get(&self, key: &Q) -> Option> 143 | where 144 | Q: ?Sized + Comparable>; 145 | 146 | /// Returns `true` if the memtable contains the specified pointer. 147 | fn contains(&self, key: &Q) -> bool 148 | where 149 | Q: ?Sized + Comparable>; 150 | 151 | /// Returns an iterator over the memtable. 152 | fn iter(&self) -> Self::Iterator<'_>; 153 | 154 | /// Returns an iterator over a subset of the memtable. 155 | fn range<'a, Q, R>(&'a self, range: R) -> Self::Range<'a, Q, R> 156 | where 157 | R: RangeBounds + 'a, 158 | Q: ?Sized + Comparable>; 159 | } 160 | 161 | /// A memory table which is used to store pointers to the underlying entries. 162 | pub trait MultipleVersionMemtable: BaseTable 163 | where 164 | for<'a> Self::Item<'a>: VersionedMemtableEntry<'a>, 165 | { 166 | /// The item returned by the iterator or query methods. 167 | type VersionedItem<'a>: VersionedMemtableEntry<'a, Key = Self::Key, Value = Self::Value> + Clone 168 | where 169 | KeyPointer: 'a, 170 | Self: 'a; 171 | 172 | /// The iterator type which can yields all the entries in the memtable. 173 | type IterAll<'a>: DoubleEndedIterator> 174 | where 175 | KeyPointer: 'a, 176 | Self: 'a; 177 | 178 | /// The range iterator type which can yields all the entries in the memtable. 179 | type RangeAll<'a, Q, R>: DoubleEndedIterator> 180 | where 181 | KeyPointer: 'a, 182 | Self: 'a, 183 | R: RangeBounds + 'a, 184 | Q: ?Sized + Comparable>; 185 | 186 | /// Returns the maximum version of the memtable. 187 | fn maximum_version(&self) -> u64; 188 | 189 | /// Returns the minimum version of the memtable. 190 | fn minimum_version(&self) -> u64; 191 | 192 | /// Returns `true` if the memtable may contain an entry whose version is less than or equal to the specified version. 193 | fn may_contain_version(&self, version: u64) -> bool; 194 | 195 | /// Returns the upper bound of the memtable. 196 | fn upper_bound(&self, version: u64, bound: Bound<&Q>) -> Option> 197 | where 198 | Q: ?Sized + Comparable>; 199 | 200 | /// Returns the upper bound of the memtable. 201 | fn upper_bound_versioned( 202 | &self, 203 | version: u64, 204 | bound: Bound<&Q>, 205 | ) -> Option> 206 | where 207 | Q: ?Sized + Comparable>; 208 | 209 | /// Returns the lower bound of the memtable. 210 | fn lower_bound(&self, version: u64, bound: Bound<&Q>) -> Option> 211 | where 212 | Q: ?Sized + Comparable>; 213 | 214 | /// Returns the lower bound of the memtable. 215 | fn lower_bound_versioned( 216 | &self, 217 | version: u64, 218 | bound: Bound<&Q>, 219 | ) -> Option> 220 | where 221 | Q: ?Sized + Comparable>; 222 | 223 | /// Returns the first pointer in the memtable. 224 | fn first(&self, version: u64) -> Option> 225 | where 226 | KeyPointer: Ord; 227 | 228 | /// Returns the first pointer in the memtable. 229 | fn first_versioned(&self, version: u64) -> Option> 230 | where 231 | KeyPointer: Ord; 232 | 233 | /// Returns the last pointer in the memtable. 234 | fn last(&self, version: u64) -> Option> 235 | where 236 | KeyPointer: Ord; 237 | 238 | /// Returns the last pointer in the memtable. 239 | fn last_versioned(&self, version: u64) -> Option> 240 | where 241 | KeyPointer: Ord; 242 | 243 | /// Returns the pointer associated with the key. 244 | fn get(&self, version: u64, key: &Q) -> Option> 245 | where 246 | Q: ?Sized + Comparable>; 247 | 248 | /// Returns the pointer associated with the key. 249 | fn get_versioned(&self, version: u64, key: &Q) -> Option> 250 | where 251 | Q: ?Sized + Comparable>; 252 | 253 | /// Returns `true` if the memtable contains the specified pointer. 254 | fn contains(&self, version: u64, key: &Q) -> bool 255 | where 256 | Q: ?Sized + Comparable>; 257 | 258 | /// Returns `true` if the memtable contains the specified pointer. 259 | fn contains_versioned(&self, version: u64, key: &Q) -> bool 260 | where 261 | Q: ?Sized + Comparable>; 262 | 263 | /// Returns an iterator over the memtable. 264 | fn iter(&self, version: u64) -> Self::Iterator<'_>; 265 | 266 | /// Returns an iterator over all the entries in the memtable. 267 | fn iter_all_versions(&self, version: u64) -> Self::IterAll<'_>; 268 | 269 | /// Returns an iterator over a subset of the memtable. 270 | fn range<'a, Q, R>(&'a self, version: u64, range: R) -> Self::Range<'a, Q, R> 271 | where 272 | R: RangeBounds + 'a, 273 | Q: ?Sized + Comparable>; 274 | 275 | /// Returns an iterator over all the entries in a subset of the memtable. 276 | fn range_all_versions<'a, Q, R>(&'a self, version: u64, range: R) -> Self::RangeAll<'a, Q, R> 277 | where 278 | R: RangeBounds + 'a, 279 | Q: ?Sized + Comparable>; 280 | } 281 | -------------------------------------------------------------------------------- /src/memtable/alternative/multiple_version.rs: -------------------------------------------------------------------------------- 1 | use core::ops::{Bound, RangeBounds}; 2 | 3 | use crate::{ 4 | memtable::{ 5 | arena::{ 6 | multiple_version::{ 7 | Entry as ArenaEntry, Iter as ArenaIter, IterAll as ArenaIterAll, Range as ArenaRange, 8 | RangeAll as ArenaRangeAll, VersionedEntry as ArenaVersionedEntry, 9 | }, 10 | MultipleVersionTable as ArenaTable, 11 | }, 12 | BaseEntry, BaseTable, MultipleVersionMemtable, VersionedMemtableEntry, 13 | }, 14 | sealed::WithVersion, 15 | types::Kind, 16 | wal::{KeyPointer, ValuePointer}, 17 | }; 18 | 19 | #[cfg(feature = "std")] 20 | use crate::memtable::linked::{ 21 | multiple_version::{ 22 | Entry as LinkedEntry, Iter as LinkedIter, IterAll as LinkedIterAll, Range as LinkedRange, 23 | RangeAll as LinkedRangeAll, VersionedEntry as LinkedVersionedEntry, 24 | }, 25 | MultipleVersionTable as LinkedTable, 26 | }; 27 | 28 | use dbutils::{ 29 | equivalent::Comparable, 30 | types::{KeyRef, Type}, 31 | }; 32 | 33 | use super::TableOptions; 34 | 35 | base_entry!( 36 | enum Entry { 37 | Arena(ArenaEntry), 38 | Linked(LinkedEntry), 39 | } 40 | ); 41 | 42 | impl<'a, K, V> VersionedMemtableEntry<'a> for Entry<'a, K, V> 43 | where 44 | K: ?Sized + Type + Ord, 45 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 46 | V: ?Sized + Type, 47 | { 48 | #[inline] 49 | fn value(&self) -> Option> { 50 | Some(*match_op!(self.value())) 51 | } 52 | 53 | #[inline] 54 | fn version(&self) -> u64 { 55 | match_op!(self.version()) 56 | } 57 | } 58 | 59 | impl WithVersion for Entry<'_, K, V> {} 60 | 61 | base_entry!( 62 | enum VersionedEntry { 63 | Arena(ArenaVersionedEntry), 64 | Linked(LinkedVersionedEntry), 65 | } 66 | ); 67 | 68 | impl<'a, K, V> VersionedMemtableEntry<'a> for VersionedEntry<'a, K, V> 69 | where 70 | K: ?Sized + Type + Ord, 71 | KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 72 | V: ?Sized + Type, 73 | { 74 | #[inline] 75 | fn value(&self) -> Option> { 76 | match_op!(self.value()).copied() 77 | } 78 | 79 | #[inline] 80 | fn version(&self) -> u64 { 81 | match_op!(self.version()) 82 | } 83 | } 84 | 85 | impl WithVersion for VersionedEntry<'_, K, V> {} 86 | 87 | iter!( 88 | enum Iter { 89 | Arena(ArenaIter), 90 | Linked(LinkedIter), 91 | } -> Entry 92 | ); 93 | 94 | range!( 95 | enum Range { 96 | Arena(ArenaRange), 97 | Linked(LinkedRange), 98 | } -> Entry 99 | ); 100 | 101 | iter!( 102 | enum IterAll { 103 | Arena(ArenaIterAll), 104 | Linked(LinkedIterAll), 105 | } -> VersionedEntry 106 | ); 107 | 108 | range!( 109 | enum RangeAll { 110 | Arena(ArenaRangeAll), 111 | Linked(LinkedRangeAll), 112 | } -> VersionedEntry 113 | ); 114 | 115 | /// A sum type for different memtable implementations. 116 | #[non_exhaustive] 117 | pub enum MultipleVersionTable { 118 | /// Arena memtable 119 | Arena(ArenaTable), 120 | /// Linked memtable 121 | #[cfg(feature = "std")] 122 | #[cfg_attr(docsrs, doc(cfg(feature = "std")))] 123 | Linked(LinkedTable), 124 | } 125 | 126 | impl BaseTable for MultipleVersionTable 127 | where 128 | K: ?Sized + Type + Ord + 'static, 129 | for<'a> K::Ref<'a>: KeyRef<'a, K>, 130 | for<'a> KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 131 | V: ?Sized + Type + 'static, 132 | { 133 | type Key = K; 134 | 135 | type Value = V; 136 | 137 | type Options = TableOptions; 138 | 139 | type Error = super::Error; 140 | 141 | type Item<'a> 142 | = Entry<'a, K, V> 143 | where 144 | Self: 'a; 145 | 146 | type Iterator<'a> 147 | = Iter<'a, K, V> 148 | where 149 | Self: 'a; 150 | 151 | type Range<'a, Q, R> 152 | = Range<'a, K, V, Q, R> 153 | where 154 | Self: 'a, 155 | R: RangeBounds + 'a, 156 | Q: ?Sized + Comparable>; 157 | 158 | #[inline] 159 | fn new(opts: Self::Options) -> Result 160 | where 161 | Self: Sized, 162 | { 163 | match_op!(new(opts)) 164 | } 165 | 166 | #[inline] 167 | fn insert( 168 | &self, 169 | version: Option, 170 | kp: KeyPointer, 171 | vp: ValuePointer, 172 | ) -> Result<(), Self::Error> 173 | where 174 | KeyPointer: Ord + 'static, 175 | { 176 | match_op!(update(self.insert(version, kp, vp))) 177 | } 178 | 179 | #[inline] 180 | fn remove(&self, version: Option, key: KeyPointer) -> Result<(), Self::Error> 181 | where 182 | KeyPointer: Ord + 'static, 183 | { 184 | match_op!(update(self.remove(version, key))) 185 | } 186 | 187 | #[inline] 188 | fn kind() -> Kind { 189 | Kind::MultipleVersion 190 | } 191 | } 192 | 193 | impl MultipleVersionMemtable for MultipleVersionTable 194 | where 195 | K: ?Sized + Type + Ord + 'static, 196 | for<'a> K::Ref<'a>: KeyRef<'a, K>, 197 | for<'a> KeyPointer: Type = KeyPointer> + KeyRef<'a, KeyPointer>, 198 | V: ?Sized + Type + 'static, 199 | { 200 | type VersionedItem<'a> 201 | = VersionedEntry<'a, K, V> 202 | where 203 | KeyPointer: 'a, 204 | Self: 'a; 205 | 206 | type IterAll<'a> 207 | = IterAll<'a, K, V> 208 | where 209 | KeyPointer: 'a, 210 | Self: 'a; 211 | 212 | type RangeAll<'a, Q, R> 213 | = RangeAll<'a, K, V, Q, R> 214 | where 215 | KeyPointer: 'a, 216 | Self: 'a, 217 | R: RangeBounds + 'a, 218 | Q: ?Sized + Comparable>; 219 | 220 | #[inline] 221 | fn maximum_version(&self) -> u64 { 222 | match_op!(self.maximum_version()) 223 | } 224 | 225 | #[inline] 226 | fn minimum_version(&self) -> u64 { 227 | match_op!(self.minimum_version()) 228 | } 229 | 230 | #[inline] 231 | fn may_contain_version(&self, version: u64) -> bool { 232 | match_op!(self.may_contain_version(version)) 233 | } 234 | 235 | #[inline] 236 | fn upper_bound(&self, version: u64, bound: Bound<&Q>) -> Option> 237 | where 238 | Q: ?Sized + Comparable>, 239 | { 240 | match_op!(self.upper_bound(version, bound).map(Item)) 241 | } 242 | 243 | fn upper_bound_versioned( 244 | &self, 245 | version: u64, 246 | bound: Bound<&Q>, 247 | ) -> Option> 248 | where 249 | Q: ?Sized + Comparable>, 250 | { 251 | match_op!(self 252 | .upper_bound_versioned(version, bound) 253 | .map(VersionedItem)) 254 | } 255 | 256 | #[inline] 257 | fn lower_bound(&self, version: u64, bound: Bound<&Q>) -> Option> 258 | where 259 | Q: ?Sized + Comparable>, 260 | { 261 | match_op!(self.lower_bound(version, bound).map(Item)) 262 | } 263 | 264 | fn lower_bound_versioned( 265 | &self, 266 | version: u64, 267 | bound: Bound<&Q>, 268 | ) -> Option> 269 | where 270 | Q: ?Sized + Comparable>, 271 | { 272 | match_op!(self 273 | .lower_bound_versioned(version, bound) 274 | .map(VersionedItem)) 275 | } 276 | 277 | #[inline] 278 | fn first(&self, version: u64) -> Option> 279 | where 280 | KeyPointer: Ord, 281 | { 282 | match_op!(self.first(version).map(Item)) 283 | } 284 | 285 | fn first_versioned(&self, version: u64) -> Option> 286 | where 287 | KeyPointer: Ord, 288 | { 289 | match_op!(self.first_versioned(version).map(VersionedItem)) 290 | } 291 | 292 | #[inline] 293 | fn last(&self, version: u64) -> Option> 294 | where 295 | KeyPointer: Ord, 296 | { 297 | match_op!(self.last(version).map(Item)) 298 | } 299 | 300 | fn last_versioned(&self, version: u64) -> Option> 301 | where 302 | KeyPointer: Ord, 303 | { 304 | match_op!(self.last_versioned(version).map(VersionedItem)) 305 | } 306 | 307 | #[inline] 308 | fn get(&self, version: u64, key: &Q) -> Option> 309 | where 310 | Q: ?Sized + Comparable>, 311 | { 312 | match_op!(self.get(version, key).map(Item)) 313 | } 314 | 315 | fn get_versioned(&self, version: u64, key: &Q) -> Option> 316 | where 317 | Q: ?Sized + Comparable>, 318 | { 319 | match_op!(self.get_versioned(version, key).map(VersionedItem)) 320 | } 321 | 322 | #[inline] 323 | fn contains(&self, version: u64, key: &Q) -> bool 324 | where 325 | Q: ?Sized + Comparable>, 326 | { 327 | match_op!(self.contains(version, key)) 328 | } 329 | 330 | fn contains_versioned(&self, version: u64, key: &Q) -> bool 331 | where 332 | Q: ?Sized + Comparable>, 333 | { 334 | match_op!(self.contains_versioned(version, key)) 335 | } 336 | 337 | #[inline] 338 | fn iter(&self, version: u64) -> Self::Iterator<'_> { 339 | match_op!(Dispatch::Iterator(self.iter(version))) 340 | } 341 | 342 | fn iter_all_versions(&self, version: u64) -> Self::IterAll<'_> { 343 | match_op!(Dispatch::IterAll(self.iter_all_versions(version))) 344 | } 345 | 346 | #[inline] 347 | fn range<'a, Q, R>(&'a self, version: u64, range: R) -> Self::Range<'a, Q, R> 348 | where 349 | R: RangeBounds + 'a, 350 | Q: ?Sized + Comparable>, 351 | { 352 | match_op!(Dispatch::Range(self.range(version, range))) 353 | } 354 | 355 | fn range_all_versions<'a, Q, R>(&'a self, version: u64, range: R) -> Self::RangeAll<'a, Q, R> 356 | where 357 | R: RangeBounds + 'a, 358 | Q: ?Sized + Comparable>, 359 | { 360 | match_op!(Dispatch::RangeAll(self.range_all_versions(version, range))) 361 | } 362 | } 363 | -------------------------------------------------------------------------------- /src/swmr/tests.rs: -------------------------------------------------------------------------------- 1 | use core::cmp; 2 | #[cfg(feature = "std")] 3 | use std::thread::spawn; 4 | 5 | use std::{ 6 | string::{String, ToString}, 7 | vec, 8 | vec::Vec, 9 | }; 10 | 11 | use base::{AlternativeTable, OrderWal, OrderWalReader}; 12 | use dbutils::{ 13 | equivalent::{Comparable, Equivalent}, 14 | leb128::{decode_u64_varint, encode_u64_varint, encoded_u64_varint_len}, 15 | types::{KeyRef, Type, TypeRef}, 16 | }; 17 | 18 | use super::*; 19 | 20 | const MB: u32 = 1024 * 1024; 21 | 22 | macro_rules! expand_unit_tests { 23 | ($prefix:literal: $wal:ty [$memtable_opts:expr]: $table:ty { $($name:ident $({ $($tt:tt)* })?), +$(,)? }) => { 24 | $( 25 | paste::paste! { 26 | #[test] 27 | fn [< test_ $prefix _ $name _inmemory >]() { 28 | $name(&mut $crate::Builder::<$table>::new() 29 | .with_capacity(MB) 30 | .with_memtable_options($memtable_opts) 31 | .alloc::<$wal>() 32 | .unwrap() 33 | ); 34 | } 35 | 36 | #[test] 37 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 38 | fn [< test_ $prefix _ $name _map_anon >]() { 39 | $name(&mut $crate::Builder::<$table>::new() 40 | .with_capacity(MB) 41 | .with_memtable_options($memtable_opts) 42 | .map_anon::<$wal>().unwrap() 43 | ); 44 | } 45 | 46 | #[test] 47 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 48 | #[cfg_attr(miri, ignore)] 49 | fn [< test_ $prefix _ $name _map_file >]() { 50 | let dir = ::tempfile::tempdir().unwrap(); 51 | $name( 52 | &mut unsafe { 53 | $crate::Builder::<$table>::new() 54 | .with_create_new(true) 55 | .with_read(true) 56 | .with_write(true) 57 | .with_capacity(MB as u32) 58 | .with_memtable_options($memtable_opts) 59 | .map_mut::<$wal, _>( 60 | dir.path().join(concat!("test_", $prefix, "_", stringify!($name), "_map_file") 61 | ), 62 | ) 63 | .unwrap() }, 64 | ); 65 | } 66 | } 67 | )* 68 | }; 69 | (move $prefix:literal: $wal:ty [$memtable_opts:expr]: $table:ty { $($name:ident $($block:expr)?), +$(,)? }) => { 70 | $( 71 | paste::paste! { 72 | #[test] 73 | fn [< test_ $prefix _ $name _inmemory >]() { 74 | $name($crate::Builder::<$table>::new().with_memtable_options($memtable_opts).with_capacity(MB).alloc::<$wal>().unwrap()); 75 | } 76 | 77 | #[test] 78 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 79 | fn [< test_ $prefix _ $name _map_anon >]() { 80 | $name($crate::Builder::<$table>::new().with_memtable_options($memtable_opts).with_capacity(MB).map_anon::<$wal>().unwrap()); 81 | } 82 | 83 | #[test] 84 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 85 | #[cfg_attr(miri, ignore)] 86 | fn [< test_ $prefix _ $name _map_file >]() { 87 | let dir = ::tempfile::tempdir().unwrap(); 88 | let p = dir.path().join(concat!("test_", $prefix, "_", stringify!($name), "_map_file")); 89 | let wal = unsafe { 90 | $crate::Builder::<$table>::new() 91 | .with_memtable_options($memtable_opts) 92 | .with_create_new(true) 93 | .with_read(true) 94 | .with_write(true) 95 | .with_capacity(MB as u32) 96 | .map_mut::<$wal, _>( 97 | &p, 98 | ) 99 | .unwrap() 100 | }; 101 | 102 | let res = $name(wal); 103 | 104 | $( 105 | { 106 | let f = |p, res| { $block(p, res) }; 107 | f(p, res); 108 | } 109 | )? 110 | } 111 | } 112 | )* 113 | }; 114 | ($prefix:literal: $wal:ty [$memtable_opts:expr]: $table:ty { $($name:ident($builder:expr) $({ $($tt:tt)* })?), +$(,)? }) => { 115 | $( 116 | paste::paste! { 117 | #[test] 118 | fn [< test_ $prefix _ $name _inmemory >]() { 119 | $name(&mut $builder.alloc::<$wal>().unwrap()); 120 | } 121 | 122 | #[test] 123 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 124 | fn [< test_ $prefix _ $name _map_anon >]() { 125 | $name(&mut $builder 126 | .map_anon::<$wal>().unwrap() 127 | ); 128 | } 129 | 130 | #[test] 131 | #[cfg_attr(miri, ignore)] 132 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 133 | fn [< test_ $prefix _ $name _map_file >]() { 134 | let dir = ::tempfile::tempdir().unwrap(); 135 | $name( 136 | &mut unsafe { 137 | $builder 138 | .with_create_new(true) 139 | .with_read(true) 140 | .with_write(true) 141 | .with_capacity(MB as u32) 142 | .map_mut::<$wal, _>( 143 | dir.path().join(concat!("test_", $prefix, "_", stringify!($name), "_map_file") 144 | ), 145 | ) 146 | .unwrap() }, 147 | ); 148 | } 149 | } 150 | )* 151 | }; 152 | } 153 | 154 | type OrderWalAlternativeTable = OrderWal>; 155 | type OrderWalReaderAlternativeTable = OrderWalReader>; 156 | 157 | type MultipleVersionOrderWalAlternativeTable = 158 | multiple_version::OrderWal>; 159 | type MultipleVersionOrderWalReaderAlternativeTable = 160 | multiple_version::OrderWalReader>; 161 | 162 | #[doc(hidden)] 163 | #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] 164 | pub struct Person { 165 | #[doc(hidden)] 166 | pub id: u64, 167 | #[doc(hidden)] 168 | pub name: String, 169 | } 170 | 171 | impl Person { 172 | #[doc(hidden)] 173 | #[cfg(test)] 174 | pub fn random() -> Self { 175 | Self { 176 | id: rand::random(), 177 | name: names::Generator::default().next().unwrap(), 178 | } 179 | } 180 | 181 | #[doc(hidden)] 182 | pub fn as_ref(&self) -> PersonRef<'_> { 183 | PersonRef { 184 | id: self.id, 185 | name: &self.name, 186 | } 187 | } 188 | 189 | #[doc(hidden)] 190 | #[cfg(test)] 191 | #[allow(dead_code)] 192 | fn to_vec(&self) -> Vec { 193 | let mut buf = vec![0; self.encoded_len()]; 194 | self.encode(&mut buf).unwrap(); 195 | buf 196 | } 197 | } 198 | 199 | #[doc(hidden)] 200 | #[derive(Debug, Clone, Copy)] 201 | pub struct PersonRef<'a> { 202 | id: u64, 203 | name: &'a str, 204 | } 205 | 206 | impl PartialEq for PersonRef<'_> { 207 | fn eq(&self, other: &Self) -> bool { 208 | self.id == other.id && self.name == other.name 209 | } 210 | } 211 | 212 | impl Eq for PersonRef<'_> {} 213 | 214 | impl PartialOrd for PersonRef<'_> { 215 | fn partial_cmp(&self, other: &Self) -> Option { 216 | Some(self.cmp(other)) 217 | } 218 | } 219 | 220 | impl Ord for PersonRef<'_> { 221 | fn cmp(&self, other: &Self) -> cmp::Ordering { 222 | self 223 | .id 224 | .cmp(&other.id) 225 | .then_with(|| self.name.cmp(other.name)) 226 | } 227 | } 228 | 229 | impl Equivalent for PersonRef<'_> { 230 | fn equivalent(&self, key: &Person) -> bool { 231 | self.id == key.id && self.name == key.name 232 | } 233 | } 234 | 235 | impl Comparable for PersonRef<'_> { 236 | fn compare(&self, key: &Person) -> core::cmp::Ordering { 237 | self.id.cmp(&key.id).then_with(|| self.name.cmp(&key.name)) 238 | } 239 | } 240 | 241 | impl Equivalent> for Person { 242 | fn equivalent(&self, key: &PersonRef<'_>) -> bool { 243 | self.id == key.id && self.name == key.name 244 | } 245 | } 246 | 247 | impl Comparable> for Person { 248 | fn compare(&self, key: &PersonRef<'_>) -> core::cmp::Ordering { 249 | self 250 | .id 251 | .cmp(&key.id) 252 | .then_with(|| self.name.as_str().cmp(key.name)) 253 | } 254 | } 255 | 256 | impl KeyRef<'_, Person> for PersonRef<'_> { 257 | fn compare(&self, a: &Q) -> cmp::Ordering 258 | where 259 | Q: ?Sized + Comparable, 260 | { 261 | Comparable::compare(a, self).reverse() 262 | } 263 | 264 | unsafe fn compare_binary(this: &[u8], other: &[u8]) -> cmp::Ordering { 265 | let (this_id_size, this_id) = decode_u64_varint(this).unwrap(); 266 | let (other_id_size, other_id) = decode_u64_varint(other).unwrap(); 267 | PersonRef { 268 | id: this_id, 269 | name: std::str::from_utf8(&this[this_id_size..]).unwrap(), 270 | } 271 | .cmp(&PersonRef { 272 | id: other_id, 273 | name: std::str::from_utf8(&other[other_id_size..]).unwrap(), 274 | }) 275 | } 276 | } 277 | 278 | impl Type for Person { 279 | type Ref<'a> = PersonRef<'a>; 280 | type Error = dbutils::error::InsufficientBuffer; 281 | 282 | fn encoded_len(&self) -> usize { 283 | encoded_u64_varint_len(self.id) + self.name.len() 284 | } 285 | 286 | fn encode(&self, buf: &mut [u8]) -> Result { 287 | let id_size = encode_u64_varint(self.id, buf)?; 288 | buf[id_size..].copy_from_slice(self.name.as_bytes()); 289 | Ok(id_size + self.name.len()) 290 | } 291 | 292 | #[inline] 293 | fn encode_to_buffer( 294 | &self, 295 | buf: &mut dbutils::buffer::VacantBuffer<'_>, 296 | ) -> Result { 297 | let id_size = buf.put_u64_varint(self.id)?; 298 | buf.put_slice_unchecked(self.name.as_bytes()); 299 | Ok(id_size + self.name.len()) 300 | } 301 | } 302 | 303 | impl<'a> TypeRef<'a> for PersonRef<'a> { 304 | unsafe fn from_slice(src: &'a [u8]) -> Self { 305 | let (id_size, id) = decode_u64_varint(src).unwrap(); 306 | let name = std::str::from_utf8(&src[id_size..]).unwrap(); 307 | PersonRef { id, name } 308 | } 309 | } 310 | 311 | impl PersonRef<'_> { 312 | #[cfg(test)] 313 | #[allow(dead_code)] 314 | fn encode_into_vec(&self) -> Result, dbutils::error::InsufficientBuffer> { 315 | let mut buf = vec![0; encoded_u64_varint_len(self.id) + self.name.len()]; 316 | let id_size = encode_u64_varint(self.id, &mut buf)?; 317 | buf[id_size..].copy_from_slice(self.name.as_bytes()); 318 | Ok(buf) 319 | } 320 | } 321 | 322 | #[cfg(all(test, any(test_swmr_constructor, all_orderwal_tests)))] 323 | mod constructor; 324 | 325 | #[cfg(all(test, any(test_swmr_insert, all_orderwal_tests)))] 326 | mod insert; 327 | 328 | #[cfg(all(test, any(test_swmr_iters, all_orderwal_tests)))] 329 | mod iters; 330 | 331 | #[cfg(all(test, any(test_swmr_get, all_orderwal_tests)))] 332 | mod get; 333 | 334 | #[cfg(all(test, any(test_swmr_multiple_version_constructor, all_orderwal_tests)))] 335 | mod multiple_version_constructor; 336 | 337 | #[cfg(all(test, any(test_swmr_multiple_version_get, all_orderwal_tests)))] 338 | mod multiple_version_get; 339 | 340 | #[cfg(all(test, any(test_swmr_multiple_version_insert, all_orderwal_tests)))] 341 | mod multiple_version_insert; 342 | 343 | #[cfg(all(test, any(test_swmr_multiple_version_iters, all_orderwal_tests)))] 344 | mod multiple_version_iters; 345 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /src/builder.rs: -------------------------------------------------------------------------------- 1 | use dbutils::checksum::Crc32; 2 | use skl::KeySize; 3 | 4 | use super::{ 5 | error::Error, 6 | memtable::BaseTable, 7 | options::{arena_options, Options}, 8 | sealed::Constructable, 9 | }; 10 | 11 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 12 | #[cfg_attr(docsrs, doc(cfg(all(feature = "memmap", not(target_family = "wasm")))))] 13 | mod memmap; 14 | 15 | /// A write-ahead log builder. 16 | pub struct Builder 17 | where 18 | M: BaseTable, 19 | { 20 | pub(super) opts: Options, 21 | pub(super) cks: S, 22 | pub(super) memtable_opts: M::Options, 23 | } 24 | 25 | impl Default for Builder 26 | where 27 | M: BaseTable, 28 | M::Options: Default, 29 | { 30 | #[inline] 31 | fn default() -> Self { 32 | Self::new() 33 | } 34 | } 35 | 36 | impl Builder 37 | where 38 | M: BaseTable, 39 | M::Options: Default, 40 | { 41 | /// Returns a new write-ahead log builder with the given options. 42 | #[inline] 43 | pub fn new() -> Self { 44 | Self { 45 | opts: Options::default(), 46 | cks: Crc32::default(), 47 | memtable_opts: M::Options::default(), 48 | } 49 | } 50 | } 51 | 52 | impl Builder 53 | where 54 | M: BaseTable, 55 | { 56 | /// Returns a new write-ahead log builder with the new checksumer 57 | /// 58 | /// ## Example 59 | /// 60 | /// ```rust 61 | /// use orderwal::{Builder, Crc32, multiple_version::DefaultTable}; 62 | /// 63 | /// let opts = Builder::>::new().with_checksumer(Crc32::new()); 64 | /// ``` 65 | #[inline] 66 | pub fn with_checksumer(self, cks: NS) -> Builder { 67 | Builder { 68 | opts: self.opts, 69 | cks, 70 | memtable_opts: self.memtable_opts, 71 | } 72 | } 73 | 74 | /// Returns a new write-ahead log builder with the new options 75 | /// 76 | /// ## Example 77 | /// 78 | /// ```rust 79 | /// use orderwal::{Builder, Options, multiple_version::DefaultTable}; 80 | /// 81 | /// let opts = Builder::>::new().with_options(Options::default()); 82 | /// ``` 83 | #[inline] 84 | pub fn with_options(self, opts: Options) -> Self { 85 | Self { 86 | opts, 87 | cks: self.cks, 88 | memtable_opts: self.memtable_opts, 89 | } 90 | } 91 | 92 | /// Returns a new write-ahead log builder with the new options 93 | /// 94 | /// ## Example 95 | /// 96 | /// ```rust 97 | /// use orderwal::{Builder, multiple_version::{ArenaTable, ArenaTableOptions}}; 98 | /// 99 | /// let opts = Builder::>::new().with_memtable_options(ArenaTableOptions::default()); 100 | /// ``` 101 | #[inline] 102 | pub fn with_memtable_options(self, opts: M::Options) -> Self { 103 | Self { 104 | opts: self.opts, 105 | cks: self.cks, 106 | memtable_opts: opts, 107 | } 108 | } 109 | 110 | /// Returns a new write-ahead log builder with the new memtable. 111 | /// 112 | /// ## Example 113 | /// 114 | /// ```rust 115 | /// use orderwal::{Builder, multiple_version::{DefaultTable, ArenaTable}}; 116 | /// 117 | /// let opts = Builder::>::new().change_memtable::>(); 118 | /// ``` 119 | #[inline] 120 | pub fn change_memtable(self) -> Builder 121 | where 122 | NM: BaseTable, 123 | NM::Options: Default, 124 | { 125 | Builder { 126 | opts: self.opts, 127 | cks: self.cks, 128 | memtable_opts: NM::Options::default(), 129 | } 130 | } 131 | 132 | /// Returns a new write-ahead log builder with the new memtable and its options 133 | /// 134 | /// ## Example 135 | /// 136 | /// ```rust 137 | /// use orderwal::{Builder, multiple_version::{DefaultTable, ArenaTable, ArenaTableOptions}}; 138 | /// 139 | /// let opts = Builder::>::new().change_memtable_with_options::>(ArenaTableOptions::default().with_capacity(1000)); 140 | /// ``` 141 | #[inline] 142 | pub fn change_memtable_with_options(self, opts: NM::Options) -> Builder 143 | where 144 | NM: BaseTable, 145 | { 146 | Builder { 147 | opts: self.opts, 148 | cks: self.cks, 149 | memtable_opts: opts, 150 | } 151 | } 152 | 153 | /// Set the reserved bytes of the WAL. 154 | /// 155 | /// The `reserved` is used to configure the start position of the WAL. This is useful 156 | /// when you want to add some bytes as your own WAL's header. 157 | /// 158 | /// The default reserved is `0`. 159 | /// 160 | /// ## Example 161 | /// 162 | /// ```rust 163 | /// use orderwal::{Builder, multiple_version::DefaultTable}; 164 | /// 165 | /// let opts = Builder::>::new().with_reserved(8); 166 | /// ``` 167 | #[inline] 168 | pub const fn with_reserved(mut self, reserved: u32) -> Self { 169 | self.opts = self.opts.with_reserved(reserved); 170 | self 171 | } 172 | 173 | /// Get the reserved of the WAL. 174 | /// 175 | /// The `reserved` is used to configure the start position of the WAL. This is useful 176 | /// when you want to add some bytes as your own WAL's header. 177 | /// 178 | /// The default reserved is `0`. 179 | /// 180 | /// ## Example 181 | /// 182 | /// ```rust 183 | /// use orderwal::{Builder, multiple_version::DefaultTable}; 184 | /// 185 | /// let opts = Builder::>::new().with_reserved(8); 186 | /// 187 | /// assert_eq!(opts.reserved(), 8); 188 | /// ``` 189 | #[inline] 190 | pub const fn reserved(&self) -> u32 { 191 | self.opts.reserved() 192 | } 193 | 194 | /// Returns the magic version. 195 | /// 196 | /// The default value is `0`. 197 | /// 198 | /// ## Example 199 | /// 200 | /// ```rust 201 | /// use orderwal::{Builder, multiple_version::DefaultTable}; 202 | /// 203 | /// let options = Builder::>::new().with_magic_version(1); 204 | /// assert_eq!(options.magic_version(), 1); 205 | /// ``` 206 | #[inline] 207 | pub const fn magic_version(&self) -> u16 { 208 | self.opts.magic_version() 209 | } 210 | 211 | /// Returns the capacity of the WAL. 212 | /// 213 | /// The default value is `0`. 214 | /// 215 | /// ## Example 216 | /// 217 | /// ```rust 218 | /// use orderwal::{Builder, multiple_version::DefaultTable}; 219 | /// 220 | /// let options = Builder::>::new().with_capacity(1000); 221 | /// assert_eq!(options.capacity(), 1000); 222 | /// ``` 223 | #[inline] 224 | pub const fn capacity(&self) -> u32 { 225 | self.opts.capacity() 226 | } 227 | 228 | /// Returns the maximum key length. 229 | /// 230 | /// The default value is `u16::MAX`. 231 | /// 232 | /// ## Example 233 | /// 234 | /// ```rust 235 | /// use orderwal::{Builder, KeySize, multiple_version::DefaultTable}; 236 | /// 237 | /// let options = Builder::>::new().with_maximum_key_size(KeySize::with(1024)); 238 | /// assert_eq!(options.maximum_key_size(), KeySize::with(1024)); 239 | /// ``` 240 | #[inline] 241 | pub const fn maximum_key_size(&self) -> KeySize { 242 | self.opts.maximum_key_size() 243 | } 244 | 245 | /// Returns the maximum value length. 246 | /// 247 | /// The default value is `u32::MAX`. 248 | /// 249 | /// ## Example 250 | /// 251 | /// ```rust 252 | /// use orderwal::{Builder, multiple_version::DefaultTable}; 253 | /// 254 | /// let options = Builder::>::new().with_maximum_value_size(1024); 255 | /// assert_eq!(options.maximum_value_size(), 1024); 256 | /// ``` 257 | #[inline] 258 | pub const fn maximum_value_size(&self) -> u32 { 259 | self.opts.maximum_value_size() 260 | } 261 | 262 | /// Returns `true` if the WAL syncs on write. 263 | /// 264 | /// The default value is `true`. 265 | /// 266 | /// ## Example 267 | /// 268 | /// ```rust 269 | /// use orderwal::{Builder, multiple_version::DefaultTable}; 270 | /// 271 | /// let options = Builder::>::new(); 272 | /// assert_eq!(options.sync(), true); 273 | /// ``` 274 | #[inline] 275 | pub const fn sync(&self) -> bool { 276 | self.opts.sync() 277 | } 278 | 279 | /// Sets the capacity of the WAL. 280 | /// 281 | /// This configuration will be ignored when using file-backed memory maps. 282 | /// 283 | /// The default value is `0`. 284 | /// 285 | /// ## Example 286 | /// 287 | /// ```rust 288 | /// use orderwal::{Builder, multiple_version::DefaultTable}; 289 | /// 290 | /// let options = Builder::>::new().with_capacity(100); 291 | /// assert_eq!(options.capacity(), 100); 292 | /// ``` 293 | #[inline] 294 | pub const fn with_capacity(mut self, cap: u32) -> Self { 295 | self.opts = self.opts.with_capacity(cap); 296 | self 297 | } 298 | 299 | /// Sets the maximum key length. 300 | /// 301 | /// ## Example 302 | /// 303 | /// ```rust 304 | /// use orderwal::{Builder, KeySize, multiple_version::DefaultTable}; 305 | /// 306 | /// let options = Builder::>::new().with_maximum_key_size(KeySize::with(1024)); 307 | /// assert_eq!(options.maximum_key_size(), KeySize::with(1024)); 308 | /// ``` 309 | #[inline] 310 | pub const fn with_maximum_key_size(mut self, size: KeySize) -> Self { 311 | self.opts = self.opts.with_maximum_key_size(size); 312 | self 313 | } 314 | 315 | /// Sets the maximum value length. 316 | /// 317 | /// ## Example 318 | /// 319 | /// ```rust 320 | /// use orderwal::{Builder, multiple_version::DefaultTable}; 321 | /// 322 | /// let options = Builder::>::new().with_maximum_value_size(1024); 323 | /// assert_eq!(options.maximum_value_size(), 1024); 324 | /// ``` 325 | #[inline] 326 | pub const fn with_maximum_value_size(mut self, size: u32) -> Self { 327 | self.opts = self.opts.with_maximum_value_size(size); 328 | self 329 | } 330 | 331 | /// Sets the WAL to sync on write. 332 | /// 333 | /// The default value is `true`. 334 | /// 335 | /// ## Example 336 | /// 337 | /// ```rust 338 | /// use orderwal::{Builder, multiple_version::DefaultTable}; 339 | /// 340 | /// let options = Builder::>::new().with_sync(false); 341 | /// assert_eq!(options.sync(), false); 342 | /// ``` 343 | #[inline] 344 | pub const fn with_sync(mut self, sync: bool) -> Self { 345 | self.opts = self.opts.with_sync(sync); 346 | self 347 | } 348 | 349 | /// Sets the magic version. 350 | /// 351 | /// The default value is `0`. 352 | /// 353 | /// ## Example 354 | /// 355 | /// ```rust 356 | /// 357 | /// use orderwal::{Builder, multiple_version::DefaultTable}; 358 | /// 359 | /// let options = Builder::>::new().with_magic_version(1); 360 | /// assert_eq!(options.magic_version(), 1); 361 | /// ``` 362 | #[inline] 363 | pub const fn with_magic_version(mut self, version: u16) -> Self { 364 | self.opts = self.opts.with_magic_version(version); 365 | self 366 | } 367 | } 368 | 369 | impl Builder 370 | where 371 | M: BaseTable, 372 | { 373 | /// Creates a new in-memory write-ahead log backed by an aligned vec. 374 | /// 375 | /// ## Example 376 | /// 377 | /// ```rust 378 | /// 379 | /// use orderwal::{base::OrderWal, Builder}; 380 | /// 381 | /// let wal = Builder::new() 382 | /// .with_capacity(1024) 383 | /// .alloc::>() 384 | /// .unwrap(); 385 | /// ``` 386 | pub fn alloc(self) -> Result> 387 | where 388 | W: Constructable, 389 | { 390 | let Self { 391 | opts, 392 | cks, 393 | memtable_opts, 394 | } = self; 395 | arena_options(opts.reserved()) 396 | .with_capacity(opts.capacity()) 397 | .alloc() 398 | .map_err(Error::from_insufficient_space) 399 | .and_then(|arena| W::new_in(arena, opts, memtable_opts, cks).map(W::from_core)) 400 | } 401 | } 402 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | use among::Among; 2 | use dbutils::error::InsufficientBuffer; 3 | use derive_where::derive_where; 4 | 5 | use crate::memtable::BaseTable; 6 | 7 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 8 | use crate::types::Kind; 9 | 10 | /// The batch error type. 11 | #[derive(Debug)] 12 | pub enum BatchError { 13 | /// Returned when the expected batch encoding size does not match the actual size. 14 | EncodedSizeMismatch { 15 | /// The expected size. 16 | expected: u32, 17 | /// The actual size. 18 | actual: u32, 19 | }, 20 | /// Larger encoding size than the expected batch encoding size. 21 | LargerEncodedSize(u32), 22 | } 23 | 24 | impl core::fmt::Display for BatchError { 25 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 26 | match self { 27 | Self::EncodedSizeMismatch { expected, actual } => { 28 | write!( 29 | f, 30 | "the expected batch encoding size ({}) does not match the actual size {}", 31 | expected, actual 32 | ) 33 | } 34 | Self::LargerEncodedSize(size) => { 35 | write!( 36 | f, 37 | "larger encoding size than the expected batch encoding size {}", 38 | size 39 | ) 40 | } 41 | } 42 | } 43 | } 44 | 45 | impl core::error::Error for BatchError {} 46 | 47 | /// The error type. 48 | #[derive_where(Debug; T::Error)] 49 | pub enum Error { 50 | /// Insufficient space in the WAL 51 | InsufficientSpace(InsufficientBuffer), 52 | /// Memtable does not have enough space. 53 | Memtable(T::Error), 54 | /// The key is too large. 55 | KeyTooLarge { 56 | /// The size of the key. 57 | size: u64, 58 | /// The maximum key size. 59 | maximum_key_size: u32, 60 | }, 61 | /// The value is too large. 62 | ValueTooLarge { 63 | /// The size of the value. 64 | size: u64, 65 | /// The maximum value size. 66 | maximum_value_size: u32, 67 | }, 68 | /// The entry is too large. 69 | EntryTooLarge { 70 | /// The size of the entry. 71 | size: u64, 72 | /// The maximum entry size. 73 | maximum_entry_size: u64, 74 | }, 75 | 76 | /// Returned when the expected batch encoding size does not match the actual size. 77 | Batch(BatchError), 78 | 79 | /// The WAL is read-only. 80 | ReadOnly, 81 | 82 | /// Unknown WAL kind. 83 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 84 | #[cfg_attr(docsrs, doc(cfg(all(feature = "memmap", not(target_family = "wasm")))))] 85 | UnknownKind(UnknownKind), 86 | 87 | /// WAL kind mismatch. 88 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 89 | #[cfg_attr(docsrs, doc(cfg(all(feature = "memmap", not(target_family = "wasm")))))] 90 | KindMismatch { 91 | /// The WAL was created with this kind. 92 | create: Kind, 93 | /// Trying to open the WAL with this kind. 94 | open: Kind, 95 | }, 96 | 97 | /// I/O error. 98 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 99 | #[cfg_attr(docsrs, doc(cfg(all(feature = "memmap", not(target_family = "wasm")))))] 100 | IO(std::io::Error), 101 | } 102 | 103 | impl From for Error { 104 | #[inline] 105 | fn from(e: BatchError) -> Self { 106 | Self::Batch(e) 107 | } 108 | } 109 | 110 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 111 | impl From for Error { 112 | #[inline] 113 | fn from(e: UnknownKind) -> Self { 114 | Self::UnknownKind(e) 115 | } 116 | } 117 | 118 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 119 | impl From for Error { 120 | #[inline] 121 | fn from(e: std::io::Error) -> Self { 122 | Self::IO(e) 123 | } 124 | } 125 | 126 | impl core::fmt::Display for Error 127 | where 128 | T: BaseTable, 129 | T::Error: core::fmt::Display, 130 | { 131 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 132 | match self { 133 | Self::InsufficientSpace(e) => write!(f, "insufficient space in the WAL: {e}"), 134 | Self::Memtable(e) => write!(f, "{e}"), 135 | Self::KeyTooLarge { 136 | size, 137 | maximum_key_size, 138 | } => write!( 139 | f, 140 | "the key size is {} larger than the maximum key size {}", 141 | size, maximum_key_size 142 | ), 143 | Self::ValueTooLarge { 144 | size, 145 | maximum_value_size, 146 | } => write!( 147 | f, 148 | "the value size is {} larger than the maximum value size {}", 149 | size, maximum_value_size 150 | ), 151 | Self::EntryTooLarge { 152 | size, 153 | maximum_entry_size, 154 | } => write!( 155 | f, 156 | "the entry size is {} larger than the maximum entry size {}", 157 | size, maximum_entry_size 158 | ), 159 | Self::Batch(e) => write!(f, "{e}"), 160 | Self::ReadOnly => write!(f, "The WAL is read-only"), 161 | 162 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 163 | Self::UnknownKind(e) => write!(f, "{e}"), 164 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 165 | Self::KindMismatch { create, open } => write!( 166 | f, 167 | "the wal was {}, cannot be {}", 168 | create.display_created_err_msg(), 169 | open.display_open_err_msg() 170 | ), 171 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 172 | Self::IO(e) => write!(f, "{e}"), 173 | } 174 | } 175 | } 176 | 177 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 178 | impl Kind { 179 | #[inline] 180 | const fn display_created_err_msg(&self) -> &'static str { 181 | match self { 182 | Self::Plain => "created without multiple versions support", 183 | Self::MultipleVersion => "created with multiple versions support", 184 | } 185 | } 186 | 187 | #[inline] 188 | const fn display_open_err_msg(&self) -> &'static str { 189 | match self { 190 | Self::Plain => "opened without multiple versions support", 191 | Self::MultipleVersion => "opened with multiple versions support", 192 | } 193 | } 194 | } 195 | 196 | impl core::error::Error for Error 197 | where 198 | T: BaseTable, 199 | T::Error: core::error::Error + 'static, 200 | { 201 | fn source(&self) -> Option<&(dyn core::error::Error + 'static)> { 202 | match self { 203 | Self::InsufficientSpace(e) => Some(e), 204 | Self::Memtable(e) => Some(e), 205 | Self::KeyTooLarge { .. } => None, 206 | Self::ValueTooLarge { .. } => None, 207 | Self::EntryTooLarge { .. } => None, 208 | Self::Batch(e) => Some(e), 209 | Self::ReadOnly => None, 210 | 211 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 212 | Self::UnknownKind(e) => Some(e), 213 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 214 | Self::KindMismatch { .. } => None, 215 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 216 | Self::IO(e) => Some(e), 217 | } 218 | } 219 | } 220 | 221 | impl From>> for Error { 222 | #[inline] 223 | fn from(value: Among>) -> Self { 224 | match value { 225 | Among::Left(a) => Self::InsufficientSpace(a), 226 | Among::Middle(b) => Self::InsufficientSpace(b), 227 | Among::Right(c) => c, 228 | } 229 | } 230 | } 231 | 232 | impl Error { 233 | /// Create a new `Error::InsufficientSpace` instance. 234 | #[inline] 235 | pub(crate) const fn insufficient_space(requested: u64, available: u32) -> Self { 236 | Self::InsufficientSpace(InsufficientBuffer::with_information( 237 | requested, 238 | available as u64, 239 | )) 240 | } 241 | 242 | /// Create a new `Error::MemtableInsufficientSpace` instance. 243 | #[inline] 244 | pub(crate) const fn memtable(e: T::Error) -> Self { 245 | Self::Memtable(e) 246 | } 247 | 248 | /// Create a new `Error::KeyTooLarge` instance. 249 | #[inline] 250 | pub(crate) const fn key_too_large(size: u64, maximum_key_size: u32) -> Self { 251 | Self::KeyTooLarge { 252 | size, 253 | maximum_key_size, 254 | } 255 | } 256 | 257 | /// Create a new `Error::ValueTooLarge` instance. 258 | #[inline] 259 | pub(crate) const fn value_too_large(size: u64, maximum_value_size: u32) -> Self { 260 | Self::ValueTooLarge { 261 | size, 262 | maximum_value_size, 263 | } 264 | } 265 | 266 | /// Create a new `Error::EntryTooLarge` instance. 267 | #[inline] 268 | pub(crate) const fn entry_too_large(size: u64, maximum_entry_size: u64) -> Self { 269 | Self::EntryTooLarge { 270 | size, 271 | maximum_entry_size, 272 | } 273 | } 274 | 275 | #[inline] 276 | pub(crate) const fn from_insufficient_space(error: rarena_allocator::Error) -> Self { 277 | match error { 278 | rarena_allocator::Error::InsufficientSpace { 279 | requested, 280 | available, 281 | } => Self::insufficient_space(requested as u64, available), 282 | _ => unreachable!(), 283 | } 284 | } 285 | 286 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 287 | #[inline] 288 | pub(crate) const fn wal_kind_mismatch(create: Kind, open: Kind) -> Self { 289 | Self::KindMismatch { create, open } 290 | } 291 | 292 | /// Create a new corrupted error. 293 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 294 | #[inline] 295 | pub(crate) fn corrupted(e: E) -> Self 296 | where 297 | E: Into>, 298 | { 299 | #[derive(Debug)] 300 | struct Corrupted(Box); 301 | 302 | impl std::fmt::Display for Corrupted { 303 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 304 | write!(f, "corrupted write-ahead log: {}", self.0) 305 | } 306 | } 307 | 308 | impl std::error::Error for Corrupted {} 309 | 310 | Self::IO(std::io::Error::new( 311 | std::io::ErrorKind::InvalidData, 312 | Corrupted(e.into()), 313 | )) 314 | } 315 | 316 | /// Create a new batch size mismatch error. 317 | #[inline] 318 | pub(crate) const fn batch_size_mismatch(expected: u32, actual: u32) -> Self { 319 | Self::Batch(BatchError::EncodedSizeMismatch { expected, actual }) 320 | } 321 | 322 | /// Create a new larger batch size error. 323 | #[inline] 324 | pub(crate) const fn larger_batch_size(size: u32) -> Self { 325 | Self::Batch(BatchError::LargerEncodedSize(size)) 326 | } 327 | 328 | /// Create a read-only error. 329 | #[inline] 330 | pub(crate) const fn read_only() -> Self { 331 | Self::ReadOnly 332 | } 333 | 334 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 335 | #[inline] 336 | pub(crate) fn magic_text_mismatch() -> Self { 337 | Self::IO(std::io::Error::new( 338 | std::io::ErrorKind::InvalidData, 339 | "magic text of orderwal does not match", 340 | )) 341 | } 342 | 343 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 344 | #[inline] 345 | pub(crate) fn magic_version_mismatch() -> Self { 346 | Self::IO(std::io::Error::new( 347 | std::io::ErrorKind::InvalidData, 348 | "magic version of orderwal does not match", 349 | )) 350 | } 351 | } 352 | 353 | /// Unknown WAL kind error. 354 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 355 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 356 | #[cfg_attr(docsrs, doc(cfg(all(feature = "memmap", not(target_family = "wasm")))))] 357 | pub struct UnknownKind(pub(super) u8); 358 | 359 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 360 | impl core::fmt::Display for UnknownKind { 361 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 362 | write!(f, "unknown WAL kind: {}", self.0) 363 | } 364 | } 365 | 366 | #[cfg(all(feature = "memmap", not(target_family = "wasm")))] 367 | impl core::error::Error for UnknownKind {} 368 | --------------------------------------------------------------------------------