├── .github └── workflows │ └── rust.yml ├── .gitignore ├── .gitmodules ├── .rustfmt.toml ├── Cargo.toml ├── LICENSE ├── README.md ├── assets ├── heed-pigeon-logo.png └── heed-pigeon.ico ├── convert-to-heed3.sh ├── heed-traits ├── Cargo.toml └── src │ └── lib.rs ├── heed-types ├── Cargo.toml └── src │ ├── bytes.rs │ ├── decode_ignore.rs │ ├── integer.rs │ ├── lazy_decode.rs │ ├── lib.rs │ ├── serde_bincode.rs │ ├── serde_json.rs │ ├── serde_rmp.rs │ ├── str.rs │ └── unit.rs ├── heed ├── Cargo.toml ├── build.rs ├── examples │ ├── all-types.rs │ ├── clear-database.rs │ ├── cursor-append.rs │ ├── custom-comparator.rs │ ├── custom-dupsort-comparator.rs │ ├── multi-env.rs │ ├── nested.rs │ ├── prev-snapshot.rs │ └── rmp-serde.rs └── src │ ├── cookbook.rs │ ├── cursor.rs │ ├── databases │ ├── database.rs │ ├── encrypted_database.rs │ └── mod.rs │ ├── envs │ ├── encrypted_env.rs │ ├── env.rs │ ├── env_open_options.rs │ └── mod.rs │ ├── iteration_method.rs │ ├── iterator │ ├── iter.rs │ ├── mod.rs │ ├── prefix.rs │ └── range.rs │ ├── lib.rs │ ├── mdb │ ├── lmdb_error.rs │ ├── lmdb_ffi.rs │ ├── lmdb_flags.rs │ └── mod.rs │ ├── reserved_space.rs │ └── txn.rs ├── heed3 ├── Cargo.toml └── examples │ ├── heed3-all-types.rs │ ├── heed3-encrypted.rs │ └── prev-snapshot.rs ├── lmdb-master-sys ├── .rustfmt.toml ├── Cargo.toml ├── README.md ├── bindgen.rs ├── build.rs ├── src │ ├── bindings.rs │ └── lib.rs └── tests │ ├── fixtures │ └── testdb-32 │ │ ├── data.mdb │ │ └── lock.mdb │ ├── lmdb.rs │ └── simple.rs └── lmdb-master3-sys ├── .rustfmt.toml ├── Cargo.toml ├── README.md ├── bindgen.rs ├── build.rs └── src ├── bindings.rs └── lib.rs /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | merge_group: 4 | 5 | name: Rust 6 | jobs: 7 | heed-test: 8 | name: Test the heed project 9 | runs-on: ${{ matrix.os }} 10 | strategy: 11 | matrix: 12 | os: [ubuntu-latest, windows-latest, macos-latest] 13 | include: 14 | - os: ubuntu-latest 15 | - os: windows-latest 16 | - os: macos-latest 17 | steps: 18 | - uses: actions/checkout@v2 19 | with: 20 | submodules: recursive 21 | - uses: actions-rs/toolchain@v1 22 | with: 23 | profile: minimal 24 | toolchain: stable 25 | override: true 26 | - name: Run cargo test 27 | run: | 28 | cargo clean 29 | cargo test 30 | 31 | heed3-test: 32 | name: Test the heed3 project 33 | runs-on: ${{ matrix.os }} 34 | strategy: 35 | matrix: 36 | os: [ubuntu-latest, windows-latest, macos-latest] 37 | include: 38 | - os: ubuntu-latest 39 | - os: windows-latest 40 | - os: macos-latest 41 | steps: 42 | - uses: actions/checkout@v2 43 | with: 44 | submodules: recursive 45 | - uses: actions-rs/toolchain@v1 46 | with: 47 | profile: minimal 48 | toolchain: stable 49 | override: true 50 | - name: Run cargo test 51 | run: | 52 | cargo clean 53 | bash convert-to-heed3.sh 54 | cargo test 55 | 56 | check-heed3: 57 | name: Check the heed3 project 58 | runs-on: ${{ matrix.os }} 59 | strategy: 60 | matrix: 61 | os: [ubuntu-latest, windows-latest, macos-latest] 62 | include: 63 | - os: ubuntu-latest 64 | - os: windows-latest 65 | - os: macos-latest 66 | steps: 67 | - uses: actions/checkout@v2 68 | with: 69 | submodules: recursive 70 | - uses: actions-rs/toolchain@v1 71 | with: 72 | profile: minimal 73 | toolchain: stable 74 | override: true 75 | - name: Run cargo check 76 | run: | 77 | cargo clean 78 | bash convert-to-heed3.sh 79 | cargo check -p heed3 80 | 81 | check-all-features: 82 | name: Check all the features of the heed project 83 | runs-on: ubuntu-latest 84 | env: 85 | RUSTFLAGS: -D warnings 86 | steps: 87 | - uses: actions/checkout@v2 88 | with: 89 | submodules: recursive 90 | - uses: actions-rs/toolchain@v1 91 | with: 92 | profile: minimal 93 | toolchain: stable 94 | override: true 95 | - id: check_toml 96 | run: | 97 | if grep -q 'name = "heed3"' heed/Cargo.toml; then 98 | echo "should_skip=true" >> $GITHUB_OUTPUT 99 | else 100 | echo "should_skip=false" >> $GITHUB_OUTPUT 101 | fi 102 | - name: Run cargo test 103 | # Skip it if the CI is running with the heed3 Cargo.toml 104 | if: ${{ steps.check_toml.outputs.should_skip == 'false' }} 105 | run: | 106 | sudo apt install -y valgrind 107 | cargo clean 108 | cargo check --all-features -p heed 109 | 110 | check-all-features-heed3: 111 | name: Check all the features of the heed3 project 112 | runs-on: ubuntu-latest 113 | env: 114 | RUSTFLAGS: -D warnings 115 | steps: 116 | - uses: actions/checkout@v2 117 | with: 118 | submodules: recursive 119 | - uses: actions-rs/toolchain@v1 120 | with: 121 | profile: minimal 122 | toolchain: stable 123 | override: true 124 | - name: Run cargo test 125 | run: | 126 | sudo apt install -y valgrind 127 | cargo clean 128 | bash convert-to-heed3.sh 129 | cargo check --all-features -p heed3 130 | 131 | examples: 132 | name: Run the heed examples 133 | runs-on: ${{ matrix.os }} 134 | strategy: 135 | matrix: 136 | os: [ubuntu-latest, macos-latest] 137 | include: 138 | - os: ubuntu-latest 139 | - os: macos-latest 140 | steps: 141 | - uses: actions/checkout@v2 142 | with: 143 | submodules: recursive 144 | - uses: actions-rs/toolchain@v1 145 | with: 146 | profile: minimal 147 | toolchain: stable 148 | override: true 149 | - id: check_toml 150 | run: | 151 | if grep -q 'name = "heed3"' heed/Cargo.toml; then 152 | echo "should_skip=true" >> $GITHUB_OUTPUT 153 | else 154 | echo "should_skip=false" >> $GITHUB_OUTPUT 155 | fi 156 | - name: Run the examples 157 | # Skip it if the CI is running with the heed3 Cargo.toml 158 | if: ${{ steps.check_toml.outputs.should_skip == 'false' }} 159 | run: | 160 | cargo clean 161 | # rmp-serde needs a feature activated, so we'll just run it separately. 162 | cargo run --example 2>&1 | grep -E '^ ' | awk '!/rmp-serde/' | xargs -n1 cargo run --example 163 | cargo run --example rmp-serde --features serde-rmp 164 | 165 | heed3-examples: 166 | name: Run the heed3 examples 167 | runs-on: ${{ matrix.os }} 168 | strategy: 169 | matrix: 170 | os: [ubuntu-latest, macos-latest] 171 | include: 172 | - os: ubuntu-latest 173 | - os: macos-latest 174 | steps: 175 | - uses: actions/checkout@v2 176 | with: 177 | submodules: recursive 178 | - uses: actions-rs/toolchain@v1 179 | with: 180 | profile: minimal 181 | toolchain: stable 182 | override: true 183 | - name: Run the examples 184 | run: | 185 | cargo clean 186 | bash convert-to-heed3.sh 187 | cargo run --example 2>&1 | grep -E '^ '| xargs -n1 cargo run --example 188 | 189 | clippy: 190 | name: Ensure clippy is happy on heed and heed3 191 | runs-on: ubuntu-latest 192 | steps: 193 | - uses: actions/checkout@v2 194 | with: 195 | submodules: recursive 196 | - uses: actions-rs/toolchain@v1 197 | with: 198 | profile: minimal 199 | toolchain: stable 200 | override: true 201 | - name: Run the examples 202 | run: | 203 | cargo clippy --all-targets -- --deny warnings 204 | bash convert-to-heed3.sh 205 | cargo clippy --all-targets -- --deny warnings 206 | 207 | fmt: 208 | name: Ensure the heed and heed3 project are formatted 209 | runs-on: ubuntu-latest 210 | steps: 211 | - uses: actions/checkout@v2 212 | - uses: actions-rs/toolchain@v1 213 | with: 214 | profile: minimal 215 | toolchain: nightly 216 | override: true 217 | components: rustfmt 218 | - name: Run cargo fmt 219 | run: | 220 | cargo fmt --check 221 | bash convert-to-heed3.sh 222 | cargo fmt --check 223 | 224 | no-heed3-in-heed-folder: 225 | name: Ensure heed3 is not erasing heed 226 | runs-on: ubuntu-latest 227 | steps: 228 | - uses: actions/checkout@v2 229 | - name: Check name is heed with grep 230 | run: grep -q 'name = "heed"' heed/Cargo.toml 231 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | heed/target 3 | **/*.rs.bk 4 | Cargo.lock 5 | *.mdb 6 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "lmdb-master-sys/lmdb"] 2 | path = lmdb-master-sys/lmdb 3 | url = https://git.openldap.org/openldap/openldap 4 | branch = mdb.master 5 | [submodule "lmdb-master3-sys/lmdb"] 6 | path = lmdb-master3-sys/lmdb 7 | url = https://git.openldap.org/openldap/openldap 8 | branch = mdb.master3 9 | -------------------------------------------------------------------------------- /.rustfmt.toml: -------------------------------------------------------------------------------- 1 | unstable_features = true 2 | 3 | use_small_heuristics = "max" 4 | imports_granularity = "Module" 5 | group_imports = "StdExternalCrate" 6 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["lmdb-master-sys", "lmdb-master3-sys", "heed", "heed-traits", "heed-types"] 3 | resolver = "2" 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Clément Renault 4 | Copyright (c) 2019-2025 Meili SAS 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 |

heed & heed3

3 | 4 | [![License](https://img.shields.io/badge/license-MIT-green)](#LICENSE) 5 | [![Crates.io](https://img.shields.io/crates/v/heed)](https://crates.io/crates/heed) 6 | [![Docs](https://docs.rs/heed/badge.svg)](https://docs.rs/heed) 7 | [![dependency status](https://deps.rs/repo/github/meilisearch/heed/status.svg)](https://deps.rs/repo/github/meilisearch/heed) 8 | [![Build](https://github.com/meilisearch/heed/actions/workflows/rust.yml/badge.svg)](https://github.com/meilisearch/heed/actions/workflows/rust.yml) 9 | [![Discord](https://img.shields.io/discord/1006923006964154428?style=flat&logo=discord&logoColor=ffffff&label=&labelColor=6A7EC2&color=7389D8)](https://discord.com/channels/1006923006964154428/1347203493106024528) 10 | 11 | Rust-centric [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database) abstractions with minimal overhead. These libraries enable the storage of various Rust types within LMDB, extending support to include Serde-compatible types. It supports not only the LMDB `mdb.master` branch but also the `mdb.master3` branch, which features encryption-at-rest. 12 | 13 | ## Simple Example Usage 14 | 15 | Here is an example on how to store and read entries into LMDB in a safe and ACID way. For usage examples, see [examples/](examples/). To see more advanced usage techniques go check our [Cookbook](https://docs.rs/heed/latest/heed/cookbook/index.html). 16 | 17 | ```rust 18 | use std::fs; 19 | use std::path::Path; 20 | use heed::{EnvOpenOptions, Database}; 21 | use heed::types::*; 22 | 23 | fn main() -> Result<(), Box> { 24 | let env = unsafe { EnvOpenOptions::new().open("my-first-db")? }; 25 | 26 | // We open the default unnamed database 27 | let mut wtxn = env.write_txn()?; 28 | let db: Database> = env.create_database(&mut wtxn, None)?; 29 | 30 | // We open a write transaction 31 | db.put(&mut wtxn, "seven", &7)?; 32 | db.put(&mut wtxn, "zero", &0)?; 33 | db.put(&mut wtxn, "five", &5)?; 34 | db.put(&mut wtxn, "three", &3)?; 35 | wtxn.commit()?; 36 | 37 | // We open a read transaction to check if those values are now available 38 | let mut rtxn = env.read_txn()?; 39 | 40 | let ret = db.get(&rtxn, "zero")?; 41 | assert_eq!(ret, Some(0)); 42 | 43 | let ret = db.get(&rtxn, "five")?; 44 | assert_eq!(ret, Some(5)); 45 | 46 | Ok(()) 47 | } 48 | ``` 49 | 50 | ## Working with two Crates: heed and heed3 51 | 52 | The heed and heed3 crates manage a shared codebase. Within the heed3 folder, you can find the Cargo.toml specific to the heed3 crate. 53 | To facilitate work on heed3, utilize the `convert-to-heed3.sh` script. 54 | 55 | This script conveniently moves the `heed3/Cargo.toml` file to the `heed/` folder, updates the `heed::` references to `heed3::`, and generates a commit for easy rollback if needed. 56 | 57 | ## Building from Source 58 | 59 | You can use this command to clone the repository: 60 | 61 | ```bash 62 | git clone --recursive https://github.com/meilisearch/heed.git 63 | cd heed 64 | cargo build 65 | ``` 66 | 67 | However, if you already cloned it and forgot to initialize the submodules, execute the following command: 68 | 69 | ```bash 70 | git submodule update --init 71 | ``` 72 | -------------------------------------------------------------------------------- /assets/heed-pigeon-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meilisearch/heed/5a10a00e02669d28edd6d11d35b617a9128189ca/assets/heed-pigeon-logo.png -------------------------------------------------------------------------------- /assets/heed-pigeon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meilisearch/heed/5a10a00e02669d28edd6d11d35b617a9128189ca/assets/heed-pigeon.ico -------------------------------------------------------------------------------- /convert-to-heed3.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script is meant to setup the heed3 crate. 4 | # 5 | 6 | if [[ -n $(git status -s) ]]; then 7 | echo "Error: Repository is git dirty, please commit or stash changes before running this script." 8 | exit 1 9 | fi 10 | 11 | set -e 12 | 13 | # It basically copy the heed3/Cargo.toml file into 14 | # the heed folder... 15 | if [[ "$OSTYPE" == "cygwin" || "$OSTYPE" == "msys" ]]; then 16 | cp heed3\\Cargo.toml heed\\Cargo.toml 17 | rm -rf heed\\examples 18 | cp -R heed3\\examples heed\\examples 19 | else 20 | cp heed3/Cargo.toml heed/Cargo.toml 21 | rm -rf heed/examples 22 | cp -R heed3/examples heed/examples 23 | fi 24 | 25 | # ...and replaces the `heed::` string by the `heed3::` one. 26 | for file in $(find heed/src -type f -name "*.rs"); do 27 | if [[ "$OSTYPE" == "darwin"* ]]; then 28 | sed -i '' 's/heed::/heed3::/g' "$file" 29 | else 30 | sed -i 's/heed::/heed3::/g' "$file" 31 | fi 32 | done 33 | 34 | # Make it easier to rollback by doing a commit 35 | git config --local user.email "ci@github.com" 36 | git config --local user.name "The CI" 37 | 38 | # Adds the new examples to the changes 39 | if [[ "$OSTYPE" == "cygwin" || "$OSTYPE" == "msys" ]]; then 40 | git add heed\\examples 41 | else 42 | git add heed/examples 43 | fi 44 | 45 | # But only commit if there is something to commit 46 | git diff -q --exit-code || git commit -am 'remove-me: heed3 changes generated by the convert-to-heed3.sh script' 47 | 48 | git config --local --unset user.email 49 | git config --local --unset user.name 50 | 51 | echo "Heed3 crate setup completed successfully. Configurations for the heed crate have been copied and modified." 52 | echo "A commit (starting with remove-me) has been generated and must be deleted before merging into the main branch." 53 | -------------------------------------------------------------------------------- /heed-traits/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "heed-traits" 3 | version = "0.20.0" 4 | authors = ["Kerollmops "] 5 | description = "The traits used inside of the fully typed LMDB wrapper, heed" 6 | license = "MIT" 7 | repository = "https://github.com/Kerollmops/heed" 8 | readme = "../README.md" 9 | edition = "2021" 10 | 11 | [dependencies] 12 | -------------------------------------------------------------------------------- /heed-traits/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc( 2 | html_favicon_url = "https://raw.githubusercontent.com/meilisearch/heed/main/assets//heed-pigeon.ico?raw=true" 3 | )] 4 | #![doc( 5 | html_logo_url = "https://raw.githubusercontent.com/meilisearch/heed/main/assets/heed-pigeon-logo.png?raw=true" 6 | )] 7 | 8 | //! Contains the traits used to encode and decode database content. 9 | 10 | #![warn(missing_docs)] 11 | 12 | use std::borrow::Cow; 13 | use std::cmp::{Ord, Ordering}; 14 | use std::error::Error as StdError; 15 | 16 | /// A boxed `Send + Sync + 'static` error. 17 | pub type BoxedError = Box; 18 | 19 | /// A trait that represents an encoding structure. 20 | pub trait BytesEncode<'a> { 21 | /// The type to encode. 22 | type EItem: ?Sized + 'a; 23 | 24 | /// Encode the given item as bytes. 25 | fn bytes_encode(item: &'a Self::EItem) -> Result, BoxedError>; 26 | } 27 | 28 | /// A trait that represents a decoding structure. 29 | pub trait BytesDecode<'a> { 30 | /// The type to decode. 31 | type DItem: 'a; 32 | 33 | /// Decode the given bytes as `DItem`. 34 | fn bytes_decode(bytes: &'a [u8]) -> Result; 35 | } 36 | 37 | /// Define a custom key comparison function for a database. 38 | /// 39 | /// The comparison function is called whenever it is necessary to compare a key specified 40 | /// by the application with a key currently stored in the database. If no comparison function 41 | /// is specified, and no special key flags were specified, the keys are compared lexically, 42 | /// with shorter keys collating before longer keys. 43 | pub trait Comparator { 44 | /// Compares the raw bytes representation of two keys. 45 | fn compare(a: &[u8], b: &[u8]) -> Ordering; 46 | } 47 | 48 | /// Define a lexicographic comparator, which is a special case of [`Comparator`]. 49 | /// 50 | /// Types that implements [`LexicographicComparator`] will automatically have [`Comparator`] 51 | /// implemented as well, where the [`Comparator::compare`] is implemented per the definition 52 | /// of lexicographic ordering with [`LexicographicComparator::compare_elem`]. 53 | /// 54 | /// This trait is introduced to support prefix iterators, which implicit assumes that the 55 | /// underlying key comparator is lexicographic. 56 | pub trait LexicographicComparator: Comparator { 57 | /// Compare a single byte; this function is used to implement [`Comparator::compare`] 58 | /// by definition of lexicographic ordering. 59 | fn compare_elem(a: u8, b: u8) -> Ordering; 60 | 61 | /// Advances the given `elem` to its immediate lexicographic successor, if possible. 62 | /// Returns `None` if `elem` is already at its maximum value with respect to the 63 | /// lexicographic order defined by this comparator. 64 | fn successor(elem: u8) -> Option; 65 | 66 | /// Moves the given `elem` to its immediate lexicographic predecessor, if possible. 67 | /// Returns `None` if `elem` is already at its minimum value with respect to the 68 | /// lexicographic order defined by this comparator. 69 | fn predecessor(elem: u8) -> Option; 70 | 71 | /// Returns the maximum byte value per the comparator's lexicographic order. 72 | fn max_elem() -> u8; 73 | 74 | /// Returns the minimum byte value per the comparator's lexicographic order. 75 | fn min_elem() -> u8; 76 | } 77 | 78 | impl Comparator for C { 79 | fn compare(a: &[u8], b: &[u8]) -> Ordering { 80 | for idx in 0..std::cmp::min(a.len(), b.len()) { 81 | if a[idx] != b[idx] { 82 | return C::compare_elem(a[idx], b[idx]); 83 | } 84 | } 85 | Ord::cmp(&a.len(), &b.len()) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /heed-types/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "heed-types" 3 | version = "0.21.0" 4 | authors = ["Kerollmops "] 5 | description = "The types used with the fully typed LMDB wrapper, heed" 6 | license = "MIT" 7 | repository = "https://github.com/Kerollmops/heed" 8 | readme = "../README.md" 9 | edition = "2021" 10 | 11 | [dependencies] 12 | bincode = { version = "1.3.3", optional = true } 13 | byteorder = "1.5.0" 14 | heed-traits = { version = "0.20.0", path = "../heed-traits" } 15 | serde = { version = "1.0.218", optional = true } 16 | serde_json = { version = "1.0.140", optional = true } 17 | rmp-serde = { version = "1.3.0", optional = true } 18 | 19 | [features] 20 | default = ["serde-bincode", "serde-json"] 21 | serde-bincode = ["serde", "bincode"] 22 | serde-json = ["serde", "serde_json"] 23 | serde-rmp = ["serde", "rmp-serde"] 24 | # serde_json features 25 | preserve_order = ["serde_json/preserve_order"] 26 | arbitrary_precision = ["serde_json/arbitrary_precision"] 27 | raw_value = ["serde_json/raw_value"] 28 | unbounded_depth = ["serde_json/unbounded_depth"] 29 | -------------------------------------------------------------------------------- /heed-types/src/bytes.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::Cow; 2 | 3 | use heed_traits::{BoxedError, BytesDecode, BytesEncode}; 4 | 5 | /// Describes a byte slice `[u8]` that is totally borrowed and doesn't depend on 6 | /// any [memory alignment]. 7 | /// 8 | /// [memory alignment]: std::mem::align_of() 9 | pub enum Bytes {} 10 | 11 | impl<'a> BytesEncode<'a> for Bytes { 12 | type EItem = [u8]; 13 | 14 | fn bytes_encode(item: &'a Self::EItem) -> Result, BoxedError> { 15 | Ok(Cow::Borrowed(item)) 16 | } 17 | } 18 | 19 | impl<'a> BytesDecode<'a> for Bytes { 20 | type DItem = &'a [u8]; 21 | 22 | fn bytes_decode(bytes: &'a [u8]) -> Result { 23 | Ok(bytes) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /heed-types/src/decode_ignore.rs: -------------------------------------------------------------------------------- 1 | use heed_traits::BoxedError; 2 | 3 | /// A convenient struct made to ignore the type when decoding it. 4 | /// 5 | /// For example, it is appropriate to be used to count keys or to ensure that an 6 | /// entry exists. 7 | pub enum DecodeIgnore {} 8 | 9 | impl heed_traits::BytesDecode<'_> for DecodeIgnore { 10 | type DItem = (); 11 | 12 | fn bytes_decode(_bytes: &[u8]) -> Result { 13 | Ok(()) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /heed-types/src/integer.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::Cow; 2 | use std::marker::PhantomData; 3 | use std::mem::size_of; 4 | 5 | use byteorder::{ByteOrder, ReadBytesExt}; 6 | use heed_traits::{BoxedError, BytesDecode, BytesEncode}; 7 | 8 | /// Encodable version of [`u8`]. 9 | pub struct U8; 10 | 11 | impl BytesEncode<'_> for U8 { 12 | type EItem = u8; 13 | 14 | fn bytes_encode(item: &Self::EItem) -> Result, BoxedError> { 15 | Ok(Cow::from([*item].to_vec())) 16 | } 17 | } 18 | 19 | impl BytesDecode<'_> for U8 { 20 | type DItem = u8; 21 | 22 | fn bytes_decode(mut bytes: &'_ [u8]) -> Result { 23 | bytes.read_u8().map_err(Into::into) 24 | } 25 | } 26 | 27 | /// Encodable version of [`i8`]. 28 | pub struct I8; 29 | 30 | impl BytesEncode<'_> for I8 { 31 | type EItem = i8; 32 | 33 | fn bytes_encode(item: &Self::EItem) -> Result, BoxedError> { 34 | Ok(Cow::from([*item as u8].to_vec())) 35 | } 36 | } 37 | 38 | impl BytesDecode<'_> for I8 { 39 | type DItem = i8; 40 | 41 | fn bytes_decode(mut bytes: &'_ [u8]) -> Result { 42 | bytes.read_i8().map_err(Into::into) 43 | } 44 | } 45 | 46 | macro_rules! define_type { 47 | ($name:ident, $native:ident, $read_method:ident, $write_method:ident) => { 48 | #[doc = "Encodable version of [`"] 49 | #[doc = stringify!($native)] 50 | #[doc = "`]."] 51 | 52 | pub struct $name(PhantomData); 53 | 54 | impl BytesEncode<'_> for $name { 55 | type EItem = $native; 56 | 57 | fn bytes_encode(item: &Self::EItem) -> Result, BoxedError> { 58 | let mut buf = vec![0; size_of::()]; 59 | O::$write_method(&mut buf, *item); 60 | Ok(Cow::from(buf)) 61 | } 62 | } 63 | 64 | impl BytesDecode<'_> for $name { 65 | type DItem = $native; 66 | 67 | fn bytes_decode(mut bytes: &'_ [u8]) -> Result { 68 | bytes.$read_method::().map_err(Into::into) 69 | } 70 | } 71 | }; 72 | } 73 | 74 | define_type!(U16, u16, read_u16, write_u16); 75 | define_type!(U32, u32, read_u32, write_u32); 76 | define_type!(U64, u64, read_u64, write_u64); 77 | define_type!(U128, u128, read_u128, write_u128); 78 | define_type!(I16, i16, read_i16, write_i16); 79 | define_type!(I32, i32, read_i32, write_i32); 80 | define_type!(I64, i64, read_i64, write_i64); 81 | define_type!(I128, i128, read_i128, write_i128); 82 | -------------------------------------------------------------------------------- /heed-types/src/lazy_decode.rs: -------------------------------------------------------------------------------- 1 | use std::marker; 2 | 3 | use heed_traits::BoxedError; 4 | 5 | /// Lazily decodes the data bytes. 6 | /// 7 | /// It can be used to avoid CPU-intensive decoding before making sure that it 8 | /// actually needs to be decoded (e.g. based on the key). 9 | #[derive(Default)] 10 | pub struct LazyDecode(marker::PhantomData); 11 | 12 | impl<'a, C: 'static> heed_traits::BytesDecode<'a> for LazyDecode { 13 | type DItem = Lazy<'a, C>; 14 | 15 | fn bytes_decode(bytes: &'a [u8]) -> Result { 16 | Ok(Lazy { data: bytes, _phantom: marker::PhantomData }) 17 | } 18 | } 19 | 20 | /// Owns bytes that can be decoded on demand. 21 | #[derive(Copy, Clone)] 22 | pub struct Lazy<'a, C> { 23 | data: &'a [u8], 24 | _phantom: marker::PhantomData, 25 | } 26 | 27 | impl<'a, C> Lazy<'a, C> { 28 | /// Change the codec type of the given bytes, specifying the new codec. 29 | pub fn remap(&self) -> Lazy<'a, NC> { 30 | Lazy { data: self.data, _phantom: marker::PhantomData } 31 | } 32 | } 33 | 34 | impl<'a, C: heed_traits::BytesDecode<'a>> Lazy<'a, C> { 35 | /// Decode the given bytes as `DItem`. 36 | pub fn decode(&self) -> Result { 37 | C::bytes_decode(self.data) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /heed-types/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc( 2 | html_favicon_url = "https://raw.githubusercontent.com/meilisearch/heed/main/assets/heed-pigeon.ico?raw=true" 3 | )] 4 | #![doc( 5 | html_logo_url = "https://raw.githubusercontent.com/meilisearch/heed/main/assets/heed-pigeon-logo.png?raw=true" 6 | )] 7 | 8 | //! Types that can be used to serialize and deserialize types inside databases. 9 | 10 | #![warn(missing_docs)] 11 | 12 | mod bytes; 13 | mod decode_ignore; 14 | mod integer; 15 | mod lazy_decode; 16 | mod str; 17 | mod unit; 18 | 19 | #[cfg(feature = "serde-bincode")] 20 | mod serde_bincode; 21 | 22 | #[cfg(feature = "serde-json")] 23 | mod serde_json; 24 | 25 | #[cfg(feature = "serde-rmp")] 26 | mod serde_rmp; 27 | 28 | pub use self::bytes::Bytes; 29 | pub use self::decode_ignore::DecodeIgnore; 30 | pub use self::integer::*; 31 | pub use self::lazy_decode::{Lazy, LazyDecode}; 32 | #[cfg(feature = "serde-bincode")] 33 | pub use self::serde_bincode::SerdeBincode; 34 | #[cfg(feature = "serde-json")] 35 | pub use self::serde_json::SerdeJson; 36 | #[cfg(feature = "serde-rmp")] 37 | pub use self::serde_rmp::SerdeRmp; 38 | pub use self::str::Str; 39 | pub use self::unit::Unit; 40 | -------------------------------------------------------------------------------- /heed-types/src/serde_bincode.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::Cow; 2 | 3 | use heed_traits::{BoxedError, BytesDecode, BytesEncode}; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | /// Describes a type that is [`Serialize`]/[`Deserialize`] and uses `bincode` to do so. 7 | /// 8 | /// It can borrow bytes from the original slice. 9 | pub struct SerdeBincode(std::marker::PhantomData); 10 | 11 | impl<'a, T: 'a> BytesEncode<'a> for SerdeBincode 12 | where 13 | T: Serialize, 14 | { 15 | type EItem = T; 16 | 17 | fn bytes_encode(item: &'a Self::EItem) -> Result, BoxedError> { 18 | bincode::serialize(item).map(Cow::Owned).map_err(Into::into) 19 | } 20 | } 21 | 22 | impl<'a, T: 'a> BytesDecode<'a> for SerdeBincode 23 | where 24 | T: Deserialize<'a>, 25 | { 26 | type DItem = T; 27 | 28 | fn bytes_decode(bytes: &'a [u8]) -> Result { 29 | bincode::deserialize(bytes).map_err(Into::into) 30 | } 31 | } 32 | 33 | unsafe impl Send for SerdeBincode {} 34 | 35 | unsafe impl Sync for SerdeBincode {} 36 | -------------------------------------------------------------------------------- /heed-types/src/serde_json.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::Cow; 2 | 3 | use heed_traits::{BoxedError, BytesDecode, BytesEncode}; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | /// Describes a type that is [`Serialize`]/[`Deserialize`] and uses `serde_json` to do so. 7 | /// 8 | /// It can borrow bytes from the original slice. 9 | pub struct SerdeJson(std::marker::PhantomData); 10 | 11 | impl<'a, T: 'a> BytesEncode<'a> for SerdeJson 12 | where 13 | T: Serialize, 14 | { 15 | type EItem = T; 16 | 17 | fn bytes_encode(item: &Self::EItem) -> Result, BoxedError> { 18 | serde_json::to_vec(item).map(Cow::Owned).map_err(Into::into) 19 | } 20 | } 21 | 22 | impl<'a, T: 'a> BytesDecode<'a> for SerdeJson 23 | where 24 | T: Deserialize<'a>, 25 | { 26 | type DItem = T; 27 | 28 | fn bytes_decode(bytes: &'a [u8]) -> Result { 29 | serde_json::from_slice(bytes).map_err(Into::into) 30 | } 31 | } 32 | 33 | unsafe impl Send for SerdeJson {} 34 | 35 | unsafe impl Sync for SerdeJson {} 36 | -------------------------------------------------------------------------------- /heed-types/src/serde_rmp.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::Cow; 2 | 3 | use heed_traits::{BoxedError, BytesDecode, BytesEncode}; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | /// Describes a type that is [`Serialize`]/[`Deserialize`] and uses `rmp_serde` to do so. 7 | /// 8 | /// It can borrow bytes from the original slice. 9 | pub struct SerdeRmp(std::marker::PhantomData); 10 | 11 | impl<'a, T: 'a> BytesEncode<'a> for SerdeRmp 12 | where 13 | T: Serialize, 14 | { 15 | type EItem = T; 16 | 17 | fn bytes_encode(item: &Self::EItem) -> Result, BoxedError> { 18 | rmp_serde::to_vec(item).map(Cow::Owned).map_err(Into::into) 19 | } 20 | } 21 | 22 | impl<'a, T: 'a> BytesDecode<'a> for SerdeRmp 23 | where 24 | T: Deserialize<'a>, 25 | { 26 | type DItem = T; 27 | 28 | fn bytes_decode(bytes: &'a [u8]) -> Result { 29 | rmp_serde::from_slice(bytes).map_err(Into::into) 30 | } 31 | } 32 | 33 | unsafe impl Send for SerdeRmp {} 34 | 35 | unsafe impl Sync for SerdeRmp {} 36 | -------------------------------------------------------------------------------- /heed-types/src/str.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::Cow; 2 | use std::str; 3 | 4 | use heed_traits::{BoxedError, BytesDecode, BytesEncode}; 5 | 6 | /// Describes a [`prim@str`]. 7 | pub enum Str {} 8 | 9 | impl BytesEncode<'_> for Str { 10 | type EItem = str; 11 | 12 | fn bytes_encode(item: &Self::EItem) -> Result, BoxedError> { 13 | Ok(Cow::Borrowed(item.as_bytes())) 14 | } 15 | } 16 | 17 | impl<'a> BytesDecode<'a> for Str { 18 | type DItem = &'a str; 19 | 20 | fn bytes_decode(bytes: &'a [u8]) -> Result { 21 | str::from_utf8(bytes).map_err(Into::into) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /heed-types/src/unit.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::Cow; 2 | use std::{error, fmt}; 3 | 4 | use heed_traits::{BoxedError, BytesDecode, BytesEncode}; 5 | 6 | /// Describes the unit `()` type. 7 | pub enum Unit {} 8 | 9 | impl BytesEncode<'_> for Unit { 10 | type EItem = (); 11 | 12 | fn bytes_encode(_item: &Self::EItem) -> Result, BoxedError> { 13 | Ok(Cow::Borrowed(&[])) 14 | } 15 | } 16 | 17 | impl BytesDecode<'_> for Unit { 18 | type DItem = (); 19 | 20 | fn bytes_decode(bytes: &[u8]) -> Result { 21 | if bytes.is_empty() { 22 | Ok(()) 23 | } else { 24 | Err(NonEmptyError.into()) 25 | } 26 | } 27 | } 28 | 29 | /// The slice of bytes is non-empty and therefore is not a unit `()` type. 30 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] 31 | pub struct NonEmptyError; 32 | 33 | impl fmt::Display for NonEmptyError { 34 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 35 | f.write_str("the slice of bytes is non-empty and therefore is not a unit `()` type") 36 | } 37 | } 38 | 39 | impl error::Error for NonEmptyError {} 40 | -------------------------------------------------------------------------------- /heed/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "heed" 3 | version = "0.22.0" 4 | authors = ["Kerollmops "] 5 | description = "A fully typed LMDB (mdb.master) wrapper with minimum overhead" 6 | license = "MIT" 7 | repository = "https://github.com/Kerollmops/heed" 8 | keywords = ["lmdb", "database", "storage", "typed"] 9 | categories = ["database", "data-structures"] 10 | readme = "../README.md" 11 | edition = "2021" 12 | 13 | [dependencies] 14 | bitflags = { version = "2.9.0", features = ["serde"] } 15 | byteorder = { version = "1.5.0", default-features = false } 16 | heed-traits = { version = "0.20.0", path = "../heed-traits" } 17 | heed-types = { version = "0.21.0", default-features = false, path = "../heed-types" } 18 | libc = "0.2.170" 19 | lmdb-master-sys = { version = "0.2.5", path = "../lmdb-master-sys" } 20 | once_cell = "1.20.3" 21 | page_size = "0.6.0" 22 | serde = { version = "1.0.218", features = ["derive"], optional = true } 23 | synchronoise = "1.0.1" 24 | 25 | [dev-dependencies] 26 | memchr = "2.7.4" 27 | serde = { version = "1.0.218", features = ["derive"] } 28 | tempfile = "3.18.0" 29 | 30 | [target.'cfg(windows)'.dependencies] 31 | url = "2.5.4" 32 | 33 | [features] 34 | # The `serde` feature makes some types serializable, 35 | # like the `EnvOpenOptions` struct. 36 | default = ["serde", "serde-bincode", "serde-json"] 37 | serde = ["bitflags/serde", "dep:serde"] 38 | 39 | # Enable the serde en/decoders for bincode, serde_json, or rmp_serde 40 | serde-bincode = ["heed-types/serde-bincode"] 41 | serde-json = ["heed-types/serde-json"] 42 | serde-rmp = ["heed-types/serde-rmp"] 43 | 44 | # serde_json features 45 | preserve_order = ["heed-types/preserve_order"] 46 | arbitrary_precision = ["heed-types/arbitrary_precision"] 47 | raw_value = ["heed-types/raw_value"] 48 | unbounded_depth = ["heed-types/unbounded_depth"] 49 | 50 | # Whether to tell LMDB to use POSIX semaphores during compilation 51 | # (instead of the default, which are System V semaphores). 52 | # POSIX semaphores are required for Apple's App Sandbox on iOS & macOS, 53 | # and are possibly faster and more appropriate for single-process use. 54 | # There are tradeoffs for both POSIX and SysV semaphores; which you 55 | # should look into before enabling this feature. Also, see here: 56 | # 57 | posix-sem = ["lmdb-master-sys/posix-sem"] 58 | 59 | # These features configure the MDB_IDL_LOGN macro, which determines 60 | # the size of the free and dirty page lists (and thus the amount of memory 61 | # allocated when opening an LMDB environment in read-write mode). 62 | # 63 | # Each feature defines MDB_IDL_LOGN as the value in the name of the feature. 64 | # That means these features are mutually exclusive, and you must not specify 65 | # more than one at the same time (or the crate will fail to compile). 66 | # 67 | # For more information on the motivation for these features (and their effect), 68 | # see https://github.com/mozilla/lmdb/pull/2. 69 | mdb_idl_logn_8 = ["lmdb-master-sys/mdb_idl_logn_8"] 70 | mdb_idl_logn_9 = ["lmdb-master-sys/mdb_idl_logn_9"] 71 | mdb_idl_logn_10 = ["lmdb-master-sys/mdb_idl_logn_10"] 72 | mdb_idl_logn_11 = ["lmdb-master-sys/mdb_idl_logn_11"] 73 | mdb_idl_logn_12 = ["lmdb-master-sys/mdb_idl_logn_12"] 74 | mdb_idl_logn_13 = ["lmdb-master-sys/mdb_idl_logn_13"] 75 | mdb_idl_logn_14 = ["lmdb-master-sys/mdb_idl_logn_14"] 76 | mdb_idl_logn_15 = ["lmdb-master-sys/mdb_idl_logn_15"] 77 | mdb_idl_logn_16 = ["lmdb-master-sys/mdb_idl_logn_16"] 78 | 79 | # Setting this enables you to use keys longer than 511 bytes. The exact limit 80 | # is computed by LMDB at compile time. You can find the exact value by calling 81 | # Env::max_key_size(). This value varies by architecture. 82 | # 83 | # Example max key sizes: 84 | # - Apple M1 (ARM64): 8126 bytes 85 | # - Apple Intel (AMD64): 1982 bytes 86 | # - Linux Intel (AMD64): 1982 bytes 87 | # 88 | # Setting this also enables you to use values larger than 511 bytes when using 89 | # a Database with the DatabaseFlags::DUP_SORT flag. 90 | # 91 | # This builds LMDB with the -DMDB_MAXKEYSIZE=0 option. 92 | # 93 | # Note: If you are moving database files between architectures then your longest 94 | # stored key must fit within the smallest limit of all architectures used. For 95 | # example, if you are moving databases between Apple M1 and Apple Intel 96 | # computers then you need to keep your keys within the smaller 1982 byte limit. 97 | longer-keys = ["lmdb-master-sys/longer-keys"] 98 | 99 | # Enable a better Valgrind support. This builds LMDB with the -DUSE_VALGRIND=1 option. 100 | # 101 | # You have to install the RPM valgrind-devel which contains memcheck.h. 102 | # 103 | # More information can be found at: 104 | # 105 | use-valgrind = ["lmdb-master-sys/use-valgrind"] 106 | 107 | [[example]] 108 | name = "all-types" 109 | 110 | [[example]] 111 | name = "clear-database" 112 | 113 | [[example]] 114 | name = "cursor-append" 115 | 116 | [[example]] 117 | name = "custom-comparator" 118 | 119 | [[example]] 120 | name = "custom-dupsort-comparator" 121 | 122 | [[example]] 123 | name = "multi-env" 124 | 125 | [[example]] 126 | name = "nested" 127 | 128 | [[example]] 129 | name = "prev-snapshot" 130 | 131 | [[example]] 132 | name = "rmp-serde" 133 | required-features = ["serde-rmp"] 134 | -------------------------------------------------------------------------------- /heed/build.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | fn main() { 4 | println!("cargo:rerun-if-changed=build.rs"); 5 | println!("cargo::rustc-check-cfg=cfg(master3)"); 6 | 7 | let pkgname = env::var("CARGO_PKG_NAME").expect("Cargo didn't set the CARGO_PKG_NAME env var!"); 8 | match pkgname.as_str() { 9 | "heed" => (), 10 | "heed3" => println!("cargo:rustc-cfg=master3"), 11 | _ => panic!("unexpected package name!"), 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /heed/examples/all-types.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use heed::byteorder::BE; 4 | use heed::types::*; 5 | use heed::{Database, EnvOpenOptions}; 6 | use serde::{Deserialize, Serialize}; 7 | 8 | fn main() -> Result<(), Box> { 9 | let path = tempfile::tempdir()?; 10 | 11 | let env = unsafe { 12 | EnvOpenOptions::new() 13 | .map_size(10 * 1024 * 1024) // 10MB 14 | .max_dbs(3000) 15 | .open(path)? 16 | }; 17 | 18 | // here the key will be an str and the data will be a slice of u8 19 | let mut wtxn = env.write_txn()?; 20 | let db: Database = env.create_database(&mut wtxn, Some("kiki"))?; 21 | 22 | db.put(&mut wtxn, "hello", &[2, 3][..])?; 23 | let ret: Option<&[u8]> = db.get(&wtxn, "hello")?; 24 | 25 | println!("{:?}", ret); 26 | wtxn.commit()?; 27 | 28 | // serde types are also supported!!! 29 | #[derive(Debug, Serialize, Deserialize)] 30 | struct Hello<'a> { 31 | string: &'a str, 32 | } 33 | 34 | let mut wtxn = env.write_txn()?; 35 | let db: Database> = 36 | env.create_database(&mut wtxn, Some("serde-bincode"))?; 37 | 38 | let hello = Hello { string: "hi" }; 39 | db.put(&mut wtxn, "hello", &hello)?; 40 | 41 | let ret: Option = db.get(&wtxn, "hello")?; 42 | println!("serde-bincode:\t{:?}", ret); 43 | 44 | wtxn.commit()?; 45 | 46 | let mut wtxn = env.write_txn()?; 47 | let db: Database> = env.create_database(&mut wtxn, Some("serde-json"))?; 48 | 49 | let hello = Hello { string: "hi" }; 50 | db.put(&mut wtxn, "hello", &hello)?; 51 | 52 | let ret: Option = db.get(&wtxn, "hello")?; 53 | println!("serde-json:\t{:?}", ret); 54 | 55 | wtxn.commit()?; 56 | 57 | // you can ignore the data 58 | let mut wtxn = env.write_txn()?; 59 | let db: Database = env.create_database(&mut wtxn, Some("ignored-data"))?; 60 | 61 | db.put(&mut wtxn, "hello", &())?; 62 | let ret: Option<()> = db.get(&wtxn, "hello")?; 63 | 64 | println!("{:?}", ret); 65 | 66 | let ret: Option<()> = db.get(&wtxn, "non-existant")?; 67 | 68 | println!("{:?}", ret); 69 | wtxn.commit()?; 70 | 71 | // database opening and types are tested in a safe way 72 | // 73 | // we try to open a database twice with the same types 74 | let mut wtxn = env.write_txn()?; 75 | let _db: Database = env.create_database(&mut wtxn, Some("ignored-data"))?; 76 | 77 | // you can iterate over keys in order 78 | type BEI64 = I64; 79 | 80 | let db: Database = env.create_database(&mut wtxn, Some("big-endian-iter"))?; 81 | 82 | db.put(&mut wtxn, &0, &())?; 83 | db.put(&mut wtxn, &68, &())?; 84 | db.put(&mut wtxn, &35, &())?; 85 | db.put(&mut wtxn, &42, &())?; 86 | 87 | let rets: Result, _> = db.iter(&wtxn)?.collect(); 88 | 89 | println!("{:?}", rets); 90 | 91 | // or iterate over ranges too!!! 92 | let range = 35..=42; 93 | let rets: Result, _> = db.range(&wtxn, &range)?.collect(); 94 | 95 | println!("{:?}", rets); 96 | 97 | // delete a range of key 98 | let range = 35..=42; 99 | let deleted: usize = db.delete_range(&mut wtxn, &range)?; 100 | 101 | let rets: Result, _> = db.iter(&wtxn)?.collect(); 102 | 103 | println!("deleted: {:?}, {:?}", deleted, rets); 104 | wtxn.commit()?; 105 | 106 | Ok(()) 107 | } 108 | -------------------------------------------------------------------------------- /heed/examples/clear-database.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use heed::types::*; 4 | use heed::{Database, EnvOpenOptions}; 5 | 6 | // In this test we are checking that we can clear database entries and 7 | // write just after in the same transaction without loosing the writes. 8 | fn main() -> Result<(), Box> { 9 | let env_path = tempfile::tempdir()?; 10 | 11 | let env = unsafe { 12 | EnvOpenOptions::new() 13 | .map_size(10 * 1024 * 1024) // 10MB 14 | .max_dbs(3) 15 | .open(env_path)? 16 | }; 17 | 18 | let mut wtxn = env.write_txn()?; 19 | let db: Database = env.create_database(&mut wtxn, Some("first"))?; 20 | 21 | // We fill the db database with entries. 22 | db.put(&mut wtxn, "I am here", "to test things")?; 23 | db.put(&mut wtxn, "I am here too", "for the same purpose")?; 24 | 25 | wtxn.commit()?; 26 | 27 | let mut wtxn = env.write_txn()?; 28 | db.clear(&mut wtxn)?; 29 | db.put(&mut wtxn, "And I come back", "to test things")?; 30 | 31 | let mut iter = db.iter(&wtxn)?; 32 | assert_eq!(iter.next().transpose()?, Some(("And I come back", "to test things"))); 33 | assert_eq!(iter.next().transpose()?, None); 34 | 35 | drop(iter); 36 | wtxn.commit()?; 37 | 38 | let rtxn = env.read_txn()?; 39 | let mut iter = db.iter(&rtxn)?; 40 | assert_eq!(iter.next().transpose()?, Some(("And I come back", "to test things"))); 41 | assert_eq!(iter.next().transpose()?, None); 42 | 43 | Ok(()) 44 | } 45 | -------------------------------------------------------------------------------- /heed/examples/cursor-append.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use heed::types::*; 4 | use heed::{Database, EnvOpenOptions, PutFlags}; 5 | 6 | // In this test we are checking that we can append ordered entries in one 7 | // database even if there is multiple databases which already contain entries. 8 | fn main() -> Result<(), Box> { 9 | let env_path = tempfile::tempdir()?; 10 | 11 | let env = unsafe { 12 | EnvOpenOptions::new() 13 | .map_size(10 * 1024 * 1024) // 10MB 14 | .max_dbs(3) 15 | .open(env_path)? 16 | }; 17 | 18 | let mut wtxn = env.write_txn()?; 19 | let first: Database = env.create_database(&mut wtxn, Some("first"))?; 20 | let second: Database = env.create_database(&mut wtxn, Some("second"))?; 21 | 22 | // We fill the first database with entries. 23 | first.put(&mut wtxn, "I am here", "to test things")?; 24 | first.put(&mut wtxn, "I am here too", "for the same purpose")?; 25 | 26 | // We try to append ordered entries in the second database. 27 | let mut iter = second.iter_mut(&mut wtxn)?; 28 | 29 | unsafe { iter.put_current_with_options::(PutFlags::APPEND, "aaaa", "lol")? }; 30 | unsafe { iter.put_current_with_options::(PutFlags::APPEND, "abcd", "lol")? }; 31 | unsafe { iter.put_current_with_options::(PutFlags::APPEND, "bcde", "lol")? }; 32 | 33 | drop(iter); 34 | 35 | wtxn.commit()?; 36 | 37 | Ok(()) 38 | } 39 | -------------------------------------------------------------------------------- /heed/examples/custom-comparator.rs: -------------------------------------------------------------------------------- 1 | use std::cmp::Ordering; 2 | use std::error::Error; 3 | use std::str; 4 | 5 | use heed::EnvOpenOptions; 6 | use heed_traits::Comparator; 7 | use heed_types::{Str, Unit}; 8 | 9 | enum StringAsIntCmp {} 10 | 11 | // This function takes two strings which represent positive numbers, 12 | // parses them into i32s and compare the parsed value. 13 | // Therefore "-1000" < "-100" must be true even without '0' padding. 14 | impl Comparator for StringAsIntCmp { 15 | fn compare(a: &[u8], b: &[u8]) -> Ordering { 16 | let a: i32 = str::from_utf8(a).unwrap().parse().unwrap(); 17 | let b: i32 = str::from_utf8(b).unwrap().parse().unwrap(); 18 | a.cmp(&b) 19 | } 20 | } 21 | 22 | fn main() -> Result<(), Box> { 23 | let env_path = tempfile::tempdir()?; 24 | 25 | let env = unsafe { 26 | EnvOpenOptions::new() 27 | .map_size(10 * 1024 * 1024) // 10MB 28 | .max_dbs(3) 29 | .open(env_path)? 30 | }; 31 | 32 | let mut wtxn = env.write_txn()?; 33 | let db = env 34 | .database_options() 35 | .types::() 36 | .key_comparator::() 37 | .create(&mut wtxn)?; 38 | wtxn.commit()?; 39 | 40 | let mut wtxn = env.write_txn()?; 41 | 42 | // We fill our database with entries. 43 | db.put(&mut wtxn, "-100000", &())?; 44 | db.put(&mut wtxn, "-10000", &())?; 45 | db.put(&mut wtxn, "-1000", &())?; 46 | db.put(&mut wtxn, "-100", &())?; 47 | db.put(&mut wtxn, "100", &())?; 48 | 49 | // We check that the key are in the right order ("-100" < "-1000" < "-10000"...) 50 | let mut iter = db.iter(&wtxn)?; 51 | assert_eq!(iter.next().transpose()?, Some(("-100000", ()))); 52 | assert_eq!(iter.next().transpose()?, Some(("-10000", ()))); 53 | assert_eq!(iter.next().transpose()?, Some(("-1000", ()))); 54 | assert_eq!(iter.next().transpose()?, Some(("-100", ()))); 55 | assert_eq!(iter.next().transpose()?, Some(("100", ()))); 56 | drop(iter); 57 | 58 | Ok(()) 59 | } 60 | -------------------------------------------------------------------------------- /heed/examples/custom-dupsort-comparator.rs: -------------------------------------------------------------------------------- 1 | use std::cmp::Ordering; 2 | use std::error::Error; 3 | use std::fs; 4 | use std::path::Path; 5 | 6 | use byteorder::BigEndian; 7 | use heed::{DatabaseFlags, EnvOpenOptions}; 8 | use heed_traits::Comparator; 9 | use heed_types::{Str, U128}; 10 | 11 | enum DescendingIntCmp {} 12 | 13 | impl Comparator for DescendingIntCmp { 14 | fn compare(a: &[u8], b: &[u8]) -> Ordering { 15 | a.cmp(b).reverse() 16 | } 17 | } 18 | 19 | fn main() -> Result<(), Box> { 20 | let env_path = Path::new("target").join("custom-dupsort-cmp.mdb"); 21 | 22 | let _ = fs::remove_dir_all(&env_path); 23 | 24 | fs::create_dir_all(&env_path)?; 25 | let env = unsafe { 26 | EnvOpenOptions::new() 27 | .map_size(10 * 1024 * 1024) // 10MB 28 | .max_dbs(3) 29 | .open(env_path)? 30 | }; 31 | 32 | let mut wtxn = env.write_txn()?; 33 | let db = env 34 | .database_options() 35 | .types::>() 36 | .flags(DatabaseFlags::DUP_SORT) 37 | .dup_sort_comparator::() 38 | .create(&mut wtxn)?; 39 | wtxn.commit()?; 40 | 41 | let mut wtxn = env.write_txn()?; 42 | 43 | // We fill our database with entries. 44 | db.put(&mut wtxn, "1", &1)?; 45 | db.put(&mut wtxn, "1", &2)?; 46 | db.put(&mut wtxn, "1", &3)?; 47 | db.put(&mut wtxn, "2", &4)?; 48 | db.put(&mut wtxn, "1", &5)?; 49 | db.put(&mut wtxn, "0", &0)?; 50 | 51 | // We check that the keys are in lexicographic and values in descending order. 52 | let mut iter = db.iter(&wtxn)?; 53 | assert_eq!(iter.next().transpose()?, Some(("0", 0))); 54 | assert_eq!(iter.next().transpose()?, Some(("1", 5))); 55 | assert_eq!(iter.next().transpose()?, Some(("1", 3))); 56 | assert_eq!(iter.next().transpose()?, Some(("1", 2))); 57 | assert_eq!(iter.next().transpose()?, Some(("1", 1))); 58 | assert_eq!(iter.next().transpose()?, Some(("2", 4))); 59 | drop(iter); 60 | 61 | Ok(()) 62 | } 63 | -------------------------------------------------------------------------------- /heed/examples/multi-env.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use byteorder::BE; 4 | use heed::types::*; 5 | use heed::{Database, EnvOpenOptions}; 6 | 7 | type BEU32 = U32; 8 | 9 | fn main() -> Result<(), Box> { 10 | let env1_path = tempfile::tempdir()?; 11 | let env2_path = tempfile::tempdir()?; 12 | let env1 = unsafe { 13 | EnvOpenOptions::new() 14 | .map_size(10 * 1024 * 1024) // 10MB 15 | .max_dbs(3000) 16 | .open(env1_path)? 17 | }; 18 | 19 | let env2 = unsafe { 20 | EnvOpenOptions::new() 21 | .map_size(10 * 1024 * 1024) // 10MB 22 | .max_dbs(3000) 23 | .open(env2_path)? 24 | }; 25 | 26 | let mut wtxn1 = env1.write_txn()?; 27 | let mut wtxn2 = env2.write_txn()?; 28 | let db1: Database = env1.create_database(&mut wtxn1, Some("hello"))?; 29 | let db2: Database = env2.create_database(&mut wtxn2, Some("hello"))?; 30 | 31 | // clear db 32 | db1.clear(&mut wtxn1)?; 33 | wtxn1.commit()?; 34 | 35 | // clear db 36 | db2.clear(&mut wtxn2)?; 37 | wtxn2.commit()?; 38 | 39 | // ----- 40 | 41 | let mut wtxn1 = env1.write_txn()?; 42 | 43 | db1.put(&mut wtxn1, "what", &[4, 5][..])?; 44 | db1.get(&wtxn1, "what")?; 45 | wtxn1.commit()?; 46 | 47 | let rtxn2 = env2.read_txn()?; 48 | let ret = db2.last(&rtxn2)?; 49 | assert_eq!(ret, None); 50 | 51 | Ok(()) 52 | } 53 | -------------------------------------------------------------------------------- /heed/examples/nested.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use heed::types::*; 4 | use heed::{Database, EnvOpenOptions}; 5 | 6 | fn main() -> Result<(), Box> { 7 | let path = tempfile::tempdir()?; 8 | 9 | let env = unsafe { 10 | EnvOpenOptions::new() 11 | .map_size(10 * 1024 * 1024) // 10MB 12 | .max_dbs(3000) 13 | .open(path)? 14 | }; 15 | 16 | // here the key will be an str and the data will be a slice of u8 17 | let mut wtxn = env.write_txn()?; 18 | let db: Database = env.create_database(&mut wtxn, None)?; 19 | 20 | // clear db 21 | db.clear(&mut wtxn)?; 22 | wtxn.commit()?; 23 | 24 | // ----- 25 | 26 | let mut wtxn = env.write_txn()?; 27 | let mut nwtxn = env.nested_write_txn(&mut wtxn)?; 28 | 29 | db.put(&mut nwtxn, "what", &[4, 5][..])?; 30 | let ret = db.get(&nwtxn, "what")?; 31 | println!("nested(1) \"what\": {:?}", ret); 32 | 33 | println!("nested(1) abort"); 34 | nwtxn.abort(); 35 | 36 | let ret = db.get(&wtxn, "what")?; 37 | println!("parent \"what\": {:?}", ret); 38 | 39 | // ------ 40 | println!(); 41 | 42 | // also try with multiple levels of nesting 43 | let mut nwtxn = env.nested_write_txn(&mut wtxn)?; 44 | let mut nnwtxn = env.nested_write_txn(&mut nwtxn)?; 45 | 46 | db.put(&mut nnwtxn, "humm...", &[6, 7][..])?; 47 | let ret = db.get(&nnwtxn, "humm...")?; 48 | println!("nested(2) \"humm...\": {:?}", ret); 49 | 50 | println!("nested(2) commit"); 51 | nnwtxn.commit()?; 52 | nwtxn.commit()?; 53 | 54 | let ret = db.get(&wtxn, "humm...")?; 55 | println!("parent \"humm...\": {:?}", ret); 56 | 57 | db.put(&mut wtxn, "hello", &[2, 3][..])?; 58 | 59 | let ret = db.get(&wtxn, "hello")?; 60 | println!("parent \"hello\": {:?}", ret); 61 | 62 | println!("parent commit"); 63 | wtxn.commit()?; 64 | 65 | // ------ 66 | println!(); 67 | 68 | let rtxn = env.read_txn()?; 69 | 70 | let ret = db.get(&rtxn, "hello")?; 71 | println!("parent (reader) \"hello\": {:?}", ret); 72 | 73 | let ret = db.get(&rtxn, "humm...")?; 74 | println!("parent (reader) \"humm...\": {:?}", ret); 75 | 76 | Ok(()) 77 | } 78 | -------------------------------------------------------------------------------- /heed/examples/prev-snapshot.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use heed::types::*; 4 | use heed::{Database, EnvFlags, EnvOpenOptions}; 5 | 6 | // In this test we are checking that we can move to a previous environement snapshot. 7 | fn main() -> Result<(), Box> { 8 | let env_path = tempfile::tempdir()?; 9 | 10 | let env = unsafe { 11 | EnvOpenOptions::new() 12 | .map_size(10 * 1024 * 1024) // 10MB 13 | .max_dbs(3) 14 | .open(&env_path)? 15 | }; 16 | 17 | let mut wtxn = env.write_txn()?; 18 | let db: Database = env.create_database(&mut wtxn, None)?; 19 | 20 | // We fill the db database with entries. 21 | db.put(&mut wtxn, "I am here", "to test things")?; 22 | db.put(&mut wtxn, "I am here too", "for the same purpose")?; 23 | 24 | wtxn.commit()?; 25 | 26 | env.prepare_for_closing().wait(); 27 | 28 | // We can get the env state from before the last commit 29 | // and therefore see an empty env. 30 | let env = unsafe { 31 | EnvOpenOptions::new() 32 | .map_size(10 * 1024 * 1024) // 10MB 33 | .max_dbs(3) 34 | .flags(EnvFlags::PREV_SNAPSHOT) 35 | .open(&env_path)? 36 | }; 37 | 38 | let mut wtxn = env.write_txn()?; 39 | let db: Database = env.create_database(&mut wtxn, None)?; 40 | 41 | assert!(db.is_empty(&wtxn)?); 42 | 43 | wtxn.abort(); 44 | env.prepare_for_closing().wait(); 45 | 46 | // However, if we don't commit we can still get 47 | // back the latest version of the env. 48 | let env = unsafe { 49 | EnvOpenOptions::new() 50 | .map_size(10 * 1024 * 1024) // 10MB 51 | .max_dbs(3) 52 | .open(&env_path)? 53 | }; 54 | 55 | let mut wtxn = env.write_txn()?; 56 | let db: Database = env.create_database(&mut wtxn, None)?; 57 | 58 | assert_eq!(db.get(&wtxn, "I am here")?, Some("to test things")); 59 | assert_eq!(db.get(&wtxn, "I am here too")?, Some("for the same purpose")); 60 | 61 | // And write new stuff in the env. 62 | db.put(&mut wtxn, "I will fade away", "I am so sad")?; 63 | 64 | wtxn.commit()?; 65 | env.prepare_for_closing().wait(); 66 | 67 | // Once again we can get back the previous version 68 | // of the env and see some entries disappear. 69 | let env = unsafe { 70 | EnvOpenOptions::new() 71 | .map_size(10 * 1024 * 1024) // 10MB 72 | .max_dbs(3) 73 | .flags(EnvFlags::PREV_SNAPSHOT) 74 | .open(&env_path)? 75 | }; 76 | 77 | let rtxn = env.read_txn()?; 78 | let db: Database = env.open_database(&rtxn, None)?.unwrap(); 79 | 80 | assert_eq!(db.get(&rtxn, "I am here")?, Some("to test things")); 81 | assert_eq!(db.get(&rtxn, "I am here too")?, Some("for the same purpose")); 82 | assert_eq!(db.get(&rtxn, "I will fade away")?, None); 83 | 84 | Ok(()) 85 | } 86 | -------------------------------------------------------------------------------- /heed/examples/rmp-serde.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use heed::types::{SerdeRmp, Str}; 4 | use heed::{Database, EnvOpenOptions}; 5 | use serde::{Deserialize, Serialize}; 6 | 7 | fn main() -> Result<(), Box> { 8 | let path = tempfile::tempdir()?; 9 | 10 | let env = unsafe { 11 | EnvOpenOptions::new() 12 | .map_size(10 * 1024 * 1024) // 10MB 13 | .max_dbs(3000) 14 | .open(path)? 15 | }; 16 | 17 | // you can specify that a database will support some typed key/data 18 | // serde types are also supported!!! 19 | #[derive(Debug, Serialize, Deserialize)] 20 | struct Hello<'a> { 21 | string: &'a str, 22 | } 23 | 24 | let mut wtxn = env.write_txn()?; 25 | let db: Database> = env.create_database(&mut wtxn, Some("serde-rmp"))?; 26 | 27 | let hello = Hello { string: "hi" }; 28 | db.put(&mut wtxn, "hello", &hello)?; 29 | 30 | let ret: Option = db.get(&wtxn, "hello")?; 31 | println!("serde-rmp:\t{:?}", ret); 32 | 33 | wtxn.commit()?; 34 | 35 | Ok(()) 36 | } 37 | -------------------------------------------------------------------------------- /heed/src/cursor.rs: -------------------------------------------------------------------------------- 1 | use std::ops::{Deref, DerefMut}; 2 | use std::{marker, mem, ptr}; 3 | 4 | use crate::mdb::error::mdb_result; 5 | use crate::mdb::ffi; 6 | use crate::*; 7 | 8 | pub struct RoCursor<'txn> { 9 | cursor: *mut ffi::MDB_cursor, 10 | _marker: marker::PhantomData<&'txn ()>, 11 | } 12 | 13 | impl<'txn> RoCursor<'txn> { 14 | pub(crate) fn new(txn: &'txn RoTxn, dbi: ffi::MDB_dbi) -> Result> { 15 | let mut cursor: *mut ffi::MDB_cursor = ptr::null_mut(); 16 | let mut txn = txn.txn_ptr(); 17 | unsafe { mdb_result(ffi::mdb_cursor_open(txn.as_mut(), dbi, &mut cursor))? } 18 | Ok(RoCursor { cursor, _marker: marker::PhantomData }) 19 | } 20 | 21 | pub fn current(&mut self) -> Result> { 22 | let mut key_val = mem::MaybeUninit::uninit(); 23 | let mut data_val = mem::MaybeUninit::uninit(); 24 | 25 | // Move the cursor on the first database key 26 | let result = unsafe { 27 | mdb_result(ffi::mdb_cursor_get( 28 | self.cursor, 29 | key_val.as_mut_ptr(), 30 | data_val.as_mut_ptr(), 31 | ffi::cursor_op::MDB_GET_CURRENT, 32 | )) 33 | }; 34 | 35 | match result { 36 | Ok(()) => { 37 | let key = unsafe { crate::from_val(key_val.assume_init()) }; 38 | let data = unsafe { crate::from_val(data_val.assume_init()) }; 39 | Ok(Some((key, data))) 40 | } 41 | Err(e) if e.not_found() => Ok(None), 42 | Err(e) => Err(e.into()), 43 | } 44 | } 45 | 46 | pub fn move_on_first(&mut self, op: MoveOperation) -> Result> { 47 | let mut key_val = mem::MaybeUninit::uninit(); 48 | let mut data_val = mem::MaybeUninit::uninit(); 49 | 50 | let flag = match op { 51 | MoveOperation::Any => ffi::cursor_op::MDB_FIRST, 52 | MoveOperation::Dup => { 53 | unsafe { 54 | mdb_result(ffi::mdb_cursor_get( 55 | self.cursor, 56 | ptr::null_mut(), 57 | &mut ffi::MDB_val { mv_size: 0, mv_data: ptr::null_mut() }, 58 | ffi::cursor_op::MDB_FIRST_DUP, 59 | ))? 60 | }; 61 | ffi::cursor_op::MDB_GET_CURRENT 62 | } 63 | MoveOperation::NoDup => ffi::cursor_op::MDB_FIRST, 64 | }; 65 | 66 | // Move the cursor on the first database key 67 | let result = unsafe { 68 | mdb_result(ffi::mdb_cursor_get( 69 | self.cursor, 70 | key_val.as_mut_ptr(), 71 | data_val.as_mut_ptr(), 72 | flag, 73 | )) 74 | }; 75 | 76 | match result { 77 | Ok(()) => { 78 | let key = unsafe { crate::from_val(key_val.assume_init()) }; 79 | let data = unsafe { crate::from_val(data_val.assume_init()) }; 80 | Ok(Some((key, data))) 81 | } 82 | Err(e) if e.not_found() => Ok(None), 83 | Err(e) => Err(e.into()), 84 | } 85 | } 86 | 87 | pub fn move_on_last(&mut self, op: MoveOperation) -> Result> { 88 | let mut key_val = mem::MaybeUninit::uninit(); 89 | let mut data_val = mem::MaybeUninit::uninit(); 90 | 91 | let flag = match op { 92 | MoveOperation::Any => ffi::cursor_op::MDB_LAST, 93 | MoveOperation::Dup => { 94 | unsafe { 95 | mdb_result(ffi::mdb_cursor_get( 96 | self.cursor, 97 | ptr::null_mut(), 98 | &mut ffi::MDB_val { mv_size: 0, mv_data: ptr::null_mut() }, 99 | ffi::cursor_op::MDB_LAST_DUP, 100 | ))? 101 | }; 102 | ffi::cursor_op::MDB_GET_CURRENT 103 | } 104 | MoveOperation::NoDup => ffi::cursor_op::MDB_LAST, 105 | }; 106 | 107 | // Move the cursor on the first database key 108 | let result = unsafe { 109 | mdb_result(ffi::mdb_cursor_get( 110 | self.cursor, 111 | key_val.as_mut_ptr(), 112 | data_val.as_mut_ptr(), 113 | flag, 114 | )) 115 | }; 116 | 117 | match result { 118 | Ok(()) => { 119 | let key = unsafe { crate::from_val(key_val.assume_init()) }; 120 | let data = unsafe { crate::from_val(data_val.assume_init()) }; 121 | Ok(Some((key, data))) 122 | } 123 | Err(e) if e.not_found() => Ok(None), 124 | Err(e) => Err(e.into()), 125 | } 126 | } 127 | 128 | pub fn move_on_key(&mut self, key: &[u8]) -> Result { 129 | let mut key_val = unsafe { crate::into_val(key) }; 130 | 131 | // Move the cursor to the specified key 132 | let result = unsafe { 133 | mdb_result(ffi::mdb_cursor_get( 134 | self.cursor, 135 | &mut key_val, 136 | &mut ffi::MDB_val { mv_size: 0, mv_data: ptr::null_mut() }, 137 | ffi::cursor_op::MDB_SET, 138 | )) 139 | }; 140 | 141 | match result { 142 | Ok(()) => Ok(true), 143 | Err(e) if e.not_found() => Ok(false), 144 | Err(e) => Err(e.into()), 145 | } 146 | } 147 | 148 | pub fn move_on_key_greater_than_or_equal_to( 149 | &mut self, 150 | key: &[u8], 151 | ) -> Result> { 152 | let mut key_val = unsafe { crate::into_val(key) }; 153 | let mut data_val = mem::MaybeUninit::uninit(); 154 | 155 | // Move the cursor to the specified key 156 | let result = unsafe { 157 | mdb_result(ffi::mdb_cursor_get( 158 | self.cursor, 159 | &mut key_val, 160 | data_val.as_mut_ptr(), 161 | ffi::cursor_op::MDB_SET_RANGE, 162 | )) 163 | }; 164 | 165 | match result { 166 | Ok(()) => { 167 | let key = unsafe { crate::from_val(key_val) }; 168 | let data = unsafe { crate::from_val(data_val.assume_init()) }; 169 | Ok(Some((key, data))) 170 | } 171 | Err(e) if e.not_found() => Ok(None), 172 | Err(e) => Err(e.into()), 173 | } 174 | } 175 | 176 | pub fn move_on_prev(&mut self, op: MoveOperation) -> Result> { 177 | let mut key_val = mem::MaybeUninit::uninit(); 178 | let mut data_val = mem::MaybeUninit::uninit(); 179 | 180 | let flag = match op { 181 | MoveOperation::Any => ffi::cursor_op::MDB_PREV, 182 | MoveOperation::Dup => ffi::cursor_op::MDB_PREV_DUP, 183 | MoveOperation::NoDup => ffi::cursor_op::MDB_PREV_NODUP, 184 | }; 185 | 186 | // Move the cursor to the previous non-dup key 187 | let result = unsafe { 188 | mdb_result(ffi::mdb_cursor_get( 189 | self.cursor, 190 | key_val.as_mut_ptr(), 191 | data_val.as_mut_ptr(), 192 | flag, 193 | )) 194 | }; 195 | 196 | match result { 197 | Ok(()) => { 198 | let key = unsafe { crate::from_val(key_val.assume_init()) }; 199 | let data = unsafe { crate::from_val(data_val.assume_init()) }; 200 | Ok(Some((key, data))) 201 | } 202 | Err(e) if e.not_found() => Ok(None), 203 | Err(e) => Err(e.into()), 204 | } 205 | } 206 | 207 | pub fn move_on_next(&mut self, op: MoveOperation) -> Result> { 208 | let mut key_val = mem::MaybeUninit::uninit(); 209 | let mut data_val = mem::MaybeUninit::uninit(); 210 | 211 | let flag = match op { 212 | MoveOperation::Any => ffi::cursor_op::MDB_NEXT, 213 | MoveOperation::Dup => ffi::cursor_op::MDB_NEXT_DUP, 214 | MoveOperation::NoDup => ffi::cursor_op::MDB_NEXT_NODUP, 215 | }; 216 | 217 | // Move the cursor to the next non-dup key 218 | let result = unsafe { 219 | mdb_result(ffi::mdb_cursor_get( 220 | self.cursor, 221 | key_val.as_mut_ptr(), 222 | data_val.as_mut_ptr(), 223 | flag, 224 | )) 225 | }; 226 | 227 | match result { 228 | Ok(()) => { 229 | let key = unsafe { crate::from_val(key_val.assume_init()) }; 230 | let data = unsafe { crate::from_val(data_val.assume_init()) }; 231 | Ok(Some((key, data))) 232 | } 233 | Err(e) if e.not_found() => Ok(None), 234 | Err(e) => Err(e.into()), 235 | } 236 | } 237 | } 238 | 239 | impl Drop for RoCursor<'_> { 240 | fn drop(&mut self) { 241 | unsafe { ffi::mdb_cursor_close(self.cursor) } 242 | } 243 | } 244 | 245 | pub struct RwCursor<'txn> { 246 | cursor: RoCursor<'txn>, 247 | } 248 | 249 | impl<'txn> RwCursor<'txn> { 250 | pub(crate) fn new(txn: &'txn RwTxn, dbi: ffi::MDB_dbi) -> Result> { 251 | Ok(RwCursor { cursor: RoCursor::new(txn, dbi)? }) 252 | } 253 | 254 | /// Delete the entry the cursor is currently pointing to. 255 | /// 256 | /// Returns `true` if the entry was successfully deleted. 257 | /// 258 | /// # Safety 259 | /// 260 | /// It is _[undefined behavior]_ to keep a reference of a value from this database 261 | /// while modifying it. 262 | /// 263 | /// > [Values returned from the database are valid only until a subsequent update operation, 264 | /// > or the end of the transaction.](http://www.lmdb.tech/doc/group__mdb.html#structMDB__val) 265 | /// 266 | /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html 267 | pub unsafe fn del_current(&mut self) -> Result { 268 | // Delete the current entry 269 | let result = mdb_result(ffi::mdb_cursor_del(self.cursor.cursor, 0)); 270 | 271 | match result { 272 | Ok(()) => Ok(true), 273 | Err(e) if e.not_found() => Ok(false), 274 | Err(e) => Err(e.into()), 275 | } 276 | } 277 | 278 | /// Write a new value to the current entry. 279 | /// 280 | /// The given key **must** be equal to the one this cursor is pointing otherwise the database 281 | /// can be put into an inconsistent state. 282 | /// 283 | /// Returns `true` if the entry was successfully written. 284 | /// 285 | /// > This is intended to be used when the new data is the same size as the old. 286 | /// > Otherwise it will simply perform a delete of the old record followed by an insert. 287 | /// 288 | /// # Safety 289 | /// 290 | /// It is _[undefined behavior]_ to keep a reference of a value from this database while 291 | /// modifying it, so you can't use the key/value that comes from the cursor to feed 292 | /// this function. 293 | /// 294 | /// In other words: Transform the key and value that you borrow from this database into an owned 295 | /// version of them (e.g. `&str` into `String`). 296 | /// 297 | /// > [Values returned from the database are valid only until a subsequent update operation, 298 | /// > or the end of the transaction.](http://www.lmdb.tech/doc/group__mdb.html#structMDB__val) 299 | /// 300 | /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html 301 | pub unsafe fn put_current(&mut self, key: &[u8], data: &[u8]) -> Result { 302 | let mut key_val = crate::into_val(key); 303 | let mut data_val = crate::into_val(data); 304 | 305 | // Modify the pointed data 306 | let result = mdb_result(ffi::mdb_cursor_put( 307 | self.cursor.cursor, 308 | &mut key_val, 309 | &mut data_val, 310 | ffi::MDB_CURRENT, 311 | )); 312 | 313 | match result { 314 | Ok(()) => Ok(true), 315 | Err(e) if e.not_found() => Ok(false), 316 | Err(e) => Err(e.into()), 317 | } 318 | } 319 | 320 | /// Write a new value to the current entry. 321 | /// 322 | /// The given key **must** be equal to the one this cursor is pointing otherwise the database 323 | /// can be put into an inconsistent state. 324 | /// 325 | /// Returns `true` if the entry was successfully written. 326 | /// 327 | /// > This is intended to be used when the new data is the same size as the old. 328 | /// > Otherwise it will simply perform a delete of the old record followed by an insert. 329 | /// 330 | /// # Safety 331 | /// 332 | /// Please read the safety notes of the [`Self::put_current`] method. 333 | pub unsafe fn put_current_reserved_with_flags( 334 | &mut self, 335 | flags: PutFlags, 336 | key: &[u8], 337 | data_size: usize, 338 | write_func: F, 339 | ) -> Result 340 | where 341 | F: FnOnce(&mut ReservedSpace) -> io::Result<()>, 342 | { 343 | let mut key_val = crate::into_val(key); 344 | let mut reserved = ffi::reserve_size_val(data_size); 345 | let flags = ffi::MDB_RESERVE | flags.bits(); 346 | 347 | let result = 348 | mdb_result(ffi::mdb_cursor_put(self.cursor.cursor, &mut key_val, &mut reserved, flags)); 349 | 350 | let found = match result { 351 | Ok(()) => true, 352 | Err(e) if e.not_found() => false, 353 | Err(e) => return Err(e.into()), 354 | }; 355 | 356 | let mut reserved = ReservedSpace::from_val(reserved); 357 | write_func(&mut reserved)?; 358 | 359 | if reserved.remaining() == 0 { 360 | Ok(found) 361 | } else { 362 | Err(io::Error::from(io::ErrorKind::UnexpectedEof).into()) 363 | } 364 | } 365 | 366 | /// Append the given key/value pair to the end of the database. 367 | /// 368 | /// If a key is inserted that is less than any previous key a `KeyExist` error 369 | /// is returned and the key is not inserted into the database. 370 | /// 371 | /// # Safety 372 | /// 373 | /// It is _[undefined behavior]_ to keep a reference of a value from this database while 374 | /// modifying it, so you can't use the key/value that comes from the cursor to feed 375 | /// this function. 376 | /// 377 | /// In other words: Transform the key and value that you borrow from this database into an owned 378 | /// version of them (e.g. `&str` into `String`). 379 | /// 380 | /// > [Values returned from the database are valid only until a subsequent update operation, 381 | /// > or the end of the transaction.](http://www.lmdb.tech/doc/group__mdb.html#structMDB__val) 382 | /// 383 | /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html 384 | pub unsafe fn put_current_with_flags( 385 | &mut self, 386 | flags: PutFlags, 387 | key: &[u8], 388 | data: &[u8], 389 | ) -> Result<()> { 390 | let mut key_val = crate::into_val(key); 391 | let mut data_val = crate::into_val(data); 392 | 393 | // Modify the pointed data 394 | let result = mdb_result(ffi::mdb_cursor_put( 395 | self.cursor.cursor, 396 | &mut key_val, 397 | &mut data_val, 398 | flags.bits(), 399 | )); 400 | 401 | result.map_err(Into::into) 402 | } 403 | } 404 | 405 | impl<'txn> Deref for RwCursor<'txn> { 406 | type Target = RoCursor<'txn>; 407 | 408 | fn deref(&self) -> &Self::Target { 409 | &self.cursor 410 | } 411 | } 412 | 413 | impl DerefMut for RwCursor<'_> { 414 | fn deref_mut(&mut self) -> &mut Self::Target { 415 | &mut self.cursor 416 | } 417 | } 418 | 419 | /// The way the `Iterator::next/prev` method behaves towards DUP data. 420 | #[derive(Debug, Clone, Copy)] 421 | pub enum MoveOperation { 422 | /// Move on the next/prev entry, wether it's the same key or not. 423 | Any, 424 | /// Move on the next/prev data of the current key. 425 | Dup, 426 | /// Move on the next/prev entry which is the next/prev key. 427 | /// Skip the multiple values of the current key. 428 | NoDup, 429 | } 430 | -------------------------------------------------------------------------------- /heed/src/databases/mod.rs: -------------------------------------------------------------------------------- 1 | pub use database::{Database, DatabaseOpenOptions}; 2 | #[cfg(master3)] 3 | pub use encrypted_database::{EncryptedDatabase, EncryptedDatabaseOpenOptions}; 4 | 5 | mod database; 6 | #[cfg(master3)] 7 | mod encrypted_database; 8 | 9 | /// Statistics for a database in the environment. 10 | #[derive(Debug, Clone, Copy)] 11 | pub struct DatabaseStat { 12 | /// Size of a database page. 13 | /// This is currently the same for all databases. 14 | pub page_size: u32, 15 | /// Depth (height) of the B-tree. 16 | pub depth: u32, 17 | /// Number of internal (non-leaf) pages 18 | pub branch_pages: usize, 19 | /// Number of leaf pages. 20 | pub leaf_pages: usize, 21 | /// Number of overflow pages. 22 | pub overflow_pages: usize, 23 | /// Number of data items. 24 | pub entries: usize, 25 | } 26 | -------------------------------------------------------------------------------- /heed/src/envs/encrypted_env.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | use std::fs::File; 3 | use std::panic::catch_unwind; 4 | use std::path::Path; 5 | 6 | use aead::generic_array::typenum::Unsigned; 7 | use aead::{AeadMutInPlace, Key, KeyInit, Nonce, Tag}; 8 | 9 | use super::{Env, EnvClosingEvent, EnvInfo, FlagSetMode}; 10 | use crate::databases::{EncryptedDatabase, EncryptedDatabaseOpenOptions}; 11 | use crate::mdb::ffi::{self}; 12 | use crate::{CompactionOption, EnvFlags, Result, RoTxn, RwTxn, Unspecified, WithTls}; 13 | #[allow(unused)] // fro cargo auto doc links 14 | use crate::{Database, EnvOpenOptions}; 15 | 16 | /// An environment handle constructed by using [`EnvOpenOptions::open_encrypted`]. 17 | #[derive(Clone)] 18 | pub struct EncryptedEnv { 19 | pub(crate) inner: Env, 20 | } 21 | 22 | impl EncryptedEnv { 23 | /// The size of the data file on disk. 24 | /// 25 | /// # Example 26 | /// 27 | /// ``` 28 | /// use heed::EnvOpenOptions; 29 | /// 30 | /// # fn main() -> Result<(), Box> { 31 | /// let dir = tempfile::tempdir()?; 32 | /// let size_in_bytes = 1024 * 1024; 33 | /// let env = unsafe { EnvOpenOptions::new().map_size(size_in_bytes).open(dir.path())? }; 34 | /// 35 | /// let actual_size = env.real_disk_size()? as usize; 36 | /// assert!(actual_size < size_in_bytes); 37 | /// # Ok(()) } 38 | /// ``` 39 | pub fn real_disk_size(&self) -> Result { 40 | self.inner.real_disk_size() 41 | } 42 | 43 | /// Return the raw flags the environment was opened with. 44 | /// 45 | /// Returns `None` if the environment flags are different from the [`EnvFlags`] set. 46 | pub fn flags(&self) -> Result> { 47 | self.inner.flags() 48 | } 49 | 50 | /// Enable or disable the environment's currently active [`EnvFlags`]. 51 | /// 52 | /// ``` 53 | /// use std::fs; 54 | /// use std::path::Path; 55 | /// use heed::{EnvOpenOptions, Database, EnvFlags, FlagSetMode}; 56 | /// use heed::types::*; 57 | /// 58 | /// # fn main() -> Result<(), Box> { 59 | /// let mut env_builder = EnvOpenOptions::new(); 60 | /// let dir = tempfile::tempdir().unwrap(); 61 | /// let env = unsafe { env_builder.open(dir.path())? }; 62 | /// 63 | /// // Env was opened without flags. 64 | /// assert_eq!(env.get_flags().unwrap(), EnvFlags::empty().bits()); 65 | /// 66 | /// // Enable a flag after opening. 67 | /// unsafe { env.set_flags(EnvFlags::NO_SYNC, FlagSetMode::Enable).unwrap(); } 68 | /// assert_eq!(env.get_flags().unwrap(), EnvFlags::NO_SYNC.bits()); 69 | /// 70 | /// // Disable a flag after opening. 71 | /// unsafe { env.set_flags(EnvFlags::NO_SYNC, FlagSetMode::Disable).unwrap(); } 72 | /// assert_eq!(env.get_flags().unwrap(), EnvFlags::empty().bits()); 73 | /// # Ok(()) } 74 | /// ``` 75 | /// 76 | /// # Safety 77 | /// 78 | /// It is unsafe to use unsafe LMDB flags such as `NO_SYNC`, `NO_META_SYNC`, or `NO_LOCK`. 79 | /// 80 | /// LMDB also requires that only 1 thread calls this function at any given moment. 81 | /// Neither `heed` or LMDB check for this condition, so the caller must ensure it explicitly. 82 | pub unsafe fn set_flags(&self, flags: EnvFlags, mode: FlagSetMode) -> Result<()> { 83 | self.inner.set_flags(flags, mode) 84 | } 85 | 86 | /// Return the raw flags the environment is currently set with. 87 | pub fn get_flags(&self) -> Result { 88 | self.inner.get_flags() 89 | } 90 | 91 | /// Returns some basic informations about this environment. 92 | pub fn info(&self) -> EnvInfo { 93 | self.inner.info() 94 | } 95 | 96 | /// Returns the size used by all the databases in the environment without the free pages. 97 | /// 98 | /// It is crucial to configure [`EnvOpenOptions::max_dbs`] with a sufficiently large value 99 | /// before invoking this function. All databases within the environment will be opened 100 | /// and remain so. 101 | pub fn non_free_pages_size(&self) -> Result { 102 | self.inner.non_free_pages_size() 103 | } 104 | 105 | /// Options and flags which can be used to configure how a [`Database`] is opened. 106 | pub fn database_options(&self) -> EncryptedDatabaseOpenOptions { 107 | EncryptedDatabaseOpenOptions::new(self) 108 | } 109 | 110 | /// Opens a typed database that already exists in this environment. 111 | /// 112 | /// If the database was previously opened in this program run, types will be checked. 113 | /// 114 | /// ## Important Information 115 | /// 116 | /// LMDB has an important restriction on the unnamed database when named ones are opened. 117 | /// The names of the named databases are stored as keys in the unnamed one and are immutable, 118 | /// and these keys can only be read and not written. 119 | /// 120 | /// ## LMDB read-only access of existing database 121 | /// 122 | /// In the case of accessing a database in a read-only manner from another process 123 | /// where you wrote, you might need to manually call [`RoTxn::commit`] to get metadata 124 | /// and the database handles opened and shared with the global [`Env`] handle. 125 | /// 126 | /// If not done, you might raise `Io(Os { code: 22, kind: InvalidInput, message: "Invalid argument" })` 127 | /// known as `EINVAL`. 128 | pub fn open_database( 129 | &self, 130 | rtxn: &RoTxn, 131 | name: Option<&str>, 132 | ) -> Result>> 133 | where 134 | KC: 'static, 135 | DC: 'static, 136 | { 137 | let mut options = self.database_options().types::(); 138 | if let Some(name) = name { 139 | options.name(name); 140 | } 141 | options.open(rtxn) 142 | } 143 | 144 | /// Creates a typed database that can already exist in this environment. 145 | /// 146 | /// If the database was previously opened during this program run, types will be checked. 147 | /// 148 | /// ## Important Information 149 | /// 150 | /// LMDB has an important restriction on the unnamed database when named ones are opened. 151 | /// The names of the named databases are stored as keys in the unnamed one and are immutable, 152 | /// and these keys can only be read and not written. 153 | pub fn create_database( 154 | &self, 155 | wtxn: &mut RwTxn, 156 | name: Option<&str>, 157 | ) -> Result> 158 | where 159 | KC: 'static, 160 | DC: 'static, 161 | { 162 | let mut options = self.database_options().types::(); 163 | if let Some(name) = name { 164 | options.name(name); 165 | } 166 | options.create(wtxn) 167 | } 168 | 169 | /// Create a transaction with read and write access for use with the environment. 170 | /// 171 | /// ## LMDB Limitations 172 | /// 173 | /// Only one [`RwTxn`] may exist simultaneously in the current environment. 174 | /// If another write transaction is initiated, while another write transaction exists 175 | /// the thread initiating the new one will wait on a mutex upon completion of the previous 176 | /// transaction. 177 | pub fn write_txn(&self) -> Result { 178 | self.inner.write_txn() 179 | } 180 | 181 | /// Create a nested transaction with read and write access for use with the environment. 182 | /// 183 | /// The new transaction will be a nested transaction, with the transaction indicated by parent 184 | /// as its parent. Transactions may be nested to any level. 185 | /// 186 | /// A parent transaction and its cursors may not issue any other operations than _commit_ and 187 | /// _abort_ while it has active child transactions. 188 | pub fn nested_write_txn<'p>(&'p self, parent: &'p mut RwTxn) -> Result> { 189 | self.inner.nested_write_txn(parent) 190 | } 191 | 192 | /// Create a transaction with read-only access for use with the environment. 193 | /// 194 | /// You can make this transaction `Send`able between threads by opening 195 | /// the environment with the [`EnvOpenOptions::read_txn_without_tls`] 196 | /// method. 197 | /// 198 | /// See [`Self::static_read_txn`] if you want the txn to own the environment. 199 | /// 200 | /// ## LMDB Limitations 201 | /// 202 | /// It's possible to have multiple read transactions in the same environment 203 | /// while there is a write transaction ongoing. 204 | /// 205 | /// But read transactions prevent reuse of pages freed by newer write transactions, 206 | /// thus the database can grow quickly. Write transactions prevent other write transactions, 207 | /// since writes are serialized. 208 | /// 209 | /// So avoid long-lived read transactions. 210 | /// 211 | /// ## Errors 212 | /// 213 | /// * [`crate::MdbError::Panic`]: A fatal error occurred earlier, and the environment must be shut down 214 | /// * [`crate::MdbError::MapResized`]: Another process wrote data beyond this [`Env`] mapsize and this env 215 | /// map must be resized 216 | /// * [`crate::MdbError::ReadersFull`]: a read-only transaction was requested, and the reader lock table is 217 | /// full 218 | pub fn read_txn(&self) -> Result> { 219 | self.inner.read_txn() 220 | } 221 | 222 | /// Create a transaction with read-only access for use with the environment. 223 | /// Contrary to [`Self::read_txn`], this version **owns** the environment, which 224 | /// means you won't be able to close the environment while this transaction is alive. 225 | /// 226 | /// You can make this transaction `Send`able between threads by opening 227 | /// the environment with the [`EnvOpenOptions::read_txn_without_tls`] 228 | /// method. 229 | /// 230 | /// ## LMDB Limitations 231 | /// 232 | /// It's possible to have multiple read transactions in the same environment 233 | /// while there is a write transaction ongoing. 234 | /// 235 | /// But read transactions prevent reuse of pages freed by newer write transactions, 236 | /// thus the database can grow quickly. Write transactions prevent other write transactions, 237 | /// since writes are serialized. 238 | /// 239 | /// So avoid long-lived read transactions. 240 | /// 241 | /// ## Errors 242 | /// 243 | /// * [`crate::MdbError::Panic`]: A fatal error occurred earlier, and the environment must be shut down 244 | /// * [`crate::MdbError::MapResized`]: Another process wrote data beyond this [`Env`] mapsize and this env 245 | /// map must be resized 246 | /// * [`crate::MdbError::ReadersFull`]: a read-only transaction was requested, and the reader lock table is 247 | /// full 248 | pub fn static_read_txn(self) -> Result> { 249 | self.inner.static_read_txn() 250 | } 251 | 252 | /// Copy an LMDB environment to the specified path, with options. 253 | /// 254 | /// This function may be used to make a backup of an existing environment. 255 | /// No lockfile is created, since it gets recreated at need. 256 | /// 257 | /// Note that the file must be seek to the beginning after the copy is complete. 258 | /// 259 | /// ``` 260 | /// use std::fs; 261 | /// use std::io::{Read, Seek, SeekFrom}; 262 | /// use std::path::Path; 263 | /// use heed3::{EnvOpenOptions, Database, EnvFlags, FlagSetMode, CompactionOption}; 264 | /// use heed3::types::*; 265 | /// use memchr::memmem::find_iter; 266 | /// 267 | /// # fn main() -> Result<(), Box> { 268 | /// # let dir = tempfile::tempdir()?; 269 | /// # let env = unsafe { EnvOpenOptions::new() 270 | /// # .map_size(10 * 1024 * 1024) // 10MB 271 | /// # .max_dbs(3000) 272 | /// # .open(dir.path())? 273 | /// # }; 274 | /// 275 | /// let mut wtxn = env.write_txn()?; 276 | /// let db: Database = env.create_database(&mut wtxn, None)?; 277 | /// 278 | /// db.put(&mut wtxn, &"hello0", &"world0")?; 279 | /// db.put(&mut wtxn, &"hello1", &"world1")?; 280 | /// db.put(&mut wtxn, &"hello2", &"world2")?; 281 | /// db.put(&mut wtxn, &"hello3", &"world3")?; 282 | /// 283 | /// wtxn.commit()?; 284 | /// 285 | /// let mut tmp_file = tempfile::tempfile()?; 286 | /// env.copy_to_file(&mut tmp_file, CompactionOption::Enabled)?; 287 | /// let offset = tmp_file.seek(SeekFrom::Current(0))?; 288 | /// assert_ne!(offset, 0); 289 | /// 290 | /// let offset = tmp_file.seek(SeekFrom::Start(0))?; 291 | /// assert_eq!(offset, 0); 292 | /// 293 | /// let mut content = Vec::new(); 294 | /// tmp_file.read_to_end(&mut content)?; 295 | /// assert!(content.len() > 8 * 6); // more than 8 times hellox + worldx 296 | /// # Ok(()) } 297 | /// ``` 298 | pub fn copy_to_file(&self, file: &mut File, option: CompactionOption) -> Result<()> { 299 | self.inner.copy_to_file(file, option) 300 | } 301 | 302 | /// Copy an LMDB environment to the specified file descriptor, with compaction option. 303 | /// 304 | /// This function may be used to make a backup of an existing environment. 305 | /// No lockfile is created, since it gets recreated at need. 306 | /// 307 | /// # Safety 308 | /// 309 | /// The [`ffi::mdb_filehandle_t`] must have already been opened for Write access. 310 | pub unsafe fn copy_to_fd( 311 | &self, 312 | fd: ffi::mdb_filehandle_t, 313 | option: CompactionOption, 314 | ) -> Result<()> { 315 | self.inner.copy_to_fd(fd, option) 316 | } 317 | 318 | /// Flush the data buffers to disk. 319 | pub fn force_sync(&self) -> Result<()> { 320 | self.inner.force_sync() 321 | } 322 | 323 | /// Returns the canonicalized path where this env lives. 324 | pub fn path(&self) -> &Path { 325 | self.inner.path() 326 | } 327 | 328 | /// Returns the maximum number of threads/reader slots for the environment. 329 | pub fn max_readers(&self) -> u32 { 330 | self.inner.max_readers() 331 | } 332 | 333 | /// Get the maximum size of keys and MDB_DUPSORT data we can write. 334 | /// 335 | /// Depends on the compile-time constant MDB_MAXKEYSIZE. Default 511 336 | pub fn max_key_size(&self) -> usize { 337 | self.inner.max_key_size() 338 | } 339 | 340 | /// Returns an `EnvClosingEvent` that can be used to wait for the closing event, 341 | /// multiple threads can wait on this event. 342 | /// 343 | /// Make sure that you drop all the copies of `Env`s you have, env closing are triggered 344 | /// when all references are dropped, the last one will eventually close the environment. 345 | pub fn prepare_for_closing(self) -> EnvClosingEvent { 346 | self.inner.prepare_for_closing() 347 | } 348 | 349 | /// Check for stale entries in the reader lock table and clear them. 350 | /// 351 | /// Returns the number of stale readers cleared. 352 | pub fn clear_stale_readers(&self) -> Result { 353 | self.inner.clear_stale_readers() 354 | } 355 | 356 | /// Resize the memory map to a new size. 357 | /// 358 | /// # Safety 359 | /// 360 | /// According to the [LMDB documentation](http://www.lmdb.tech/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5), 361 | /// it is okay to call `mdb_env_set_mapsize` for an open environment as long as no transactions are active, 362 | /// but the library does not check for this condition, so the caller must ensure it explicitly. 363 | pub unsafe fn resize(&self, new_size: usize) -> Result<()> { 364 | self.inner.resize(new_size) 365 | } 366 | } 367 | 368 | unsafe impl Send for EncryptedEnv {} 369 | unsafe impl Sync for EncryptedEnv {} 370 | 371 | impl fmt::Debug for EncryptedEnv { 372 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 373 | f.debug_struct("EncryptedEnv") 374 | .field("path", &self.inner.path().display()) 375 | .finish_non_exhaustive() 376 | } 377 | } 378 | 379 | fn encrypt( 380 | key: &[u8], 381 | nonce: &[u8], 382 | aad: &[u8], 383 | plaintext: &[u8], 384 | chipertext_out: &mut [u8], 385 | auth_out: &mut [u8], 386 | ) -> aead::Result<()> { 387 | chipertext_out.copy_from_slice(plaintext); 388 | let key: &Key = key.into(); 389 | let nonce: &Nonce = if nonce.len() >= A::NonceSize::USIZE { 390 | nonce[..A::NonceSize::USIZE].into() 391 | } else { 392 | return Err(aead::Error); 393 | }; 394 | let mut aead = A::new(key); 395 | let tag = aead.encrypt_in_place_detached(nonce, aad, chipertext_out)?; 396 | auth_out.copy_from_slice(&tag); 397 | Ok(()) 398 | } 399 | 400 | fn decrypt( 401 | key: &[u8], 402 | nonce: &[u8], 403 | aad: &[u8], 404 | chipher_text: &[u8], 405 | output: &mut [u8], 406 | auth_in: &[u8], 407 | ) -> aead::Result<()> { 408 | output.copy_from_slice(chipher_text); 409 | let key: &Key = key.into(); 410 | let nonce: &Nonce = if nonce.len() >= A::NonceSize::USIZE { 411 | nonce[..A::NonceSize::USIZE].into() 412 | } else { 413 | return Err(aead::Error); 414 | }; 415 | let tag: &Tag = auth_in.into(); 416 | let mut aead = A::new(key); 417 | aead.decrypt_in_place_detached(nonce, aad, output, tag) 418 | } 419 | 420 | /// The wrapper function that is called by LMDB that directly calls 421 | /// the Rust idiomatic function internally. 422 | pub(crate) unsafe extern "C" fn encrypt_func_wrapper( 423 | src: *const ffi::MDB_val, 424 | dst: *mut ffi::MDB_val, 425 | key_ptr: *const ffi::MDB_val, 426 | encdec: i32, 427 | ) -> i32 { 428 | let result = catch_unwind(|| { 429 | let input = std::slice::from_raw_parts((*src).mv_data as *const u8, (*src).mv_size); 430 | let output = std::slice::from_raw_parts_mut((*dst).mv_data as *mut u8, (*dst).mv_size); 431 | let key = std::slice::from_raw_parts((*key_ptr).mv_data as *const u8, (*key_ptr).mv_size); 432 | let iv = std::slice::from_raw_parts( 433 | (*key_ptr.offset(1)).mv_data as *const u8, 434 | (*key_ptr.offset(1)).mv_size, 435 | ); 436 | let auth = std::slice::from_raw_parts_mut( 437 | (*key_ptr.offset(2)).mv_data as *mut u8, 438 | (*key_ptr.offset(2)).mv_size, 439 | ); 440 | 441 | let aad = []; 442 | let nonce = iv; 443 | let result = if encdec == 1 { 444 | encrypt::(key, nonce, &aad, input, output, auth) 445 | } else { 446 | decrypt::(key, nonce, &aad, input, output, auth) 447 | }; 448 | 449 | result.is_err() as i32 450 | }); 451 | 452 | result.unwrap_or(1) 453 | } 454 | -------------------------------------------------------------------------------- /heed/src/envs/mod.rs: -------------------------------------------------------------------------------- 1 | use std::cmp::Ordering; 2 | use std::collections::HashMap; 3 | use std::ffi::c_void; 4 | use std::fs::{File, Metadata}; 5 | #[cfg(unix)] 6 | use std::os::unix::io::{AsRawFd, BorrowedFd, RawFd}; 7 | use std::panic::catch_unwind; 8 | use std::path::{Path, PathBuf}; 9 | use std::process::abort; 10 | use std::sync::{Arc, LazyLock, RwLock}; 11 | use std::time::Duration; 12 | #[cfg(windows)] 13 | use std::{ 14 | ffi::OsStr, 15 | os::windows::io::{AsRawHandle as _, BorrowedHandle, RawHandle}, 16 | }; 17 | use std::{fmt, io}; 18 | 19 | use heed_traits::{Comparator, LexicographicComparator}; 20 | use synchronoise::event::SignalEvent; 21 | 22 | use crate::mdb::ffi; 23 | #[allow(unused)] // for cargo auto doc links 24 | use crate::{Database, DatabaseFlags}; 25 | 26 | #[cfg(master3)] 27 | mod encrypted_env; 28 | mod env; 29 | mod env_open_options; 30 | 31 | #[cfg(master3)] 32 | pub use encrypted_env::EncryptedEnv; 33 | pub use env::Env; 34 | pub(crate) use env::EnvInner; 35 | pub use env_open_options::EnvOpenOptions; 36 | 37 | /// Records the current list of opened environments for tracking purposes. The canonical 38 | /// path of an environment is removed when either an `Env` or `EncryptedEnv` is closed. 39 | static OPENED_ENV: LazyLock>>> = 40 | LazyLock::new(RwLock::default); 41 | 42 | /// Returns a struct that allows to wait for the effective closing of an environment. 43 | pub fn env_closing_event>(path: P) -> Option { 44 | let lock = OPENED_ENV.read().unwrap(); 45 | lock.get(path.as_ref()).map(|signal_event| EnvClosingEvent(signal_event.clone())) 46 | } 47 | 48 | /// Contains information about the environment. 49 | #[derive(Debug, Clone, Copy)] 50 | pub struct EnvInfo { 51 | /// Address of the map, if fixed. 52 | pub map_addr: *mut c_void, 53 | /// Size of the data memory map. 54 | pub map_size: usize, 55 | /// ID of the last used page. 56 | pub last_page_number: usize, 57 | /// ID of the last committed transaction. 58 | pub last_txn_id: usize, 59 | /// Maximum number of reader slots in the environment. 60 | pub maximum_number_of_readers: u32, 61 | /// Number of reader slots used in the environment. 62 | pub number_of_readers: u32, 63 | } 64 | 65 | /// A structure that can be used to wait for the closing event. 66 | /// Multiple threads can wait on this event. 67 | #[derive(Clone)] 68 | pub struct EnvClosingEvent(Arc); 69 | 70 | impl EnvClosingEvent { 71 | /// Blocks this thread until the environment is effectively closed. 72 | /// 73 | /// # Safety 74 | /// 75 | /// Make sure that you don't have any copy of the environment in the thread 76 | /// that is waiting for a close event. If you do, you will have a deadlock. 77 | pub fn wait(&self) { 78 | self.0.wait() 79 | } 80 | 81 | /// Blocks this thread until either the environment has been closed 82 | /// or until the timeout elapses. Returns `true` if the environment 83 | /// has been effectively closed. 84 | pub fn wait_timeout(&self, timeout: Duration) -> bool { 85 | self.0.wait_timeout(timeout) 86 | } 87 | } 88 | 89 | impl fmt::Debug for EnvClosingEvent { 90 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 91 | f.debug_struct("EnvClosingEvent").finish() 92 | } 93 | } 94 | 95 | // Thanks to the mozilla/rkv project 96 | // Workaround the UNC path on Windows, see https://github.com/rust-lang/rust/issues/42869. 97 | // Otherwise, `Env::from_env()` will panic with error_no(123). 98 | #[cfg(not(windows))] 99 | fn canonicalize_path(path: &Path) -> io::Result { 100 | path.canonicalize() 101 | } 102 | 103 | #[cfg(windows)] 104 | fn canonicalize_path(path: &Path) -> io::Result { 105 | let canonical = path.canonicalize()?; 106 | let url = url::Url::from_file_path(&canonical) 107 | .map_err(|_e| io::Error::new(io::ErrorKind::Other, "URL passing error"))?; 108 | url.to_file_path() 109 | .map_err(|_e| io::Error::new(io::ErrorKind::Other, "path canonicalization error")) 110 | } 111 | 112 | #[cfg(windows)] 113 | /// Adding a 'missing' trait from windows OsStrExt 114 | trait OsStrExtLmdb { 115 | fn as_bytes(&self) -> &[u8]; 116 | } 117 | #[cfg(windows)] 118 | impl OsStrExtLmdb for OsStr { 119 | fn as_bytes(&self) -> &[u8] { 120 | &self.to_str().unwrap().as_bytes() 121 | } 122 | } 123 | 124 | #[cfg(unix)] 125 | fn get_file_fd(file: &File) -> RawFd { 126 | file.as_raw_fd() 127 | } 128 | 129 | #[cfg(windows)] 130 | fn get_file_fd(file: &File) -> RawHandle { 131 | file.as_raw_handle() 132 | } 133 | 134 | #[cfg(unix)] 135 | /// Get metadata from a file descriptor. 136 | unsafe fn metadata_from_fd(raw_fd: RawFd) -> io::Result { 137 | let fd = BorrowedFd::borrow_raw(raw_fd); 138 | let owned = fd.try_clone_to_owned()?; 139 | File::from(owned).metadata() 140 | } 141 | 142 | #[cfg(windows)] 143 | /// Get metadata from a file descriptor. 144 | unsafe fn metadata_from_fd(raw_fd: RawHandle) -> io::Result { 145 | let fd = BorrowedHandle::borrow_raw(raw_fd); 146 | let owned = fd.try_clone_to_owned()?; 147 | File::from(owned).metadata() 148 | } 149 | 150 | /// A helper function that transforms the LMDB types into Rust types (`MDB_val` into slices) 151 | /// and vice versa, the Rust types into C types (`Ordering` into an integer). 152 | /// 153 | /// # Safety 154 | /// 155 | /// `a` and `b` should both properly aligned, valid for reads and should point to a valid 156 | /// [`MDB_val`][ffi::MDB_val]. An [`MDB_val`][ffi::MDB_val] (consists of a pointer and size) is 157 | /// valid when its pointer (`mv_data`) is valid for reads of `mv_size` bytes and is not null. 158 | unsafe extern "C" fn custom_key_cmp_wrapper( 159 | a: *const ffi::MDB_val, 160 | b: *const ffi::MDB_val, 161 | ) -> i32 { 162 | let a = unsafe { ffi::from_val(*a) }; 163 | let b = unsafe { ffi::from_val(*b) }; 164 | match catch_unwind(|| C::compare(a, b)) { 165 | Ok(Ordering::Less) => -1, 166 | Ok(Ordering::Equal) => 0, 167 | Ok(Ordering::Greater) => 1, 168 | Err(_) => abort(), 169 | } 170 | } 171 | 172 | /// A representation of LMDB's default comparator behavior. 173 | /// 174 | /// This enum is used to indicate the absence of a custom comparator for an LMDB 175 | /// database instance. When a [`Database`] is created or opened with 176 | /// [`DefaultComparator`], it signifies that the comparator should not be explicitly 177 | /// set via [`ffi::mdb_set_compare`]. Consequently, the database 178 | /// instance utilizes LMDB's built-in default comparator, which inherently performs 179 | /// lexicographic comparison of keys. 180 | /// 181 | /// This comparator's lexicographic implementation is employed in scenarios involving 182 | /// prefix iterators. Specifically, methods other than [`Comparator::compare`] are utilized 183 | /// to determine the lexicographic successors and predecessors of byte sequences, which 184 | /// is essential for these iterators' operation. 185 | /// 186 | /// When a custom comparator is provided, the wrapper is responsible for setting 187 | /// it with the [`ffi::mdb_set_compare`] function, which overrides the default comparison 188 | /// behavior of LMDB with the user-defined logic. 189 | #[derive(Debug)] 190 | pub enum DefaultComparator {} 191 | 192 | impl LexicographicComparator for DefaultComparator { 193 | #[inline] 194 | fn compare_elem(a: u8, b: u8) -> Ordering { 195 | a.cmp(&b) 196 | } 197 | 198 | #[inline] 199 | fn successor(elem: u8) -> Option { 200 | match elem { 201 | u8::MAX => None, 202 | elem => Some(elem + 1), 203 | } 204 | } 205 | 206 | #[inline] 207 | fn predecessor(elem: u8) -> Option { 208 | match elem { 209 | u8::MIN => None, 210 | elem => Some(elem - 1), 211 | } 212 | } 213 | 214 | #[inline] 215 | fn max_elem() -> u8 { 216 | u8::MAX 217 | } 218 | 219 | #[inline] 220 | fn min_elem() -> u8 { 221 | u8::MIN 222 | } 223 | } 224 | 225 | /// A representation of LMDB's `MDB_INTEGERKEY` and `MDB_INTEGERDUP` comparator behavior. 226 | /// 227 | /// This enum is used to indicate a table should be sorted by the keys numeric 228 | /// value in native byte order. When a [`Database`] is created or opened with 229 | /// [`IntegerComparator`], it signifies that the comparator should not be explicitly 230 | /// set via [`ffi::mdb_set_compare`], instead the flag [`DatabaseFlags::INTEGER_KEY`] 231 | /// or [`DatabaseFlags::INTEGER_DUP`] is set on the table. 232 | /// 233 | /// This can only be used on certain types: either `u32` or `usize`. 234 | /// The keys must all be of the same size. 235 | #[derive(Debug)] 236 | pub enum IntegerComparator {} 237 | 238 | impl Comparator for IntegerComparator { 239 | fn compare(a: &[u8], b: &[u8]) -> Ordering { 240 | #[cfg(target_endian = "big")] 241 | return a.cmp(b); 242 | 243 | #[cfg(target_endian = "little")] 244 | { 245 | let len = a.len(); 246 | 247 | for i in (0..len).rev() { 248 | match a[i].cmp(&b[i]) { 249 | Ordering::Equal => continue, 250 | other => return other, 251 | } 252 | } 253 | 254 | Ordering::Equal 255 | } 256 | } 257 | } 258 | 259 | /// Whether to perform compaction while copying an environment. 260 | #[derive(Debug, Copy, Clone)] 261 | pub enum CompactionOption { 262 | /// Omit free pages and sequentially renumber all pages in output. 263 | /// 264 | /// This option consumes more CPU and runs more slowly than the default. 265 | /// Currently it fails if the environment has suffered a page leak. 266 | Enabled, 267 | 268 | /// Copy everything without taking any special action about free pages. 269 | Disabled, 270 | } 271 | 272 | /// Whether to enable or disable flags in [`Env::set_flags`]. 273 | #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] 274 | pub enum FlagSetMode { 275 | /// Enable the flags. 276 | Enable, 277 | /// Disable the flags. 278 | Disable, 279 | } 280 | 281 | impl FlagSetMode { 282 | /// Convert the enum into the `i32` required by LMDB. 283 | /// "A non-zero value sets the flags, zero clears them." 284 | /// 285 | fn as_mdb_env_set_flags_input(self) -> i32 { 286 | match self { 287 | Self::Enable => 1, 288 | Self::Disable => 0, 289 | } 290 | } 291 | } 292 | -------------------------------------------------------------------------------- /heed/src/iteration_method.rs: -------------------------------------------------------------------------------- 1 | //! The set of possible iteration methods for the different iterators. 2 | 3 | use crate::cursor::MoveOperation; 4 | 5 | /// The trait used to define the way iterators behave. 6 | pub trait IterationMethod { 7 | /// The internal operation to move the cursor through entries. 8 | const MOVE_OPERATION: MoveOperation; 9 | } 10 | 11 | /// Moves to the next or previous key if there 12 | /// are no more values associated with the current key. 13 | #[derive(Debug, Clone, Copy)] 14 | pub enum MoveThroughDuplicateValues {} 15 | 16 | impl IterationMethod for MoveThroughDuplicateValues { 17 | const MOVE_OPERATION: MoveOperation = MoveOperation::Any; 18 | } 19 | 20 | /// Moves between keys and ignores the duplicate values of keys. 21 | #[derive(Debug, Clone, Copy)] 22 | pub enum MoveBetweenKeys {} 23 | 24 | impl IterationMethod for MoveBetweenKeys { 25 | const MOVE_OPERATION: MoveOperation = MoveOperation::NoDup; 26 | } 27 | 28 | /// Moves only on the duplicate values of a given key and ignores other keys. 29 | #[derive(Debug, Clone, Copy)] 30 | pub enum MoveOnCurrentKeyDuplicates {} 31 | 32 | impl IterationMethod for MoveOnCurrentKeyDuplicates { 33 | const MOVE_OPERATION: MoveOperation = MoveOperation::Dup; 34 | } 35 | -------------------------------------------------------------------------------- /heed/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc( 2 | html_favicon_url = "https://raw.githubusercontent.com/meilisearch/heed/main/assets/heed-pigeon.ico?raw=true" 3 | )] 4 | #![doc( 5 | html_logo_url = "https://raw.githubusercontent.com/meilisearch/heed/main/assets/heed-pigeon-logo.png?raw=true" 6 | )] 7 | 8 | //! `heed` and `heed3` are high-level wrappers of [LMDB]. 9 | //! 10 | //! - `heed` is a wrapper around LMDB on the `mdb.master` branch, 11 | //! - `heed3` derives from the `heed` wrapper but on the `mdb.master3` branch. 12 | //! 13 | //! The `heed3` crate will be stable once the LMDB version on the `mdb.master3` branch 14 | //! will be officially released. It features encryption-at-rest and checksumming features 15 | //! that the `heed` crate doesn't. 16 | //! 17 | //! The [cookbook] will give you a variety of complete Rust programs to use with `heed`. 18 | //! 19 | //! ---- 20 | //! 21 | //! This crate simply facilitates the use of LMDB by providing a mechanism to store and 22 | //! retrieve Rust types. It abstracts away some of the complexities of the raw LMDB usage 23 | //! while retaining its performance characteristics. The functionality is achieved with the help 24 | //! of the serde library for data serialization concerns. 25 | //! 26 | //! LMDB stands for Lightning Memory-Mapped Database, which utilizes memory-mapped files 27 | //! for efficient data storage and retrieval by mapping file content directly into the virtual 28 | //! address space. `heed` derives its efficiency from the underlying LMDB without imposing 29 | //! additional runtime costs. 30 | //! 31 | //! [LMDB]: https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database 32 | //! 33 | //! # Examples 34 | //! 35 | //! Open a database that will support some typed key/data and ensure, at compile time, 36 | //! that you'll write those types and not others. 37 | //! 38 | //! ``` 39 | //! use std::fs; 40 | //! use std::path::Path; 41 | //! use heed::{EnvOpenOptions, Database}; 42 | //! use heed::types::*; 43 | //! 44 | //! # fn main() -> Result<(), Box> { 45 | //! let dir = tempfile::tempdir()?; 46 | //! let env = unsafe { EnvOpenOptions::new().open(dir.path())? }; 47 | //! 48 | //! // we will open the default unnamed database 49 | //! let mut wtxn = env.write_txn()?; 50 | //! let db: Database> = env.create_database(&mut wtxn, None)?; 51 | //! 52 | //! // opening a write transaction 53 | //! db.put(&mut wtxn, "seven", &7)?; 54 | //! db.put(&mut wtxn, "zero", &0)?; 55 | //! db.put(&mut wtxn, "five", &5)?; 56 | //! db.put(&mut wtxn, "three", &3)?; 57 | //! wtxn.commit()?; 58 | //! 59 | //! // opening a read transaction 60 | //! // to check if those values are now available 61 | //! let mut rtxn = env.read_txn()?; 62 | //! 63 | //! let ret = db.get(&rtxn, "zero")?; 64 | //! assert_eq!(ret, Some(0)); 65 | //! 66 | //! let ret = db.get(&rtxn, "five")?; 67 | //! assert_eq!(ret, Some(5)); 68 | //! # Ok(()) } 69 | //! ``` 70 | #![warn(missing_docs)] 71 | 72 | pub mod cookbook; 73 | mod cursor; 74 | mod databases; 75 | mod envs; 76 | pub mod iteration_method; 77 | mod iterator; 78 | mod mdb; 79 | mod reserved_space; 80 | mod txn; 81 | 82 | use std::ffi::CStr; 83 | use std::{error, fmt, io, mem, result}; 84 | 85 | use heed_traits as traits; 86 | pub use {byteorder, heed_types as types}; 87 | 88 | use self::cursor::{RoCursor, RwCursor}; 89 | pub use self::databases::{Database, DatabaseOpenOptions, DatabaseStat}; 90 | #[cfg(master3)] 91 | pub use self::databases::{EncryptedDatabase, EncryptedDatabaseOpenOptions}; 92 | #[cfg(master3)] 93 | pub use self::envs::EncryptedEnv; 94 | pub use self::envs::{ 95 | env_closing_event, CompactionOption, DefaultComparator, Env, EnvClosingEvent, EnvInfo, 96 | EnvOpenOptions, FlagSetMode, IntegerComparator, 97 | }; 98 | pub use self::iterator::{ 99 | RoIter, RoPrefix, RoRange, RoRevIter, RoRevPrefix, RoRevRange, RwIter, RwPrefix, RwRange, 100 | RwRevIter, RwRevPrefix, RwRevRange, 101 | }; 102 | pub use self::mdb::error::Error as MdbError; 103 | use self::mdb::ffi::{from_val, into_val}; 104 | pub use self::mdb::flags::{DatabaseFlags, EnvFlags, PutFlags}; 105 | pub use self::reserved_space::ReservedSpace; 106 | pub use self::traits::{BoxedError, BytesDecode, BytesEncode, Comparator, LexicographicComparator}; 107 | pub use self::txn::{AnyTls, RoTxn, RwTxn, TlsUsage, WithTls, WithoutTls}; 108 | 109 | /// The underlying LMDB library version information. 110 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 111 | pub struct LmdbVersion { 112 | /// The library version as a string. 113 | pub string: &'static str, 114 | /// The library major version number. 115 | pub major: i32, 116 | /// The library minor version number. 117 | pub minor: i32, 118 | /// The library patch version number. 119 | pub patch: i32, 120 | } 121 | 122 | /// Return the LMDB library version information. 123 | /// 124 | /// ``` 125 | /// use heed::{lmdb_version, LmdbVersion}; 126 | /// 127 | /// let expected_master = LmdbVersion { 128 | /// string: "LMDB 0.9.70: (December 19, 2015)", 129 | /// major: 0, 130 | /// minor: 9, 131 | /// patch: 70, 132 | /// }; 133 | /// 134 | /// let expected_master3 = LmdbVersion { 135 | /// string: "LMDB 0.9.90: (May 1, 2017)", 136 | /// major: 0, 137 | /// minor: 9, 138 | /// patch: 90, 139 | /// }; 140 | /// 141 | /// let actual = lmdb_version(); 142 | /// assert!(actual == expected_master || actual == expected_master3); 143 | /// ``` 144 | pub fn lmdb_version() -> LmdbVersion { 145 | let mut major = mem::MaybeUninit::uninit(); 146 | let mut minor = mem::MaybeUninit::uninit(); 147 | let mut patch = mem::MaybeUninit::uninit(); 148 | 149 | unsafe { 150 | let string_ptr = 151 | mdb::ffi::mdb_version(major.as_mut_ptr(), minor.as_mut_ptr(), patch.as_mut_ptr()); 152 | LmdbVersion { 153 | string: CStr::from_ptr(string_ptr).to_str().unwrap(), 154 | major: major.assume_init(), 155 | minor: minor.assume_init(), 156 | patch: patch.assume_init(), 157 | } 158 | } 159 | } 160 | 161 | /// An error that encapsulates all possible errors in this crate. 162 | #[derive(Debug)] 163 | pub enum Error { 164 | /// I/O error: can come from the standard library or be a rewrapped [`MdbError`]. 165 | Io(io::Error), 166 | /// LMDB error. 167 | Mdb(MdbError), 168 | /// Encoding error. 169 | Encoding(BoxedError), 170 | /// Decoding error. 171 | Decoding(BoxedError), 172 | /// The environment is already open in this program; 173 | /// close it to be able to open it again with different options. 174 | EnvAlreadyOpened, 175 | } 176 | 177 | impl fmt::Display for Error { 178 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 179 | match self { 180 | Error::Io(error) => write!(f, "{error}"), 181 | Error::Mdb(error) => write!(f, "{error}"), 182 | Error::Encoding(error) => write!(f, "error while encoding: {error}"), 183 | Error::Decoding(error) => write!(f, "error while decoding: {error}"), 184 | Error::EnvAlreadyOpened => f.write_str( 185 | "environment already open in this program; \ 186 | close it to be able to open it again with different options", 187 | ), 188 | } 189 | } 190 | } 191 | 192 | impl error::Error for Error {} 193 | 194 | impl From for Error { 195 | fn from(error: MdbError) -> Error { 196 | match error { 197 | MdbError::Other(e) => Error::Io(io::Error::from_raw_os_error(e)), 198 | _ => Error::Mdb(error), 199 | } 200 | } 201 | } 202 | 203 | impl From for Error { 204 | fn from(error: io::Error) -> Error { 205 | Error::Io(error) 206 | } 207 | } 208 | 209 | /// Either a success or an [`Error`]. 210 | pub type Result = result::Result; 211 | 212 | /// An unspecified type. 213 | /// 214 | /// It is used as placeholders when creating a database. 215 | /// It does not implement the [`BytesEncode`] and [`BytesDecode`] traits 216 | /// and therefore can't be used as codecs. You must use the [`Database::remap_types`] 217 | /// to properly define them. 218 | pub enum Unspecified {} 219 | 220 | macro_rules! assert_eq_env_db_txn { 221 | ($database:ident, $txn:ident) => { 222 | assert!( 223 | $database.env_ident == unsafe { $txn.env_mut_ptr().as_mut() as *mut _ as usize }, 224 | "The database environment doesn't match the transaction's environment" 225 | ); 226 | }; 227 | } 228 | 229 | macro_rules! assert_eq_env_txn { 230 | ($env:expr, $txn:ident) => { 231 | assert!( 232 | $env.env_mut_ptr() == $txn.env_mut_ptr(), 233 | "The environment doesn't match the transaction's environment" 234 | ); 235 | }; 236 | } 237 | 238 | pub(crate) use {assert_eq_env_db_txn, assert_eq_env_txn}; 239 | 240 | #[cfg(test)] 241 | mod tests { 242 | use super::*; 243 | 244 | #[test] 245 | fn error_is_send_sync() { 246 | fn give_me_send_sync(_: T) {} 247 | 248 | let error = Error::Encoding(Box::from("There is an issue, you know?")); 249 | give_me_send_sync(error); 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /heed/src/mdb/lmdb_error.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error as StdError; 2 | use std::ffi::CStr; 3 | use std::os::raw::c_char; 4 | use std::{fmt, str}; 5 | 6 | use libc::c_int; 7 | #[cfg(master3)] 8 | use lmdb_master3_sys as ffi; 9 | #[cfg(not(master3))] 10 | use lmdb_master_sys as ffi; 11 | 12 | /// An LMDB error kind. 13 | #[derive(Debug, Copy, Clone, Eq, PartialEq)] 14 | pub enum Error { 15 | /// A key/data pair already exists. 16 | /// 17 | /// May be also returned by append functions. Data passed 18 | /// doesn't respect the database ordering. 19 | KeyExist, 20 | /// A key/data pair was not found (EOF). 21 | NotFound, 22 | /// Requested page not found - this usually indicates corruption. 23 | PageNotFound, 24 | /// Located page was wrong type. 25 | Corrupted, 26 | /// Update of meta page failed or environment had fatal error. 27 | Panic, 28 | /// Environment version mismatch. 29 | VersionMismatch, 30 | /// File is not a valid LMDB file. 31 | Invalid, 32 | /// Environment mapsize reached. 33 | MapFull, 34 | /// Environment maxdbs reached. 35 | DbsFull, 36 | /// Environment maxreaders reached. 37 | ReadersFull, 38 | /// Too many TLS keys in use - Windows only. 39 | TlsFull, 40 | /// Txn has too many dirty pages. 41 | TxnFull, 42 | /// Cursor stack too deep - internal error. 43 | CursorFull, 44 | /// Page has not enough space - internal error. 45 | PageFull, 46 | /// Database contents grew beyond environment mapsize. 47 | MapResized, 48 | /// Operation and DB incompatible, or DB type changed. This can mean: 49 | /// - The operation expects an MDB_DUPSORT / MDB_DUPFIXED database. 50 | /// - Opening a named DB when the unnamed DB has MDB_DUPSORT / MDB_INTEGERKEY. 51 | /// - Accessing a data record as a database, or vice versa. 52 | /// - The database was dropped and recreated with different flags. 53 | Incompatible, 54 | /// Invalid reuse of reader locktable slot. 55 | BadRslot, 56 | /// Transaction cannot recover - it must be aborted. 57 | BadTxn, 58 | /// Unsupported size of key/DB name/data, or wrong DUP_FIXED size. 59 | /// 60 | /// Common causes of this error: 61 | /// - You tried to store a zero-length key 62 | /// - You tried to store a key longer than the max allowed key (511 bytes by default) 63 | /// - You are using [DUP_SORT](crate::DatabaseFlags::DUP_SORT) and trying to store a 64 | /// value longer than the max allowed key size (511 bytes by default) 65 | /// 66 | /// In the last two cases you can enable the `longer-keys` feature to increase the max allowed key size. 67 | BadValSize, 68 | /// The specified DBI was changed unexpectedly. 69 | BadDbi, 70 | /// Unexpected problem - transaction should abort. 71 | Problem, 72 | /// Page checksum incorrect. 73 | #[cfg(master3)] 74 | BadChecksum, 75 | /// Encryption/decryption failed. 76 | #[cfg(master3)] 77 | CryptoFail, 78 | /// Environment encryption mismatch. 79 | #[cfg(master3)] 80 | EnvEncryption, 81 | /// Other error. 82 | Other(c_int), 83 | } 84 | 85 | impl Error { 86 | /// Returns `true` if the given error is [`Error::NotFound`]. 87 | pub fn not_found(&self) -> bool { 88 | *self == Error::NotFound 89 | } 90 | 91 | /// Converts a raw error code to an `Error`. 92 | pub fn from_err_code(err_code: c_int) -> Error { 93 | match err_code { 94 | ffi::MDB_KEYEXIST => Error::KeyExist, 95 | ffi::MDB_NOTFOUND => Error::NotFound, 96 | ffi::MDB_PAGE_NOTFOUND => Error::PageNotFound, 97 | ffi::MDB_CORRUPTED => Error::Corrupted, 98 | ffi::MDB_PANIC => Error::Panic, 99 | ffi::MDB_VERSION_MISMATCH => Error::VersionMismatch, 100 | ffi::MDB_INVALID => Error::Invalid, 101 | ffi::MDB_MAP_FULL => Error::MapFull, 102 | ffi::MDB_DBS_FULL => Error::DbsFull, 103 | ffi::MDB_READERS_FULL => Error::ReadersFull, 104 | ffi::MDB_TLS_FULL => Error::TlsFull, 105 | ffi::MDB_TXN_FULL => Error::TxnFull, 106 | ffi::MDB_CURSOR_FULL => Error::CursorFull, 107 | ffi::MDB_PAGE_FULL => Error::PageFull, 108 | ffi::MDB_MAP_RESIZED => Error::MapResized, 109 | ffi::MDB_INCOMPATIBLE => Error::Incompatible, 110 | ffi::MDB_BAD_RSLOT => Error::BadRslot, 111 | ffi::MDB_BAD_TXN => Error::BadTxn, 112 | ffi::MDB_BAD_VALSIZE => Error::BadValSize, 113 | ffi::MDB_BAD_DBI => Error::BadDbi, 114 | ffi::MDB_PROBLEM => Error::Problem, 115 | #[cfg(master3)] 116 | ffi::MDB_BAD_CHECKSUM => Error::BadChecksum, 117 | #[cfg(master3)] 118 | ffi::MDB_CRYPTO_FAIL => Error::CryptoFail, 119 | #[cfg(master3)] 120 | ffi::MDB_ENV_ENCRYPTION => Error::EnvEncryption, 121 | other => Error::Other(other), 122 | } 123 | } 124 | 125 | /// Converts an `Error` to the raw error code. 126 | #[allow(clippy::trivially_copy_pass_by_ref)] 127 | pub fn to_err_code(&self) -> c_int { 128 | match *self { 129 | Error::KeyExist => ffi::MDB_KEYEXIST, 130 | Error::NotFound => ffi::MDB_NOTFOUND, 131 | Error::PageNotFound => ffi::MDB_PAGE_NOTFOUND, 132 | Error::Corrupted => ffi::MDB_CORRUPTED, 133 | Error::Panic => ffi::MDB_PANIC, 134 | Error::VersionMismatch => ffi::MDB_VERSION_MISMATCH, 135 | Error::Invalid => ffi::MDB_INVALID, 136 | Error::MapFull => ffi::MDB_MAP_FULL, 137 | Error::DbsFull => ffi::MDB_DBS_FULL, 138 | Error::ReadersFull => ffi::MDB_READERS_FULL, 139 | Error::TlsFull => ffi::MDB_TLS_FULL, 140 | Error::TxnFull => ffi::MDB_TXN_FULL, 141 | Error::CursorFull => ffi::MDB_CURSOR_FULL, 142 | Error::PageFull => ffi::MDB_PAGE_FULL, 143 | Error::MapResized => ffi::MDB_MAP_RESIZED, 144 | Error::Incompatible => ffi::MDB_INCOMPATIBLE, 145 | Error::BadRslot => ffi::MDB_BAD_RSLOT, 146 | Error::BadTxn => ffi::MDB_BAD_TXN, 147 | Error::BadValSize => ffi::MDB_BAD_VALSIZE, 148 | Error::BadDbi => ffi::MDB_BAD_DBI, 149 | Error::Problem => ffi::MDB_PROBLEM, 150 | #[cfg(master3)] 151 | Error::BadChecksum => ffi::MDB_BAD_CHECKSUM, 152 | #[cfg(master3)] 153 | Error::CryptoFail => ffi::MDB_CRYPTO_FAIL, 154 | #[cfg(master3)] 155 | Error::EnvEncryption => ffi::MDB_ENV_ENCRYPTION, 156 | Error::Other(err_code) => err_code, 157 | } 158 | } 159 | } 160 | 161 | impl fmt::Display for Error { 162 | fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { 163 | let description = unsafe { 164 | // This is safe since the error messages returned from mdb_strerror are static. 165 | let err: *const c_char = ffi::mdb_strerror(self.to_err_code()) as *const c_char; 166 | str::from_utf8_unchecked(CStr::from_ptr(err).to_bytes()) 167 | }; 168 | 169 | fmt.write_str(description) 170 | } 171 | } 172 | 173 | impl StdError for Error {} 174 | 175 | pub fn mdb_result(err_code: c_int) -> Result<(), Error> { 176 | if err_code == ffi::MDB_SUCCESS { 177 | Ok(()) 178 | } else { 179 | Err(Error::from_err_code(err_code)) 180 | } 181 | } 182 | 183 | #[cfg(test)] 184 | mod test { 185 | use super::*; 186 | 187 | #[test] 188 | fn test_description() { 189 | assert_eq!("Permission denied", Error::from_err_code(13).to_string()); 190 | assert_eq!("MDB_NOTFOUND: No matching key/data pair found", Error::NotFound.to_string()); 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /heed/src/mdb/lmdb_ffi.rs: -------------------------------------------------------------------------------- 1 | use std::ptr; 2 | 3 | pub use ffi::{ 4 | mdb_cursor_close, mdb_cursor_del, mdb_cursor_get, mdb_cursor_open, mdb_cursor_put, 5 | mdb_dbi_open, mdb_del, mdb_drop, mdb_env_close, mdb_env_copyfd2, mdb_env_create, 6 | mdb_env_get_fd, mdb_env_get_flags, mdb_env_get_maxkeysize, mdb_env_get_maxreaders, 7 | mdb_env_info, mdb_env_open, mdb_env_set_flags, mdb_env_set_mapsize, mdb_env_set_maxdbs, 8 | mdb_env_set_maxreaders, mdb_env_stat, mdb_env_sync, mdb_filehandle_t, mdb_get, mdb_put, 9 | mdb_reader_check, mdb_set_compare, mdb_set_dupsort, mdb_stat, mdb_txn_abort, mdb_txn_begin, 10 | mdb_txn_commit, mdb_txn_id, mdb_version, MDB_cursor, MDB_dbi, MDB_env, MDB_stat, MDB_txn, 11 | MDB_val, MDB_CP_COMPACT, MDB_CURRENT, MDB_RDONLY, MDB_RESERVE, 12 | }; 13 | #[cfg(master3)] 14 | pub use ffi::{mdb_env_set_encrypt, MDB_enc_func}; 15 | #[cfg(master3)] 16 | use lmdb_master3_sys as ffi; 17 | #[cfg(not(master3))] 18 | use lmdb_master_sys as ffi; 19 | 20 | pub mod cursor_op { 21 | use super::ffi::{self, MDB_cursor_op}; 22 | 23 | pub const MDB_FIRST: MDB_cursor_op = ffi::MDB_FIRST; 24 | pub const MDB_FIRST_DUP: MDB_cursor_op = ffi::MDB_FIRST_DUP; 25 | pub const MDB_LAST: MDB_cursor_op = ffi::MDB_LAST; 26 | pub const MDB_LAST_DUP: MDB_cursor_op = ffi::MDB_LAST_DUP; 27 | pub const MDB_SET_RANGE: MDB_cursor_op = ffi::MDB_SET_RANGE; 28 | pub const MDB_SET: MDB_cursor_op = ffi::MDB_SET; 29 | pub const MDB_PREV: MDB_cursor_op = ffi::MDB_PREV; 30 | pub const MDB_PREV_NODUP: MDB_cursor_op = ffi::MDB_PREV_NODUP; 31 | pub const MDB_PREV_DUP: MDB_cursor_op = ffi::MDB_PREV_DUP; 32 | pub const MDB_NEXT: MDB_cursor_op = ffi::MDB_NEXT; 33 | pub const MDB_NEXT_NODUP: MDB_cursor_op = ffi::MDB_NEXT_NODUP; 34 | pub const MDB_NEXT_DUP: MDB_cursor_op = ffi::MDB_NEXT_DUP; 35 | pub const MDB_GET_CURRENT: MDB_cursor_op = ffi::MDB_GET_CURRENT; 36 | } 37 | 38 | pub fn reserve_size_val(size: usize) -> ffi::MDB_val { 39 | ffi::MDB_val { mv_size: size, mv_data: ptr::null_mut() } 40 | } 41 | 42 | pub unsafe fn into_val(value: &[u8]) -> ffi::MDB_val { 43 | ffi::MDB_val { mv_data: value.as_ptr() as *mut libc::c_void, mv_size: value.len() } 44 | } 45 | 46 | pub unsafe fn from_val<'a>(value: ffi::MDB_val) -> &'a [u8] { 47 | std::slice::from_raw_parts(value.mv_data as *const u8, value.mv_size) 48 | } 49 | -------------------------------------------------------------------------------- /heed/src/mdb/lmdb_flags.rs: -------------------------------------------------------------------------------- 1 | use bitflags::bitflags; 2 | #[cfg(master3)] 3 | use lmdb_master3_sys as ffi; 4 | #[cfg(not(master3))] 5 | use lmdb_master_sys as ffi; 6 | 7 | #[allow(unused)] // for cargo auto doc links 8 | use crate::{Database, IntegerComparator}; 9 | 10 | bitflags! { 11 | /// LMDB environment flags (see for more details). 12 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 13 | #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] 14 | #[repr(transparent)] 15 | pub struct EnvFlags: u32 { 16 | /// mmap at a fixed address (experimental). 17 | const FIXED_MAP = ffi::MDB_FIXEDMAP; 18 | /// No environment directory. 19 | const NO_SUB_DIR = ffi::MDB_NOSUBDIR; 20 | /// Don't fsync after commit. 21 | const NO_SYNC = ffi::MDB_NOSYNC; 22 | /// Open the previous transaction. 23 | const PREV_SNAPSHOT = ffi::MDB_PREVSNAPSHOT; 24 | /// Read only. 25 | const READ_ONLY = ffi::MDB_RDONLY; 26 | /// Don't fsync metapage after commit. 27 | const NO_META_SYNC = ffi::MDB_NOMETASYNC; 28 | /// Use writable mmap. 29 | const WRITE_MAP = ffi::MDB_WRITEMAP; 30 | /// Use asynchronous msync when MDB_WRITEMAP is used. 31 | const MAP_ASYNC = ffi::MDB_MAPASYNC; 32 | /// Tie reader locktable slots to MDB_txn objects instead of to threads. 33 | // Note to self: When removing this flag from here, we must introduce an 34 | // internal-only AllEnvFlags akin to the AllDatabaseFlags bitflags. 35 | #[deprecated(since="0.22.0", note="please use `EnvOpenOptions::read_txn_with_tls` or `EnvOpenOptions::read_txn_without_tls` instead")] 36 | const NO_TLS = ffi::MDB_NOTLS; 37 | /// Don't do any locking, caller must manage their own locks. 38 | const NO_LOCK = ffi::MDB_NOLOCK; 39 | /// Don't do readahead (no effect on Windows). 40 | const NO_READ_AHEAD = ffi::MDB_NORDAHEAD; 41 | /// Don't initialize malloc'd memory before writing to datafile. 42 | const NO_MEM_INIT = ffi::MDB_NOMEMINIT; 43 | } 44 | } 45 | 46 | bitflags! { 47 | /// LMDB database flags (see for more details). 48 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 49 | #[repr(transparent)] 50 | pub struct AllDatabaseFlags: u32 { 51 | /// Use reverse string keys. 52 | const REVERSE_KEY = ffi::MDB_REVERSEKEY; 53 | /// Use sorted duplicates. 54 | const DUP_SORT = ffi::MDB_DUPSORT; 55 | /// Numeric keys in native byte order: either `u32` or `usize`. 56 | /// The keys must all be of the same size. 57 | /// 58 | /// It is recommended to set the comparator to [`IntegerComparator`](crate::IntegerComparator), 59 | /// rather than setting this flag manually. 60 | const INTEGER_KEY = ffi::MDB_INTEGERKEY; 61 | /// With [`DatabaseFlags::DUP_SORT`], sorted dup items have fixed size. 62 | const DUP_FIXED = ffi::MDB_DUPFIXED; 63 | /// With [`DatabaseFlags::DUP_SORT`], dups are [`DatabaseFlags::INTEGER_KEY`]-style integers. 64 | const INTEGER_DUP = ffi::MDB_INTEGERDUP; 65 | /// With [`DatabaseFlags::DUP_SORT`], use reverse string dups. 66 | const REVERSE_DUP = ffi::MDB_REVERSEDUP; 67 | /// Create DB if not already existing. 68 | const CREATE = ffi::MDB_CREATE; 69 | } 70 | } 71 | 72 | bitflags! { 73 | /// LMDB database flags (see for more details). 74 | // It is a subset of the whole list of possible flags LMDB exposes but 75 | // we only want users to be able to specify these with the DUP flags. 76 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 77 | #[repr(transparent)] 78 | pub struct DatabaseFlags: u32 { 79 | /// Use reverse string keys. 80 | /// 81 | /// ``` 82 | /// # use std::fs; 83 | /// # use std::path::Path; 84 | /// # use heed::{DatabaseFlags, EnvOpenOptions}; 85 | /// use heed::types::*; 86 | /// use heed::byteorder::BigEndian; 87 | /// 88 | /// # fn main() -> Result<(), Box> { 89 | /// # let dir = tempfile::tempdir()?; 90 | /// # let env = unsafe { EnvOpenOptions::new() 91 | /// # .map_size(10 * 1024 * 1024) // 10MB 92 | /// # .max_dbs(3000) 93 | /// # .open(dir.path())? 94 | /// # }; 95 | /// 96 | /// let mut wtxn = env.write_txn()?; 97 | /// let db = env.database_options() 98 | /// .types::() 99 | /// .flags(DatabaseFlags::REVERSE_KEY) 100 | /// .name("reverse-key") 101 | /// .create(&mut wtxn)?; 102 | /// 103 | /// # db.clear(&mut wtxn)?; 104 | /// db.put(&mut wtxn, &"bonjour", &())?; 105 | /// db.put(&mut wtxn, &"hello", &())?; 106 | /// db.put(&mut wtxn, &"hola", &())?; 107 | /// 108 | /// let mut iter = db.iter(&wtxn)?; 109 | /// assert_eq!(iter.next().transpose()?, Some(("hola", ()))); 110 | /// assert_eq!(iter.next().transpose()?, Some(("hello", ()))); 111 | /// assert_eq!(iter.next().transpose()?, Some(("bonjour", ()))); 112 | /// assert_eq!(iter.next().transpose()?, None); 113 | /// drop(iter); 114 | /// 115 | /// let mut iter = db.rev_iter(&wtxn)?; 116 | /// assert_eq!(iter.next().transpose()?, Some(("bonjour", ()))); 117 | /// assert_eq!(iter.next().transpose()?, Some(("hello", ()))); 118 | /// assert_eq!(iter.next().transpose()?, Some(("hola", ()))); 119 | /// assert_eq!(iter.next().transpose()?, None); 120 | /// drop(iter); 121 | /// 122 | /// wtxn.commit()?; 123 | /// # Ok(()) } 124 | /// ``` 125 | const REVERSE_KEY = ffi::MDB_REVERSEKEY; 126 | /// Use sorted duplicates. 127 | /// 128 | /// ``` 129 | /// # use std::fs; 130 | /// # use std::path::Path; 131 | /// # use heed::{DatabaseFlags, EnvOpenOptions}; 132 | /// use heed::types::*; 133 | /// use heed::byteorder::BigEndian; 134 | /// 135 | /// # fn main() -> Result<(), Box> { 136 | /// # let dir = tempfile::tempdir()?; 137 | /// # let env = unsafe { EnvOpenOptions::new() 138 | /// # .map_size(10 * 1024 * 1024) // 10MB 139 | /// # .max_dbs(3000) 140 | /// # .open(dir.path())? 141 | /// # }; 142 | /// type BEI64 = I64; 143 | /// 144 | /// let mut wtxn = env.write_txn()?; 145 | /// let db = env.database_options() 146 | /// .types::() 147 | /// .flags(DatabaseFlags::DUP_SORT) 148 | /// .name("dup-sort") 149 | /// .create(&mut wtxn)?; 150 | /// 151 | /// # db.clear(&mut wtxn)?; 152 | /// db.put(&mut wtxn, &68, &120)?; 153 | /// db.put(&mut wtxn, &68, &121)?; 154 | /// db.put(&mut wtxn, &68, &122)?; 155 | /// db.put(&mut wtxn, &68, &123)?; 156 | /// db.put(&mut wtxn, &92, &32)?; 157 | /// db.put(&mut wtxn, &35, &120)?; 158 | /// db.put(&mut wtxn, &0, &120)?; 159 | /// db.put(&mut wtxn, &42, &120)?; 160 | /// 161 | /// let mut iter = db.get_duplicates(&wtxn, &68)?.expect("the key exists"); 162 | /// assert_eq!(iter.next().transpose()?, Some((68, 120))); 163 | /// assert_eq!(iter.next().transpose()?, Some((68, 121))); 164 | /// assert_eq!(iter.next().transpose()?, Some((68, 122))); 165 | /// assert_eq!(iter.next().transpose()?, Some((68, 123))); 166 | /// assert_eq!(iter.next().transpose()?, None); 167 | /// drop(iter); 168 | /// 169 | /// let mut iter = db.get_duplicates(&wtxn, &68)?.expect("the key exists"); 170 | /// assert_eq!(iter.last().transpose()?, Some((68, 123))); 171 | /// 172 | /// assert!(db.delete_one_duplicate(&mut wtxn, &68, &121)?, "The entry must exist"); 173 | /// 174 | /// let mut iter = db.get_duplicates(&wtxn, &68)?.expect("the key exists"); 175 | /// assert_eq!(iter.next().transpose()?, Some((68, 120))); 176 | /// // No more (68, 121) returned here! 177 | /// assert_eq!(iter.next().transpose()?, Some((68, 122))); 178 | /// assert_eq!(iter.next().transpose()?, Some((68, 123))); 179 | /// assert_eq!(iter.next().transpose()?, None); 180 | /// drop(iter); 181 | /// 182 | /// wtxn.commit()?; 183 | /// # Ok(()) } 184 | /// ``` 185 | const DUP_SORT = ffi::MDB_DUPSORT; 186 | /// Numeric keys in native byte order: either `u32` or `usize`. 187 | /// The keys must all be of the same size. 188 | /// 189 | /// It is recommended to use the [`IntegerComparator`] when 190 | /// opening the [`Database`] instead. This comparator provides 191 | /// better support for ranges and does not allow for prefix 192 | /// iteration, as it is not applicable in this context. 193 | /// 194 | /// ``` 195 | /// # use std::fs; 196 | /// # use std::path::Path; 197 | /// # use heed::{DatabaseFlags, EnvOpenOptions}; 198 | /// use heed::types::*; 199 | /// use heed::byteorder::BigEndian; 200 | /// 201 | /// # fn main() -> Result<(), Box> { 202 | /// # let dir = tempfile::tempdir()?; 203 | /// # let env = unsafe { EnvOpenOptions::new() 204 | /// # .map_size(10 * 1024 * 1024) // 10MB 205 | /// # .max_dbs(3000) 206 | /// # .open(dir.path())? 207 | /// # }; 208 | /// type BEI32 = I32; 209 | /// 210 | /// let mut wtxn = env.write_txn()?; 211 | /// let db = env.database_options() 212 | /// .types::() 213 | /// .flags(DatabaseFlags::INTEGER_KEY) 214 | /// .name("integer-key") 215 | /// .create(&mut wtxn)?; 216 | /// 217 | /// # db.clear(&mut wtxn)?; 218 | /// db.put(&mut wtxn, &68, &120)?; 219 | /// db.put(&mut wtxn, &92, &32)?; 220 | /// db.put(&mut wtxn, &35, &120)?; 221 | /// db.put(&mut wtxn, &0, &120)?; 222 | /// db.put(&mut wtxn, &42, &120)?; 223 | /// 224 | /// let mut iter = db.iter(&wtxn)?; 225 | /// assert_eq!(iter.next().transpose()?, Some((0, 120))); 226 | /// assert_eq!(iter.next().transpose()?, Some((35, 120))); 227 | /// assert_eq!(iter.next().transpose()?, Some((42, 120))); 228 | /// assert_eq!(iter.next().transpose()?, Some((68, 120))); 229 | /// assert_eq!(iter.next().transpose()?, Some((92, 32))); 230 | /// assert_eq!(iter.next().transpose()?, None); 231 | /// drop(iter); 232 | /// 233 | /// wtxn.commit()?; 234 | /// # Ok(()) } 235 | /// ``` 236 | #[deprecated(since="0.21.0", note="prefer using `IntegerComparator` with the `DatabaseOpenOptions::key_comparator` method instead")] 237 | const INTEGER_KEY = ffi::MDB_INTEGERKEY; 238 | /// With [`DatabaseFlags::DUP_SORT`], sorted dup items have fixed size. 239 | /// 240 | /// ``` 241 | /// # use std::fs; 242 | /// # use std::path::Path; 243 | /// # use heed::{DatabaseFlags, EnvOpenOptions}; 244 | /// use heed::types::*; 245 | /// use heed::byteorder::BigEndian; 246 | /// 247 | /// # fn main() -> Result<(), Box> { 248 | /// # let dir = tempfile::tempdir()?; 249 | /// # let env = unsafe { EnvOpenOptions::new() 250 | /// # .map_size(10 * 1024 * 1024) // 10MB 251 | /// # .max_dbs(3000) 252 | /// # .open(dir.path())? 253 | /// # }; 254 | /// type BEI64 = I64; 255 | /// 256 | /// let mut wtxn = env.write_txn()?; 257 | /// let db = env.database_options() 258 | /// .types::() 259 | /// .flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED) 260 | /// .name("dup-sort-fixed") 261 | /// .create(&mut wtxn)?; 262 | /// 263 | /// # db.clear(&mut wtxn)?; 264 | /// db.put(&mut wtxn, &68, &120)?; 265 | /// db.put(&mut wtxn, &68, &121)?; 266 | /// db.put(&mut wtxn, &68, &122)?; 267 | /// db.put(&mut wtxn, &68, &123)?; 268 | /// db.put(&mut wtxn, &92, &32)?; 269 | /// db.put(&mut wtxn, &35, &120)?; 270 | /// db.put(&mut wtxn, &0, &120)?; 271 | /// db.put(&mut wtxn, &42, &120)?; 272 | /// 273 | /// let mut iter = db.get_duplicates(&wtxn, &68)?.expect("the key exists"); 274 | /// assert_eq!(iter.next().transpose()?, Some((68, 120))); 275 | /// assert_eq!(iter.next().transpose()?, Some((68, 121))); 276 | /// assert_eq!(iter.next().transpose()?, Some((68, 122))); 277 | /// assert_eq!(iter.next().transpose()?, Some((68, 123))); 278 | /// assert_eq!(iter.next().transpose()?, None); 279 | /// drop(iter); 280 | /// 281 | /// let mut iter = db.get_duplicates(&wtxn, &68)?.expect("the key exists"); 282 | /// assert_eq!(iter.last().transpose()?, Some((68, 123))); 283 | /// 284 | /// assert!(db.delete_one_duplicate(&mut wtxn, &68, &121)?, "The entry must exist"); 285 | /// 286 | /// let mut iter = db.get_duplicates(&wtxn, &68)?.expect("the key exists"); 287 | /// assert_eq!(iter.next().transpose()?, Some((68, 120))); 288 | /// // No more (68, 121) returned here! 289 | /// assert_eq!(iter.next().transpose()?, Some((68, 122))); 290 | /// assert_eq!(iter.next().transpose()?, Some((68, 123))); 291 | /// assert_eq!(iter.next().transpose()?, None); 292 | /// drop(iter); 293 | /// 294 | /// wtxn.commit()?; 295 | /// # Ok(()) } 296 | /// ``` 297 | const DUP_FIXED = ffi::MDB_DUPFIXED; 298 | /// With [`DatabaseFlags::DUP_SORT`], dups are [`DatabaseFlags::INTEGER_KEY`]-style integers. 299 | /// 300 | /// ``` 301 | /// # use std::fs; 302 | /// # use std::path::Path; 303 | /// # use heed::{DatabaseFlags, EnvOpenOptions}; 304 | /// use heed::types::*; 305 | /// use heed::byteorder::BigEndian; 306 | /// 307 | /// # fn main() -> Result<(), Box> { 308 | /// # let dir = tempfile::tempdir()?; 309 | /// # let env = unsafe { EnvOpenOptions::new() 310 | /// # .map_size(10 * 1024 * 1024) // 10MB 311 | /// # .max_dbs(3000) 312 | /// # .open(dir.path())? 313 | /// # }; 314 | /// type BEI32 = I32; 315 | /// 316 | /// let mut wtxn = env.write_txn()?; 317 | /// let db = env.database_options() 318 | /// .types::() 319 | /// .flags(DatabaseFlags::DUP_SORT | DatabaseFlags::INTEGER_DUP) 320 | /// .name("dup-sort-integer-dup") 321 | /// .create(&mut wtxn)?; 322 | /// 323 | /// # db.clear(&mut wtxn)?; 324 | /// db.put(&mut wtxn, &68, &120)?; 325 | /// db.put(&mut wtxn, &68, &121)?; 326 | /// db.put(&mut wtxn, &68, &122)?; 327 | /// db.put(&mut wtxn, &68, &123)?; 328 | /// db.put(&mut wtxn, &92, &32)?; 329 | /// db.put(&mut wtxn, &35, &120)?; 330 | /// db.put(&mut wtxn, &0, &120)?; 331 | /// db.put(&mut wtxn, &42, &120)?; 332 | /// 333 | /// let mut iter = db.get_duplicates(&wtxn, &68)?.expect("the key exists"); 334 | /// assert_eq!(iter.next().transpose()?, Some((68, 120))); 335 | /// assert_eq!(iter.next().transpose()?, Some((68, 121))); 336 | /// assert_eq!(iter.next().transpose()?, Some((68, 122))); 337 | /// assert_eq!(iter.next().transpose()?, Some((68, 123))); 338 | /// assert_eq!(iter.next().transpose()?, None); 339 | /// drop(iter); 340 | /// 341 | /// let mut iter = db.get_duplicates(&wtxn, &68)?.expect("the key exists"); 342 | /// assert_eq!(iter.last().transpose()?, Some((68, 123))); 343 | /// 344 | /// assert!(db.delete_one_duplicate(&mut wtxn, &68, &121)?, "The entry must exist"); 345 | /// 346 | /// let mut iter = db.get_duplicates(&wtxn, &68)?.expect("the key exists"); 347 | /// assert_eq!(iter.next().transpose()?, Some((68, 120))); 348 | /// // No more (68, 121) returned here! 349 | /// assert_eq!(iter.next().transpose()?, Some((68, 122))); 350 | /// assert_eq!(iter.next().transpose()?, Some((68, 123))); 351 | /// assert_eq!(iter.next().transpose()?, None); 352 | /// drop(iter); 353 | /// 354 | /// wtxn.commit()?; 355 | /// # Ok(()) } 356 | /// ``` 357 | #[deprecated(since="0.22.0", note="prefer using `IntegerComparator` with the `DatabaseOpenOptions::dup_sort_comparator` method instead")] 358 | const INTEGER_DUP = ffi::MDB_INTEGERDUP; 359 | /// With [`DatabaseFlags::DUP_SORT`], use reverse string dups. 360 | /// 361 | /// ``` 362 | /// # use std::fs; 363 | /// # use std::path::Path; 364 | /// # use heed::{DatabaseFlags, EnvOpenOptions}; 365 | /// use heed::types::*; 366 | /// use heed::byteorder::BigEndian; 367 | /// 368 | /// # fn main() -> Result<(), Box> { 369 | /// # let dir = tempfile::tempdir()?; 370 | /// # let env = unsafe { EnvOpenOptions::new() 371 | /// # .map_size(10 * 1024 * 1024) // 10MB 372 | /// # .max_dbs(3000) 373 | /// # .open(dir.path())? 374 | /// # }; 375 | /// type BEI64 = I64; 376 | /// 377 | /// let mut wtxn = env.write_txn()?; 378 | /// let db = env.database_options() 379 | /// .types::() 380 | /// .flags(DatabaseFlags::DUP_SORT | DatabaseFlags::REVERSE_DUP) 381 | /// .name("dup-sort") 382 | /// .create(&mut wtxn)?; 383 | /// 384 | /// # db.clear(&mut wtxn)?; 385 | /// db.put(&mut wtxn, &68, &"bonjour")?; 386 | /// db.put(&mut wtxn, &68, &"hola")?; 387 | /// db.put(&mut wtxn, &68, &"hello")?; 388 | /// db.put(&mut wtxn, &92, &"hallo")?; 389 | /// 390 | /// let mut iter = db.get_duplicates(&wtxn, &68)?.expect("the key exists"); 391 | /// assert_eq!(iter.next().transpose()?, Some((68, "hola"))); 392 | /// assert_eq!(iter.next().transpose()?, Some((68, "hello"))); 393 | /// assert_eq!(iter.next().transpose()?, Some((68, "bonjour"))); 394 | /// assert_eq!(iter.next().transpose()?, None); 395 | /// drop(iter); 396 | /// 397 | /// wtxn.commit()?; 398 | /// # Ok(()) } 399 | /// ``` 400 | const REVERSE_DUP = ffi::MDB_REVERSEDUP; 401 | } 402 | } 403 | 404 | bitflags! { 405 | /// LMDB put flags (see 406 | /// or for more details). 407 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 408 | #[repr(transparent)] 409 | pub struct PutFlags: u32 { 410 | /// Enter the new key/data pair only if it does not already appear in the database. 411 | /// 412 | /// This flag may only be specified if the database was opened with MDB_DUPSORT. 413 | /// The function will return MDB_KEYEXIST if the key/data pair already appears in the database. 414 | const NO_DUP_DATA = ffi::MDB_NODUPDATA; 415 | /// Enter the new key/data pair only if the key does not already appear in the database. 416 | /// 417 | /// The function will return MDB_KEYEXIST if the key already appears in the database, 418 | /// even if the database supports duplicates (MDB_DUPSORT). 419 | /// The data parameter will be set to point to the existing item. 420 | const NO_OVERWRITE = ffi::MDB_NOOVERWRITE; 421 | /// Append the given key/data pair to the end of the database. 422 | /// 423 | /// This option allows fast bulk loading when keys are already known to be in the correct order. 424 | /// Loading unsorted keys with this flag will cause a MDB_KEYEXIST error. 425 | const APPEND = ffi::MDB_APPEND; 426 | /// Append the given key/data pair to the end of the database but for sorted dup data. 427 | /// 428 | /// This option allows fast bulk loading when keys and dup data are already known to be in the correct order. 429 | /// Loading unsorted key/values with this flag will cause a MDB_KEYEXIST error. 430 | const APPEND_DUP = ffi::MDB_APPENDDUP; 431 | } 432 | } 433 | -------------------------------------------------------------------------------- /heed/src/mdb/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod lmdb_error; 2 | pub mod lmdb_ffi; 3 | pub mod lmdb_flags; 4 | 5 | pub use self::{lmdb_error as error, lmdb_ffi as ffi, lmdb_flags as flags}; 6 | -------------------------------------------------------------------------------- /heed/src/reserved_space.rs: -------------------------------------------------------------------------------- 1 | use std::mem::MaybeUninit; 2 | use std::{fmt, io}; 3 | 4 | use crate::mdb::ffi; 5 | 6 | /// A structure that is used to improve the write speed in LMDB. 7 | /// 8 | /// You must write the exact amount of bytes, no less, no more. 9 | pub struct ReservedSpace<'a> { 10 | bytes: &'a mut [MaybeUninit], 11 | /// Number of bytes which have been written: all bytes in `0..written`. 12 | written: usize, 13 | /// Index of the byte which should be written next. 14 | /// 15 | /// # Safety 16 | /// 17 | /// To ensure there are no unwritten gaps in the buffer this should be kept in the range 18 | /// `0..=written` at all times. 19 | write_head: usize, 20 | } 21 | 22 | impl ReservedSpace<'_> { 23 | pub(crate) unsafe fn from_val<'a>(val: ffi::MDB_val) -> ReservedSpace<'a> { 24 | let len = val.mv_size; 25 | let ptr = val.mv_data; 26 | 27 | ReservedSpace { 28 | bytes: std::slice::from_raw_parts_mut(ptr.cast(), len), 29 | written: 0, 30 | write_head: 0, 31 | } 32 | } 33 | 34 | /// The total number of bytes that this memory buffer has. 35 | #[inline] 36 | pub fn size(&self) -> usize { 37 | self.bytes.len() 38 | } 39 | 40 | /// The remaining number of bytes that this memory buffer has. 41 | #[inline] 42 | pub fn remaining(&self) -> usize { 43 | self.bytes.len() - self.write_head 44 | } 45 | 46 | /// Get a slice of all the bytes that have previously been written. 47 | /// 48 | /// This can be used to write information which cannot be known until the very end of 49 | /// serialization. For example, this method can be used to serialize a value, then compute a 50 | /// checksum over the bytes, and then write that checksum to a header at the start of the 51 | /// reserved space. 52 | #[inline] 53 | pub fn written_mut(&mut self) -> &mut [u8] { 54 | let ptr = self.bytes.as_mut_ptr(); 55 | let len = self.written; 56 | unsafe { std::slice::from_raw_parts_mut(ptr.cast(), len) } 57 | } 58 | 59 | /// Fills the remaining reserved space with zeroes. 60 | /// 61 | /// This can be used together with [`written_mut`](Self::written_mut) to get a mutable view of 62 | /// the entire reserved space. 63 | /// 64 | /// ### Note 65 | /// 66 | /// After calling this function, the entire space is considered to be filled and any 67 | /// further attempt to [`write`](std::io::Write::write) anything else will fail. 68 | #[inline] 69 | pub fn fill_zeroes(&mut self) { 70 | self.bytes[self.write_head..].fill(MaybeUninit::new(0)); 71 | self.written = self.bytes.len(); 72 | self.write_head = self.bytes.len(); 73 | } 74 | 75 | /// Get a slice of bytes corresponding to the *entire* reserved space. 76 | /// 77 | /// It is safe to write to any byte within the slice. However, for a write past the end of the 78 | /// prevously written bytes to take effect, [`assume_written`](Self::assume_written) has to be 79 | /// called to mark those bytes as initialized. 80 | /// 81 | /// # Safety 82 | /// 83 | /// As the memory comes from within the database itself, the bytes may not yet be 84 | /// initialized. Thus, it is up to the caller to ensure that only initialized memory is read 85 | /// (ensured by the [`MaybeUninit`] API). 86 | #[inline] 87 | pub fn as_uninit_mut(&mut self) -> &mut [MaybeUninit] { 88 | self.bytes 89 | } 90 | 91 | /// Marks the bytes in the range `0..len` as being initialized by advancing the internal write 92 | /// pointer. 93 | /// 94 | /// # Safety 95 | /// 96 | /// The caller guarantees that all bytes in the range have been initialized. 97 | #[inline] 98 | pub unsafe fn assume_written(&mut self, len: usize) { 99 | debug_assert!(len <= self.bytes.len()); 100 | self.written = len; 101 | self.write_head = len; 102 | } 103 | } 104 | 105 | impl io::Write for ReservedSpace<'_> { 106 | #[inline] 107 | fn write(&mut self, buf: &[u8]) -> io::Result { 108 | self.write_all(buf)?; 109 | Ok(buf.len()) 110 | } 111 | 112 | #[inline] 113 | fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { 114 | let remaining = unsafe { self.bytes.get_unchecked_mut(self.write_head..) }; 115 | 116 | if buf.len() > remaining.len() { 117 | return Err(io::Error::from(io::ErrorKind::WriteZero)); 118 | } 119 | 120 | unsafe { 121 | // SAFETY: we can always cast `T` -> `MaybeUninit` as it's a transparent wrapper 122 | let buf_uninit = std::slice::from_raw_parts(buf.as_ptr().cast(), buf.len()); 123 | remaining.as_mut_ptr().copy_from_nonoverlapping(buf_uninit.as_ptr(), buf.len()); 124 | } 125 | 126 | self.write_head += buf.len(); 127 | self.written = usize::max(self.written, self.write_head); 128 | 129 | Ok(()) 130 | } 131 | 132 | #[inline(always)] 133 | fn flush(&mut self) -> io::Result<()> { 134 | Ok(()) 135 | } 136 | } 137 | 138 | /// ## Note 139 | /// 140 | /// May only seek within the previously written space. 141 | /// Attempts to do otherwise will result in an error. 142 | impl io::Seek for ReservedSpace<'_> { 143 | #[inline] 144 | fn seek(&mut self, pos: io::SeekFrom) -> io::Result { 145 | let (base, offset) = match pos { 146 | io::SeekFrom::Start(start) => (start, 0), 147 | io::SeekFrom::End(offset) => (self.written as u64, offset), 148 | io::SeekFrom::Current(offset) => (self.write_head as u64, offset), 149 | }; 150 | 151 | let Some(new_pos) = base.checked_add_signed(offset) else { 152 | return Err(std::io::Error::new( 153 | std::io::ErrorKind::InvalidInput, 154 | "cannot seek before start of reserved space", 155 | )); 156 | }; 157 | 158 | if new_pos > self.written as u64 { 159 | return Err(std::io::Error::new( 160 | std::io::ErrorKind::InvalidInput, 161 | "cannot seek past end of reserved space", 162 | )); 163 | } 164 | 165 | self.write_head = new_pos as usize; 166 | 167 | Ok(new_pos) 168 | } 169 | 170 | #[inline] 171 | fn rewind(&mut self) -> io::Result<()> { 172 | self.write_head = 0; 173 | Ok(()) 174 | } 175 | 176 | #[inline] 177 | fn stream_position(&mut self) -> io::Result { 178 | Ok(self.write_head as u64) 179 | } 180 | } 181 | 182 | impl fmt::Debug for ReservedSpace<'_> { 183 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 184 | f.debug_struct("ReservedSpace").finish() 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /heed/src/txn.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::Cow; 2 | use std::marker::PhantomData; 3 | use std::ops::Deref; 4 | use std::ptr::{self, NonNull}; 5 | use std::sync::Arc; 6 | 7 | use crate::envs::{Env, EnvInner}; 8 | use crate::mdb::error::mdb_result; 9 | use crate::mdb::ffi; 10 | use crate::Result; 11 | 12 | /// A read-only transaction. 13 | /// 14 | /// ## LMDB Limitations 15 | /// 16 | /// It's a must to keep read transactions short-lived. 17 | /// 18 | /// Active Read transactions prevent the reuse of pages freed 19 | /// by newer write transactions, thus the database can grow quickly. 20 | /// 21 | /// ## OSX/Darwin Limitation 22 | /// 23 | /// At least 10 transactions can be active at the same time in the same process, since only 10 POSIX semaphores can 24 | /// be active at the same time for a process. Threads are in the same process space. 25 | /// 26 | /// If the process crashes in the POSIX semaphore locking section of the transaction, the semaphore will be kept locked. 27 | /// 28 | /// Note: if your program already use POSIX semaphores, you will have less available for heed/LMDB! 29 | /// 30 | /// You may increase the limit by editing it **at your own risk**: `/Library/LaunchDaemons/sysctl.plist` 31 | /// 32 | /// ## This struct is covariant 33 | /// 34 | /// ```rust 35 | /// #[allow(dead_code)] 36 | /// trait CovariantMarker<'a>: 'static { 37 | /// type R: 'a; 38 | /// 39 | /// fn is_covariant(&'a self) -> &'a Self::R; 40 | /// } 41 | /// 42 | /// impl<'a, T> CovariantMarker<'a> for heed::RoTxn<'static, T> { 43 | /// type R = heed::RoTxn<'a, T>; 44 | /// 45 | /// fn is_covariant(&'a self) -> &'a heed::RoTxn<'a, T> { 46 | /// self 47 | /// } 48 | /// } 49 | /// ``` 50 | #[repr(transparent)] 51 | pub struct RoTxn<'e, T = AnyTls> { 52 | inner: RoTxnInner<'e>, 53 | _tls_marker: PhantomData<&'e T>, 54 | } 55 | 56 | struct RoTxnInner<'e> { 57 | /// Makes the struct covariant and !Sync 58 | pub(crate) txn: Option>, 59 | env: Cow<'e, Arc>, 60 | } 61 | 62 | impl<'e, T> RoTxn<'e, T> { 63 | pub(crate) fn new(env: &'e Env) -> Result> { 64 | let mut txn: *mut ffi::MDB_txn = ptr::null_mut(); 65 | 66 | unsafe { 67 | mdb_result(ffi::mdb_txn_begin( 68 | env.env_mut_ptr().as_mut(), 69 | ptr::null_mut(), 70 | ffi::MDB_RDONLY, 71 | &mut txn, 72 | ))? 73 | }; 74 | 75 | Ok(RoTxn { 76 | inner: RoTxnInner { txn: NonNull::new(txn), env: Cow::Borrowed(&env.inner) }, 77 | _tls_marker: PhantomData, 78 | }) 79 | } 80 | 81 | pub(crate) fn static_read_txn(env: Env) -> Result> { 82 | let mut txn: *mut ffi::MDB_txn = ptr::null_mut(); 83 | 84 | unsafe { 85 | mdb_result(ffi::mdb_txn_begin( 86 | env.env_mut_ptr().as_mut(), 87 | ptr::null_mut(), 88 | ffi::MDB_RDONLY, 89 | &mut txn, 90 | ))? 91 | }; 92 | 93 | Ok(RoTxn { 94 | inner: RoTxnInner { txn: NonNull::new(txn), env: Cow::Owned(env.inner) }, 95 | _tls_marker: PhantomData, 96 | }) 97 | } 98 | 99 | pub(crate) fn txn_ptr(&self) -> NonNull { 100 | self.inner.txn.unwrap() 101 | } 102 | 103 | pub(crate) fn env_mut_ptr(&self) -> NonNull { 104 | self.inner.env.env_mut_ptr() 105 | } 106 | 107 | /// Return the transaction's ID. 108 | /// 109 | /// This returns the identifier associated with this transaction. For a 110 | /// [`RoTxn`], this corresponds to the snapshot being read; 111 | /// concurrent readers will frequently have the same transaction ID. 112 | pub fn id(&self) -> usize { 113 | unsafe { ffi::mdb_txn_id(self.inner.txn.unwrap().as_ptr()) } 114 | } 115 | 116 | /// Commit a read transaction. 117 | /// 118 | /// Synchronizing some [`Env`] metadata with the global handle. 119 | /// 120 | /// ## LMDB 121 | /// 122 | /// It's mandatory in a multi-process setup to call [`RoTxn::commit`] upon read-only database opening. 123 | /// After the transaction opening, the database is dropped. The next transaction might return 124 | /// `Io(Os { code: 22, kind: InvalidInput, message: "Invalid argument" })` known as `EINVAL`. 125 | pub fn commit(mut self) -> Result<()> { 126 | // Asserts that the transaction hasn't been already 127 | // committed/aborter and ensure we cannot use it twice. 128 | let mut txn = self.inner.txn.take().unwrap(); 129 | let result = unsafe { mdb_result(ffi::mdb_txn_commit(txn.as_mut())) }; 130 | result.map_err(Into::into) 131 | } 132 | } 133 | 134 | impl<'a> Deref for RoTxn<'a, WithTls> { 135 | type Target = RoTxn<'a, AnyTls>; 136 | 137 | fn deref(&self) -> &Self::Target { 138 | // SAFETY: OK because repr(transparent) means RoTxn always has the same layout 139 | // as RoTxnInner. 140 | unsafe { std::mem::transmute(self) } 141 | } 142 | } 143 | 144 | #[cfg(master3)] 145 | impl std::ops::DerefMut for RoTxn<'_, WithTls> { 146 | fn deref_mut(&mut self) -> &mut Self::Target { 147 | // SAFETY: OK because repr(transparent) means RoTxn always has the same layout 148 | // as RoTxnInner. 149 | unsafe { std::mem::transmute(self) } 150 | } 151 | } 152 | 153 | impl<'a> Deref for RoTxn<'a, WithoutTls> { 154 | type Target = RoTxn<'a, AnyTls>; 155 | 156 | fn deref(&self) -> &Self::Target { 157 | // SAFETY: OK because repr(transparent) means RoTxn always has the same layout 158 | // as RoTxnInner. 159 | unsafe { std::mem::transmute(self) } 160 | } 161 | } 162 | 163 | #[cfg(master3)] 164 | impl std::ops::DerefMut for RoTxn<'_, WithoutTls> { 165 | fn deref_mut(&mut self) -> &mut Self::Target { 166 | // SAFETY: OK because repr(transparent) means RoTxn always has the same layout 167 | // as RoTxnInner. 168 | unsafe { std::mem::transmute(self) } 169 | } 170 | } 171 | 172 | impl Drop for RoTxn<'_, T> { 173 | fn drop(&mut self) { 174 | if let Some(mut txn) = self.inner.txn.take() { 175 | // Asserts that the transaction hasn't been already 176 | // committed/aborter and ensure we cannot use it twice. 177 | unsafe { ffi::mdb_txn_abort(txn.as_mut()) } 178 | } 179 | } 180 | } 181 | 182 | /// Parameter defining that read transactions are opened with 183 | /// Thread Local Storage (TLS) and cannot be sent between threads 184 | /// `!Send`. It is often faster to open TLS-backed transactions. 185 | /// 186 | /// When used to open transactions: A thread can only use one transaction 187 | /// at a time, plus any child (nested) transactions. Each transaction belongs 188 | /// to one thread. A `BadRslot` error will be thrown when multiple read 189 | /// transactions exists on the same thread. 190 | #[derive(Debug, PartialEq, Eq)] 191 | pub enum WithTls {} 192 | 193 | /// Parameter defining that read transactions are opened without 194 | /// Thread Local Storage (TLS) and are therefore `Send`. 195 | /// 196 | /// When used to open transactions: A thread can use any number 197 | /// of read transactions at a time on the same thread. Read transactions 198 | /// can be moved in between threads (`Send`). 199 | #[derive(Debug, PartialEq, Eq)] 200 | pub enum WithoutTls {} 201 | 202 | /// Parameter defining that read transactions might have been opened with or 203 | /// without Thread Local Storage (TLS). 204 | /// 205 | /// `RwTxn`s and any `RoTxn` dereference to `&RoTxn`. 206 | pub enum AnyTls {} 207 | 208 | /// Specificies if Thread Local Storage (TLS) must be used when 209 | /// opening transactions. It is often faster to open TLS-backed 210 | /// transactions but makes them `!Send`. 211 | /// 212 | /// The `#MDB_NOTLS` flag is set on `Env` opening, `RoTxn`s and 213 | /// iterators implements the `Send` trait. This allows the user to 214 | /// move `RoTxn`s and iterators between threads as read transactions 215 | /// will no more use thread local storage and will tie reader 216 | /// locktable slots to transaction objects instead of to threads. 217 | pub trait TlsUsage { 218 | /// True if TLS must be used, false otherwise. 219 | const ENABLED: bool; 220 | } 221 | 222 | impl TlsUsage for WithTls { 223 | const ENABLED: bool = true; 224 | } 225 | 226 | impl TlsUsage for WithoutTls { 227 | const ENABLED: bool = false; 228 | } 229 | 230 | impl TlsUsage for AnyTls { 231 | // Users cannot open environments with AnyTls; therefore, this will never be read. 232 | // We prefer to put the most restrictive value. 233 | const ENABLED: bool = false; 234 | } 235 | 236 | /// Is sendable only if `MDB_NOTLS` has been used to open this transaction. 237 | unsafe impl Send for RoTxn<'_, WithoutTls> {} 238 | 239 | /// A read-write transaction. 240 | /// 241 | /// ## LMDB Limitations 242 | /// 243 | /// Only one [`RwTxn`] may exist in the same environment at the same time. 244 | /// If two exist, the new one may wait on a mutex for [`RwTxn::commit`] or [`RwTxn::abort`] to 245 | /// be called for the first one. 246 | /// 247 | /// ## OSX/Darwin Limitation 248 | /// 249 | /// At least 10 transactions can be active at the same time in the same process, since only 10 POSIX semaphores can 250 | /// be active at the same time for a process. Threads are in the same process space. 251 | /// 252 | /// If the process crashes in the POSIX semaphore locking section of the transaction, the semaphore will be kept locked. 253 | /// 254 | /// Note: if your program already use POSIX semaphores, you will have less available for heed/LMDB! 255 | /// 256 | /// You may increase the limit by editing it **at your own risk**: `/Library/LaunchDaemons/sysctl.plist` 257 | /// 258 | /// ## This struct is covariant 259 | /// 260 | /// ```rust 261 | /// #[allow(dead_code)] 262 | /// trait CovariantMarker<'a>: 'static { 263 | /// type T: 'a; 264 | /// 265 | /// fn is_covariant(&'a self) -> &'a Self::T; 266 | /// } 267 | /// 268 | /// impl<'a> CovariantMarker<'a> for heed::RwTxn<'static> { 269 | /// type T = heed::RwTxn<'a>; 270 | /// 271 | /// fn is_covariant(&'a self) -> &'a heed::RwTxn<'a> { 272 | /// self 273 | /// } 274 | /// } 275 | /// ``` 276 | pub struct RwTxn<'p> { 277 | pub(crate) txn: RoTxn<'p, WithoutTls>, 278 | } 279 | 280 | impl<'p> RwTxn<'p> { 281 | pub(crate) fn new(env: &'p Env) -> Result> { 282 | let mut txn: *mut ffi::MDB_txn = ptr::null_mut(); 283 | 284 | unsafe { 285 | mdb_result(ffi::mdb_txn_begin( 286 | env.env_mut_ptr().as_mut(), 287 | ptr::null_mut(), 288 | 0, 289 | &mut txn, 290 | ))? 291 | }; 292 | 293 | Ok(RwTxn { 294 | txn: RoTxn { 295 | inner: RoTxnInner { txn: NonNull::new(txn), env: Cow::Borrowed(&env.inner) }, 296 | _tls_marker: PhantomData, 297 | }, 298 | }) 299 | } 300 | 301 | pub(crate) fn nested(env: &'p Env, parent: &'p mut RwTxn) -> Result> { 302 | let mut txn: *mut ffi::MDB_txn = ptr::null_mut(); 303 | let parent_ptr: *mut ffi::MDB_txn = unsafe { parent.txn.inner.txn.unwrap().as_mut() }; 304 | 305 | unsafe { 306 | mdb_result(ffi::mdb_txn_begin(env.env_mut_ptr().as_mut(), parent_ptr, 0, &mut txn))? 307 | }; 308 | 309 | Ok(RwTxn { 310 | txn: RoTxn { 311 | inner: RoTxnInner { txn: NonNull::new(txn), env: Cow::Borrowed(&env.inner) }, 312 | _tls_marker: PhantomData, 313 | }, 314 | }) 315 | } 316 | 317 | pub(crate) fn env_mut_ptr(&self) -> NonNull { 318 | self.txn.inner.env.env_mut_ptr() 319 | } 320 | 321 | /// Commit all the operations of a transaction into the database. 322 | /// The transaction is reset. 323 | pub fn commit(mut self) -> Result<()> { 324 | // Asserts that the transaction hasn't been already 325 | // committed/aborter and ensure we cannot use it two times. 326 | let mut txn = self.txn.inner.txn.take().unwrap(); 327 | let result = unsafe { mdb_result(ffi::mdb_txn_commit(txn.as_mut())) }; 328 | result.map_err(Into::into) 329 | } 330 | 331 | /// Abandon all the operations of the transaction instead of saving them. 332 | /// The transaction is reset. 333 | pub fn abort(mut self) { 334 | // Asserts that the transaction hasn't been already 335 | // committed/aborter and ensure we cannot use it twice. 336 | let mut txn = self.txn.inner.txn.take().unwrap(); 337 | unsafe { ffi::mdb_txn_abort(txn.as_mut()) } 338 | } 339 | } 340 | 341 | impl<'p> Deref for RwTxn<'p> { 342 | type Target = RoTxn<'p, WithoutTls>; 343 | 344 | fn deref(&self) -> &Self::Target { 345 | &self.txn 346 | } 347 | } 348 | 349 | // TODO can't we just always implement it? 350 | #[cfg(master3)] 351 | impl std::ops::DerefMut for RwTxn<'_> { 352 | fn deref_mut(&mut self) -> &mut Self::Target { 353 | &mut self.txn 354 | } 355 | } 356 | 357 | #[cfg(test)] 358 | mod tests { 359 | #[test] 360 | fn ro_txns_are_send() { 361 | use crate::{RoTxn, WithoutTls}; 362 | 363 | fn is_send() {} 364 | 365 | is_send::>(); 366 | } 367 | 368 | #[test] 369 | fn rw_txns_are_send() { 370 | use crate::RwTxn; 371 | 372 | fn is_send() {} 373 | 374 | is_send::(); 375 | } 376 | } 377 | -------------------------------------------------------------------------------- /heed3/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "heed3" 3 | version = "0.22.0" 4 | authors = ["Kerollmops "] 5 | description = "A fully typed LMDB (mdb.master3) wrapper with minimum overhead with support for encryption" 6 | license = "MIT" 7 | repository = "https://github.com/Kerollmops/heed" 8 | keywords = ["lmdb", "database", "storage", "typed", "encryption"] 9 | categories = ["database", "data-structures"] 10 | readme = "../README.md" 11 | edition = "2021" 12 | 13 | [dependencies] 14 | aead = { version = "0.5.2", default-features = false } 15 | bitflags = { version = "2.6.0", features = ["serde"] } 16 | byteorder = { version = "1.5.0", default-features = false } 17 | generic-array = { version = "0.14.7", features = ["serde"] } 18 | heed-traits = { version = "0.20.0", path = "../heed-traits" } 19 | heed-types = { version = "0.21.0", default-features = false, path = "../heed-types" } 20 | libc = "0.2.169" 21 | lmdb-master3-sys = { version = "0.2.5", path = "../lmdb-master3-sys" } 22 | once_cell = "1.20.2" 23 | page_size = "0.6.0" 24 | serde = { version = "1.0.217", features = ["derive"], optional = true } 25 | synchronoise = "1.0.1" 26 | 27 | [dev-dependencies] 28 | argon2 = { version = "0.5.3", features = ["std"] } 29 | memchr = "2.7.4" 30 | serde = { version = "1.0.217", features = ["derive"] } 31 | chacha20poly1305 = "0.10.1" 32 | tempfile = "3.15.0" 33 | 34 | [target.'cfg(windows)'.dependencies] 35 | url = "2.5.4" 36 | 37 | [features] 38 | # The `serde` feature makes some types serializable, 39 | # like the `EnvOpenOptions` struct. 40 | default = ["serde", "serde-bincode", "serde-json"] 41 | serde = ["bitflags/serde", "dep:serde"] 42 | 43 | # Enable the serde en/decoders for bincode, serde_json, or rmp_serde 44 | serde-bincode = ["heed-types/serde-bincode"] 45 | serde-json = ["heed-types/serde-json"] 46 | serde-rmp = ["heed-types/serde-rmp"] 47 | 48 | # serde_json features 49 | preserve_order = ["heed-types/preserve_order"] 50 | arbitrary_precision = ["heed-types/arbitrary_precision"] 51 | raw_value = ["heed-types/raw_value"] 52 | unbounded_depth = ["heed-types/unbounded_depth"] 53 | 54 | # Whether to tell LMDB to use POSIX semaphores during compilation 55 | # (instead of the default, which are System V semaphores). 56 | # POSIX semaphores are required for Apple's App Sandbox on iOS & macOS, 57 | # and are possibly faster and more appropriate for single-process use. 58 | # There are tradeoffs for both POSIX and SysV semaphores; which you 59 | # should look into before enabling this feature. Also, see here: 60 | # 61 | posix-sem = ["lmdb-master3-sys/posix-sem"] 62 | 63 | # These features configure the MDB_IDL_LOGN macro, which determines 64 | # the size of the free and dirty page lists (and thus the amount of memory 65 | # allocated when opening an LMDB environment in read-write mode). 66 | # 67 | # Each feature defines MDB_IDL_LOGN as the value in the name of the feature. 68 | # That means these features are mutually exclusive, and you must not specify 69 | # more than one at the same time (or the crate will fail to compile). 70 | # 71 | # For more information on the motivation for these features (and their effect), 72 | # see https://github.com/mozilla/lmdb/pull/2. 73 | mdb_idl_logn_8 = ["lmdb-master3-sys/mdb_idl_logn_8"] 74 | mdb_idl_logn_9 = ["lmdb-master3-sys/mdb_idl_logn_9"] 75 | mdb_idl_logn_10 = ["lmdb-master3-sys/mdb_idl_logn_10"] 76 | mdb_idl_logn_11 = ["lmdb-master3-sys/mdb_idl_logn_11"] 77 | mdb_idl_logn_12 = ["lmdb-master3-sys/mdb_idl_logn_12"] 78 | mdb_idl_logn_13 = ["lmdb-master3-sys/mdb_idl_logn_13"] 79 | mdb_idl_logn_14 = ["lmdb-master3-sys/mdb_idl_logn_14"] 80 | mdb_idl_logn_15 = ["lmdb-master3-sys/mdb_idl_logn_15"] 81 | mdb_idl_logn_16 = ["lmdb-master3-sys/mdb_idl_logn_16"] 82 | 83 | # Setting this enables you to use keys longer than 511 bytes. The exact limit 84 | # is computed by LMDB at compile time. You can find the exact value by calling 85 | # Env::max_key_size(). This value varies by architecture. 86 | # 87 | # Example max key sizes: 88 | # - Apple M1 (ARM64): 8126 bytes 89 | # - Apple Intel (AMD64): 1982 bytes 90 | # - Linux Intel (AMD64): 1982 bytes 91 | # 92 | # Setting this also enables you to use values larger than 511 bytes when using 93 | # a Database with the DatabaseFlags::DUP_SORT flag. 94 | # 95 | # This builds LMDB with the -DMDB_MAXKEYSIZE=0 option. 96 | # 97 | # Note: If you are moving database files between architectures then your longest 98 | # stored key must fit within the smallest limit of all architectures used. For 99 | # example, if you are moving databases between Apple M1 and Apple Intel 100 | # computers then you need to keep your keys within the smaller 1982 byte limit. 101 | longer-keys = ["lmdb-master3-sys/longer-keys"] 102 | 103 | # Enable a better Valgrind support. This builds LMDB with the -DUSE_VALGRIND=1 option. 104 | # 105 | # You have to install the RPM valgrind-devel which contains memcheck.h. 106 | # 107 | # More information can be found at: 108 | # 109 | use-valgrind = ["lmdb-master3-sys/use-valgrind"] 110 | 111 | # Examples are located outside the standard heed/examples directory to prevent 112 | # conflicts between heed3 and heed examples when working on both crates. 113 | [[example]] 114 | name = "prev-snapshot" 115 | 116 | [[example]] 117 | name = "heed3-encrypted" 118 | 119 | [[example]] 120 | name = "heed3-all-types" 121 | -------------------------------------------------------------------------------- /heed3/examples/heed3-all-types.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use heed3::byteorder::BE; 4 | use heed3::types::*; 5 | use heed3::{Database, EnvOpenOptions}; 6 | use serde::{Deserialize, Serialize}; 7 | 8 | fn main() -> Result<(), Box> { 9 | let path = tempfile::tempdir()?; 10 | 11 | let env = unsafe { 12 | EnvOpenOptions::new() 13 | .map_size(10 * 1024 * 1024) // 10MB 14 | .max_dbs(3000) 15 | .open(path)? 16 | }; 17 | 18 | // here the key will be an str and the data will be a slice of u8 19 | let mut wtxn = env.write_txn()?; 20 | let db: Database = env.create_database(&mut wtxn, Some("kiki"))?; 21 | 22 | db.put(&mut wtxn, "hello", &[2, 3][..])?; 23 | let ret: Option<&[u8]> = db.get(&wtxn, "hello")?; 24 | 25 | println!("{:?}", ret); 26 | wtxn.commit()?; 27 | 28 | // serde types are also supported!!! 29 | #[derive(Debug, Serialize, Deserialize)] 30 | struct Hello<'a> { 31 | string: &'a str, 32 | } 33 | 34 | let mut wtxn = env.write_txn()?; 35 | let db: Database> = 36 | env.create_database(&mut wtxn, Some("serde-bincode"))?; 37 | 38 | let hello = Hello { string: "hi" }; 39 | db.put(&mut wtxn, "hello", &hello)?; 40 | 41 | let ret: Option = db.get(&wtxn, "hello")?; 42 | println!("serde-bincode:\t{:?}", ret); 43 | 44 | wtxn.commit()?; 45 | 46 | let mut wtxn = env.write_txn()?; 47 | let db: Database> = env.create_database(&mut wtxn, Some("serde-json"))?; 48 | 49 | let hello = Hello { string: "hi" }; 50 | db.put(&mut wtxn, "hello", &hello)?; 51 | 52 | let ret: Option = db.get(&wtxn, "hello")?; 53 | println!("serde-json:\t{:?}", ret); 54 | 55 | wtxn.commit()?; 56 | 57 | // you can ignore the data 58 | let mut wtxn = env.write_txn()?; 59 | let db: Database = env.create_database(&mut wtxn, Some("ignored-data"))?; 60 | 61 | db.put(&mut wtxn, "hello", &())?; 62 | let ret: Option<()> = db.get(&wtxn, "hello")?; 63 | 64 | println!("{:?}", ret); 65 | 66 | let ret: Option<()> = db.get(&wtxn, "non-existant")?; 67 | 68 | println!("{:?}", ret); 69 | wtxn.commit()?; 70 | 71 | // database opening and types are tested in a safe way 72 | // 73 | // we try to open a database twice with the same types 74 | let mut wtxn = env.write_txn()?; 75 | let _db: Database = env.create_database(&mut wtxn, Some("ignored-data"))?; 76 | 77 | // you can iterate over keys in order 78 | type BEI64 = I64; 79 | 80 | let db: Database = env.create_database(&mut wtxn, Some("big-endian-iter"))?; 81 | 82 | db.put(&mut wtxn, &0, &())?; 83 | db.put(&mut wtxn, &68, &())?; 84 | db.put(&mut wtxn, &35, &())?; 85 | db.put(&mut wtxn, &42, &())?; 86 | 87 | let rets: Result, _> = db.iter(&wtxn)?.collect(); 88 | 89 | println!("{:?}", rets); 90 | 91 | // or iterate over ranges too!!! 92 | let range = 35..=42; 93 | let rets: Result, _> = db.range(&wtxn, &range)?.collect(); 94 | 95 | println!("{:?}", rets); 96 | 97 | // delete a range of key 98 | let range = 35..=42; 99 | let deleted: usize = db.delete_range(&mut wtxn, &range)?; 100 | 101 | let rets: Result, _> = db.iter(&wtxn)?.collect(); 102 | 103 | println!("deleted: {:?}, {:?}", deleted, rets); 104 | wtxn.commit()?; 105 | 106 | Ok(()) 107 | } 108 | -------------------------------------------------------------------------------- /heed3/examples/heed3-encrypted.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use argon2::Argon2; 4 | use chacha20poly1305::{ChaCha20Poly1305, Key}; 5 | use heed3::types::*; 6 | use heed3::EnvOpenOptions; 7 | 8 | fn main() -> Result<(), Box> { 9 | let env_path = tempfile::tempdir()?; 10 | let password = "This is the password that will be hashed by the argon2 algorithm"; 11 | let salt = "The salt added to the password hashes to add more security when stored"; 12 | 13 | // We choose to use argon2 as our Key Derivation Function, but you can choose whatever you want. 14 | // 15 | let mut key = Key::default(); 16 | Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; 17 | 18 | // We open the environment 19 | let mut options = EnvOpenOptions::new(); 20 | let env = unsafe { 21 | options 22 | .map_size(10 * 1024 * 1024) // 10MB 23 | .max_dbs(3) 24 | .open_encrypted::(key, &env_path)? 25 | }; 26 | 27 | let key1 = "first-key"; 28 | let val1 = "this is a secret info"; 29 | let key2 = "second-key"; 30 | let val2 = "this is another secret info"; 31 | 32 | // We create database and write secret values in it 33 | let mut wtxn = env.write_txn()?; 34 | let db = env.create_database::(&mut wtxn, Some("first"))?; 35 | db.put(&mut wtxn, key1, val1)?; 36 | db.put(&mut wtxn, key2, val2)?; 37 | wtxn.commit()?; 38 | env.prepare_for_closing().wait(); 39 | 40 | // We reopen the environment now 41 | let env = unsafe { options.open_encrypted::(key, &env_path)? }; 42 | 43 | // We check that the secret entries are correctly decrypted 44 | let mut rtxn = env.read_txn()?; 45 | let db = env.open_database::(&rtxn, Some("first"))?.unwrap(); 46 | let mut iter = db.iter(&mut rtxn)?; 47 | assert_eq!(iter.next().transpose()?, Some((key1, val1))); 48 | assert_eq!(iter.next().transpose()?, Some((key2, val2))); 49 | assert_eq!(iter.next().transpose()?, None); 50 | 51 | eprintln!("Successful test!"); 52 | 53 | Ok(()) 54 | } 55 | -------------------------------------------------------------------------------- /heed3/examples/prev-snapshot.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use heed3::types::*; 4 | use heed3::{Database, EnvFlags, EnvOpenOptions}; 5 | 6 | // In this test we are checking that we can move to a previous environement snapshot. 7 | fn main() -> Result<(), Box> { 8 | let env_path = tempfile::tempdir()?; 9 | 10 | let env = unsafe { 11 | EnvOpenOptions::new() 12 | .map_size(10 * 1024 * 1024) // 10MB 13 | .max_dbs(3) 14 | .open(&env_path)? 15 | }; 16 | 17 | let mut wtxn = env.write_txn()?; 18 | let db: Database = env.create_database(&mut wtxn, None)?; 19 | 20 | // We fill the db database with entries. 21 | db.put(&mut wtxn, "I am here", "to test things")?; 22 | db.put(&mut wtxn, "I am here too", "for the same purpose")?; 23 | 24 | wtxn.commit()?; 25 | 26 | env.prepare_for_closing().wait(); 27 | 28 | // We can get the env state from before the last commit 29 | // and therefore see an empty env. 30 | let env = unsafe { 31 | EnvOpenOptions::new() 32 | .map_size(10 * 1024 * 1024) // 10MB 33 | .max_dbs(3) 34 | .flags(EnvFlags::PREV_SNAPSHOT) 35 | .open(&env_path)? 36 | }; 37 | 38 | let mut wtxn = env.write_txn()?; 39 | let db: Database = env.create_database(&mut wtxn, None)?; 40 | 41 | assert!(db.is_empty(&wtxn)?); 42 | 43 | wtxn.abort(); 44 | env.prepare_for_closing().wait(); 45 | 46 | // However, if we don't commit we can still get 47 | // back the latest version of the env. 48 | let env = unsafe { 49 | EnvOpenOptions::new() 50 | .map_size(10 * 1024 * 1024) // 10MB 51 | .max_dbs(3) 52 | .open(&env_path)? 53 | }; 54 | 55 | let mut wtxn = env.write_txn()?; 56 | let db: Database = env.create_database(&mut wtxn, None)?; 57 | 58 | assert_eq!(db.get(&wtxn, "I am here")?, Some("to test things")); 59 | assert_eq!(db.get(&wtxn, "I am here too")?, Some("for the same purpose")); 60 | 61 | // And write new stuff in the env. 62 | db.put(&mut wtxn, "I will fade away", "I am so sad")?; 63 | 64 | wtxn.commit()?; 65 | env.prepare_for_closing().wait(); 66 | 67 | // Once again we can get back the previous version 68 | // of the env and see some entries disappear. 69 | let env = unsafe { 70 | EnvOpenOptions::new() 71 | .map_size(10 * 1024 * 1024) // 10MB 72 | .max_dbs(3) 73 | .flags(EnvFlags::PREV_SNAPSHOT) 74 | .open(&env_path)? 75 | }; 76 | 77 | let rtxn = env.read_txn()?; 78 | let db: Database = env.open_database(&rtxn, None)?.unwrap(); 79 | 80 | assert_eq!(db.get(&rtxn, "I am here")?, Some("to test things")); 81 | assert_eq!(db.get(&rtxn, "I am here too")?, Some("for the same purpose")); 82 | assert_eq!(db.get(&rtxn, "I will fade away")?, None); 83 | 84 | Ok(()) 85 | } 86 | -------------------------------------------------------------------------------- /lmdb-master-sys/.rustfmt.toml: -------------------------------------------------------------------------------- 1 | ignore = [ 2 | "src/bindings.rs" 3 | ] -------------------------------------------------------------------------------- /lmdb-master-sys/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "lmdb-master-sys" 3 | # NB: When modifying, also modify html_root_url in lib.rs 4 | version = "0.2.5" 5 | authors = [ 6 | "Kerollmops ", 7 | "Dan Burkert ", 8 | "Victor Porof ", 9 | ] 10 | license = "Apache-2.0" 11 | description = "Rust bindings for liblmdb on the mdb.master branch." 12 | documentation = "https://docs.rs/lmdb-master-sys" 13 | repository = "https://github.com/meilisearch/heed/tree/main/lmdb-master-sys" 14 | readme = "README.md" 15 | keywords = ["LMDB", "database", "storage-engine", "bindings", "library"] 16 | categories = ["database", "external-ffi-bindings"] 17 | edition = "2021" 18 | 19 | # NB: Use "--features bindgen" to generate bindings. 20 | build = "build.rs" 21 | 22 | [lib] 23 | name = "lmdb_master_sys" 24 | doctest = false 25 | 26 | [dependencies] 27 | libc = "0.2.170" 28 | 29 | [build-dependencies] 30 | bindgen = { version = "0.71.1", default-features = false, optional = true, features = [ 31 | "runtime", 32 | ] } 33 | cc = "1.2.16" 34 | doxygen-rs = "0.4.2" 35 | 36 | [dev-dependencies] 37 | cstr = "0.2.12" 38 | 39 | [features] 40 | default = [] 41 | asan = [] 42 | fuzzer = [] 43 | fuzzer-no-link = [] 44 | posix-sem = [] 45 | 46 | # These features configure the MDB_IDL_LOGN macro, which determines 47 | # the size of the free and dirty page lists (and thus the amount of memory 48 | # allocated when opening an LMDB environment in read-write mode). 49 | # 50 | # Each feature defines MDB_IDL_LOGN as the value in the name of the feature. 51 | # That means these features are mutually exclusive, and you must not specify 52 | # more than one at the same time (or the crate will fail to compile). 53 | # 54 | # For more information on the motivation for these features (and their effect), 55 | # see https://github.com/mozilla/lmdb/pull/2. 56 | mdb_idl_logn_8 = [] 57 | mdb_idl_logn_9 = [] 58 | mdb_idl_logn_10 = [] 59 | mdb_idl_logn_11 = [] 60 | mdb_idl_logn_12 = [] 61 | mdb_idl_logn_13 = [] 62 | mdb_idl_logn_14 = [] 63 | mdb_idl_logn_15 = [] 64 | mdb_idl_logn_16 = [] 65 | 66 | # Enable the USE_VALGRIND feature. 67 | # 68 | # You have to install the RPM valgrind-devel which contains memcheck.h. 69 | use-valgrind = [] 70 | 71 | # Setting this enables you to use keys longer than 511 bytes. The exact limit 72 | # is computed by LMDB at compile time. You can find the exact value by calling 73 | # Env::max_key_size(). This value varies by architecture. 74 | # 75 | # Example max key sizes: 76 | # - Apple M1 (ARM64): 8126 bytes 77 | # - Apple Intel (AMD64): 1982 bytes 78 | # - Linux Intel (AMD64): 1982 bytes 79 | # 80 | # Setting this also enables you to use values larger than 511 bytes when using 81 | # a Database with the DatabaseFlags::DUP_SORT flag. 82 | # 83 | # This builds LMDB with the -DMDB_MAXKEYSIZE=0 option. 84 | # 85 | # Note: If you are moving database files between architectures then your longest 86 | # stored key must fit within the smallest limit of all architectures used. For 87 | # example, if you are moving databases between Apple M1 and Apple Intel 88 | # computers then you need to keep your keys within the smaller 1982 byte limit. 89 | longer-keys = [] 90 | -------------------------------------------------------------------------------- /lmdb-master-sys/README.md: -------------------------------------------------------------------------------- 1 | # lmdb-master-sys 2 | 3 | Rust bindings for liblmdb on the mdb.master branch. 4 | -------------------------------------------------------------------------------- /lmdb-master-sys/bindgen.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | use std::path::PathBuf; 3 | 4 | use bindgen::callbacks::{IntKind, ParseCallbacks}; 5 | 6 | #[derive(Debug)] 7 | struct Callbacks; 8 | 9 | impl ParseCallbacks for Callbacks { 10 | fn process_comment(&self, comment: &str) -> Option { 11 | Some(doxygen_rs::transform(comment)) 12 | } 13 | 14 | fn int_macro(&self, name: &str, _value: i64) -> Option { 15 | match name { 16 | "MDB_SUCCESS" 17 | | "MDB_KEYEXIST" 18 | | "MDB_NOTFOUND" 19 | | "MDB_PAGE_NOTFOUND" 20 | | "MDB_CORRUPTED" 21 | | "MDB_PANIC" 22 | | "MDB_VERSION_MISMATCH" 23 | | "MDB_INVALID" 24 | | "MDB_MAP_FULL" 25 | | "MDB_DBS_FULL" 26 | | "MDB_READERS_FULL" 27 | | "MDB_TLS_FULL" 28 | | "MDB_TXN_FULL" 29 | | "MDB_CURSOR_FULL" 30 | | "MDB_PAGE_FULL" 31 | | "MDB_MAP_RESIZED" 32 | | "MDB_INCOMPATIBLE" 33 | | "MDB_BAD_RSLOT" 34 | | "MDB_BAD_TXN" 35 | | "MDB_BAD_VALSIZE" 36 | | "MDB_BAD_DBI" 37 | | "MDB_LAST_ERRCODE" => Some(IntKind::Int), 38 | "MDB_SIZE_MAX" => Some(IntKind::U64), 39 | "MDB_PROBLEM" | "MDB_BAD_CHECKSUM" | "MDB_CRYPTO_FAIL" | "MDB_ENV_ENCRYPTION" => { 40 | Some(IntKind::Int) 41 | } 42 | _ => Some(IntKind::UInt), 43 | } 44 | } 45 | } 46 | 47 | pub fn generate() { 48 | let mut lmdb = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); 49 | lmdb.push("lmdb"); 50 | lmdb.push("libraries"); 51 | lmdb.push("liblmdb"); 52 | 53 | let mut out_path = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); 54 | out_path.push("src"); 55 | 56 | let bindings = bindgen::Builder::default() 57 | .header(lmdb.join("lmdb.h").to_string_lossy()) 58 | .allowlist_var("^(MDB|mdb)_.*") 59 | .allowlist_type("^(MDB|mdb)_.*") 60 | .allowlist_function("^(MDB|mdb)_.*") 61 | .size_t_is_usize(true) 62 | .ctypes_prefix("::libc") 63 | .blocklist_item("mode_t") 64 | .blocklist_item("mdb_mode_t") 65 | .blocklist_item("mdb_filehandle_t") 66 | .blocklist_item("^__.*") 67 | .parse_callbacks(Box::new(Callbacks {})) 68 | .layout_tests(false) 69 | .prepend_enum_name(false) 70 | .generate() 71 | .expect("Unable to generate bindings"); 72 | 73 | bindings 74 | .write_to_file(out_path.join("bindings.rs")) 75 | .expect("Couldn't write bindings!"); 76 | } 77 | -------------------------------------------------------------------------------- /lmdb-master-sys/build.rs: -------------------------------------------------------------------------------- 1 | extern crate cc; 2 | 3 | #[cfg(feature = "bindgen")] 4 | extern crate bindgen; 5 | 6 | #[cfg(feature = "bindgen")] 7 | #[path = "bindgen.rs"] 8 | mod generate; 9 | 10 | use std::env; 11 | use std::path::PathBuf; 12 | 13 | #[cfg(all( 14 | feature = "mdb_idl_logn_8", 15 | not(any( 16 | feature = "mdb_idl_logn_9", 17 | feature = "mdb_idl_logn_10", 18 | feature = "mdb_idl_logn_11", 19 | feature = "mdb_idl_logn_12", 20 | feature = "mdb_idl_logn_13", 21 | feature = "mdb_idl_logn_14", 22 | feature = "mdb_idl_logn_15", 23 | feature = "mdb_idl_logn_16" 24 | )) 25 | ))] 26 | const MDB_IDL_LOGN: u8 = 8; 27 | #[cfg(all( 28 | feature = "mdb_idl_logn_9", 29 | not(any( 30 | feature = "mdb_idl_logn_10", 31 | feature = "mdb_idl_logn_11", 32 | feature = "mdb_idl_logn_12", 33 | feature = "mdb_idl_logn_13", 34 | feature = "mdb_idl_logn_14", 35 | feature = "mdb_idl_logn_15", 36 | feature = "mdb_idl_logn_16" 37 | )) 38 | ))] 39 | const MDB_IDL_LOGN: u8 = 9; 40 | #[cfg(all( 41 | feature = "mdb_idl_logn_10", 42 | not(any( 43 | feature = "mdb_idl_logn_11", 44 | feature = "mdb_idl_logn_12", 45 | feature = "mdb_idl_logn_13", 46 | feature = "mdb_idl_logn_14", 47 | feature = "mdb_idl_logn_15", 48 | feature = "mdb_idl_logn_16" 49 | )) 50 | ))] 51 | const MDB_IDL_LOGN: u8 = 10; 52 | #[cfg(all( 53 | feature = "mdb_idl_logn_11", 54 | not(any( 55 | feature = "mdb_idl_logn_12", 56 | feature = "mdb_idl_logn_13", 57 | feature = "mdb_idl_logn_14", 58 | feature = "mdb_idl_logn_15", 59 | feature = "mdb_idl_logn_16" 60 | )) 61 | ))] 62 | const MDB_IDL_LOGN: u8 = 11; 63 | #[cfg(all( 64 | feature = "mdb_idl_logn_12", 65 | not(any( 66 | feature = "mdb_idl_logn_13", 67 | feature = "mdb_idl_logn_14", 68 | feature = "mdb_idl_logn_15", 69 | feature = "mdb_idl_logn_16" 70 | )) 71 | ))] 72 | const MDB_IDL_LOGN: u8 = 12; 73 | #[cfg(all( 74 | feature = "mdb_idl_logn_13", 75 | not(any( 76 | feature = "mdb_idl_logn_14", 77 | feature = "mdb_idl_logn_15", 78 | feature = "mdb_idl_logn_16" 79 | )) 80 | ))] 81 | const MDB_IDL_LOGN: u8 = 13; 82 | #[cfg(all( 83 | feature = "mdb_idl_logn_14", 84 | not(any(feature = "mdb_idl_logn_15", feature = "mdb_idl_logn_16")) 85 | ))] 86 | const MDB_IDL_LOGN: u8 = 14; 87 | #[cfg(all(feature = "mdb_idl_logn_15", not(any(feature = "mdb_idl_logn_16"))))] 88 | const MDB_IDL_LOGN: u8 = 15; 89 | #[cfg(any( 90 | feature = "mdb_idl_logn_16", 91 | not(any( 92 | feature = "mdb_idl_logn_8", 93 | feature = "mdb_idl_logn_9", 94 | feature = "mdb_idl_logn_10", 95 | feature = "mdb_idl_logn_11", 96 | feature = "mdb_idl_logn_12", 97 | feature = "mdb_idl_logn_13", 98 | feature = "mdb_idl_logn_14", 99 | feature = "mdb_idl_logn_15", 100 | feature = "mdb_idl_logn_16", 101 | )) 102 | ))] 103 | const MDB_IDL_LOGN: u8 = 16; 104 | 105 | macro_rules! warn { 106 | ($message:expr) => { 107 | println!("cargo:warning={}", $message); 108 | }; 109 | } 110 | 111 | fn main() { 112 | #[cfg(feature = "bindgen")] 113 | generate::generate(); 114 | 115 | println!("cargo::rerun-if-changed=lmdb"); 116 | 117 | let mut lmdb = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); 118 | lmdb.push("lmdb"); 119 | lmdb.push("libraries"); 120 | lmdb.push("liblmdb"); 121 | 122 | if cfg!(feature = "fuzzer") && cfg!(feature = "fuzzer-no-link") { 123 | warn!("Features `fuzzer` and `fuzzer-no-link` are mutually exclusive."); 124 | warn!("Building with `-fsanitize=fuzzer`."); 125 | } 126 | 127 | let mut builder = cc::Build::new(); 128 | 129 | builder 130 | .define("MDB_IDL_LOGN", Some(MDB_IDL_LOGN.to_string().as_str())) 131 | .file(lmdb.join("mdb.c")) 132 | .file(lmdb.join("midl.c")) 133 | // https://github.com/mozilla/lmdb/blob/b7df2cac50fb41e8bd16aab4cc5fd167be9e032a/libraries/liblmdb/Makefile#L23 134 | .flag_if_supported("-Wno-unused-parameter") 135 | .flag_if_supported("-Wbad-function-cast") 136 | .flag_if_supported("-Wuninitialized"); 137 | 138 | if cfg!(feature = "posix-sem") { 139 | builder.define("MDB_USE_POSIX_SEM", None); 140 | } 141 | 142 | if cfg!(feature = "use-valgrind") { 143 | builder.define("USE_VALGRIND", None); 144 | } 145 | 146 | if cfg!(feature = "asan") { 147 | builder.flag("-fsanitize=address"); 148 | } 149 | 150 | if cfg!(feature = "fuzzer") { 151 | builder.flag("-fsanitize=fuzzer"); 152 | } else if cfg!(feature = "fuzzer-no-link") { 153 | builder.flag("-fsanitize=fuzzer-no-link"); 154 | } 155 | 156 | if cfg!(feature = "longer-keys") { 157 | builder.define("MDB_MAXKEYSIZE", "0"); 158 | } 159 | 160 | if !cfg!(debug_assertions) { 161 | builder.define("NDEBUG", None); 162 | } 163 | 164 | builder.compile("liblmdb.a") 165 | } 166 | -------------------------------------------------------------------------------- /lmdb-master-sys/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(warnings)] 2 | #![allow(rustdoc::broken_intra_doc_links)] 3 | #![allow(rustdoc::invalid_html_tags)] 4 | #![allow(non_camel_case_types)] 5 | #![allow(clippy::all)] 6 | #![doc(html_root_url = "https://docs.rs/lmdb-master-sys/0.2.5")] 7 | 8 | extern crate libc; 9 | 10 | #[cfg(unix)] 11 | #[allow(non_camel_case_types)] 12 | pub type mdb_mode_t = ::libc::mode_t; 13 | #[cfg(windows)] 14 | #[allow(non_camel_case_types)] 15 | pub type mdb_mode_t = ::libc::c_int; 16 | 17 | #[cfg(unix)] 18 | #[allow(non_camel_case_types)] 19 | pub type mdb_filehandle_t = ::libc::c_int; 20 | #[cfg(windows)] 21 | #[allow(non_camel_case_types)] 22 | pub type mdb_filehandle_t = *mut ::libc::c_void; 23 | 24 | include!("bindings.rs"); 25 | -------------------------------------------------------------------------------- /lmdb-master-sys/tests/fixtures/testdb-32/data.mdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meilisearch/heed/5a10a00e02669d28edd6d11d35b617a9128189ca/lmdb-master-sys/tests/fixtures/testdb-32/data.mdb -------------------------------------------------------------------------------- /lmdb-master-sys/tests/fixtures/testdb-32/lock.mdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meilisearch/heed/5a10a00e02669d28edd6d11d35b617a9128189ca/lmdb-master-sys/tests/fixtures/testdb-32/lock.mdb -------------------------------------------------------------------------------- /lmdb-master-sys/tests/lmdb.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | use std::path::PathBuf; 3 | use std::process::Command; 4 | 5 | #[test] 6 | fn test_lmdb() { 7 | let mut lmdb = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); 8 | lmdb.push("lmdb"); 9 | lmdb.push("libraries"); 10 | lmdb.push("liblmdb"); 11 | 12 | let make_cmd = Command::new("make") 13 | .current_dir(&lmdb) 14 | .status() 15 | .expect("failed to execute process"); 16 | 17 | assert!(make_cmd.success()); 18 | 19 | let make_test_cmd = Command::new("make") 20 | .arg("test") 21 | .current_dir(&lmdb) 22 | .status() 23 | .expect("failed to execute process"); 24 | 25 | assert!(make_test_cmd.success()); 26 | } 27 | -------------------------------------------------------------------------------- /lmdb-master-sys/tests/simple.rs: -------------------------------------------------------------------------------- 1 | use std::ffi::{c_void, CString}; 2 | use std::fs::{self, File}; 3 | use std::ptr; 4 | 5 | use cstr::cstr; 6 | use lmdb_master_sys::*; 7 | 8 | // https://github.com/victorporof/lmdb/blob/mdb.master/libraries/liblmdb/moz-test.c 9 | 10 | macro_rules! E { 11 | ($expr:expr) => {{ 12 | match $expr { 13 | lmdb_master_sys::MDB_SUCCESS => (), 14 | err_code => assert!(false, "Failed with code {}", err_code), 15 | } 16 | }}; 17 | } 18 | 19 | #[test] 20 | #[cfg(target_pointer_width = "32")] 21 | fn test_simple_32() { 22 | test_simple("./tests/fixtures/testdb-32") 23 | } 24 | 25 | #[test] 26 | #[cfg(target_pointer_width = "64")] 27 | fn test_simple_64() { 28 | test_simple("./tests/fixtures/testdb") 29 | } 30 | 31 | #[cfg(windows)] 32 | fn get_file_fd(file: &File) -> std::os::windows::io::RawHandle { 33 | use std::os::windows::io::AsRawHandle; 34 | file.as_raw_handle() 35 | } 36 | 37 | #[cfg(unix)] 38 | fn get_file_fd(file: &File) -> std::os::unix::io::RawFd { 39 | use std::os::unix::io::AsRawFd; 40 | file.as_raw_fd() 41 | } 42 | 43 | fn test_simple(env_path: &str) { 44 | let _ = fs::remove_dir_all(env_path); 45 | fs::create_dir_all(env_path).unwrap(); 46 | 47 | let mut env: *mut MDB_env = ptr::null_mut(); 48 | let mut dbi: MDB_dbi = 0; 49 | let mut key = MDB_val { 50 | mv_size: 0, 51 | mv_data: ptr::null_mut(), 52 | }; 53 | let mut data = MDB_val { 54 | mv_size: 0, 55 | mv_data: ptr::null_mut(), 56 | }; 57 | let mut txn: *mut MDB_txn = ptr::null_mut(); 58 | let sval = cstr!("foo").as_ptr() as *mut c_void; 59 | let dval = cstr!("bar").as_ptr() as *mut c_void; 60 | 61 | unsafe { 62 | E!(mdb_env_create(&mut env)); 63 | E!(mdb_env_set_maxdbs(env, 2)); 64 | let env_path = CString::new(env_path).unwrap(); 65 | E!(mdb_env_open(env, env_path.as_ptr(), 0, 0o664)); 66 | 67 | E!(mdb_txn_begin(env, ptr::null_mut(), 0, &mut txn)); 68 | E!(mdb_dbi_open( 69 | txn, 70 | cstr!("subdb").as_ptr(), 71 | MDB_CREATE, 72 | &mut dbi 73 | )); 74 | E!(mdb_txn_commit(txn)); 75 | 76 | key.mv_size = 3; 77 | key.mv_data = sval; 78 | data.mv_size = 3; 79 | data.mv_data = dval; 80 | 81 | E!(mdb_txn_begin(env, ptr::null_mut(), 0, &mut txn)); 82 | E!(mdb_put(txn, dbi, &mut key, &mut data, 0)); 83 | 84 | if cfg!(feature = "longer-keys") { 85 | // Try storing a key larger than 511 bytes (the default if MDB_MAXKEYSIZE is not set) 86 | let sval = cstr!("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut pharetra sit amet aliquam. Sit amet nisl purus in mollis nunc. Eget egestas purus viverra accumsan in nisl nisi scelerisque. Duis ultricies lacus sed turpis tincidunt. Sem nulla pharetra diam sit. Leo vel orci porta non pulvinar. Erat pellentesque adipiscing commodo elit at imperdiet dui. Suspendisse ultrices gravida dictum fusce ut placerat orci nulla. Diam donec adipiscing tristique risus nec feugiat. In fermentum et sollicitudin ac orci. Ut sem nulla pharetra diam sit amet. Aliquam purus sit amet luctus venenatis lectus. Erat pellentesque adipiscing commodo elit at imperdiet dui accumsan. Urna duis convallis convallis tellus id interdum velit laoreet id. Ac feugiat sed lectus vestibulum mattis ullamcorper velit sed. Tincidunt arcu non sodales neque. Habitant morbi tristique senectus et netus et malesuada fames.").as_ptr() as *mut c_void; 87 | 88 | key.mv_size = 952; 89 | key.mv_data = sval; 90 | 91 | E!(mdb_put(txn, dbi, &mut key, &mut data, 0)); 92 | } 93 | 94 | E!(mdb_txn_commit(txn)); 95 | } 96 | 97 | let file = File::create("./tests/fixtures/copytestdb.mdb").unwrap(); 98 | 99 | unsafe { 100 | let fd = get_file_fd(&file); 101 | E!(mdb_env_copyfd(env, fd)); 102 | 103 | mdb_dbi_close(env, dbi); 104 | mdb_env_close(env); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /lmdb-master3-sys/.rustfmt.toml: -------------------------------------------------------------------------------- 1 | ignore = [ 2 | "src/bindings.rs" 3 | ] -------------------------------------------------------------------------------- /lmdb-master3-sys/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "lmdb-master3-sys" 3 | # NB: When modifying, also modify html_root_url in lib.rs 4 | version = "0.2.5" 5 | authors = [ 6 | "Kerollmops ", 7 | "Dan Burkert ", 8 | "Victor Porof ", 9 | ] 10 | license = "Apache-2.0" 11 | description = "Rust bindings for liblmdb on the mdb.master3 branch." 12 | documentation = "https://docs.rs/lmdb-master3-sys" 13 | repository = "https://github.com/meilisearch/heed/tree/main/lmdb-master3-sys" 14 | readme = "README.md" 15 | keywords = ["LMDB", "database", "storage-engine", "bindings", "library"] 16 | categories = ["database", "external-ffi-bindings"] 17 | edition = "2021" 18 | 19 | # NB: Use "--features bindgen" to generate bindings. 20 | build = "build.rs" 21 | 22 | [lib] 23 | name = "lmdb_master3_sys" 24 | doctest = false 25 | 26 | [dependencies] 27 | libc = "0.2.170" 28 | 29 | [build-dependencies] 30 | bindgen = { version = "0.71.1", default-features = false, optional = true, features = [ 31 | "runtime", 32 | ] } 33 | cc = "1.2.16" 34 | doxygen-rs = "0.4.2" 35 | 36 | [dev-dependencies] 37 | cstr = "0.2.12" 38 | 39 | [features] 40 | default = [] 41 | asan = [] 42 | fuzzer = [] 43 | fuzzer-no-link = [] 44 | posix-sem = [] 45 | 46 | # These features configure the MDB_IDL_LOGN macro, which determines 47 | # the size of the free and dirty page lists (and thus the amount of memory 48 | # allocated when opening an LMDB environment in read-write mode). 49 | # 50 | # Each feature defines MDB_IDL_LOGN as the value in the name of the feature. 51 | # That means these features are mutually exclusive, and you must not specify 52 | # more than one at the same time (or the crate will fail to compile). 53 | # 54 | # For more information on the motivation for these features (and their effect), 55 | # see https://github.com/mozilla/lmdb/pull/2. 56 | mdb_idl_logn_8 = [] 57 | mdb_idl_logn_9 = [] 58 | mdb_idl_logn_10 = [] 59 | mdb_idl_logn_11 = [] 60 | mdb_idl_logn_12 = [] 61 | mdb_idl_logn_13 = [] 62 | mdb_idl_logn_14 = [] 63 | mdb_idl_logn_15 = [] 64 | mdb_idl_logn_16 = [] 65 | 66 | # Enable the USE_VALGRIND feature. 67 | # 68 | # You have to install the RPM valgrind-devel which contains memcheck.h. 69 | use-valgrind = [] 70 | 71 | # Setting this enables you to use keys longer than 511 bytes. The exact limit 72 | # is computed by LMDB at compile time. You can find the exact value by calling 73 | # Env::max_key_size(). This value varies by architecture. 74 | # 75 | # Example max key sizes: 76 | # - Apple M1 (ARM64): 8126 bytes 77 | # - Apple Intel (AMD64): 1982 bytes 78 | # - Linux Intel (AMD64): 1982 bytes 79 | # 80 | # Setting this also enables you to use values larger than 511 bytes when using 81 | # a Database with the DatabaseFlags::DUP_SORT flag. 82 | # 83 | # This builds LMDB with the -DMDB_MAXKEYSIZE=0 option. 84 | # 85 | # Note: If you are moving database files between architectures then your longest 86 | # stored key must fit within the smallest limit of all architectures used. For 87 | # example, if you are moving databases between Apple M1 and Apple Intel 88 | # computers then you need to keep your keys within the smaller 1982 byte limit. 89 | longer-keys = [] 90 | -------------------------------------------------------------------------------- /lmdb-master3-sys/README.md: -------------------------------------------------------------------------------- 1 | # lmdb-master3-sys 2 | 3 | Rust bindings for liblmdb on the mdb.master3 branch. 4 | -------------------------------------------------------------------------------- /lmdb-master3-sys/bindgen.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | use std::path::PathBuf; 3 | 4 | use bindgen::callbacks::{IntKind, ParseCallbacks}; 5 | 6 | #[derive(Debug)] 7 | struct Callbacks; 8 | 9 | impl ParseCallbacks for Callbacks { 10 | fn process_comment(&self, comment: &str) -> Option { 11 | Some(doxygen_rs::transform(comment)) 12 | } 13 | 14 | fn int_macro(&self, name: &str, _value: i64) -> Option { 15 | match name { 16 | "MDB_SUCCESS" 17 | | "MDB_KEYEXIST" 18 | | "MDB_NOTFOUND" 19 | | "MDB_PAGE_NOTFOUND" 20 | | "MDB_CORRUPTED" 21 | | "MDB_PANIC" 22 | | "MDB_VERSION_MISMATCH" 23 | | "MDB_INVALID" 24 | | "MDB_MAP_FULL" 25 | | "MDB_DBS_FULL" 26 | | "MDB_READERS_FULL" 27 | | "MDB_TLS_FULL" 28 | | "MDB_TXN_FULL" 29 | | "MDB_CURSOR_FULL" 30 | | "MDB_PAGE_FULL" 31 | | "MDB_MAP_RESIZED" 32 | | "MDB_INCOMPATIBLE" 33 | | "MDB_BAD_RSLOT" 34 | | "MDB_BAD_TXN" 35 | | "MDB_BAD_VALSIZE" 36 | | "MDB_BAD_DBI" 37 | | "MDB_LAST_ERRCODE" => Some(IntKind::Int), 38 | "MDB_SIZE_MAX" => Some(IntKind::U64), 39 | "MDB_PROBLEM" | "MDB_BAD_CHECKSUM" | "MDB_CRYPTO_FAIL" | "MDB_ENV_ENCRYPTION" => { 40 | Some(IntKind::Int) 41 | } 42 | _ => Some(IntKind::UInt), 43 | } 44 | } 45 | } 46 | 47 | pub fn generate() { 48 | let mut lmdb = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); 49 | lmdb.push("lmdb"); 50 | lmdb.push("libraries"); 51 | lmdb.push("liblmdb"); 52 | 53 | let mut out_path = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); 54 | out_path.push("src"); 55 | 56 | let bindings = bindgen::Builder::default() 57 | .header(lmdb.join("lmdb.h").to_string_lossy()) 58 | .allowlist_var("^(MDB|mdb)_.*") 59 | .allowlist_type("^(MDB|mdb)_.*") 60 | .allowlist_function("^(MDB|mdb)_.*") 61 | .size_t_is_usize(true) 62 | .ctypes_prefix("::libc") 63 | .blocklist_item("mode_t") 64 | .blocklist_item("mdb_mode_t") 65 | .blocklist_item("mdb_filehandle_t") 66 | .blocklist_item("^__.*") 67 | .parse_callbacks(Box::new(Callbacks {})) 68 | .layout_tests(false) 69 | .prepend_enum_name(false) 70 | .generate() 71 | .expect("Unable to generate bindings"); 72 | 73 | bindings 74 | .write_to_file(out_path.join("bindings.rs")) 75 | .expect("Couldn't write bindings!"); 76 | } 77 | -------------------------------------------------------------------------------- /lmdb-master3-sys/build.rs: -------------------------------------------------------------------------------- 1 | extern crate cc; 2 | 3 | #[cfg(feature = "bindgen")] 4 | extern crate bindgen; 5 | 6 | #[cfg(feature = "bindgen")] 7 | #[path = "bindgen.rs"] 8 | mod generate; 9 | 10 | use std::env; 11 | use std::path::PathBuf; 12 | 13 | #[cfg(all( 14 | feature = "mdb_idl_logn_8", 15 | not(any( 16 | feature = "mdb_idl_logn_9", 17 | feature = "mdb_idl_logn_10", 18 | feature = "mdb_idl_logn_11", 19 | feature = "mdb_idl_logn_12", 20 | feature = "mdb_idl_logn_13", 21 | feature = "mdb_idl_logn_14", 22 | feature = "mdb_idl_logn_15", 23 | feature = "mdb_idl_logn_16" 24 | )) 25 | ))] 26 | const MDB_IDL_LOGN: u8 = 8; 27 | #[cfg(all( 28 | feature = "mdb_idl_logn_9", 29 | not(any( 30 | feature = "mdb_idl_logn_10", 31 | feature = "mdb_idl_logn_11", 32 | feature = "mdb_idl_logn_12", 33 | feature = "mdb_idl_logn_13", 34 | feature = "mdb_idl_logn_14", 35 | feature = "mdb_idl_logn_15", 36 | feature = "mdb_idl_logn_16" 37 | )) 38 | ))] 39 | const MDB_IDL_LOGN: u8 = 9; 40 | #[cfg(all( 41 | feature = "mdb_idl_logn_10", 42 | not(any( 43 | feature = "mdb_idl_logn_11", 44 | feature = "mdb_idl_logn_12", 45 | feature = "mdb_idl_logn_13", 46 | feature = "mdb_idl_logn_14", 47 | feature = "mdb_idl_logn_15", 48 | feature = "mdb_idl_logn_16" 49 | )) 50 | ))] 51 | const MDB_IDL_LOGN: u8 = 10; 52 | #[cfg(all( 53 | feature = "mdb_idl_logn_11", 54 | not(any( 55 | feature = "mdb_idl_logn_12", 56 | feature = "mdb_idl_logn_13", 57 | feature = "mdb_idl_logn_14", 58 | feature = "mdb_idl_logn_15", 59 | feature = "mdb_idl_logn_16" 60 | )) 61 | ))] 62 | const MDB_IDL_LOGN: u8 = 11; 63 | #[cfg(all( 64 | feature = "mdb_idl_logn_12", 65 | not(any( 66 | feature = "mdb_idl_logn_13", 67 | feature = "mdb_idl_logn_14", 68 | feature = "mdb_idl_logn_15", 69 | feature = "mdb_idl_logn_16" 70 | )) 71 | ))] 72 | const MDB_IDL_LOGN: u8 = 12; 73 | #[cfg(all( 74 | feature = "mdb_idl_logn_13", 75 | not(any( 76 | feature = "mdb_idl_logn_14", 77 | feature = "mdb_idl_logn_15", 78 | feature = "mdb_idl_logn_16" 79 | )) 80 | ))] 81 | const MDB_IDL_LOGN: u8 = 13; 82 | #[cfg(all( 83 | feature = "mdb_idl_logn_14", 84 | not(any(feature = "mdb_idl_logn_15", feature = "mdb_idl_logn_16")) 85 | ))] 86 | const MDB_IDL_LOGN: u8 = 14; 87 | #[cfg(all(feature = "mdb_idl_logn_15", not(any(feature = "mdb_idl_logn_16"))))] 88 | const MDB_IDL_LOGN: u8 = 15; 89 | #[cfg(any( 90 | feature = "mdb_idl_logn_16", 91 | not(any( 92 | feature = "mdb_idl_logn_8", 93 | feature = "mdb_idl_logn_9", 94 | feature = "mdb_idl_logn_10", 95 | feature = "mdb_idl_logn_11", 96 | feature = "mdb_idl_logn_12", 97 | feature = "mdb_idl_logn_13", 98 | feature = "mdb_idl_logn_14", 99 | feature = "mdb_idl_logn_15", 100 | feature = "mdb_idl_logn_16", 101 | )) 102 | ))] 103 | const MDB_IDL_LOGN: u8 = 16; 104 | 105 | macro_rules! warn { 106 | ($message:expr) => { 107 | println!("cargo:warning={}", $message); 108 | }; 109 | } 110 | 111 | fn main() { 112 | #[cfg(feature = "bindgen")] 113 | generate::generate(); 114 | 115 | println!("cargo::rerun-if-changed=lmdb"); 116 | 117 | let mut lmdb = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); 118 | lmdb.push("lmdb"); 119 | lmdb.push("libraries"); 120 | lmdb.push("liblmdb"); 121 | 122 | if cfg!(feature = "fuzzer") && cfg!(feature = "fuzzer-no-link") { 123 | warn!("Features `fuzzer` and `fuzzer-no-link` are mutually exclusive."); 124 | warn!("Building with `-fsanitize=fuzzer`."); 125 | } 126 | 127 | let mut builder = cc::Build::new(); 128 | 129 | builder 130 | .define("MDB_IDL_LOGN", Some(MDB_IDL_LOGN.to_string().as_str())) 131 | .file(lmdb.join("mdb.c")) 132 | .file(lmdb.join("midl.c")) 133 | // https://github.com/mozilla/lmdb/blob/b7df2cac50fb41e8bd16aab4cc5fd167be9e032a/libraries/liblmdb/Makefile#L23 134 | .flag_if_supported("-Wno-unused-parameter") 135 | .flag_if_supported("-Wbad-function-cast") 136 | .flag_if_supported("-Wuninitialized"); 137 | 138 | if cfg!(feature = "posix-sem") { 139 | builder.define("MDB_USE_POSIX_SEM", None); 140 | } 141 | 142 | if cfg!(feature = "use-valgrind") { 143 | builder.define("USE_VALGRIND", None); 144 | } 145 | 146 | if cfg!(feature = "asan") { 147 | builder.flag("-fsanitize=address"); 148 | } 149 | 150 | if cfg!(feature = "fuzzer") { 151 | builder.flag("-fsanitize=fuzzer"); 152 | } else if cfg!(feature = "fuzzer-no-link") { 153 | builder.flag("-fsanitize=fuzzer-no-link"); 154 | } 155 | 156 | if cfg!(feature = "longer-keys") { 157 | builder.define("MDB_MAXKEYSIZE", "0"); 158 | } 159 | 160 | if !cfg!(debug_assertions) { 161 | builder.define("NDEBUG", None); 162 | } 163 | 164 | builder.compile("liblmdb.a") 165 | } 166 | -------------------------------------------------------------------------------- /lmdb-master3-sys/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(warnings)] 2 | #![allow(rustdoc::broken_intra_doc_links)] 3 | #![allow(rustdoc::invalid_html_tags)] 4 | #![allow(non_camel_case_types)] 5 | #![allow(clippy::all)] 6 | #![doc(html_root_url = "https://docs.rs/lmdb-master-sys/0.2.5")] 7 | 8 | extern crate libc; 9 | 10 | #[cfg(unix)] 11 | #[allow(non_camel_case_types)] 12 | pub type mdb_mode_t = ::libc::mode_t; 13 | #[cfg(windows)] 14 | #[allow(non_camel_case_types)] 15 | pub type mdb_mode_t = ::libc::c_int; 16 | 17 | #[cfg(unix)] 18 | #[allow(non_camel_case_types)] 19 | pub type mdb_filehandle_t = ::libc::c_int; 20 | #[cfg(windows)] 21 | #[allow(non_camel_case_types)] 22 | pub type mdb_filehandle_t = *mut ::libc::c_void; 23 | 24 | include!("bindings.rs"); 25 | --------------------------------------------------------------------------------