├── .config └── nextest.toml ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ ├── commit.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── .rustfmt.toml ├── .vscode └── settings.json ├── CONTRIBUTING.md ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── _typos.toml ├── benches └── fjall.rs ├── commit.nu ├── compile_examples.mjs ├── examples ├── actix-kv │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ └── src │ │ ├── error.rs │ │ └── main.rs ├── axum-kv │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ └── src │ │ ├── error.rs │ │ └── main.rs ├── basic │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── gc-simple │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── migration │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── partition-rotation │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── permuterm │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── rolling-log │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── secondary-index │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── structured │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── suffix-search │ ├── .run │ ├── Cargo.toml │ ├── README.md │ ├── english_words.txt │ └── src │ │ └── main.rs ├── tokio │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── triplestore │ ├── .gitignore │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── tx-atomic-counter │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── tx-blob-cas │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── tx-mpmc-queue │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── tx-partition-move │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── tx-ssi-atomic-counter │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── tx-ssi-cc │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── tx-ssi-mpmc-queue │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── tx-ssi-partition-move │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs └── unique-index │ ├── .run │ ├── Cargo.toml │ ├── README.md │ └── src │ └── main.rs ├── kawaii.png ├── logo.png ├── renovate.json ├── src ├── background_worker.rs ├── batch │ ├── item.rs │ └── mod.rs ├── compaction │ ├── manager.rs │ ├── mod.rs │ └── worker.rs ├── config.rs ├── drop.rs ├── error.rs ├── file.rs ├── flush │ ├── manager.rs │ ├── mod.rs │ ├── queue.rs │ └── worker.rs ├── gc │ └── mod.rs ├── iter.rs ├── journal │ ├── batch_reader.rs │ ├── error.rs │ ├── manager.rs │ ├── marker.rs │ ├── mod.rs │ ├── reader.rs │ ├── recovery.rs │ └── writer.rs ├── keyspace.rs ├── lib.rs ├── monitor.rs ├── partition │ ├── mod.rs │ ├── name.rs │ ├── options.rs │ └── write_delay.rs ├── path.rs ├── poison_dart.rs ├── recovery.rs ├── snapshot_nonce.rs ├── snapshot_tracker.rs ├── stats.rs ├── tracked_snapshot.rs ├── tx │ ├── conflict_manager.rs │ ├── keyspace.rs │ ├── mod.rs │ ├── oracle.rs │ ├── partition.rs │ ├── read_tx.rs │ ├── write │ │ ├── mod.rs │ │ ├── single_writer.rs │ │ └── ssi.rs │ └── write_tx.rs ├── version.rs └── write_buffer_manager.rs ├── test_fixture ├── v1_keyspace │ ├── journals │ │ └── 2 │ │ │ ├── 0 │ │ │ ├── 1 │ │ │ ├── 2 │ │ │ └── 3 │ ├── partitions │ │ ├── a │ │ │ ├── config │ │ │ ├── levels │ │ │ ├── segments │ │ │ │ └── 0 │ │ │ └── version │ │ ├── b │ │ │ ├── config │ │ │ ├── levels │ │ │ ├── segments │ │ │ │ └── 1 │ │ │ └── version │ │ └── c │ │ │ ├── config │ │ │ ├── levels │ │ │ └── version │ └── version ├── v1_keyspace_corrupt_journal │ ├── journals │ │ └── 2 │ │ │ ├── 0 │ │ │ ├── 1 │ │ │ ├── 2 │ │ │ └── 3 │ ├── partitions │ │ ├── a │ │ │ ├── config │ │ │ ├── levels │ │ │ ├── segments │ │ │ │ └── 0 │ │ │ └── version │ │ ├── b │ │ │ ├── config │ │ │ ├── levels │ │ │ ├── segments │ │ │ │ └── 1 │ │ │ └── version │ │ └── c │ │ │ ├── config │ │ │ ├── levels │ │ │ └── version │ └── version ├── v2_keyspace │ ├── journals │ │ └── 2 │ ├── partitions │ │ ├── default1 │ │ │ ├── config │ │ │ ├── levels │ │ │ ├── manifest │ │ │ └── segments │ │ │ │ └── 0 │ │ └── default2 │ │ │ ├── blobs │ │ │ ├── .vlog │ │ │ ├── segments │ │ │ │ └── 0 │ │ │ └── vlog_manifest │ │ │ ├── config │ │ │ ├── levels │ │ │ ├── manifest │ │ │ └── segments │ │ │ └── 0 │ └── version ├── v2_keyspace_corrupt_journal │ ├── journals │ │ └── 2 │ ├── partitions │ │ ├── default1 │ │ │ ├── config │ │ │ ├── levels │ │ │ ├── manifest │ │ │ └── segments │ │ │ │ └── 0 │ │ └── default2 │ │ │ ├── blobs │ │ │ ├── .vlog │ │ │ ├── segments │ │ │ │ └── 0 │ │ │ └── vlog_manifest │ │ │ ├── config │ │ │ ├── levels │ │ │ ├── manifest │ │ │ └── segments │ │ │ └── 0 │ └── version └── v2_sealed_journal_shenanigans │ ├── journals │ ├── 2 │ ├── 3 │ ├── 0.sealed │ └── 1.sealed │ ├── partitions │ └── default │ │ ├── config │ │ ├── levels │ │ └── manifest │ └── version └── tests ├── batch.rs ├── blob_kv_simple.rs ├── gc_watermark_pull_up.rs ├── journal_large_value.rs ├── keyspace_drop.rs ├── keyspace_recover_empty.rs ├── keyspace_v1_load_fixture.rs ├── keyspace_v2_load_fixture.rs ├── memtable_recover.rs ├── partition_delete.rs ├── partition_iter_lifetime.rs ├── partition_recover.rs ├── partition_recover_from_sealed.rs ├── recover_from_different_folder.rs ├── recovery_ds_store.rs ├── sealed_recovery.rs ├── seqno_recovery.rs ├── tx_ryow.rs ├── v2_cache_api.rs ├── v2_sealed_journal_recovery.rs ├── write_buffer_size.rs └── write_during_read.rs /.config/nextest.toml: -------------------------------------------------------------------------------- 1 | [profile.default] 2 | slow-timeout = "1m" 3 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: fjall-rs 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/commit.yml: -------------------------------------------------------------------------------- 1 | name: Report commit 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | commit: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | - uses: hustcer/setup-nu@main 14 | with: 15 | version: "*" 16 | - run: nu commit.nu 17 | env: 18 | AWS_ACCESS_KEY_ID: ${{ secrets.BENCH_AWS_ACCESS_KEY_ID }} 19 | AWS_SECRET_ACCESS_KEY: ${{ secrets.BENCH_AWS_SECRET_ACCESS_KEY }} 20 | AWS_REGION: eu-central-1 21 | TABLE_NAME: ${{ secrets.BENCH_TABLE_NAME }} 22 | COMMIT: ${{ github.sha }} 23 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | release: 5 | types: created 6 | 7 | jobs: 8 | publish: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | - uses: katyo/publish-crates@v2 13 | with: 14 | registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }} 15 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - "**" 10 | 11 | env: 12 | CARGO_TERM_COLOR: always 13 | 14 | jobs: 15 | examples: 16 | timeout-minutes: 20 17 | strategy: 18 | matrix: 19 | rust_version: 20 | - stable # We cannot run with MSRV because some deps may require a higher MSRV 21 | os: 22 | - ubuntu-latest 23 | - windows-latest 24 | - macos-latest 25 | runs-on: ${{ matrix.os }} 26 | steps: 27 | - name: Checkout 28 | uses: actions/checkout@v4 29 | - uses: dtolnay/rust-toolchain@stable 30 | with: 31 | toolchain: ${{ matrix.rust_version }} 32 | - name: Set up cargo cache 33 | uses: Swatinem/rust-cache@v2 34 | with: 35 | prefix-key: ${{ runner.os }}-cargo 36 | workspaces: > 37 | . -> target 38 | examples/actix-kv -> target 39 | examples/axum-kv -> target 40 | examples/basic -> target 41 | examples/gc-simple -> target 42 | examples/partition-rotation -> target 43 | examples/permuterm -> target 44 | examples/rolling-log -> target 45 | examples/secondary-index -> target 46 | examples/structured -> target 47 | examples/suffix-search -> target 48 | examples/tokio -> target 49 | examples/triplestore -> target 50 | examples/tx-atomic-counter -> target 51 | examples/tx-blob-cas 52 | examples/tx-mpmc-queue -> target 53 | examples/tx-partition-move -> target 54 | examples/tx-ssi-atomic-counter -> target 55 | examples/tx-ssi-cc -> target 56 | examples/tx-ssi-mpmc-queue -> target 57 | examples/tx-ssi-partition-move -> target 58 | examples/unique-index -> target 59 | - name: Build & test examples 60 | run: node compile_examples.mjs 61 | test: 62 | timeout-minutes: 20 63 | strategy: 64 | matrix: 65 | rust_version: 66 | - stable 67 | - "1.76.0" # MSRV 68 | os: 69 | - ubuntu-latest 70 | - windows-latest 71 | - macos-latest 72 | runs-on: ${{ matrix.os }} 73 | steps: 74 | - name: Checkout 75 | uses: actions/checkout@v4 76 | - uses: dtolnay/rust-toolchain@stable 77 | with: 78 | toolchain: ${{ matrix.rust_version }} 79 | - name: Add rustfmt 80 | run: rustup component add rustfmt 81 | - name: Add clippy 82 | run: rustup component add clippy 83 | - name: Set up cargo cache 84 | uses: Swatinem/rust-cache@v2 85 | with: 86 | prefix-key: ${{ runner.os }}-cargo 87 | workspaces: > 88 | . -> target 89 | - name: Install cargo-all-features 90 | run: cargo install cargo-all-features 91 | - uses: taiki-e/install-action@nextest 92 | - name: Format 93 | run: cargo fmt --all -- --check 94 | - name: Clippy 95 | run: cargo clippy 96 | - name: Build permutations 97 | run: cargo build-all-features 98 | - name: Run whitebox tests 99 | run: cargo test --features __internal_whitebox -- whitebox_ --test-threads=1 100 | - name: Run tests 101 | run: cargo nextest run --features lz4,miniz,single_writer_tx 102 | - name: Run SSI tests 103 | run: cargo nextest run --no-default-features --features ssi_tx tx_ssi_ 104 | - name: Run doc tests 105 | run: cargo test --doc 106 | - name: Run SSI doc tests 107 | run: cargo test --no-default-features --features ssi_tx --doc 108 | cross: 109 | timeout-minutes: 15 110 | name: cross 111 | strategy: 112 | matrix: 113 | target: 114 | - aarch64-unknown-linux-gnu 115 | - aarch64-unknown-linux-musl 116 | - i686-unknown-linux-gnu 117 | - powerpc64-unknown-linux-gnu 118 | - riscv64gc-unknown-linux-gnu 119 | # - aarch64-linux-android 120 | # - i686-linux-android 121 | # - x86_64-linux-android 122 | #- mips64-unknown-linux-gnuabi64 123 | #- x86_64-apple-darwin 124 | #- aarch64-apple-darwin 125 | runs-on: ubuntu-latest 126 | steps: 127 | - uses: actions/checkout@v4 128 | - name: cross test 129 | run: | 130 | cargo install cross 131 | cross test --target ${{ matrix.target }} 132 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | debug/ 4 | target/ 5 | 6 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 7 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 8 | Cargo.lock 9 | 10 | # These are backup files generated by rustfmt 11 | **/*.rs.bk 12 | 13 | # MSVC Windows builds of rustc generate these, which store debugging information 14 | *.pdb 15 | 16 | # Added by cargo 17 | 18 | /target 19 | .lsm.data 20 | .fjall_data 21 | .data 22 | 23 | /old_* 24 | .test 25 | 26 | segment_history.jsonl 27 | -------------------------------------------------------------------------------- /.rustfmt.toml: -------------------------------------------------------------------------------- 1 | reorder_imports = true 2 | # group_imports = "StdExternalCrate" 3 | # imports_granularity = "crate" 4 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "rust-analyzer.showUnlinkedFileNotification": false 3 | } -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## License 4 | 5 | By contributing to this project, you agree that your contributions will be licensed under the project's license (MIT OR Apache-2.0). 6 | 7 | Thank you for your contribution! 8 | 9 | ## Looking for issues? 10 | 11 | https://github.com/fjall-rs/fjall/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 12 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fjall" 3 | description = "LSM-based key-value storage engine" 4 | license = "MIT OR Apache-2.0" 5 | version = "2.11.0" 6 | edition = "2021" 7 | rust-version = "1.76.0" 8 | readme = "README.md" 9 | include = ["src/**/*", "LICENSE-APACHE", "LICENSE-MIT", "README.md"] 10 | repository = "https://github.com/fjall-rs/fjall" 11 | homepage = "https://github.com/fjall-rs/fjall" 12 | keywords = ["database", "key-value", "lsm", "rocksdb", "leveldb"] 13 | categories = ["data-structures", "database-implementations", "algorithms"] 14 | 15 | [lib] 16 | name = "fjall" 17 | path = "src/lib.rs" 18 | 19 | [features] 20 | default = ["single_writer_tx", "lz4"] 21 | lz4 = ["lsm-tree/lz4"] 22 | miniz = ["lsm-tree/miniz"] 23 | bytes = ["lsm-tree/bytes"] 24 | single_writer_tx = [] 25 | ssi_tx = [] 26 | __internal_whitebox = [] 27 | 28 | [dependencies] 29 | byteorder = "1.5.0" 30 | byteview = "0.6.1" 31 | lsm-tree = { version = "~2.10", default-features = false, features = [] } 32 | log = "0.4.21" 33 | std-semaphore = "0.1.0" 34 | tempfile = "3.10.1" 35 | path-absolutize = "3.1.1" 36 | dashmap = "6.0.1" 37 | xxhash-rust = { version = "0.8.12", features = ["xxh3"] } 38 | 39 | [dev-dependencies] 40 | criterion = { version = "0.5.1", features = ["html_reports"] } 41 | nanoid = "0.4.0" 42 | test-log = "0.2.16" 43 | rand = "0.9.0" 44 | 45 | # half 2.5.0 has MSRV 1.81 46 | half = "=2.4.0" 47 | 48 | [package.metadata.cargo-all-features] 49 | denylist = ["__internal_whitebox"] 50 | skip_feature_sets = [["ssi_tx", "single_writer_tx"]] 51 | 52 | [[bench]] 53 | name = "lsmt" 54 | harness = false 55 | path = "benches/fjall.rs" 56 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 fjall-rs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /_typos.toml: -------------------------------------------------------------------------------- 1 | [files] 2 | extend-exclude = ["examples/**/*"] 3 | -------------------------------------------------------------------------------- /benches/fjall.rs: -------------------------------------------------------------------------------- 1 | use criterion::{criterion_group, criterion_main, Criterion}; 2 | 3 | fn batch_write(c: &mut Criterion) { 4 | let dir = tempfile::tempdir().unwrap(); 5 | 6 | let keyspace = fjall::Config::new(&dir).open().unwrap(); 7 | let items = keyspace 8 | .open_partition("default", Default::default()) 9 | .unwrap(); 10 | 11 | c.bench_function("Batch commit", |b| { 12 | b.iter(|| { 13 | let mut batch = keyspace.batch(); 14 | for item in 'a'..='z' { 15 | let item = item.to_string(); 16 | batch.insert(&items, &item, &item); 17 | } 18 | batch.commit().unwrap(); 19 | }); 20 | }); 21 | } 22 | 23 | criterion_group!(benches, batch_write); 24 | criterion_main!(benches); 25 | -------------------------------------------------------------------------------- /commit.nu: -------------------------------------------------------------------------------- 1 | let machines = [ 2 | # Fly.io performance 3 | # "fly.performance.1x", 4 | # "fly.performance.2x", 5 | "fly.performance.4x", 6 | # "fly.performance.8x", 7 | # "fly.performance.16x", 8 | 9 | # EC2 T2 10 | # "aws.ec2.t2.nano", 11 | # "aws.ec2.t2.micro", 12 | # "aws.ec2.t2.small", 13 | # "aws.ec2.t2.medium", 14 | # "aws.ec2.t2.large", 15 | # "aws.ec2.t2.xlarge", 16 | # "aws.ec2.t2.2xlarge", 17 | 18 | # EC2 T3 19 | # "aws.ec2.t3.nano", 20 | # "aws.ec2.t3.micro", 21 | # "aws.ec2.t3.small", 22 | "aws.ec2.t3.medium", 23 | # "aws.ec2.t3.large", 24 | # "aws.ec2.t3.xlarge", 25 | # "aws.ec2.t3.2xlarge", 26 | 27 | # EC2 T3a 28 | # "aws.ec2.t3a.nano", 29 | # "aws.ec2.t3a.micro", 30 | # "aws.ec2.t3a.small", 31 | # "aws.ec2.t3a.medium", 32 | # "aws.ec2.t3a.large", 33 | # "aws.ec2.t3a.xlarge", 34 | # "aws.ec2.t3a.2xlarge", 35 | 36 | # EC2 T4g 37 | # "aws.ec2.t4g.nano", 38 | # "aws.ec2.t4g.micro", 39 | # "aws.ec2.t4g.small", 40 | # "aws.ec2.t4g.medium", 41 | # "aws.ec2.t4g.large", 42 | # "aws.ec2.t4g.xlarge", 43 | # "aws.ec2.t4g.2xlarge", 44 | 45 | # EC2 M4 46 | # "aws.ec2.m4.large", 47 | ] 48 | 49 | let table = $env.TABLE_NAME 50 | let commit = $env.COMMIT 51 | 52 | print $"Queuing ($commit)" 53 | 54 | for machine in $machines { 55 | let q_pk = $"q#($machine)" 56 | 57 | print $"Adding queue item for ($machine)" 58 | let item = { 59 | pk: { S: $q_pk }, 60 | sk: { S: $commit }, 61 | version: { S: "2" }, 62 | } 63 | aws dynamodb put-item --table-name $table --item ($item | to json) 64 | } 65 | -------------------------------------------------------------------------------- /compile_examples.mjs: -------------------------------------------------------------------------------- 1 | import { spawn } from "node:child_process"; 2 | import { existsSync } from "node:fs"; 3 | import { readdir } from "node:fs/promises"; 4 | import { resolve } from "node:path"; 5 | 6 | const examplesFolder = "examples"; 7 | 8 | for (const exampleName of await readdir(examplesFolder)) { 9 | 10 | const folder = resolve(examplesFolder, exampleName); 11 | 12 | { 13 | console.error(`Testing ${exampleName}`); 14 | 15 | const proc = spawn("cargo test", { 16 | cwd: folder, 17 | shell: true, 18 | }); 19 | 20 | proc.stdout.on("data", buf => console.log(String(buf))); 21 | proc.stderr.on("data", buf => console.error(String(buf))); 22 | 23 | await new Promise((resolve, _) => { 24 | proc.on("exit", () => { 25 | if (proc.exitCode > 0) { 26 | console.error(`${exampleName} FAILED`); 27 | process.exit(1); 28 | } 29 | else { 30 | resolve(); 31 | } 32 | }) 33 | }); 34 | } 35 | 36 | if (existsSync(resolve(folder, ".run"))) { 37 | console.error(`Running ${exampleName}`); 38 | 39 | const proc = spawn("cargo run", { 40 | cwd: folder, 41 | shell: true, 42 | }); 43 | 44 | proc.stdout.on("data", buf => console.log(String(buf))); 45 | proc.stderr.on("data", buf => console.error(String(buf))); 46 | 47 | await new Promise((resolve, _) => { 48 | proc.on("exit", () => { 49 | if (proc.exitCode > 0) { 50 | console.error(`${exampleName} FAILED`); 51 | process.exit(1); 52 | } 53 | else { 54 | resolve(); 55 | } 56 | }) 57 | }); 58 | } 59 | 60 | console.error(`${exampleName} OK`); 61 | } 62 | -------------------------------------------------------------------------------- /examples/actix-kv/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | .data 3 | .test 4 | -------------------------------------------------------------------------------- /examples/actix-kv/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "actix-kv" 3 | version = "0.0.1" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | actix-web = "4.9.0" 10 | log = { version = "0.4", features = ["release_max_level_info"] } 11 | env_logger = "0.10.0" 12 | fjall = { path = "../../", features = ["bytes"] } 13 | serde = { version = "1.0.193", features = ["derive"] } 14 | serde_json = "1.0.99" 15 | 16 | [dev-dependencies] 17 | tempfile = "3.8.1" 18 | -------------------------------------------------------------------------------- /examples/actix-kv/README.md: -------------------------------------------------------------------------------- 1 | # actix-kv 2 | 3 | This example uses `fjall`, `actix_web` and `serde_json` to provide a simple key-value store with a JSON REST API. 4 | 5 | ## REST API 6 | 7 | ### `POST /batch` 8 | 9 | Upserts and/or removes items in an atomic batch. 10 | 11 | ```json 12 | { 13 | "upsert": [ 14 | ["key", "value"] 15 | ], 16 | "remove": ["another_key"] 17 | } 18 | ``` 19 | 20 | ### `PUT /{key}` 21 | 22 | Upserts an item. 23 | 24 | ```json 25 | { 26 | "item": SOME_JSON_VALUE 27 | } 28 | ``` 29 | 30 | ### `GET /{key}` 31 | 32 | Returns a single item. 33 | 34 | ### `DELETE /{key}` 35 | 36 | Deletes an item. 37 | -------------------------------------------------------------------------------- /examples/actix-kv/src/error.rs: -------------------------------------------------------------------------------- 1 | use actix_web::{HttpResponse, ResponseError}; 2 | 3 | #[derive(Debug)] 4 | pub struct MyError(fjall::Error); 5 | 6 | impl std::fmt::Display for MyError { 7 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 8 | write!(f, "{:?}", self.0) 9 | } 10 | } 11 | 12 | impl ResponseError for MyError { 13 | fn error_response(&self) -> HttpResponse { 14 | HttpResponse::InternalServerError().body("Internal Server Error") 15 | } 16 | } 17 | 18 | impl From for MyError { 19 | fn from(value: fjall::Error) -> Self { 20 | Self(value) 21 | } 22 | } 23 | 24 | pub type RouteResult = Result; 25 | -------------------------------------------------------------------------------- /examples/axum-kv/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | .data 3 | .test 4 | -------------------------------------------------------------------------------- /examples/axum-kv/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "axum-kv" 3 | version = "0.0.1" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | axum = "0.7" 8 | env_logger = "0.10" 9 | fjall = { path = "../../" } 10 | log = { version = "0.4", features = ["release_max_level_info"] } 11 | mime = "0.3" 12 | serde = { version = "1", features = ["derive"] } 13 | serde_json = "1" 14 | thiserror = "1" 15 | tokio = { version = "1", features = ["rt-multi-thread", "macros", "net"] } 16 | 17 | [dev-dependencies] 18 | http-body-util = "0.1" 19 | tempfile = "3" 20 | tower = "0.5" 21 | -------------------------------------------------------------------------------- /examples/axum-kv/README.md: -------------------------------------------------------------------------------- 1 | # axum-kv 2 | 3 | This example uses `fjall`, `axum` and `serde_json` to provide a simple key-value store with a JSON REST API. 4 | 5 | ## REST API 6 | 7 | ### `POST /batch` 8 | 9 | Upserts and/or removes items in an atomic batch. 10 | 11 | ```json 12 | { 13 | "upsert": [ 14 | ["key", "value"] 15 | ], 16 | "remove": ["another_key"] 17 | } 18 | ``` 19 | 20 | ### `PUT /{key}` 21 | 22 | Upserts an item. 23 | 24 | ```json 25 | { 26 | "item": SOME_JSON_VALUE 27 | } 28 | ``` 29 | 30 | ### `GET /{key}` 31 | 32 | Returns a single item. 33 | 34 | ### `DELETE /{key}` 35 | 36 | Deletes an item. 37 | -------------------------------------------------------------------------------- /examples/axum-kv/src/error.rs: -------------------------------------------------------------------------------- 1 | use axum::response::{IntoResponse, Response}; 2 | use thiserror::Error; 3 | 4 | #[derive(Debug, Error)] 5 | pub enum Error { 6 | #[error(transparent)] 7 | Io(#[from] std::io::Error), 8 | 9 | #[error(transparent)] 10 | Axum(#[from] axum::Error), 11 | 12 | #[error(transparent)] 13 | Fjall(#[from] fjall::Error), 14 | } 15 | 16 | impl IntoResponse for Error { 17 | fn into_response(self) -> Response { 18 | self.to_string().into_response() 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /examples/basic/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/basic/.run -------------------------------------------------------------------------------- /examples/basic/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fjall-basic" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../" } 10 | -------------------------------------------------------------------------------- /examples/basic/README.md: -------------------------------------------------------------------------------- 1 | # basic 2 | 3 | This example just opens a keyspace. Used as starting template. 4 | -------------------------------------------------------------------------------- /examples/basic/src/main.rs: -------------------------------------------------------------------------------- 1 | fn main() -> fjall::Result<()> { 2 | let keyspace = fjall::Config::default().open()?; 3 | let items = keyspace.open_partition("items", Default::default())?; 4 | 5 | assert_eq!(0, items.len()?); 6 | 7 | println!("OK"); 8 | 9 | Ok(()) 10 | } 11 | -------------------------------------------------------------------------------- /examples/gc-simple/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/gc-simple/.run -------------------------------------------------------------------------------- /examples/gc-simple/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "gc-simple" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../" } 10 | -------------------------------------------------------------------------------- /examples/gc-simple/README.md: -------------------------------------------------------------------------------- 1 | # gc-simple 2 | 3 | This example demonstrates using the garbage collection API. 4 | -------------------------------------------------------------------------------- /examples/gc-simple/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, GarbageCollection, KvSeparationOptions, PartitionCreateOptions}; 2 | use std::time::Instant; 3 | 4 | const BLOB_SIZE: usize = 10_000; 5 | 6 | fn main() -> fjall::Result<()> { 7 | let keyspace = Config::default().temporary(true).open()?; 8 | let blobs = keyspace.open_partition( 9 | "blobs", 10 | PartitionCreateOptions::default().with_kv_separation(KvSeparationOptions::default()), 11 | )?; 12 | 13 | for _ in 0..10 { 14 | blobs.insert("a", "a".repeat(BLOB_SIZE))?; 15 | 16 | // NOTE: This is just used to force the data into the value log 17 | blobs.rotate_memtable_and_wait()?; 18 | } 19 | 20 | eprintln!("Running GC for partition {:?}", blobs.name); 21 | let start = Instant::now(); 22 | 23 | let report = blobs.gc_scan()?; 24 | assert_eq!(10.0, report.space_amp()); 25 | assert_eq!(0.9, report.stale_ratio()); 26 | assert_eq!(9, report.stale_blobs); 27 | assert_eq!(9 * BLOB_SIZE as u64, report.stale_bytes); 28 | assert_eq!(9, report.stale_segment_count); 29 | 30 | let freed_bytes = blobs.gc_with_space_amp_target(1.0)?; 31 | 32 | eprintln!("GC done in {:?}, freed {freed_bytes}B", start.elapsed()); 33 | assert!( 34 | // TODO: needs to be fixed in value-log 35 | /* NOTE: freed_bytes is the amount of bytes freed on disk, not uncompressed data */ 36 | freed_bytes > 0 37 | ); 38 | 39 | let report = blobs.gc_scan()?; 40 | assert_eq!(1.0, report.space_amp()); 41 | assert_eq!(0.0, report.stale_ratio()); 42 | assert_eq!(0, report.stale_blobs); 43 | assert_eq!(0, report.stale_bytes); 44 | assert_eq!(0, report.stale_segment_count); 45 | 46 | Ok(()) 47 | } 48 | -------------------------------------------------------------------------------- /examples/migration/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/migration/.run -------------------------------------------------------------------------------- /examples/migration/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "migration" 3 | version = "0.0.1" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../" } 10 | rmp-serde = "1.1.2" 11 | serde = { version = "1.0.193", features = ["derive"] } 12 | -------------------------------------------------------------------------------- /examples/migration/README.md: -------------------------------------------------------------------------------- 1 | # migration 2 | 3 | This example uses `fjall`, `serde` and `rmp-serde` to store and load persistent, structured data, then migrate it from one schema version to the next using `ingest`. 4 | -------------------------------------------------------------------------------- /examples/migration/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::Config; 2 | use serde::{Deserialize, Serialize}; 3 | 4 | #[derive(Debug, Deserialize, Serialize)] 5 | pub struct PlanetV1 { 6 | /// ID 7 | id: String, 8 | 9 | /// Name 10 | name: String, 11 | } 12 | 13 | #[derive(Debug, Deserialize, Serialize)] 14 | pub struct PlanetV2 { 15 | /// ID 16 | id: String, 17 | 18 | /// Name 19 | name: String, 20 | 21 | /// Mass 22 | mass: Option, 23 | 24 | /// Radius 25 | radius: Option, 26 | } 27 | 28 | impl From for PlanetV2 { 29 | fn from(value: PlanetV1) -> Self { 30 | Self { 31 | id: value.id, 32 | name: value.name, 33 | mass: None, 34 | radius: None, 35 | } 36 | } 37 | } 38 | 39 | #[derive(Debug, Deserialize, Serialize)] 40 | pub enum Planet { 41 | V1(PlanetV1), 42 | V2(PlanetV2), 43 | } 44 | 45 | impl std::fmt::Display for Planet { 46 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 47 | write!(f, "{:?}", self) 48 | } 49 | } 50 | 51 | fn main() -> fjall::Result<()> { 52 | let path = std::path::Path::new(".fjall_data"); 53 | 54 | let keyspace = Config::new(path).temporary(true).open()?; 55 | let planets_v1 = keyspace.open_partition("v1_planets", Default::default())?; 56 | 57 | for (id, name) in [ 58 | ("p:earth", "Earth"), 59 | ("p:jupiter", "Jupiter"), 60 | ("p:neptune", "Mars"), 61 | ("p:neptune", "Mercury"), 62 | ("p:neptune", "Neptune"), 63 | ("p:saturn", "Saturn"), 64 | ("p:uranus", "Uranus"), 65 | ("p:venus", "Venus"), 66 | ] { 67 | planets_v1.insert( 68 | id, 69 | rmp_serde::to_vec(&Planet::V1(PlanetV1 { 70 | id: id.into(), 71 | name: name.into(), 72 | })) 73 | .unwrap(), 74 | )?; 75 | } 76 | 77 | eprintln!("--- v1 ---"); 78 | for kv in planets_v1.iter() { 79 | let (_, v) = kv?; 80 | let planet = rmp_serde::from_slice::(&v).unwrap(); 81 | eprintln!("{planet:?}"); 82 | } 83 | 84 | // Do migration from V1 -> V2 85 | let planets_v2 = keyspace.open_partition("v2_planets", Default::default())?; 86 | 87 | let stream = planets_v1.iter().map(|kv| { 88 | let (k, v) = kv.unwrap(); 89 | let planet = rmp_serde::from_slice::(&v).unwrap(); 90 | 91 | if let Planet::V1(planet) = planet { 92 | let v2: PlanetV2 = planet.into(); 93 | let v2 = Planet::V2(v2); 94 | (k, rmp_serde::to_vec(&v2).unwrap()) 95 | } else { 96 | unreachable!("v1 does not contain other versions"); 97 | } 98 | }); 99 | planets_v2.ingest(stream)?; 100 | 101 | eprintln!("--- v2 ---"); 102 | for kv in planets_v2.iter() { 103 | let (_, v) = kv?; 104 | let planet = rmp_serde::from_slice::(&v).unwrap(); 105 | eprintln!("{planet:?}"); 106 | } 107 | 108 | assert_eq!(planets_v1.len()?, planets_v2.len()?); 109 | 110 | // Delete old data 111 | keyspace.delete_partition(planets_v1)?; 112 | 113 | Ok(()) 114 | } 115 | -------------------------------------------------------------------------------- /examples/partition-rotation/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "partition-rotation" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | env_logger = "0.11.5" 8 | fjall = { path = "../.." } 9 | random-string = "1.1.0" 10 | scru128 = "3.0.2" 11 | -------------------------------------------------------------------------------- /examples/partition-rotation/README.md: -------------------------------------------------------------------------------- 1 | # partition-rotation 2 | 3 | This example demonstrates Elasticsearch-style index rollovers by splitting a data set (e.g. timeseries) into partitions. This makes it easy to, for example, drop data that is older than a certain point in time. 4 | -------------------------------------------------------------------------------- /examples/partition-rotation/src/main.rs: -------------------------------------------------------------------------------- 1 | fn get_split_name(no: usize) -> String { 2 | format!("split_{no}") 3 | } 4 | 5 | const SPLITS: usize = 1_000; 6 | const ITEMS_PER_SPLIT: usize = 1_000; 7 | 8 | fn main() -> fjall::Result<()> { 9 | env_logger::Builder::from_default_env().init(); 10 | 11 | let keyspace = fjall::Config::default().open()?; 12 | 13 | let start = std::time::Instant::now(); 14 | 15 | for no in 0..SPLITS { 16 | let split_name = get_split_name(no); 17 | eprintln!("writing into {split_name:?}"); 18 | let split = keyspace.open_partition(&split_name, Default::default())?; 19 | 20 | let before = std::time::Instant::now(); 21 | for _ in 0..ITEMS_PER_SPLIT { 22 | split.insert( 23 | scru128::new_string(), 24 | random_string::generate(50, random_string::charsets::ALPHANUMERIC), 25 | )?; 26 | } 27 | 28 | keyspace.persist(fjall::PersistMode::SyncData)?; 29 | 30 | // NOTE: Flush memtable because partition becomes immutable 31 | // This relaxes the journal GC, making everything a bit faster 32 | split.rotate_memtable()?; 33 | 34 | eprintln!( 35 | "writing {ITEMS_PER_SPLIT} took {}ms, journal size: {} MiB", 36 | before.elapsed().as_millis(), 37 | keyspace.journal_disk_space() / 1_024 / 1_024, 38 | ); 39 | } 40 | 41 | let elapsed = start.elapsed(); 42 | 43 | eprintln!( 44 | "written {} ({} MiB) in {}s, {}µs per item", 45 | SPLITS * ITEMS_PER_SPLIT, 46 | keyspace.disk_space() / 1_024 / 1_024, 47 | elapsed.as_secs_f32(), 48 | elapsed.as_micros() / (SPLITS as u128 * ITEMS_PER_SPLIT as u128) 49 | ); 50 | 51 | assert_eq!(SPLITS, keyspace.partition_count()); 52 | 53 | Ok(()) 54 | } 55 | -------------------------------------------------------------------------------- /examples/permuterm/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/permuterm/.run -------------------------------------------------------------------------------- /examples/permuterm/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fjall-permuterm" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../" } 10 | -------------------------------------------------------------------------------- /examples/permuterm/README.md: -------------------------------------------------------------------------------- 1 | # permuterm 2 | 3 | This example demonstrates using `fjall` and Permuterm indexes for wildcard full text queries. 4 | -------------------------------------------------------------------------------- /examples/permuterm/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, Result}; 2 | 3 | struct Permutermifier { 4 | term: String, 5 | count: usize, 6 | len: usize, 7 | } 8 | 9 | impl Permutermifier { 10 | pub fn new(term: &str) -> Self { 11 | Self { 12 | term: format!("{term}$"), 13 | len: term.len(), 14 | count: term.len() + 1, 15 | } 16 | } 17 | 18 | fn rotate(&mut self) { 19 | let head = &self.term[0..1]; 20 | let rest = &self.term[1..]; 21 | self.term = format!("{rest}{head}"); 22 | } 23 | } 24 | 25 | impl Iterator for Permutermifier { 26 | type Item = String; 27 | 28 | fn next(&mut self) -> Option { 29 | if self.count == 0 { 30 | return None; 31 | } 32 | 33 | self.count -= 1; 34 | 35 | if self.count < self.len { 36 | self.rotate(); 37 | } 38 | 39 | Some(self.term.clone()) 40 | } 41 | } 42 | 43 | fn permuterm(term: &str) -> impl Iterator { 44 | Permutermifier::new(term) 45 | } 46 | 47 | const WORDS: &[&str] = &[ 48 | "daughter", 49 | "laughter", 50 | "water", 51 | "blackwater", 52 | "waterway", 53 | "waterside", 54 | "waterfall", 55 | "waterfront", 56 | "arbiter", 57 | "test", 58 | "something", 59 | "mississippi", 60 | "watak", 61 | "teriyaki", 62 | "systrafoss", 63 | "skogafoss", 64 | "skiptarfoss", 65 | "seljalandsfoss", 66 | "gullfoss", 67 | "svartifoss", 68 | "hengifoss", 69 | "fagrifoss", 70 | "haifoss", 71 | "fossa", 72 | "foster", 73 | "fossil", 74 | ]; 75 | 76 | fn main() -> Result<()> { 77 | let keyspace = Config::default().temporary(true).open()?; 78 | 79 | let db = keyspace.open_partition("db", Default::default())?; 80 | 81 | for word in WORDS { 82 | for term in permuterm(word) { 83 | db.insert(format!("{term}#{word}"), *word).unwrap(); 84 | } 85 | } 86 | 87 | println!("-- Suffix query --"); 88 | { 89 | let query = "ter"; 90 | 91 | // Permuterm suffix queries are performed using: TERM$* 92 | for kv in db.prefix(format!("{query}$")) { 93 | let (_, v) = kv?; 94 | let v = std::str::from_utf8(&v).unwrap(); 95 | println!("*{query} => {v:?}"); 96 | } 97 | } 98 | println!(); 99 | 100 | println!("-- Exact query --"); 101 | { 102 | for query in ["water", "blackwater"] { 103 | // Permuterm exact queries are performed using: TERM$ 104 | for kv in db.prefix(format!("{query}$#")) { 105 | let (_, v) = kv?; 106 | let v = std::str::from_utf8(&v).unwrap(); 107 | println!("{query:?} => {v:?}"); 108 | } 109 | println!(); 110 | } 111 | } 112 | 113 | println!("-- Prefix query --"); 114 | { 115 | let query = "water"; 116 | 117 | // Permuterm prefix queries are performed using: $TERM* 118 | for kv in db.prefix(format!("${query}")) { 119 | let (_, v) = kv?; 120 | let v = std::str::from_utf8(&v).unwrap(); 121 | println!("{query}* => {v:?}"); 122 | } 123 | } 124 | println!(); 125 | 126 | println!("-- Partial query --"); 127 | { 128 | for query in ["ter", "fos"] { 129 | // Permuterm partial queries are performed using: TERM* 130 | for kv in db.prefix(format!("{query}")) { 131 | let (_, v) = kv?; 132 | let v = std::str::from_utf8(&v).unwrap(); 133 | println!("*{query}* => {v:?}"); 134 | } 135 | println!(); 136 | } 137 | } 138 | 139 | println!("-- Between query --"); 140 | { 141 | for (q1, q2) in [("s", "foss"), ("h", "foss")] { 142 | // Permuterm between queries are performed using: B$A* 143 | for kv in db.prefix(format!("{q2}${q1}")) { 144 | let (_, v) = kv?; 145 | let v = std::str::from_utf8(&v).unwrap(); 146 | println!("{q1}*{q2} => {v:?}"); 147 | } 148 | println!(); 149 | } 150 | } 151 | 152 | Ok(()) 153 | } 154 | -------------------------------------------------------------------------------- /examples/rolling-log/.run: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /examples/rolling-log/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rolling-log" 3 | version = "0.0.1" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../" } 10 | -------------------------------------------------------------------------------- /examples/rolling-log/README.md: -------------------------------------------------------------------------------- 1 | # rolling-log 2 | 3 | This example demonstrates the FIFO compaction to store a rolling log of ephemeral, ordered data. 4 | -------------------------------------------------------------------------------- /examples/rolling-log/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, PartitionCreateOptions}; 2 | use std::path::Path; 3 | 4 | const LIMIT: u64 = 10_000_000; 5 | 6 | fn main() -> fjall::Result<()> { 7 | let path = Path::new(".fjall_data"); 8 | 9 | let keyspace = Config::new(path) 10 | .temporary(true) 11 | .max_write_buffer_size(4_000_000) 12 | .open()?; 13 | 14 | let log = keyspace.open_partition( 15 | "log", 16 | PartitionCreateOptions::default().compaction_strategy(fjall::compaction::Strategy::Fifo( 17 | fjall::compaction::Fifo::new(LIMIT, None), 18 | )), 19 | )?; 20 | 21 | for x in 0u64..2_500_000 { 22 | log.insert(x.to_be_bytes(), x.to_be_bytes())?; 23 | 24 | if x % 100_000 == 0 { 25 | let (min_key, _) = log.first_key_value()?.unwrap(); 26 | let (max_key, _) = log.last_key_value()?.unwrap(); 27 | 28 | let mut buf = [0; 8]; 29 | buf.copy_from_slice(&min_key[0..8]); 30 | let min_key = u64::from_be_bytes(buf); 31 | 32 | buf.copy_from_slice(&max_key[0..8]); 33 | let max_key = u64::from_be_bytes(buf); 34 | 35 | let disk_space = log.disk_space(); 36 | 37 | println!( 38 | "key range: [{min_key}, {max_key}] - disk space used: {} MiB - # segments: {}", 39 | disk_space / 1_024 / 1_024, 40 | log.segment_count(), 41 | ); 42 | 43 | assert!(disk_space < (LIMIT as f64 * 1.5) as u64); 44 | } 45 | } 46 | 47 | Ok(()) 48 | } 49 | -------------------------------------------------------------------------------- /examples/secondary-index/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/secondary-index/.run -------------------------------------------------------------------------------- /examples/secondary-index/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "secondary-index" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | fjall = { path = "../../" } 8 | format-bytes = "0.3.0" 9 | nanoid = "0.4.0" 10 | -------------------------------------------------------------------------------- /examples/secondary-index/README.md: -------------------------------------------------------------------------------- 1 | # secondary-index 2 | 3 | This example uses a secondary partition as a secondary index to provide range search over a non-unique attribute. 4 | -------------------------------------------------------------------------------- /examples/secondary-index/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Batch, BlockCache, Config, PartitionHandle}; 2 | use format_bytes::format_bytes; 3 | use nanoid::nanoid; 4 | use std::path::Path; 5 | 6 | fn create_item( 7 | batch: &mut Batch, 8 | table: &PartitionHandle, 9 | index: &PartitionHandle, 10 | name: &str, 11 | year: u64, 12 | ) -> fjall::Result<()> { 13 | let id = nanoid!(); 14 | batch.insert(table, &id, format!("{name} [{year}]")); 15 | 16 | let ts_bytes = year.to_be_bytes(); 17 | let key = format_bytes!(b"{}\0{}", ts_bytes, id.as_bytes()); 18 | 19 | batch.insert(index, &key, ""); 20 | 21 | Ok(()) 22 | } 23 | 24 | fn main() -> fjall::Result<()> { 25 | let path = Path::new(".fjall_data"); 26 | 27 | let keyspace = Config::new(path).temporary(true).open()?; 28 | let items = keyspace.open_partition("items", Default::default())?; 29 | let sec = keyspace.open_partition("sec_idx", Default::default())?; 30 | 31 | let mut batch = keyspace.batch(); 32 | create_item(&mut batch, &items, &sec, "Remain in Light", 1_980)?; 33 | create_item(&mut batch, &items, &sec, "Seventeen Seconds", 1_980)?; 34 | create_item(&mut batch, &items, &sec, "Power, Corruption & Lies", 1_983)?; 35 | create_item(&mut batch, &items, &sec, "Hounds of Love", 1_985)?; 36 | create_item(&mut batch, &items, &sec, "Black Celebration", 1_986)?; 37 | create_item(&mut batch, &items, &sec, "Disintegration", 1_989)?; 38 | create_item(&mut batch, &items, &sec, "Violator", 1_990)?; 39 | create_item(&mut batch, &items, &sec, "Wish", 1_991)?; 40 | create_item(&mut batch, &items, &sec, "Loveless", 1_991)?; 41 | create_item(&mut batch, &items, &sec, "Dummy", 1_994)?; 42 | create_item(&mut batch, &items, &sec, "When The Pawn...", 1_999)?; 43 | create_item(&mut batch, &items, &sec, "Kid A", 2_000)?; 44 | create_item(&mut batch, &items, &sec, "Have You In My Wilderness", 2_015)?; 45 | 46 | batch.commit()?; 47 | keyspace.persist(fjall::PersistMode::SyncAll)?; 48 | 49 | // Get items from 1990 to 2000 (exclusive) 50 | let lo = 1_990_u64; 51 | let hi = 1_999_u64; 52 | 53 | println!("Searching for [{lo} - {hi}]"); 54 | 55 | let mut found_count = 0; 56 | 57 | for kv in sec.range(lo.to_be_bytes()..(hi + 1).to_be_bytes()) { 58 | let (k, _) = kv?; 59 | 60 | // Get ID 61 | let primary_key = &k[std::mem::size_of::() + 1..]; 62 | 63 | // Get from primary index 64 | let item = items.get(primary_key)?.unwrap(); 65 | 66 | println!("found: {}", std::str::from_utf8(&item).unwrap()); 67 | 68 | found_count += 1; 69 | } 70 | 71 | assert_eq!(5, found_count); 72 | 73 | Ok(()) 74 | } 75 | -------------------------------------------------------------------------------- /examples/structured/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/structured/.run -------------------------------------------------------------------------------- /examples/structured/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "structured" 3 | version = "0.0.1" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../" } 10 | rmp-serde = "1.1.2" 11 | serde = { version = "1.0.193", features = ["derive"] } 12 | -------------------------------------------------------------------------------- /examples/structured/README.md: -------------------------------------------------------------------------------- 1 | # structured 2 | 3 | This example uses `fjall`, `serde` and `rmp-serde` to store and load persistent, structured data. 4 | -------------------------------------------------------------------------------- /examples/structured/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, Keyspace, PartitionHandle, UserKey, UserValue}; 2 | use serde::{Deserialize, Serialize}; 3 | 4 | #[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] 5 | pub struct Song { 6 | /// ID 7 | #[serde(skip)] 8 | id: String, 9 | 10 | /// Artist name 11 | artist: String, 12 | 13 | /// Title 14 | title: String, 15 | 16 | /// Release year 17 | release_year: u16, 18 | } 19 | 20 | impl From<&Song> for Vec { 21 | fn from(val: &Song) -> Self { 22 | rmp_serde::to_vec(&val).expect("should serialize") 23 | } 24 | } 25 | 26 | impl From<(UserKey, UserValue)> for Song { 27 | fn from((key, value): (UserKey, UserValue)) -> Self { 28 | let key = std::str::from_utf8(&key).unwrap(); 29 | let mut item: Song = rmp_serde::from_slice(&value).expect("should deserialize"); 30 | key.clone_into(&mut item.id); 31 | item 32 | } 33 | } 34 | 35 | impl std::fmt::Display for Song { 36 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 37 | write!( 38 | f, 39 | "{} - {} ({})", 40 | self.artist, self.title, self.release_year 41 | ) 42 | } 43 | } 44 | 45 | pub struct SongDatabase { 46 | #[allow(unused)] 47 | keyspace: Keyspace, 48 | 49 | db: PartitionHandle, 50 | } 51 | 52 | impl SongDatabase { 53 | pub fn get(&self, key: &str) -> fjall::Result> { 54 | let Some(item) = self.db.get(key)? else { 55 | return Ok(None); 56 | }; 57 | 58 | let mut song: Song = rmp_serde::from_slice(&item).expect("should deserialize"); 59 | key.clone_into(&mut song.id); 60 | 61 | Ok(Some(song)) 62 | } 63 | 64 | pub fn insert(&self, song: &Song) -> fjall::Result<()> { 65 | let serialized: Vec = song.into(); 66 | self.db.insert(&song.id, serialized) 67 | } 68 | 69 | pub fn iter(&self) -> impl Iterator> + '_ { 70 | self.db.iter().map(|item| item.map(Song::from)) 71 | } 72 | 73 | pub fn len(&self) -> fjall::Result { 74 | self.db.len() 75 | } 76 | } 77 | 78 | fn main() -> fjall::Result<()> { 79 | let path = std::path::Path::new(".fjall_data"); 80 | 81 | let items = vec![ 82 | Song { 83 | id: "clairo:amoeba".to_owned(), 84 | release_year: 2021, 85 | artist: "Clairo".to_owned(), 86 | title: "Amoeba".to_owned(), 87 | }, 88 | Song { 89 | id: "clairo:zinnias".to_owned(), 90 | release_year: 2021, 91 | artist: "Clairo".to_owned(), 92 | title: "Zinnias".to_owned(), 93 | }, 94 | Song { 95 | id: "fazerdaze:break".to_owned(), 96 | release_year: 2022, 97 | artist: "Fazerdaze".to_owned(), 98 | title: "Break!".to_owned(), 99 | }, 100 | Song { 101 | id: "fazerdaze:winter".to_owned(), 102 | release_year: 2022, 103 | artist: "Fazerdaze".to_owned(), 104 | title: "Winter".to_owned(), 105 | }, 106 | ]; 107 | 108 | { 109 | let keyspace = Config::new(path).open()?; 110 | let db = keyspace.open_partition("songs", Default::default())?; 111 | 112 | let song_db = SongDatabase { 113 | keyspace: keyspace.clone(), 114 | db, 115 | }; 116 | 117 | for item_to_insert in &items { 118 | if let Some(item) = song_db.get(&item_to_insert.id)? { 119 | println!("Found: {item}"); 120 | assert_eq!(&item, item_to_insert); 121 | } else { 122 | println!("Inserting..."); 123 | song_db.insert(item_to_insert)?; 124 | } 125 | } 126 | keyspace.persist(fjall::PersistMode::SyncAll)?; 127 | 128 | assert_eq!(items.len(), song_db.len()?); 129 | } 130 | 131 | // Reload from disk 132 | { 133 | println!("\nReloading..."); 134 | 135 | let keyspace = Config::new(path).temporary(true).open()?; 136 | let db = keyspace.open_partition("songs", Default::default())?; 137 | 138 | let song_db = SongDatabase { 139 | keyspace: keyspace.clone(), 140 | db, 141 | }; 142 | 143 | println!("\nListing all items:"); 144 | 145 | for (idx, song) in song_db.iter().enumerate() { 146 | let song = song?; 147 | println!("[{idx}] {song}"); 148 | } 149 | 150 | assert_eq!(items.len(), song_db.len()?); 151 | } 152 | 153 | Ok(()) 154 | } 155 | -------------------------------------------------------------------------------- /examples/suffix-search/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/suffix-search/.run -------------------------------------------------------------------------------- /examples/suffix-search/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "suffix-search" 3 | version = "0.0.1" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../" } 10 | -------------------------------------------------------------------------------- /examples/suffix-search/README.md: -------------------------------------------------------------------------------- 1 | # suffix-search 2 | 3 | This example uses a secondary partition to provide efficient suffix search. 4 | -------------------------------------------------------------------------------- /examples/suffix-search/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::{BlockCache, Config, PersistMode}; 2 | use std::{ 3 | fs::File, 4 | io::{BufRead, BufReader}, 5 | path::Path, 6 | time::Instant, 7 | }; 8 | 9 | fn main() -> fjall::Result<()> { 10 | let path = Path::new(".fjall_data"); 11 | 12 | let keyspace = Config::new(path).temporary(true).open()?; 13 | let items = keyspace.open_partition("items", Default::default())?; 14 | 15 | // To search suffixes of keys, we store a secondary index that stores the reversed key 16 | // which will allow a .prefix() search over that key, resulting in a suffix search. 17 | let items_rev = keyspace.open_partition("items_rev", Default::default())?; 18 | 19 | if items.is_empty()? { 20 | println!("Ingesting test data"); 21 | 22 | let line_reader = BufReader::new(File::open("english_words.txt")?); 23 | 24 | for (idx, line) in line_reader.lines().enumerate() { 25 | let line = line?; 26 | 27 | // We use a write batch to keep both partitions synchronized 28 | let mut batch = keyspace.batch(); 29 | batch.insert(&items, &line, &line); 30 | batch.insert(&items_rev, line.chars().rev().collect::(), line); 31 | batch.commit()?; 32 | 33 | if idx % 50_000 == 0 { 34 | println!("Loaded {idx} words"); 35 | } 36 | } 37 | } 38 | 39 | keyspace.persist(PersistMode::SyncAll)?; 40 | 41 | let suffix = "west"; 42 | let test_runs = 5; 43 | 44 | let count = items.len()?; 45 | 46 | for i in 0..test_runs { 47 | let before = Instant::now(); 48 | let mut found_count = 0; 49 | 50 | if i == 0 { 51 | println!("\n[SLOW] Scanning all items for suffix {suffix:?}:"); 52 | } 53 | 54 | for kv in items_rev.iter() { 55 | let (_, value) = kv?; 56 | 57 | if value.ends_with(suffix.as_bytes()) { 58 | if i == 0 { 59 | println!(" -> {}", std::str::from_utf8(&value).unwrap()); 60 | } 61 | 62 | found_count += 1; 63 | } 64 | } 65 | 66 | println!( 67 | "Found {found_count:?} in {count:?} words in {:?}", 68 | before.elapsed(), 69 | ); 70 | 71 | assert_eq!(40, found_count); 72 | } 73 | println!("==============================================="); 74 | 75 | for i in 0..test_runs { 76 | let before = Instant::now(); 77 | let mut found_count = 0; 78 | 79 | if i == 0 { 80 | println!("\n[FAST] Finding all items by suffix {suffix:?}:"); 81 | } 82 | 83 | // Uses prefix, so generally faster than table scan 84 | // `------------------v 85 | for kv in items_rev.prefix(suffix.chars().rev().collect::()) { 86 | let (_, value) = kv?; 87 | 88 | if i == 0 { 89 | println!(" -> {}", std::str::from_utf8(&value).unwrap()); 90 | } 91 | 92 | found_count += 1; 93 | } 94 | 95 | println!( 96 | "Found {found_count:?} in {count:?} words in {:?}", 97 | before.elapsed(), 98 | ); 99 | 100 | assert_eq!(40, found_count); 101 | } 102 | println!("==============================================="); 103 | 104 | Ok(()) 105 | } 106 | -------------------------------------------------------------------------------- /examples/tokio/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/tokio/.run -------------------------------------------------------------------------------- /examples/tokio/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fjall-tokio" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../" } 10 | tokio = { version = "1.37.0", features = ["rt-multi-thread", "macros"] } 11 | -------------------------------------------------------------------------------- /examples/tokio/README.md: -------------------------------------------------------------------------------- 1 | # tokio 2 | 3 | This example demonstrates using `fjall` inside a Tokio runtime by using `spawn_blocking`. 4 | -------------------------------------------------------------------------------- /examples/tokio/src/main.rs: -------------------------------------------------------------------------------- 1 | use tokio::task::spawn_blocking; 2 | 3 | #[tokio::main] 4 | async fn main() -> fjall::Result<()> { 5 | let keyspace = fjall::Config::default().open()?; 6 | let items = keyspace.open_partition("items", Default::default())?; 7 | 8 | { 9 | let items = items.clone(); 10 | spawn_blocking(move || items.insert("hello", "world")) 11 | .await 12 | .expect("join failed")?; 13 | } 14 | 15 | let item = { 16 | let items = items.clone(); 17 | spawn_blocking(move || items.get("hello")) 18 | .await 19 | .expect("join failed")? 20 | }; 21 | 22 | let item = item.expect("should exist"); 23 | 24 | assert_eq!(b"world", &*item); 25 | 26 | println!("OK"); 27 | 28 | Ok(()) 29 | } 30 | -------------------------------------------------------------------------------- /examples/triplestore/.gitignore: -------------------------------------------------------------------------------- 1 | .data 2 | -------------------------------------------------------------------------------- /examples/triplestore/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/triplestore/.run -------------------------------------------------------------------------------- /examples/triplestore/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "triplestore" 3 | version = "0.0.1" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../" } 10 | serde_json = "1.0.99" 11 | serde = "1.0.193" 12 | -------------------------------------------------------------------------------- /examples/triplestore/README.md: -------------------------------------------------------------------------------- 1 | # triplestore 2 | 3 | This example uses `fjall`, `serde` and `serde_json` to provide a simple triplestore. 4 | -------------------------------------------------------------------------------- /examples/triplestore/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, Keyspace, PartitionHandle}; 2 | use serde_json::Value; 3 | use std::path::Path; 4 | 5 | struct Triplestore { 6 | #[allow(dead_code)] 7 | keyspace: Keyspace, 8 | 9 | subjects: PartitionHandle, 10 | verbs: PartitionHandle, 11 | } 12 | 13 | impl Triplestore { 14 | pub fn new>(path: P) -> fjall::Result { 15 | let keyspace = Config::new(path).open()?; 16 | let subjects = keyspace.open_partition("subjects", Default::default())?; 17 | let verbs = keyspace.open_partition("verbs", Default::default())?; 18 | 19 | Ok(Self { 20 | keyspace, 21 | subjects, 22 | verbs, 23 | }) 24 | } 25 | 26 | pub fn add_subject(&self, key: &str, data: &Value) -> fjall::Result<()> { 27 | self.subjects 28 | .insert(key, serde_json::to_string(data).expect("should serialize")) 29 | } 30 | 31 | pub fn add_triple(&self, from: &str, verb: &str, to: &str, data: &Value) -> fjall::Result<()> { 32 | self.verbs.insert( 33 | format!("{from}\0{verb}\0{to}"), 34 | serde_json::to_string(data).expect("should serialize"), 35 | ) 36 | } 37 | 38 | pub fn contains_subject(&self, key: &str) -> fjall::Result { 39 | self.subjects.contains_key(key) 40 | } 41 | 42 | pub fn get_triple( 43 | &self, 44 | subject: &str, 45 | verb: &str, 46 | object: &str, 47 | ) -> fjall::Result> { 48 | let Some(bytes) = self.verbs.get(format!("{subject}\0{verb}\0{object}"))? else { 49 | return Ok(None); 50 | }; 51 | let value = std::str::from_utf8(&bytes).expect("should be utf-8"); 52 | let value = serde_json::from_str(value).expect("should be json"); 53 | Ok(Some(value)) 54 | } 55 | 56 | pub fn out( 57 | &self, 58 | subject: &str, 59 | verb: &str, 60 | ) -> fjall::Result> { 61 | let mut result = vec![]; 62 | 63 | for kv in self.verbs.prefix(format!("{subject}\0{verb}\0")) { 64 | let (key, value) = kv?; 65 | 66 | let key = std::str::from_utf8(&key).expect("should be utf-8"); 67 | let mut splits = key.split('\0'); 68 | let s = splits.next().unwrap().to_string(); 69 | let v = splits.next().unwrap().to_string(); 70 | let o = splits.next().unwrap().to_string(); 71 | 72 | let value = std::str::from_utf8(&value).expect("should be utf-8"); 73 | let value: Value = serde_json::from_str(&value).expect("should be json"); 74 | 75 | result.push((s, v, o, value)); 76 | } 77 | 78 | Ok(result) 79 | } 80 | } 81 | 82 | fn main() -> fjall::Result<()> { 83 | let store = Triplestore::new(".data")?; 84 | 85 | if !store.contains_subject("person-1")? { 86 | store.add_subject( 87 | "person-1", 88 | &serde_json::json!({ 89 | "name": "Peter" 90 | }), 91 | )?; 92 | } 93 | if !store.contains_subject("person-2")? { 94 | store.add_subject( 95 | "person-2", 96 | &serde_json::json!({ 97 | "name": "Peter 2" 98 | }), 99 | )?; 100 | } 101 | if !store.contains_subject("person-3")? { 102 | store.add_subject( 103 | "person-3", 104 | &serde_json::json!({ 105 | "name": "Peter 3" 106 | }), 107 | )?; 108 | } 109 | if !store.contains_subject("person-4")? { 110 | store.add_subject( 111 | "person-4", 112 | &serde_json::json!({ 113 | "name": "Peter 4" 114 | }), 115 | )?; 116 | } 117 | 118 | for person in &["person-2", "person-4"] { 119 | if (store.get_triple("person-1", "knows", person)?).is_none() { 120 | store.add_triple( 121 | "person-1", 122 | "knows", 123 | person, 124 | &serde_json::json!({ 125 | "since": 2014 126 | }), 127 | )?; 128 | } 129 | } 130 | store.keyspace.persist(fjall::PersistMode::SyncAll)?; 131 | 132 | let mut count = 0; 133 | 134 | println!("Listing all person-1->knows-> relations:"); 135 | for (_, _, o, _) in store.out("person-1", "knows")? { 136 | println!("person-1 knows {o}!"); 137 | 138 | count += 1; 139 | } 140 | 141 | assert_eq!(2, count); 142 | 143 | Ok(()) 144 | } 145 | -------------------------------------------------------------------------------- /examples/tx-atomic-counter/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/tx-atomic-counter/.run -------------------------------------------------------------------------------- /examples/tx-atomic-counter/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tx-atomic-counter" 3 | version = "0.0.1" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../" } 10 | rand = "0.8.5" 11 | -------------------------------------------------------------------------------- /examples/tx-atomic-counter/README.md: -------------------------------------------------------------------------------- 1 | # tx-atomic-counter 2 | 3 | This example demonstrates using transactions for atomic updates. 4 | -------------------------------------------------------------------------------- /examples/tx-atomic-counter/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, PersistMode}; 2 | use std::path::Path; 3 | 4 | const LIMIT: u64 = 100; 5 | 6 | fn main() -> fjall::Result<()> { 7 | let path = Path::new(".fjall_data"); 8 | 9 | let keyspace = Config::new(path).temporary(true).open_transactional()?; 10 | let counters = keyspace.open_partition("counters", Default::default())?; 11 | 12 | counters.insert("c1", 0_u64.to_be_bytes())?; 13 | 14 | let workers = (0_u8..4) 15 | .map(|idx| { 16 | let keyspace = keyspace.clone(); 17 | let counters = counters.clone(); 18 | 19 | std::thread::spawn(move || { 20 | use rand::Rng; 21 | 22 | let mut rng = rand::thread_rng(); 23 | 24 | loop { 25 | let mut write_tx = keyspace.write_tx(); 26 | 27 | let item = write_tx.get(&counters, "c1")?.unwrap(); 28 | 29 | let mut bytes = [0; 8]; 30 | bytes.copy_from_slice(&item); 31 | let prev = u64::from_be_bytes(bytes); 32 | 33 | if prev >= LIMIT { 34 | return Ok::<_, fjall::Error>(()); 35 | } 36 | 37 | let next = prev + 1; 38 | 39 | write_tx.insert(&counters, "c1", next.to_be_bytes()); 40 | write_tx.commit()?; 41 | 42 | println!("worker {idx} incremented to {next}"); 43 | 44 | let ms = rng.gen_range(10..400); 45 | std::thread::sleep(std::time::Duration::from_millis(ms)); 46 | } 47 | }) 48 | }) 49 | .collect::>(); 50 | 51 | for worker in workers { 52 | worker.join().unwrap()?; 53 | } 54 | 55 | assert_eq!(&*counters.get("c1").unwrap().unwrap(), LIMIT.to_be_bytes()); 56 | 57 | Ok(()) 58 | } 59 | -------------------------------------------------------------------------------- /examples/tx-blob-cas/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/tx-blob-cas/.run -------------------------------------------------------------------------------- /examples/tx-blob-cas/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tx-blob-cas" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | env_logger = "0.11.3" 8 | fjall = { path = "../../" } 9 | format-bytes = "0.3.0" 10 | log = "0.4.21" 11 | sha2 = "0.10.8" 12 | -------------------------------------------------------------------------------- /examples/tx-blob-cas/README.md: -------------------------------------------------------------------------------- 1 | # tx-blob-cas 2 | 3 | This example demonstrates using transactions and key-value separation to provide a content-addressable store for large blobs. 4 | -------------------------------------------------------------------------------- /examples/tx-mpmc-queue/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/tx-mpmc-queue/.run -------------------------------------------------------------------------------- /examples/tx-mpmc-queue/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tx-mpmc-queue" 3 | version = "0.0.1" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../" } 10 | rand = "0.8.5" 11 | scru128 = "3.0.2" 12 | -------------------------------------------------------------------------------- /examples/tx-mpmc-queue/README.md: -------------------------------------------------------------------------------- 1 | # tx-mpmc-queue 2 | 3 | This example demonstrates implementing a FIFO-MPMC queue using transactions. 4 | -------------------------------------------------------------------------------- /examples/tx-mpmc-queue/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, PersistMode}; 2 | use std::path::Path; 3 | use std::sync::{ 4 | atomic::{AtomicUsize, Ordering::Relaxed}, 5 | Arc, 6 | }; 7 | 8 | const PRODUCER_COUNT: usize = 4; 9 | const PRODUCING_COUNT: usize = 100; 10 | 11 | const EXPECTED_COUNT: usize = PRODUCER_COUNT * PRODUCING_COUNT; 12 | 13 | fn main() -> fjall::Result<()> { 14 | let path = Path::new(".fjall_data"); 15 | 16 | let keyspace = Config::new(path).temporary(true).open_transactional()?; 17 | let tasks = keyspace.open_partition("tasks", Default::default())?; 18 | 19 | let counter = Arc::new(AtomicUsize::default()); 20 | 21 | let producers = (0..PRODUCER_COUNT) 22 | .map(|idx| { 23 | let keyspace = keyspace.clone(); 24 | let tasks = tasks.clone(); 25 | 26 | std::thread::spawn(move || { 27 | use rand::Rng; 28 | 29 | let mut rng = rand::thread_rng(); 30 | 31 | for _ in 0..PRODUCING_COUNT { 32 | let task_id = scru128::new_string(); 33 | 34 | tasks.insert(&task_id, &task_id)?; 35 | 36 | println!("producer {idx} created task {task_id}"); 37 | 38 | let ms = rng.gen_range(10..100); 39 | std::thread::sleep(std::time::Duration::from_millis(ms)); 40 | } 41 | 42 | Ok::<_, fjall::Error>(()) 43 | }) 44 | }) 45 | .collect::>(); 46 | 47 | let consumers = (0..4) 48 | .map(|idx| { 49 | let keyspace = keyspace.clone(); 50 | let tasks = tasks.clone(); 51 | let counter = counter.clone(); 52 | 53 | std::thread::spawn(move || { 54 | use rand::Rng; 55 | 56 | let mut rng = rand::thread_rng(); 57 | 58 | loop { 59 | let mut tx = keyspace.write_tx(); 60 | 61 | // TODO: NOTE: 62 | // Tombstones will add up over time, making first KV slower 63 | // Something like SingleDelete https://github.com/facebook/rocksdb/wiki/Single-Delete 64 | // would be good for this type of workload 65 | if let Some((key, _)) = tx.first_key_value(&tasks)? { 66 | let task_id = std::str::from_utf8(&key).unwrap().to_owned(); 67 | 68 | tx.remove(&tasks, key); 69 | 70 | tx.commit()?; 71 | 72 | println!("consumer {idx} completed task {task_id}"); 73 | 74 | counter.fetch_add(1, Relaxed); 75 | 76 | let ms = rng.gen_range(50..200); 77 | std::thread::sleep(std::time::Duration::from_millis(ms)); 78 | } else if counter.load(Relaxed) == EXPECTED_COUNT { 79 | return Ok::<_, fjall::Error>(()); 80 | } 81 | } 82 | }) 83 | }) 84 | .collect::>(); 85 | 86 | for t in producers { 87 | t.join().unwrap()?; 88 | } 89 | 90 | for t in consumers { 91 | t.join().unwrap()?; 92 | } 93 | 94 | assert_eq!(EXPECTED_COUNT, counter.load(Relaxed)); 95 | 96 | Ok(()) 97 | } 98 | -------------------------------------------------------------------------------- /examples/tx-partition-move/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/tx-partition-move/.run -------------------------------------------------------------------------------- /examples/tx-partition-move/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tx-partition-move" 3 | version = "0.0.1" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../" } 10 | rand = "0.8.5" 11 | scru128 = "3.0.2" 12 | -------------------------------------------------------------------------------- /examples/tx-partition-move/README.md: -------------------------------------------------------------------------------- 1 | # tx-partition-move 2 | 3 | This example demonstrates atomically moving items between partitions using transactions. 4 | -------------------------------------------------------------------------------- /examples/tx-partition-move/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, PersistMode}; 2 | use std::path::Path; 3 | 4 | const ITEM_COUNT: u64 = 200; 5 | 6 | fn main() -> fjall::Result<()> { 7 | let path = Path::new(".fjall_data"); 8 | 9 | let keyspace = Config::new(path).temporary(true).open_transactional()?; 10 | let src = keyspace.open_partition("src", Default::default())?; 11 | let dst = keyspace.open_partition("dst", Default::default())?; 12 | 13 | for _ in 0..ITEM_COUNT { 14 | src.insert(scru128::new_string(), "")?; 15 | } 16 | 17 | let movers = (0..4) 18 | .map(|idx| { 19 | let keyspace = keyspace.clone(); 20 | let src = src.clone(); 21 | let dst = dst.clone(); 22 | 23 | std::thread::spawn(move || { 24 | use rand::Rng; 25 | 26 | let mut rng = rand::thread_rng(); 27 | 28 | loop { 29 | let mut tx = keyspace.write_tx(); 30 | 31 | // TODO: NOTE: 32 | // Tombstones will add up over time, making first KV slower 33 | // Something like SingleDelete https://github.com/facebook/rocksdb/wiki/Single-Delete 34 | // would be good for this type of workload 35 | if let Some((key, value)) = tx.first_key_value(&src)? { 36 | let task_id = std::str::from_utf8(&key).unwrap().to_owned(); 37 | 38 | tx.remove(&src, key.clone()); 39 | tx.insert(&dst, key, value); 40 | 41 | tx.commit()?; 42 | 43 | println!("consumer {idx} moved {task_id}"); 44 | 45 | let ms = rng.gen_range(10..100); 46 | std::thread::sleep(std::time::Duration::from_millis(ms)); 47 | } else { 48 | return Ok::<_, fjall::Error>(()); 49 | } 50 | } 51 | }) 52 | }) 53 | .collect::>(); 54 | 55 | for t in movers { 56 | t.join().unwrap()?; 57 | } 58 | 59 | assert_eq!(ITEM_COUNT, keyspace.read_tx().len(&dst)? as u64); 60 | assert!(keyspace.read_tx().is_empty(&src)?); 61 | 62 | Ok(()) 63 | } 64 | -------------------------------------------------------------------------------- /examples/tx-ssi-atomic-counter/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/tx-ssi-atomic-counter/.run -------------------------------------------------------------------------------- /examples/tx-ssi-atomic-counter/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tx-atomic-counter" 3 | version = "0.0.1" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../", default-features = false, features = [ 10 | "lz4", 11 | "ssi_tx", 12 | ] } 13 | rand = "0.8.5" 14 | -------------------------------------------------------------------------------- /examples/tx-ssi-atomic-counter/README.md: -------------------------------------------------------------------------------- 1 | # tx-ssi-atomic-counter 2 | 3 | This example demonstrates using transactions for atomic updates. 4 | -------------------------------------------------------------------------------- /examples/tx-ssi-atomic-counter/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, PersistMode}; 2 | use std::path::Path; 3 | 4 | const LIMIT: u64 = 100; 5 | 6 | fn main() -> fjall::Result<()> { 7 | let path = Path::new(".fjall_data"); 8 | 9 | let keyspace = Config::new(path).temporary(true).open_transactional()?; 10 | let counters = keyspace.open_partition("counters", Default::default())?; 11 | 12 | counters.insert("c1", 0_u64.to_be_bytes())?; 13 | 14 | let workers = (0_u8..4) 15 | .map(|idx| { 16 | let keyspace = keyspace.clone(); 17 | let counters = counters.clone(); 18 | 19 | std::thread::spawn(move || { 20 | use rand::Rng; 21 | 22 | let mut rng = rand::thread_rng(); 23 | 24 | loop { 25 | let mut write_tx = keyspace.write_tx().unwrap(); 26 | 27 | let item = write_tx.get(&counters, "c1")?.unwrap(); 28 | 29 | let mut bytes = [0; 8]; 30 | bytes.copy_from_slice(&item); 31 | let prev = u64::from_be_bytes(bytes); 32 | 33 | if prev >= LIMIT { 34 | return Ok::<_, fjall::Error>(()); 35 | } 36 | 37 | let next = prev + 1; 38 | 39 | write_tx.insert(&counters, "c1", next.to_be_bytes()); 40 | write_tx.commit()?.ok(); 41 | 42 | println!("worker {idx} incremented to {next}"); 43 | 44 | let ms = rng.gen_range(10..400); 45 | std::thread::sleep(std::time::Duration::from_millis(ms)); 46 | } 47 | }) 48 | }) 49 | .collect::>(); 50 | 51 | for worker in workers { 52 | worker.join().unwrap()?; 53 | } 54 | 55 | assert_eq!(&*counters.get("c1").unwrap().unwrap(), LIMIT.to_be_bytes()); 56 | 57 | Ok(()) 58 | } 59 | -------------------------------------------------------------------------------- /examples/tx-ssi-cc/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/tx-ssi-cc/.run -------------------------------------------------------------------------------- /examples/tx-ssi-cc/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tx-ssi-cc" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../", default-features = false, features = [ 10 | "lz4", 11 | "ssi_tx", 12 | ] } 13 | -------------------------------------------------------------------------------- /examples/tx-ssi-cc/README.md: -------------------------------------------------------------------------------- 1 | # tx-ssi-cc 2 | 3 | This example demonstrates concurrent transactions using SSI (serializable snapshot isolation). 4 | -------------------------------------------------------------------------------- /examples/tx-ssi-cc/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::time::{Duration, Instant}; 2 | 3 | fn main() -> fjall::Result<()> { 4 | let keyspace = fjall::Config::default() 5 | .temporary(true) 6 | .open_transactional()?; 7 | let items = keyspace.open_partition("items", Default::default())?; 8 | 9 | let start = Instant::now(); 10 | 11 | let t1 = { 12 | let keyspace = keyspace.clone(); 13 | let items = items.clone(); 14 | 15 | std::thread::spawn(move || { 16 | let mut wtx = keyspace.write_tx().unwrap(); 17 | println!("Started tx1"); 18 | std::thread::sleep(Duration::from_secs(3)); 19 | wtx.insert(&items, "a", "a"); 20 | wtx.commit() 21 | }) 22 | }; 23 | 24 | let t2 = { 25 | let keyspace = keyspace.clone(); 26 | let items = items.clone(); 27 | 28 | std::thread::spawn(move || { 29 | let mut wtx = keyspace.write_tx().unwrap(); 30 | println!("Started tx2"); 31 | std::thread::sleep(Duration::from_secs(3)); 32 | wtx.insert(&items, "b", "b"); 33 | wtx.commit() 34 | }) 35 | }; 36 | 37 | t1.join() 38 | .expect("should join")? 39 | .expect("tx should not fail"); 40 | 41 | t2.join() 42 | .expect("should join")? 43 | .expect("tx should not fail"); 44 | 45 | // NOTE: We would expect a single writer tx implementation to finish in 46 | // ~6 seconds 47 | println!("Done in {:?}, items.len={}", start.elapsed(), { 48 | let rtx = keyspace.read_tx(); 49 | rtx.len(&items)? 50 | }); 51 | 52 | Ok(()) 53 | } 54 | -------------------------------------------------------------------------------- /examples/tx-ssi-mpmc-queue/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/tx-ssi-mpmc-queue/.run -------------------------------------------------------------------------------- /examples/tx-ssi-mpmc-queue/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tx-mpmc-queue" 3 | version = "0.0.1" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../", default-features = false, features = [ 10 | "lz4", 11 | "ssi_tx", 12 | ] } 13 | rand = "0.8.5" 14 | scru128 = "3.0.2" 15 | -------------------------------------------------------------------------------- /examples/tx-ssi-mpmc-queue/README.md: -------------------------------------------------------------------------------- 1 | # tx-ssi-mpmc-queue 2 | 3 | This example demonstrates implementing a FIFO-MPMC queue using transactions. 4 | -------------------------------------------------------------------------------- /examples/tx-ssi-mpmc-queue/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, PersistMode}; 2 | use std::path::Path; 3 | use std::sync::{ 4 | atomic::{AtomicUsize, Ordering::Relaxed}, 5 | Arc, 6 | }; 7 | 8 | const PRODUCER_COUNT: usize = 4; 9 | const PRODUCING_COUNT: usize = 100; 10 | 11 | const EXPECTED_COUNT: usize = PRODUCER_COUNT * PRODUCING_COUNT; 12 | 13 | fn main() -> fjall::Result<()> { 14 | let path = Path::new(".fjall_data"); 15 | 16 | let keyspace = Config::new(path).temporary(true).open_transactional()?; 17 | let tasks = keyspace.open_partition("tasks", Default::default())?; 18 | 19 | let counter = Arc::new(AtomicUsize::default()); 20 | 21 | let producers = (0..PRODUCER_COUNT) 22 | .map(|idx| { 23 | let keyspace = keyspace.clone(); 24 | let tasks = tasks.clone(); 25 | 26 | std::thread::spawn(move || { 27 | use rand::Rng; 28 | 29 | let mut rng = rand::thread_rng(); 30 | 31 | for _ in 0..PRODUCING_COUNT { 32 | let task_id = scru128::new_string(); 33 | 34 | tasks.insert(&task_id, &task_id)?; 35 | 36 | println!("producer {idx} created task {task_id}"); 37 | 38 | let ms = rng.gen_range(10..100); 39 | std::thread::sleep(std::time::Duration::from_millis(ms)); 40 | } 41 | 42 | println!("producer {idx} done"); 43 | 44 | Ok::<_, fjall::Error>(()) 45 | }) 46 | }) 47 | .collect::>(); 48 | 49 | let consumers = (0..4) 50 | .map(|idx| { 51 | let keyspace = keyspace.clone(); 52 | let tasks = tasks.clone(); 53 | let counter = counter.clone(); 54 | 55 | std::thread::spawn(move || { 56 | use rand::Rng; 57 | 58 | let mut rng = rand::thread_rng(); 59 | 60 | loop { 61 | let mut tx = keyspace.write_tx().unwrap(); 62 | 63 | // TODO: NOTE: 64 | // Tombstones will add up over time, making first KV slower 65 | // Something like SingleDelete https://github.com/facebook/rocksdb/wiki/Single-Delete 66 | // would be good for this type of workload 67 | if let Some((key, _)) = tx.first_key_value(&tasks)? { 68 | let task_id = std::str::from_utf8(&key).unwrap().to_owned(); 69 | 70 | tx.remove(&tasks, key); 71 | 72 | if tx.commit()?.is_ok() { 73 | counter.fetch_add(1, Relaxed); 74 | } 75 | 76 | println!("consumer {idx} completed task {task_id}"); 77 | 78 | let ms = rng.gen_range(50..200); 79 | std::thread::sleep(std::time::Duration::from_millis(ms)); 80 | } else if counter.load(Relaxed) == EXPECTED_COUNT { 81 | return Ok::<_, fjall::Error>(()); 82 | } 83 | } 84 | }) 85 | }) 86 | .collect::>(); 87 | 88 | for t in producers { 89 | t.join().unwrap()?; 90 | } 91 | 92 | for t in consumers { 93 | t.join().unwrap()?; 94 | } 95 | 96 | assert_eq!(EXPECTED_COUNT, counter.load(Relaxed)); 97 | 98 | Ok(()) 99 | } 100 | -------------------------------------------------------------------------------- /examples/tx-ssi-partition-move/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/tx-ssi-partition-move/.run -------------------------------------------------------------------------------- /examples/tx-ssi-partition-move/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tx-partition-move" 3 | version = "0.0.1" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fjall = { path = "../../", default-features = false, features = [ 10 | "lz4", 11 | "ssi_tx", 12 | ] } 13 | rand = "0.8.5" 14 | scru128 = "3.0.2" 15 | -------------------------------------------------------------------------------- /examples/tx-ssi-partition-move/README.md: -------------------------------------------------------------------------------- 1 | # tx-ssi-partition-move 2 | 3 | This example demonstrates atomically moving items between partitions using transactions. 4 | -------------------------------------------------------------------------------- /examples/tx-ssi-partition-move/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, PersistMode}; 2 | use std::path::Path; 3 | 4 | const ITEM_COUNT: u64 = 200; 5 | 6 | fn main() -> fjall::Result<()> { 7 | let path = Path::new(".fjall_data"); 8 | 9 | let keyspace = Config::new(path).temporary(true).open_transactional()?; 10 | let src = keyspace.open_partition("src", Default::default())?; 11 | let dst = keyspace.open_partition("dst", Default::default())?; 12 | 13 | for _ in 0..ITEM_COUNT { 14 | src.insert(scru128::new_string(), "")?; 15 | } 16 | 17 | let movers = (0..4) 18 | .map(|idx| { 19 | let keyspace = keyspace.clone(); 20 | let src = src.clone(); 21 | let dst = dst.clone(); 22 | 23 | std::thread::spawn(move || { 24 | use rand::Rng; 25 | 26 | let mut rng = rand::thread_rng(); 27 | 28 | loop { 29 | let mut tx = keyspace.write_tx().unwrap(); 30 | 31 | // TODO: NOTE: 32 | // Tombstones will add up over time, making first KV slower 33 | // Something like SingleDelete https://github.com/facebook/rocksdb/wiki/Single-Delete 34 | // would be good for this type of workload 35 | if let Some((key, value)) = tx.first_key_value(&src)? { 36 | let task_id = std::str::from_utf8(&key).unwrap().to_owned(); 37 | 38 | tx.remove(&src, key.clone()); 39 | tx.insert(&dst, key, value); 40 | 41 | tx.commit()?.ok(); 42 | 43 | println!("consumer {idx} moved {task_id}"); 44 | 45 | let ms = rng.gen_range(10..100); 46 | std::thread::sleep(std::time::Duration::from_millis(ms)); 47 | } else { 48 | return Ok::<_, fjall::Error>(()); 49 | } 50 | } 51 | }) 52 | }) 53 | .collect::>(); 54 | 55 | for t in movers { 56 | t.join().unwrap()?; 57 | } 58 | 59 | assert_eq!(ITEM_COUNT, keyspace.read_tx().len(&dst)? as u64); 60 | assert!(keyspace.read_tx().is_empty(&src)?); 61 | 62 | Ok(()) 63 | } 64 | -------------------------------------------------------------------------------- /examples/unique-index/.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/examples/unique-index/.run -------------------------------------------------------------------------------- /examples/unique-index/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "unique-index" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | fjall = { path = "../../" } 8 | -------------------------------------------------------------------------------- /examples/unique-index/README.md: -------------------------------------------------------------------------------- 1 | # unique-index 2 | 3 | This example uses a secondary partition as a unique, secondary index & transactions to guarantee an attribute being unique. 4 | -------------------------------------------------------------------------------- /examples/unique-index/src/main.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, TxKeyspace, TxPartition}; 2 | use std::path::Path; 3 | 4 | #[derive(Debug)] 5 | pub enum Error { 6 | Storage(fjall::Error), 7 | UniqueConstraintFailed, 8 | } 9 | 10 | impl From for Error { 11 | fn from(value: fjall::Error) -> Self { 12 | Self::Storage(value) 13 | } 14 | } 15 | 16 | fn maybe_create_item( 17 | keyspace: &TxKeyspace, 18 | items: &TxPartition, 19 | uniq: &TxPartition, 20 | id: &str, 21 | name: &str, 22 | ) -> Result<(), Error> { 23 | let mut tx = keyspace.write_tx(); 24 | 25 | if uniq.contains_key(name)? { 26 | return Err(Error::UniqueConstraintFailed); 27 | } 28 | 29 | tx.insert(items, id, name); 30 | tx.insert(uniq, name, id); 31 | 32 | tx.commit()?; 33 | 34 | Ok(()) 35 | } 36 | 37 | fn main() -> Result<(), Error> { 38 | let path = Path::new(".fjall_data"); 39 | 40 | let keyspace = Config::new(path).temporary(true).open_transactional()?; 41 | let items = keyspace.open_partition("items", Default::default())?; 42 | let uniq = keyspace.open_partition("uniq_idx", Default::default())?; 43 | 44 | maybe_create_item(&keyspace, &items, &uniq, "a", "Item A")?; 45 | maybe_create_item(&keyspace, &items, &uniq, "b", "Item B")?; 46 | maybe_create_item(&keyspace, &items, &uniq, "c", "Item C")?; 47 | 48 | assert!(matches!( 49 | maybe_create_item(&keyspace, &items, &uniq, "d", "Item A"), 50 | Err(Error::UniqueConstraintFailed), 51 | )); 52 | 53 | println!("Listing all unique values and their owners"); 54 | 55 | let mut found_count = 0; 56 | 57 | for kv in keyspace.read_tx().iter(&uniq) { 58 | let (k, v) = kv?; 59 | 60 | println!( 61 | "unique value: {:?} -> {:?}", 62 | std::str::from_utf8(&k).unwrap(), 63 | std::str::from_utf8(&v).unwrap(), 64 | ); 65 | 66 | found_count += 1; 67 | } 68 | 69 | assert_eq!(3, found_count); 70 | 71 | Ok(()) 72 | } 73 | -------------------------------------------------------------------------------- /kawaii.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/kawaii.png -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/logo.png -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /src/background_worker.rs: -------------------------------------------------------------------------------- 1 | use crate::poison_dart::PoisonDart; 2 | use lsm_tree::stop_signal::StopSignal; 3 | use std::sync::{atomic::AtomicUsize, Arc}; 4 | 5 | pub trait Activity { 6 | /// Gets the name of the activity. 7 | fn name(&self) -> &'static str; 8 | 9 | /// Runs the activity once. 10 | fn run(&mut self) -> crate::Result<()>; 11 | } 12 | 13 | pub struct BackgroundWorker { 14 | activity: A, 15 | poison_dart: PoisonDart, 16 | thread_counter: Arc, 17 | stop_signal: StopSignal, 18 | } 19 | 20 | impl BackgroundWorker { 21 | pub fn new( 22 | activity: A, 23 | poison_dart: PoisonDart, 24 | thread_counter: Arc, 25 | stop_signal: StopSignal, 26 | ) -> Self { 27 | Self { 28 | activity, 29 | poison_dart, 30 | thread_counter, 31 | stop_signal, 32 | } 33 | } 34 | 35 | /// Starts the background activity. 36 | /// 37 | /// Does not start a thread; this function is blocking. 38 | pub fn start(mut self) { 39 | log::debug!("Starting background worker {:?}", self.activity.name()); 40 | 41 | self.thread_counter 42 | .fetch_add(1, std::sync::atomic::Ordering::Release); 43 | 44 | loop { 45 | if self.stop_signal.is_stopped() { 46 | log::trace!( 47 | "Background worker {:?} exiting because keyspace is dropping", 48 | self.activity.name(), 49 | ); 50 | return; 51 | } 52 | 53 | if let Err(e) = self.activity.run() { 54 | eprintln!("Background worker {:?} failed: {e:?}", self.activity.name()); 55 | self.poison_dart.poison(); 56 | return; 57 | } 58 | } 59 | } 60 | } 61 | 62 | impl Drop for BackgroundWorker { 63 | fn drop(&mut self) { 64 | self.thread_counter 65 | .fetch_sub(1, std::sync::atomic::Ordering::Release); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/batch/item.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use super::PartitionKey; 6 | use lsm_tree::{UserKey, UserValue, ValueType}; 7 | 8 | #[derive(Clone, PartialEq, Eq)] 9 | pub struct Item { 10 | /// Partition key - an arbitrary byte array 11 | /// 12 | /// Supports up to 2^8 bytes 13 | pub partition: PartitionKey, 14 | 15 | /// User-defined key - an arbitrary byte array 16 | /// 17 | /// Supports up to 2^16 bytes 18 | pub key: UserKey, 19 | 20 | /// User-defined value - an arbitrary byte array 21 | /// 22 | /// Supports up to 65535 bytes 23 | pub value: UserValue, 24 | 25 | /// Tombstone marker - if this is true, the value has been deleted 26 | pub value_type: ValueType, 27 | } 28 | 29 | impl std::fmt::Debug for Item { 30 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 31 | write!( 32 | f, 33 | "{}:{:?}:{} => {:?}", 34 | self.partition, 35 | self.key, 36 | match self.value_type { 37 | ValueType::Value => "V", 38 | ValueType::Tombstone => "T", 39 | ValueType::WeakTombstone => "W", 40 | }, 41 | self.value 42 | ) 43 | } 44 | } 45 | 46 | impl Item { 47 | pub fn new, K: Into, V: Into>( 48 | partition: P, 49 | key: K, 50 | value: V, 51 | value_type: ValueType, 52 | ) -> Self { 53 | let p = partition.into(); 54 | let k = key.into(); 55 | let v = value.into(); 56 | 57 | assert!(!p.is_empty()); 58 | assert!(!k.is_empty()); 59 | 60 | assert!(u8::try_from(p.len()).is_ok(), "Partition name too long"); 61 | assert!( 62 | u16::try_from(k.len()).is_ok(), 63 | "Keys can be up to 65535 bytes long" 64 | ); 65 | assert!( 66 | u32::try_from(v.len()).is_ok(), 67 | "Values can be up to 2^32 bytes long" 68 | ); 69 | 70 | Self { 71 | partition: p, 72 | key: k, 73 | value: v, 74 | value_type, 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/batch/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | pub mod item; 6 | 7 | use crate::{Keyspace, PartitionHandle, PersistMode}; 8 | use item::Item; 9 | use lsm_tree::{AbstractTree, UserKey, UserValue, ValueType}; 10 | use std::collections::HashSet; 11 | 12 | /// Partition key (a.k.a. column family, locality group) 13 | pub type PartitionKey = byteview::StrView; 14 | 15 | /// An atomic write batch 16 | /// 17 | /// Allows atomically writing across partitions inside the [`Keyspace`]. 18 | #[doc(alias = "WriteBatch")] 19 | pub struct Batch { 20 | pub(crate) data: Vec, 21 | keyspace: Keyspace, 22 | durability: Option, 23 | } 24 | 25 | impl Batch { 26 | /// Initializes a new write batch. 27 | /// 28 | /// This function is called by [`Keyspace::batch`]. 29 | pub(crate) fn new(keyspace: Keyspace) -> Self { 30 | Self { 31 | data: Vec::new(), 32 | keyspace, 33 | durability: None, 34 | } 35 | } 36 | 37 | /// Initializes a new write batch with preallocated capacity. 38 | /// 39 | /// ### Note 40 | /// 41 | /// "Capacity" refers to the number of batch item slots, not their size in memory. 42 | #[must_use] 43 | pub fn with_capacity(keyspace: Keyspace, capacity: usize) -> Self { 44 | Self { 45 | data: Vec::with_capacity(capacity), 46 | keyspace, 47 | durability: None, 48 | } 49 | } 50 | 51 | /// Gets the number of batched items. 52 | #[must_use] 53 | pub fn len(&self) -> usize { 54 | self.data.len() 55 | } 56 | 57 | /// Returns `true` if there are no batches items (yet). 58 | #[must_use] 59 | pub fn is_empty(&self) -> bool { 60 | self.len() == 0 61 | } 62 | 63 | /// Sets the durability level. 64 | #[must_use] 65 | pub fn durability(mut self, mode: Option) -> Self { 66 | self.durability = mode; 67 | self 68 | } 69 | 70 | /// Inserts a key-value pair into the batch. 71 | pub fn insert, V: Into>( 72 | &mut self, 73 | p: &PartitionHandle, 74 | key: K, 75 | value: V, 76 | ) { 77 | self.data 78 | .push(Item::new(p.name.clone(), key, value, ValueType::Value)); 79 | } 80 | 81 | /// Adds a tombstone marker for a key 82 | pub fn remove>(&mut self, p: &PartitionHandle, key: K) { 83 | self.data 84 | .push(Item::new(p.name.clone(), key, vec![], ValueType::Tombstone)); 85 | } 86 | 87 | /// Commits the batch to the [`Keyspace`] atomically. 88 | /// 89 | /// # Errors 90 | /// 91 | /// Will return `Err` if an IO error occurs. 92 | pub fn commit(mut self) -> crate::Result<()> { 93 | use std::sync::atomic::Ordering; 94 | 95 | log::trace!("batch: Acquiring journal writer"); 96 | let mut journal_writer = self.keyspace.journal.get_writer(); 97 | 98 | // IMPORTANT: Check the poisoned flag after getting journal mutex, otherwise TOCTOU 99 | if self.keyspace.is_poisoned.load(Ordering::Relaxed) { 100 | return Err(crate::Error::Poisoned); 101 | } 102 | 103 | let batch_seqno = self.keyspace.seqno.next(); 104 | 105 | let _ = journal_writer.write_batch(self.data.iter(), self.data.len(), batch_seqno); 106 | 107 | if let Some(mode) = self.durability { 108 | if let Err(e) = journal_writer.persist(mode) { 109 | self.keyspace.is_poisoned.store(true, Ordering::Release); 110 | 111 | log::error!( 112 | "persist failed, which is a FATAL, and possibly hardware-related, failure: {e:?}" 113 | ); 114 | 115 | return Err(crate::Error::Poisoned); 116 | } 117 | } 118 | 119 | #[allow(clippy::mutable_key_type)] 120 | let mut partitions_with_possible_stall = HashSet::new(); 121 | let partitions = self.keyspace.partitions.read().expect("lock is poisoned"); 122 | 123 | let mut batch_size = 0u64; 124 | 125 | log::trace!("Applying {} batched items to memtable(s)", self.data.len()); 126 | 127 | for item in std::mem::take(&mut self.data) { 128 | let Some(partition) = partitions.get(&item.partition) else { 129 | continue; 130 | }; 131 | 132 | // TODO: need a better, generic write op 133 | let (item_size, _) = match item.value_type { 134 | ValueType::Value => partition.tree.insert(item.key, item.value, batch_seqno), 135 | ValueType::Tombstone => partition.tree.remove(item.key, batch_seqno), 136 | ValueType::WeakTombstone => partition.tree.remove_weak(item.key, batch_seqno), 137 | }; 138 | 139 | batch_size += u64::from(item_size); 140 | 141 | // IMPORTANT: Clone the handle, because we don't want to keep the partitions lock open 142 | partitions_with_possible_stall.insert(partition.clone()); 143 | } 144 | 145 | self.keyspace 146 | .visible_seqno 147 | .fetch_max(batch_seqno + 1, Ordering::AcqRel); 148 | 149 | drop(journal_writer); 150 | 151 | log::trace!("batch: Freed journal writer"); 152 | 153 | drop(partitions); 154 | 155 | // IMPORTANT: Add batch size to current write buffer size 156 | // Otherwise write buffer growth is unbounded when using batches 157 | self.keyspace.write_buffer_manager.allocate(batch_size); 158 | 159 | // Check each affected partition for write stall/halt 160 | for partition in partitions_with_possible_stall { 161 | let memtable_size = partition.tree.active_memtable_size(); 162 | 163 | if let Err(e) = partition.check_memtable_overflow(memtable_size) { 164 | log::error!("Failed memtable rotate check: {e:?}"); 165 | } 166 | 167 | // IMPORTANT: Check write buffer as well 168 | // Otherwise batch writes are never stalled/halted 169 | let write_buffer_size = self.keyspace.write_buffer_manager.get(); 170 | partition.check_write_buffer_size(write_buffer_size); 171 | } 172 | 173 | Ok(()) 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /src/compaction/manager.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use crate::PartitionHandle; 6 | use std::{ 7 | collections::VecDeque, 8 | sync::{Arc, Mutex}, 9 | }; 10 | use std_semaphore::Semaphore; 11 | 12 | pub struct CompactionManagerInner { 13 | partitions: Mutex>, 14 | semaphore: Semaphore, 15 | } 16 | 17 | impl Drop for CompactionManagerInner { 18 | fn drop(&mut self) { 19 | log::trace!("Dropping compaction manager"); 20 | } 21 | } 22 | 23 | impl Default for CompactionManagerInner { 24 | fn default() -> Self { 25 | Self { 26 | partitions: Mutex::new(VecDeque::with_capacity(10)), 27 | semaphore: Semaphore::new(0), 28 | } 29 | } 30 | } 31 | 32 | /// The compaction manager keeps track of which partitions 33 | /// have recently been flushed in a FIFO queue. 34 | /// 35 | /// Its semaphore notifies compaction threads which will wake 36 | /// up and consume the queue items. 37 | /// 38 | /// The semaphore is incremented by the flush worker and optionally 39 | /// by the individual partitions in case of write halting. 40 | #[derive(Clone, Default)] 41 | #[allow(clippy::module_name_repetitions)] 42 | pub struct CompactionManager(Arc); 43 | 44 | impl std::ops::Deref for CompactionManager { 45 | type Target = CompactionManagerInner; 46 | 47 | fn deref(&self) -> &Self::Target { 48 | &self.0 49 | } 50 | } 51 | 52 | impl CompactionManager { 53 | pub fn clear(&self) { 54 | self.partitions.lock().expect("lock is poisoned").clear(); 55 | } 56 | 57 | pub fn remove_partition(&self, name: &str) { 58 | let mut lock = self.partitions.lock().expect("lock is poisoned"); 59 | lock.retain(|x| &*x.name != name); 60 | } 61 | 62 | pub fn wait_for(&self) { 63 | self.semaphore.acquire(); 64 | } 65 | 66 | pub fn notify(&self, partition: PartitionHandle) { 67 | let mut lock = self.partitions.lock().expect("lock is poisoned"); 68 | lock.push_back(partition); 69 | self.semaphore.release(); 70 | } 71 | 72 | pub fn notify_empty(&self) { 73 | self.semaphore.release(); 74 | } 75 | 76 | pub fn pop(&self) -> Option { 77 | let mut lock = self.partitions.lock().expect("lock is poisoned"); 78 | lock.pop_front() 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/compaction/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | pub(crate) mod manager; 6 | pub(crate) mod worker; 7 | 8 | use std::sync::Arc; 9 | 10 | pub use lsm_tree::compaction::{Fifo, Leveled, Levelled, SizeTiered}; 11 | 12 | /// Compaction strategy 13 | #[derive(Clone)] 14 | #[allow(clippy::module_name_repetitions)] 15 | pub enum Strategy { 16 | /// Leveled compaction 17 | Leveled(crate::compaction::Leveled), 18 | 19 | /// Size-tiered compaction 20 | SizeTiered(crate::compaction::SizeTiered), 21 | 22 | /// FIFO compaction 23 | Fifo(crate::compaction::Fifo), 24 | } 25 | 26 | impl std::fmt::Debug for Strategy { 27 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 28 | write!( 29 | f, 30 | "{}", 31 | match self { 32 | Self::SizeTiered(_) => "SizeTieredStrategy", 33 | Self::Leveled(_) => "LeveledStrategy", 34 | Self::Fifo(_) => "FifoStrategy", 35 | } 36 | ) 37 | } 38 | } 39 | 40 | impl Default for Strategy { 41 | fn default() -> Self { 42 | Self::Leveled(crate::compaction::Leveled::default()) 43 | } 44 | } 45 | 46 | impl Strategy { 47 | pub(crate) fn inner(&self) -> Arc { 48 | match self { 49 | Self::Leveled(s) => Arc::new(s.clone()), 50 | Self::SizeTiered(s) => Arc::new(s.clone()), 51 | Self::Fifo(s) => Arc::new(s.clone()), 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/compaction/worker.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use super::manager::CompactionManager; 6 | use crate::{snapshot_tracker::SnapshotTracker, stats::Stats}; 7 | use lsm_tree::AbstractTree; 8 | use std::time::Instant; 9 | 10 | /// Runs a single run of compaction. 11 | pub fn run( 12 | compaction_manager: &CompactionManager, 13 | snapshot_tracker: &SnapshotTracker, 14 | stats: &Stats, 15 | ) -> crate::Result<()> { 16 | use std::sync::atomic::Ordering::Relaxed; 17 | 18 | let Some(item) = compaction_manager.pop() else { 19 | return Ok(()); 20 | }; 21 | 22 | log::trace!( 23 | "compactor: calling compaction strategy for partition {:?}", 24 | item.0.name, 25 | ); 26 | 27 | let strategy = item.config.compaction_strategy.clone(); 28 | 29 | stats.active_compaction_count.fetch_add(1, Relaxed); 30 | 31 | let start = Instant::now(); 32 | 33 | if let Err(e) = item 34 | .tree 35 | .compact(strategy.inner(), snapshot_tracker.get_seqno_safe_to_gc()) 36 | { 37 | log::error!("Compaction failed: {e:?}"); 38 | return Err(e.into()); 39 | } 40 | 41 | #[allow(clippy::cast_possible_truncation)] 42 | stats 43 | .time_compacting 44 | .fetch_add(start.elapsed().as_micros() as u64, Relaxed); 45 | 46 | stats.active_compaction_count.fetch_sub(1, Relaxed); 47 | stats.compactions_completed.fetch_add(1, Relaxed); 48 | 49 | // TODO: loop if there's more work to do? 50 | 51 | Ok(()) 52 | } 53 | -------------------------------------------------------------------------------- /src/drop.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use std::sync::atomic::Ordering::Relaxed; 6 | use std::sync::{atomic::AtomicUsize, OnceLock}; 7 | 8 | static DROP_COUNTER: OnceLock = OnceLock::new(); 9 | 10 | pub fn increment_drop_counter() { 11 | get_drop_counter().fetch_add(1, Relaxed); 12 | } 13 | 14 | pub fn decrement_drop_counter() { 15 | get_drop_counter().fetch_sub(1, Relaxed); 16 | } 17 | 18 | pub fn get_drop_counter<'a>() -> &'a AtomicUsize { 19 | DROP_COUNTER.get_or_init(AtomicUsize::default) 20 | } 21 | 22 | #[must_use] 23 | pub fn load_drop_counter() -> usize { 24 | get_drop_counter().load(Relaxed) 25 | } 26 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use crate::{journal::error::RecoveryError as JournalRecoveryError, version::Version}; 6 | use lsm_tree::{DecodeError, EncodeError}; 7 | 8 | /// Errors that may occur in the storage engine 9 | #[derive(Debug)] 10 | #[non_exhaustive] 11 | pub enum Error { 12 | /// Error inside LSM-tree 13 | Storage(lsm_tree::Error), 14 | 15 | /// I/O error 16 | Io(std::io::Error), 17 | 18 | /// Serialization failed 19 | Encode(EncodeError), 20 | 21 | /// Deserialization failed 22 | Decode(DecodeError), 23 | 24 | /// Error during journal recovery 25 | JournalRecovery(JournalRecoveryError), 26 | 27 | /// Invalid or unparsable data format version 28 | InvalidVersion(Option), 29 | 30 | /// A previous flush / commit operation failed, indicating a hardware-related failure 31 | /// 32 | /// Future writes will not be accepted as consistency cannot be guaranteed. 33 | /// 34 | /// **At this point, it's best to let the application crash and try to recover.** 35 | /// 36 | /// More info: 37 | Poisoned, 38 | 39 | /// Partition is deleted 40 | PartitionDeleted, 41 | } 42 | 43 | impl std::fmt::Display for Error { 44 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 45 | write!(f, "FjallError: {self:?}") 46 | } 47 | } 48 | 49 | impl From for Error { 50 | fn from(inner: std::io::Error) -> Self { 51 | Self::Io(inner) 52 | } 53 | } 54 | 55 | impl From for Error { 56 | fn from(value: EncodeError) -> Self { 57 | Self::Encode(value) 58 | } 59 | } 60 | 61 | impl From for Error { 62 | fn from(value: DecodeError) -> Self { 63 | Self::Decode(value) 64 | } 65 | } 66 | 67 | impl From for Error { 68 | fn from(inner: lsm_tree::Error) -> Self { 69 | Self::Storage(inner) 70 | } 71 | } 72 | 73 | impl std::error::Error for Error { 74 | fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { 75 | match self { 76 | Self::Storage(inner) => Some(inner), 77 | Self::Io(inner) => Some(inner), 78 | Self::Encode(inner) => Some(inner), 79 | Self::Decode(inner) => Some(inner), 80 | Self::JournalRecovery(inner) => Some(inner), 81 | Self::InvalidVersion(_) => None, 82 | Self::Poisoned => None, 83 | Self::PartitionDeleted => None, 84 | } 85 | } 86 | } 87 | 88 | /// Result helper type 89 | pub type Result = std::result::Result; 90 | -------------------------------------------------------------------------------- /src/file.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use std::path::Path; 6 | 7 | pub const MAGIC_BYTES: &[u8] = &[b'F', b'J', b'L', 2]; 8 | 9 | pub const JOURNALS_FOLDER: &str = "journals"; 10 | pub const PARTITIONS_FOLDER: &str = "partitions"; 11 | 12 | pub const FJALL_MARKER: &str = "version"; 13 | pub const PARTITION_DELETED_MARKER: &str = ".deleted"; 14 | pub const PARTITION_CONFIG_FILE: &str = "config"; 15 | 16 | pub const LSM_MANIFEST_FILE: &str = "manifest"; 17 | 18 | #[cfg(not(target_os = "windows"))] 19 | pub fn fsync_directory>(path: P) -> std::io::Result<()> { 20 | let path = path.as_ref(); 21 | 22 | let file = std::fs::File::open(path).inspect_err(|e| { 23 | log::error!("Failed to open directory at {path:?}: {e:?}"); 24 | })?; 25 | 26 | debug_assert!(file.metadata()?.is_dir()); 27 | 28 | file.sync_all().inspect_err(|e| { 29 | log::error!("Failed to fsync directory at {path:?}: {e:?}"); 30 | }) 31 | } 32 | 33 | #[cfg(target_os = "windows")] 34 | pub fn fsync_directory>(_path: P) -> std::io::Result<()> { 35 | // Cannot fsync directory on Windows 36 | Ok(()) 37 | } 38 | -------------------------------------------------------------------------------- /src/flush/manager.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use super::queue::FlushQueue; 6 | use crate::{batch::PartitionKey, HashMap, HashSet, PartitionHandle}; 7 | use lsm_tree::{Memtable, SegmentId}; 8 | use std::sync::Arc; 9 | 10 | pub struct Task { 11 | /// ID of memtable 12 | pub(crate) id: SegmentId, 13 | 14 | /// Memtable to flush 15 | pub(crate) sealed_memtable: Arc, 16 | 17 | /// Partition 18 | pub(crate) partition: PartitionHandle, 19 | } 20 | 21 | impl std::fmt::Debug for Task { 22 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 23 | write!(f, "FlushTask {}:{}", self.partition.name, self.id) 24 | } 25 | } 26 | 27 | // TODO: accessing flush manager shouldn't take RwLock... but changing its internals should 28 | 29 | /// The [`FlushManager`] stores a dictionary of queues, each queue 30 | /// containing some flush tasks. 31 | /// 32 | /// Each flush task references a sealed memtable and the given partition. 33 | #[allow(clippy::module_name_repetitions)] 34 | #[derive(Debug)] 35 | pub struct FlushManager { 36 | queues: HashMap, 37 | } 38 | 39 | impl Drop for FlushManager { 40 | fn drop(&mut self) { 41 | log::trace!("Dropping flush manager"); 42 | 43 | #[cfg(feature = "__internal_whitebox")] 44 | crate::drop::decrement_drop_counter(); 45 | } 46 | } 47 | 48 | impl FlushManager { 49 | pub(crate) fn new() -> Self { 50 | #[cfg(feature = "__internal_whitebox")] 51 | crate::drop::increment_drop_counter(); 52 | 53 | Self { 54 | queues: HashMap::default(), 55 | } 56 | } 57 | 58 | pub(crate) fn clear(&mut self) { 59 | self.queues.clear(); 60 | } 61 | 62 | /// Gets the names of partitions that have queued tasks. 63 | pub(crate) fn get_partitions_with_tasks(&self) -> HashSet { 64 | self.queues 65 | .iter() 66 | .filter(|(_, v)| !v.is_empty()) 67 | .map(|(k, _)| k) 68 | .cloned() 69 | .collect() 70 | } 71 | 72 | /// Returns the amount of queues. 73 | pub(crate) fn queue_count(&self) -> usize { 74 | self.queues.len() 75 | } 76 | 77 | /// Returns the amount of bytes queued. 78 | pub(crate) fn queued_size(&self) -> u64 { 79 | self.queues.values().map(FlushQueue::size).sum::() 80 | } 81 | 82 | // NOTE: is actually used in tests 83 | #[allow(dead_code)] 84 | /// Returns the amount of tasks that are queued to be flushed. 85 | pub(crate) fn len(&self) -> usize { 86 | self.queues.values().map(FlushQueue::len).sum::() 87 | } 88 | 89 | // NOTE: is actually used in tests 90 | #[allow(dead_code)] 91 | #[must_use] 92 | pub(crate) fn is_empty(&self) -> bool { 93 | self.len() == 0 94 | } 95 | 96 | pub(crate) fn remove_partition(&mut self, name: &str) { 97 | self.queues.remove(name); 98 | } 99 | 100 | pub(crate) fn enqueue_task(&mut self, partition_name: PartitionKey, task: Task) { 101 | log::debug!( 102 | "Enqueuing {partition_name}:{} for flushing ({} B)", 103 | task.id, 104 | task.sealed_memtable.size() 105 | ); 106 | 107 | self.queues 108 | .entry(partition_name) 109 | .or_default() 110 | .enqueue(Arc::new(task)); 111 | } 112 | 113 | /// Returns a list of tasks per partition. 114 | pub(crate) fn collect_tasks(&mut self, limit: usize) -> HashMap>> { 115 | let mut collected: HashMap<_, Vec<_>> = HashMap::default(); 116 | let mut cnt = 0; 117 | 118 | // NOTE: Returning multiple tasks per partition is fine and will 119 | // help with flushing very active partitions. 120 | // 121 | // Because we are flushing them atomically inside one batch, 122 | // we will never cover up a lower seqno of some other segment. 123 | // For this to work, all tasks need to be successful and atomically 124 | // applied (all-or-nothing). 125 | 'outer: for (partition_name, queue) in &self.queues { 126 | for item in queue.iter() { 127 | if cnt == limit { 128 | break 'outer; 129 | } 130 | 131 | collected 132 | .entry(partition_name.clone()) 133 | .or_default() 134 | .push(item.clone()); 135 | 136 | cnt += 1; 137 | } 138 | } 139 | 140 | collected 141 | } 142 | 143 | pub(crate) fn dequeue_tasks(&mut self, partition_name: PartitionKey, cnt: usize) { 144 | self.queues.entry(partition_name).or_default().dequeue(cnt); 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/flush/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | pub mod manager; 6 | pub mod queue; 7 | pub mod worker; 8 | -------------------------------------------------------------------------------- /src/flush/queue.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use super::manager::Task; 6 | use std::sync::Arc; 7 | 8 | /// A FIFO queue of flush tasks. 9 | /// 10 | /// Allows peeking N items into the queue head to allow 11 | /// parallel processing of tasks of a single partition, 12 | /// which allows processing N tasks even if there is just 13 | /// one partition. 14 | /// 15 | /// This only really works because there is one flush thread 16 | /// that spawns flush workers for each partition it collects tasks for. 17 | #[derive(Default, Debug)] 18 | #[allow(clippy::module_name_repetitions)] 19 | pub struct FlushQueue { 20 | items: Vec>, 21 | } 22 | 23 | impl FlushQueue { 24 | pub fn iter(&self) -> impl Iterator> + '_ { 25 | self.items.iter() 26 | } 27 | 28 | pub fn dequeue(&mut self, cnt: usize) { 29 | for _ in 0..cnt { 30 | self.items.remove(0); 31 | } 32 | } 33 | 34 | pub fn enqueue(&mut self, item: Arc) { 35 | self.items.push(item); 36 | } 37 | 38 | pub fn is_empty(&self) -> bool { 39 | self.items.is_empty() 40 | } 41 | 42 | pub fn len(&self) -> usize { 43 | self.items.len() 44 | } 45 | 46 | pub fn size(&self) -> u64 { 47 | self.items 48 | .iter() 49 | .map(|x| x.sealed_memtable.size()) 50 | .map(u64::from) 51 | .sum::() 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/iter.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use crate::snapshot_nonce::SnapshotNonce; 6 | 7 | /// A wrapper around iterators to hold a snapshot moment 8 | /// 9 | /// We need to hold the snapshot nonce so the GC watermark does not 10 | /// move past this snapshot moment, removing data that may still be read. 11 | /// 12 | /// This may not be strictly needed because an iterator holds a read lock to a memtable anyway 13 | /// but for correctness it's probably better. 14 | pub struct Iter>> { 15 | iter: I, 16 | 17 | #[allow(unused)] 18 | nonce: SnapshotNonce, 19 | } 20 | 21 | impl>> Iter { 22 | pub fn new(nonce: SnapshotNonce, iter: I) -> Self { 23 | Self { iter, nonce } 24 | } 25 | } 26 | 27 | impl>> Iterator for Iter { 28 | type Item = crate::Result; 29 | 30 | fn next(&mut self) -> Option { 31 | self.iter.next() 32 | } 33 | } 34 | 35 | impl>> DoubleEndedIterator for Iter { 36 | fn next_back(&mut self) -> Option { 37 | self.iter.next_back() 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/journal/error.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | /// Recovery mode to use 6 | /// 7 | /// Based on `RocksDB`'s WAL Recovery Modes: 8 | #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] 9 | #[non_exhaustive] 10 | pub enum RecoveryMode { 11 | /// The last batch in the journal may be corrupt on crash, 12 | /// and will be discarded without error. 13 | /// 14 | /// This mode will error on any other IO or consistency error, so 15 | /// any data up to the tail will be consistent. 16 | /// 17 | /// This is the default mode. 18 | #[default] 19 | TolerateCorruptTail, 20 | // TODO: in the future? 21 | /* /// Skips corrupt (invalid checksum) batches. This may violate 22 | /// consistency, but will recover as much data as possible. 23 | SkipInvalidBatches, */ 24 | // TODO: absolute consistency 25 | } 26 | 27 | /// Errors that can occur during journal recovery 28 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] 29 | #[allow(clippy::module_name_repetitions)] 30 | pub enum RecoveryError { 31 | /// Batch had less items than expected, so it's incomplete 32 | InsufficientLength, 33 | 34 | /* /// Batch was not terminated, so it's possibly incomplete 35 | MissingTerminator, */ 36 | /// Too many items in batch 37 | TooManyItems, 38 | 39 | /// The checksum value does not match the expected value 40 | ChecksumMismatch, 41 | } 42 | 43 | impl std::fmt::Display for RecoveryError { 44 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 45 | write!(f, "RecoveryError({self:?})") 46 | } 47 | } 48 | 49 | impl std::error::Error for RecoveryError {} 50 | -------------------------------------------------------------------------------- /src/journal/manager.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use super::writer::Writer; 6 | use crate::PartitionHandle; 7 | use lsm_tree::{AbstractTree, SeqNo}; 8 | use std::{path::PathBuf, sync::MutexGuard}; 9 | 10 | /// Stores the highest seqno of a partition found in a journal. 11 | #[derive(Clone)] 12 | pub struct EvictionWatermark { 13 | pub(crate) partition: PartitionHandle, 14 | pub(crate) lsn: SeqNo, 15 | } 16 | 17 | impl std::fmt::Debug for EvictionWatermark { 18 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 19 | write!(f, "{}:{}", self.partition.name, self.lsn) 20 | } 21 | } 22 | 23 | pub struct Item { 24 | pub(crate) path: PathBuf, 25 | pub(crate) size_in_bytes: u64, 26 | pub(crate) watermarks: Vec, 27 | } 28 | 29 | impl std::fmt::Debug for Item { 30 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 31 | write!( 32 | f, 33 | "JournalManagerItem {:?} => {:#?}", 34 | self.path, self.watermarks 35 | ) 36 | } 37 | } 38 | 39 | // TODO: accessing journal manager shouldn't take RwLock... but changing its internals should 40 | 41 | /// The [`JournalManager`] keeps track of sealed journals that are being flushed. 42 | /// 43 | /// Each journal may contain items of different partitions. 44 | #[allow(clippy::module_name_repetitions)] 45 | #[derive(Debug)] 46 | pub struct JournalManager { 47 | active_path: PathBuf, // TODO: remove? 48 | items: Vec, 49 | 50 | // TODO: should be taking into account active journal, which is preallocated... 51 | disk_space_in_bytes: u64, 52 | } 53 | 54 | impl Drop for JournalManager { 55 | fn drop(&mut self) { 56 | log::trace!("Dropping journal manager"); 57 | 58 | #[cfg(feature = "__internal_whitebox")] 59 | crate::drop::decrement_drop_counter(); 60 | } 61 | } 62 | 63 | impl JournalManager { 64 | pub(crate) fn from_active>(path: P) -> Self { 65 | #[cfg(feature = "__internal_whitebox")] 66 | crate::drop::increment_drop_counter(); 67 | 68 | Self { 69 | active_path: path.into(), 70 | items: Vec::with_capacity(10), 71 | disk_space_in_bytes: 0, 72 | } 73 | } 74 | 75 | pub(crate) fn clear(&mut self) { 76 | self.items.clear(); 77 | } 78 | 79 | pub(crate) fn enqueue(&mut self, item: Item) { 80 | self.disk_space_in_bytes = self.disk_space_in_bytes.saturating_add(item.size_in_bytes); 81 | self.items.push(item); 82 | } 83 | 84 | /// Returns the amount of journals 85 | pub(crate) fn journal_count(&self) -> usize { 86 | // NOTE: + 1 = active journal 87 | self.sealed_journal_count() + 1 88 | } 89 | 90 | /// Returns the amount of sealed journals 91 | pub(crate) fn sealed_journal_count(&self) -> usize { 92 | self.items.len() 93 | } 94 | 95 | /// Returns the amount of bytes used on disk by journals 96 | pub(crate) fn disk_space_used(&self) -> u64 { 97 | self.disk_space_in_bytes 98 | } 99 | 100 | /// Performs maintenance, maybe deleting some old journals 101 | pub(crate) fn maintenance(&mut self) -> crate::Result<()> { 102 | loop { 103 | let Some(item) = self.items.first() else { 104 | return Ok(()); 105 | }; 106 | 107 | // TODO: unit test: check deleted partition does not prevent journal eviction 108 | for item in &item.watermarks { 109 | // Only check partition seqno if not deleted 110 | if !item 111 | .partition 112 | .is_deleted 113 | .load(std::sync::atomic::Ordering::Acquire) 114 | { 115 | let Some(partition_seqno) = item.partition.tree.get_highest_persisted_seqno() 116 | else { 117 | return Ok(()); 118 | }; 119 | 120 | if partition_seqno < item.lsn { 121 | return Ok(()); 122 | } 123 | } 124 | } 125 | 126 | // NOTE: Once the LSN of *every* partition's segments [1] is higher than the journal's stored partition seqno, 127 | // it can be deleted from disk, as we know the entire journal has been flushed to segments [2]. 128 | // 129 | // [1] We cannot use the partition's max seqno, because the memtable will get writes, which increase the seqno. 130 | // We *need* to check the disk segments specifically, they are the source of truth for flushed data. 131 | // 132 | // [2] Checking the seqno is safe because the queues inside the flush manager are FIFO. 133 | // 134 | // IMPORTANT: On recovery, the journals need to be flushed from oldest to newest. 135 | log::trace!("Removing fully flushed journal at {:?}", item.path); 136 | std::fs::remove_file(&item.path).inspect_err(|e| { 137 | log::error!( 138 | "Failed to clean up stale journal file at {:?}: {e:?}", 139 | item.path, 140 | ); 141 | })?; 142 | 143 | self.disk_space_in_bytes = self.disk_space_in_bytes.saturating_sub(item.size_in_bytes); 144 | self.items.remove(0); 145 | } 146 | } 147 | 148 | pub(crate) fn rotate_journal( 149 | &mut self, 150 | journal_writer: &mut MutexGuard, 151 | watermarks: Vec, 152 | ) -> crate::Result<()> { 153 | let journal_size = journal_writer.len()?; 154 | 155 | let (sealed_path, next_journal_path) = journal_writer.rotate()?; 156 | self.active_path = next_journal_path; 157 | 158 | self.enqueue(Item { 159 | path: sealed_path, 160 | watermarks, 161 | size_in_bytes: journal_size, 162 | }); 163 | 164 | Ok(()) 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/journal/reader.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use super::marker::Marker; 6 | use lsm_tree::{coding::Decode, DecodeError}; 7 | use std::{ 8 | fs::{File, OpenOptions}, 9 | io::{BufReader, Seek}, 10 | path::{Path, PathBuf}, 11 | }; 12 | 13 | macro_rules! fail_iter { 14 | ($e:expr) => { 15 | match $e { 16 | Ok(v) => v, 17 | Err(e) => return Some(Err(e.into())), 18 | } 19 | }; 20 | } 21 | 22 | /// Reads and emits through the entries in a journal file, but doesn't 23 | /// check the validity of batches 24 | /// 25 | /// Will truncate the file to the last valid position to prevent corrupt 26 | /// bytes at the end of the file, which would jeopardize future writes into the file. 27 | #[allow(clippy::module_name_repetitions)] 28 | pub struct JournalReader { 29 | pub(crate) path: PathBuf, 30 | pub(crate) reader: BufReader, 31 | pub(crate) last_valid_pos: u64, 32 | } 33 | 34 | impl JournalReader { 35 | pub fn new>(path: P) -> crate::Result { 36 | let file = OpenOptions::new().read(true).write(true).open(&path)?; 37 | 38 | Ok(Self { 39 | path: path.as_ref().into(), 40 | reader: BufReader::new(file), 41 | last_valid_pos: 0, 42 | }) 43 | } 44 | 45 | fn truncate_file(&mut self, pos: u64) -> crate::Result<()> { 46 | log::debug!("truncating journal to {pos}"); 47 | self.reader.get_mut().set_len(pos)?; 48 | self.reader.get_mut().sync_all()?; 49 | Ok(()) 50 | } 51 | 52 | fn maybe_truncate_file_to_last_valid_pos(&mut self) -> crate::Result<()> { 53 | let stream_pos = self.reader.stream_position()?; 54 | 55 | if stream_pos > self.last_valid_pos { 56 | self.truncate_file(self.last_valid_pos)?; 57 | } 58 | 59 | Ok(()) 60 | } 61 | } 62 | 63 | impl Iterator for JournalReader { 64 | type Item = crate::Result; 65 | 66 | fn next(&mut self) -> Option { 67 | match Marker::decode_from(&mut self.reader) { 68 | Ok(item) => { 69 | self.last_valid_pos = fail_iter!(self.reader.stream_position()); 70 | Some(Ok(item)) 71 | } 72 | Err(e) => { 73 | if let DecodeError::Io(e) = e { 74 | match e.kind() { 75 | std::io::ErrorKind::UnexpectedEof | std::io::ErrorKind::Other => { 76 | fail_iter!(self.maybe_truncate_file_to_last_valid_pos()); 77 | None 78 | } 79 | _ => Some(Err(crate::Error::Io(e))), 80 | } 81 | } else { 82 | fail_iter!(self.maybe_truncate_file_to_last_valid_pos()); 83 | None 84 | } 85 | } 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/journal/recovery.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use super::Journal; 6 | use std::path::{Path, PathBuf}; 7 | 8 | pub type JournalId = u64; 9 | 10 | #[derive(Debug)] 11 | #[allow(clippy::module_name_repetitions)] 12 | pub struct RecoveryResult { 13 | pub(crate) active: Journal, 14 | pub(crate) sealed: Vec<(JournalId, PathBuf)>, 15 | pub(crate) was_active_created: bool, 16 | } 17 | 18 | pub fn recover_journals>(path: P) -> crate::Result { 19 | let path = path.as_ref(); 20 | 21 | let mut max_journal_id: JournalId = 0; 22 | let mut journal_fragments = Vec::<(JournalId, PathBuf)>::new(); 23 | 24 | log::warn!("{journal_fragments:?}"); 25 | 26 | for dirent in std::fs::read_dir(path)? { 27 | let dirent = dirent?; 28 | let path = dirent.path(); 29 | let filename = dirent.file_name(); 30 | 31 | if filename == ".DS_Store" { 32 | continue; 33 | } 34 | 35 | assert!(dirent.file_type()?.is_file()); 36 | 37 | let filename = filename.to_str().expect("should be utf-8"); 38 | 39 | let journal_id = filename 40 | .strip_suffix(".sealed") // TODO: 3.0.0 remove in V3 41 | .unwrap_or(filename) 42 | .parse::() 43 | .expect("should be valid journal ID"); 44 | 45 | max_journal_id = max_journal_id.max(journal_id); 46 | 47 | journal_fragments.push((journal_id, path)); 48 | } 49 | 50 | // NOTE: Sort ascending, so the last item is the active journal 51 | journal_fragments.sort_by(|(a, _), (b, _)| a.cmp(b)); 52 | 53 | Ok(match journal_fragments.pop() { 54 | Some((_, active)) => RecoveryResult { 55 | active: Journal::from_file(active)?, 56 | sealed: journal_fragments, 57 | was_active_created: false, 58 | }, 59 | None => RecoveryResult { 60 | active: { 61 | let id: JournalId = max_journal_id + 1; 62 | Journal::create_new(path.join(id.to_string()))? 63 | }, 64 | sealed: vec![], 65 | was_active_created: true, 66 | }, 67 | }) 68 | } 69 | -------------------------------------------------------------------------------- /src/partition/name.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | const VALID_CHARACTERS: &str = 6 | "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-.#$"; 7 | 8 | /// Partition names can be up to 255 characters long, can not be empty and 9 | /// can only contain alphanumerics, underscore (`_`), dash (`-`), dot (`.`), hash tag (`#`) and dollar (`$`). 10 | #[allow(clippy::module_name_repetitions)] 11 | pub fn is_valid_partition_name(s: &str) -> bool { 12 | if s.is_empty() { 13 | return false; 14 | } 15 | 16 | if u8::try_from(s.len()).is_err() { 17 | return false; 18 | } 19 | 20 | s.chars().all(|c| VALID_CHARACTERS.contains(c)) 21 | } 22 | -------------------------------------------------------------------------------- /src/partition/write_delay.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | /// Gets the write delay based on L0 segments. 6 | /// 7 | /// The write delay increases linearly as L0 approaches 20 segments. 8 | #[allow(clippy::module_name_repetitions)] 9 | pub fn get_write_delay(l0_segments: usize) -> u64 { 10 | match l0_segments { 11 | 20 => 1, 12 | 21 => 2, 13 | 22 => 4, 14 | 23 => 8, 15 | 24 => 16, 16 | 25 => 32, 17 | 26 => 64, 18 | 27 => 128, 19 | 28 => 256, 20 | 29 => 512, 21 | _ => 0, 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/path.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use path_absolutize::Absolutize; 6 | use std::path::{Path, PathBuf}; 7 | 8 | #[allow(clippy::module_name_repetitions)] 9 | pub fn absolute_path>(path: P) -> PathBuf { 10 | // TODO: replace with https://doc.rust-lang.org/std/path/fn.absolute.html once stable 11 | path.as_ref() 12 | .absolutize() 13 | .expect("should be absolute path") 14 | .into() 15 | } 16 | -------------------------------------------------------------------------------- /src/poison_dart.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{atomic::AtomicBool, Arc}; 2 | 3 | type PoisonSignal = Arc; 4 | 5 | /// RAII guard to catch panics in background workers 6 | /// and poison a keyspace 7 | pub struct PoisonDart { 8 | name: &'static str, 9 | signal: PoisonSignal, 10 | } 11 | 12 | impl PoisonDart { 13 | pub fn new(name: &'static str, signal: PoisonSignal) -> Self { 14 | Self { name, signal } 15 | } 16 | 17 | pub fn poison(&self) { 18 | self.signal 19 | .store(true, std::sync::atomic::Ordering::Release); 20 | } 21 | } 22 | 23 | impl Drop for PoisonDart { 24 | fn drop(&mut self) { 25 | if std::thread::panicking() { 26 | log::error!( 27 | "Poisoning keyspace because of panic in background worker {:?}", 28 | self.name 29 | ); 30 | self.poison(); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/snapshot_nonce.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use crate::{snapshot_tracker::SnapshotTracker, Instant}; 6 | 7 | /// Holds a snapshot instant and automatically frees it from the snapshot tracker when dropped 8 | pub struct SnapshotNonce { 9 | pub(crate) instant: Instant, 10 | tracker: SnapshotTracker, 11 | } 12 | 13 | impl Clone for SnapshotNonce { 14 | fn clone(&self) -> Self { 15 | // IMPORTANT: Increment snapshot count in tracker 16 | self.tracker.open(self.instant); 17 | 18 | Self { 19 | instant: self.instant, 20 | tracker: self.tracker.clone(), 21 | } 22 | } 23 | } 24 | 25 | impl Drop for SnapshotNonce { 26 | fn drop(&mut self) { 27 | self.tracker.close(self.instant); 28 | } 29 | } 30 | 31 | impl SnapshotNonce { 32 | pub fn new(instant: Instant, tracker: SnapshotTracker) -> Self { 33 | tracker.open(instant); 34 | Self { instant, tracker } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/snapshot_tracker.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use crate::Instant; 6 | use dashmap::DashMap; 7 | use std::sync::{atomic::AtomicU64, Arc, RwLock}; 8 | 9 | /// Keeps track of open snapshots 10 | #[allow(clippy::module_name_repetitions)] 11 | pub struct SnapshotTrackerInner { 12 | // TODO: maybe use rustc_hash or ahash 13 | pub(crate) data: DashMap, 14 | 15 | #[doc(hidden)] 16 | pub(crate) freed_count: AtomicU64, 17 | safety_gap: u64, 18 | 19 | #[doc(hidden)] 20 | pub(crate) lowest_freed_instant: RwLock, 21 | } 22 | 23 | #[derive(Clone, Default)] 24 | pub struct SnapshotTracker(Arc); 25 | 26 | impl std::ops::Deref for SnapshotTracker { 27 | type Target = SnapshotTrackerInner; 28 | 29 | fn deref(&self) -> &Self::Target { 30 | &self.0 31 | } 32 | } 33 | 34 | impl Default for SnapshotTrackerInner { 35 | fn default() -> Self { 36 | Self { 37 | data: DashMap::default(), 38 | safety_gap: 50, 39 | freed_count: AtomicU64::default(), 40 | lowest_freed_instant: RwLock::default(), 41 | } 42 | } 43 | } 44 | 45 | impl SnapshotTrackerInner { 46 | pub fn open(&self, seqno: Instant) { 47 | log::trace!("open snapshot {seqno}"); 48 | 49 | self.data 50 | .entry(seqno) 51 | .and_modify(|x| { 52 | *x += 1; 53 | }) 54 | .or_insert(1); 55 | } 56 | 57 | pub fn close(&self, seqno: Instant) { 58 | log::trace!("close snapshot {seqno}"); 59 | 60 | self.data.alter(&seqno, |_, v| v.saturating_sub(1)); 61 | 62 | let freed = self 63 | .freed_count 64 | .fetch_add(1, std::sync::atomic::Ordering::Relaxed) 65 | + 1; 66 | 67 | if (freed % self.safety_gap) == 0 { 68 | self.gc(seqno); 69 | } 70 | } 71 | 72 | pub fn get_seqno_safe_to_gc(&self) -> Instant { 73 | *self.lowest_freed_instant.read().expect("lock is poisoned") 74 | } 75 | 76 | fn gc(&self, watermark: Instant) { 77 | log::trace!("snapshot gc, watermark={watermark}"); 78 | 79 | let mut lock = self.lowest_freed_instant.write().expect("lock is poisoned"); 80 | 81 | let seqno_threshold = watermark.saturating_sub(self.safety_gap); 82 | 83 | let mut lowest_retained = 0; 84 | 85 | self.data.retain(|&k, v| { 86 | let should_be_retained = *v > 0 || k > seqno_threshold; 87 | 88 | if should_be_retained { 89 | lowest_retained = match lowest_retained { 90 | 0 => k, 91 | lo => lo.min(k), 92 | }; 93 | } 94 | 95 | should_be_retained 96 | }); 97 | 98 | log::trace!("lowest retained snapshot={lowest_retained}"); 99 | 100 | *lock = match *lock { 101 | 0 => lowest_retained.saturating_sub(1), 102 | lo => lo.max(lowest_retained.saturating_sub(1)), 103 | }; 104 | 105 | log::trace!("gc threshold now {}", *lock); 106 | } 107 | } 108 | 109 | #[cfg(test)] 110 | mod tests { 111 | use super::*; 112 | use test_log::test; 113 | 114 | #[test] 115 | #[allow(clippy::field_reassign_with_default)] 116 | fn seqno_tracker_one_shot() { 117 | let mut map = SnapshotTrackerInner::default(); 118 | map.safety_gap = 5; 119 | 120 | map.open(1); 121 | map.close(1); 122 | 123 | assert_eq!(map.get_seqno_safe_to_gc(), 0); 124 | } 125 | 126 | #[test] 127 | #[allow(clippy::field_reassign_with_default)] 128 | fn seqno_tracker_reverse_order() { 129 | let mut map = SnapshotTrackerInner::default(); 130 | map.safety_gap = 5; 131 | 132 | map.open(1); 133 | map.open(2); 134 | map.open(3); 135 | map.open(4); 136 | map.open(5); 137 | map.open(6); 138 | map.open(7); 139 | map.open(8); 140 | map.open(9); 141 | map.open(10); 142 | 143 | map.close(10); 144 | map.close(9); 145 | map.close(8); 146 | map.close(7); 147 | map.close(6); 148 | map.close(5); 149 | map.close(4); 150 | map.close(3); 151 | map.close(2); 152 | map.close(1); 153 | 154 | map.open(11); 155 | map.close(11); 156 | map.gc(11); 157 | 158 | assert_eq!(map.get_seqno_safe_to_gc(), 6); 159 | } 160 | 161 | #[test] 162 | #[allow(clippy::field_reassign_with_default)] 163 | fn seqno_tracker_simple_2() { 164 | let mut map = SnapshotTrackerInner::default(); 165 | map.safety_gap = 5; 166 | 167 | map.open(1); 168 | assert_eq!(map.get_seqno_safe_to_gc(), 0); 169 | 170 | map.open(2); 171 | assert_eq!(map.get_seqno_safe_to_gc(), 0); 172 | 173 | map.open(3); 174 | assert_eq!(map.get_seqno_safe_to_gc(), 0); 175 | 176 | map.open(4); 177 | assert_eq!(map.get_seqno_safe_to_gc(), 0); 178 | 179 | map.open(5); 180 | assert_eq!(map.get_seqno_safe_to_gc(), 0); 181 | 182 | map.open(6); 183 | assert_eq!(map.get_seqno_safe_to_gc(), 0); 184 | 185 | map.close(1); 186 | assert_eq!(map.get_seqno_safe_to_gc(), 0); 187 | 188 | map.close(2); 189 | assert_eq!(map.get_seqno_safe_to_gc(), 0); 190 | 191 | map.close(3); 192 | assert_eq!(map.get_seqno_safe_to_gc(), 0); 193 | 194 | map.close(4); 195 | assert_eq!(map.get_seqno_safe_to_gc(), 0); 196 | 197 | map.close(5); 198 | assert_eq!(map.get_seqno_safe_to_gc(), 0); 199 | 200 | map.close(6); 201 | map.gc(6); 202 | assert_eq!(map.get_seqno_safe_to_gc(), 1); 203 | 204 | map.open(7); 205 | map.close(7); 206 | map.gc(7); 207 | assert_eq!(map.get_seqno_safe_to_gc(), 2); 208 | 209 | map.open(8); 210 | map.open(9); 211 | map.close(9); 212 | map.gc(9); 213 | assert_eq!(map.get_seqno_safe_to_gc(), 4); 214 | 215 | map.close(8); 216 | map.gc(8); 217 | assert_eq!(map.get_seqno_safe_to_gc(), 4); 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /src/stats.rs: -------------------------------------------------------------------------------- 1 | use std::sync::atomic::{AtomicU64, AtomicUsize}; 2 | 3 | /// Ephemeral, runtime stats 4 | #[derive(Default)] 5 | pub struct Stats { 6 | /// Active compaction conter 7 | pub(crate) active_compaction_count: AtomicUsize, 8 | 9 | /// Time spent in compactions (in µs) 10 | pub(crate) time_compacting: AtomicU64, 11 | 12 | /// Time spent in garbage collection (in µs) 13 | pub(crate) time_gc: AtomicU64, 14 | 15 | /// Number of created memtable flush tasks 16 | pub(crate) flushes_enqueued: AtomicUsize, 17 | 18 | /// Number of completed memtable flushes 19 | pub(crate) flushes_completed: AtomicUsize, 20 | 21 | /// Number of completed compactions 22 | pub(crate) compactions_completed: AtomicUsize, 23 | } 24 | 25 | impl Stats { 26 | pub fn outstanding_flushes(&self) -> usize { 27 | self.flushes_enqueued 28 | .load(std::sync::atomic::Ordering::Relaxed) 29 | - self 30 | .flushes_completed 31 | .load(std::sync::atomic::Ordering::Relaxed) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/tracked_snapshot.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use crate::snapshot_nonce::SnapshotNonce; 6 | 7 | /// A snapshot captures a read-only point-in-time view of the tree at the time the snapshot was created 8 | /// 9 | /// As long as the snapshot is open, old versions of objects will not be evicted as to 10 | /// keep the snapshot consistent. Thus, snapshots should only be kept around for as little as possible. 11 | /// 12 | /// Snapshots do not persist across restarts. 13 | pub struct TrackedSnapshot { 14 | inner: lsm_tree::Snapshot, 15 | 16 | #[allow(unused)] 17 | nonce: SnapshotNonce, 18 | } 19 | 20 | impl std::ops::Deref for TrackedSnapshot { 21 | type Target = lsm_tree::Snapshot; 22 | 23 | fn deref(&self) -> &Self::Target { 24 | &self.inner 25 | } 26 | } 27 | 28 | impl TrackedSnapshot { 29 | pub(crate) fn new(snapshot: lsm_tree::Snapshot, nonce: SnapshotNonce) -> Self { 30 | Self { 31 | inner: snapshot, 32 | nonce, 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/tx/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | pub mod keyspace; 6 | pub mod partition; 7 | 8 | #[allow(clippy::module_name_repetitions)] 9 | pub mod read_tx; 10 | 11 | #[allow(clippy::module_name_repetitions)] 12 | pub mod write_tx; 13 | 14 | #[cfg(feature = "ssi_tx")] 15 | mod conflict_manager; 16 | 17 | #[cfg(feature = "ssi_tx")] 18 | mod oracle; 19 | 20 | pub mod write; 21 | -------------------------------------------------------------------------------- /src/tx/oracle.rs: -------------------------------------------------------------------------------- 1 | use crate::snapshot_tracker::SnapshotTracker; 2 | use crate::Instant; 3 | 4 | use super::conflict_manager::ConflictManager; 5 | use lsm_tree::SequenceNumberCounter; 6 | use std::collections::BTreeMap; 7 | use std::fmt; 8 | use std::sync::{Mutex, MutexGuard, PoisonError}; 9 | 10 | pub enum CommitOutcome { 11 | Ok, 12 | Aborted(E), 13 | Conflicted, 14 | } 15 | 16 | pub struct Oracle { 17 | pub(super) write_serialize_lock: Mutex>, 18 | pub(super) seqno: SequenceNumberCounter, 19 | pub(super) snapshot_tracker: SnapshotTracker, 20 | } 21 | 22 | impl Oracle { 23 | #[allow(clippy::nursery)] 24 | pub(super) fn with_commit Result<(), E>>( 25 | &self, 26 | instant: Instant, 27 | conflict_checker: ConflictManager, 28 | f: F, 29 | ) -> crate::Result> { 30 | let mut committed_txns = self 31 | .write_serialize_lock 32 | .lock() 33 | .map_err(|_| crate::Error::Poisoned)?; 34 | 35 | // If the committed_txn.ts is less than Instant that implies that the 36 | // committed_txn finished before the current transaction started. 37 | // We don't need to check for conflict in that case. 38 | // This change assumes linearizability. Lack of linearizability could 39 | // cause the read ts of a new txn to be lower than the commit ts of 40 | // a txn before it. 41 | let conflicted = 42 | committed_txns 43 | .range((instant + 1)..) 44 | .any(|(_ts, other_conflict_checker)| { 45 | conflict_checker.has_conflict(other_conflict_checker) 46 | }); 47 | 48 | self.snapshot_tracker.close(instant); 49 | let safe_to_gc = self.snapshot_tracker.get_seqno_safe_to_gc(); 50 | committed_txns.retain(|ts, _| *ts > safe_to_gc); 51 | 52 | if conflicted { 53 | return Ok(CommitOutcome::Conflicted); 54 | } 55 | 56 | if let Err(e) = f() { 57 | return Ok(CommitOutcome::Aborted(e)); 58 | } 59 | 60 | committed_txns.insert(self.seqno.get(), conflict_checker); 61 | 62 | Ok(CommitOutcome::Ok) 63 | } 64 | 65 | pub(super) fn write_serialize_lock( 66 | &self, 67 | ) -> crate::Result>> { 68 | self.write_serialize_lock 69 | .lock() 70 | .map_err(|_| crate::Error::Poisoned) 71 | } 72 | } 73 | 74 | #[cfg(test)] 75 | mod tests { 76 | use crate::{Config, PartitionCreateOptions, TxKeyspace, TxPartitionHandle}; 77 | 78 | #[allow(clippy::unwrap_used)] 79 | #[test] 80 | fn oracle_committed_txns_does_not_leak() -> crate::Result<()> { 81 | let tmpdir = tempfile::tempdir()?; 82 | let ks = Config::new(tmpdir.path()).open_transactional()?; 83 | 84 | let part = ks.open_partition("foo", PartitionCreateOptions::default())?; 85 | 86 | for _ in 0..250 { 87 | run_tx(&ks, &part).unwrap(); 88 | } 89 | 90 | assert!(dbg!(ks.oracle.write_serialize_lock.lock().unwrap().len()) < 200); 91 | 92 | for _ in 0..200 { 93 | run_tx(&ks, &part).unwrap(); 94 | } 95 | 96 | assert!(dbg!(ks.oracle.write_serialize_lock.lock().unwrap().len()) < 200); 97 | 98 | Ok(()) 99 | } 100 | 101 | fn run_tx(ks: &TxKeyspace, part: &TxPartitionHandle) -> Result<(), Box> { 102 | let mut tx1 = ks.write_tx()?; 103 | let mut tx2 = ks.write_tx()?; 104 | tx1.insert(part, "hello", "world"); 105 | 106 | tx1.commit()??; 107 | assert!(part.contains_key("hello")?); 108 | 109 | _ = tx2.get(part, "hello")?; 110 | 111 | tx2.insert(part, "hello", "world2"); 112 | assert!(tx2.commit()?.is_err()); // intended to conflict 113 | 114 | Ok(()) 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/tx/write_tx.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | #[cfg(all(feature = "single_writer_tx", feature = "ssi_tx"))] 6 | compile_error!("Either single_writer_tx or ssi_tx can be enabled at once"); 7 | 8 | #[cfg(feature = "single_writer_tx")] 9 | pub use super::write::single_writer::WriteTransaction; 10 | 11 | #[cfg(feature = "ssi_tx")] 12 | pub use super::write::ssi::WriteTransaction; 13 | 14 | // TODO: 15 | // use https://github.com/rust-lang/rust/issues/43781 16 | // when stable 17 | // 18 | // #[doc(cfg(feature = "single_writer_tx"))] 19 | // 20 | // #[doc(cfg(feature = "ssi_tx"))] 21 | -------------------------------------------------------------------------------- /src/version.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use byteorder::WriteBytesExt; 6 | 7 | /// Disk format version 8 | #[derive(Copy, Clone, Debug, Eq, PartialEq)] 9 | pub enum Version { 10 | /// Version for 1.x.x releases 11 | V1, 12 | 13 | /// Version for 2.x.x releases 14 | V2, 15 | } 16 | 17 | impl std::fmt::Display for Version { 18 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 19 | write!(f, "{}", u8::from(*self)) 20 | } 21 | } 22 | 23 | impl From for u8 { 24 | fn from(value: Version) -> Self { 25 | match value { 26 | Version::V1 => 1, 27 | Version::V2 => 2, 28 | } 29 | } 30 | } 31 | 32 | impl TryFrom for Version { 33 | type Error = (); 34 | 35 | fn try_from(value: u8) -> Result { 36 | match value { 37 | 1 => Ok(Self::V1), 38 | 2 => Ok(Self::V2), 39 | _ => Err(()), 40 | } 41 | } 42 | } 43 | 44 | const MAGIC_BYTES: [u8; 3] = [b'F', b'J', b'L']; 45 | 46 | impl Version { 47 | pub(crate) fn parse_file_header(bytes: &[u8]) -> Option { 48 | let first_three = bytes.get(0..3)?; 49 | 50 | if first_three == MAGIC_BYTES { 51 | let version = *bytes.get(3)?; 52 | let version = Self::try_from(version).ok()?; 53 | 54 | Some(version) 55 | } else { 56 | None 57 | } 58 | } 59 | 60 | pub(crate) fn write_file_header( 61 | self, 62 | writer: &mut W, 63 | ) -> std::io::Result<()> { 64 | writer.write_all(&MAGIC_BYTES)?; 65 | writer.write_u8(u8::from(self))?; 66 | Ok(()) 67 | } 68 | } 69 | 70 | #[cfg(test)] 71 | mod tests { 72 | use super::*; 73 | use test_log::test; 74 | 75 | #[test] 76 | #[allow(clippy::expect_used)] 77 | pub fn version_serialize() -> crate::Result<()> { 78 | let mut bytes = vec![]; 79 | Version::V1.write_file_header(&mut bytes)?; 80 | assert_eq!(bytes, &[b'F', b'J', b'L', 1]); 81 | Ok(()) 82 | } 83 | 84 | #[test] 85 | #[allow(clippy::expect_used)] 86 | pub fn version_serialize_2() -> crate::Result<()> { 87 | let mut bytes = vec![]; 88 | Version::V2.write_file_header(&mut bytes)?; 89 | assert_eq!(bytes, &[b'F', b'J', b'L', 2]); 90 | Ok(()) 91 | } 92 | 93 | #[test] 94 | #[allow(clippy::expect_used)] 95 | pub fn version_deserialize_success() { 96 | let version = Version::parse_file_header(&[b'F', b'J', b'L', 1]); 97 | assert_eq!(version, Some(Version::V1)); 98 | } 99 | 100 | #[test] 101 | #[allow(clippy::expect_used)] 102 | pub fn version_deserialize_success_2() { 103 | let version = Version::parse_file_header(&[b'F', b'J', b'L', 2]); 104 | assert_eq!(version, Some(Version::V2)); 105 | } 106 | 107 | #[test] 108 | #[allow(clippy::expect_used)] 109 | pub fn version_deserialize_fail() { 110 | let version = Version::parse_file_header(&[b'F', b'J', b'X', 1]); 111 | assert!(version.is_none()); 112 | } 113 | 114 | #[test] 115 | #[allow(clippy::expect_used)] 116 | pub fn version_serde_round_trip() { 117 | let mut buf = vec![]; 118 | Version::V1.write_file_header(&mut buf).expect("can't fail"); 119 | 120 | let version = Version::parse_file_header(&buf); 121 | assert_eq!(version, Some(Version::V1)); 122 | } 123 | 124 | #[test] 125 | #[allow(clippy::expect_used)] 126 | pub fn version_len() { 127 | let mut buf = vec![]; 128 | Version::V1.write_file_header(&mut buf).expect("can't fail"); 129 | assert_eq!(4, buf.len()); 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/write_buffer_manager.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, fjall-rs 2 | // This source code is licensed under both the Apache 2.0 and MIT License 3 | // (found in the LICENSE-* files in the repository) 4 | 5 | use std::sync::{atomic::AtomicU64, Arc}; 6 | 7 | /// Keeps track of the size of the keyspace's write buffer 8 | #[derive(Clone, Default, Debug)] 9 | pub struct WriteBufferManager(Arc); 10 | 11 | impl std::ops::Deref for WriteBufferManager { 12 | type Target = AtomicU64; 13 | 14 | fn deref(&self) -> &Self::Target { 15 | &self.0 16 | } 17 | } 18 | 19 | impl WriteBufferManager { 20 | pub fn get(&self) -> u64 { 21 | self.load(std::sync::atomic::Ordering::Acquire) 22 | } 23 | 24 | // Adds some bytes to the write buffer counter. 25 | // 26 | // Returns the counter *after* incrementing. 27 | pub fn allocate(&self, n: u64) -> u64 { 28 | let before = self.fetch_add(n, std::sync::atomic::Ordering::AcqRel); 29 | before + n 30 | } 31 | 32 | // Frees some bytes from the write buffer counter. 33 | // 34 | // Returns the counter *after* decrementing. 35 | pub fn free(&self, n: u64) -> u64 { 36 | use std::sync::atomic::Ordering::{Acquire, SeqCst}; 37 | 38 | loop { 39 | let now = self.load(Acquire); 40 | let subbed = now.saturating_sub(n); 41 | 42 | if self.compare_exchange(now, subbed, SeqCst, SeqCst).is_ok() { 43 | return subbed; 44 | } 45 | } 46 | } 47 | } 48 | 49 | #[cfg(test)] 50 | mod tests { 51 | use super::*; 52 | use test_log::test; 53 | 54 | #[test] 55 | fn write_buffer_manager_increment() { 56 | let m = WriteBufferManager::default(); 57 | m.allocate(5); 58 | assert_eq!(m.get(), 5); 59 | 60 | m.allocate(15); 61 | assert_eq!(m.get(), 20); 62 | } 63 | 64 | #[test] 65 | fn write_buffer_manager_decrement() { 66 | let m = WriteBufferManager::default(); 67 | m.allocate(20); 68 | assert_eq!(m.get(), 20); 69 | 70 | m.free(5); 71 | assert_eq!(m.get(), 15); 72 | 73 | m.free(20); 74 | assert_eq!(m.get(), 0); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/journals/2/0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v1_keyspace/journals/2/0 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/journals/2/1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v1_keyspace/journals/2/1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/journals/2/2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v1_keyspace/journals/2/2 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/journals/2/3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v1_keyspace/journals/2/3 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/partitions/a/config: -------------------------------------------------------------------------------- 1 | FJLLCFG1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/partitions/a/levels: -------------------------------------------------------------------------------- 1 | FJLLLVL1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/partitions/a/segments/0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v1_keyspace/partitions/a/segments/0 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/partitions/a/version: -------------------------------------------------------------------------------- 1 | FJL -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/partitions/b/config: -------------------------------------------------------------------------------- 1 | FJLLCFG1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/partitions/b/levels: -------------------------------------------------------------------------------- 1 | FJLLLVL1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/partitions/b/segments/1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v1_keyspace/partitions/b/segments/1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/partitions/b/version: -------------------------------------------------------------------------------- 1 | FJL -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/partitions/c/config: -------------------------------------------------------------------------------- 1 | FJLLCFG1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/partitions/c/levels: -------------------------------------------------------------------------------- 1 | FJLLLVL1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/partitions/c/version: -------------------------------------------------------------------------------- 1 | FJL -------------------------------------------------------------------------------- /test_fixture/v1_keyspace/version: -------------------------------------------------------------------------------- 1 | FJL -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/journals/2/0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v1_keyspace_corrupt_journal/journals/2/0 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/journals/2/1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v1_keyspace_corrupt_journal/journals/2/1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/journals/2/2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v1_keyspace_corrupt_journal/journals/2/2 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/journals/2/3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v1_keyspace_corrupt_journal/journals/2/3 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/partitions/a/config: -------------------------------------------------------------------------------- 1 | FJLLCFG1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/partitions/a/levels: -------------------------------------------------------------------------------- 1 | FJLLLVL1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/partitions/a/segments/0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v1_keyspace_corrupt_journal/partitions/a/segments/0 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/partitions/a/version: -------------------------------------------------------------------------------- 1 | FJL -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/partitions/b/config: -------------------------------------------------------------------------------- 1 | FJLLCFG1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/partitions/b/levels: -------------------------------------------------------------------------------- 1 | FJLLLVL1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/partitions/b/segments/1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v1_keyspace_corrupt_journal/partitions/b/segments/1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/partitions/b/version: -------------------------------------------------------------------------------- 1 | FJL -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/partitions/c/config: -------------------------------------------------------------------------------- 1 | FJLLCFG1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/partitions/c/levels: -------------------------------------------------------------------------------- 1 | FJLLLVL1 -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/partitions/c/version: -------------------------------------------------------------------------------- 1 | FJL -------------------------------------------------------------------------------- /test_fixture/v1_keyspace_corrupt_journal/version: -------------------------------------------------------------------------------- 1 | FJL -------------------------------------------------------------------------------- /test_fixture/v2_keyspace/journals/2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v2_keyspace/journals/2 -------------------------------------------------------------------------------- /test_fixture/v2_keyspace/partitions/default1/config: -------------------------------------------------------------------------------- 1 | FJL 2 |  -------------------------------------------------------------------------------- /test_fixture/v2_keyspace/partitions/default1/levels: -------------------------------------------------------------------------------- 1 | LSM -------------------------------------------------------------------------------- /test_fixture/v2_keyspace/partitions/default1/manifest: -------------------------------------------------------------------------------- 1 | LSM -------------------------------------------------------------------------------- /test_fixture/v2_keyspace/partitions/default1/segments/0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v2_keyspace/partitions/default1/segments/0 -------------------------------------------------------------------------------- /test_fixture/v2_keyspace/partitions/default2/blobs/.vlog: -------------------------------------------------------------------------------- 1 | VLG -------------------------------------------------------------------------------- /test_fixture/v2_keyspace/partitions/default2/blobs/segments/0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v2_keyspace/partitions/default2/blobs/segments/0 -------------------------------------------------------------------------------- /test_fixture/v2_keyspace/partitions/default2/blobs/vlog_manifest: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /test_fixture/v2_keyspace/partitions/default2/config: -------------------------------------------------------------------------------- 1 | FJL 2 |  -------------------------------------------------------------------------------- /test_fixture/v2_keyspace/partitions/default2/levels: -------------------------------------------------------------------------------- 1 | LSM -------------------------------------------------------------------------------- /test_fixture/v2_keyspace/partitions/default2/manifest: -------------------------------------------------------------------------------- 1 | LSM -------------------------------------------------------------------------------- /test_fixture/v2_keyspace/partitions/default2/segments/0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v2_keyspace/partitions/default2/segments/0 -------------------------------------------------------------------------------- /test_fixture/v2_keyspace/version: -------------------------------------------------------------------------------- 1 | FJL -------------------------------------------------------------------------------- /test_fixture/v2_keyspace_corrupt_journal/journals/2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v2_keyspace_corrupt_journal/journals/2 -------------------------------------------------------------------------------- /test_fixture/v2_keyspace_corrupt_journal/partitions/default1/config: -------------------------------------------------------------------------------- 1 | FJL 2 |  -------------------------------------------------------------------------------- /test_fixture/v2_keyspace_corrupt_journal/partitions/default1/levels: -------------------------------------------------------------------------------- 1 | LSM -------------------------------------------------------------------------------- /test_fixture/v2_keyspace_corrupt_journal/partitions/default1/manifest: -------------------------------------------------------------------------------- 1 | LSM -------------------------------------------------------------------------------- /test_fixture/v2_keyspace_corrupt_journal/partitions/default1/segments/0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v2_keyspace_corrupt_journal/partitions/default1/segments/0 -------------------------------------------------------------------------------- /test_fixture/v2_keyspace_corrupt_journal/partitions/default2/blobs/.vlog: -------------------------------------------------------------------------------- 1 | VLG -------------------------------------------------------------------------------- /test_fixture/v2_keyspace_corrupt_journal/partitions/default2/blobs/segments/0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v2_keyspace_corrupt_journal/partitions/default2/blobs/segments/0 -------------------------------------------------------------------------------- /test_fixture/v2_keyspace_corrupt_journal/partitions/default2/blobs/vlog_manifest: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /test_fixture/v2_keyspace_corrupt_journal/partitions/default2/config: -------------------------------------------------------------------------------- 1 | FJL 2 |  -------------------------------------------------------------------------------- /test_fixture/v2_keyspace_corrupt_journal/partitions/default2/levels: -------------------------------------------------------------------------------- 1 | LSM -------------------------------------------------------------------------------- /test_fixture/v2_keyspace_corrupt_journal/partitions/default2/manifest: -------------------------------------------------------------------------------- 1 | LSM -------------------------------------------------------------------------------- /test_fixture/v2_keyspace_corrupt_journal/partitions/default2/segments/0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v2_keyspace_corrupt_journal/partitions/default2/segments/0 -------------------------------------------------------------------------------- /test_fixture/v2_keyspace_corrupt_journal/version: -------------------------------------------------------------------------------- 1 | FJL -------------------------------------------------------------------------------- /test_fixture/v2_sealed_journal_shenanigans/journals/0.sealed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v2_sealed_journal_shenanigans/journals/0.sealed -------------------------------------------------------------------------------- /test_fixture/v2_sealed_journal_shenanigans/journals/1.sealed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v2_sealed_journal_shenanigans/journals/1.sealed -------------------------------------------------------------------------------- /test_fixture/v2_sealed_journal_shenanigans/journals/2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v2_sealed_journal_shenanigans/journals/2 -------------------------------------------------------------------------------- /test_fixture/v2_sealed_journal_shenanigans/journals/3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v2_sealed_journal_shenanigans/journals/3 -------------------------------------------------------------------------------- /test_fixture/v2_sealed_journal_shenanigans/partitions/default/config: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjall-rs/fjall/b87e01649f8dcbb62bcb5c0cd30630467bbccc5b/test_fixture/v2_sealed_journal_shenanigans/partitions/default/config -------------------------------------------------------------------------------- /test_fixture/v2_sealed_journal_shenanigans/partitions/default/levels: -------------------------------------------------------------------------------- 1 | LSM -------------------------------------------------------------------------------- /test_fixture/v2_sealed_journal_shenanigans/partitions/default/manifest: -------------------------------------------------------------------------------- 1 | LSM -------------------------------------------------------------------------------- /test_fixture/v2_sealed_journal_shenanigans/version: -------------------------------------------------------------------------------- 1 | FJL -------------------------------------------------------------------------------- /tests/batch.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, KvSeparationOptions, PartitionCreateOptions}; 2 | use test_log::test; 3 | 4 | #[test] 5 | fn batch_simple() -> fjall::Result<()> { 6 | let folder = tempfile::tempdir()?; 7 | 8 | let keyspace = Config::new(folder).open()?; 9 | let partition = keyspace.open_partition("default", PartitionCreateOptions::default())?; 10 | let mut batch = keyspace.batch(); 11 | 12 | assert_eq!(partition.len()?, 0); 13 | batch.insert(&partition, "1", "abc"); 14 | batch.insert(&partition, "3", "abc"); 15 | batch.insert(&partition, "5", "abc"); 16 | assert_eq!(partition.len()?, 0); 17 | 18 | batch.commit()?; 19 | assert_eq!(partition.len()?, 3); 20 | 21 | Ok(()) 22 | } 23 | 24 | #[test] 25 | fn blob_batch_simple() -> fjall::Result<()> { 26 | let folder = tempfile::tempdir()?; 27 | 28 | let keyspace = Config::new(folder).open()?; 29 | let partition = keyspace.open_partition( 30 | "default", 31 | PartitionCreateOptions::default().with_kv_separation(KvSeparationOptions::default()), 32 | )?; 33 | 34 | let blob = "oxygen".repeat(128_000); 35 | 36 | let mut batch = keyspace.batch(); 37 | 38 | assert_eq!(partition.len()?, 0); 39 | batch.insert(&partition, "1", &blob); 40 | batch.insert(&partition, "3", "abc"); 41 | batch.insert(&partition, "5", "abc"); 42 | assert_eq!(partition.len()?, 0); 43 | 44 | batch.commit()?; 45 | assert_eq!(partition.len()?, 3); 46 | 47 | assert_eq!(&*partition.get("1")?.unwrap(), blob.as_bytes()); 48 | 49 | Ok(()) 50 | } 51 | -------------------------------------------------------------------------------- /tests/blob_kv_simple.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, KvSeparationOptions, PartitionCreateOptions}; 2 | use lsm_tree::AbstractTree; 3 | use test_log::test; 4 | 5 | #[test] 6 | fn blob_kv_simple() -> fjall::Result<()> { 7 | let folder = tempfile::tempdir()?; 8 | 9 | let keyspace = Config::new(folder).open()?; 10 | let partition = keyspace.open_partition( 11 | "default", 12 | PartitionCreateOptions::default().with_kv_separation(KvSeparationOptions::default()), 13 | )?; 14 | 15 | assert_eq!(partition.len()?, 0); 16 | partition.insert("1", "oxygen".repeat(1_000_000))?; 17 | partition.insert("3", "abc")?; 18 | partition.insert("5", "abc")?; 19 | assert_eq!(partition.len()?, 3); 20 | 21 | partition.rotate_memtable_and_wait()?; 22 | 23 | if let fjall::AnyTree::Blob(tree) = &partition.tree { 24 | assert!(tree.index.disk_space() < 200); 25 | 26 | // NOTE: The data is compressed quite well, so it's way less than 1M 27 | assert!(tree.disk_space() > 5_000); 28 | 29 | assert!(tree.blobs.manifest.disk_space_used() > 5_000); 30 | assert_eq!(1, tree.blobs.segment_count()); 31 | } else { 32 | panic!("nope"); 33 | } 34 | 35 | Ok(()) 36 | } 37 | -------------------------------------------------------------------------------- /tests/gc_watermark_pull_up.rs: -------------------------------------------------------------------------------- 1 | use fjall::Config; 2 | use std::time::Duration; 3 | use test_log::test; 4 | 5 | #[test] 6 | fn keyspace_recover_empty() -> fjall::Result<()> { 7 | let folder = tempfile::tempdir()?; 8 | 9 | let keyspace = Config::new(&folder).open()?; 10 | let partition = keyspace.open_partition("default", Default::default())?; 11 | 12 | for _ in 0..10_000 { 13 | partition.insert("a", "a")?; 14 | } 15 | 16 | // NOTE: Wait for monitor thread tick to kick in 17 | std::thread::sleep(Duration::from_secs(1)); 18 | 19 | assert!(keyspace.snapshot_tracker.get_seqno_safe_to_gc() > 0); 20 | 21 | Ok(()) 22 | } 23 | -------------------------------------------------------------------------------- /tests/journal_large_value.rs: -------------------------------------------------------------------------------- 1 | // Regression test for https://github.com/fjall-rs/fjall/issues/68 2 | 3 | use fjall::{Config, KvSeparationOptions, PartitionCreateOptions}; 4 | 5 | #[test_log::test] 6 | fn journal_recover_large_value() -> fjall::Result<()> { 7 | let folder = tempfile::tempdir()?; 8 | 9 | let large_value = "a".repeat(128_000); 10 | 11 | { 12 | let keyspace = Config::new(&folder).open()?; 13 | let partition = keyspace.open_partition("default", PartitionCreateOptions::default())?; 14 | partition.insert("a", &large_value)?; 15 | partition.insert("b", "b")?; 16 | } 17 | 18 | { 19 | let keyspace = Config::new(&folder).open()?; 20 | let partition = keyspace.open_partition("default", PartitionCreateOptions::default())?; 21 | assert_eq!(large_value.as_bytes(), &*partition.get("a")?.unwrap()); 22 | assert_eq!(b"b", &*partition.get("b")?.unwrap()); 23 | } 24 | 25 | Ok(()) 26 | } 27 | 28 | #[test_log::test] 29 | fn journal_recover_large_value_blob() -> fjall::Result<()> { 30 | let folder = tempfile::tempdir()?; 31 | 32 | let large_value = "a".repeat(128_000); 33 | 34 | { 35 | let keyspace = Config::new(&folder).open()?; 36 | let partition = keyspace.open_partition( 37 | "default", 38 | PartitionCreateOptions::default().with_kv_separation(KvSeparationOptions::default()), 39 | )?; 40 | partition.insert("a", &large_value)?; 41 | partition.insert("b", "b")?; 42 | } 43 | 44 | { 45 | let keyspace = Config::new(&folder).open()?; 46 | let partition = keyspace.open_partition( 47 | "default", 48 | PartitionCreateOptions::default().with_kv_separation(KvSeparationOptions::default()), 49 | )?; 50 | assert_eq!(large_value.as_bytes(), &*partition.get("a")?.unwrap()); 51 | assert_eq!(b"b", &*partition.get("b")?.unwrap()); 52 | } 53 | 54 | Ok(()) 55 | } 56 | -------------------------------------------------------------------------------- /tests/keyspace_drop.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "__internal_whitebox")] 2 | #[test_log::test] 3 | fn whitebox_keyspace_drop() -> fjall::Result<()> { 4 | use fjall::Config; 5 | 6 | { 7 | let folder = tempfile::tempdir()?; 8 | 9 | assert_eq!(0, fjall::drop::load_drop_counter()); 10 | let keyspace = Config::new(folder).open()?; 11 | assert_eq!(5, fjall::drop::load_drop_counter()); 12 | 13 | drop(keyspace); 14 | assert_eq!(0, fjall::drop::load_drop_counter()); 15 | } 16 | 17 | { 18 | let folder = tempfile::tempdir()?; 19 | 20 | assert_eq!(0, fjall::drop::load_drop_counter()); 21 | let keyspace = Config::new(folder).open()?; 22 | assert_eq!(5, fjall::drop::load_drop_counter()); 23 | 24 | let partition = keyspace.open_partition("default", Default::default())?; 25 | assert_eq!(6, fjall::drop::load_drop_counter()); 26 | 27 | drop(partition); 28 | drop(keyspace); 29 | assert_eq!(0, fjall::drop::load_drop_counter()); 30 | } 31 | 32 | { 33 | let folder = tempfile::tempdir()?; 34 | 35 | assert_eq!(0, fjall::drop::load_drop_counter()); 36 | let keyspace = Config::new(folder).open()?; 37 | assert_eq!(5, fjall::drop::load_drop_counter()); 38 | 39 | let _partition = keyspace.open_partition("default", Default::default())?; 40 | assert_eq!(6, fjall::drop::load_drop_counter()); 41 | 42 | let _partition2 = keyspace.open_partition("different", Default::default())?; 43 | assert_eq!(7, fjall::drop::load_drop_counter()); 44 | } 45 | 46 | assert_eq!(0, fjall::drop::load_drop_counter()); 47 | 48 | Ok(()) 49 | } 50 | 51 | #[cfg(feature = "__internal_whitebox")] 52 | #[test_log::test] 53 | fn whitebox_keyspace_drop_2() -> fjall::Result<()> { 54 | use fjall::{Config, PartitionCreateOptions}; 55 | 56 | let folder = tempfile::tempdir()?; 57 | 58 | { 59 | let keyspace = Config::new(&folder).open()?; 60 | 61 | let partition = keyspace.open_partition("tree", PartitionCreateOptions::default())?; 62 | let partition2 = keyspace.open_partition("tree1", PartitionCreateOptions::default())?; 63 | 64 | partition.insert("a", "a")?; 65 | partition2.insert("b", "b")?; 66 | 67 | partition.rotate_memtable_and_wait()?; 68 | } 69 | 70 | assert_eq!(0, fjall::drop::load_drop_counter()); 71 | 72 | Ok(()) 73 | } 74 | -------------------------------------------------------------------------------- /tests/keyspace_recover_empty.rs: -------------------------------------------------------------------------------- 1 | use fjall::Config; 2 | use test_log::test; 3 | 4 | #[test] 5 | fn keyspace_recover_empty() -> fjall::Result<()> { 6 | let folder = tempfile::tempdir()?; 7 | 8 | for _ in 0..10 { 9 | let _keyspace = Config::new(&folder).open()?; 10 | assert_eq!(0, _keyspace.partition_count()); 11 | } 12 | 13 | Ok(()) 14 | } 15 | -------------------------------------------------------------------------------- /tests/keyspace_v1_load_fixture.rs: -------------------------------------------------------------------------------- 1 | use fjall::Config; 2 | use test_log::test; 3 | 4 | #[test] 5 | fn keyspace_load_v1() -> fjall::Result<()> { 6 | let folder = "test_fixture/v1_keyspace"; 7 | 8 | let result = Config::new(folder).open(); 9 | 10 | matches!( 11 | result, 12 | Err(fjall::Error::InvalidVersion(Some(fjall::Version::V1))) 13 | ); 14 | 15 | Ok(()) 16 | } 17 | 18 | #[test] 19 | fn keyspace_load_v1_corrupt_journal() -> fjall::Result<()> { 20 | let folder = "test_fixture/v1_keyspace_corrupt_journal"; 21 | 22 | let result = Config::new(folder).open(); 23 | 24 | matches!( 25 | result, 26 | Err(fjall::Error::InvalidVersion(Some(fjall::Version::V1))) 27 | ); 28 | 29 | Ok(()) 30 | } 31 | -------------------------------------------------------------------------------- /tests/keyspace_v2_load_fixture.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, KvSeparationOptions, PartitionCreateOptions, RecoveryError}; 2 | use test_log::test; 3 | 4 | #[test] 5 | fn keyspace_load_v2() -> fjall::Result<()> { 6 | let folder = "test_fixture/v2_keyspace"; 7 | 8 | let keyspace = Config::new(folder).open()?; 9 | let tree1 = keyspace.open_partition("default1", PartitionCreateOptions::default())?; 10 | let tree2 = keyspace.open_partition( 11 | "default2", 12 | PartitionCreateOptions::default().with_kv_separation(KvSeparationOptions::default()), 13 | )?; 14 | 15 | assert_eq!(6, tree1.len()?); 16 | assert_eq!(6, tree2.len()?); 17 | 18 | Ok(()) 19 | } 20 | 21 | #[test] 22 | fn keyspace_load_v2_corrupt_journal() -> fjall::Result<()> { 23 | let folder = "test_fixture/v2_keyspace_corrupt_journal"; 24 | 25 | let result = Config::new(folder).open(); 26 | matches!( 27 | result, 28 | Err(fjall::Error::JournalRecovery( 29 | RecoveryError::ChecksumMismatch 30 | )), 31 | ); 32 | 33 | Ok(()) 34 | } 35 | -------------------------------------------------------------------------------- /tests/memtable_recover.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, PartitionCreateOptions}; 2 | use test_log::test; 3 | 4 | const ITEM_COUNT: usize = 10_000; 5 | 6 | #[test] 7 | fn reload_with_memtable() -> fjall::Result<()> { 8 | let folder = tempfile::tempdir()?; 9 | 10 | // NOTE: clippy bug 11 | #[allow(unused_assignments)] 12 | { 13 | let keyspace = Config::new(&folder).open()?; 14 | let tree = keyspace.open_partition("default", PartitionCreateOptions::default())?; 15 | 16 | for x in 0..ITEM_COUNT as u64 { 17 | let key = x.to_be_bytes(); 18 | let value = nanoid::nanoid!(); 19 | tree.insert(key, value.as_bytes())?; 20 | } 21 | 22 | for x in 0..ITEM_COUNT as u64 { 23 | let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); 24 | let value = nanoid::nanoid!(); 25 | tree.insert(key, value.as_bytes())?; 26 | } 27 | 28 | assert_eq!(tree.len()?, ITEM_COUNT * 2); 29 | assert_eq!(tree.iter().flatten().count(), ITEM_COUNT * 2); 30 | assert_eq!(tree.iter().rev().flatten().count(), ITEM_COUNT * 2); 31 | } 32 | 33 | for _ in 0..5 { 34 | let keyspace = Config::new(&folder).open()?; 35 | let tree = keyspace.open_partition("default", PartitionCreateOptions::default())?; 36 | 37 | assert_eq!(tree.len()?, ITEM_COUNT * 2); 38 | assert_eq!(tree.iter().flatten().count(), ITEM_COUNT * 2); 39 | assert_eq!(tree.iter().rev().flatten().count(), ITEM_COUNT * 2); 40 | } 41 | 42 | Ok(()) 43 | } 44 | -------------------------------------------------------------------------------- /tests/partition_delete.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, PartitionCreateOptions}; 2 | use test_log::test; 3 | 4 | const ITEM_COUNT: usize = 10; 5 | 6 | #[test] 7 | fn partition_delete() -> fjall::Result<()> { 8 | let folder = tempfile::tempdir()?; 9 | 10 | let path; 11 | 12 | // NOTE: clippy bug 13 | #[allow(unused_assignments)] 14 | { 15 | let keyspace = Config::new(&folder).open()?; 16 | 17 | let tree = keyspace.open_partition("default", PartitionCreateOptions::default())?; 18 | path = tree.path().to_path_buf(); 19 | 20 | assert!(path.try_exists()?); 21 | 22 | for x in 0..ITEM_COUNT as u64 { 23 | let key = x.to_be_bytes(); 24 | let value = nanoid::nanoid!(); 25 | tree.insert(key, value.as_bytes())?; 26 | } 27 | 28 | for x in 0..ITEM_COUNT as u64 { 29 | let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); 30 | let value = nanoid::nanoid!(); 31 | tree.insert(key, value.as_bytes())?; 32 | } 33 | 34 | assert_eq!(tree.len()?, ITEM_COUNT * 2); 35 | assert_eq!(tree.iter().flatten().count(), ITEM_COUNT * 2); 36 | assert_eq!(tree.iter().rev().flatten().count(), ITEM_COUNT * 2); 37 | } 38 | 39 | for _ in 0..10 { 40 | let keyspace = Config::new(&folder).open()?; 41 | 42 | let tree = keyspace.open_partition("default", PartitionCreateOptions::default())?; 43 | 44 | assert_eq!(tree.len()?, ITEM_COUNT * 2); 45 | assert_eq!(tree.iter().flatten().count(), ITEM_COUNT * 2); 46 | assert_eq!(tree.iter().rev().flatten().count(), ITEM_COUNT * 2); 47 | 48 | assert!(path.try_exists()?); 49 | } 50 | 51 | { 52 | let keyspace = Config::new(&folder).open()?; 53 | 54 | let tree = keyspace.open_partition("default", PartitionCreateOptions::default())?; 55 | 56 | assert!(path.try_exists()?); 57 | 58 | keyspace.delete_partition(tree)?; 59 | 60 | assert!(!path.try_exists()?); 61 | } 62 | 63 | { 64 | let _keyspace = Config::new(&folder).open()?; 65 | assert!(!path.try_exists()?); 66 | } 67 | 68 | Ok(()) 69 | } 70 | 71 | #[test] 72 | #[cfg(feature = "single_writer_tx")] 73 | fn tx_partition_delete() -> fjall::Result<()> { 74 | let folder = tempfile::tempdir()?; 75 | 76 | let path; 77 | 78 | // NOTE: clippy bug 79 | #[allow(unused_assignments)] 80 | { 81 | let keyspace = Config::new(&folder).open_transactional()?; 82 | 83 | let tree = keyspace.open_partition("default", PartitionCreateOptions::default())?; 84 | path = tree.path(); 85 | 86 | assert!(path.try_exists()?); 87 | 88 | for x in 0..ITEM_COUNT as u64 { 89 | let key = x.to_be_bytes(); 90 | let value = nanoid::nanoid!(); 91 | tree.insert(key, value.as_bytes())?; 92 | } 93 | 94 | for x in 0..ITEM_COUNT as u64 { 95 | let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); 96 | let value = nanoid::nanoid!(); 97 | tree.insert(key, value.as_bytes())?; 98 | } 99 | 100 | assert_eq!(keyspace.read_tx().len(&tree)?, ITEM_COUNT * 2); 101 | assert_eq!( 102 | keyspace.read_tx().iter(&tree).flatten().count(), 103 | ITEM_COUNT * 2, 104 | ); 105 | assert_eq!( 106 | keyspace.read_tx().iter(&tree).rev().flatten().count(), 107 | ITEM_COUNT * 2, 108 | ); 109 | } 110 | 111 | for _ in 0..5 { 112 | let keyspace = Config::new(&folder).open_transactional()?; 113 | 114 | let tree = keyspace.open_partition("default", PartitionCreateOptions::default())?; 115 | 116 | assert_eq!(keyspace.read_tx().len(&tree)?, ITEM_COUNT * 2); 117 | assert_eq!( 118 | keyspace.read_tx().iter(&tree).flatten().count(), 119 | ITEM_COUNT * 2, 120 | ); 121 | assert_eq!( 122 | keyspace.read_tx().iter(&tree).rev().flatten().count(), 123 | ITEM_COUNT * 2, 124 | ); 125 | 126 | assert!(path.try_exists()?); 127 | } 128 | 129 | { 130 | let keyspace = Config::new(&folder).open_transactional()?; 131 | 132 | let tree = keyspace.open_partition("default", PartitionCreateOptions::default())?; 133 | 134 | assert!(path.try_exists()?); 135 | 136 | keyspace.delete_partition(tree)?; 137 | 138 | assert!(!path.try_exists()?); 139 | } 140 | 141 | { 142 | let _keyspace = Config::new(&folder).open_transactional()?; 143 | assert!(!path.try_exists()?); 144 | } 145 | 146 | Ok(()) 147 | } 148 | 149 | #[test] 150 | fn partition_deletion_and_reopening_behavior() -> fjall::Result<()> { 151 | let partition_name = "default"; 152 | let folder = tempfile::tempdir()?; 153 | 154 | let partition_exists = || -> std::io::Result { 155 | folder 156 | .path() 157 | .join("partitions") 158 | .join(partition_name) 159 | .try_exists() 160 | }; 161 | 162 | let keyspace = Config::new(&folder).open()?; 163 | assert!(!partition_exists()?); 164 | 165 | let partition = keyspace.open_partition(partition_name, Default::default())?; 166 | assert!(partition_exists()?); 167 | 168 | keyspace.delete_partition(partition.clone())?; 169 | assert!(partition_exists()?); 170 | 171 | // NOTE: Partition is marked as deleted but still referenced, so it's not cleaned up 172 | assert!(matches!( 173 | keyspace.open_partition(partition_name, Default::default()), 174 | Err(fjall::Error::PartitionDeleted) 175 | )); 176 | assert!(partition_exists()?); 177 | 178 | // NOTE: Remove last handle, will drop partition folder, allowing us to recreate again 179 | drop(partition); 180 | assert!(!partition_exists()?); 181 | 182 | assert!(keyspace 183 | .open_partition("default", Default::default()) 184 | .is_ok()); 185 | assert!(partition_exists()?); 186 | 187 | Ok(()) 188 | } 189 | -------------------------------------------------------------------------------- /tests/partition_iter_lifetime.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, PartitionCreateOptions}; 2 | use test_log::test; 3 | 4 | struct Counter>> { 5 | iter: I, 6 | } 7 | 8 | impl>> Counter { 9 | pub fn execute(self) -> usize { 10 | self.iter.count() 11 | } 12 | } 13 | 14 | #[test] 15 | fn partition_iter_lifetime() -> fjall::Result<()> { 16 | let folder = tempfile::tempdir()?; 17 | 18 | let keyspace = Config::new(&folder).open()?; 19 | 20 | let tree = keyspace.open_partition("default", PartitionCreateOptions::default())?; 21 | assert_eq!(0, keyspace.write_buffer_size()); 22 | 23 | tree.insert("asd", "def")?; 24 | tree.insert("efg", "hgf")?; 25 | tree.insert("hij", "wer")?; 26 | 27 | { 28 | let iter = tree.iter(); 29 | let counter = Counter { iter }; 30 | assert_eq!(3, counter.execute()); 31 | } 32 | 33 | Ok(()) 34 | } 35 | -------------------------------------------------------------------------------- /tests/partition_recover.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, KvSeparationOptions, PartitionCreateOptions}; 2 | use test_log::test; 3 | 4 | const ITEM_COUNT: usize = 100; 5 | 6 | #[test] 7 | fn reload_partition_config() -> fjall::Result<()> { 8 | use lsm_tree::coding::Encode; 9 | 10 | let folder = tempfile::tempdir()?; 11 | 12 | let serialized_config = { 13 | let keyspace = Config::new(&folder).open()?; 14 | 15 | let tree = keyspace.open_partition( 16 | "default", 17 | PartitionCreateOptions::default() 18 | .compaction_strategy(fjall::compaction::Strategy::SizeTiered( 19 | fjall::compaction::SizeTiered::default(), 20 | )) 21 | .block_size(10_000) 22 | .with_kv_separation( 23 | KvSeparationOptions::default() 24 | .separation_threshold(4_000) 25 | .file_target_size(150_000_000), 26 | ), 27 | )?; 28 | 29 | tree.config.encode_into_vec() 30 | }; 31 | 32 | { 33 | let keyspace = Config::new(&folder).open()?; 34 | let tree = keyspace.open_partition("default", PartitionCreateOptions::default())?; 35 | assert_eq!(serialized_config, tree.config.encode_into_vec()); 36 | } 37 | 38 | Ok(()) 39 | } 40 | 41 | #[test] 42 | fn reload_with_partitions() -> fjall::Result<()> { 43 | let folder = tempfile::tempdir()?; 44 | 45 | // NOTE: clippy bug 46 | #[allow(unused_assignments)] 47 | { 48 | let keyspace = Config::new(&folder).open()?; 49 | 50 | let partitions = &[ 51 | keyspace.open_partition("default1", PartitionCreateOptions::default())?, 52 | keyspace.open_partition("default2", PartitionCreateOptions::default())?, 53 | keyspace.open_partition("default3", PartitionCreateOptions::default())?, 54 | ]; 55 | 56 | for tree in partitions { 57 | for x in 0..ITEM_COUNT as u64 { 58 | let key = x.to_be_bytes(); 59 | let value = nanoid::nanoid!(); 60 | tree.insert(key, value.as_bytes())?; 61 | } 62 | 63 | for x in 0..ITEM_COUNT as u64 { 64 | let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); 65 | let value = nanoid::nanoid!(); 66 | tree.insert(key, value.as_bytes())?; 67 | } 68 | } 69 | 70 | for tree in partitions { 71 | assert_eq!(tree.len()?, ITEM_COUNT * 2); 72 | assert_eq!(tree.iter().flatten().count(), ITEM_COUNT * 2); 73 | assert_eq!(tree.iter().rev().flatten().count(), ITEM_COUNT * 2); 74 | } 75 | } 76 | 77 | for _ in 0..10 { 78 | let keyspace = Config::new(&folder).open()?; 79 | 80 | let partitions = &[ 81 | keyspace.open_partition("default1", PartitionCreateOptions::default())?, 82 | keyspace.open_partition("default2", PartitionCreateOptions::default())?, 83 | keyspace.open_partition("default3", PartitionCreateOptions::default())?, 84 | ]; 85 | 86 | for tree in partitions { 87 | assert_eq!(tree.len()?, ITEM_COUNT * 2); 88 | assert_eq!(tree.iter().flatten().count(), ITEM_COUNT * 2); 89 | assert_eq!(tree.iter().rev().flatten().count(), ITEM_COUNT * 2); 90 | } 91 | } 92 | 93 | Ok(()) 94 | } 95 | -------------------------------------------------------------------------------- /tests/partition_recover_from_sealed.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, PartitionCreateOptions}; 2 | use test_log::test; 3 | 4 | const ITEM_COUNT: usize = 10; 5 | 6 | #[test] 7 | fn recover_sealed_journal() -> fjall::Result<()> { 8 | let folder = tempfile::tempdir()?; 9 | 10 | // NOTE: clippy bug 11 | #[allow(unused_assignments)] 12 | { 13 | let keyspace = Config::new(&folder) 14 | .flush_workers(0) 15 | .compaction_workers(0) 16 | .open()?; 17 | 18 | let partitions = &[ 19 | keyspace.open_partition("tree1", PartitionCreateOptions::default())?, 20 | keyspace.open_partition("tree2", PartitionCreateOptions::default())?, 21 | keyspace.open_partition("tree3", PartitionCreateOptions::default())?, 22 | ]; 23 | 24 | for tree in partitions { 25 | for x in 0..ITEM_COUNT as u64 { 26 | let key = x.to_be_bytes(); 27 | let value = nanoid::nanoid!(); 28 | tree.insert(key, value.as_bytes())?; 29 | } 30 | } 31 | 32 | for tree in partitions { 33 | assert_eq!(tree.len()?, ITEM_COUNT); 34 | assert_eq!(tree.iter().flatten().count(), ITEM_COUNT); 35 | assert_eq!(tree.iter().rev().flatten().count(), ITEM_COUNT); 36 | } 37 | 38 | { 39 | use lsm_tree::AbstractTree; 40 | 41 | let first = partitions.first().unwrap(); 42 | first.rotate_memtable()?; 43 | assert_eq!(1, first.tree.sealed_memtable_count()); 44 | } 45 | 46 | for tree in partitions { 47 | for x in 0..ITEM_COUNT as u64 { 48 | let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); 49 | let value = nanoid::nanoid!(); 50 | tree.insert(key, value.as_bytes())?; 51 | } 52 | } 53 | 54 | for tree in partitions { 55 | assert_eq!(tree.len()?, ITEM_COUNT * 2); 56 | assert_eq!(tree.iter().flatten().count(), ITEM_COUNT * 2); 57 | assert_eq!(tree.iter().rev().flatten().count(), ITEM_COUNT * 2); 58 | } 59 | 60 | { 61 | use lsm_tree::AbstractTree; 62 | 63 | let first = partitions.first().unwrap(); 64 | assert_eq!(1, first.tree.sealed_memtable_count()); 65 | } 66 | } 67 | 68 | for _ in 0..10 { 69 | let keyspace = Config::new(&folder) 70 | .flush_workers(0) 71 | .compaction_workers(0) 72 | .open()?; 73 | 74 | let partitions = &[ 75 | keyspace.open_partition("tree1", PartitionCreateOptions::default())?, 76 | keyspace.open_partition("tree2", PartitionCreateOptions::default())?, 77 | keyspace.open_partition("tree3", PartitionCreateOptions::default())?, 78 | ]; 79 | 80 | { 81 | use lsm_tree::AbstractTree; 82 | 83 | let first = partitions.first().unwrap(); 84 | assert_eq!(1, first.tree.sealed_memtable_count()); 85 | } 86 | 87 | for tree in partitions { 88 | assert_eq!(tree.len()?, ITEM_COUNT * 2); 89 | assert_eq!(tree.iter().flatten().count(), ITEM_COUNT * 2); 90 | assert_eq!(tree.iter().rev().flatten().count(), ITEM_COUNT * 2); 91 | } 92 | 93 | { 94 | use lsm_tree::AbstractTree; 95 | 96 | let first = partitions.first().unwrap(); 97 | assert_eq!(1, first.tree.sealed_memtable_count()); 98 | } 99 | } 100 | 101 | Ok(()) 102 | } 103 | 104 | #[test] 105 | fn recover_sealed_journal_blob() -> fjall::Result<()> { 106 | let folder = tempfile::tempdir()?; 107 | 108 | // NOTE: clippy bug 109 | #[allow(unused_assignments)] 110 | { 111 | let keyspace = Config::new(&folder) 112 | .flush_workers(0) 113 | .compaction_workers(0) 114 | .open()?; 115 | 116 | let partitions = &[ 117 | keyspace.open_partition("tree1", PartitionCreateOptions::default())?, 118 | keyspace.open_partition("tree2", PartitionCreateOptions::default())?, 119 | keyspace.open_partition("tree3", PartitionCreateOptions::default())?, 120 | ]; 121 | 122 | for tree in partitions { 123 | for x in 0..ITEM_COUNT as u64 { 124 | let key = x.to_be_bytes(); 125 | let value = nanoid::nanoid!().repeat(1_000); 126 | tree.insert(key, value.as_bytes())?; 127 | } 128 | } 129 | 130 | for tree in partitions { 131 | assert_eq!(tree.len()?, ITEM_COUNT); 132 | assert_eq!(tree.iter().flatten().count(), ITEM_COUNT); 133 | assert_eq!(tree.iter().rev().flatten().count(), ITEM_COUNT); 134 | } 135 | 136 | { 137 | use lsm_tree::AbstractTree; 138 | 139 | let first = partitions.first().unwrap(); 140 | first.rotate_memtable()?; 141 | assert_eq!(1, first.tree.sealed_memtable_count()); 142 | } 143 | 144 | for tree in partitions { 145 | for x in 0..ITEM_COUNT as u64 { 146 | let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); 147 | let value = nanoid::nanoid!().repeat(1_000); 148 | tree.insert(key, value.as_bytes())?; 149 | } 150 | } 151 | 152 | for tree in partitions { 153 | assert_eq!(tree.len()?, ITEM_COUNT * 2); 154 | assert_eq!(tree.iter().flatten().count(), ITEM_COUNT * 2); 155 | assert_eq!(tree.iter().rev().flatten().count(), ITEM_COUNT * 2); 156 | } 157 | 158 | { 159 | use lsm_tree::AbstractTree; 160 | 161 | let first = partitions.first().unwrap(); 162 | assert_eq!(1, first.tree.sealed_memtable_count()); 163 | } 164 | } 165 | 166 | for _ in 0..10 { 167 | let keyspace = Config::new(&folder) 168 | .flush_workers(0) 169 | .compaction_workers(0) 170 | .open()?; 171 | 172 | let partitions = &[ 173 | keyspace.open_partition("tree1", PartitionCreateOptions::default())?, 174 | keyspace.open_partition("tree2", PartitionCreateOptions::default())?, 175 | keyspace.open_partition("tree3", PartitionCreateOptions::default())?, 176 | ]; 177 | 178 | { 179 | use lsm_tree::AbstractTree; 180 | 181 | let first = partitions.first().unwrap(); 182 | assert_eq!(1, first.tree.sealed_memtable_count()); 183 | } 184 | 185 | for tree in partitions { 186 | assert_eq!(tree.len()?, ITEM_COUNT * 2); 187 | assert_eq!(tree.iter().flatten().count(), ITEM_COUNT * 2); 188 | assert_eq!(tree.iter().rev().flatten().count(), ITEM_COUNT * 2); 189 | } 190 | 191 | { 192 | use lsm_tree::AbstractTree; 193 | 194 | let first = partitions.first().unwrap(); 195 | assert_eq!(1, first.tree.sealed_memtable_count()); 196 | } 197 | } 198 | 199 | Ok(()) 200 | } 201 | -------------------------------------------------------------------------------- /tests/recover_from_different_folder.rs: -------------------------------------------------------------------------------- 1 | use fjall::Config; 2 | use test_log::test; 3 | 4 | #[test] 5 | fn recover_from_different_folder() -> fjall::Result<()> { 6 | if std::path::Path::new(".test").try_exists()? { 7 | std::fs::remove_dir_all(".test")?; 8 | } 9 | 10 | let folder = ".test/asd"; 11 | 12 | { 13 | let keyspace = Config::new(folder).open()?; 14 | let partition = keyspace.open_partition("default", Default::default())?; 15 | 16 | partition.insert("abc", "def")?; 17 | partition.insert("wqewe", "def")?; 18 | partition.insert("ewewq", "def")?; 19 | partition.insert("asddas", "def")?; 20 | partition.insert("ycxycx", "def")?; 21 | partition.insert("asdsda", "def")?; 22 | partition.insert("wewqe", "def")?; 23 | } 24 | 25 | let absolute_folder = std::path::Path::new(folder).canonicalize()?; 26 | 27 | std::fs::create_dir_all(".test/def")?; 28 | std::env::set_current_dir(".test/def")?; 29 | 30 | for _ in 0..100 { 31 | let _keyspace = Config::new(&absolute_folder) 32 | .max_write_buffer_size(1_024 * 1_024) 33 | .open()?; 34 | } 35 | 36 | Ok(()) 37 | } 38 | -------------------------------------------------------------------------------- /tests/recovery_ds_store.rs: -------------------------------------------------------------------------------- 1 | use fjall::Config; 2 | use test_log::test; 3 | 4 | #[test] 5 | fn keyspace_recover_ds_store() -> fjall::Result<()> { 6 | let folder = tempfile::tempdir()?; 7 | 8 | { 9 | let _keyspace = Config::new(&folder).open()?; 10 | } 11 | 12 | std::fs::File::create(folder.path().join("partitions").join(".DS_Store"))?; 13 | 14 | { 15 | let _keyspace = Config::new(&folder).open()?; 16 | } 17 | 18 | Ok(()) 19 | } 20 | -------------------------------------------------------------------------------- /tests/sealed_recovery.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, Keyspace, KvSeparationOptions, PartitionCreateOptions}; 2 | use test_log::test; 3 | 4 | #[test] 5 | fn recover_sealed() -> fjall::Result<()> { 6 | let folder = tempfile::tempdir()?; 7 | 8 | for item in 0_u128..100 { 9 | let config = Config::new(&folder); 10 | let keyspace = Keyspace::create_or_recover(config)?; 11 | 12 | let tree = keyspace.open_partition( 13 | "default", 14 | PartitionCreateOptions::default().max_memtable_size(1_000), 15 | )?; 16 | 17 | assert_eq!(item, tree.len()?.try_into().unwrap()); 18 | 19 | tree.insert(item.to_be_bytes(), item.to_be_bytes())?; 20 | assert_eq!(item + 1, tree.len()?.try_into().unwrap()); 21 | 22 | tree.rotate_memtable()?; 23 | keyspace.force_flush()?; 24 | } 25 | 26 | Ok(()) 27 | } 28 | 29 | #[test] 30 | fn recover_sealed_blob() -> fjall::Result<()> { 31 | let folder = tempfile::tempdir()?; 32 | 33 | for item in 0_u128..100 { 34 | let config = Config::new(&folder); 35 | let keyspace = Keyspace::create_or_recover(config)?; 36 | 37 | let tree = keyspace.open_partition( 38 | "default", 39 | PartitionCreateOptions::default() 40 | .max_memtable_size(1_000) 41 | .with_kv_separation(KvSeparationOptions::default()), 42 | )?; 43 | 44 | assert_eq!(item, tree.len()?.try_into().unwrap()); 45 | 46 | tree.insert(item.to_be_bytes(), item.to_be_bytes().repeat(1_000))?; 47 | assert_eq!(item + 1, tree.len()?.try_into().unwrap()); 48 | 49 | tree.rotate_memtable()?; 50 | keyspace.force_flush()?; 51 | } 52 | 53 | Ok(()) 54 | } 55 | 56 | #[test] 57 | fn recover_sealed_pair_1() -> fjall::Result<()> { 58 | let folder = tempfile::tempdir()?; 59 | 60 | for item in 0_u128..100 { 61 | let config = Config::new(&folder); 62 | let keyspace = Keyspace::create_or_recover(config)?; 63 | 64 | let tree = keyspace.open_partition( 65 | "default", 66 | PartitionCreateOptions::default().max_memtable_size(1_000), 67 | )?; 68 | let tree2 = keyspace.open_partition( 69 | "default2", 70 | PartitionCreateOptions::default() 71 | .max_memtable_size(1_000) 72 | .with_kv_separation(KvSeparationOptions::default()), 73 | )?; 74 | 75 | assert_eq!(item, tree.len()?.try_into().unwrap()); 76 | assert_eq!(item, tree2.len()?.try_into().unwrap()); 77 | 78 | let mut batch = keyspace.batch(); 79 | batch.insert(&tree, item.to_be_bytes(), item.to_be_bytes()); 80 | batch.insert(&tree2, item.to_be_bytes(), item.to_be_bytes().repeat(1_000)); 81 | batch.commit()?; 82 | 83 | assert_eq!(item + 1, tree.len()?.try_into().unwrap()); 84 | assert_eq!(item + 1, tree2.len()?.try_into().unwrap()); 85 | 86 | tree.rotate_memtable()?; 87 | keyspace.force_flush()?; 88 | } 89 | 90 | Ok(()) 91 | } 92 | 93 | #[test] 94 | fn recover_sealed_pair_2() -> fjall::Result<()> { 95 | use lsm_tree::AbstractTree; 96 | 97 | let folder = tempfile::tempdir()?; 98 | 99 | { 100 | let config = Config::new(&folder); 101 | let keyspace = Keyspace::create_or_recover(config)?; 102 | 103 | let tree = keyspace.open_partition("default", PartitionCreateOptions::default())?; 104 | let tree2 = keyspace.open_partition("default2", PartitionCreateOptions::default())?; 105 | 106 | tree.insert(0u8.to_be_bytes(), 0u8.to_be_bytes())?; 107 | tree2.insert(0u8.to_be_bytes(), 0u8.to_be_bytes())?; 108 | assert_eq!(1, tree.len()?.try_into().unwrap()); 109 | assert_eq!(1, tree2.len()?.try_into().unwrap()); 110 | assert_eq!(1, tree.tree.lock_active_memtable().len()); 111 | 112 | tree.rotate_memtable()?; 113 | assert_eq!(1, tree.tree.sealed_memtable_count()); 114 | 115 | keyspace.force_flush()?; 116 | assert_eq!(0, tree.tree.sealed_memtable_count()); 117 | 118 | tree.insert(1u8.to_be_bytes(), 1u8.to_be_bytes())?; 119 | 120 | assert_eq!(2, tree.len()?.try_into().unwrap()); 121 | assert_eq!(1, tree2.len()?.try_into().unwrap()); 122 | assert_eq!(1, tree.tree.lock_active_memtable().len()); 123 | 124 | assert_eq!(2, keyspace.journal_count()); 125 | } 126 | 127 | { 128 | let config = Config::new(&folder); 129 | let keyspace = Keyspace::create_or_recover(config)?; 130 | 131 | let tree = keyspace.open_partition("default", PartitionCreateOptions::default())?; 132 | let tree2 = keyspace.open_partition("default2", PartitionCreateOptions::default())?; 133 | 134 | assert_eq!(2, tree.len()?.try_into().unwrap()); 135 | assert_eq!(1, tree2.len()?.try_into().unwrap()); 136 | assert_eq!(1, tree.tree.lock_active_memtable().len()); 137 | 138 | assert_eq!(2, keyspace.journal_count()); 139 | } 140 | 141 | Ok(()) 142 | } 143 | 144 | #[test] 145 | fn recover_sealed_pair_3() -> fjall::Result<()> { 146 | let folder = tempfile::tempdir()?; 147 | 148 | for item in 0_u128..100 { 149 | let config = Config::new(&folder); 150 | let keyspace = Keyspace::create_or_recover(config)?; 151 | 152 | let tree = keyspace.open_partition( 153 | "default", 154 | PartitionCreateOptions::default().max_memtable_size(1_000), 155 | )?; 156 | let tree2 = keyspace.open_partition( 157 | "default2", 158 | PartitionCreateOptions::default() 159 | .max_memtable_size(1_000) 160 | .with_kv_separation(KvSeparationOptions::default()), 161 | )?; 162 | 163 | assert_eq!(item, tree.len()?.try_into().unwrap()); 164 | assert_eq!(item, tree2.len()?.try_into().unwrap()); 165 | 166 | let mut batch = keyspace.batch(); 167 | batch.insert(&tree, item.to_be_bytes(), item.to_be_bytes()); 168 | batch.insert(&tree2, item.to_be_bytes(), item.to_be_bytes().repeat(1_000)); 169 | batch.commit()?; 170 | 171 | log::info!("item now {item}"); 172 | 173 | use lsm_tree::AbstractTree; 174 | assert!(tree2.tree.l0_run_count() == 1); 175 | 176 | assert_eq!(item + 1, tree.len()?.try_into().unwrap()); 177 | assert_eq!(item + 1, tree2.len()?.try_into().unwrap()); 178 | 179 | tree2.rotate_memtable()?; 180 | assert_eq!(1, tree2.tree.sealed_memtable_count()); 181 | assert!(tree2.tree.lock_active_memtable().is_empty()); 182 | 183 | log::error!("-- MANUAL FLUSH --"); 184 | keyspace.force_flush()?; 185 | assert_eq!(0, tree2.tree.sealed_memtable_count()); 186 | } 187 | 188 | Ok(()) 189 | } 190 | -------------------------------------------------------------------------------- /tests/seqno_recovery.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, PartitionCreateOptions}; 2 | use test_log::test; 3 | 4 | const ITEM_COUNT: usize = 100; 5 | 6 | #[test] 7 | fn recover_seqno() -> fjall::Result<()> { 8 | let folder = tempfile::tempdir()?; 9 | 10 | let mut seqno = 0; 11 | 12 | // NOTE: clippy bug 13 | #[allow(unused_assignments)] 14 | { 15 | let keyspace = Config::new(&folder).open()?; 16 | 17 | let partitions = &[ 18 | keyspace.open_partition("default1", PartitionCreateOptions::default())?, 19 | keyspace.open_partition("default2", PartitionCreateOptions::default())?, 20 | keyspace.open_partition("default3", PartitionCreateOptions::default())?, 21 | ]; 22 | 23 | for tree in partitions { 24 | for x in 0..ITEM_COUNT as u64 { 25 | let key = x.to_be_bytes(); 26 | let value = nanoid::nanoid!(); 27 | tree.insert(key, value.as_bytes())?; 28 | 29 | seqno += 1; 30 | assert_eq!(seqno, keyspace.instant()); 31 | } 32 | 33 | for x in 0..ITEM_COUNT as u64 { 34 | let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); 35 | let value = nanoid::nanoid!(); 36 | tree.insert(key, value.as_bytes())?; 37 | 38 | seqno += 1; 39 | assert_eq!(seqno, keyspace.instant()); 40 | } 41 | } 42 | 43 | for tree in partitions { 44 | assert_eq!(tree.len()?, ITEM_COUNT * 2); 45 | assert_eq!(tree.iter().flatten().count(), ITEM_COUNT * 2); 46 | assert_eq!(tree.iter().rev().flatten().count(), ITEM_COUNT * 2); 47 | } 48 | } 49 | 50 | for _ in 0..10 { 51 | let keyspace = Config::new(&folder).open()?; 52 | assert_eq!(seqno, keyspace.instant()); 53 | 54 | let partitions = &[ 55 | keyspace.open_partition("default1", PartitionCreateOptions::default())?, 56 | keyspace.open_partition("default2", PartitionCreateOptions::default())?, 57 | keyspace.open_partition("default3", PartitionCreateOptions::default())?, 58 | ]; 59 | 60 | for tree in partitions { 61 | assert_eq!(tree.len()?, ITEM_COUNT * 2); 62 | assert_eq!(tree.iter().flatten().count(), ITEM_COUNT * 2); 63 | assert_eq!(tree.iter().rev().flatten().count(), ITEM_COUNT * 2); 64 | } 65 | } 66 | 67 | Ok(()) 68 | } 69 | 70 | #[test] 71 | fn recover_seqno_tombstone() -> fjall::Result<()> { 72 | let folder = tempfile::tempdir()?; 73 | 74 | let mut seqno = 0; 75 | 76 | // NOTE: clippy bug 77 | #[allow(unused_assignments)] 78 | { 79 | let keyspace = Config::new(&folder).open()?; 80 | 81 | let partitions = &[ 82 | keyspace.open_partition("default1", PartitionCreateOptions::default())?, 83 | keyspace.open_partition("default2", PartitionCreateOptions::default())?, 84 | keyspace.open_partition("default3", PartitionCreateOptions::default())?, 85 | ]; 86 | 87 | for tree in partitions { 88 | for x in 0..ITEM_COUNT as u64 { 89 | let key = x.to_be_bytes(); 90 | tree.remove(key)?; 91 | 92 | seqno += 1; 93 | assert_eq!(seqno, keyspace.instant()); 94 | } 95 | 96 | for x in 0..ITEM_COUNT as u64 { 97 | let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); 98 | tree.remove(key)?; 99 | 100 | seqno += 1; 101 | assert_eq!(seqno, keyspace.instant()); 102 | } 103 | } 104 | } 105 | 106 | for _ in 0..10 { 107 | let keyspace = Config::new(&folder).open()?; 108 | assert_eq!(seqno, keyspace.instant()); 109 | } 110 | 111 | Ok(()) 112 | } 113 | -------------------------------------------------------------------------------- /tests/tx_ryow.rs: -------------------------------------------------------------------------------- 1 | #[test_log::test] 2 | #[cfg(feature = "single_writer_tx")] 3 | fn tx_ryow() -> fjall::Result<()> { 4 | use fjall::{Config, PartitionCreateOptions}; 5 | 6 | let folder = tempfile::tempdir()?; 7 | 8 | let keyspace = Config::new(&folder).open_transactional()?; 9 | 10 | let tree = keyspace.open_partition("default", PartitionCreateOptions::default())?; 11 | 12 | let mut tx = keyspace.write_tx(); 13 | 14 | assert!(!tx.contains_key(&tree, "a")?); 15 | 16 | tx.insert(&tree, "a", "a"); 17 | assert!(tx.contains_key(&tree, "a")?); 18 | 19 | tx.remove(&tree, "a"); 20 | assert!(!tx.contains_key(&tree, "a")?); 21 | 22 | tx.insert(&tree, "a", "a"); 23 | tx.insert(&tree, "a", "c"); 24 | assert_eq!(b"c", &*tx.get(&tree, "a")?.unwrap()); 25 | 26 | tx.remove(&tree, "a"); 27 | assert!(!tx.contains_key(&tree, "a")?); 28 | 29 | Ok(()) 30 | } 31 | -------------------------------------------------------------------------------- /tests/v2_cache_api.rs: -------------------------------------------------------------------------------- 1 | // TODO: remove in V3 2 | 3 | #[allow(deprecated)] 4 | use fjall::{BlobCache, BlockCache}; 5 | use std::sync::Arc; 6 | 7 | #[test_log::test] 8 | #[allow(deprecated)] 9 | fn v2_cache_api() -> fjall::Result<()> { 10 | use fjall::Config; 11 | 12 | let folder = tempfile::tempdir()?; 13 | 14 | let keyspace = Config::new(&folder) 15 | .blob_cache(Arc::new(BlobCache::with_capacity_bytes(64_000))) 16 | .block_cache(Arc::new(BlockCache::with_capacity_bytes(64_000))) 17 | .open()?; 18 | 19 | assert_eq!(keyspace.cache_capacity(), 128_000); 20 | 21 | Ok(()) 22 | } 23 | -------------------------------------------------------------------------------- /tests/v2_sealed_journal_recovery.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, Keyspace, PartitionCreateOptions}; 2 | use test_log::test; 3 | 4 | // TODO: 3.0.0 remove in v3 5 | #[test] 6 | fn v2_awkward_sealed_journal_recovery() -> fjall::Result<()> { 7 | let config = Config::new("test_fixture/v2_sealed_journal_shenanigans") 8 | .flush_workers(0) 9 | .compaction_workers(0); 10 | 11 | let keyspace = Keyspace::create_or_recover(config)?; 12 | 13 | let tree = keyspace.open_partition( 14 | "default", 15 | PartitionCreateOptions::default().max_memtable_size(1_000), 16 | )?; 17 | 18 | /* tree.insert("a", "a")?; 19 | tree.rotate_memtable()?; 20 | 21 | tree.insert("b", "b")?; 22 | tree.rotate_memtable()?; 23 | 24 | tree.insert("c", "c")?; 25 | tree.rotate_memtable()?; */ 26 | 27 | assert_eq!(4, keyspace.journal_count()); 28 | assert_eq!(3, tree.len()?); 29 | 30 | Ok(()) 31 | } 32 | -------------------------------------------------------------------------------- /tests/write_buffer_size.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, KvSeparationOptions, PartitionCreateOptions}; 2 | use test_log::test; 3 | 4 | #[test] 5 | fn write_buffer_size_after_insert() -> fjall::Result<()> { 6 | let folder = tempfile::tempdir()?; 7 | 8 | let keyspace = Config::new(&folder).open()?; 9 | 10 | let tree = keyspace.open_partition("default", PartitionCreateOptions::default())?; 11 | assert_eq!(0, keyspace.write_buffer_size()); 12 | 13 | tree.insert("asd", "def")?; 14 | 15 | let write_buffer_size_after = keyspace.write_buffer_size(); 16 | assert!(write_buffer_size_after > 0); 17 | 18 | let mut batch = keyspace.batch(); 19 | batch.insert(&tree, "dsa", "qwe"); 20 | batch.commit()?; 21 | 22 | let write_buffer_size_after_batch = keyspace.write_buffer_size(); 23 | assert!(write_buffer_size_after_batch > write_buffer_size_after); 24 | 25 | tree.rotate_memtable_and_wait()?; 26 | assert_eq!(0, keyspace.write_buffer_size()); 27 | 28 | Ok(()) 29 | } 30 | 31 | #[test] 32 | fn write_buffer_size_blob() -> fjall::Result<()> { 33 | let folder = tempfile::tempdir()?; 34 | 35 | let keyspace = Config::new(&folder).open()?; 36 | 37 | let tree = keyspace.open_partition( 38 | "default", 39 | PartitionCreateOptions::default().with_kv_separation(KvSeparationOptions::default()), 40 | )?; 41 | assert_eq!(0, keyspace.write_buffer_size()); 42 | 43 | tree.insert("asd", "def")?; 44 | 45 | let write_buffer_size_after = keyspace.write_buffer_size(); 46 | assert!(write_buffer_size_after > 0); 47 | 48 | let mut batch = keyspace.batch(); 49 | batch.insert(&tree, "dsa", "qwe"); 50 | batch.commit()?; 51 | 52 | let write_buffer_size_after_batch = keyspace.write_buffer_size(); 53 | assert!(write_buffer_size_after_batch > write_buffer_size_after); 54 | 55 | tree.rotate_memtable_and_wait()?; 56 | assert_eq!(0, keyspace.write_buffer_size()); 57 | 58 | Ok(()) 59 | } 60 | -------------------------------------------------------------------------------- /tests/write_during_read.rs: -------------------------------------------------------------------------------- 1 | use fjall::{Config, PartitionCreateOptions}; 2 | use test_log::test; 3 | 4 | #[test] 5 | fn write_during_read() -> fjall::Result<()> { 6 | let folder = tempfile::tempdir()?; 7 | 8 | let keyspace = Config::new(&folder).open()?; 9 | 10 | let tree = keyspace.open_partition( 11 | "default", 12 | PartitionCreateOptions::default().max_memtable_size(128_000), 13 | )?; 14 | 15 | for x in 0u64..50_000 { 16 | tree.insert(x.to_be_bytes(), x.to_be_bytes())?; 17 | } 18 | tree.rotate_memtable_and_wait()?; 19 | 20 | for kv in tree.iter() { 21 | let (k, v) = kv?; 22 | tree.insert(k, v.repeat(4))?; 23 | } 24 | 25 | Ok(()) 26 | } 27 | --------------------------------------------------------------------------------