├── .cargo └── config.toml ├── .dockerignore ├── .github ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml └── workflows │ ├── ci.yml │ └── web.yml ├── .gitignore ├── CHANGELOG.md ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── benches ├── bench_main.rs └── cache │ ├── juice.rs │ └── mod.rs ├── cliff.toml ├── components ├── binary │ ├── Cargo.toml │ ├── build.rs │ └── src │ │ ├── build_info.rs │ │ ├── cmd │ │ ├── format.rs │ │ ├── mod.rs │ │ ├── mount.rs │ │ └── unmount.rs │ │ └── main.rs ├── common │ ├── Cargo.toml │ └── src │ │ └── lib.rs ├── fuse │ ├── Cargo.toml │ └── src │ │ ├── config.rs │ │ ├── err.rs │ │ ├── lib.rs │ │ └── null.rs ├── meta │ ├── Cargo.toml │ ├── justfile │ └── src │ │ ├── backend │ │ ├── key.rs │ │ ├── mod.rs │ │ └── rocksdb.rs │ │ ├── config.rs │ │ ├── context.rs │ │ ├── engine.rs │ │ ├── err.rs │ │ ├── id_table.rs │ │ ├── lib.rs │ │ └── open_files.rs ├── storage │ ├── Cargo.toml │ ├── benches │ │ └── bench_main.rs │ └── src │ │ ├── cache.rs │ │ ├── cache │ │ ├── README.md │ │ ├── file_cache.rs │ │ ├── juice_cache.rs │ │ └── mem_cache.rs │ │ ├── err.rs │ │ ├── lib.rs │ │ ├── pool │ │ ├── disk_pool.rs │ │ ├── memory_pool.rs │ │ └── mod.rs │ │ ├── raw_buffer.rs │ │ └── slice_buffer.rs ├── types │ ├── Cargo.toml │ └── src │ │ ├── attr.rs │ │ ├── entry.rs │ │ ├── ino.rs │ │ ├── internal_nodes.rs │ │ ├── lib.rs │ │ ├── setting.rs │ │ ├── slice.rs │ │ └── stat.rs ├── utils │ ├── Cargo.toml │ └── src │ │ ├── align.rs │ │ ├── env.rs │ │ ├── lib.rs │ │ ├── logger.rs │ │ ├── object_storage.rs │ │ ├── panic_hook.rs │ │ ├── pyroscope_init.rs │ │ ├── readable_size.rs │ │ ├── runtime.rs │ │ └── sentry_init.rs └── vfs │ ├── Cargo.toml │ └── src │ ├── config.rs │ ├── data_manager.rs │ ├── err.rs │ ├── handle.rs │ ├── kiseki.rs │ ├── lib.rs │ ├── reader.rs │ ├── writer.rs │ └── writer_v1.rs ├── criterion.toml ├── dev ├── lima-k8s.yml └── lima-ubuntu.yml ├── docker ├── Dockerfile.ubuntu ├── Dockerfile.ubuntu.builder └── entrypoint.sh ├── docs ├── book.toml └── src │ ├── SUMMARY.md │ ├── background.md │ ├── bench.md │ ├── design.md │ ├── flox.md │ └── quick_start.md ├── justfile ├── licenserc.toml ├── oranda.json ├── rust-toolchain.toml ├── rustfmt.toml ├── script └── ubuntu │ └── dep.sh ├── taplo.toml ├── tests ├── Cargo.toml └── src │ └── lib.rs └── typos.toml /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | #rustflags = ["--cfg", "tokio_unstable"] -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # macOS trash 2 | .DS_Store 3 | 4 | # Visual Studio Code 5 | .vscode/ 6 | .devcontainer/ 7 | 8 | # Eclipse files 9 | .classpath 10 | .project 11 | .settings/** 12 | 13 | # Vim swap files 14 | *.swp 15 | 16 | # Files generated by JetBrains IDEs, e.g. IntelliJ IDEA 17 | .idea/ 18 | *.iml 19 | out/ 20 | 21 | # Rust 22 | target/ 23 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Description 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | ## How are the changes test-covered 10 | 11 | - [ ] N/A 12 | - [ ] Automated tests (unit and/or integration tests) 13 | - [ ] Manual tests 14 | - [ ] Details are described below 15 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Maintain dependencies for GitHub Actions 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "monthly" 8 | 9 | # Maintain dependencies for rust 10 | - package-ecosystem: "cargo" 11 | directory: "/" 12 | schedule: 13 | interval: "monthly" 14 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - release/** 8 | paths-ignore: 9 | - '**.md' 10 | - '.licenserc.yaml' 11 | - '.commitlintrc.yaml' 12 | - 'LICENSE' 13 | - 'NOTICE' 14 | pull_request: 15 | branches: 16 | - main 17 | paths-ignore: 18 | - '**.md' 19 | - '.licenserc.yaml' 20 | - '.commitlintrc.yaml' 21 | - 'LICENSE' 22 | - 'NOTICE' 23 | 24 | concurrency: 25 | group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} 26 | cancel-in-progress: true 27 | 28 | 29 | jobs: 30 | check: 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: actions/checkout@v4 34 | 35 | - name: Check License Header 36 | uses: apache/skywalking-eyes/header@v0.6.0 37 | 38 | - name: Format 39 | run: cargo fmt --all -- --check 40 | 41 | - name: Clippy 42 | run: cargo clippy --all-targets --workspace -- -D warnings 43 | build: 44 | runs-on: ${{ matrix.os }} 45 | strategy: 46 | matrix: 47 | os: 48 | - ubuntu-latest 49 | steps: 50 | - uses: actions/checkout@v4 51 | - uses: extractions/setup-just@v2 52 | - name: Build 53 | run: just build 54 | 55 | rust-tests: 56 | strategy: 57 | fail-fast: false 58 | matrix: 59 | os: 60 | - ubuntu-22.04 61 | runs-on: ${{ matrix.os }} 62 | steps: 63 | - uses: actions/checkout@v4 64 | - uses: cargo-bins/cargo-binstall@main 65 | - name: Install tarpaulin 66 | run: cargo binstall -y cargo-tarpaulin 67 | - name: Rust unit tests with coverage report 68 | run: cargo tarpaulin --engine llvm --no-dead-code --no-fail-fast --all-features --workspace -o xml --output-dir ./cov-reports 69 | - name: Upload coverage report 70 | uses: actions/upload-artifact@v4 71 | with: 72 | name: cov-report-rust-tests-${{ runner.os }} 73 | path: ./cov-reports 74 | if-no-files-found: 'error' 75 | -------------------------------------------------------------------------------- /.github/workflows/web.yml: -------------------------------------------------------------------------------- 1 | # Workflow to build your docs with oranda (and mdbook) 2 | # and deploy them to Github Pages 3 | name: Web 4 | 5 | # We're going to push to the gh-pages branch, so we need that permission 6 | permissions: 7 | contents: write 8 | 9 | # What situations do we want to build docs in? 10 | # All of these work independently and can be removed / commented out 11 | # if you don't want oranda/mdbook running in that situation 12 | on: 13 | # Check that a PR didn't break docs! 14 | # 15 | # Note that the "Deploy to Github Pages" step won't run in this mode, 16 | # so this won't have any side-effects. But it will tell you if a PR 17 | # completely broke oranda/mdbook. Sadly we don't provide previews (yet)! 18 | pull_request: 19 | 20 | # Whenever something gets pushed to main, update the docs! 21 | # This is great for getting docs changes live without cutting a full release. 22 | # 23 | # Note that if you're using cargo-dist, this will "race" the Release workflow 24 | # that actually builds the Github Release that oranda tries to read (and 25 | # this will almost certainly complete first). As a result you will publish 26 | # docs for the latest commit but the oranda landing page won't know about 27 | # the latest release. The workflow_run trigger below will properly wait for 28 | # cargo-dist, and so this half-published state will only last for ~10 minutes. 29 | # 30 | # If you only want docs to update with releases, disable this, or change it to 31 | # a "release" branch. You can, of course, also manually trigger a workflow run 32 | # when you want the docs to update. 33 | push: 34 | branches: 35 | - main 36 | 37 | 38 | # Whenever a workflow called "Release" completes, update the docs! 39 | # 40 | # If you're using cargo-dist, this is recommended, as it will ensure that 41 | # oranda always sees the latest release right when it's available. Note 42 | # however that Github's UI is wonky when you use workflow_run, and won't 43 | # show this workflow as part of any commit. You have to go to the "actions" 44 | # tab for your repo to see this one running (the gh-pages deploy will also 45 | # only show up there). 46 | workflow_run: 47 | workflows: [ "Release" ] 48 | types: 49 | - completed 50 | 51 | # Alright, let's do it! 52 | jobs: 53 | web: 54 | name: Build and deploy site and docs 55 | runs-on: ubuntu-latest 56 | steps: 57 | # Setup 58 | - uses: actions/checkout@v3 59 | with: 60 | fetch-depth: 0 61 | - uses: dtolnay/rust-toolchain@stable 62 | - uses: swatinem/rust-cache@v2 63 | 64 | # Install mdbook plugins 65 | - name: Install mdbook plugins 66 | run: | 67 | cargo install --locked mdbook-toc@0.11.2 68 | cargo install --locked mdbook-linkcheck@0.7.7 69 | 70 | # Install and run oranda (and mdbook) 71 | # This will write all output to ./public/ (including copying mdbook's output to there) 72 | - name: Install and run oranda 73 | run: | 74 | cargo install oranda --locked --profile=dist 75 | oranda build 76 | 77 | # Deploy to our gh-pages branch (creating it if it doesn't exist) 78 | # the "public" dir that oranda made above will become the root dir 79 | # of this branch. 80 | # 81 | # Note that once the gh-pages branch exists, you must 82 | # go into repo's settings > pages and set "deploy from branch: gh-pages" 83 | # the other defaults work fine. 84 | - name: Deploy to Github Pages 85 | uses: JamesIves/github-pages-deploy-action@v4.4.1 86 | # ONLY if we're on main (so no PRs or feature branches allowed!) 87 | # if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/topic/') }} 88 | if: ${{ github.ref == 'refs/heads/main' }} 89 | with: 90 | branch: gh-pages 91 | # Gotta tell the action where to find oranda's output 92 | folder: public 93 | token: ${{ secrets.GITHUB_TOKEN }} 94 | single-commit: true -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .idea 3 | .DS_Store 4 | target 5 | debug 6 | out/ 7 | book/ 8 | docs/node_modules/ 9 | .vagrant/ 10 | public/ 11 | 12 | # OSX leaves these everywhere on SMB shares 13 | ._* 14 | *.o 15 | *.so 16 | *.dump 17 | *.out 18 | *.bak 19 | *.tmp 20 | *.temp 21 | *.log 22 | *.swp 23 | __debug_bin 24 | 25 | .env -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [0.0.1] - 2024-04-21 9 | 10 | ### Added 11 | 12 | - Add monitor to check why it is slow 13 | - Add otpl 14 | - Add access test after set_attr 15 | - Add quick start with virtual machine 16 | 17 | ### Changed 18 | 19 | - How to alloc data? 20 | - Write will dead lock 21 | - Implement write back logic 22 | - Figure out why we get stuck 23 | - We can rmdir 24 | - Get rid of dashmap on handle table 25 | - Refact openfile cache 26 | - Implement truncate function when set_attr change the file length 27 | - Implement link; TODO: read link 28 | - Implement basic link and unlink 29 | - Implement rename 30 | - Implement vfs:symlink 31 | - Implement symlink & readlink 32 | - Update licenserc 33 | - Fmt 34 | - Try1 35 | - Try2 36 | - Try3 37 | - Try4 38 | 39 | ### Fixed 40 | 41 | - Read & truncate read length after write 42 | - Fix vfs::truncate 43 | - 1. attr mode; 2. link; 3. unlink 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = [ 4 | "components/types", 5 | "components/storage", 6 | "components/utils", 7 | "tests", 8 | "components/vfs", 9 | "components/meta", 10 | "components/common", 11 | "components/fuse", 12 | "components/binary", 13 | ] 14 | [workspace.dependencies] 15 | async-trait = "0.1.77" 16 | bincode = "1.3.3" 17 | bitflags = "2.4.2" 18 | byteorder = "1.5.0" 19 | bytes = "1.5.0" 20 | criterion = { version = "0.5.1", features = ["async_tokio", "async_futures"] } 21 | crossbeam = "0.8.4" 22 | crossbeam-channel = "0.5.11" 23 | crossbeam-queue = "0.3.11" 24 | dashmap = "5.5.3" 25 | features = { version = "0.10.0" } 26 | fuser = { version = "0.14.0", features = ["libfuse", "serializable"] } 27 | futures = "0.3.30" 28 | lazy_static = "1.4.0" 29 | libc = "0.2.152" 30 | opendal = { version = "0.45.0", features = ["services-sled", "services-s3"] } 31 | rand = "0.8.5" 32 | rangemap = "1.4.0" 33 | rustix = "0.38.30" 34 | scopeguard = "1.2.0" 35 | serde = "1.0.195" 36 | serde_json = "1.0.111" 37 | snafu = "0.8.0" 38 | sonyflake = "0.2.0" 39 | tempfile = "3.9.0" 40 | tokio = { version = "1.35.1", features = ["full", "tracing"] } 41 | tokio-stream = "0.1.14" 42 | tokio-util = { version = "0.7.10", features = ["rt", "io"] } 43 | tracing = "0.1.40" 44 | tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } 45 | 46 | [workspace.package] 47 | version = "0.1.0" 48 | edition = "2021" 49 | authors = ["Ryan Tan"] 50 | license = "Apache-2.0" 51 | description = "A fuse filesystem" 52 | exclude = ["benches"] 53 | 54 | [profile.release] 55 | debug = true 56 | panic = 'abort' 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kiseki 2 | 3 | Go check the [doc](https://crrow.github.io/kisekifs/). 4 | 5 | Kiseki is my learning rust project, a simple 'distributed' fuse file system that is port 6 | of [JuiceFS](https://github.com/juicedata/juicefs). 7 | 8 | **It's just a rust learning project** 9 | 10 | If you don't know juicefs very much, the following is the introduction of juicefs: 11 | 12 | ``` 13 | JuiceFS is an open-source, high-performance distributed file system designed for the cloud. By providing full POSIX 14 | compatibility, it allows almost all kinds of object storage to be used as massive 15 | local disks and to be mounted and accessed on different hosts across platforms and regions. 16 | 17 | JuiceFS separates "data" and "metadata" storage. Files are split into chunks and stored in object storage like Amazon 18 | S3. The corresponding metadata can be stored in various databases such as Redis, MySQL, TiKV, and SQLite, based on the 19 | scenarios and requirements. 20 | ``` 21 | 22 | FUSE must be installed to build or run programs that use fuse-rs (i.e. kernel driver and libraries. Some platforms may 23 | also require userland utils like `fusermount`). A default installation of FUSE is usually sufficient. 24 | 25 | To build fuse-rs or any program that depends on it, `pkg-config` needs to be installed as well. 26 | 27 | ## Difference with juicefs 28 | 29 | ### 1. Write Buffer 30 | 31 | JuiceFS uses a pre-allocated memory and growable bytes pool as write buffer, 32 | but this pool is also used for make reading buffer. 33 | 34 | Kiseki's write buffer pool is fixed-size, and it is consist of a in-memory bytes 35 | pool and a mmap file. 36 | 37 | ### 2. Cache 38 | 39 | JuiceFs use disk-eviction mechinism to manage the writeback cache, 40 | in Kisekifs, it employs [moka](https://github.com/moka-rs/moka) to implement the 41 | write back cache, much cleaner and efficient. 42 | 43 | ### 3. How to read slices 44 | 45 | JuiceFs reorganize the slices into a linkedlist, kisekifs use [rangemap](https://github.com/jeffparsons/rangemap) to 46 | handle the trick part. 47 | 48 | 49 | # Posix Compliance 50 | 51 | Totally not compliant with posix, go to the issue page check the details. 52 | 53 | ## Disclaimer 54 | 55 | Kiseki is an independent learning project 56 | and is not endorsed by or affiliated with the Juice company. 57 | 58 | # License 59 | 60 | Apache-2.0 61 | -------------------------------------------------------------------------------- /benches/bench_main.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | mod cache; 18 | 19 | use criterion::criterion_main; 20 | 21 | criterion_main!(cache::juice::benches); 22 | -------------------------------------------------------------------------------- /benches/cache/juice.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::sync::Arc; 18 | 19 | use criterion::{criterion_group, BenchmarkId, Criterion, Throughput}; 20 | use kisekifs::{ 21 | common::readable_size::ReadableSize, 22 | meta::types::{random_slice_id, SliceID}, 23 | vfs::storage::{new_juice_builder, Cache}, 24 | }; 25 | use tokio::{io::AsyncReadExt, runtime}; 26 | 27 | // async fn juice_rw(c: &mut Criterion, rt: &runtime::Runtime) { 28 | // const BLOCK_SIZE: u64 = ReadableSize::mb(4).as_bytes(); 29 | // let cache = make_juice_cache(); 30 | // 31 | // c.bench_function("write", |b| { 32 | // b.to_async(rt).iter(|| { 33 | // let block = Arc::new((0..BLOCK_SIZE).map(|_| 34 | // rand::random::()).collect()); let slice_id = 35 | // random_slice_id(); cache.cache(slice_id, block).unwrap(); 36 | // }) 37 | // }); 38 | // 39 | // cache.wait_on_all_flush_finish(); 40 | // } 41 | 42 | fn make_juice_cache() -> Arc { 43 | let b = new_juice_builder(); 44 | let c = b.build().unwrap(); 45 | c 46 | } 47 | 48 | async fn read_cache(cache: Arc, req: &CacheReq) { 49 | let mut reader = cache.get(req.slice_id).await.unwrap(); 50 | let mut buf = vec![0u8; req.block.len()]; 51 | reader.read_exact(&mut buf).await.unwrap(); 52 | } 53 | 54 | struct CacheReq { 55 | slice_id: SliceID, 56 | block: Arc>, 57 | } 58 | 59 | fn bench(c: &mut Criterion) { 60 | let rt = runtime::Builder::new_multi_thread() 61 | .worker_threads(3) 62 | .thread_name("cache-bench-async-runtime") 63 | .thread_stack_size(3 * 1024 * 1024) 64 | .enable_all() 65 | .build() 66 | .unwrap(); 67 | 68 | let reqs = (0..block_cnt) 69 | .map(|_| CacheReq { 70 | slice_id: random_slice_id(), 71 | block: Arc::new( 72 | (0..block_size.as_bytes()) 73 | .map(|_| rand::random::()) 74 | .collect(), 75 | ), 76 | }) 77 | .collect::>(); 78 | 79 | let mut group = c.benchmark_group("simple cache rw"); 80 | for i in reqs.iter() { 81 | group.throughput(Throughput::Bytes(block_size.as_bytes())); 82 | group.bench_with_input( 83 | BenchmarkId::new("write", block_size), 84 | &block_size, 85 | |b, s| { 86 | b.to_async(&rt) 87 | .iter(|| cache.cache(i.slice_id, i.block.clone())); 88 | }, 89 | ); 90 | group.bench_with_input(BenchmarkId::new("read", block_size), &block_size, |b, s| { 91 | b.iter(|| async { 92 | let mut reader = cache.get(i.slice_id).await.unwrap(); 93 | let mut buf = vec![0u8; block_size.as_bytes() as usize]; 94 | reader.read_to_end(&mut buf).await.unwrap(); 95 | }); 96 | }); 97 | } 98 | group.finish(); 99 | } 100 | 101 | criterion_group!(benches, bench); 102 | -------------------------------------------------------------------------------- /benches/cache/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | pub mod juice; 18 | -------------------------------------------------------------------------------- /cliff.toml: -------------------------------------------------------------------------------- 1 | # git-cliff ~ configuration file 2 | # https://git-cliff.org/docs/configuration 3 | 4 | [changelog] 5 | # changelog header 6 | header = """ 7 | # Changelog\n 8 | All notable changes to this project will be documented in this file. 9 | 10 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 11 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n 12 | """ 13 | # template for the changelog body 14 | # https://keats.github.io/tera/docs/#introduction 15 | body = """ 16 | {% if version -%} 17 | ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} 18 | {% else -%} 19 | ## [Unreleased] 20 | {% endif -%} 21 | {% for group, commits in commits | group_by(attribute="group") %} 22 | ### {{ group | upper_first }} 23 | {% for commit in commits %} 24 | - {{ commit.message | upper_first }}\ 25 | {% endfor %} 26 | {% endfor %}\n 27 | """ 28 | # template for the changelog footer 29 | footer = """ 30 | {% for release in releases -%} 31 | {% if release.version -%} 32 | {% if release.previous.version -%} 33 | [{{ release.version | trim_start_matches(pat="v") }}]: \ 34 | https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}\ 35 | /compare/{{ release.previous.version }}..{{ release.version }} 36 | {% endif -%} 37 | {% else -%} 38 | [unreleased]: https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}\ 39 | /compare/{{ release.previous.version }}..HEAD 40 | {% endif -%} 41 | {% endfor %} 42 | 43 | """ 44 | # remove the leading and trailing whitespace from the templates 45 | trim = true 46 | 47 | [git] 48 | # parse the commits based on https://www.conventionalcommits.org 49 | conventional_commits = true 50 | # filter out the commits that are not conventional 51 | filter_unconventional = true 52 | # process each line of a commit as an individual commit 53 | split_commits = false 54 | # regex for parsing and grouping commits 55 | commit_parsers = [ 56 | { message = "^.*: add", group = "Added" }, 57 | { message = "^.*: support", group = "Added" }, 58 | { message = "^.*: remove", group = "Removed" }, 59 | { message = "^.*: delete", group = "Removed" }, 60 | { message = "^test", group = "Fixed" }, 61 | { message = "^fix", group = "Fixed" }, 62 | { message = "^.*: fix", group = "Fixed" }, 63 | { message = "^.*", group = "Changed" }, 64 | ] 65 | # protect breaking changes from being skipped due to matching a skipping commit_parser 66 | protect_breaking_commits = false 67 | # filter out the commits that are not matched by commit parsers 68 | filter_commits = true 69 | # regex for matching git tags 70 | tag_pattern = "v[0-9].*" 71 | # regex for skipping tags 72 | skip_tags = "v0.1.0-beta.1" 73 | # regex for ignoring tags 74 | ignore_tags = "" 75 | # sort the tags topologically 76 | topo_order = false 77 | # sort the commits inside sections by oldest/newest order 78 | sort_commits = "oldest" 79 | -------------------------------------------------------------------------------- /components/binary/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kiseki-binary" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fuser.workspace = true 10 | rustix = { workspace = true, features = ["mount"] } 11 | snafu.workspace = true 12 | tokio.workspace = true 13 | tracing.workspace = true 14 | 15 | #kiseki-utils = {path = "../../components/utils", features = ["tokio-console"]} 16 | kiseki-common = { path = "../../components/common" } 17 | kiseki-fuse = { path = "../../components/fuse" } 18 | kiseki-meta = { path = "../../components/meta", features = ["meta-rocksdb"] } 19 | kiseki-storage = { path = "../../components/storage" } 20 | kiseki-types = { path = "../../components/types" } 21 | kiseki-utils = { path = "../../components/utils" } 22 | kiseki-vfs = { path = "../../components/vfs" } 23 | 24 | clap = { version = "4.4.16", features = ["derive"] } 25 | clap-num = "1.0.2" 26 | const_format = "0.2.32" 27 | ctrlc = "3.4.2" 28 | human-panic = "1.2.3" 29 | procfs = "0.16.0" 30 | regex = "1.10.2" 31 | 32 | [build-dependencies] 33 | built = { version = "0.7.1", features = ["git2"] } 34 | const_format = "0.2.32" 35 | -------------------------------------------------------------------------------- /components/binary/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { built::write_built_file().expect("Failed to acquire build-time information"); } 2 | -------------------------------------------------------------------------------- /components/binary/src/build_info.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | mod built { 18 | include!(concat!(env!("OUT_DIR"), "/built.rs")); 19 | } 20 | 21 | pub use built::*; 22 | 23 | pub const AUTHOR: &str = built::PKG_AUTHORS; 24 | 25 | /// Valid SemVer version constructed using declared Cargo version and short 26 | /// commit hash if needed. 27 | pub const FULL_VERSION: &str = { 28 | if is_official_release() { 29 | built::PKG_VERSION 30 | } else { 31 | // A little hacky so we can pull out the hash as a const 32 | const COMMIT_HASH_STR: &str = match built::GIT_COMMIT_HASH_SHORT { 33 | Some(hash) => hash, 34 | None => "", 35 | }; 36 | const COMMIT_DIRTY_STR: &str = match built::GIT_DIRTY { 37 | Some(true) => "-dirty", 38 | _ => "", 39 | }; 40 | const UNOFFICIAL_SUFFIX: &str = if COMMIT_HASH_STR.is_empty() { 41 | "-unofficial" 42 | } else { 43 | const_format::concatcp!("-unofficial+", COMMIT_HASH_STR, COMMIT_DIRTY_STR) 44 | }; 45 | const_format::concatcp!(built::PKG_VERSION, UNOFFICIAL_SUFFIX) 46 | } 47 | }; 48 | const fn is_official_release() -> bool { option_env!("KISEKI_RELEASE").is_some() } 49 | -------------------------------------------------------------------------------- /components/binary/src/cmd/format.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::str::FromStr; 18 | 19 | use clap::Args; 20 | use kiseki_meta::MetaConfig; 21 | use kiseki_types::setting::Format; 22 | use kiseki_utils::{ 23 | align::{align4k, align_to_block}, 24 | readable_size::ReadableSize, 25 | }; 26 | use regex::Regex; 27 | use snafu::{ensure, ensure_whatever, OptionExt, ResultExt, Whatever}; 28 | use tokio::runtime; 29 | use tracing::{debug, info, level_filters::LevelFilter, warn, Instrument}; 30 | 31 | const FORMAT_OPTIONS_HEADER: &str = "DATA FORMAT"; 32 | const MANAGEMENT_OPTIONS_HEADER: &str = "MANAGEMENT"; 33 | 34 | #[derive(Debug, Clone, Args)] 35 | #[command(args_conflicts_with_subcommands = true)] 36 | #[command(flatten_help = true)] 37 | #[command(long_about = r" 38 | 39 | Create a new KisekiFS volume. Here META_ADDRESS is used to set up the 40 | metadata engine location, the SCHEME is used to specific the meta sto 41 | engine, and NAME is the prefix of all objects in data storage. 42 | ")] 43 | pub struct FormatArgs { 44 | #[arg( 45 | value_name = "FILE_SYSTEM_NAME", 46 | help = r"Your file system name, like 'demo-fs'.", 47 | value_parser = validate_name, 48 | default_value = "hello-fs", 49 | )] 50 | pub name: String, 51 | 52 | #[arg( 53 | long, 54 | help = "Specify the address of the meta store", 55 | help_heading = FORMAT_OPTIONS_HEADER, 56 | default_value = kiseki_common::KISEKI_DEBUG_META_ADDR, 57 | )] 58 | pub meta_dsn: Option, 59 | 60 | #[arg(long, short, help = "overwrite existing format", help_heading = FORMAT_OPTIONS_HEADER)] 61 | pub force: bool, 62 | 63 | // #[arg(long, help = "compression algorithm", help_heading = FORMAT_OPTIONS_HEADER)] 64 | // pub compression: Option, 65 | #[arg( 66 | long, 67 | help = "size of block in KiB", 68 | help_heading = FORMAT_OPTIONS_HEADER, 69 | default_value = "4M", 70 | value_parser = validate_block_size, 71 | )] 72 | pub block_size: ReadableSize, 73 | 74 | #[arg( 75 | long, 76 | short, 77 | help = "hard quota of the volume limiting its usage of space in GiB", 78 | help_heading = MANAGEMENT_OPTIONS_HEADER, 79 | value_parser = validate_capacity, 80 | )] 81 | pub capacity: Option, 82 | 83 | #[arg( 84 | long, 85 | short, 86 | help = "hard quota of the volume limiting its number of inodes", 87 | help_heading = MANAGEMENT_OPTIONS_HEADER, 88 | )] 89 | pub inodes: Option, 90 | } 91 | 92 | impl FormatArgs { 93 | fn generate_format(&self) -> Format { 94 | let mut format = Format::default(); 95 | format.max_capacity = self.capacity.map(|s| s.as_bytes_usize()); 96 | if let Some(inodes) = self.inodes { 97 | format.max_inodes = Some(inodes); 98 | } 99 | format.block_size = self.block_size.as_bytes_usize(); 100 | format.name = self.name.clone(); 101 | format 102 | } 103 | 104 | pub fn run(&self) -> Result<(), Whatever> { 105 | kiseki_utils::logger::install_fmt_log(); 106 | let dsn = self 107 | .meta_dsn 108 | .clone() 109 | .expect("meta_dsn should be validated in the argument parser"); 110 | let format = self.generate_format(); 111 | kiseki_meta::update_format(&dsn, format, true).unwrap(); 112 | info!("format file system {:?} success", self.name); 113 | Ok(()) 114 | } 115 | } 116 | 117 | const NAME_REGEX: &str = r"^[a-z0-9][a-z0-9\-]{1,61}[a-z0-9]$"; 118 | 119 | // Validation function for file system names 120 | fn validate_name(name: &str) -> Result { 121 | if name.len() <= 3 { 122 | return Err(format!("File system name {:?} is too short", name)); 123 | } 124 | if name.len() >= 30 { 125 | return Err(format!("File system name {:?} is too long", name)); 126 | } 127 | let reg = Regex::new(NAME_REGEX).unwrap(); 128 | if !reg.is_match(name) { 129 | return Err(format!("File system name {:?} is invalid", name)); 130 | } 131 | 132 | Ok(name.to_string()) 133 | } 134 | 135 | fn validate_trash_day(s: &str) -> Result { 136 | clap_num::number_range(s, 1, u64::MAX as usize) 137 | } 138 | 139 | fn validate_block_size(s: &str) -> Result { 140 | let n = ReadableSize::from_str(s).map_err(|e| format!("invalid block size: {}", e))?; 141 | let n = n.as_bytes() as usize; 142 | let n = align_to_block(n); 143 | if n < kiseki_common::MIN_BLOCK_SIZE { 144 | return Err(format!("block size {} too small", n)); 145 | } 146 | if n > kiseki_common::MAX_BLOCK_SIZE { 147 | return Err(format!("block size {} too large", n)); 148 | } 149 | Ok(ReadableSize(n as u64)) 150 | } 151 | 152 | fn validate_capacity(s: &str) -> Result { 153 | let n = ReadableSize::from_str(s).map_err(|e| format!("invalid capacity: {}", e))?; 154 | let n = n.as_bytes() as usize; 155 | 156 | Ok(ReadableSize(align4k(n as u64) as u64)) 157 | } 158 | -------------------------------------------------------------------------------- /components/binary/src/cmd/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | pub mod format; 18 | pub mod mount; 19 | pub mod unmount; 20 | -------------------------------------------------------------------------------- /components/binary/src/cmd/mount.rs: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::{ 16 | path::{Path, PathBuf}, 17 | str::FromStr, 18 | }; 19 | 20 | use clap::{Args, Parser}; 21 | use fuser::MountOption; 22 | use kiseki_common::{KISEKI, KISEKI_DEBUG_META_ADDR}; 23 | use kiseki_fuse::{null, FuseConfig}; 24 | use kiseki_meta::MetaConfig; 25 | use kiseki_utils::logger::{LoggingOptions, DEFAULT_LOG_DIR}; 26 | use kiseki_vfs::{Config as VFSConfig, KisekiVFS}; 27 | use snafu::{whatever, ResultExt, Whatever}; 28 | use tracing::info; 29 | 30 | use crate::build_info; 31 | 32 | const MOUNT_OPTIONS_HEADER: &str = "Mount options"; 33 | const LOGGING_OPTIONS_HEADER: &str = "Logging options"; 34 | const META_OPTIONS_HEADER: &str = "Meta options"; 35 | 36 | #[derive(Debug, Clone, Args)] 37 | #[command(flatten_help = true)] 38 | #[command(long_about = r" 39 | 40 | Mount the target volume at the mount point. 41 | Examples: 42 | 43 | # Mount in foreground 44 | kiseki mount -f /tmp/kiseki 45 | ")] 46 | pub struct MountArgs { 47 | #[arg( 48 | help = "Directory to mount the fs at", 49 | value_name = "MOUNT_POINT", 50 | default_value = "/tmp/kiseki" 51 | )] 52 | pub mount_point: PathBuf, 53 | 54 | #[arg( 55 | long, 56 | help = "Mount file system in read-only mode", 57 | help_heading = MOUNT_OPTIONS_HEADER 58 | )] 59 | pub read_only: bool, 60 | 61 | #[arg( 62 | long, 63 | help = "Automatically unmount on exit", 64 | help_heading = MOUNT_OPTIONS_HEADER, 65 | default_value = "true", 66 | )] 67 | pub auto_unmount: bool, 68 | 69 | #[arg(long, help = "Allow root user to access file system", help_heading = MOUNT_OPTIONS_HEADER)] 70 | pub allow_root: bool, 71 | 72 | #[arg( 73 | long, 74 | help = "Allow other users, including root, to access file system", 75 | help_heading = MOUNT_OPTIONS_HEADER, 76 | conflicts_with = "allow_root", 77 | default_value = "true", 78 | )] 79 | pub allow_other: bool, 80 | 81 | #[arg( 82 | long, 83 | help = "Number of threads to use for tokio async runtime", 84 | help_heading = MOUNT_OPTIONS_HEADER, 85 | default_value = "10", 86 | )] 87 | pub async_work_threads: usize, 88 | 89 | #[clap( 90 | long, 91 | help = "Write log files to a directory [default: logs written to syslog]", 92 | help_heading = LOGGING_OPTIONS_HEADER, 93 | value_name = "DIRECTORY", 94 | default_value = "/tmp/kiseki.log" 95 | )] 96 | pub log_directory: String, 97 | 98 | #[clap( 99 | short, 100 | long, 101 | help = "Log level", 102 | help_heading = LOGGING_OPTIONS_HEADER, 103 | value_name = "LEVEL", 104 | default_value = "info" 105 | )] 106 | pub level: Option, 107 | 108 | #[clap( 109 | long, 110 | help = "Enable OTLP tracing", 111 | help_heading = LOGGING_OPTIONS_HEADER, 112 | default_value = "true" 113 | )] 114 | pub enable_otlp_tracing: bool, 115 | 116 | #[clap( 117 | long, 118 | help = "Specify the OTLP endpoint", 119 | help_heading = LOGGING_OPTIONS_HEADER, 120 | value_name = "URL", 121 | default_value = "localhost:4317", 122 | )] 123 | pub otlp_endpoint: Option, 124 | 125 | #[clap( 126 | long, 127 | help = "Specify the tracing sample ratio", 128 | help_heading = LOGGING_OPTIONS_HEADER, 129 | default_value = "0.5", 130 | value_name = "RATIO", 131 | )] 132 | pub tracing_sample_ratio: Option, 133 | 134 | #[clap( 135 | long, 136 | help = "Append stdout to log files", 137 | help_heading = LOGGING_OPTIONS_HEADER, 138 | default_value = "true", 139 | )] 140 | pub append_stdout: bool, 141 | 142 | #[clap( 143 | long, 144 | help = "Disable all logging. You will still see stdout messages.", 145 | help_heading = LOGGING_OPTIONS_HEADER, 146 | conflicts_with_all(["log_directory", "level", "enable_otlp_tracing"]) 147 | )] 148 | pub no_log: bool, 149 | 150 | #[clap( 151 | short, 152 | long, 153 | help = "Run as foreground process", 154 | default_value = "true" 155 | )] 156 | pub foreground: bool, 157 | 158 | #[arg( 159 | long, 160 | help = "Specify the address of the meta store", 161 | help_heading = META_OPTIONS_HEADER, 162 | default_value = kiseki_common::KISEKI_DEBUG_META_ADDR, 163 | )] 164 | pub meta_dsn: String, 165 | } 166 | 167 | impl MountArgs { 168 | fn fuse_config(&self) -> FuseConfig { 169 | let mut options = vec![ 170 | MountOption::DefaultPermissions, 171 | MountOption::FSName(KISEKI.to_string()), 172 | MountOption::NoAtime, 173 | ]; 174 | if self.read_only { 175 | options.push(MountOption::RO); 176 | } 177 | if self.auto_unmount { 178 | options.push(MountOption::AutoUnmount); 179 | } 180 | if self.allow_root { 181 | options.push(MountOption::AllowRoot); 182 | } 183 | if self.allow_other { 184 | options.push(MountOption::AllowOther); 185 | } 186 | FuseConfig { 187 | mount_point: self.mount_point.clone(), 188 | mount_options: options, 189 | async_work_threads: self.async_work_threads, 190 | } 191 | } 192 | 193 | fn meta_config(&self) -> Result { 194 | let mut mc = MetaConfig::default(); 195 | mc.with_dsn(&self.meta_dsn); 196 | Ok(mc) 197 | } 198 | 199 | fn load_logging_opts(&self) -> Option { 200 | if self.no_log { 201 | return None; 202 | } 203 | let mut opts = LoggingOptions { 204 | dir: self.log_directory.clone(), 205 | level: self.level.clone(), 206 | enable_otlp_tracing: self.enable_otlp_tracing.clone(), 207 | otlp_endpoint: self.otlp_endpoint.clone(), 208 | tracing_sample_ratio: self.tracing_sample_ratio, 209 | append_stdout: self.append_stdout, 210 | tokio_console_addr: Some( 211 | kiseki_utils::logger::DEFAULT_TOKIO_CONSOLE_ADDR.to_string(), 212 | ), 213 | }; 214 | Some(opts) 215 | } 216 | 217 | fn vfs_config(&self) -> VFSConfig { VFSConfig::default() } 218 | 219 | pub fn run(self) -> Result<(), Whatever> { 220 | human_panic::setup_panic!(); 221 | kiseki_utils::panic_hook::set_panic_hook(); 222 | 223 | if self.foreground { 224 | let (_guard, _sentry_guard) = if let Some(opts) = self.load_logging_opts() { 225 | kiseki_utils::logger::init_global_logging_without_runtime("kiseki-fuse", &opts) 226 | } else { 227 | (vec![], None) 228 | }; 229 | 230 | let pyroscope_guard = kiseki_utils::pyroscope_init::init_pyroscope()?; 231 | 232 | mount(self)?; 233 | 234 | if let Some(agent_running) = pyroscope_guard { 235 | // Stop Agent 236 | let agent_ready = agent_running 237 | .stop() 238 | .with_whatever_context(|e| format!("failed to stop pyroscope agent {} ", e))?; 239 | 240 | // Shutdown the Agent 241 | agent_ready.shutdown(); 242 | } 243 | } 244 | Ok(()) 245 | } 246 | } 247 | 248 | pub fn print_versions() { 249 | // Report app version as gauge. 250 | // APP_VERSION 251 | // .with_label_values(&[short_version(), full_version()]) 252 | // .inc(); 253 | 254 | // Log version and argument flags. 255 | println!( 256 | "PKG_VERSION: {}, FULL_VERSION: {}", 257 | build_info::PKG_VERSION, 258 | build_info::FULL_VERSION, 259 | ); 260 | 261 | print_args(); 262 | } 263 | 264 | fn print_args() { 265 | println!("command line arguments"); 266 | for argument in std::env::args() { 267 | println!("argument: {}", argument); 268 | } 269 | } 270 | 271 | fn mount(args: MountArgs) -> Result<(), Whatever> { 272 | info!("try to mount kiseki on {:?}", &args.mount_point); 273 | print_versions(); 274 | 275 | validate_mount_point(&args.mount_point)?; 276 | 277 | let fuse_config = args.fuse_config(); 278 | let meta_config = args.meta_config()?; 279 | let vfs_config = args.vfs_config(); 280 | 281 | let meta = kiseki_meta::open(meta_config) 282 | .with_whatever_context(|e| format!("failed to open meta, {:?}", e))?; 283 | let file_system = KisekiVFS::new(vfs_config, meta) 284 | .with_whatever_context(|e| format!("failed to create file system, {:?}", e))?; 285 | 286 | let fs = kiseki_fuse::KisekiFuse::create(fuse_config.clone(), file_system)?; 287 | fuser::mount2(fs, &args.mount_point, &fuse_config.mount_options).with_whatever_context( 288 | |e| { 289 | format!( 290 | "failed to mount kiseki on {}; {}", 291 | args.mount_point.display(), 292 | e 293 | ) 294 | }, 295 | )?; 296 | Ok(()) 297 | } 298 | fn validate_mount_point(path: impl AsRef) -> Result<(), Whatever> { 299 | let mount_point = path.as_ref(); 300 | if !mount_point.exists() { 301 | whatever!("mount point {} does not exist", mount_point.display()); 302 | } 303 | 304 | if !mount_point.is_dir() { 305 | whatever!("mount point {} is not a directory", mount_point.display()); 306 | } 307 | 308 | #[cfg(target_os = "linux")] 309 | { 310 | use procfs::process::Process; 311 | 312 | // This is a best-effort validation, so don't fail if we can't read 313 | // /proc/self/mountinfo for some reason. 314 | let mounts = match Process::myself().and_then(|me| me.mountinfo()) { 315 | Ok(mounts) => mounts, 316 | Err(e) => { 317 | tracing::debug!( 318 | "failed to read mountinfo, not checking for existing mounts: {e:?}" 319 | ); 320 | return Ok(()); 321 | } 322 | }; 323 | 324 | if mounts 325 | .into_iter() 326 | .any(|mount| mount.mount_point == path.as_ref()) 327 | { 328 | whatever!("mount point {} is already mounted", path.as_ref().display()); 329 | } 330 | } 331 | 332 | null::mount_check(path)?; 333 | 334 | Ok(()) 335 | } 336 | -------------------------------------------------------------------------------- /components/binary/src/cmd/unmount.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::path::PathBuf; 18 | 19 | use clap::Args; 20 | use snafu::{ResultExt, Whatever}; 21 | 22 | #[derive(Debug, Clone, Args)] 23 | #[command(long_about = r" 24 | 25 | Unmount kiseki-fs from the specified directory. 26 | ")] 27 | pub struct UmountArgs { 28 | #[arg( 29 | help = "Directory to umount the fs at", 30 | value_name = "MOUNT_POINT", 31 | default_value = "/tmp/kiseki" 32 | )] 33 | pub mount_point: PathBuf, 34 | #[arg(long, short, help = "Force unmount even if the directory is not empty")] 35 | pub force: bool, 36 | } 37 | 38 | impl UmountArgs { 39 | pub fn run(&self) -> Result<(), Whatever> { 40 | let path = &self.mount_point; 41 | rustix::mount::unmount(path, rustix::mount::UnmountFlags::FORCE) 42 | .with_whatever_context(|_| format!("could not umount { }", path.display()))?; 43 | Ok(()) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /components/binary/src/main.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | mod build_info; 18 | mod cmd; 19 | 20 | use clap::{Parser, Subcommand}; 21 | use snafu::Whatever; 22 | 23 | use crate::cmd::{format::FormatArgs, mount::MountArgs, unmount::UmountArgs}; 24 | 25 | #[derive(Debug, Parser)] 26 | #[clap( 27 | name = "kiseki", 28 | about= "kiseki-fs client", 29 | author = build_info::AUTHOR, 30 | version = build_info::FULL_VERSION)] 31 | struct Cli { 32 | #[command(subcommand)] 33 | commands: Commands, 34 | } 35 | 36 | #[derive(Debug, Subcommand)] 37 | enum Commands { 38 | Mount(MountArgs), 39 | Umount(UmountArgs), 40 | Format(FormatArgs), 41 | } 42 | 43 | fn main() -> Result<(), Whatever> { 44 | let cli = Cli::parse(); 45 | match cli.commands { 46 | Commands::Mount(mount_args) => mount_args.run(), 47 | Commands::Umount(umount_args) => umount_args.run(), 48 | Commands::Format(format_args) => format_args.run(), 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /components/common/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kiseki-common" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | lazy_static.workspace = true 10 | -------------------------------------------------------------------------------- /components/common/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | pub const MAX_NAME_LENGTH: usize = 255; 18 | pub const DOT: &str = "."; 19 | pub const DOT_DOT: &str = ".."; 20 | 21 | pub const MODE_MASK_R: u8 = 0b100; 22 | pub const MODE_MASK_W: u8 = 0b010; 23 | pub const MODE_MASK_X: u8 = 0b001; 24 | 25 | pub const KISEKI: &str = "kiseki"; 26 | pub const KISEKI_DEBUG_META_ADDR: &str = "rocksdb://:/tmp/kiseki.meta"; 27 | pub const KISEKI_DEBUG_STAGE_CACHE: &str = "/tmp/kiseki.stage_cache"; 28 | pub const KISEKI_DEBUG_CACHE: &str = "/tmp/kiseki.cache"; 29 | pub const KISEKI_DEBUG_OBJECT_STORAGE: &str = "/tmp/kiseki.data"; 30 | 31 | pub const PAGE_BUFFER_SIZE: usize = 300 << 20; 32 | // 300MiB 33 | // pub const PAGE_SIZE: usize = 64 << 10; 34 | pub const PAGE_SIZE: usize = 128 << 10; 35 | // 128 KiB 36 | // The max block size is 4MB. 37 | pub const BLOCK_SIZE: usize = 4 << 20; // 4 MiB 38 | 39 | pub const MIN_BLOCK_SIZE: usize = PAGE_SIZE; // 128 KiB 40 | 41 | pub const MAX_BLOCK_SIZE: usize = 16 << 20; // 16 MB 42 | 43 | // The max size of a slice buffer can grow. 44 | pub const CHUNK_SIZE: usize = 64 << 20; // 64 MiB 45 | 46 | pub const MAX_FILE_SIZE: usize = CHUNK_SIZE << 31; // 64 MiB * 2^31 = 64 PiB 47 | 48 | pub const MIN_FILE_SYSTEM_CAPACITY: usize = 1 << 30; // 1 GiB 49 | 50 | pub const MAX_SYMLINK_LEN: usize = 4096; 51 | 52 | pub fn cal_chunk_idx(offset: usize, chunk_size: usize) -> usize { offset / chunk_size } 53 | 54 | pub fn cal_chunk_offset(offset: usize, chunk_size: usize) -> usize { offset % chunk_size } 55 | 56 | pub type PageSize = usize; 57 | pub type BlockIndex = usize; 58 | pub type BlockSize = usize; 59 | pub type ChunkIndex = usize; 60 | pub type ChunkOffset = usize; 61 | pub type ChunkSize = usize; 62 | pub type FileOffset = usize; 63 | 64 | pub type FH = u64; 65 | -------------------------------------------------------------------------------- /components/fuse/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kiseki-fuse" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fuser.workspace = true 10 | libc.workspace = true 11 | snafu.workspace = true 12 | tokio.workspace = true 13 | tracing.workspace = true 14 | 15 | kiseki-common = { path = "../../components/common" } 16 | kiseki-meta = { path = "../../components/meta" } 17 | kiseki-storage = { path = "../../components/storage" } 18 | kiseki-types = { path = "../../components/types" } 19 | kiseki-utils = { path = "../../components/utils" } 20 | kiseki-vfs = { path = "../../components/vfs" } 21 | -------------------------------------------------------------------------------- /components/fuse/src/config.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::path::PathBuf; 18 | 19 | /// Configuration for a FUSE background session. 20 | #[derive(Debug, Clone)] 21 | pub struct FuseConfig { 22 | pub mount_point: PathBuf, 23 | pub mount_options: Vec, 24 | /// work threads count for tokio runtime. 25 | pub async_work_threads: usize, 26 | } 27 | -------------------------------------------------------------------------------- /components/fuse/src/err.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use libc::c_int; 18 | use snafu::{location, Location, Snafu}; 19 | 20 | pub type Result = std::result::Result; 21 | 22 | #[derive(Debug, Snafu)] 23 | #[snafu(visibility(pub))] 24 | pub enum Error { 25 | Unknown { 26 | #[snafu(implicit)] 27 | location: Location, 28 | source: Box, 29 | }, 30 | } 31 | 32 | impl kiseki_types::ToErrno for Error { 33 | fn to_errno(&self) -> c_int { 34 | match self { 35 | Error::Unknown { .. } => libc::EINTR, 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /components/fuse/src/null.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::path::Path; 18 | 19 | use fuser::{spawn_mount2, Filesystem, MountOption}; 20 | use kiseki_common::KISEKI; 21 | use snafu::{ResultExt, Whatever}; 22 | 23 | /// An empty FUSE file system. It can be used in a mounting test aimed to 24 | /// determine whether or not the real file system can be mounted as well. If the 25 | /// test fails, the application can fail early instead of wasting time 26 | /// constructing the real file system. 27 | struct NullFs {} 28 | 29 | impl Filesystem for NullFs {} 30 | 31 | pub fn mount_check>(mountpoint: P) -> Result<(), Whatever> { 32 | let mountpoint = mountpoint.as_ref(); 33 | let options = [ 34 | MountOption::FSName(String::from(KISEKI)), 35 | MountOption::AllowRoot, 36 | ]; 37 | let session = spawn_mount2(NullFs {}, mountpoint, &options).with_whatever_context(|e| { 38 | format!("failed to mount null fs on {}; {}", mountpoint.display(), e) 39 | })?; 40 | drop(session); 41 | 42 | Ok(()) 43 | } 44 | -------------------------------------------------------------------------------- /components/meta/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kiseki-meta" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [features] 9 | # Public features 10 | meta-tikv = ["dep:tikv-client"] 11 | meta-rocksdb = ["dep:rocksdb"] 12 | 13 | [dependencies] 14 | async-trait.workspace = true 15 | bincode.workspace = true 16 | bitflags.workspace = true 17 | byteorder.workspace = true 18 | bytes.workspace = true 19 | crossbeam.workspace = true 20 | dashmap.workspace = true 21 | futures.workspace = true 22 | lazy_static.workspace = true 23 | libc.workspace = true 24 | opendal.workspace = true 25 | scopeguard.workspace = true 26 | serde.workspace = true 27 | serde_json.workspace = true 28 | snafu.workspace = true 29 | sonyflake.workspace = true 30 | tokio.workspace = true 31 | tokio-util.workspace = true 32 | tracing.workspace = true 33 | 34 | kiseki-common = { path = "../../components/common" } 35 | kiseki-types = { path = "../../components/types" } 36 | kiseki-utils = { path = "../../components/utils" } 37 | 38 | log = "0.4.20" 39 | rocksdb = { version = "0.22.0", features = ["lz4", "snappy"], optional = true } 40 | strum = "0.26" 41 | strum_macros = "0.26" 42 | tikv-client = { version = "0.3.0", optional = true } 43 | 44 | [dev-dependencies] 45 | tempfile.workspace = true 46 | -------------------------------------------------------------------------------- /components/meta/justfile: -------------------------------------------------------------------------------- 1 | test: 2 | @echo "Running tests" 3 | @cargo test --features meta-rocksdb -- --nocapture -------------------------------------------------------------------------------- /components/meta/src/backend/key.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use kiseki_types::{ino::Ino, slice::SliceID}; 18 | 19 | pub const CURRENT_FORMAT: &str = "current_format"; 20 | pub const USED_SPACE: &str = "used_space"; 21 | pub const TOTAL_INODES: &str = "total_inodes"; 22 | pub const LEGACY_SESSIONS: &str = "legacy_sessions"; 23 | pub const NEXT_TRASH: &str = "next_trash"; 24 | pub const NEXT_INODE: &str = "next_inode"; 25 | pub const NEXT_SLICE: &str = "next_slice"; 26 | 27 | #[derive(Debug, Eq, PartialEq, Copy, Clone, Hash)] 28 | pub(crate) enum Counter { 29 | UsedSpace, 30 | TotalInodes, 31 | LegacySessions, 32 | NextTrash, 33 | NextInode, 34 | NextSlice, 35 | } 36 | 37 | impl Into> for Counter { 38 | fn into(self) -> Vec { 39 | match self { 40 | Counter::UsedSpace => USED_SPACE.as_bytes().to_vec(), 41 | Counter::TotalInodes => TOTAL_INODES.as_bytes().to_vec(), 42 | Counter::LegacySessions => LEGACY_SESSIONS.as_bytes().to_vec(), 43 | Counter::NextTrash => NEXT_TRASH.as_bytes().to_vec(), 44 | Counter::NextInode => NEXT_INODE.as_bytes().to_vec(), 45 | Counter::NextSlice => NEXT_SLICE.as_bytes().to_vec(), 46 | } 47 | } 48 | } 49 | 50 | impl AsRef<[u8]> for Counter { 51 | fn as_ref(&self) -> &[u8] { 52 | match self { 53 | Counter::UsedSpace => USED_SPACE.as_bytes(), 54 | Counter::TotalInodes => TOTAL_INODES.as_bytes(), 55 | Counter::LegacySessions => LEGACY_SESSIONS.as_bytes(), 56 | Counter::NextTrash => NEXT_TRASH.as_bytes(), 57 | Counter::NextInode => NEXT_INODE.as_bytes(), 58 | Counter::NextSlice => NEXT_SLICE.as_bytes(), 59 | } 60 | } 61 | } 62 | 63 | impl Counter { 64 | pub fn get_step(&self) -> usize { 65 | match self { 66 | Counter::NextTrash => 1, 67 | Counter::NextInode => 1 << 10, 68 | Counter::NextSlice => 4 << 10, 69 | _ => panic!("Counter {:?} does not have a step", self), 70 | } 71 | } 72 | } 73 | 74 | pub fn attr(inode: Ino) -> Vec { format!("A{:0>8}I", inode.0).into_bytes() } 75 | 76 | pub fn xattr(inode: Ino, name: &str) -> Vec { 77 | format!("A{:0>8}X{}", inode.0, name).into_bytes() 78 | } 79 | pub fn xattr_prefix(inode: Ino) -> Vec { format!("A{:0>8}X", inode.0).into_bytes() } 80 | 81 | pub fn dentry(parent: Ino, name: &str) -> Vec { 82 | format!("A{:0>8}D/{}", parent.0, name).into_bytes() 83 | } 84 | 85 | pub fn dentry_prefix(parent: Ino) -> Vec { format!("A{:0>8}D/", parent.0).into_bytes() } 86 | 87 | /// [parent] generate key for hard links. 88 | /// 89 | /// This key is used to store the hard link count of a file. 90 | pub fn parent(inode: Ino, parent: Ino) -> Vec { 91 | // AiiiiiiiiPiiiiiiii parents 92 | format!("A{:0>8}P{:0>8}", inode.0, parent.0).into_bytes() 93 | } 94 | pub fn parent_prefix(inode: Ino) -> Vec { format!("A{:0>8}P", inode.0).into_bytes() } 95 | 96 | pub fn symlink(inode: Ino) -> Vec { format!("A{:0>8}S", inode.0).into_bytes() } 97 | 98 | // sustained is used when we cannot delete inode since it was being used at 99 | // right now. 100 | pub fn sustained(sid: u64, inode: Ino) -> Vec { 101 | format!("SS{:0>8}{:0>8}", sid, inode.0).into_bytes() 102 | } 103 | 104 | // delete_chunk_after is a marker used to indicate that when we need to delete 105 | // the chunks. 106 | // 107 | // Key: Diiiiiiii 108 | // Val: timestamp 109 | // 110 | // Don't know why juicefs doesn't delete the chunk info directly, it setups a 111 | // background job to delete the chunk info which has been deleted for a while. 112 | // Maybe a performance optimization since the slices amount is huge. 113 | // 114 | // Juice also put the length in the key, doesn't get the point. 115 | pub fn delete_chunk_after(inode: Ino) -> Vec { format!("D{:0>8}", inode.0).into_bytes() } 116 | 117 | pub fn chunk_slices(inode: Ino, chunk_idx: kiseki_common::ChunkIndex) -> Vec { 118 | format!("A{:0>8}C/{}", inode.0, chunk_idx).into_bytes() 119 | } 120 | pub fn chunk_slices_prefix(inode: Ino) -> Vec { format!("A{:0>8}C/", inode.0).into_bytes() } 121 | 122 | /// slice_ref tracks how many borrow slices are referencing to an Owned slice. 123 | /// We can only delete the slice when the reference count is zero. 124 | pub fn slice_ref(slice_id: SliceID) -> Vec { format!("K{:0>8}", slice_id).into_bytes() } 125 | 126 | pub fn dir_stat(inode: Ino) -> Vec { format!("U{:0>8}I", inode.0).into_bytes() } 127 | -------------------------------------------------------------------------------- /components/meta/src/backend/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::{path::Path, str::FromStr, sync::Arc, time::Duration}; 18 | 19 | use bytes::Bytes; 20 | use kiseki_common::ChunkIndex; 21 | use kiseki_types::{ 22 | attr::InodeAttr, entry::DEntry, ino::Ino, setting::Format, slice::Slices, stat, stat::DirStat, 23 | FileType, 24 | }; 25 | use snafu::ensure; 26 | use strum_macros::EnumString; 27 | use tracing::debug; 28 | 29 | use crate::{backend::key::Counter, context::FuseContext, err::Result}; 30 | 31 | pub mod key; 32 | #[cfg(feature = "meta-rocksdb")] 33 | mod rocksdb; 34 | 35 | use crate::{engine::RenameFlags, err::UnsupportedMetaDSNSnafu, open_files::OpenFilesRef}; 36 | 37 | // TODO: optimize me 38 | pub fn open_backend(dsn: &str, skip_dir_mtime: Duration) -> Result { 39 | let x = dsn.splitn(2, "://:").collect::>(); 40 | ensure!(x.len() == 2, UnsupportedMetaDSNSnafu { dsn: dsn.clone() }); 41 | let backend_kind = x[0]; 42 | let path = x[1]; 43 | 44 | let backend = BackendKinds::from_str(backend_kind).expect("unsupported backend kind"); 45 | backend.build(path, skip_dir_mtime) 46 | } 47 | 48 | #[derive(Debug, EnumString)] 49 | enum BackendKinds { 50 | #[cfg(feature = "meta-rocksdb")] 51 | #[strum(serialize = "rocksdb", serialize = "Rocksdb")] 52 | Rocksdb, 53 | } 54 | 55 | impl BackendKinds { 56 | fn build(&self, path: &str, skip_dir_mtime: Duration) -> Result { 57 | match self { 58 | #[cfg(feature = "meta-rocksdb")] 59 | BackendKinds::Rocksdb => { 60 | let mut builder = rocksdb::Builder::default(); 61 | builder.with_path(path).with_skip_dir_mtime(skip_dir_mtime); 62 | debug!("backend [rocksdb] is built with path: {}", path); 63 | builder.build() 64 | } 65 | _ => unimplemented!("unsupported backend"), 66 | } 67 | } 68 | } 69 | 70 | pub type BackendRef = Arc; 71 | 72 | #[async_trait::async_trait] 73 | pub trait Backend: Send + Sync { 74 | fn set_format(&self, format: &Format) -> Result<()>; 75 | fn load_format(&self) -> Result; 76 | 77 | fn increase_count_by(&self, counter: Counter, step: usize) -> Result; 78 | fn load_count(&self, counter: Counter) -> Result; 79 | 80 | fn get_attr(&self, inode: Ino) -> Result; 81 | fn set_attr(&self, inode: Ino, attr: &InodeAttr) -> Result<()>; 82 | 83 | fn get_dentry(&self, parent: Ino, name: &str) -> Result; 84 | fn set_dentry(&self, parent: Ino, name: &str, inode: Ino, typ: FileType) -> Result<()>; 85 | fn list_dentry(&self, parent: Ino, limit: i64) -> Result>; 86 | 87 | fn set_symlink(&self, inode: Ino, path: String) -> Result<()>; 88 | fn get_symlink(&self, inode: Ino) -> Result; 89 | 90 | fn set_chunk_slices(&self, inode: Ino, chunk_index: ChunkIndex, slices: Slices) -> Result<()>; 91 | fn set_raw_chunk_slices(&self, inode: Ino, chunk_index: ChunkIndex, buf: Vec) 92 | -> Result<()>; 93 | fn get_raw_chunk_slices(&self, inode: Ino, chunk_index: ChunkIndex) -> Result>>; 94 | fn get_chunk_slices(&self, inode: Ino, chunk_index: ChunkIndex) -> Result; 95 | 96 | fn set_dir_stat(&self, inode: Ino, dir_stat: DirStat) -> Result<()>; 97 | fn get_dir_stat(&self, inode: Ino) -> Result; 98 | 99 | /// [do_mknod] creates a node in a directory with given name, type and 100 | /// permissions. 101 | fn do_mknod( 102 | &self, 103 | ctx: Arc, 104 | new_inode: Ino, 105 | new_inode_attr: InodeAttr, 106 | parent: Ino, 107 | name: &str, 108 | typ: FileType, 109 | path: String, 110 | ) -> Result<(Ino, InodeAttr)>; 111 | 112 | /// [do_rmdir] removes a directory from the filesystem. The directory must 113 | /// be empty. return the removed directory entry and its attribute 114 | fn do_rmdir( 115 | &self, 116 | ctx: Arc, 117 | parent: Ino, 118 | name: &str, 119 | // skip updating attribute of a directory if the mtime difference is smaller 120 | // than this value 121 | skip_dir_mtime: Duration, 122 | ) -> Result<(DEntry, InodeAttr)>; 123 | 124 | /// [truncate] changes the length for given file. 125 | fn do_truncate( 126 | &self, 127 | ctx: Arc, 128 | inode: Ino, 129 | length: u64, 130 | skip_perm_check: bool, 131 | ) -> Result; 132 | 133 | /// [do_link] creates an entry for the inode, return the new [InodeAttr]. 134 | /// Creating another directory entry (filename) that points directly to the 135 | /// same inode as the original file. 136 | fn do_link( 137 | &self, 138 | ctx: Arc, 139 | inode: Ino, 140 | new_parent: Ino, 141 | new_name: &str, 142 | ) -> Result; 143 | 144 | /// [do_unlink] removes a file entry from a directory. 145 | /// return the freed space size and inode count. 146 | async fn do_unlink( 147 | &self, 148 | ctx: Arc, 149 | parent: Ino, 150 | name: String, 151 | session_id: u64, 152 | open_files_ref: OpenFilesRef, 153 | ) -> Result; 154 | 155 | /// do_delete_chunks try to delete all [free] slices of a file, 156 | /// free means that slice is not been borrowed. 157 | fn do_delete_chunks(&self, inode: Ino); 158 | 159 | async fn do_rename( 160 | &self, 161 | ctx: Arc, 162 | session_id: u64, 163 | old_parent: Ino, 164 | old_name: &str, 165 | new_parent: Ino, 166 | new_name: &str, 167 | flags: RenameFlags, 168 | open_files_ref: OpenFilesRef, 169 | ) -> Result; 170 | 171 | fn do_readlink(&self, inode: Ino) -> Result; 172 | } 173 | 174 | pub struct UnlinkResult { 175 | // The inode of the file 176 | pub inode: Ino, 177 | // the removed inode attr, if the file is removed 178 | pub removed: Option, 179 | // the freed space size 180 | pub freed_space: u64, 181 | // the freed inode count 182 | pub freed_inode: u64, 183 | // whether the file is opened while we're trying to unlink 184 | pub is_opened: bool, 185 | } 186 | 187 | pub struct RenameResult { 188 | // may need to delete the replaced file 189 | pub need_delete: Option<(Ino, bool)>, 190 | pub freed_space: u64, 191 | pub freed_inode: u64, 192 | } 193 | -------------------------------------------------------------------------------- /components/meta/src/config.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::{fmt::Display, path::PathBuf, str::FromStr, time::Duration}; 18 | 19 | use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer}; 20 | 21 | use crate::err::Result; 22 | 23 | /// Atime (Access Time): 24 | /// Every file has three timestamps: 25 | /// atime (access time): The last time the file was read or accessed. 26 | /// mtime (modification time): The last time the file's content was modified. 27 | /// ctime (change time): The last time the file's metadata (e.g., permissions, 28 | /// owner) was changed. 29 | #[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] 30 | pub enum AccessTimeMode { 31 | /// Disables atime updates entirely. 32 | /// Reading a file doesn't update its atime timestamp. 33 | /// Improves performance, especially for frequently accessed files. 34 | /// Can make it difficult to determine when a file was last accessed. 35 | Never, 36 | /// Default atime mode on many Linux systems. 37 | /// Updates atime only if: 38 | /// The file's atime is older than its mtime or ctime. 39 | /// The file has been accessed more than a certain time threshold (usually 1 40 | /// day). Balances performance and access time tracking. 41 | Relative, 42 | /// Always updates atime whenever a file is read. 43 | /// Accurately tracks file access times. 44 | /// Can impact performance, especially on storage systems with slow write 45 | /// speeds. 46 | Everytime, 47 | } 48 | 49 | impl Default for AccessTimeMode { 50 | fn default() -> Self { Self::Never } 51 | } 52 | 53 | #[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq)] 54 | pub struct MetaConfig { 55 | pub dsn: String, 56 | 57 | pub read_only: bool, 58 | /// The duration to reuse open file without checking update (0 means disable 59 | /// this feature) 60 | pub open_cache: Duration, 61 | /// max number of open files to cache (soft limit, 0 means unlimited) 62 | pub open_cache_limit: usize, 63 | /// [skip_dir_mtime] skip updating attribute of a directory if the mtime 64 | /// difference is smaller than this value 65 | pub skip_dir_mtime: Duration, 66 | } 67 | 68 | impl MetaConfig { 69 | pub fn with_dsn(&mut self, dsn: &str) -> &mut Self { 70 | self.dsn = dsn.to_string(); 71 | self 72 | } 73 | } 74 | 75 | impl Default for MetaConfig { 76 | fn default() -> Self { 77 | Self { 78 | dsn: kiseki_common::KISEKI_DEBUG_META_ADDR.to_string(), 79 | read_only: false, 80 | open_cache: Duration::default(), 81 | open_cache_limit: 10_000, 82 | skip_dir_mtime: Duration::from_millis(100), 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /components/meta/src/context.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::time::Instant; 18 | 19 | use kiseki_types::{attr::InodeAttr, ino::Ino}; 20 | use lazy_static::lazy_static; 21 | use snafu::ensure; 22 | use tokio_util::sync::CancellationToken; 23 | 24 | use crate::err::{LibcSnafu, Result}; 25 | 26 | lazy_static! { 27 | pub static ref EMPTY_CONTEXT: FuseContext = FuseContext::background(); 28 | } 29 | 30 | #[derive(Debug, Clone)] 31 | pub struct FuseContext { 32 | pub unique: u64, 33 | pub gid: u32, 34 | pub gid_list: Vec, 35 | pub uid: u32, 36 | pub pid: u32, 37 | pub check_permission: bool, 38 | pub start_at: Instant, 39 | pub cancellation_token: CancellationToken, 40 | } 41 | 42 | impl<'a> From<&'a kiseki_types::Request<'a>> for FuseContext { 43 | fn from(req: &'a kiseki_types::Request) -> Self { 44 | Self { 45 | unique: req.unique(), 46 | gid: req.gid(), 47 | gid_list: vec![], 48 | uid: req.uid(), 49 | pid: req.pid(), 50 | check_permission: true, 51 | start_at: Instant::now(), 52 | cancellation_token: CancellationToken::new(), 53 | } 54 | } 55 | } 56 | 57 | impl FuseContext { 58 | // Access checks the access permission on given inode. 59 | pub fn check_access(&self, attr: &InodeAttr, perm_mask: u8) -> Result<()> { 60 | if self.uid == 0 { 61 | return Ok(()); 62 | } 63 | if !self.check_permission { 64 | return Ok(()); 65 | } 66 | 67 | let mode = attr.access_mode(self.uid, &self.gid_list); 68 | // This condition checks if all the bits set in mmask (requested permissions) 69 | // are also set in mode (file's permissions). 70 | // 71 | // perm = 0o644 (rw-r--r--) 72 | // perm_mask = 0o4 (read permission) 73 | // perm & perm_mask = 0o4 (read permission is granted) 74 | ensure!( 75 | mode & perm_mask == perm_mask, 76 | LibcSnafu { 77 | errno: libc::EACCES, 78 | } 79 | ); 80 | return Ok(()); 81 | } 82 | 83 | pub fn is_cancelled(&self) -> bool { self.cancellation_token.is_cancelled() } 84 | } 85 | 86 | impl FuseContext { 87 | #[allow(dead_code)] 88 | pub fn background() -> Self { 89 | Self { 90 | unique: 0, 91 | gid: 2, 92 | gid_list: vec![2, 3], 93 | uid: 1, 94 | pid: 10, 95 | check_permission: true, 96 | start_at: Instant::now(), 97 | cancellation_token: CancellationToken::new(), 98 | } 99 | } 100 | 101 | pub fn contains_gid(&self, gid: u32) -> bool { self.gid_list.contains(&gid) } 102 | } 103 | -------------------------------------------------------------------------------- /components/meta/src/err.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use kiseki_types::ToErrno; 18 | use libc::c_int; 19 | use snafu::{location, Location, Snafu}; 20 | 21 | pub type Result = std::result::Result; 22 | 23 | #[derive(Debug, Snafu)] 24 | #[snafu(visibility(pub))] 25 | pub enum Error { 26 | Unknown { 27 | #[snafu(implicit)] 28 | location: Location, 29 | source: Box, 30 | }, 31 | UnsupportedMetaDSN { 32 | #[snafu(implicit)] 33 | location: Location, 34 | dsn: String, 35 | }, 36 | 37 | TokioJoinError { 38 | #[snafu(implicit)] 39 | location: Location, 40 | source: tokio::task::JoinError, 41 | }, 42 | 43 | #[cfg(feature = "meta-rocksdb")] 44 | RocksdbError { 45 | #[snafu(implicit)] 46 | location: Location, 47 | source: rocksdb::Error, 48 | }, 49 | 50 | // Model Error 51 | #[snafu(display("Model error: {:?}, {:?}", source, location))] 52 | ModelError { 53 | #[snafu(implicit)] 54 | location: Location, 55 | source: model_err::Error, 56 | }, 57 | 58 | // Setting 59 | #[snafu(display("FileSystem has not been initialized yet. Location: {}", location))] 60 | UninitializedEngine { 61 | #[snafu(implicit)] 62 | location: Location, 63 | }, 64 | 65 | #[snafu(display("Invalid setting: {:?}, {:?}", String::from_utf8_lossy(key.as_slice()).to_string(), location))] 66 | InvalidSetting { 67 | #[snafu(implicit)] 68 | location: Location, 69 | key: Vec, 70 | }, 71 | 72 | LibcError { 73 | #[snafu(implicit)] 74 | location: Location, 75 | errno: libc::c_int, 76 | }, 77 | } 78 | 79 | impl Error { 80 | pub fn is_not_found(&self) -> bool { 81 | matches!(self, Error::ModelError { source, .. } if source.is_not_found()) 82 | } 83 | } 84 | 85 | pub mod model_err { 86 | use snafu::Snafu; 87 | 88 | #[derive(Debug)] 89 | pub enum ModelKind { 90 | Attr, 91 | DEntry, 92 | Symlink, 93 | Setting, 94 | Counter, 95 | ChunkSlices, 96 | DirStat, 97 | HardLinkCount, 98 | Sustained, 99 | DeleteInode, 100 | } 101 | 102 | #[derive(Debug, Snafu)] 103 | #[snafu(visibility(pub))] 104 | pub enum Error { 105 | NotFound { 106 | kind: ModelKind, 107 | key: String, 108 | }, 109 | Corruption { 110 | kind: ModelKind, 111 | key: String, 112 | source: bincode::Error, 113 | }, 114 | CorruptionString { 115 | kind: ModelKind, 116 | key: String, 117 | reason: String, 118 | }, 119 | } 120 | 121 | impl Error { 122 | pub fn is_not_found(&self) -> bool { matches!(self, Error::NotFound { .. }) } 123 | } 124 | } 125 | 126 | impl ToErrno for Error { 127 | fn to_errno(&self) -> libc::c_int { 128 | match self { 129 | Error::Unknown { .. } => libc::EINTR, 130 | Error::UnsupportedMetaDSN { .. } => libc::EINTR, 131 | Error::TokioJoinError { .. } => libc::EINTR, 132 | #[cfg(feature = "meta-rocksdb")] 133 | Error::RocksdbError { .. } => libc::EINTR, 134 | Error::ModelError { source, .. } => { 135 | if source.is_not_found() { 136 | libc::ENOENT 137 | } else { 138 | libc::EINTR 139 | } 140 | } 141 | Error::UninitializedEngine { .. } => libc::EINTR, 142 | Error::InvalidSetting { .. } => libc::EINTR, 143 | Error::LibcError { errno, .. } => *errno, 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /components/meta/src/id_table.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use tokio::sync::RwLock; 18 | 19 | use crate::{ 20 | backend::{key::Counter, BackendRef}, 21 | err::Result, 22 | }; 23 | 24 | /// A table for allocating inode numbers. 25 | /// It starts at 2 since the root inode is 1. 26 | pub struct IdTable { 27 | next_max_pair: RwLock<(u64, u64)>, 28 | backend: BackendRef, 29 | counter: Counter, 30 | } 31 | 32 | impl IdTable { 33 | /// Return a new empty `IdTable`. 34 | pub(crate) fn new(backend: BackendRef, counter: Counter) -> Self { 35 | Self { 36 | next_max_pair: RwLock::new((0, 0)), 37 | backend, 38 | counter, 39 | } 40 | } 41 | 42 | /// Return the next unused ID from the table. 43 | pub async fn next(&self) -> Result { 44 | let mut next_max_pair = self.next_max_pair.write().await; 45 | if next_max_pair.0 >= next_max_pair.1 { 46 | let step = self.counter.get_step(); 47 | let new_max = self.backend.increase_count_by(self.counter.clone(), step)?; 48 | next_max_pair.0 = new_max - step as u64; 49 | next_max_pair.1 = new_max; 50 | } 51 | let mut next = next_max_pair.0; 52 | next_max_pair.0 += 1; 53 | while next <= 1 { 54 | next = next_max_pair.0; 55 | next_max_pair.0 += 1; 56 | } 57 | Ok(next) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /components/meta/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | pub mod backend; 18 | mod config; 19 | pub use config::MetaConfig; 20 | pub mod context; 21 | mod engine; 22 | pub use engine::{open, update_format, MetaEngineRef}; 23 | mod err; 24 | pub use err::Error; 25 | mod id_table; 26 | mod open_files; 27 | -------------------------------------------------------------------------------- /components/meta/src/open_files.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::{ 18 | collections::HashMap, 19 | sync::Arc, 20 | time::{Duration, SystemTime, UNIX_EPOCH}, 21 | }; 22 | 23 | use kiseki_common::ChunkIndex; 24 | use kiseki_types::{attr::InodeAttr, ino::Ino, slice::Slices}; 25 | use tokio::{sync::RwLock, time::Instant}; 26 | 27 | /// [OpenFile] represents an opened file in the cache. 28 | /// It is used for accelerating the query of the file's slices and attr. 29 | #[derive(Clone)] 30 | pub(crate) struct OpenFile(Arc>); 31 | 32 | impl OpenFile { 33 | async fn refresh_access(&self, attr: &mut InodeAttr) { 34 | let mut write_guard = self.0.write().await; 35 | if attr.mtime != write_guard.attr.mtime { 36 | write_guard.chunks.clear(); 37 | } else { 38 | attr.keep_cache = write_guard.attr.keep_cache; 39 | } 40 | write_guard.attr = attr.clone(); 41 | write_guard.last_check = SystemTime::now(); 42 | } 43 | 44 | async fn refresh_slices(&self, chunk_index: ChunkIndex, slices: Arc) { 45 | let mut write_guard = self.0.write().await; 46 | write_guard.chunks.insert(chunk_index, slices); 47 | write_guard.last_check = UNIX_EPOCH; 48 | } 49 | 50 | async fn invalid_slices(&self, chunk_index: ChunkIndex) { 51 | let mut write_guard = self.0.write().await; 52 | write_guard.chunks.remove(&chunk_index); 53 | } 54 | 55 | async fn invalid_all_chunk(&self) { 56 | let mut write_guard = self.0.write().await; 57 | write_guard.chunks.clear(); 58 | } 59 | 60 | async fn invalid_attr(&self) { 61 | let mut write_guard = self.0.write().await; 62 | write_guard.attr.keep_cache = false; 63 | write_guard.last_check = UNIX_EPOCH; 64 | } 65 | 66 | // decreases the reference count of the open file. 67 | async fn decrease_ref(&self) -> usize { 68 | let mut write_guard = self.0.write().await; 69 | write_guard.reference_count -= 1; 70 | write_guard.reference_count 71 | } 72 | 73 | pub(crate) async fn read_guard(&self) -> tokio::sync::RwLockReadGuard { 74 | self.0.read().await 75 | } 76 | 77 | pub(crate) async fn is_opened(&self) -> bool { 78 | let read_guard = self.0.read().await; 79 | read_guard.reference_count > 0 80 | } 81 | } 82 | 83 | pub(crate) struct OpenFileInner { 84 | pub(crate) attr: InodeAttr, 85 | reference_count: usize, 86 | last_check: SystemTime, 87 | chunks: HashMap>, 88 | } 89 | 90 | pub(crate) type OpenFilesRef = Arc; 91 | 92 | pub(crate) struct OpenFiles { 93 | ttl: Duration, 94 | limit: usize, 95 | files: RwLock>, 96 | // TODO: background clean up 97 | } 98 | 99 | impl OpenFiles { 100 | pub(crate) fn new(ttl: Duration, limit: usize) -> Self { 101 | Self { 102 | ttl, 103 | limit, 104 | files: Default::default(), 105 | } 106 | } 107 | 108 | /// [load] fetches the [OpenFile] from the cache. 109 | pub(crate) async fn load(&self, inode: &Ino) -> Option { 110 | let read_guard = self.files.read().await; 111 | let of = read_guard.get(inode).map(|v| v.clone()); 112 | drop(read_guard); // explicit drop to release the lock 113 | of 114 | } 115 | 116 | /// [open] create a new [OpenFile] if it does not exist, otherwise increase 117 | /// the reference count. 118 | pub(crate) async fn open(&self, inode: Ino, attr: &mut InodeAttr) { 119 | let read_guard = self.files.read().await; 120 | let of = match read_guard.get(&inode) { 121 | Some(of) => { 122 | let of = of.clone(); 123 | drop(read_guard); 124 | of 125 | } 126 | None => { 127 | drop(read_guard); 128 | let mut outer_write_guard = self.files.write().await; 129 | // check again 130 | let of = match outer_write_guard.get(&inode) { 131 | None => { 132 | outer_write_guard.insert( 133 | inode, 134 | OpenFile(Arc::new(RwLock::new(OpenFileInner { 135 | attr: attr.keep_cache().clone(), 136 | reference_count: 1, 137 | last_check: SystemTime::now(), 138 | chunks: Default::default(), 139 | }))), 140 | ); 141 | return; 142 | } 143 | Some(of) => of.clone(), 144 | }; 145 | of 146 | } 147 | }; 148 | // exists case 149 | let read_guard = of.0.read().await; 150 | if read_guard.attr.mtime == attr.mtime { 151 | attr.keep_cache = read_guard.attr.keep_cache; 152 | } 153 | drop(read_guard); 154 | let mut write_guard = of.0.write().await; 155 | write_guard.attr.keep_cache = true; 156 | write_guard.reference_count += 1; 157 | write_guard.last_check = SystemTime::now(); 158 | } 159 | 160 | /// [load_attr] fetches the [InodeAttr] from the cache, if it is not 161 | /// expired. 162 | pub(crate) async fn load_attr(&self, ino: Ino, add_ref: bool) -> Option { 163 | let outer_read_guard = self.files.read().await; 164 | if let Some(of) = outer_read_guard.get(&ino).map(|v| v.clone()) { 165 | drop(outer_read_guard); 166 | 167 | let read_guard = of.0.read().await; 168 | if read_guard.last_check.elapsed().unwrap() < self.ttl { 169 | let attr = read_guard.attr.clone(); 170 | drop(read_guard); 171 | if add_ref { 172 | let mut write_guard = of.0.write().await; 173 | write_guard.reference_count += 1; 174 | } 175 | return Some(attr); 176 | } 177 | } 178 | None 179 | } 180 | 181 | /// [load_slices] fetches the [Slices] from the cache, if it is not expired. 182 | pub(crate) async fn load_slices( 183 | &self, 184 | inode: Ino, 185 | chunk_index: ChunkIndex, 186 | ) -> Option> { 187 | let outer_read_guard = self.files.read().await; 188 | if let Some(of) = outer_read_guard.get(&inode).map(|of| of.clone()) { 189 | drop(outer_read_guard); 190 | 191 | let read_guard = of.0.read().await; 192 | if read_guard.last_check.elapsed().unwrap() < self.ttl { 193 | return read_guard.chunks.get(&chunk_index).map(|s| s.clone()); 194 | } 195 | } 196 | None 197 | } 198 | 199 | /// [refresh_attr] refresh the open file's [InodeAttr]. 200 | pub(crate) async fn refresh_attr(&self, ino: Ino, attr: &mut InodeAttr) { 201 | let read_guard = self.files.read().await; 202 | if let Some(of) = read_guard.get(&ino).map(|of| of.clone()) { 203 | drop(read_guard); 204 | of.refresh_access(attr).await; 205 | } 206 | } 207 | 208 | pub(crate) async fn refresh_slices(&self, ino: Ino, chunk_idx: ChunkIndex, views: Arc) { 209 | let read_guard = self.files.read().await; 210 | if let Some(of) = read_guard.get(&ino) { 211 | let of = of.clone(); 212 | drop(read_guard); 213 | of.refresh_slices(chunk_idx, views).await; 214 | } 215 | } 216 | 217 | pub(crate) async fn invalid(&self, inode: Ino, req: InvalidReq) { 218 | let read_guard = self.files.read().await; 219 | if let Some(of) = read_guard.get(&inode).map(|of| of.clone()) { 220 | drop(read_guard); 221 | match req { 222 | InvalidReq::OneChunk(idx) => { 223 | of.invalid_slices(idx).await; 224 | } 225 | InvalidReq::All => { 226 | of.invalid_all_chunk().await; 227 | } 228 | InvalidReq::OnlyAttr => { 229 | of.invalid_attr().await; 230 | } 231 | } 232 | } 233 | } 234 | 235 | /// [close] a file, under the hood, it just decreases the reference count. 236 | pub(crate) async fn close(&self, ino: Ino) -> bool { 237 | let read_guard = self.files.read().await; 238 | if let Some(of) = read_guard.get(&ino) { 239 | let of = of.clone(); 240 | drop(read_guard); 241 | 242 | let new_ref_cnt = of.decrease_ref().await; 243 | return new_ref_cnt <= 0; 244 | } 245 | true 246 | } 247 | } 248 | 249 | pub(crate) enum InvalidReq { 250 | OneChunk(ChunkIndex), 251 | All, 252 | OnlyAttr, 253 | } 254 | -------------------------------------------------------------------------------- /components/storage/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kiseki-storage" 3 | version = "0.1.0" 4 | edition = "2021" 5 | description = "The core storage implementation" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | async-trait.workspace = true 11 | bytes.workspace = true 12 | crossbeam.workspace = true 13 | crossbeam-queue.workspace = true 14 | dashmap.workspace = true 15 | futures.workspace = true 16 | lazy_static.workspace = true 17 | opendal.workspace = true 18 | rustix.workspace = true 19 | scopeguard.workspace = true 20 | serde.workspace = true 21 | snafu.workspace = true 22 | sonyflake.workspace = true 23 | tokio.workspace = true 24 | tokio-stream.workspace = true 25 | tokio-util.workspace = true 26 | tracing.workspace = true 27 | 28 | kiseki-common = { path = "../../components/common" } 29 | kiseki-meta = { path = "../../components/meta" } 30 | kiseki-types = { path = "../../components/types" } 31 | kiseki-utils = { path = "../../components/utils" } 32 | 33 | byte-pool = "0.2.4" 34 | crc32fast = "1.3.2" 35 | fmmap = { version = "0.3.3", features = ["tokio", "tokio-async"] } 36 | log = "0.4.20" 37 | moka = { version = "0.12.5", features = ["future"] } 38 | rand = "0.8.5" 39 | 40 | [dev-dependencies] 41 | criterion.workspace = true 42 | tempfile.workspace = true 43 | 44 | kiseki-utils = { path = "../../components/utils" } 45 | 46 | uuid = "1.7.0" 47 | 48 | #[[bench]] 49 | #name = "bench_main" 50 | #harness = false 51 | #path = "benches/bench_main.rs" 52 | -------------------------------------------------------------------------------- /components/storage/benches/bench_main.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | // use criterion::criterion_main; 18 | 19 | // criterion_main!(buffer::write::benches); 20 | -------------------------------------------------------------------------------- /components/storage/src/cache.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | // mod juice_cache; 18 | pub mod file_cache; 19 | pub mod mem_cache; 20 | -------------------------------------------------------------------------------- /components/storage/src/cache/README.md: -------------------------------------------------------------------------------- 1 | # Possible Disk Eviction implementation 2 | 1. https://github.com/neondatabase/neon/blob/main/pageserver/src/disk_usage_eviction_task.rs 3 | -------------------------------------------------------------------------------- /components/storage/src/cache/mem_cache.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::sync::Arc; 18 | 19 | use bytes::Bytes; 20 | use kiseki_types::slice::SliceKey; 21 | use kiseki_utils::readable_size::ReadableSize; 22 | use snafu::ResultExt; 23 | 24 | use crate::err::{Error::CacheError, ObjectStorageSnafu, Result}; 25 | 26 | #[derive(Debug)] 27 | pub struct Config { 28 | pub capacity: ReadableSize, 29 | } 30 | 31 | impl Default for Config { 32 | fn default() -> Self { 33 | Self { 34 | capacity: ReadableSize::gb(1), 35 | } 36 | } 37 | } 38 | 39 | pub type MemCacheRef = Arc; 40 | 41 | /// MemCache is responsible for caching the object block in memory. 42 | pub struct MemCache { 43 | inner: moka::future::Cache, 44 | remote_storage: kiseki_utils::object_storage::ObjectStorage, 45 | } 46 | 47 | impl MemCache { 48 | pub fn new( 49 | config: Config, 50 | remote_storage: kiseki_utils::object_storage::ObjectStorage, 51 | ) -> Self { 52 | let inner = moka::future::Cache::builder() 53 | .weigher(|_, value: &Bytes| -> u32 { value.len() as u32 }) 54 | .max_capacity(config.capacity.as_bytes()) 55 | // only one minute for the object to be alive 56 | .time_to_idle(std::time::Duration::from_secs(60)) 57 | .build(); 58 | Self { 59 | inner, 60 | remote_storage, 61 | } 62 | } 63 | 64 | pub async fn get(&self, key: &SliceKey) -> Result> { 65 | match self 66 | .inner 67 | .try_get_with_by_ref(key, async { 68 | let path = key.make_object_storage_path(); 69 | let object = self 70 | .remote_storage 71 | .get(&path) 72 | .await 73 | .context(ObjectStorageSnafu)?; 74 | let v = object.bytes().await.context(ObjectStorageSnafu)?; 75 | Ok(v) as Result 76 | }) 77 | .await 78 | { 79 | Ok(v) => Ok(Some(v)), 80 | Err(e) => { 81 | if e.is_not_found() { 82 | Ok(None) 83 | } else { 84 | Err(CacheError { 85 | error: format!("failed to get from cache: {}", e), 86 | }) 87 | } 88 | } 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /components/storage/src/err.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use snafu::{Location, Snafu}; 18 | 19 | #[derive(Snafu, Debug)] 20 | #[snafu(visibility(pub))] 21 | pub enum Error { 22 | #[snafu(display("OpenDAL operator failed"))] 23 | OpenDal { 24 | #[snafu(implicit)] 25 | location: Location, 26 | #[snafu(source)] 27 | error: opendal::Error, 28 | }, 29 | 30 | ObjectStorageError { 31 | #[snafu(implicit)] 32 | location: Location, 33 | source: kiseki_utils::object_storage::ObjectStorageError, 34 | }, 35 | 36 | CacheError { 37 | error: String, 38 | }, 39 | 40 | UnknownIOError { 41 | #[snafu(implicit)] 42 | location: Location, 43 | source: std::io::Error, 44 | }, 45 | 46 | JoinErr { 47 | #[snafu(implicit)] 48 | location: Location, 49 | source: tokio::task::JoinError, 50 | }, 51 | 52 | FlushBlockFailed { 53 | #[snafu(implicit)] 54 | location: Location, 55 | }, 56 | 57 | InvalidSliceBufferWriteOffset { 58 | #[snafu(implicit)] 59 | location: Location, 60 | }, 61 | 62 | DiskPoolMmapError { 63 | #[snafu(implicit)] 64 | location: Location, 65 | source: fmmap::error::Error, 66 | }, 67 | 68 | #[snafu(display("no more space in cache dir {}", cache_dir))] 69 | ErrStageNoMoreSpace { 70 | cache_dir: String, 71 | #[snafu(implicit)] 72 | location: Location, 73 | }, 74 | 75 | MetaError { 76 | #[snafu(implicit)] 77 | location: Location, 78 | source: kiseki_meta::Error, 79 | }, 80 | } 81 | 82 | impl Error { 83 | pub fn is_not_found(&self) -> bool { 84 | matches!(self, Error::ObjectStorageError { source, .. } if kiseki_utils::object_storage::is_not_found_error(source)) 85 | } 86 | } 87 | 88 | pub type Result = std::result::Result; 89 | -------------------------------------------------------------------------------- /components/storage/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | pub mod err; 18 | mod pool; 19 | 20 | pub fn get_pool_free_ratio() -> f64 { pool::GLOBAL_HYBRID_PAGE_POOL.free_ratio() } 21 | 22 | pub mod slice_buffer; 23 | 24 | pub mod cache; 25 | 26 | // pub mod raw_buffer; 27 | 28 | pub struct Storage {} 29 | 30 | impl Storage {} 31 | -------------------------------------------------------------------------------- /components/storage/src/pool/disk_pool.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::{ 18 | fmt::{Display, Formatter}, 19 | io::Read, 20 | path::{Path, PathBuf}, 21 | sync::Arc, 22 | }; 23 | 24 | use crossbeam_queue::ArrayQueue; 25 | use fmmap::tokio::{AsyncMmapFileExt, AsyncMmapFileMut, AsyncMmapFileMutExt, AsyncOptions}; 26 | use kiseki_utils::readable_size::ReadableSize; 27 | use snafu::ResultExt; 28 | use tokio::{ 29 | io::AsyncWriteExt, 30 | sync::{Notify, RwLock}, 31 | time::Instant, 32 | }; 33 | use tracing::debug; 34 | 35 | use crate::err::{DiskPoolMmapSnafu, Result, UnknownIOSnafu}; 36 | 37 | pub(crate) struct DiskPagePool { 38 | // the file path of the pool. 39 | filepath: PathBuf, 40 | // the size of each page. 41 | page_size: usize, 42 | // the total space of the file will use. 43 | capacity: usize, 44 | // the queue of the pages. 45 | queue: ArrayQueue, 46 | // ready notify. 47 | notify: Notify, 48 | // the underlying persistent storage support 49 | file: RwLock, 50 | } 51 | 52 | impl DiskPagePool { 53 | pub(crate) async fn new>( 54 | path: P, 55 | page_size: usize, 56 | capacity: usize, 57 | ) -> Result> { 58 | let start = Instant::now(); 59 | debug_assert!( 60 | page_size > 0 && capacity > 0 && capacity % page_size == 0 && capacity > page_size, 61 | "invalid page pool" 62 | ); 63 | let path_buf = path.as_ref().to_path_buf(); 64 | let cnt = capacity / page_size; 65 | let mut file = AsyncOptions::new() 66 | .create(true) 67 | .read(true) 68 | .write(true) 69 | .truncate(true) 70 | .max_size(capacity as u64) 71 | .open_mmap_file_mut(path) 72 | .await 73 | .context(DiskPoolMmapSnafu)?; 74 | 75 | file.truncate(capacity as u64) 76 | .await 77 | .context(DiskPoolMmapSnafu)?; 78 | let queue = ArrayQueue::new(cnt); 79 | (0..cnt as u64).for_each(|page_id| { 80 | queue.push(page_id).unwrap(); 81 | }); 82 | debug!("create disk pool finished, cost: {:?}", start.elapsed()); 83 | Ok(Arc::new(Self { 84 | filepath: path_buf, 85 | page_size, 86 | capacity, 87 | queue, 88 | notify: Default::default(), 89 | file: RwLock::new(file), 90 | })) 91 | } 92 | 93 | pub(crate) fn try_acquire_page(self: &Arc) -> Option { 94 | let page_id = self.queue.pop(); 95 | page_id.map(|page_id| Page { 96 | page_id, 97 | pool: self.clone(), 98 | }) 99 | } 100 | 101 | pub(crate) async fn acquire_page(self: &Arc) -> Page { 102 | let mut page_id = self.queue.pop(); 103 | while page_id.is_none() { 104 | self.notify.notified().await; 105 | page_id = self.queue.pop(); 106 | } 107 | Page { 108 | page_id: page_id.unwrap(), 109 | pool: self.clone(), 110 | } 111 | } 112 | 113 | pub(crate) fn remain_page_cnt(&self) -> usize { self.queue.len() } 114 | 115 | pub(crate) fn total_page_cnt(&self) -> usize { self.capacity / self.page_size } 116 | } 117 | 118 | impl Display for DiskPagePool { 119 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 120 | write!( 121 | f, 122 | "DiskPool {{ page_size: {}, capacity: {}, remain: {}, total_cnt: {} }}", 123 | ReadableSize(self.page_size as u64), 124 | ReadableSize(self.capacity as u64), 125 | self.remain_page_cnt(), 126 | self.total_page_cnt(), 127 | ) 128 | } 129 | } 130 | 131 | pub(crate) struct Page { 132 | page_id: u64, 133 | pool: Arc, 134 | } 135 | 136 | impl Page { 137 | pub(crate) async fn copy_to_writer( 138 | &self, 139 | offset: usize, 140 | length: usize, 141 | writer: &mut W, 142 | ) -> Result<()> 143 | where 144 | W: tokio::io::AsyncWrite + Unpin + ?Sized, 145 | { 146 | let guard = self.pool.file.read().await; 147 | let mut reader = guard 148 | .range_reader(self.page_id as usize * self.pool.page_size + offset, length) 149 | .context(DiskPoolMmapSnafu)?; 150 | let copy_len = tokio::io::copy(&mut reader, writer) 151 | .await 152 | .context(UnknownIOSnafu)?; 153 | debug_assert_eq!(copy_len as usize, length); 154 | Ok(()) 155 | } 156 | 157 | pub(crate) async fn copy_from_reader( 158 | &self, 159 | offset: usize, 160 | length: usize, 161 | reader: &mut R, 162 | ) -> Result<()> 163 | where 164 | R: tokio::io::AsyncRead + Unpin + ?Sized, 165 | { 166 | let mut guard = self.pool.file.write().await; 167 | let mut writer = guard 168 | .range_writer(self.cal_offset() + offset, length) 169 | .context(DiskPoolMmapSnafu)?; 170 | let copy_len = tokio::io::copy(reader, &mut writer) 171 | .await 172 | .context(UnknownIOSnafu)?; 173 | assert_eq!(copy_len as usize, length); 174 | Ok(()) 175 | } 176 | 177 | fn cal_offset(&self) -> usize { self.page_id as usize * self.pool.page_size } 178 | } 179 | 180 | impl Drop for Page { 181 | fn drop(&mut self) { 182 | self.pool.queue.push(self.page_id).unwrap(); 183 | self.pool.notify.notify_one(); 184 | } 185 | } 186 | 187 | #[cfg(test)] 188 | mod tests { 189 | use std::fs; 190 | 191 | use bytes::Bytes; 192 | use kiseki_utils::logger::install_fmt_log; 193 | use tokio_util::io::StreamReader; 194 | use tracing::info; 195 | 196 | use super::*; 197 | 198 | #[tokio::test(flavor = "multi_thread", worker_threads = 2)] 199 | async fn basic() { 200 | install_fmt_log(); 201 | 202 | let tempfile = tempfile::NamedTempFile::new().unwrap(); 203 | let path = tempfile.path(); 204 | 205 | let page_size = 128 << 10; 206 | let cap = 300 << 20; 207 | 208 | let pool = DiskPagePool::new(path, page_size, cap).await.unwrap(); 209 | let meta = fs::metadata(path).unwrap(); 210 | assert_eq!(meta.len(), cap as u64); 211 | 212 | assert_eq!(pool.remain_page_cnt(), pool.total_page_cnt()); 213 | let page = pool.acquire_page().await; 214 | assert_eq!(pool.remain_page_cnt(), pool.total_page_cnt() - 1); 215 | drop(page); 216 | assert_eq!(pool.remain_page_cnt(), pool.total_page_cnt()); 217 | } 218 | 219 | #[tokio::test] 220 | async fn get_page_concurrently() { 221 | install_fmt_log(); 222 | let tempfile = tempfile::NamedTempFile::new().unwrap(); 223 | let path = tempfile.path(); 224 | let page_size = 128 << 10; 225 | let cap = 300 << 20; 226 | 227 | let pool = DiskPagePool::new(path, page_size, cap).await.unwrap(); 228 | let start = std::time::Instant::now(); 229 | let mut handles = vec![]; 230 | for _ in 0..pool.total_page_cnt() { 231 | let pool = pool.clone(); 232 | let handle = tokio::spawn(async move { 233 | let page = pool.acquire_page().await; 234 | // tokio::time::sleep(Duration::from_millis(1)).await; 235 | let mut reader = StreamReader::new(tokio_stream::iter(vec![std::io::Result::Ok( 236 | Bytes::from_static(b"hello"), 237 | )])); 238 | 239 | page.copy_from_reader(0, page_size, &mut reader) 240 | .await 241 | .unwrap(); 242 | let mut test = vec![0u8; 5]; 243 | page.copy_to_writer(0, 5, &mut test).await.unwrap(); 244 | }); 245 | handles.push(handle); 246 | } 247 | 248 | assert!(pool.remain_page_cnt() <= pool.total_page_cnt()); 249 | let _ = futures::future::join_all(handles).await; 250 | 251 | info!( 252 | "fill the whole pool {} cost: {:?}", 253 | ReadableSize(pool.capacity as u64), 254 | start.elapsed(), 255 | ); 256 | 257 | assert_eq!(pool.remain_page_cnt(), pool.total_page_cnt()); 258 | } 259 | } 260 | -------------------------------------------------------------------------------- /components/storage/src/pool/memory_pool.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::{ 18 | cell::UnsafeCell, 19 | fmt::{Display, Formatter}, 20 | io::{Cursor, Write}, 21 | sync::Arc, 22 | }; 23 | 24 | use crossbeam_queue::ArrayQueue; 25 | use kiseki_utils::readable_size::ReadableSize; 26 | use snafu::ResultExt; 27 | use tokio::{sync::Notify, time::Instant}; 28 | use tracing::debug; 29 | 30 | use crate::err::UnknownIOSnafu; 31 | 32 | pub struct MemoryPagePool { 33 | page_size: usize, 34 | capacity: usize, 35 | queue: ArrayQueue, 36 | raw_pages: Box<[Slot]>, 37 | notify: Notify, 38 | } 39 | 40 | struct Slot { 41 | inner: UnsafeCell<&'static mut [u8]>, 42 | page_size: usize, 43 | } 44 | 45 | unsafe impl Send for Slot {} 46 | 47 | unsafe impl Sync for Slot {} 48 | 49 | impl Slot { 50 | fn get_inner_slice(&self, offset: usize, len: usize) -> &[u8] { 51 | unsafe { 52 | std::slice::from_raw_parts( 53 | ((*self.inner.get()).as_ptr() as usize + offset) as *const u8, 54 | len, 55 | ) 56 | } 57 | } 58 | 59 | fn get_mut_inner_slice(&self, offset: usize, len: usize) -> &mut [u8] { 60 | unsafe { 61 | std::slice::from_raw_parts_mut( 62 | ((*self.inner.get()).as_mut_ptr() as usize + offset) as *mut u8, 63 | len, 64 | ) 65 | } 66 | } 67 | 68 | fn clear(&self) { 69 | unsafe { 70 | let slice = std::slice::from_raw_parts_mut( 71 | ((*self.inner.get()).as_mut_ptr() as usize) as *mut u8, 72 | self.page_size, 73 | ); 74 | slice.fill(0); 75 | }; 76 | } 77 | } 78 | 79 | impl MemoryPagePool { 80 | pub fn new(page_size: usize, capacity: usize) -> Arc { 81 | let start_at = Instant::now(); 82 | debug_assert!( 83 | page_size > 0 && capacity > 0 && capacity % page_size == 0 && capacity > page_size, 84 | "invalid page pool" 85 | ); 86 | 87 | debug!( 88 | "page pool: page_size: {}, capacity: {}", 89 | ReadableSize(page_size as u64), 90 | ReadableSize(capacity as u64) 91 | ); 92 | let page_cnt = capacity / page_size; 93 | 94 | let page_buffer = Box::leak(vec![0u8; page_cnt * page_size].into_boxed_slice()); 95 | let slots = page_buffer 96 | .chunks_exact_mut(page_size) 97 | .map(|chunk| { 98 | let buf: &mut [u8] = chunk.try_into().unwrap(); 99 | Slot { 100 | inner: UnsafeCell::new(buf), 101 | page_size, 102 | } 103 | }) 104 | .collect(); 105 | 106 | let pool = Arc::new(Self { 107 | page_size, 108 | capacity, 109 | queue: ArrayQueue::new(page_cnt), 110 | raw_pages: slots, 111 | notify: Default::default(), 112 | }); 113 | 114 | (0..page_cnt as u64).for_each(|page_id| { 115 | pool.queue.push(page_id).unwrap(); 116 | }); 117 | 118 | debug!( 119 | "{} initialize finished, cost: {:?}", 120 | &pool, 121 | start_at.elapsed(), 122 | ); 123 | pool 124 | } 125 | 126 | pub fn try_acquire_page(self: &Arc) -> Option { 127 | Some(Page { 128 | page_id: self.queue.pop()?, 129 | _pool: self.clone(), 130 | }) 131 | } 132 | 133 | pub async fn acquire_page(self: &Arc) -> Page { 134 | loop { 135 | if let Some(page_id) = self.queue.pop() { 136 | return Page { 137 | page_id, 138 | _pool: self.clone(), 139 | }; 140 | } 141 | self.notify.notified().await; 142 | } 143 | } 144 | 145 | fn notify_page_ready(self: &Arc) { self.notify.notify_one(); } 146 | 147 | fn recycle(self: &Arc, page_id: u64) { 148 | let slot = &self.raw_pages[page_id as usize]; 149 | slot.clear(); 150 | self.queue.push(page_id).unwrap(); 151 | self.notify_page_ready(); 152 | } 153 | 154 | pub fn remain_page_cnt(&self) -> usize { self.queue.len() } 155 | 156 | #[inline] 157 | pub fn total_page_cnt(&self) -> usize { self.capacity / self.page_size } 158 | 159 | #[inline] 160 | pub fn capacity(&self) -> usize { self.capacity } 161 | } 162 | 163 | impl Display for MemoryPagePool { 164 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 165 | write!( 166 | f, 167 | "PagePool {{ page_size: {}, capacity: {}, remain: {}, total_cnt: {} }}", 168 | ReadableSize(self.page_size as u64), 169 | ReadableSize(self.capacity as u64), 170 | self.remain_page_cnt(), 171 | self.total_page_cnt(), 172 | ) 173 | } 174 | } 175 | 176 | /// The value returned by an allocation of the pool. 177 | /// When it is dropped the memory gets returned into the pool, and is not 178 | /// zeroed. If that is a concern, you must clear the data yourself. 179 | pub struct Page { 180 | page_id: u64, 181 | _pool: Arc, 182 | } 183 | 184 | impl Page { 185 | pub(crate) async fn copy_to_writer( 186 | &self, 187 | offset: usize, 188 | length: usize, 189 | writer: &mut W, 190 | ) -> crate::err::Result<()> 191 | where 192 | W: tokio::io::AsyncWrite + Unpin + ?Sized, 193 | { 194 | let slot = &self._pool.raw_pages[self.page_id as usize]; 195 | let slice = slot.get_inner_slice(offset, length); 196 | let mut cursor = Cursor::new(slice); 197 | let copy_len = tokio::io::copy(&mut cursor, writer) 198 | .await 199 | .context(UnknownIOSnafu)?; 200 | debug_assert_eq!(copy_len as usize, length); 201 | Ok(()) 202 | } 203 | 204 | pub(crate) async fn copy_from_reader( 205 | &self, 206 | offset: usize, 207 | length: usize, 208 | reader: &mut R, 209 | ) -> crate::err::Result<()> 210 | where 211 | R: tokio::io::AsyncRead + Unpin + ?Sized, 212 | { 213 | let slot = &self._pool.raw_pages[self.page_id as usize]; 214 | let slice = slot.get_mut_inner_slice(offset, length); 215 | let mut cursor = Cursor::new(slice); 216 | let copy_len = tokio::io::copy(reader, &mut cursor) 217 | .await 218 | .context(UnknownIOSnafu)?; 219 | debug_assert_eq!(copy_len as usize, length); 220 | Ok(()) 221 | } 222 | 223 | pub(crate) fn size(&self) -> usize { self._pool.page_size } 224 | } 225 | 226 | impl Drop for Page { 227 | fn drop(&mut self) { self._pool.recycle(self.page_id); } 228 | } 229 | 230 | #[cfg(test)] 231 | mod tests { 232 | use std::{io::Write, time::Duration}; 233 | 234 | use kiseki_utils::logger::install_fmt_log; 235 | use tracing::info; 236 | 237 | use super::*; 238 | 239 | #[test] 240 | fn slot() { 241 | let page_size = 1024; 242 | let page_cnt = 10; 243 | let page_buffer = Box::leak(vec![0u8; page_cnt * page_size].into_boxed_slice()); 244 | let slots: Box<[Slot]> = page_buffer 245 | .chunks_exact_mut(page_size) 246 | .map(|chunk| { 247 | let buf: &mut [u8] = chunk.try_into().unwrap(); 248 | Slot { 249 | inner: UnsafeCell::new(buf), 250 | page_size, 251 | } 252 | }) 253 | .collect(); 254 | 255 | let slot = &slots[0]; 256 | let mut buf = slot.get_mut_inner_slice(0, 5); 257 | buf.write_all(b"hello").unwrap(); 258 | let slice = slot.get_inner_slice(0, 5); 259 | assert_eq!(slice, b"hello"); 260 | } 261 | 262 | #[tokio::test(flavor = "multi_thread", worker_threads = 2)] 263 | async fn basic() { 264 | install_fmt_log(); 265 | 266 | let pool = MemoryPagePool::new(128 << 10, 300 << 20); 267 | let page = pool.acquire_page().await; 268 | assert_eq!(page.size(), 128 << 10); 269 | assert_eq!(pool.remain_page_cnt(), pool.total_page_cnt() - 1); 270 | drop(page); 271 | assert_eq!(pool.remain_page_cnt(), pool.total_page_cnt()); 272 | } 273 | 274 | #[tokio::test] 275 | async fn get_page_concurrently() { 276 | install_fmt_log(); 277 | let pool = MemoryPagePool::new(128 << 10, 300 << 20); 278 | 279 | let start = std::time::Instant::now(); 280 | let mut handles = vec![]; 281 | for _ in 0..pool.total_page_cnt() { 282 | let pool = pool.clone(); 283 | let handle = tokio::spawn(async move { 284 | let _page2 = pool.acquire_page().await; 285 | let page = pool.acquire_page().await; 286 | tokio::time::sleep(Duration::from_millis(1)).await; 287 | let mut cursor = Cursor::new(b"hello"); 288 | page.copy_from_reader(0, 5, &mut cursor).await.unwrap(); 289 | // let mut buf = page.as_mut_slice(); 290 | // let write_len = buf.write(b"hello").unwrap(); 291 | // assert_eq!(write_len, 5); 292 | }); 293 | handles.push(handle); 294 | } 295 | 296 | assert!(pool.remain_page_cnt() <= pool.total_page_cnt()); 297 | let _ = futures::future::join_all(handles).await; 298 | 299 | info!( 300 | "fill the whole pool {} cost: {:?}", 301 | ReadableSize(pool.capacity() as u64), 302 | start.elapsed(), 303 | ); 304 | 305 | assert_eq!(pool.remain_page_cnt(), pool.total_page_cnt()); 306 | } 307 | } 308 | -------------------------------------------------------------------------------- /components/storage/src/pool/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | pub mod disk_pool; 18 | pub mod memory_pool; 19 | 20 | use std::{ 21 | fmt::{Debug, Formatter}, 22 | sync::Arc, 23 | thread, 24 | }; 25 | 26 | use kiseki_common::{PAGE_BUFFER_SIZE, PAGE_SIZE}; 27 | use kiseki_utils::readable_size::ReadableSize; 28 | use lazy_static::lazy_static; 29 | use tracing::debug; 30 | 31 | use crate::err::Result; 32 | 33 | lazy_static! { 34 | pub static ref GLOBAL_MEMORY_PAGE_POOL: Arc = 35 | memory_pool::MemoryPagePool::new(PAGE_SIZE, PAGE_BUFFER_SIZE); 36 | pub static ref GLOBAL_HYBRID_PAGE_POOL: Arc = thread::spawn(|| { 37 | let runtime = tokio::runtime::Runtime::new().unwrap(); 38 | runtime.handle().block_on(async { 39 | Arc::new( 40 | PagePoolBuilder::default() 41 | .with_page_size(PAGE_SIZE) 42 | .with_memory_capacity(1 << 30) 43 | .with_disk_capacity(1 << 30) 44 | .build() 45 | .await 46 | .unwrap(), 47 | ) 48 | }) 49 | }) 50 | .join() 51 | .unwrap(); 52 | } 53 | 54 | const DEFAULT_DISK_PAGE_POOL_PATH: &str = "/tmp/kiseki.page_pool"; 55 | 56 | #[derive(Debug, Default)] 57 | pub struct PagePoolBuilder { 58 | page_size: usize, 59 | memory_capacity: usize, 60 | // disk page pool is optional 61 | disk_capacity: Option, 62 | } 63 | 64 | impl PagePoolBuilder { 65 | pub fn with_page_size(mut self, page_size: usize) -> Self { 66 | self.page_size = page_size; 67 | self 68 | } 69 | 70 | pub fn with_memory_capacity(mut self, memory_capacity: usize) -> Self { 71 | self.memory_capacity = memory_capacity; 72 | self 73 | } 74 | 75 | pub fn with_disk_capacity(mut self, disk_capacity: usize) -> Self { 76 | self.disk_capacity = Some(disk_capacity); 77 | self 78 | } 79 | 80 | pub async fn build(self) -> Result { 81 | let mut total_page_cnt = self.memory_capacity / self.page_size; 82 | let memory_pool = memory_pool::MemoryPagePool::new(self.page_size, self.memory_capacity); 83 | let (disk_pool, disk_capacity) = if let Some(disk_capacity) = self.disk_capacity { 84 | total_page_cnt += disk_capacity / self.page_size; 85 | let disk_pool = disk_pool::DiskPagePool::new( 86 | DEFAULT_DISK_PAGE_POOL_PATH, 87 | self.page_size, 88 | disk_capacity, 89 | ) 90 | .await?; 91 | (Some(disk_pool), disk_capacity) 92 | } else { 93 | (None, 0) 94 | }; 95 | 96 | Ok(HybridPagePool { 97 | page_size: self.page_size, 98 | memory_capacity: self.memory_capacity, 99 | disk_capacity, 100 | total_page_cnt, 101 | memory_pool, 102 | disk_pool, 103 | }) 104 | } 105 | } 106 | 107 | /// HybridPagePool is a hybrid page pool that can store pages in memory and on 108 | /// disk. It is used to store pages in memory when the memory is sufficient, and 109 | /// to store pages on disk when the memory is insufficient. 110 | pub struct HybridPagePool { 111 | page_size: usize, 112 | memory_capacity: usize, 113 | disk_capacity: usize, 114 | total_page_cnt: usize, 115 | 116 | memory_pool: Arc, 117 | disk_pool: Option>, 118 | } 119 | 120 | impl Debug for HybridPagePool { 121 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 122 | write!( 123 | f, 124 | "HybridPagePool {{ page_size: {}, memory_capacity: {}, disk_capacity: {}, disk_path: \ 125 | {}, remain_page_cnt: {}, total_page_cnt: {} }}", 126 | ReadableSize(self.page_size as u64), 127 | ReadableSize(self.memory_capacity as u64), 128 | ReadableSize(self.disk_capacity as u64), 129 | DEFAULT_DISK_PAGE_POOL_PATH, 130 | self.remain(), 131 | self.total_page_cnt, 132 | ) 133 | } 134 | } 135 | 136 | impl HybridPagePool { 137 | pub fn try_acquire_page(self: &Arc) -> Option { 138 | if let Some(page) = self.memory_pool.try_acquire_page() { 139 | return Some(Page::Memory(page)); 140 | } 141 | 142 | if let Some(disk_pool) = &self.disk_pool { 143 | if let Some(page) = disk_pool.try_acquire_page() { 144 | return Some(Page::Disk(page)); 145 | } 146 | } 147 | 148 | None 149 | } 150 | 151 | /// acquire_page will wait and acquire a page from the page pool. 152 | pub async fn acquire_page(self: &Arc) -> Page { 153 | // let disk_pool = self.disk_pool.as_ref().unwrap(); 154 | // let page = disk_pool.acquire_page().await; 155 | // return Page::Disk(page); 156 | 157 | debug!("pool free ratio {:?}", self.free_ratio()); 158 | 159 | if self.memory_pool.remain_page_cnt() > 0 { 160 | if let Some(page) = self.try_acquire_page() { 161 | return page; 162 | } 163 | } 164 | 165 | if let Some(disk_pool) = &self.disk_pool { 166 | let page = disk_pool.acquire_page().await; 167 | return Page::Disk(page); 168 | } 169 | let page = self.memory_pool.acquire_page().await; 170 | Page::Memory(page) 171 | } 172 | 173 | pub fn remain(&self) -> usize { 174 | self.memory_pool.remain_page_cnt() 175 | + self 176 | .disk_pool 177 | .as_ref() 178 | .map_or(0, |pool| pool.remain_page_cnt()) 179 | } 180 | 181 | pub fn total_page_cnt(&self) -> usize { self.total_page_cnt } 182 | 183 | pub fn capacity(&self) -> usize { self.memory_capacity + self.disk_capacity } 184 | 185 | pub fn free_ratio(&self) -> f64 { self.remain() as f64 / self.total_page_cnt as f64 } 186 | } 187 | 188 | pub enum Page { 189 | Memory(memory_pool::Page), 190 | Disk(disk_pool::Page), 191 | } 192 | 193 | impl Page { 194 | pub(crate) async fn copy_to_writer( 195 | &self, 196 | offset: usize, 197 | length: usize, 198 | writer: &mut W, 199 | ) -> Result<()> 200 | where 201 | W: tokio::io::AsyncWrite + Unpin + ?Sized, 202 | { 203 | match self { 204 | Page::Memory(page) => page.copy_to_writer(offset, length, writer).await, 205 | Page::Disk(page) => page.copy_to_writer(offset, length, writer).await, 206 | } 207 | } 208 | 209 | pub(crate) async fn copy_from_reader( 210 | &self, 211 | offset: usize, 212 | length: usize, 213 | reader: &mut R, 214 | ) -> Result<()> 215 | where 216 | R: tokio::io::AsyncRead + Unpin + ?Sized, 217 | { 218 | match self { 219 | Page::Memory(page) => page.copy_from_reader(offset, length, reader).await, 220 | Page::Disk(page) => page.copy_from_reader(offset, length, reader).await, 221 | } 222 | } 223 | } 224 | 225 | #[cfg(test)] 226 | mod tests { 227 | use std::io::Cursor; 228 | 229 | use kiseki_utils::logger::install_fmt_log; 230 | use tokio::time::Instant; 231 | use tracing::debug; 232 | 233 | use super::*; 234 | 235 | #[tokio::test] 236 | async fn basic() { 237 | install_fmt_log(); 238 | 239 | let start = Instant::now(); 240 | 241 | let pool = Arc::new( 242 | PagePoolBuilder::default() 243 | .with_page_size(PAGE_SIZE) 244 | .with_memory_capacity(300 << 20) 245 | .with_disk_capacity(1 << 30) 246 | .build() 247 | .await 248 | .unwrap(), 249 | ); 250 | 251 | let total_page_cnt = pool.total_page_cnt(); 252 | let handles = (0..total_page_cnt) 253 | .map(|i| { 254 | let pool = pool.clone(); 255 | tokio::spawn(async move { 256 | let page = pool.acquire_page().await; 257 | let mut data = Vec::from(format!("hello {}", i)); 258 | let data_len = data.len(); 259 | let mut cursor = Cursor::new(&mut data); 260 | page.copy_from_reader(0, data_len, &mut cursor) 261 | .await 262 | .unwrap(); 263 | 264 | let mut dst = vec![0u8; data_len]; 265 | let mut writer = Cursor::new(&mut dst); 266 | page.copy_to_writer(0, data_len, &mut writer).await.unwrap(); 267 | assert_eq!(dst, data); 268 | }) 269 | }) 270 | .collect::>(); 271 | 272 | futures::future::join_all(handles).await; 273 | 274 | debug!( 275 | "total time: {:?} for {}", 276 | start.elapsed(), 277 | ReadableSize(pool.capacity() as u64) 278 | ); 279 | } 280 | } 281 | -------------------------------------------------------------------------------- /components/types/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kiseki-types" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | bincode.workspace = true 10 | bitflags.workspace = true 11 | byteorder.workspace = true 12 | fuser.workspace = true 13 | lazy_static.workspace = true 14 | libc.workspace = true 15 | opendal.workspace = true 16 | rangemap.workspace = true 17 | serde.workspace = true 18 | snafu.workspace = true 19 | sonyflake.workspace = true 20 | tokio-util.workspace = true 21 | tracing.workspace = true 22 | 23 | kiseki-common = { path = "../../components/common" } 24 | kiseki-utils = { path = "../../components/utils" } 25 | -------------------------------------------------------------------------------- /components/types/src/entry.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use fuser::FileType; 18 | use kiseki_common::{DOT, DOT_DOT}; 19 | use serde::{Deserialize, Serialize}; 20 | 21 | use crate::{ 22 | attr::InodeAttr, 23 | ino::{Ino, ZERO_INO}, 24 | }; 25 | 26 | #[derive(Clone, Debug)] 27 | pub enum Entry { 28 | Full(FullEntry), 29 | DEntry(DEntry), 30 | } 31 | 32 | /// lookup 33 | #[derive(Debug, Serialize, Deserialize, Clone)] 34 | pub struct FullEntry { 35 | pub inode: Ino, 36 | pub name: String, 37 | pub attr: InodeAttr, 38 | } 39 | 40 | impl FullEntry { 41 | pub fn new(ino: Ino, name: &str, attr: InodeAttr) -> Self { 42 | FullEntry { 43 | inode: ino, 44 | name: name.to_string(), 45 | attr, 46 | } 47 | } 48 | 49 | pub fn to_fuse_attr>(&self, ino: I) -> fuser::FileAttr { 50 | self.attr.to_fuse_attr(ino) 51 | } 52 | } 53 | 54 | /// DEntry represents the storage structure of an entry in a directory. 55 | #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] 56 | pub struct DEntry { 57 | pub parent: Ino, 58 | pub name: String, 59 | pub inode: Ino, 60 | pub typ: FileType, 61 | } 62 | 63 | impl Entry { 64 | pub fn new_basic_entry_pair(ino: Ino, parent: Ino) -> Vec { 65 | vec![ 66 | Entry::DEntry(DEntry { 67 | parent: ZERO_INO, 68 | name: DOT.to_string(), 69 | inode: ino, 70 | typ: FileType::Directory, 71 | }), 72 | Entry::DEntry(DEntry { 73 | parent: ZERO_INO, 74 | name: DOT_DOT.to_string(), 75 | inode: parent, 76 | typ: FileType::Directory, 77 | }), 78 | ] 79 | } 80 | 81 | pub fn get_inode(&self) -> Ino { 82 | match self { 83 | Entry::Full(e) => e.inode, 84 | Entry::DEntry(e) => e.inode, 85 | } 86 | } 87 | 88 | pub fn is_file(&self) -> bool { 89 | match self { 90 | Entry::Full(e) => e.attr.kind == FileType::RegularFile, 91 | Entry::DEntry(e) => e.typ == FileType::RegularFile, 92 | } 93 | } 94 | 95 | pub fn get_file_type(&self) -> FileType { 96 | match self { 97 | Entry::Full(e) => e.attr.kind, 98 | Entry::DEntry(e) => e.typ, 99 | } 100 | } 101 | 102 | pub fn get_name(&self) -> &str { 103 | match self { 104 | Entry::Full(e) => &e.name, 105 | Entry::DEntry(e) => &e.name, 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /components/types/src/ino.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::{ 18 | fmt::{Display, Formatter}, 19 | ops::{Add, AddAssign}, 20 | }; 21 | 22 | use byteorder::{LittleEndian, WriteBytesExt}; 23 | use serde::{Deserialize, Serialize}; 24 | 25 | pub const ZERO_INO: Ino = Ino(0); 26 | pub const ROOT_INO: Ino = Ino(1); 27 | 28 | pub const MIN_INTERNAL_INODE: Ino = Ino(0x7FFFFFFF00000000); 29 | pub const LOG_INODE: Ino = Ino(0x7FFFFFFF00000001); 30 | pub const CONTROL_INODE: Ino = Ino(0x7FFFFFFF00000002); 31 | pub const STATS_INODE: Ino = Ino(0x7FFFFFFF00000003); 32 | pub const CONFIG_INODE: Ino = Ino(0x7FFFFFFF00000004); 33 | pub const MAX_INTERNAL_INODE: Ino = Ino(0x7FFFFFFF10000000); 34 | 35 | const INO_SIZE: usize = std::mem::size_of::(); 36 | 37 | #[derive( 38 | Debug, Default, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize, 39 | )] 40 | pub struct Ino(pub u64); 41 | 42 | impl Display for Ino { 43 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.0) } 44 | } 45 | 46 | impl AddAssign for Ino { 47 | fn add_assign(&mut self, rhs: Self) { self.0 += rhs.0; } 48 | } 49 | 50 | impl Add for Ino { 51 | type Output = Ino; 52 | 53 | fn add(self, rhs: Self) -> Self::Output { Self(self.0 + rhs.0) } 54 | } 55 | 56 | impl From for Ino { 57 | fn from(value: u64) -> Self { Self(value) } 58 | } 59 | impl From for u64 { 60 | fn from(val: Ino) -> Self { val.0 } 61 | } 62 | 63 | impl Ino { 64 | pub fn is_special(&self) -> bool { *self >= MIN_INTERNAL_INODE } 65 | 66 | pub fn is_normal(&self) -> bool { !self.is_special() } 67 | 68 | pub fn is_zero(&self) -> bool { self.0 == 0 } 69 | 70 | pub fn is_root(&self) -> bool { self.0 == ROOT_INO.0 } 71 | 72 | // FIXME: use a better way 73 | // key: AiiiiiiiiI 74 | // key-len: 10 75 | pub fn generate_key(&self) -> Vec { 76 | let mut buf = vec![0u8; 10]; 77 | buf.write_u8(b'A').unwrap(); 78 | buf.write_u64::(self.0).unwrap(); 79 | buf.write_u8(b'I').unwrap(); 80 | buf 81 | } 82 | 83 | pub fn generate_key_str(&self) -> String { 84 | // let key_buf = self.generate_key(); 85 | // String::from_utf8_lossy(&key_buf).to_string() 86 | format!("A{:x}I", self.0) 87 | } 88 | } 89 | 90 | #[cfg(test)] 91 | mod tests { 92 | use super::*; 93 | 94 | #[test] 95 | fn ino() { 96 | let key = ROOT_INO.generate_key(); 97 | println!("{:?}", key); 98 | let key_str = ROOT_INO.generate_key_str(); 99 | println!("{:?}", key_str) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /components/types/src/internal_nodes.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::{collections::HashMap, time::Duration}; 18 | 19 | use crate::{ 20 | attr::InodeAttr, 21 | entry::FullEntry, 22 | ino::{Ino, CONFIG_INODE, CONTROL_INODE, LOG_INODE, STATS_INODE}, 23 | }; 24 | 25 | pub const LOG_INODE_NAME: &str = ".accesslog"; 26 | pub const CONTROL_INODE_NAME: &str = ".control"; 27 | pub const STATS_INODE_NAME: &str = ".stats"; 28 | pub const CONFIG_INODE_NAME: &str = ".config"; 29 | #[derive(Debug)] 30 | pub struct InternalNodeTable { 31 | nodes: HashMap<&'static str, InternalNode>, 32 | } 33 | 34 | impl InternalNodeTable { 35 | pub fn new(_entry_timeout: (Duration, Duration)) -> Self { 36 | let mut map = HashMap::new(); 37 | let control_inode: InternalNode = InternalNode(FullEntry { 38 | inode: CONTROL_INODE, 39 | name: CONTROL_INODE_NAME.to_string(), 40 | attr: InodeAttr::default().set_mode(0o666).to_owned(), 41 | }); 42 | let log_inode: InternalNode = InternalNode(FullEntry { 43 | inode: LOG_INODE, 44 | name: LOG_INODE_NAME.to_string(), 45 | attr: InodeAttr::default().set_mode(0o400).to_owned(), 46 | }); 47 | let stats_inode: InternalNode = InternalNode(FullEntry { 48 | inode: STATS_INODE, 49 | name: STATS_INODE_NAME.to_string(), 50 | attr: InodeAttr::default().set_mode(0o400).to_owned(), 51 | }); 52 | let config_inode: InternalNode = InternalNode(FullEntry { 53 | inode: CONFIG_INODE, 54 | name: CONFIG_INODE_NAME.to_string(), 55 | attr: InodeAttr::default().set_mode(0o400).to_owned(), 56 | }); 57 | map.insert(LOG_INODE_NAME, log_inode); 58 | map.insert(CONTROL_INODE_NAME, control_inode); 59 | map.insert(STATS_INODE_NAME, stats_inode); 60 | map.insert(CONFIG_INODE_NAME, config_inode); 61 | Self { nodes: map } 62 | } 63 | } 64 | 65 | impl InternalNodeTable { 66 | pub fn get_internal_node_by_name(&self, name: &str) -> Option<&InternalNode> { 67 | self.nodes.get(name) 68 | } 69 | 70 | pub fn get_mut_internal_node_by_name(&mut self, name: &str) -> Option<&mut InternalNode> { 71 | self.nodes.get_mut(name) 72 | } 73 | 74 | pub fn get_internal_node(&self, ino: Ino) -> Option<&InternalNode> { 75 | self.nodes.values().find(|node| node.0.inode == ino) 76 | } 77 | 78 | pub fn add_prefix(&mut self) { 79 | for n in self.nodes.values_mut() { 80 | n.0.name = format!(".kfs{}", n.0.name); 81 | } 82 | } 83 | 84 | pub fn contains_name(&self, name: &str) -> bool { self.nodes.contains_key(name) } 85 | } 86 | 87 | #[derive(Debug)] 88 | pub struct InternalNode(pub FullEntry); 89 | 90 | impl InternalNode { 91 | pub fn get_attr(&self) -> InodeAttr { self.0.attr.clone() } 92 | } 93 | -------------------------------------------------------------------------------- /components/types/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | pub mod attr; 18 | pub mod entry; 19 | pub mod ino; 20 | pub mod internal_nodes; 21 | pub mod setting; 22 | pub mod slice; 23 | pub mod stat; 24 | 25 | pub use fuser::{FileType, Request}; 26 | pub use libc::c_int as Errno; 27 | 28 | /// Errors that can be converted to a raw OS error (errno) 29 | pub trait ToErrno { 30 | fn to_errno(&self) -> Errno; 31 | } 32 | -------------------------------------------------------------------------------- /components/types/src/setting.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use kiseki_common::{BLOCK_SIZE, CHUNK_SIZE, KISEKI, PAGE_SIZE}; 18 | use serde::{Deserialize, Serialize}; 19 | 20 | /// [Format] can be thought of as the configuration of the filesystem. 21 | /// We can set up different filesystems with different configurations 22 | /// on the same infrastructure, kind of like tenants. We can use 23 | /// Rocksdb's column family to implement this feature. But tikv doesn't 24 | /// open that feature yet. So there may some work to implement that. 25 | #[derive(Debug, Deserialize, Serialize, Clone)] 26 | pub struct Format { 27 | /// [name] of the filesystem 28 | pub name: String, 29 | 30 | /// [chunk_size] is the max size can one buffer 31 | /// hold no matter it is for reading or writing. 32 | pub chunk_size: usize, 33 | /// [block_size] is the max object size when we upload 34 | /// the file content data to the remote. 35 | /// 36 | /// When the data is not enough to fill the block, 37 | /// then the block size is equal to the data size, 38 | /// for example, the last block of the file. 39 | pub block_size: usize, 40 | /// [page_size] can be also called as the MIN_BLOCK_SIZE, 41 | /// which is the min size of the block. Since under the hood, 42 | /// each block is divided into fixed size pages. 43 | pub page_size: usize, 44 | 45 | /// [max_capacity] set limit on the capacity of the filesystem 46 | pub max_capacity: Option, 47 | /// [max_inodes] set limit on the number of inodes 48 | pub max_inodes: Option, 49 | } 50 | 51 | impl Default for Format { 52 | fn default() -> Self { 53 | Format { 54 | name: String::from(KISEKI), 55 | chunk_size: CHUNK_SIZE, // 64MB 56 | block_size: BLOCK_SIZE, // 4MB 57 | page_size: PAGE_SIZE, // 64KB 58 | max_capacity: None, 59 | max_inodes: None, 60 | } 61 | } 62 | } 63 | 64 | impl Format { 65 | pub fn with_name(&mut self, name: &str) -> &mut Self { 66 | self.name = name.to_string(); 67 | self 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /components/types/src/stat.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::fmt::{Debug, Formatter}; 18 | 19 | use kiseki_utils::readable_size::ReadableSize; 20 | use serde::{Deserialize, Serialize}; 21 | 22 | #[derive(Debug, Default, Serialize, Deserialize)] 23 | pub struct DirStat { 24 | pub length: i64, 25 | pub space: i64, 26 | pub inodes: i64, 27 | } 28 | 29 | /// [FSStat] represents the filesystem statistics. 30 | #[derive(Clone, Copy)] 31 | pub struct FSStat { 32 | /// Represents the total available size. 33 | pub total_size: u64, 34 | /// Represents the used size. 35 | pub used_size: u64, 36 | /// Represents the total used file count. 37 | pub file_count: u64, 38 | } 39 | 40 | impl Debug for FSStat { 41 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 42 | f.debug_struct("FSStat") 43 | .field("total_size", &ReadableSize(self.total_size)) 44 | .field("used_size", &ReadableSize(self.used_size)) 45 | .field("file_count", &self.file_count) 46 | .finish() 47 | } 48 | } 49 | 50 | impl Default for FSStat { 51 | fn default() -> Self { 52 | FSStat { 53 | total_size: u64::MAX, 54 | used_size: 0, 55 | file_count: 0, 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /components/utils/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kiseki-utils" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [features] 9 | tokio-console = ["console-subscriber", "tokio/tracing"] 10 | 11 | [dependencies] 12 | bytes.workspace = true 13 | lazy_static.workspace = true 14 | serde.workspace = true 15 | snafu.workspace = true 16 | sonyflake.workspace = true 17 | tokio.workspace = true 18 | tracing.workspace = true 19 | tracing-subscriber.workspace = true 20 | 21 | kiseki-common = { path = "../../components/common" } 22 | 23 | backtrace = "0.3.60" 24 | console-subscriber = { version = "0.2", optional = true } 25 | dotenvy = "0.15.7" 26 | num_cpus = "1.14.0" 27 | object_store = { version = "0.9.0", features = ["aws"] } 28 | once_cell = "1.19.0" 29 | opentelemetry = { version = "0.21.0" } 30 | opentelemetry-otlp = { version = "0.14.0", default_features = false, features = [ 31 | "http-proto", 32 | "trace", 33 | "http", 34 | "reqwest-client", 35 | "grpc-tonic", 36 | ] } 37 | opentelemetry-semantic-conventions = "0.13.0" 38 | opentelemetry_sdk = { version = "0.21.0", features = ["rt-tokio"] } 39 | proc-macro2 = "1.0.66" 40 | pyroscope = "0.5.7" 41 | pyroscope_pprofrs = "0.2.7" 42 | quote = "1.0" 43 | sentry = "0.32.1" 44 | sentry-tracing = "0.32.2" 45 | supports-color = "3.0.0" 46 | syn = "1.0" 47 | syn2 = { version = "2.0", package = "syn", features = [ 48 | "derive", 49 | "parsing", 50 | "printing", 51 | "clone-impls", 52 | "proc-macro", 53 | "extra-traits", 54 | "full", 55 | ] } 56 | toml = "0.8.9" 57 | tracing-appender = "0.2.3" 58 | tracing-core = "0.1.32" 59 | tracing-opentelemetry = "0.22.0" 60 | url = "2.5.0" 61 | users = "0.11.0" 62 | 63 | [dev-dependencies] 64 | tempfile.workspace = true 65 | -------------------------------------------------------------------------------- /components/utils/src/align.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use kiseki_common::{MAX_BLOCK_SIZE, MIN_BLOCK_SIZE}; 18 | use tracing::warn; 19 | 20 | pub fn align4k(length: u64) -> i64 { 21 | if length == 0 { 22 | return 1 << 12; // 4096 23 | } 24 | 25 | // Calculate the number of 4K blocks needed to hold the data 26 | let blocks_needed = (length - 1) / 4096 + 1; 27 | 28 | // Return the aligned length (number of blocks * block size) as i64 29 | (blocks_needed * 4096) as i64 30 | } 31 | 32 | pub fn align_to_block(s: usize) -> usize { 33 | let mut s = s; 34 | let mut bits = 0; 35 | while s > 1 { 36 | bits += 1; 37 | s >>= 1; 38 | } 39 | 40 | let adjusted_size = s << bits; 41 | 42 | if adjusted_size < MIN_BLOCK_SIZE { 43 | warn!( 44 | "Block size is too small: {}, using {} instead", 45 | adjusted_size, MIN_BLOCK_SIZE 46 | ); 47 | MIN_BLOCK_SIZE 48 | } else if adjusted_size > MAX_BLOCK_SIZE { 49 | warn!( 50 | "Block size is too large: {}, using {} instead", 51 | adjusted_size, MAX_BLOCK_SIZE 52 | ); 53 | MAX_BLOCK_SIZE 54 | } else { 55 | adjusted_size 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /components/utils/src/env.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::{error::Error, str::FromStr}; 18 | 19 | use snafu::{whatever, ResultExt, Whatever}; 20 | 21 | /// Reads an environment variable for the current process. 22 | /// 23 | /// Compared to [std::env::var] there are a couple of differences: 24 | /// 25 | /// - [var] uses [dotenvy] which loads the `.env` file from the current or 26 | /// parent directories before returning the value. 27 | /// 28 | /// - [var] returns `Ok(None)` (instead of `Err`) if an environment variable 29 | /// wasn't set. 30 | #[track_caller] 31 | pub fn var(key: &str) -> Result, Whatever> { 32 | match dotenvy::var(key) { 33 | Ok(content) => Ok(Some(content)), 34 | Err(dotenvy::Error::EnvVar(std::env::VarError::NotPresent)) => Ok(None), 35 | Err(error) => whatever!(Err(error), "Failed to read {key} environment variable"), 36 | } 37 | } 38 | 39 | /// Reads an environment variable for the current process, and parses it if 40 | /// it is set. 41 | /// 42 | /// Compared to [std::env::var] there are a couple of differences: 43 | /// 44 | /// - [var] uses [dotenvy] which loads the `.env` file from the current or 45 | /// parent directories before returning the value. 46 | /// 47 | /// - [var] returns `Ok(None)` (instead of `Err`) if an environment variable 48 | /// wasn't set. 49 | #[track_caller] 50 | pub fn var_parsed(key: &str) -> Result, Whatever> 51 | where 52 | R: FromStr, 53 | R::Err: Error + Send + Sync + 'static, 54 | { 55 | match var(key) { 56 | Ok(Some(content)) => Ok(Some(content.parse().with_whatever_context(|e| { 57 | format!("Failed to parse {key} environment variable; {e}") 58 | })?)), 59 | Ok(None) => Ok(None), 60 | Err(error) => Err(error), 61 | } 62 | } 63 | 64 | /// Reads an environment variable for the current process, and fails if it was 65 | /// not found. 66 | /// 67 | /// Compared to [std::env::var] there are a couple of differences: 68 | /// 69 | /// - [var] uses [dotenvy] which loads the `.env` file from the current or 70 | /// parent directories before returning the value. 71 | #[track_caller] 72 | pub fn required_var(key: &str) -> Result { required(var(key), key) } 73 | 74 | fn required(res: Result, Whatever>, key: &str) -> Result { 75 | match res { 76 | Ok(opt) => { 77 | if opt.is_none() { 78 | whatever!("Failed to find required {key} environment variable") 79 | } 80 | Ok(opt.unwrap()) 81 | } 82 | Err(error) => Err(error), 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /components/utils/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | pub mod align; 18 | pub mod env; 19 | pub mod logger; 20 | pub mod object_storage; 21 | pub mod panic_hook; 22 | pub mod pyroscope_init; 23 | pub mod readable_size; 24 | pub mod runtime; 25 | pub mod sentry_init; 26 | 27 | lazy_static::lazy_static! { 28 | pub static ref RANDOM_ID_GENERATOR: sonyflake::Sonyflake = 29 | sonyflake::Sonyflake::new().expect("failed to create id generator"); 30 | 31 | /// the user ID for the user running the process. 32 | static ref UID: u32 = users::get_current_uid(); 33 | /// the group ID for the user running the process. 34 | static ref GID: u32 = users::get_current_gid(); 35 | 36 | /// the number of available CPUs(number of logical cores.) of the current system. 37 | static ref NUM_CPUS: usize = num_cpus::get(); 38 | /// the number of physical cores of the current system. 39 | /// This will always return at least 1. 40 | static ref NUM_PHYSICAL_CPUS: usize = num_cpus::get_physical(); 41 | } 42 | 43 | pub fn random_id() -> u64 { 44 | RANDOM_ID_GENERATOR 45 | .next_id() 46 | .expect("failed to generate id") 47 | } 48 | 49 | #[inline(always)] 50 | pub fn num_cpus() -> usize { *NUM_CPUS } 51 | #[inline(always)] 52 | pub fn num_physical_cpus() -> usize { *NUM_PHYSICAL_CPUS } 53 | 54 | #[inline(always)] 55 | pub fn uid() -> u32 { *UID } 56 | 57 | #[inline(always)] 58 | pub fn gid() -> u32 { *GID } 59 | -------------------------------------------------------------------------------- /components/utils/src/object_storage.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::{path::Path, sync::Arc}; 18 | 19 | use object_store::{aws::AmazonS3Builder, ObjectStore}; 20 | 21 | pub type ObjectStorage = Arc; 22 | 23 | pub type LocalStorage = Arc; 24 | 25 | pub type ObjectStorageError = object_store::Error; 26 | 27 | pub type ObjectStoragePath = object_store::path::Path; 28 | 29 | pub type ObjectReader = object_store::GetResult; 30 | 31 | pub fn is_not_found_error(e: &ObjectStorageError) -> bool { 32 | matches!(e, ObjectStorageError::NotFound { .. }) 33 | } 34 | 35 | pub fn new_memory_object_store() -> ObjectStorage { 36 | Arc::new(object_store::memory::InMemory::new()) 37 | } 38 | 39 | pub fn new_local_object_store>( 40 | path: P, 41 | ) -> Result { 42 | let path = path.as_ref(); 43 | std::fs::create_dir_all(path).unwrap(); 44 | let path = path.to_str().unwrap(); 45 | let object_sto: Arc = 46 | Arc::new(object_store::local::LocalFileSystem::new_with_prefix(path)?); 47 | Ok(object_sto) 48 | } 49 | 50 | pub fn new_minio_store() -> Result { 51 | let object_sto: Arc = Arc::new( 52 | AmazonS3Builder::new() 53 | .with_region("auto") 54 | .with_endpoint("http://localhost:9000") 55 | .with_allow_http(true) 56 | .with_bucket_name("test") 57 | .with_access_key_id("minioadmin") 58 | .with_secret_access_key("minioadmin") 59 | .build()?, 60 | ); 61 | Ok(object_sto) 62 | } 63 | 64 | #[cfg(test)] 65 | mod tests { 66 | use bytes::Bytes; 67 | use object_store::{path::Path, ObjectStore}; 68 | use tokio::io::AsyncWriteExt; 69 | 70 | use super::*; 71 | 72 | #[tokio::test] 73 | async fn basic() { 74 | // let object_sto = debug_minio_store().unwrap(); 75 | // let object_sto = 76 | // new_local_object_store(kiseki_common::KISEKI_DEBUG_OBJECT_STORAGE). 77 | // unwrap(); 78 | let object_sto = new_memory_object_store(); 79 | 80 | let bytes = Bytes::from_static(b"hello"); 81 | let path = Path::parse("data/large_file").unwrap(); 82 | object_sto.put(&path, bytes).await.unwrap(); 83 | 84 | let bytes = object_sto.get(&path).await.unwrap(); 85 | let result = bytes.bytes().await.unwrap(); 86 | assert_eq!(result.as_ref(), b"hello".as_slice()); 87 | 88 | let path = Path::parse("data/large_file_multipart").unwrap(); 89 | let (_id, mut writer) = object_sto.put_multipart(&path).await.unwrap(); 90 | let bytes = Bytes::from_static(b"hello"); 91 | writer.write_all(&bytes).await.unwrap(); 92 | writer.flush().await.unwrap(); 93 | writer.shutdown().await.unwrap(); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /components/utils/src/panic_hook.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::panic; 18 | #[cfg(feature = "deadlock_detection")] 19 | use std::time::Duration; 20 | 21 | use backtrace::Backtrace; 22 | 23 | pub fn set_panic_hook() { 24 | // Set a panic hook that records the panic as a `tracing` event at the 25 | // `ERROR` verbosity level. 26 | // 27 | // If we are currently in a span when the panic occurred, the logged event 28 | // will include the current span, allowing the context in which the panic 29 | // occurred to be recorded. 30 | let default_hook = panic::take_hook(); 31 | panic::set_hook(Box::new(move |panic| { 32 | let backtrace = Backtrace::new(); 33 | let backtrace = format!("{backtrace:?}"); 34 | if let Some(location) = panic.location() { 35 | tracing::error!( 36 | message = %panic, 37 | backtrace = %backtrace, 38 | panic.file = location.file(), 39 | panic.line = location.line(), 40 | panic.column = location.column(), 41 | ); 42 | } else { 43 | tracing::error!(message = %panic, backtrace = %backtrace); 44 | } 45 | default_hook(panic); 46 | })); 47 | 48 | #[cfg(feature = "deadlock_detection")] 49 | let _ = std::thread::spawn(move || { 50 | loop { 51 | std::thread::sleep(Duration::from_secs(5)); 52 | let deadlocks = parking_lot::deadlock::check_deadlock(); 53 | if deadlocks.is_empty() { 54 | continue; 55 | } 56 | 57 | tracing::info!("{} deadlocks detected", deadlocks.len()); 58 | for (i, threads) in deadlocks.iter().enumerate() { 59 | tracing::info!("Deadlock #{}", i); 60 | for t in threads { 61 | tracing::info!("Thread Id {:#?}", t.thread_id()); 62 | tracing::info!("{:#?}", t.backtrace()); 63 | } 64 | } 65 | } 66 | }); 67 | } 68 | -------------------------------------------------------------------------------- /components/utils/src/pyroscope_init.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use pyroscope::{pyroscope::PyroscopeAgentRunning, PyroscopeAgent}; 18 | use pyroscope_pprofrs::{pprof_backend, PprofConfig}; 19 | use snafu::{ResultExt, Whatever}; 20 | 21 | use crate::env::var; 22 | 23 | pub type Guard = PyroscopeAgent; 24 | 25 | #[must_use] 26 | pub fn init_pyroscope() -> Result, Whatever> { 27 | let url = var("PYROSCOPE_SERVER_URL")?; 28 | if url.is_none() { 29 | return Ok(None); 30 | } 31 | 32 | let url = url.unwrap(); 33 | println!("found pyroscope url: {}, starting pyroscope agent ...", url); 34 | 35 | let agent = PyroscopeAgent::builder(url, String::from("kiseki")) 36 | .backend(pprof_backend(PprofConfig::new().sample_rate(100))) 37 | .build() 38 | .with_whatever_context(|_e| "failed to setup pyroscope agent")?; 39 | 40 | let agent_running = agent 41 | .start() 42 | .with_whatever_context(|_e| "failed to start pyroscope agent")?; 43 | 44 | Ok(Some(agent_running)) 45 | } 46 | -------------------------------------------------------------------------------- /components/utils/src/readable_size.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, PingCAP, Inc. Licensed under Apache-2.0. 2 | 3 | // Copyright 2023 Greptime Team 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | // This file is copied from https://github.com/tikv/raft-engine/blob/8dd2a39f359ff16f5295f35343f626e0c10132fa/src/util.rs 18 | 19 | use std::{ 20 | fmt::{self, Debug, Display, Write}, 21 | ops::{Div, Mul}, 22 | str::FromStr, 23 | }; 24 | 25 | use serde::{ 26 | de, 27 | de::{Unexpected, Visitor}, 28 | Deserialize, Deserializer, Serialize, Serializer, 29 | }; 30 | 31 | const UNIT: u64 = 1; 32 | 33 | const BINARY_DATA_MAGNITUDE: u64 = 1024; 34 | pub const B: u64 = UNIT; 35 | pub const KIB: u64 = B * BINARY_DATA_MAGNITUDE; 36 | pub const MIB: u64 = KIB * BINARY_DATA_MAGNITUDE; 37 | pub const GIB: u64 = MIB * BINARY_DATA_MAGNITUDE; 38 | pub const TIB: u64 = GIB * BINARY_DATA_MAGNITUDE; 39 | pub const PIB: u64 = TIB * BINARY_DATA_MAGNITUDE; 40 | 41 | #[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd)] 42 | pub struct ReadableSize(pub u64); 43 | 44 | impl ReadableSize { 45 | pub const fn kb(count: u64) -> ReadableSize { ReadableSize(count * KIB) } 46 | 47 | pub const fn mb(count: u64) -> ReadableSize { ReadableSize(count * MIB) } 48 | 49 | pub const fn gb(count: u64) -> ReadableSize { ReadableSize(count * GIB) } 50 | 51 | pub const fn as_mb(self) -> u64 { self.0 / MIB } 52 | 53 | pub const fn as_bytes(self) -> u64 { self.0 } 54 | 55 | pub const fn as_bytes_usize(self) -> usize { self.0 as usize } 56 | 57 | pub fn to_string(self) -> String { format!("{:?}", self) } 58 | } 59 | 60 | impl Div for ReadableSize { 61 | type Output = ReadableSize; 62 | 63 | fn div(self, rhs: u64) -> ReadableSize { ReadableSize(self.0 / rhs) } 64 | } 65 | 66 | impl Div for ReadableSize { 67 | type Output = u64; 68 | 69 | fn div(self, rhs: ReadableSize) -> u64 { self.0 / rhs.0 } 70 | } 71 | 72 | impl Mul for ReadableSize { 73 | type Output = ReadableSize; 74 | 75 | fn mul(self, rhs: u64) -> ReadableSize { ReadableSize(self.0 * rhs) } 76 | } 77 | 78 | impl Serialize for ReadableSize { 79 | fn serialize(&self, serializer: S) -> Result 80 | where 81 | S: Serializer, 82 | { 83 | let size = self.0; 84 | let mut buffer = String::new(); 85 | if size == 0 { 86 | write!(buffer, "{}KiB", size).unwrap(); 87 | } else if size % PIB == 0 { 88 | write!(buffer, "{}PiB", size / PIB).unwrap(); 89 | } else if size % TIB == 0 { 90 | write!(buffer, "{}TiB", size / TIB).unwrap(); 91 | } else if size % GIB == 0 { 92 | write!(buffer, "{}GiB", size / GIB).unwrap(); 93 | } else if size % MIB == 0 { 94 | write!(buffer, "{}MiB", size / MIB).unwrap(); 95 | } else if size % KIB == 0 { 96 | write!(buffer, "{}KiB", size / KIB).unwrap(); 97 | } else { 98 | return serializer.serialize_u64(size); 99 | } 100 | serializer.serialize_str(&buffer) 101 | } 102 | } 103 | 104 | impl FromStr for ReadableSize { 105 | type Err = String; 106 | 107 | // This method parses value in binary unit. 108 | fn from_str(s: &str) -> Result { 109 | let size_str = s.trim(); 110 | if size_str.is_empty() { 111 | return Err(format!("{:?} is not a valid size.", s)); 112 | } 113 | 114 | if !size_str.is_ascii() { 115 | return Err(format!("ASCII string is expected, but got {:?}", s)); 116 | } 117 | 118 | // size: digits and '.' as decimal separator 119 | let size_len = size_str 120 | .to_string() 121 | .chars() 122 | .take_while(|c| char::is_ascii_digit(c) || ['.', 'e', 'E', '-', '+'].contains(c)) 123 | .count(); 124 | 125 | // unit: alphabetic characters 126 | let (size, unit) = size_str.split_at(size_len); 127 | 128 | let unit = match unit.trim() { 129 | "K" | "KB" | "KiB" => KIB, 130 | "M" | "MB" | "MiB" => MIB, 131 | "G" | "GB" | "GiB" => GIB, 132 | "T" | "TB" | "TiB" => TIB, 133 | "P" | "PB" | "PiB" => PIB, 134 | "B" | "" => B, 135 | _ => { 136 | return Err(format!( 137 | "only B, KB, KiB, MB, MiB, GB, GiB, TB, TiB, PB, and PiB are supported: {:?}", 138 | s 139 | )); 140 | } 141 | }; 142 | 143 | match size.parse::() { 144 | Ok(n) => Ok(ReadableSize((n * unit as f64) as u64)), 145 | Err(_) => Err(format!("invalid size string: {:?}", s)), 146 | } 147 | } 148 | } 149 | 150 | impl Debug for ReadableSize { 151 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self) } 152 | } 153 | 154 | impl Display for ReadableSize { 155 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 156 | if self.0 >= PIB { 157 | write!(f, "{:.1}PiB", self.0 as f64 / PIB as f64) 158 | } else if self.0 >= TIB { 159 | write!(f, "{:.1}TiB", self.0 as f64 / TIB as f64) 160 | } else if self.0 >= GIB { 161 | write!(f, "{:.1}GiB", self.0 as f64 / GIB as f64) 162 | } else if self.0 >= MIB { 163 | write!(f, "{:.1}MiB", self.0 as f64 / MIB as f64) 164 | } else if self.0 >= KIB { 165 | write!(f, "{:.1}KiB", self.0 as f64 / KIB as f64) 166 | } else { 167 | write!(f, "{}B", self.0) 168 | } 169 | } 170 | } 171 | 172 | impl<'de> Deserialize<'de> for ReadableSize { 173 | fn deserialize(deserializer: D) -> Result 174 | where 175 | D: Deserializer<'de>, 176 | { 177 | struct SizeVisitor; 178 | 179 | impl<'de> Visitor<'de> for SizeVisitor { 180 | type Value = ReadableSize; 181 | 182 | fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 183 | formatter.write_str("valid size") 184 | } 185 | 186 | fn visit_i64(self, size: i64) -> Result 187 | where 188 | E: de::Error, 189 | { 190 | if size >= 0 { 191 | self.visit_u64(size as u64) 192 | } else { 193 | Err(E::invalid_value(Unexpected::Signed(size), &self)) 194 | } 195 | } 196 | 197 | fn visit_u64(self, size: u64) -> Result 198 | where 199 | E: de::Error, 200 | { 201 | Ok(ReadableSize(size)) 202 | } 203 | 204 | fn visit_str(self, size_str: &str) -> Result 205 | where 206 | E: de::Error, 207 | { 208 | size_str.parse().map_err(E::custom) 209 | } 210 | } 211 | 212 | deserializer.deserialize_any(SizeVisitor) 213 | } 214 | } 215 | 216 | #[cfg(test)] 217 | mod tests { 218 | use super::*; 219 | 220 | #[test] 221 | fn test_readable_size() { 222 | let s = ReadableSize::kb(2); 223 | assert_eq!(s.0, 2048); 224 | assert_eq!(s.as_mb(), 0); 225 | let s = ReadableSize::mb(2); 226 | assert_eq!(s.0, 2 * 1024 * 1024); 227 | assert_eq!(s.as_mb(), 2); 228 | let s = ReadableSize::gb(2); 229 | assert_eq!(s.0, 2 * 1024 * 1024 * 1024); 230 | assert_eq!(s.as_mb(), 2048); 231 | 232 | assert_eq!((ReadableSize::mb(2) / 2).0, MIB); 233 | assert_eq!((ReadableSize::mb(1) / 2).0, 512 * KIB); 234 | assert_eq!(ReadableSize::mb(2) / ReadableSize::kb(1), 2048); 235 | } 236 | } 237 | -------------------------------------------------------------------------------- /components/utils/src/runtime.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::future::Future; 18 | 19 | use once_cell::sync::Lazy; 20 | use tokio::task::JoinHandle; 21 | use tracing::debug; 22 | 23 | static GLOBAL_RUNTIME: Lazy = Lazy::new(|| { 24 | debug!("start tokio runtime"); 25 | tokio::runtime::Builder::new_multi_thread() 26 | .worker_threads(8) 27 | .enable_all() 28 | .build() 29 | .unwrap() 30 | }); 31 | 32 | pub fn handle() -> tokio::runtime::Handle { GLOBAL_RUNTIME.handle().clone() } 33 | 34 | pub fn spawn(future: F) -> JoinHandle 35 | where 36 | F: Future + Send + 'static, 37 | F::Output: Send + 'static, 38 | { 39 | GLOBAL_RUNTIME.spawn(future) 40 | } 41 | 42 | #[allow(dead_code)] 43 | pub fn spawn_blocking(func: F) -> JoinHandle 44 | where 45 | F: FnOnce() -> R + Send + 'static, 46 | R: Send + 'static, 47 | { 48 | GLOBAL_RUNTIME.spawn_blocking(func) 49 | } 50 | 51 | #[allow(dead_code)] 52 | pub fn block_on(future: F) -> F::Output { GLOBAL_RUNTIME.block_on(future) } 53 | -------------------------------------------------------------------------------- /components/utils/src/sentry_init.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | pub use sentry::release_name; 18 | use sentry::{types::Dsn, ClientInitGuard, IntoDsn}; 19 | use snafu::{ResultExt, Whatever}; 20 | use tracing::error; 21 | 22 | use crate::env::{required_var, var, var_parsed}; 23 | 24 | #[derive(Debug)] 25 | pub struct SentryConfig { 26 | pub dsn: Option, 27 | pub environment: Option, 28 | pub release: Option, 29 | pub traces_sample_rate: f32, 30 | } 31 | 32 | impl SentryConfig { 33 | pub fn from_environment() -> Result { 34 | let dsn = var("SENTRY_DSN")?.into_dsn().with_whatever_context(|e| { 35 | format!("SENTRY_DSN_API is not a valid Sentry DSN value {}", e) 36 | })?; 37 | 38 | let environment = 39 | match dsn { 40 | None => None, 41 | Some(_) => Some(required_var("SENTRY_ENVIRONMENT").with_whatever_context( 42 | |_| "SENTRY_ENV_API must be set when using SENTRY_DSN_API", 43 | )?), 44 | }; 45 | 46 | Ok(Self { 47 | dsn, 48 | environment, 49 | release: var("SENTRY_RELEASE")?, 50 | traces_sample_rate: var_parsed("SENTRY_TRACES_SAMPLE_RATE")?.unwrap_or(1.0), 51 | }) 52 | } 53 | } 54 | 55 | /// Initializes the Sentry SDK from the environment variables. 56 | /// 57 | /// If `SENTRY_DSN_API` is not set then Sentry will not be initialized, 58 | /// otherwise it is required to be a valid DSN string. `SENTRY_ENV_API` must 59 | /// be set if a DSN is provided. 60 | /// 61 | /// `HEROKU_SLUG_COMMIT`, if present, will be used as the `release` property 62 | /// on all events. 63 | #[must_use] 64 | pub fn init_sentry() -> Option { 65 | let config = match SentryConfig::from_environment() { 66 | Ok(config) => config, 67 | Err(error) => { 68 | error!( 69 | "Failed to read Sentry configuration from environment: {}", 70 | error 71 | ); 72 | return None; 73 | } 74 | }; 75 | 76 | println!("found sentry config: {:?}", config); 77 | 78 | let opts = sentry::ClientOptions { 79 | auto_session_tracking: true, 80 | dsn: config.dsn, 81 | environment: config.environment.map(Into::into), 82 | release: config.release.map(Into::into), 83 | session_mode: sentry::SessionMode::Request, 84 | traces_sample_rate: config.traces_sample_rate, 85 | ..Default::default() 86 | }; 87 | 88 | let guard = sentry::init(opts); 89 | Some(guard) 90 | } 91 | -------------------------------------------------------------------------------- /components/vfs/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kiseki-vfs" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | kiseki-common = { path = "../../components/common" } 10 | kiseki-meta = { path = "../../components/meta", features = ["meta-rocksdb"] } 11 | kiseki-storage = { path = "../../components/storage" } 12 | kiseki-types = { path = "../../components/types" } 13 | kiseki-utils = { path = "../../components/utils" } 14 | 15 | bincode.workspace = true 16 | bytes.workspace = true 17 | crossbeam.workspace = true 18 | crossbeam-queue.workspace = true 19 | dashmap.workspace = true 20 | fuser.workspace = true 21 | futures.workspace = true 22 | libc.workspace = true 23 | opendal.workspace = true 24 | rangemap.workspace = true 25 | scopeguard.workspace = true 26 | serde.workspace = true 27 | snafu.workspace = true 28 | sonyflake.workspace = true 29 | tokio.workspace = true 30 | tokio-util.workspace = true 31 | tracing.workspace = true 32 | 33 | [dev-dependencies] 34 | tempfile.workspace = true 35 | -------------------------------------------------------------------------------- /components/vfs/src/config.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::time::Duration; 18 | 19 | use kiseki_common::{BLOCK_SIZE, CHUNK_SIZE, PAGE_BUFFER_SIZE, PAGE_SIZE}; 20 | use serde::{Deserialize, Serialize}; 21 | 22 | #[derive(Debug, Clone, Serialize, Deserialize)] 23 | pub struct Config { 24 | pub backup_meta_interval: Duration, 25 | pub prefix_internal: bool, 26 | pub hide_internal: bool, 27 | 28 | // attributes cache timeout in seconds 29 | pub attr_timeout: Duration, 30 | // dir entry cache timeout in seconds 31 | pub dir_entry_timeout: Duration, 32 | // file entry cache timeout in seconds 33 | pub file_entry_timeout: Duration, 34 | 35 | // ========Object Storage Configs ===> 36 | pub object_storage_dsn: String, 37 | 38 | // ========Cache Configs ===> 39 | pub capacity: usize, 40 | 41 | // ========Buffer configs ===> 42 | /// The total memory size for the write/read buffer. 43 | pub total_buffer_capacity: usize, 44 | /// chunk_size is the max size can one buffer 45 | /// hold no matter it is for reading or writing. 46 | pub chunk_size: usize, 47 | /// block_size is the max size when we upload 48 | /// the data to the cloud. 49 | /// 50 | /// When the data is not enough to fill the block, 51 | /// then the block size is equal to the data size, 52 | /// for example, the last block of the file. 53 | pub block_size: usize, 54 | /// The page_size can be also called as the MIN_BLOCK_SIZE, 55 | /// which is the min size of the block. 56 | /// 57 | /// And under the hood, the block is divided into pages. 58 | pub page_size: usize, 59 | } 60 | 61 | impl Default for Config { 62 | fn default() -> Self { 63 | Self { 64 | backup_meta_interval: Default::default(), 65 | prefix_internal: false, 66 | hide_internal: false, 67 | attr_timeout: Duration::from_secs(1), 68 | dir_entry_timeout: Duration::from_secs(1), 69 | file_entry_timeout: Duration::from_secs(1), 70 | object_storage_dsn: kiseki_common::KISEKI_DEBUG_OBJECT_STORAGE.to_string(), 71 | capacity: 100 << 10, 72 | total_buffer_capacity: PAGE_BUFFER_SIZE, // 300MB 73 | chunk_size: CHUNK_SIZE, // 64MB 74 | block_size: BLOCK_SIZE, // 4MB 75 | page_size: PAGE_SIZE, // 64KB 76 | } 77 | } 78 | } 79 | 80 | /// Divide cpu num by a non-zero `divisor` and returns at least 1. 81 | fn divide_num_cpus(divisor: usize) -> usize { 82 | debug_assert!(divisor > 0); 83 | let cores = kiseki_utils::num_cpus(); 84 | debug_assert!(cores > 0); 85 | (cores + divisor - 1) / divisor 86 | } 87 | -------------------------------------------------------------------------------- /components/vfs/src/data_manager.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use std::{collections::HashMap, sync::Arc, time::SystemTime}; 18 | 19 | use kiseki_common::FH; 20 | use kiseki_meta::MetaEngineRef; 21 | use kiseki_storage::{ 22 | cache, 23 | cache::{ 24 | file_cache::{FileCache, FileCacheRef}, 25 | mem_cache::{MemCache, MemCacheRef}, 26 | }, 27 | }; 28 | use kiseki_types::ino::Ino; 29 | use kiseki_utils::object_storage::ObjectStorage; 30 | use tokio::sync::RwLock; 31 | use tracing::debug; 32 | 33 | use crate::{ 34 | err::Result, 35 | reader::FileReader, 36 | writer::{FileWriter, FileWritersRef}, 37 | }; 38 | 39 | pub(crate) type DataManagerRef = Arc; 40 | 41 | /// DataManager is responsible for managing the data of the VFS. 42 | pub(crate) struct DataManager { 43 | pub(crate) page_size: usize, 44 | pub(crate) block_size: usize, 45 | pub(crate) chunk_size: usize, 46 | pub(crate) file_writers: FileWritersRef, 47 | pub(crate) file_readers: RwLock>>>>>, 48 | pub(crate) id_generator: Arc, 49 | // Dependencies 50 | pub(crate) meta_engine: MetaEngineRef, 51 | pub(crate) object_storage: ObjectStorage, 52 | pub(crate) file_cache: FileCacheRef, 53 | pub(crate) mem_cache: MemCacheRef, 54 | // pub(crate) data_cache: CacheRef, 55 | } 56 | 57 | impl DataManager { 58 | pub(crate) fn new( 59 | page_size: usize, 60 | block_size: usize, 61 | chunk_size: usize, 62 | meta_engine_ref: MetaEngineRef, 63 | object_storage: ObjectStorage, 64 | ) -> Self { 65 | let remote_storage = object_storage.clone(); 66 | Self { 67 | page_size, 68 | block_size, 69 | chunk_size, 70 | file_writers: Arc::new(Default::default()), 71 | file_readers: Default::default(), 72 | id_generator: Arc::new(sonyflake::Sonyflake::new().unwrap()), 73 | meta_engine: meta_engine_ref, 74 | object_storage, 75 | file_cache: Arc::new( 76 | FileCache::new(cache::file_cache::Config::default(), remote_storage.clone()) 77 | .unwrap(), 78 | ), 79 | mem_cache: Arc::new(MemCache::new( 80 | cache::mem_cache::Config::default(), 81 | remote_storage, 82 | )), 83 | } 84 | } 85 | 86 | pub(crate) fn find_file_writer(&self, ino: Ino) -> Option> { 87 | self.file_writers.get(&ino).map(|r| r.value().clone()) 88 | } 89 | 90 | pub(crate) fn get_length(self: &Arc, ino: Ino) -> u64 { 91 | self.file_writers 92 | .get(&(ino)) 93 | .map_or(0, |w| w.value().get_length() as u64) 94 | } 95 | 96 | pub(crate) fn update_mtime(self: &Arc, ino: Ino, mtime: SystemTime) -> Result<()> { 97 | debug!("update_mtime do nothing, {ino}: {:?}", mtime); 98 | Ok(()) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /components/vfs/src/err.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | use kiseki_types::slice::SliceKey; 18 | use snafu::{Location, Snafu}; 19 | use tracing::error; 20 | 21 | #[derive(Snafu, Debug)] 22 | #[snafu(visibility(pub))] 23 | pub enum Error { 24 | JoinErr { 25 | #[snafu(implicit)] 26 | location: Location, 27 | source: tokio::task::JoinError, 28 | }, 29 | OpenDalError { 30 | #[snafu(implicit)] 31 | location: Location, 32 | source: opendal::Error, 33 | }, 34 | 35 | ObjectStorageError { 36 | #[snafu(implicit)] 37 | location: Location, 38 | source: kiseki_utils::object_storage::ObjectStorageError, 39 | }, 40 | 41 | ObjectBlockNotFound { 42 | #[snafu(implicit)] 43 | location: Location, 44 | key: SliceKey, 45 | }, 46 | 47 | StorageError { 48 | source: kiseki_storage::err::Error, 49 | }, 50 | MetaError { 51 | source: kiseki_meta::Error, 52 | }, 53 | 54 | // ====VFS==== 55 | LibcError { 56 | errno: libc::c_int, 57 | #[snafu(implicit)] 58 | location: Location, 59 | }, 60 | } 61 | 62 | pub type Result = std::result::Result; 63 | 64 | impl From for Error { 65 | fn from(value: kiseki_meta::Error) -> Self { Self::MetaError { source: value } } 66 | } 67 | 68 | impl From for Error { 69 | fn from(value: kiseki_storage::err::Error) -> Self { Self::StorageError { source: value } } 70 | } 71 | 72 | impl kiseki_types::ToErrno for Error { 73 | fn to_errno(&self) -> kiseki_types::Errno { 74 | match self { 75 | Self::LibcError { errno, .. } => { 76 | error!("libc error: {}", errno); 77 | *errno 78 | } 79 | Self::MetaError { source } => { 80 | error!("meta error: {}", source); 81 | source.to_errno() 82 | } 83 | _ => panic!("unhandled error type in to_errno {}", self), 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /components/vfs/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 kisekifs 2 | // 3 | // JuiceFS, Copyright 2020 Juicedata, Inc. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | mod config; 18 | 19 | pub use config::Config; 20 | mod err; 21 | mod handle; 22 | mod kiseki; 23 | pub use kiseki::KisekiVFS; 24 | mod data_manager; 25 | mod reader; 26 | mod writer; 27 | // mod writer_v1; 28 | -------------------------------------------------------------------------------- /criterion.toml: -------------------------------------------------------------------------------- 1 | # This is used to override the directory where cargo-criterion saves 2 | # its data and generates reports. 3 | criterion_home = "./target/criterion" 4 | 5 | # This is used to configure the format of cargo-criterion's command-line output. 6 | # Options are: 7 | # criterion: Prints confidence intervals for measurement and throughput, and 8 | # indicates whether a change was detected from the previous run. The default. 9 | # quiet: Like criterion, but does not indicate changes. Useful for simply 10 | # presenting output numbers, eg. on a library's README. 11 | # verbose: Like criterion, but prints additional statistics. 12 | # bencher: Emulates the output format of the bencher crate and nightly-only 13 | # libtest benchmarks. 14 | output_format = "criterion" 15 | 16 | # This is used to configure the plotting backend used by cargo-criterion. 17 | # Options are "gnuplot" and "plotters", or "auto", which will use gnuplot if it's 18 | # available or plotters if it isn't. 19 | ploting_backend = "auto" 20 | 21 | # The colors table allows users to configure the colors used by the charts 22 | # cargo-criterion generates. 23 | [colors] 24 | # These are used in many charts to compare the current measurement against 25 | # the previous one. 26 | current_sample = {r = 31, g = 120, b = 180} 27 | previous_sample = {r = 7, g = 26, b = 28} 28 | 29 | # These are used by the full PDF chart to highlight which samples were outliers. 30 | not_an_outlier = {r = 31, g = 120, b = 180} 31 | mild_outlier = {r = 5, g = 127, b = 0} 32 | severe_outlier = {r = 7, g = 26, b = 28} 33 | 34 | # These are used for the line chart to compare multiple different functions. 35 | comparison_colors = [ 36 | {r = 8, g = 34, b = 34}, 37 | {r = 6, g = 139, b = 87}, 38 | {r = 0, g = 139, b = 139}, 39 | {r = 5, g = 215, b = 0}, 40 | {r = 0, g = 0, b = 139}, 41 | {r = 0, g = 20, b = 60}, 42 | {r = 9, g = 0, b = 139}, 43 | {r = 0, g = 255, b = 127}, 44 | ] 45 | 46 | -------------------------------------------------------------------------------- /dev/lima-k8s.yml: -------------------------------------------------------------------------------- 1 | # Deploy kubernetes via kubeadm. 2 | # $ limactl start ./k8s.yaml 3 | # $ limactl shell k8s kubectl 4 | 5 | # It can be accessed from the host by exporting the kubeconfig file; 6 | # the ports are already forwarded automatically by lima: 7 | # 8 | # $ export KUBECONFIG=$(limactl list k8s --format 'unix://{{.Dir}}/copied-from-guest/kubeconfig.yaml') 9 | # $ kubectl get no 10 | # NAME STATUS ROLES AGE VERSION 11 | # lima-k8s Ready control-plane,master 44s v1.22.3 12 | 13 | arch: "x86_64" 14 | os: "Linux" 15 | memory: "32GiB" 16 | mounts: 17 | - location: "/home/dh/code/personal/rust/kisekifs" 18 | writable: true 19 | - location: "/tmp/lima" 20 | writable: true 21 | - location: "~/.config/nix" 22 | writable: false 23 | 24 | # This template requires Lima v0.20.0 or later. 25 | images: 26 | # Try to use release-yyyyMMdd image if available. Note that release-yyyyMMdd will be removed after several months. 27 | - location: "https://cloud-images.ubuntu.com/releases/24.04/release-20240423/ubuntu-24.04-server-cloudimg-amd64.img" 28 | arch: "x86_64" 29 | digest: "sha256:32a9d30d18803da72f5936cf2b7b9efcb4d0bb63c67933f17e3bdfd1751de3f3" 30 | - location: "https://cloud-images.ubuntu.com/releases/24.04/release-20240423/ubuntu-24.04-server-cloudimg-arm64.img" 31 | arch: "aarch64" 32 | digest: "sha256:c841bac00925d3e6892d979798103a867931f255f28fefd9d5e07e3e22d0ef22" 33 | # Fallback to the latest release image. 34 | # Hint: run `limactl prune` to invalidate the cache 35 | - location: "https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img" 36 | arch: "x86_64" 37 | - location: "https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img" 38 | arch: "aarch64" 39 | 40 | containerd: 41 | system: true 42 | user: false 43 | provision: 44 | # See 45 | - mode: system 46 | script: | 47 | #!/bin/bash 48 | set -eux -o pipefail 49 | command -v kubeadm >/dev/null 2>&1 && exit 0 50 | # Install and configure prerequisites 51 | cat < 82 | - mode: system 83 | script: | 84 | #!/bin/bash 85 | set -eux -o pipefail 86 | grep SystemdCgroup /etc/containerd/config.toml && exit 0 87 | grep "version = 2" /etc/containerd/config.toml || exit 1 88 | # Configuring the systemd cgroup driver 89 | # Overriding the sandbox (pause) image 90 | cat <>/etc/containerd/config.toml 91 | [plugins] 92 | [plugins."io.containerd.grpc.v1.cri"] 93 | sandbox_image = "$(kubeadm config images list | grep pause | sort -r | head -n1)" 94 | [plugins."io.containerd.grpc.v1.cri".containerd] 95 | [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] 96 | [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] 97 | runtime_type = "io.containerd.runc.v2" 98 | [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] 99 | SystemdCgroup = true 100 | EOF 101 | systemctl restart containerd 102 | # See 103 | - mode: system 104 | script: | 105 | #!/bin/bash 106 | set -eux -o pipefail 107 | test -e /etc/kubernetes/admin.conf && exit 0 108 | export KUBECONFIG=/etc/kubernetes/admin.conf 109 | kubeadm config images list 110 | kubeadm config images pull --cri-socket=unix:///run/containerd/containerd.sock 111 | # Initializing your control-plane node 112 | cat <kubeadm-config.yaml 113 | kind: InitConfiguration 114 | apiVersion: kubeadm.k8s.io/v1beta3 115 | nodeRegistration: 116 | criSocket: unix:///run/containerd/containerd.sock 117 | --- 118 | kind: ClusterConfiguration 119 | apiVersion: kubeadm.k8s.io/v1beta3 120 | apiServer: 121 | certSANs: # --apiserver-cert-extra-sans 122 | - "127.0.0.1" 123 | networking: 124 | podSubnet: "10.244.0.0/16" # --pod-network-cidr 125 | --- 126 | kind: KubeletConfiguration 127 | apiVersion: kubelet.config.k8s.io/v1beta1 128 | cgroupDriver: systemd 129 | EOF 130 | kubeadm init --config kubeadm-config.yaml 131 | # Installing a Pod network add-on 132 | kubectl apply -f https://github.com/flannel-io/flannel/releases/download/v0.24.0/kube-flannel.yml 133 | # Control plane node isolation 134 | kubectl taint nodes --all node-role.kubernetes.io/control-plane- 135 | # Replace the server address with localhost, so that it works also from the host 136 | sed -e "/server:/ s|https://.*:\([0-9]*\)$|https://127.0.0.1:\1|" -i $KUBECONFIG 137 | mkdir -p ${HOME:-/root}/.kube && cp -f $KUBECONFIG ${HOME:-/root}/.kube/config 138 | - mode: system 139 | script: | 140 | #!/bin/bash 141 | set -eux -o pipefail 142 | export KUBECONFIG=/etc/kubernetes/admin.conf 143 | mkdir -p {{.Home}}/.kube 144 | cp -f $KUBECONFIG {{.Home}}/.kube/config 145 | chown -R {{.User}} {{.Home}}/.kube 146 | probes: 147 | - description: "kubeadm to be installed" 148 | script: | 149 | #!/bin/bash 150 | set -eux -o pipefail 151 | if ! timeout 30s bash -c "until command -v kubeadm >/dev/null 2>&1; do sleep 3; done"; then 152 | echo >&2 "kubeadm is not installed yet" 153 | exit 1 154 | fi 155 | hint: | 156 | See "/var/log/cloud-init-output.log" in the guest 157 | - description: "kubeadm to be completed" 158 | script: | 159 | #!/bin/bash 160 | set -eux -o pipefail 161 | if ! timeout 300s bash -c "until test -f /etc/kubernetes/admin.conf; do sleep 3; done"; then 162 | echo >&2 "k8s is not running yet" 163 | exit 1 164 | fi 165 | hint: | 166 | The k8s kubeconfig file has not yet been created. 167 | - description: "kubernetes cluster to be running" 168 | script: | 169 | #!/bin/bash 170 | set -eux -o pipefail 171 | if ! timeout 300s bash -c "until kubectl version >/dev/null 2>&1; do sleep 3; done"; then 172 | echo >&2 "kubernetes cluster is not up and running yet" 173 | exit 1 174 | fi 175 | - description: "coredns deployment to be running" 176 | script: | 177 | #!/bin/bash 178 | set -eux -o pipefail 179 | kubectl wait -n kube-system --timeout=180s --for=condition=available deploy coredns 180 | copyToHost: 181 | - guest: "/etc/kubernetes/admin.conf" 182 | host: "{{.Dir}}/copied-from-guest/kubeconfig.yaml" 183 | deleteOnStop: true 184 | message: | 185 | To run `kubectl` on the host (assumes kubectl is installed), run the following commands: 186 | ------ 187 | export KUBECONFIG="{{.Dir}}/copied-from-guest/kubeconfig.yaml" 188 | kubectl ... 189 | ------ -------------------------------------------------------------------------------- /dev/lima-ubuntu.yml: -------------------------------------------------------------------------------- 1 | # This template requires Lima v0.7.0 or later. 2 | arch: "x86_64" 3 | os: "Linux" 4 | images: 5 | # Try to use release-yyyyMMdd image if available. Note that release-yyyyMMdd will be removed after several months. 6 | - location: "https://cloud-images.ubuntu.com/releases/22.04/release-20240126/ubuntu-22.04-server-cloudimg-amd64.img" 7 | arch: "x86_64" 8 | digest: "sha256:9f8a0d84b81a1d481aafca2337cb9f0c1fdf697239ac488177cf29c97d706c25" 9 | # Fallback to the latest release image. 10 | # Hint: run `limactl prune` to invalidate the cache 11 | - location: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img" 12 | arch: "x86_64" 13 | mounts: 14 | - location: "/home/dh/code/personal/rust/kisekifs" 15 | writable: true 16 | - location: "/tmp/lima" 17 | writable: true 18 | - location: "~/.config/nix" 19 | writable: false 20 | memory: "32GiB" 21 | provision: 22 | - mode: system 23 | script: | 24 | #!/bin/bash 25 | set -eux -o pipefail 26 | export DEBIAN_FRONTEND=noninteractive -------------------------------------------------------------------------------- /docker/Dockerfile.ubuntu: -------------------------------------------------------------------------------- 1 | FROM rust:1.77.2 as builder 2 | 3 | ENV LANG en_US.utf8 4 | WORKDIR /kiseki 5 | 6 | # Install dependencies. 7 | RUN --mount=type=cache,target=/var/cache/apt \ 8 | apt-get update && apt-get install -y \ 9 | curl \ 10 | fuse3 libfuse3-dev \ 11 | openssl libssl-dev \ 12 | clang \ 13 | protobuf-compiler \ 14 | curl \ 15 | git \ 16 | build-essential \ 17 | pkg-config 18 | 19 | COPY docker/entrypoint.sh . 20 | 21 | #Build the project in release mode. 22 | RUN --mount=target=. \ 23 | --mount=type=cache,target=/root/.cargo/registry \ 24 | cargo build --release --package kiseki-binary --target-dir /out/target 25 | 26 | # Export the binary to the clean image. 27 | FROM ubuntu:22.04 as base 28 | 29 | ARG OUTPUT_DIR 30 | 31 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get \ 32 | -y install ca-certificates \ 33 | fuse3 libfuse3-dev 34 | 35 | WORKDIR /kiseki 36 | COPY --from=builder /out/target/release/kiseki-binary /kiseki/bin/ 37 | COPY --from=builder /kiseki/entrypoint.sh /kiseki/bin/entrypoint.sh 38 | RUN chmod +x /kiseki/bin/entrypoint.sh 39 | ENV PATH /kiseki/bin/:$PATH 40 | 41 | # Dummy command to keep the container running 42 | # CMD ["tail", "-f", "/dev/null"] 43 | 44 | RUN kiseki-binary format 45 | # ENTRYPOINT [ kiseki-binary ] 46 | CMD ["tail", "-f", "/dev/null"] -------------------------------------------------------------------------------- /docker/Dockerfile.ubuntu.builder: -------------------------------------------------------------------------------- 1 | FROM rust:1.77.2 2 | 3 | WORKDIR /kiseki 4 | 5 | RUN apt-get update && \ 6 | DEBIAN_FRONTEND=noninteractive apt-get install -y pkg-config \ 7 | fuse3 libfuse3-dev \ 8 | openssl libssl-dev \ 9 | clang -------------------------------------------------------------------------------- /docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kiseki-binary format 4 | mkdir -p /tmp/kiseki 5 | kiseki-binary mount -------------------------------------------------------------------------------- /docs/book.toml: -------------------------------------------------------------------------------- 1 | [book] 2 | authors = ["ryan tan"] 3 | language = "en" 4 | multilingual = false 5 | src = "src" 6 | title = "Kiseki" 7 | -------------------------------------------------------------------------------- /docs/src/SUMMARY.md: -------------------------------------------------------------------------------- 1 | # Summary 2 | 3 | - [QuickStart](./quick_start.md) 4 | - [Design](./design.md) 5 | - [Bench](./bench.md) 6 | - [Background](./background.md) -------------------------------------------------------------------------------- /docs/src/background.md: -------------------------------------------------------------------------------- 1 | # Why Kiseki? 2 | 3 | Kiseki means miracle in Japanese, and I just pick it up from a Japanese 4 | song while browsing the internet. 5 | 6 | I am just interested in Rust and passionate about distributed systems. 7 | Kiseki is my own exploration of building a file system in this language. 8 | 9 | ## Will you continue to develop it? 10 | 11 | I don't know yet, the basic storage part is done, develop is fun and I have learned 12 | a lot from it. So basically I have achieved my goal. 13 | The rest of the work is endless posix compliance and performance optimization and 14 | edge case handling, since i'm not employed by any company at present, 15 | I have to focus on my job hunting and other stuff... 16 | But who knows, we will see. 17 | 18 | ## Contribution 19 | 20 | Very welcome, just open a PR or issue. 21 | 22 | ## Why JuiceFS? 23 | 24 | While JuiceFS offers a powerful tool for data management, 25 | porting it to Rust presents a unique opportunity to delve 26 | into the inner workings of file systems and gain a deeper 27 | understanding of their design and implementation. -------------------------------------------------------------------------------- /docs/src/bench.md: -------------------------------------------------------------------------------- 1 | # Bench 2 | 3 | Just for reference, the benchmark is totally not accurate. 4 | 5 | ## Juicefs's bench 6 | 7 | MetaEngine: local rocksdb; 8 | ObjectStorage: local minio storage; 9 | WriteBackCacheSize: 10G; 10 | WriteBufferSize: 2G(1G memory + 1G mmap); 11 | 12 | Cleaning kernel cache, may ask for root privilege... 13 | 14 | BlockSize: 1 MiB, BigFileSize: 1024 MiB, SmallFileSize: 128 KiB, SmallFileCount: 100, NumThreads: 1 15 | 16 | | ITEM | VALUE | COST | 17 | |------------------|-----------------|--------------| 18 | | Write big file | 2359.03 MiB/s | 0.43 s/file | 19 | | Read big file | 1139.28 MiB/s | 0.90 s/file | 20 | | Write small file | 1001.5 files/s | 1.00 ms/file | 21 | | Read small file | 6276.5 files/s | 0.16 ms/file | 22 | | Stat file | 94324.9 files/s | 0.01 ms/file | -------------------------------------------------------------------------------- /docs/src/design.md: -------------------------------------------------------------------------------- 1 | # FileWriter in VFS 2 | 3 | ## Logic Design 4 | 5 | A File is divide into chunks, and each chunk is divide into slices. 6 | Each chunk has a fixed size, slice size is not fixed, but it cannot exceed 7 | the chunk size. Each slice is a continuous piece of data, sequentially write 8 | can extend the slice, for random write, we basically always need to create a 9 | new slice. ~~In this way, we convert the random write to sequential write.~~ 10 | 11 | Actually, it still is random write in current implementation, no matter in 12 | write-back cache or not, we need to create a new file or object for each slice 13 | block. That introduce a lot of small files. 14 | 15 | In Juice's implementation, they employee a background check to flush some files 16 | to the remote for avoiding run out of inodes in the local file system. The 17 | write-back cache has some optimization to do. Like merge multiple block file into 18 | a big segment file like what levelDB does. 19 | 20 | ### Some Limitations 21 | 22 | 1. We need to commit each slice order by order 23 | 24 | 25 | -------------------------------------------------------------------------------- /docs/src/flox.md: -------------------------------------------------------------------------------- 1 | # Quick Start 2 | 3 | ## 1. Start virtual machine locally 4 | 5 | This project provides a lima file for setting up development quickly. 6 | 7 | All you need to do is installing the [lima](https://lima-vm.io/), 8 | and run the following code. It will setup an virtual machine, and installing some dependencies, 9 | and you don't need to worry about messing up your system. 10 | 11 | But before you start, you have to modify the mount point of the configuration, replace the mount path 12 | to your's. 13 | 14 | ### 1. Create virtual machine 15 | 16 | ```shell 17 | just create-lima 18 | ``` 19 | 20 | ### 2. Start virtual machine 21 | 22 | ```shell 23 | just start-lima 24 | ``` 25 | 26 | ### 3. Go into virtual machine and install dependencies 27 | 28 | ```shell 29 | sudo bash ./script/ubuntu/dep.sh 30 | ``` 31 | 32 | ### 4. Project has prepared 33 | 34 | -------------------------------------------------------------------------------- /docs/src/quick_start.md: -------------------------------------------------------------------------------- 1 | # QuickStart 2 | 3 | Run kisekifs in a light-weight virtual machine. 4 | 5 | ## 1. Install lima 6 | 7 | https://github.com/lima-vm/lima 8 | 9 | ## 2. Start a ubuntu virtual machine 10 | 11 | 1. create a virtual machine 12 | ```shell 13 | limactl start \ 14 | --name=kiseki \ 15 | --cpus=2 \ 16 | --memory=8 \ 17 | template://ubuntu-lts 18 | ``` 19 | 20 | 2. go into the virtual machine 21 | ```shell 22 | limactl shell kiseki 23 | ``` 24 | 25 | 3. Install git and download repo 26 | 27 | ```shell 28 | sudo apt-get update -y && sudo apt-get install git 29 | 30 | git clone https://github.com/crrow/kisekifs && cd kisekifs 31 | ``` 32 | 33 | 4. Prepare environments 34 | 35 | ```shell 36 | # Set proxy: export HTTP_PROXY=http://192.168.5.2:7890,HTTPS_PROXY=http://192.168.5.2:7890 37 | sudo bash ./hack/ubuntu/dep.sh 38 | 39 | echo 'export PATH=$HOME/.cargo/bin/:$PATH' >> $HOME/.bashrc 40 | ``` 41 | 42 | 5. Build kisekifs 43 | 44 | ```shell 45 | just build 46 | ``` 47 | 48 | 6. Run 49 | 50 | ```shell 51 | # command [just mount] will mount kisekifs on /tmp/kiseki 52 | just minio && just mount 53 | ``` -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | HOME_DIR := env_var('HOME') 2 | CARGO_REGISTRY_CACHE := HOME_DIR + "/.cargo/registry" 3 | PWD := invocation_directory() 4 | HTTP_PROXY := env_var("http_proxy") 5 | HTTPS_PROXY := env_var("https_proxy") 6 | 7 | # List available just recipes 8 | @help: 9 | just -l 10 | 11 | # Calculate code 12 | @cloc: 13 | cloc . --exclude-dir=vendor,docs,tests,examples,build,scripts,tools,target 14 | 15 | # Install workspace tools 16 | @install-tools: 17 | cargo install cargo-nextest 18 | cargo install cargo-release 19 | cargo install git-cliff 20 | cargo install cargo-criterion 21 | 22 | # Lint and automatically fix what we can fix 23 | @lint: 24 | cargo clippy --all-targets --workspace -- -D warnings 25 | 26 | @fmt: 27 | cargo +nightly fmt 28 | taplo format 29 | taplo format --check 30 | hawkeye format 31 | 32 | alias c := check 33 | @check: 34 | cargo check --all --all-features --all-targets 35 | 36 | alias t := test 37 | @test: 38 | cargo test --verbose -- --nocapture --show-output 39 | 40 | alias b := bench 41 | @bench: 42 | #cargo bench -- --verbose --color always --nocapture --show-output 43 | cargo bench 44 | 45 | @book: 46 | mdbook serve docs 47 | 48 | # Applications: 49 | 50 | @build: 51 | #cargo build --bin kiseki 52 | cargo build --package kiseki-binary 53 | 54 | @build-release: 55 | cargo build --release --package kiseki-binary 56 | 57 | alias sh := show-help 58 | @show-help: 59 | cargo run --bin kiseki help 60 | #cargo run --bin kiseki 61 | 62 | # ==================================================== mount 63 | 64 | alias sh-m := help-mount 65 | @help-mount: 66 | cargo run --color=always --package kiseki-binary help mount 67 | 68 | @mount: 69 | just clean 70 | just prepare 71 | cargo run --color=always --package kiseki-binary mount --level debug 72 | 73 | @release-mount: 74 | just clean 75 | just prepare 76 | cargo run --release --color=always --package kiseki-binary mount --no-log 77 | 78 | @profile-mount: 79 | just clean 80 | just prepare 81 | cargo flamegraph --package kiseki-binary -- mount --no-log 82 | 83 | # ==================================================== umount 84 | 85 | alias sh-um := help-umount 86 | @help-umount: 87 | cargo run --color=always --packgae kiseki-binary help umount 88 | 89 | @umount: 90 | cargo run --release --color=always --package kiseki-binary umount 91 | 92 | # ==================================================== format 93 | 94 | @format: 95 | cargo run --color=always --package kiseki-binary format 96 | 97 | @prepare: 98 | mkdir -p /tmp/kiseki /tmp/kiseki.meta/ 99 | just format 100 | 101 | alias sh-f := help-format 102 | @help-format: 103 | cargo run --color=always --bin kiseki help format 104 | 105 | # ==================================================== MinIO 106 | 107 | @minio: 108 | docker run -p 9000:9000 -p 9001:9001 \ 109 | quay.io/minio/minio server /data --console-address ":9001" 110 | 111 | # ==================================================== fio test 112 | 113 | @clean: 114 | - rm -r /tmp/kiseki 115 | echo "Done: remove mount point" 116 | - rm -r /tmp/kiseki.meta/ 117 | echo "Done: remove meta dir" 118 | - rm -r /tmp/kiseki.cache/ 119 | echo "Done: remove cache dir" 120 | - rm -r /tmp/kiseki.stage_cache/ 121 | echo "Done: remove stage cache dir" 122 | - rm -r /tmp/kiseki.data/ 123 | echo "Done: remove data dir" 124 | - rm -r /tmp/kiseki.log/ 125 | echo "Done: remove log dir" 126 | - rm -r /home/dh/kiseki/kiseki.data 127 | 128 | 129 | alias sw := seq-write 130 | @seq-write: 131 | - rm -r /tmp/kiseki/fio 132 | mkdir -p /tmp/kiseki/fio 133 | fio --name=jfs-test --directory=/tmp/kiseki/fio --ioengine=libaio --rw=write --bs=1m --size=1g --numjobs=4 --direct=1 --group_reporting 134 | 135 | alias rw := random-write 136 | @random-write: 137 | - rm -r /tmp/kiseki/fio 138 | mkdir -p /tmp/kiseki/fio 139 | fio --name=jfs-test --directory=/tmp/kiseki/fio --ioengine=libaio --rw=randwrite --bs=1m --size=512m --numjobs=4 --direct=1 --group_reporting 140 | 141 | alias sr := seq-read 142 | @seq-read: 143 | - rm -r /tmp/kiseki/fio 144 | mkdir -p /tmp/kiseki/fio 145 | fio --name=jfs-test --directory=/tmp/kiseki/fio --ioengine=libaio --rw=read --bs=1m --size=1g --numjobs=4 --direct=1 --group_reporting 146 | 147 | alias rr := random-read 148 | @random-read: 149 | - rm -r /tmp/kiseki/fio 150 | mkdir -p /tmp/kiseki/fio 151 | fio --name=jfs-test --directory=/tmp/kiseki/fio --ioengine=libaio --rw=randread --bs=1m --size=1g --numjobs=4 --direct=1 --group_reporting 152 | 153 | build-base-image: 154 | docker build -t kiseki-ubuntu-builder:v0.0.1 \ 155 | --build-arg http_proxy={{HTTP_PROXY}} \ 156 | --build-arg https_proxy={{HTTPS_PROXY}} \ 157 | -f docker/Dockerfile.ubuntu.builder . 158 | 159 | build-by-docker: 160 | docker run --network=host \ 161 | -v {{PWD}}:/kiseki -v {{CARGO_REGISTRY_CACHE}}:/root/.cargo/registry \ 162 | -w /kiseki kiseki-ubuntu-builder:v0.0.1 \ 163 | cargo build --release --package kiseki-binary 164 | 165 | build-image: 166 | docker build -t kisekifs:v0.0.1 \ 167 | -f docker/Dockerfile.ubuntu . 168 | 169 | create-lima: 170 | limactl create --name=kiseki dev/lima-k8s.yml 171 | 172 | start-lima: 173 | limactl start --name=kiseki 174 | 175 | stop-lima: 176 | limactl stop kiseki 177 | -------------------------------------------------------------------------------- /licenserc.toml: -------------------------------------------------------------------------------- 1 | # cargo install hawkeye 2 | 3 | baseDir = "." 4 | 5 | inlineHeader = """ 6 | Copyright 2024 kisekifs 7 | 8 | JuiceFS, Copyright 2020 Juicedata, Inc. 9 | 10 | Licensed under the Apache License, Version 2.0 (the "License"); 11 | you may not use this file except in compliance with the License. 12 | You may obtain a copy of the License at 13 | 14 | http://www.apache.org/licenses/LICENSE-2.0 15 | 16 | Unless required by applicable law or agreed to in writing, software 17 | distributed under the License is distributed on an "AS IS" BASIS, 18 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | See the License for the specific language governing permissions and 20 | limitations under the License. 21 | """ 22 | 23 | excludes = [ 24 | "components/binary/build.rs", 25 | "components/binary/src/cmd/mount.rs", 26 | "components/utils/src/readable_size.rs", 27 | "scripts", 28 | "docker", 29 | "tests", 30 | "tools", 31 | "benchmark", 32 | # hidden files 33 | ".cargo", 34 | ".databend", 35 | ".devcontainer", 36 | ".github", 37 | ".dockerignore", 38 | ".gitignore", 39 | ".gitattributes", 40 | ".editorconfig", 41 | "LICENSE", 42 | "Makefile", 43 | # docs and generated files 44 | "**/*.md", 45 | "**/*.hbs", 46 | "**/*.template", 47 | "**/*.cue", 48 | "**/*.json", 49 | "**/*.sql", 50 | "**/*.yml", 51 | "**/*.yaml", 52 | "**/*.toml", 53 | "**/*.lock", 54 | "**/*.yapf", 55 | "**/*.test", 56 | "**/*.txt", 57 | ] 58 | 59 | [properties] 60 | inceptionYear = 2024 61 | copyrightOwner = "Paku" 62 | -------------------------------------------------------------------------------- /oranda.json: -------------------------------------------------------------------------------- 1 | { 2 | "project": { 3 | "readme_path": "README.md" 4 | }, 5 | "build": { 6 | "path_prefix": "kisekifs" 7 | }, 8 | "styles": { 9 | "theme": "axolight", 10 | "favicon": "https://www.axo.dev/favicon.ico" 11 | }, 12 | "components": { 13 | "mdbook": { 14 | "path": "./docs" 15 | }, 16 | "changelog": true, 17 | "artifacts": { 18 | "package_managers": { 19 | "preferred": { 20 | "npm": "npm install @axodotdev/oranda --save-dev", 21 | "cargo": "cargo install oranda --locked --profile=dist" 22 | }, 23 | "additional": { 24 | "npx": "npx @axodotdev/oranda", 25 | "binstall": "cargo binstall oranda", 26 | "nix-env": "nix-env -i oranda", 27 | "nix flake": "nix profile install github:axodotdev/oranda" 28 | } 29 | } 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "1.77.2" 3 | components = ["rustfmt", "clippy"] -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | version = "Two" 2 | unstable_features = true 3 | edition = "2021" 4 | 5 | indent_style = "Block" 6 | newline_style = "Unix" 7 | use_field_init_shorthand = true 8 | use_try_shorthand = true 9 | 10 | # ==== comments & docs 11 | # Maximum length of comments. Default value: 80 12 | comment_width = 80 13 | normalize_comments = true 14 | normalize_doc_attributes = true 15 | # Break comments to fit on the line 16 | # https://rust-lang.github.io/rustfmt/?version=v1.6.0&search=#wrap_comments 17 | wrap_comments = true 18 | 19 | # ==== Abount format 20 | # https://rust-lang.github.io/rustfmt/?version=v1.6.0&search=#condense_wildcard_suffixes 21 | format_code_in_doc_comments = true 22 | format_macro_bodies = true 23 | format_macro_matchers = true 24 | # https://rust-lang.github.io/rustfmt/?version=v1.6.0&search=#format_strings 25 | format_strings = true 26 | # Put single-expression functions on a single line 27 | # https://rust-lang.github.io/rustfmt/?version=v1.6.0&search=#fn_single_line 28 | fn_single_line = true 29 | condense_wildcard_suffixes = true 30 | 31 | # ==== Import 32 | imports_granularity = "Crate" 33 | group_imports = "StdExternalCrate" 34 | 35 | # ==== Reorder 36 | reorder_impl_items = true 37 | reorder_imports = true 38 | reorder_modules = true 39 | 40 | struct_field_align_threshold = 20 41 | 42 | -------------------------------------------------------------------------------- /script/ubuntu/dep.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 kisekifs 3 | # 4 | # JuiceFS, Copyright 2020 Juicedata, Inc. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | # Exit immediately if any command fails 20 | set -eux -o pipefail 21 | export DEBIAN_FRONTEND=noninteractive 22 | 23 | sudo apt-get update -y && sudo apt-get install -y \ 24 | wget curl vim \ 25 | fuse3 libfuse3-dev \ 26 | build-essential \ 27 | pkg-config \ 28 | openssl libssl-dev \ 29 | clang \ 30 | fio \ 31 | curl 32 | 33 | install_flox() { 34 | # Check if flox is already installed 35 | if command -v flox &> /dev/null; then 36 | echo "flox is already installed." 37 | else 38 | # Download and run the flox installation script 39 | wget https://downloads.flox.dev/by-env/stable/deb/flox-1.2.2.x86_64-linux.deb -o /tmp/flox.deb 40 | dpkg -i /tmp/flox.deb 41 | fi 42 | } 43 | 44 | # install rust 45 | install_rust() { 46 | # Check if rustup is already installed 47 | if command -v rustup &> /dev/null; then 48 | echo "rustup is already installed." 49 | else 50 | # Download and run the rustup installation script 51 | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y 52 | fi 53 | 54 | # Add Cargo bin to PATH if it's not already included 55 | if [[ ":$PATH:" != *":/root/.cargo/bin:"* ]]; then 56 | export PATH="/root/.cargo/bin:$PATH" 57 | fi 58 | if [[ ":$PATH:" != *":$HOME/.cargo/bin:"* ]]; then 59 | export PATH="$HOME/.cargo/bin:$PATH" 60 | fi 61 | 62 | # Update Rust to the stable version 63 | rustup update stable 64 | 65 | # install some tools 66 | cargo install just hawkeye taplo-cli cargo-nextest cargo-release git-cliff cargo-criterion 67 | } 68 | 69 | install_flox 70 | install_rust 71 | 72 | just c 73 | 74 | -------------------------------------------------------------------------------- /taplo.toml: -------------------------------------------------------------------------------- 1 | ## https://taplo.tamasfe.dev/configuration/file.html 2 | 3 | include = ["**/Cargo.toml"] 4 | 5 | [formatting] 6 | # Align consecutive entries vertically. 7 | align_entries = false 8 | # Append trailing commas for multi-line arrays. 9 | array_trailing_comma = true 10 | # Expand arrays to multiple lines that exceed the maximum column width. 11 | array_auto_expand = true 12 | # Collapse arrays that don't exceed the maximum column width and don't contain comments. 13 | array_auto_collapse = false 14 | # Omit white space padding from single-line arrays 15 | compact_arrays = true 16 | # Omit white space padding from the start and end of inline tables. 17 | compact_inline_tables = false 18 | # Maximum column width in characters, affects array expansion and collapse, this doesn't take whitespace into account. 19 | # Note that this is not set in stone, and works on a best-effort basis. 20 | column_width = 120 21 | # Indent based on tables and arrays of tables and their subtables, subtables out of order are not indented. 22 | indent_tables = false 23 | # The substring that is used for indentation, should be tabs or spaces (but technically can be anything). 24 | indent_string = ' ' 25 | # Add trailing newline at the end of the file if not present. 26 | trailing_newline = true 27 | # Alphabetically reorder keys that are not separated by empty lines. 28 | reorder_keys = false 29 | # Maximum amount of allowed consecutive blank lines. This does not affect the whitespace at the end of the document, as it is always stripped. 30 | allowed_blank_lines = 1 31 | # Use CRLF for line endings. 32 | crlf = false 33 | 34 | [[rule]] 35 | keys = ["build-dependencies", "dependencies", "dev-dependencies", "workspace.dependencies"] 36 | formatting = { reorder_keys = true } 37 | -------------------------------------------------------------------------------- /tests/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tests" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | rangemap = "1.4.0" 10 | -------------------------------------------------------------------------------- /tests/src/lib.rs: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /typos.toml: -------------------------------------------------------------------------------- 1 | [default.extend-words] 2 | ploting = "ploting" 3 | mknod = "mknod" --------------------------------------------------------------------------------