├── .config └── nextest.toml ├── .github ├── dependabot.yaml ├── pull_request_template.md └── workflows │ ├── beta.yaml │ ├── ci.yaml │ ├── cleanup.yaml │ ├── commit.yaml │ ├── docs.yaml │ ├── flaky.yaml │ └── tests.yaml ├── .gitignore ├── .img └── iroh_wordmark.svg ├── CHANGELOG.md ├── Cargo.lock ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── Makefile.toml ├── README.md ├── cliff.toml ├── code_of_conduct.md ├── deny.toml ├── docs └── img │ ├── get_machine.drawio │ └── get_machine.drawio.svg ├── examples ├── custom-protocol.rs ├── discovery-local-network.rs ├── fetch-fsm.rs ├── fetch-stream.rs ├── hello-world-fetch.rs ├── hello-world-provide.rs ├── provide-bytes.rs └── transfer.rs ├── proptest-regressions ├── protocol │ └── range_spec.txt └── provider.txt ├── release.toml ├── src ├── cli.rs ├── cli │ └── tags.rs ├── downloader.rs ├── downloader │ ├── get.rs │ ├── invariants.rs │ ├── progress.rs │ ├── test.rs │ └── test │ │ ├── dialer.rs │ │ └── getter.rs ├── export.rs ├── format.rs ├── format │ └── collection.rs ├── get.rs ├── get │ ├── db.rs │ ├── error.rs │ ├── progress.rs │ └── request.rs ├── hash.rs ├── hashseq.rs ├── lib.rs ├── metrics.rs ├── net_protocol.rs ├── protocol.rs ├── protocol │ └── range_spec.rs ├── provider.rs ├── rpc.rs ├── rpc │ ├── client.rs │ ├── client │ │ ├── blobs.rs │ │ ├── blobs │ │ │ └── batch.rs │ │ └── tags.rs │ ├── proto.rs │ └── proto │ │ ├── blobs.rs │ │ └── tags.rs ├── store.rs ├── store │ ├── bao_file.rs │ ├── fs.rs │ ├── fs │ │ ├── tables.rs │ │ ├── test_support.rs │ │ ├── tests.rs │ │ ├── util.rs │ │ └── validate.rs │ ├── mem.rs │ ├── mutable_mem_storage.rs │ ├── readonly_mem.rs │ └── traits.rs ├── ticket.rs ├── util.rs └── util │ ├── fs.rs │ ├── hexdump.rs │ ├── io.rs │ ├── local_pool.rs │ ├── mem_or_file.rs │ ├── progress.rs │ └── sparse_mem_file.rs └── tests ├── blobs.rs ├── gc.rs ├── rpc.rs └── tags.rs /.config/nextest.toml: -------------------------------------------------------------------------------- 1 | [test-groups] 2 | run-in-isolation = { max-threads = 32 } 3 | # these are tests that must not run with other tests concurrently. All tests in 4 | # this group can take up at most 32 threads among them, but each one requiring 5 | # 16 threads also. The effect should be that tests run isolated. 6 | 7 | [[profile.ci.overrides]] 8 | filter = 'test(::run_in_isolation::)' 9 | test-group = 'run-in-isolation' 10 | threads-required = 32 11 | -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | # Keep GitHub Actions up to date with GitHub's Dependabot... 2 | # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot 3 | # https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem 4 | version: 2 5 | updates: 6 | - package-ecosystem: github-actions 7 | directory: / 8 | groups: 9 | github-actions: 10 | patterns: 11 | - "*" # Group all Actions updates into a single larger pull request 12 | schedule: 13 | interval: weekly 14 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | 4 | 5 | ## Breaking Changes 6 | 7 | 8 | 9 | ## Notes & open questions 10 | 11 | 12 | 13 | ## Change checklist 14 | 15 | - [ ] Self-review. 16 | - [ ] Documentation updates following the [style guide](https://rust-lang.github.io/rfcs/1574-more-api-documentation-conventions.html#appendix-a-full-conventions-text), if relevant. 17 | - [ ] Tests if relevant. 18 | - [ ] All breaking changes documented. 19 | -------------------------------------------------------------------------------- /.github/workflows/beta.yaml: -------------------------------------------------------------------------------- 1 | # Run tests using the beta Rust compiler 2 | 3 | name: Beta Rust 4 | 5 | on: 6 | schedule: 7 | # 06:50 UTC every Monday 8 | - cron: '50 6 * * 1' 9 | workflow_dispatch: 10 | 11 | concurrency: 12 | group: beta-${{ github.workflow }}-${{ github.ref }} 13 | cancel-in-progress: true 14 | 15 | env: 16 | IROH_FORCE_STAGING_RELAYS: "1" 17 | 18 | jobs: 19 | tests: 20 | uses: './.github/workflows/tests.yaml' 21 | with: 22 | rust-version: beta 23 | notify: 24 | needs: tests 25 | if: ${{ always() }} 26 | runs-on: ubuntu-latest 27 | steps: 28 | - name: Extract test results 29 | run: | 30 | printf '${{ toJSON(needs) }}\n' 31 | result=$(echo '${{ toJSON(needs) }}' | jq -r .tests.result) 32 | echo TESTS_RESULT=$result 33 | echo "TESTS_RESULT=$result" >>"$GITHUB_ENV" 34 | - name: Notify discord on failure 35 | uses: n0-computer/discord-webhook-notify@v1 36 | if: ${{ env.TESTS_RESULT == 'failure' }} 37 | with: 38 | severity: error 39 | details: | 40 | Rustc beta tests failed in **${{ github.repository }}** 41 | See https://github.com/${{ github.repository }}/actions/workflows/beta.yaml 42 | webhookUrl: ${{ secrets.DISCORD_N0_GITHUB_CHANNEL_WEBHOOK_URL }} 43 | 44 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: 5 | types: [ 'labeled', 'unlabeled', 'opened', 'synchronize', 'reopened' ] 6 | merge_group: 7 | push: 8 | branches: 9 | - main 10 | 11 | concurrency: 12 | group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 13 | cancel-in-progress: true 14 | 15 | env: 16 | RUST_BACKTRACE: 1 17 | RUSTFLAGS: -Dwarnings 18 | RUSTDOCFLAGS: -Dwarnings 19 | MSRV: "1.81" 20 | SCCACHE_CACHE_SIZE: "50G" 21 | IROH_FORCE_STAGING_RELAYS: "1" 22 | 23 | jobs: 24 | tests: 25 | name: CI Test Suite 26 | if: "github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'flaky-test')" 27 | uses: './.github/workflows/tests.yaml' 28 | 29 | cross_build: 30 | name: Cross Build Only 31 | if: "github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'flaky-test')" 32 | timeout-minutes: 30 33 | runs-on: [self-hosted, linux, X64] 34 | strategy: 35 | fail-fast: false 36 | matrix: 37 | target: 38 | # cross tests are currently broken vor armv7 and aarch64 39 | # see https://github.com/cross-rs/cross/issues/1311 40 | # - armv7-linux-androideabi 41 | # - aarch64-linux-android 42 | # Freebsd execution fails in cross 43 | # - i686-unknown-freebsd # Linking fails :/ 44 | - x86_64-unknown-freebsd 45 | # Netbsd execution fails to link in cross 46 | # - x86_64-unknown-netbsd 47 | steps: 48 | - name: Checkout 49 | uses: actions/checkout@v4 50 | with: 51 | submodules: recursive 52 | 53 | - name: Install rust stable 54 | uses: dtolnay/rust-toolchain@stable 55 | 56 | - name: Cleanup Docker 57 | continue-on-error: true 58 | run: | 59 | docker kill $(docker ps -q) 60 | 61 | # See https://github.com/cross-rs/cross/issues/1222 62 | - uses: taiki-e/install-action@cross 63 | 64 | - name: build 65 | # cross tests are currently broken vor armv7 and aarch64 66 | # see https://github.com/cross-rs/cross/issues/1311. So on 67 | # those platforms we only build but do not run tests. 68 | run: cross build --all --target ${{ matrix.target }} 69 | env: 70 | RUST_LOG: ${{ runner.debug && 'TRACE' || 'DEBUG'}} 71 | 72 | android_build: 73 | name: Android Build Only 74 | if: "github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'flaky-test')" 75 | timeout-minutes: 30 76 | # runs-on: ubuntu-latest 77 | runs-on: [self-hosted, linux, X64] 78 | strategy: 79 | fail-fast: false 80 | matrix: 81 | target: 82 | - aarch64-linux-android 83 | - armv7-linux-androideabi 84 | steps: 85 | - name: Checkout 86 | uses: actions/checkout@v4 87 | 88 | - name: Set up Rust 89 | uses: dtolnay/rust-toolchain@stable 90 | with: 91 | target: ${{ matrix.target }} 92 | - name: Install rustup target 93 | run: rustup target add ${{ matrix.target }} 94 | 95 | - name: Setup Java 96 | uses: actions/setup-java@v4 97 | with: 98 | distribution: 'temurin' 99 | java-version: '17' 100 | 101 | - name: Setup Android SDK 102 | uses: android-actions/setup-android@v3 103 | 104 | - name: Setup Android NDK 105 | uses: arqu/setup-ndk@main 106 | id: setup-ndk 107 | with: 108 | ndk-version: r23 109 | add-to-path: true 110 | 111 | - name: Build 112 | env: 113 | ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} 114 | run: | 115 | cargo install --version 3.5.4 cargo-ndk 116 | cargo ndk --target ${{ matrix.target }} build 117 | 118 | cross_test: 119 | name: Cross Test 120 | if: "github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'flaky-test')" 121 | timeout-minutes: 30 122 | runs-on: [self-hosted, linux, X64] 123 | strategy: 124 | fail-fast: false 125 | matrix: 126 | target: 127 | - i686-unknown-linux-gnu 128 | steps: 129 | - name: Checkout 130 | uses: actions/checkout@v4 131 | with: 132 | submodules: recursive 133 | 134 | - name: Install rust stable 135 | uses: dtolnay/rust-toolchain@stable 136 | 137 | - name: Cleanup Docker 138 | continue-on-error: true 139 | run: | 140 | docker kill $(docker ps -q) 141 | 142 | # See https://github.com/cross-rs/cross/issues/1222 143 | - uses: taiki-e/install-action@cross 144 | 145 | - name: test 146 | run: cross test --all --target ${{ matrix.target }} -- --test-threads=12 147 | env: 148 | RUST_LOG: ${{ runner.debug && 'TRACE' || 'DEBUG' }} 149 | 150 | check_semver: 151 | runs-on: ubuntu-latest 152 | env: 153 | RUSTC_WRAPPER: "sccache" 154 | SCCACHE_GHA_ENABLED: "on" 155 | steps: 156 | - uses: actions/checkout@v4 157 | with: 158 | fetch-depth: 0 159 | - name: Install sccache 160 | uses: mozilla-actions/sccache-action@v0.0.9 161 | 162 | - name: Setup Environment (PR) 163 | if: ${{ github.event_name == 'pull_request' }} 164 | shell: bash 165 | run: | 166 | echo "HEAD_COMMIT_SHA=$(git rev-parse origin/${{ github.base_ref }})" >> ${GITHUB_ENV} 167 | - name: Setup Environment (Push) 168 | if: ${{ github.event_name == 'push' || github.event_name == 'merge_group' }} 169 | shell: bash 170 | run: | 171 | echo "HEAD_COMMIT_SHA=$(git rev-parse origin/main)" >> ${GITHUB_ENV} 172 | - name: Check semver 173 | # uses: obi1kenobi/cargo-semver-checks-action@v2 174 | uses: n0-computer/cargo-semver-checks-action@feat-baseline 175 | with: 176 | package: iroh-blobs 177 | baseline-rev: ${{ env.HEAD_COMMIT_SHA }} 178 | use-cache: false 179 | 180 | check_fmt: 181 | timeout-minutes: 30 182 | name: Checking fmt 183 | runs-on: ubuntu-latest 184 | env: 185 | RUSTC_WRAPPER: "sccache" 186 | SCCACHE_GHA_ENABLED: "on" 187 | steps: 188 | - uses: actions/checkout@v4 189 | - uses: dtolnay/rust-toolchain@stable 190 | with: 191 | components: rustfmt 192 | - uses: mozilla-actions/sccache-action@v0.0.9 193 | - uses: taiki-e/install-action@cargo-make 194 | - run: cargo make format-check 195 | 196 | check_docs: 197 | timeout-minutes: 30 198 | name: Checking docs 199 | runs-on: ubuntu-latest 200 | env: 201 | RUSTC_WRAPPER: "sccache" 202 | SCCACHE_GHA_ENABLED: "on" 203 | steps: 204 | - uses: actions/checkout@v4 205 | - uses: dtolnay/rust-toolchain@master 206 | with: 207 | toolchain: nightly-2024-11-30 208 | - name: Install sccache 209 | uses: mozilla-actions/sccache-action@v0.0.9 210 | 211 | - name: Docs 212 | run: cargo doc --workspace --all-features --no-deps --document-private-items 213 | env: 214 | RUSTDOCFLAGS: --cfg docsrs 215 | 216 | clippy_check: 217 | timeout-minutes: 30 218 | runs-on: ubuntu-latest 219 | env: 220 | RUSTC_WRAPPER: "sccache" 221 | SCCACHE_GHA_ENABLED: "on" 222 | steps: 223 | - uses: actions/checkout@v4 224 | - uses: dtolnay/rust-toolchain@stable 225 | with: 226 | components: clippy 227 | - name: Install sccache 228 | uses: mozilla-actions/sccache-action@v0.0.9 229 | 230 | # TODO: We have a bunch of platform-dependent code so should 231 | # probably run this job on the full platform matrix 232 | - name: clippy check (all features) 233 | run: cargo clippy --workspace --all-features --all-targets --bins --tests --benches 234 | 235 | - name: clippy check (no features) 236 | run: cargo clippy --workspace --no-default-features --lib --bins --tests 237 | 238 | - name: clippy check (default features) 239 | run: cargo clippy --workspace --all-targets 240 | 241 | msrv: 242 | if: "github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'flaky-test')" 243 | timeout-minutes: 30 244 | name: Minimal Supported Rust Version 245 | runs-on: ubuntu-latest 246 | env: 247 | RUSTC_WRAPPER: "sccache" 248 | SCCACHE_GHA_ENABLED: "on" 249 | steps: 250 | - uses: actions/checkout@v4 251 | - uses: dtolnay/rust-toolchain@master 252 | with: 253 | toolchain: ${{ env.MSRV }} 254 | - name: Install sccache 255 | uses: mozilla-actions/sccache-action@v0.0.9 256 | 257 | - name: Check MSRV all features 258 | run: | 259 | cargo +$MSRV check --workspace --all-targets 260 | 261 | cargo_deny: 262 | timeout-minutes: 30 263 | name: cargo deny 264 | runs-on: ubuntu-latest 265 | steps: 266 | - uses: actions/checkout@v4 267 | - uses: EmbarkStudios/cargo-deny-action@v2 268 | with: 269 | arguments: --workspace --all-features 270 | command: check 271 | command-arguments: "-Dwarnings" 272 | 273 | codespell: 274 | timeout-minutes: 30 275 | runs-on: ubuntu-latest 276 | steps: 277 | - uses: actions/checkout@v4 278 | - run: pip install --user codespell[toml] 279 | - run: codespell --ignore-words-list=ans,atmost,crate,inout,ratatui,ser,stayin,swarmin,worl --skip=CHANGELOG.md 280 | -------------------------------------------------------------------------------- /.github/workflows/cleanup.yaml: -------------------------------------------------------------------------------- 1 | # Run tests using the beta Rust compiler 2 | 3 | name: Cleanup 4 | 5 | on: 6 | schedule: 7 | # 06:50 UTC every Monday 8 | - cron: '50 6 * * 1' 9 | workflow_dispatch: 10 | 11 | concurrency: 12 | group: beta-${{ github.workflow }}-${{ github.ref }} 13 | cancel-in-progress: true 14 | 15 | env: 16 | IROH_FORCE_STAGING_RELAYS: "1" 17 | 18 | jobs: 19 | clean_docs_branch: 20 | permissions: 21 | issues: write 22 | contents: write 23 | runs-on: ubuntu-latest 24 | steps: 25 | - name: Checkout 26 | uses: actions/checkout@v4 27 | with: 28 | ref: generated-docs-preview 29 | - name: Clean docs branch 30 | run: | 31 | cd pr/ 32 | # keep the last 25 prs 33 | dirs=$(ls -1d [0-9]* | sort -n) 34 | total_dirs=$(echo "$dirs" | wc -l) 35 | dirs_to_remove=$(echo "$dirs" | head -n $(($total_dirs - 25))) 36 | if [ -n "$dirs_to_remove" ]; then 37 | echo "$dirs_to_remove" | xargs rm -rf 38 | fi 39 | git add . 40 | git commit -m "Cleanup old docs" 41 | git push 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /.github/workflows/commit.yaml: -------------------------------------------------------------------------------- 1 | name: Commits 2 | 3 | on: 4 | pull_request: 5 | branches: [main] 6 | types: [opened, edited, synchronize] 7 | 8 | env: 9 | IROH_FORCE_STAGING_RELAYS: "1" 10 | 11 | jobs: 12 | check-for-cc: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: check-for-cc 16 | id: check-for-cc 17 | uses: agenthunt/conventional-commit-checker-action@v2.0.0 18 | with: 19 | pr-title-regex: "^(.+)(?:(([^)s]+)))?!?: (.+)" 20 | -------------------------------------------------------------------------------- /.github/workflows/docs.yaml: -------------------------------------------------------------------------------- 1 | name: Docs Preview 2 | 3 | on: 4 | pull_request: 5 | workflow_dispatch: 6 | inputs: 7 | pr_number: 8 | required: true 9 | type: string 10 | 11 | # ensure job runs sequentially so pushing to the preview branch doesn't conflict 12 | concurrency: 13 | group: ci-docs-preview 14 | 15 | env: 16 | IROH_FORCE_STAGING_RELAYS: "1" 17 | 18 | jobs: 19 | preview_docs: 20 | permissions: write-all 21 | timeout-minutes: 30 22 | name: Docs preview 23 | if: ${{ (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' ) && !github.event.pull_request.head.repo.fork }} 24 | runs-on: ubuntu-latest 25 | env: 26 | RUSTC_WRAPPER: "sccache" 27 | SCCACHE_GHA_ENABLED: "on" 28 | SCCACHE_CACHE_SIZE: "50G" 29 | PREVIEW_PATH: pr/${{ github.event.pull_request.number || inputs.pr_number }}/docs 30 | 31 | steps: 32 | - uses: actions/checkout@v4 33 | - uses: dtolnay/rust-toolchain@master 34 | with: 35 | toolchain: nightly-2024-11-30 36 | - name: Install sccache 37 | uses: mozilla-actions/sccache-action@v0.0.9 38 | 39 | - name: Generate Docs 40 | run: cargo doc --workspace --all-features --no-deps 41 | env: 42 | RUSTDOCFLAGS: --cfg iroh_docsrs 43 | 44 | - name: Deploy Docs to Preview Branch 45 | uses: peaceiris/actions-gh-pages@v4 46 | with: 47 | github_token: ${{ secrets.GITHUB_TOKEN }} 48 | publish_dir: ./target/doc/ 49 | destination_dir: ${{ env.PREVIEW_PATH }} 50 | publish_branch: generated-docs-preview 51 | 52 | - name: Find Docs Comment 53 | uses: peter-evans/find-comment@v3 54 | id: fc 55 | with: 56 | issue-number: ${{ github.event.pull_request.number || inputs.pr_number }} 57 | comment-author: 'github-actions[bot]' 58 | body-includes: Documentation for this PR has been generated 59 | 60 | - name: Get current timestamp 61 | id: get_timestamp 62 | run: echo "TIMESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV 63 | 64 | - name: Create or Update Docs Comment 65 | uses: peter-evans/create-or-update-comment@v4 66 | with: 67 | issue-number: ${{ github.event.pull_request.number || inputs.pr_number }} 68 | comment-id: ${{ steps.fc.outputs.comment-id }} 69 | body: | 70 | Documentation for this PR has been generated and is available at: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/${{ env.PREVIEW_PATH }}/iroh_blobs/ 71 | 72 | Last updated: ${{ env.TIMESTAMP }} 73 | edit-mode: replace 74 | -------------------------------------------------------------------------------- /.github/workflows/flaky.yaml: -------------------------------------------------------------------------------- 1 | # Run all tests, including flaky test. 2 | # 3 | # The default CI workflow ignores flaky tests. This workflow will run 4 | # all tests, including ignored ones. 5 | # 6 | # To use this workflow you can either: 7 | # 8 | # - Label a PR with "flaky-test", the normal CI workflow will not run 9 | # any jobs but the jobs here will be run. Note that to merge the PR 10 | # you'll need to remove the label eventually because the normal CI 11 | # jobs are required by branch protection. 12 | # 13 | # - Manually trigger the workflow, you may choose a branch for this to 14 | # run on. 15 | # 16 | # Additionally this jobs runs once a day on a schedule. 17 | # 18 | # Currently doctests are not run by this workflow. 19 | 20 | name: Flaky CI 21 | 22 | on: 23 | pull_request: 24 | types: [ 'labeled', 'unlabeled', 'opened', 'synchronize', 'reopened' ] 25 | schedule: 26 | # 06:30 UTC every day 27 | - cron: '30 6 * * *' 28 | workflow_dispatch: 29 | inputs: 30 | branch: 31 | description: 'Branch to run on, defaults to main' 32 | required: true 33 | default: 'main' 34 | type: string 35 | 36 | concurrency: 37 | group: flaky-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 38 | cancel-in-progress: true 39 | 40 | env: 41 | IROH_FORCE_STAGING_RELAYS: "1" 42 | 43 | jobs: 44 | tests: 45 | if: "contains(github.event.pull_request.labels.*.name, 'flaky-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'" 46 | uses: './.github/workflows/tests.yaml' 47 | with: 48 | flaky: true 49 | git-ref: ${{ inputs.branch }} 50 | notify: 51 | needs: tests 52 | if: ${{ always() }} 53 | runs-on: ubuntu-latest 54 | steps: 55 | - name: Extract test results 56 | run: | 57 | printf '${{ toJSON(needs) }}\n' 58 | result=$(echo '${{ toJSON(needs) }}' | jq -r .tests.result) 59 | echo TESTS_RESULT=$result 60 | echo "TESTS_RESULT=$result" >>"$GITHUB_ENV" 61 | - name: download nextest reports 62 | uses: actions/download-artifact@v4 63 | with: 64 | pattern: libtest_run_${{ github.run_number }}-${{ github.run_attempt }}-* 65 | merge-multiple: true 66 | path: nextest-results 67 | - name: create summary report 68 | id: make_summary 69 | run: | 70 | # prevent the glob expression in the loop to match on itself when the dir is empty 71 | shopt -s nullglob 72 | # to deal with multiline outputs it's recommended to use a random EOF, the syntax is based on 73 | # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings 74 | EOF=aP51VriWCxNJ1JjvmO9i 75 | echo "summary<<$EOF" >> $GITHUB_OUTPUT 76 | echo "Flaky tests failure:" >> $GITHUB_OUTPUT 77 | echo " " >> $GITHUB_OUTPUT 78 | for report in nextest-results/*.json; do 79 | # remove the name prefix and extension, and split the parts 80 | name=$(echo ${report:16:-5} | tr _ ' ') 81 | echo $name 82 | echo "- **$name**" >> $GITHUB_OUTPUT 83 | # select the failed tests 84 | # the tests have this format "crate::module$test_name", the sed expressions remove the quotes and replace $ for :: 85 | failure=$(jq --slurp '.[] | select(.["type"] == "test" and .["event"] == "failed" ) | .["name"]' $report | sed -e 's/^"//g' -e 's/\$/::/' -e 's/"//') 86 | echo "$failure" 87 | echo "$failure" >> $GITHUB_OUTPUT 88 | done 89 | echo "" >> $GITHUB_OUTPUT 90 | echo "See https://github.com/${{ github.repository }}/actions/workflows/flaky.yaml" >> $GITHUB_OUTPUT 91 | echo "$EOF" >> $GITHUB_OUTPUT 92 | - name: Notify discord on failure 93 | uses: n0-computer/discord-webhook-notify@v1 94 | if: ${{ env.TESTS_RESULT == 'failure' || env.TESTS_RESULT == 'success' }} 95 | with: 96 | text: "Flaky tests in **${{ github.repository }}**:" 97 | severity: ${{ env.TESTS_RESULT == 'failure' && 'warn' || 'info' }} 98 | details: ${{ env.TESTS_RESULT == 'failure' && steps.make_summary.outputs.summary || 'No flaky failures!' }} 99 | webhookUrl: ${{ secrets.DISCORD_N0_GITHUB_CHANNEL_WEBHOOK_URL }} 100 | -------------------------------------------------------------------------------- /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | # Run all tests, with or without flaky tests. 2 | 3 | name: Tests 4 | 5 | on: 6 | workflow_call: 7 | inputs: 8 | rust-version: 9 | description: 'The version of the rust compiler to run' 10 | type: string 11 | default: 'stable' 12 | flaky: 13 | description: 'Whether to also run flaky tests' 14 | type: boolean 15 | default: false 16 | git-ref: 17 | description: 'Which git ref to checkout' 18 | type: string 19 | default: ${{ github.ref }} 20 | 21 | env: 22 | RUST_BACKTRACE: 1 23 | RUSTFLAGS: -Dwarnings 24 | RUSTDOCFLAGS: -Dwarnings 25 | SCCACHE_CACHE_SIZE: "50G" 26 | CRATES_LIST: "iroh-blobs" 27 | IROH_FORCE_STAGING_RELAYS: "1" 28 | 29 | jobs: 30 | build_and_test_nix: 31 | timeout-minutes: 30 32 | name: "Tests" 33 | runs-on: ${{ matrix.runner }} 34 | strategy: 35 | fail-fast: false 36 | matrix: 37 | name: [ubuntu-latest, macOS-arm-latest] 38 | rust: [ '${{ inputs.rust-version }}' ] 39 | features: [all, none, default] 40 | include: 41 | - name: ubuntu-latest 42 | os: ubuntu-latest 43 | release-os: linux 44 | release-arch: amd64 45 | runner: [self-hosted, linux, X64] 46 | - name: macOS-arm-latest 47 | os: macOS-latest 48 | release-os: darwin 49 | release-arch: aarch64 50 | runner: [self-hosted, macOS, ARM64] 51 | env: 52 | # Using self-hosted runners so use local cache for sccache and 53 | # not SCCACHE_GHA_ENABLED. 54 | RUSTC_WRAPPER: "sccache" 55 | steps: 56 | - name: Checkout 57 | uses: actions/checkout@v4 58 | with: 59 | ref: ${{ inputs.git-ref }} 60 | 61 | - name: Install ${{ matrix.rust }} rust 62 | uses: dtolnay/rust-toolchain@master 63 | with: 64 | toolchain: ${{ matrix.rust }} 65 | 66 | - name: Install cargo-nextest 67 | uses: taiki-e/install-action@v2 68 | with: 69 | tool: nextest@0.9.80 70 | 71 | - name: Install sccache 72 | uses: mozilla-actions/sccache-action@v0.0.9 73 | 74 | - name: Select features 75 | run: | 76 | case "${{ matrix.features }}" in 77 | all) 78 | echo "FEATURES=--all-features" >> "$GITHUB_ENV" 79 | ;; 80 | none) 81 | echo "FEATURES=--no-default-features" >> "$GITHUB_ENV" 82 | ;; 83 | default) 84 | echo "FEATURES=" >> "$GITHUB_ENV" 85 | ;; 86 | *) 87 | exit 1 88 | esac 89 | 90 | - name: check features 91 | if: ${{ ! inputs.flaky }} 92 | run: | 93 | for i in ${CRATES_LIST//,/ } 94 | do 95 | echo "Checking $i $FEATURES" 96 | if [ $i = "iroh-cli" ]; then 97 | targets="--bins" 98 | else 99 | targets="--lib --bins" 100 | fi 101 | echo cargo check -p $i $FEATURES $targets 102 | cargo check -p $i $FEATURES $targets 103 | done 104 | env: 105 | RUST_LOG: ${{ runner.debug && 'TRACE' || 'DEBUG'}} 106 | 107 | - name: build tests 108 | run: | 109 | cargo nextest run --workspace ${{ env.FEATURES }} --lib --bins --tests --no-run 110 | 111 | - name: list ignored tests 112 | run: | 113 | cargo nextest list --workspace ${{ env.FEATURES }} --lib --bins --tests --run-ignored ignored-only 114 | 115 | - name: run tests 116 | run: | 117 | mkdir -p output 118 | cargo nextest run --workspace ${{ env.FEATURES }} --lib --bins --tests --profile ci --run-ignored ${{ inputs.flaky && 'all' || 'default' }} --no-fail-fast --message-format ${{ inputs.flaky && 'libtest-json' || 'human' }} > output/${{ matrix.name }}_${{ matrix.features }}_${{ matrix.rust }}.json 119 | env: 120 | RUST_LOG: ${{ runner.debug && 'TRACE' || 'DEBUG'}} 121 | NEXTEST_EXPERIMENTAL_LIBTEST_JSON: 1 122 | 123 | - name: upload results 124 | if: ${{ failure() && inputs.flaky }} 125 | uses: actions/upload-artifact@v4 126 | with: 127 | name: libtest_run_${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.name }}_${{ matrix.features }}_${{ matrix.rust }}.json 128 | path: output 129 | retention-days: 45 130 | compression-level: 0 131 | 132 | - name: doctests 133 | if: ${{ (! inputs.flaky) && matrix.features == 'all' }} 134 | run: | 135 | if [ -n "${{ runner.debug }}" ]; then 136 | export RUST_LOG=TRACE 137 | else 138 | export RUST_LOG=DEBUG 139 | fi 140 | cargo test --workspace --all-features --doc 141 | 142 | build_and_test_windows: 143 | timeout-minutes: 30 144 | name: "Tests" 145 | runs-on: ${{ matrix.runner }} 146 | strategy: 147 | fail-fast: false 148 | matrix: 149 | name: [windows-latest] 150 | rust: [ '${{ inputs.rust-version}}' ] 151 | features: [all, none, default] 152 | target: 153 | - x86_64-pc-windows-msvc 154 | include: 155 | - name: windows-latest 156 | os: windows 157 | runner: [self-hosted, windows, x64] 158 | env: 159 | # Using self-hosted runners so use local cache for sccache and 160 | # not SCCACHE_GHA_ENABLED. 161 | RUSTC_WRAPPER: "sccache" 162 | steps: 163 | - name: Checkout 164 | uses: actions/checkout@v4 165 | with: 166 | ref: ${{ inputs.git-ref }} 167 | 168 | - name: Install ${{ matrix.rust }} 169 | run: | 170 | rustup toolchain install ${{ matrix.rust }} 171 | rustup toolchain default ${{ matrix.rust }} 172 | rustup target add ${{ matrix.target }} 173 | rustup set default-host ${{ matrix.target }} 174 | 175 | - name: Install cargo-nextest 176 | shell: powershell 177 | run: | 178 | $tmp = New-TemporaryFile | Rename-Item -NewName { $_ -replace 'tmp$', 'zip' } -PassThru 179 | Invoke-WebRequest -OutFile $tmp https://get.nexte.st/latest/windows 180 | $outputDir = if ($Env:CARGO_HOME) { Join-Path $Env:CARGO_HOME "bin" } else { "~/.cargo/bin" } 181 | $tmp | Expand-Archive -DestinationPath $outputDir -Force 182 | $tmp | Remove-Item 183 | 184 | - name: Select features 185 | run: | 186 | switch ("${{ matrix.features }}") { 187 | "all" { 188 | echo "FEATURES=--all-features" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append 189 | } 190 | "none" { 191 | echo "FEATURES=--no-default-features" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append 192 | } 193 | "default" { 194 | echo "FEATURES=" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append 195 | } 196 | default { 197 | Exit 1 198 | } 199 | } 200 | 201 | - name: Install sccache 202 | uses: mozilla-actions/sccache-action@v0.0.9 203 | 204 | - uses: msys2/setup-msys2@v2 205 | 206 | - name: build tests 207 | run: | 208 | cargo nextest run --workspace ${{ env.FEATURES }} --lib --bins --tests --target ${{ matrix.target }} --no-run 209 | 210 | - name: list ignored tests 211 | run: | 212 | cargo nextest list --workspace ${{ env.FEATURES }} --lib --bins --tests --target ${{ matrix.target }} --run-ignored ignored-only 213 | 214 | - name: tests 215 | run: | 216 | mkdir -p output 217 | cargo nextest run --workspace ${{ env.FEATURES }} --lib --bins --tests --profile ci --target ${{ matrix.target }} --run-ignored ${{ inputs.flaky && 'all' || 'default' }} --no-fail-fast --message-format ${{ inputs.flaky && 'libtest-json' || 'human' }} > output/${{ matrix.name }}_${{ matrix.features }}_${{ matrix.rust }}.json 218 | env: 219 | RUST_LOG: ${{ runner.debug && 'TRACE' || 'DEBUG'}} 220 | NEXTEST_EXPERIMENTAL_LIBTEST_JSON: 1 221 | 222 | - name: upload results 223 | if: ${{ failure() && inputs.flaky }} 224 | uses: actions/upload-artifact@v4 225 | with: 226 | name: libtest_run_${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.name }}_${{ matrix.features }}_${{ matrix.rust }}.json 227 | path: output 228 | retention-days: 1 229 | compression-level: 0 230 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | iroh.config.toml 3 | .vscode/* 4 | -------------------------------------------------------------------------------- /.img/iroh_wordmark.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "iroh-blobs" 3 | version = "0.35.0" 4 | edition = "2021" 5 | readme = "README.md" 6 | description = "blob and collection transfer support for iroh" 7 | license = "MIT OR Apache-2.0" 8 | authors = ["dignifiedquire ", "n0 team"] 9 | repository = "https://github.com/n0-computer/iroh-blobs" 10 | keywords = ["hashing", "quic", "blake3"] 11 | 12 | # Sadly this also needs to be updated in .github/workflows/ci.yml 13 | rust-version = "1.81" 14 | 15 | [dependencies] 16 | anyhow = { version = "1" } 17 | async-channel = "2.3.1" 18 | bao-tree = { version = "0.15.1", features = [ 19 | "tokio_fsm", 20 | "validate", 21 | ], default-features = false } 22 | blake3 = { version = "1.8" } 23 | bytes = { version = "1.7", features = ["serde"] } 24 | chrono = "0.4.31" 25 | clap = { version = "4.5.20", features = ["derive"], optional = true } 26 | data-encoding = { version = "2.3.3" } 27 | derive_more = { version = "1.0.0", features = [ 28 | "debug", 29 | "display", 30 | "deref", 31 | "deref_mut", 32 | "from", 33 | "try_into", 34 | "into", 35 | ] } 36 | futures-buffered = "0.2.4" 37 | futures-lite = "2.3" 38 | futures-util = { version = "0.3.30", optional = true } 39 | genawaiter = { version = "0.99.1", features = ["futures03"] } 40 | hashlink = { version = "0.9.0", optional = true } 41 | hex = "0.4.3" 42 | indicatif = { version = "0.17.8", optional = true } 43 | iroh-base = "0.35" 44 | iroh-io = { version = "0.6.0", features = ["stats"] } 45 | iroh-metrics = { version = "0.34", default-features = false } 46 | iroh = "0.35" 47 | nested_enum_utils = { version = "0.1.0", optional = true } 48 | num_cpus = "1.15.0" 49 | oneshot = "0.1.8" 50 | parking_lot = { version = "0.12.1", optional = true } 51 | portable-atomic = { version = "1", optional = true } 52 | postcard = { version = "1", default-features = false, features = [ 53 | "alloc", 54 | "use-std", 55 | "experimental-derive", 56 | ] } 57 | quic-rpc = { version = "0.20", optional = true } 58 | quic-rpc-derive = { version = "0.20", optional = true } 59 | rand = "0.8" 60 | range-collections = "0.4.0" 61 | redb = { version = "=2.4", optional = true } 62 | reflink-copy = { version = "0.1.8", optional = true } 63 | self_cell = "1.0.1" 64 | serde = { version = "1", features = ["derive"] } 65 | serde-error = "0.1.3" 66 | smallvec = { version = "1.10.0", features = ["serde", "const_new"] } 67 | strum = { version = "0.26.3", optional = true } 68 | ssh-key = { version = "0.6", optional = true, features = ["ed25519"] } 69 | tempfile = { version = "3.10.0", optional = true } 70 | thiserror = "2" 71 | tokio = { version = "1", features = ["fs"] } 72 | tokio-util = { version = "0.7", features = ["io-util", "io"] } 73 | tracing = "0.1" 74 | tracing-futures = "0.2.5" 75 | walkdir = { version = "2.5.0", optional = true } 76 | 77 | # Examples 78 | console = { version = "0.15.8", optional = true } 79 | tracing-test = "0.2.5" 80 | 81 | [dev-dependencies] 82 | http-body = "1.0" 83 | iroh = { version = "0.35", features = ["test-utils"] } 84 | quinn = { package = "iroh-quinn", version = "0.13", features = ["ring"] } 85 | futures-buffered = "0.2.4" 86 | proptest = "1.0.0" 87 | serde_json = "1.0.107" 88 | serde_test = "1.0.176" 89 | testresult = "0.4.0" 90 | tokio = { version = "1", features = ["macros", "test-util"] } 91 | tracing-subscriber = { version = "0.3", features = ["env-filter"] } 92 | rcgen = "0.13" 93 | rustls = { version = "0.23", default-features = false, features = ["ring"] } 94 | tempfile = "3.10.0" 95 | futures-util = "0.3.30" 96 | testdir = "0.9.1" 97 | 98 | [features] 99 | default = ["fs-store", "net_protocol", "rpc"] 100 | downloader = ["dep:parking_lot", "tokio-util/time", "dep:hashlink"] 101 | net_protocol = ["downloader", "dep:futures-util"] 102 | fs-store = ["dep:reflink-copy", "redb", "dep:tempfile"] 103 | metrics = ["iroh-metrics/metrics"] 104 | redb = ["dep:redb"] 105 | cli = ["rpc", "dep:clap", "dep:indicatif", "dep:console"] 106 | rpc = [ 107 | "dep:quic-rpc", 108 | "dep:quic-rpc-derive", 109 | "dep:nested_enum_utils", 110 | "dep:strum", 111 | "dep:futures-util", 112 | "dep:portable-atomic", 113 | "dep:walkdir", 114 | "dep:ssh-key", 115 | "downloader", 116 | ] 117 | 118 | example-iroh = [ 119 | "dep:clap", 120 | "dep:indicatif", 121 | "dep:console", 122 | "iroh/discovery-local-network" 123 | ] 124 | test = ["quic-rpc/quinn-transport", "quic-rpc/test-utils"] 125 | 126 | [package.metadata.docs.rs] 127 | all-features = true 128 | rustdoc-args = ["--cfg", "iroh_docsrs"] 129 | 130 | [[example]] 131 | name = "provide-bytes" 132 | 133 | [[example]] 134 | name = "fetch-fsm" 135 | 136 | [[example]] 137 | name = "fetch-stream" 138 | 139 | [[example]] 140 | name = "transfer" 141 | required-features = ["rpc"] 142 | 143 | [[example]] 144 | name = "hello-world-fetch" 145 | required-features = ["example-iroh"] 146 | 147 | [[example]] 148 | name = "hello-world-provide" 149 | required-features = ["example-iroh"] 150 | 151 | [[example]] 152 | name = "discovery-local-network" 153 | required-features = ["example-iroh"] 154 | 155 | [[example]] 156 | name = "custom-protocol" 157 | required-features = ["example-iroh"] 158 | 159 | [lints.rust] 160 | missing_debug_implementations = "warn" 161 | 162 | # We use this --cfg for documenting the cargo features on which an API 163 | # is available. To preview this locally use: RUSTFLAGS="--cfg 164 | # iroh_docsrs cargo +nightly doc --all-features". We use our own 165 | # iroh_docsrs instead of the common docsrs to avoid also enabling this 166 | # feature in any dependencies, because some indirect dependencies 167 | # require a feature enabled when using `--cfg docsrs` which we can not 168 | # do. To enable for a crate set `#![cfg_attr(iroh_docsrs, 169 | # feature(doc_cfg))]` in the crate. 170 | unexpected_cfgs = { level = "warn", check-cfg = ["cfg(iroh_docsrs)"] } 171 | 172 | [lints.clippy] 173 | unused-async = "warn" 174 | 175 | [profile.dev-ci] 176 | inherits = 'dev' 177 | opt-level = 1 178 | 179 | [profile.optimized-release] 180 | inherits = 'release' 181 | debug = false 182 | lto = true 183 | debug-assertions = false 184 | opt-level = 3 185 | panic = 'abort' 186 | incremental = false 187 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright 2023 N0, INC. 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /Makefile.toml: -------------------------------------------------------------------------------- 1 | # Use cargo-make to run tasks here: https://crates.io/crates/cargo-make 2 | 3 | [tasks.format] 4 | workspace = false 5 | command = "cargo" 6 | args = [ 7 | "fmt", 8 | "--all", 9 | "--", 10 | "--config", 11 | "unstable_features=true", 12 | "--config", 13 | "imports_granularity=Crate,group_imports=StdExternalCrate,reorder_imports=true", 14 | ] 15 | 16 | [tasks.format-check] 17 | workspace = false 18 | command = "cargo" 19 | args = [ 20 | "fmt", 21 | "--all", 22 | "--check", 23 | "--", 24 | "--config", 25 | "unstable_features=true", 26 | "--config", 27 | "imports_granularity=Crate,group_imports=StdExternalCrate,reorder_imports=true", 28 | ] 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # iroh-blobs 2 | 3 | This crate provides blob and blob sequence transfer support for iroh. It implements a simple request-response protocol based on BLAKE3 verified streaming. 4 | 5 | A request describes data in terms of BLAKE3 hashes and byte ranges. It is possible to request blobs or ranges of blobs, as well as entire sequences of blobs in one request. 6 | 7 | The requester opens a QUIC stream to the provider and sends the request. The provider answers with the requested data, encoded as [BLAKE3](https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf) verified streams, on the same QUIC stream. 8 | 9 | This crate is used together with [iroh](https://crates.io/crates/iroh). Connection establishment is left up to the user or higher level APIs. 10 | 11 | ## Concepts 12 | 13 | - **Blob:** a sequence of bytes of arbitrary size, without any metadata. 14 | 15 | - **Link:** a 32 byte BLAKE3 hash of a blob. 16 | 17 | - **HashSeq:** a blob that contains a sequence of links. Its size is a multiple of 32. 18 | 19 | - **Provider:** The side that provides data and answers requests. Providers wait for incoming requests from Requests. 20 | 21 | - **Requester:** The side that asks for data. It is initiating requests to one or many providers. 22 | 23 | 24 | ## Getting started 25 | 26 | The `iroh-blobs` protocol was designed to be used in conjunction with `iroh`. [Iroh](https://docs.rs/iroh) is a networking library for making direct connections, these connections are what power the data transfers in `iroh-blobs`. 27 | 28 | Iroh provides a [`Router`](https://docs.rs/iroh/latest/iroh/protocol/struct.Router.html) that takes an [`Endpoint`](https://docs.rs/iroh/latest/iroh/endpoint/struct.Endpoint.html) and any protocols needed for the application. Similar to a router in webserver library, it runs a loop accepting incoming connections and routes them to the specific protocol handler, based on `ALPN`. 29 | 30 | Here is a basic example of how to set up `iroh-blobs` with `iroh`: 31 | 32 | ```rust 33 | use iroh::{protocol::Router, Endpoint}; 34 | use iroh_blobs::{store::Store, net_protocol::Blobs}; 35 | 36 | #[tokio::main] 37 | async fn main() -> anyhow::Result<()> { 38 | // create an iroh endpoint that includes the standard discovery mechanisms 39 | // we've built at number0 40 | let endpoint = Endpoint::builder().discovery_n0().bind().await?; 41 | 42 | // create an in-memory blob store 43 | // use `iroh_blobs::net_protocol::Blobs::persistent` to load or create a 44 | // persistent blob store from a path 45 | let blobs = Blobs::memory().build(&endpoint); 46 | 47 | // turn on the "rpc" feature if you need to create blobs and tags clients 48 | let blobs_client = blobs.client(); 49 | let tags_client = blobs_client.tags(); 50 | 51 | // build the router 52 | let router = Router::builder(endpoint) 53 | .accept(iroh_blobs::ALPN, blobs.clone()) 54 | .spawn(); 55 | 56 | // do fun stuff with the blobs protocol! 57 | router.shutdown().await?; 58 | drop(tags_client); 59 | Ok(()) 60 | } 61 | ``` 62 | 63 | ## Examples 64 | 65 | Examples that use `iroh-blobs` can be found in [this repo](https://github.com/n0-computer/iroh-blobs/tree/main/examples). 66 | 67 | # License 68 | 69 | This project is licensed under either of 70 | 71 | * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or 72 | ) 73 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or 74 | ) 75 | 76 | at your option. 77 | 78 | ### Contribution 79 | 80 | Unless you explicitly state otherwise, any contribution intentionally submitted 81 | for inclusion in this project by you, as defined in the Apache-2.0 license, 82 | shall be dual licensed as above, without any additional terms or conditions. 83 | -------------------------------------------------------------------------------- /cliff.toml: -------------------------------------------------------------------------------- 1 | [changelog] 2 | # changelog header 3 | header = """ 4 | # Changelog\n 5 | All notable changes to iroh-blobs will be documented in this file.\n 6 | """ 7 | 8 | body = """ 9 | {% if version %}\ 10 | {% if previous.version %}\ 11 | ## [{{ version | trim_start_matches(pat="v") }}](/compare/{{ previous.version }}..{{ version }}) - {{ timestamp | date(format="%Y-%m-%d") }} 12 | {% else %}\ 13 | ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} 14 | {% endif %}\ 15 | {% else %}\ 16 | ## [unreleased] 17 | {% endif %}\ 18 | 19 | {% macro commit(commit) -%} 20 | - {% if commit.scope %}*({{ commit.scope }})* {% endif %}{% if commit.breaking %}[**breaking**] {% endif %}\ 21 | {{ commit.message | upper_first }} - ([{{ commit.id | truncate(length=7, end="") }}](/commit/{{ commit.id }}))\ 22 | {% endmacro -%} 23 | 24 | {% for group, commits in commits | group_by(attribute="group") %} 25 | ### {{ group | striptags | trim | upper_first }} 26 | {% for commit in commits 27 | | filter(attribute="scope") 28 | | sort(attribute="scope") %} 29 | {{ self::commit(commit=commit) }} 30 | {%- endfor -%} 31 | {% raw %}\n{% endraw %}\ 32 | {%- for commit in commits %} 33 | {%- if not commit.scope -%} 34 | {{ self::commit(commit=commit) }} 35 | {% endif -%} 36 | {% endfor -%} 37 | {% endfor %}\n 38 | """ 39 | 40 | footer = "" 41 | postprocessors = [ 42 | { pattern = '', replace = "https://github.com/n0-computer/iroh-blobs" }, 43 | { pattern = "\\(#([0-9]+)\\)", replace = "([#${1}](https://github.com/n0-computer/iroh-blobs/issues/${1}))"} 44 | ] 45 | 46 | 47 | [git] 48 | # regex for parsing and grouping commits 49 | commit_parsers = [ 50 | { message = "^feat", group = "⛰️ Features" }, 51 | { message = "^fix", group = "🐛 Bug Fixes" }, 52 | { message = "^doc", group = "📚 Documentation" }, 53 | { message = "^perf", group = "⚡ Performance" }, 54 | { message = "^refactor", group = "🚜 Refactor" }, 55 | { message = "^style", group = "🎨 Styling" }, 56 | { message = "^test", group = "🧪 Testing" }, 57 | { message = "^chore\\(release\\)", skip = true }, 58 | { message = "^chore\\(deps\\)", skip = true }, 59 | { message = "^chore\\(pr\\)", skip = true }, 60 | { message = "^chore\\(pull\\)", skip = true }, 61 | { message = "^chore|ci", group = "⚙️ Miscellaneous Tasks" }, 62 | { body = ".*security", group = "🛡️ Security" }, 63 | { message = "^revert", group = "◀️ Revert" }, 64 | ] 65 | -------------------------------------------------------------------------------- /code_of_conduct.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | Online or off, Number Zero is a harassment-free environment for everyone, regardless of gender, gender identity and expression, sexual orientation, disability, physical appearance, body size, race, age or religion or technical skill level. We do not tolerate harassment of participants in any form. 4 | 5 | Harassment includes verbal comments that reinforce social structures of domination related to gender, gender identity and expression, sexual orientation, disability, physical appearance, body size, race, age, religion, sexual images in public spaces, deliberate intimidation, stalking, following, harassing photography or recording, sustained disruption of talks or other events, inappropriate physical contact, and unwelcome sexual attention. Participants asked to stop any harassing behavior are expected to comply immediately. 6 | 7 | If a participant engages in harassing behaviour, the organizers may take any action they deem appropriate, including warning the offender or expulsion from events and online forums. 8 | 9 | If you are being harassed, notice that someone else is being harassed, or have any other concerns, please contact a member of the organizing team immediately. 10 | 11 | At offline events, organizers will identify themselves, and will help participants contact venue security or local law enforcement, provide escorts, or otherwise assist those experiencing harassment to feel safe for the duration of the event. We value your participation! 12 | 13 | This document is based on a similar code from [EDGI](https://envirodatagov.org/) and [Civic Tech Toronto](http://civictech.ca/about-us/), itself derived from the [Recurse Center’s Social Rules](https://www.recurse.com/manual#sec-environment), and the [anti-harassment policy from the Geek Feminism Wiki](http://geekfeminism.wikia.com/wiki/Conference_anti-harassment/Policy). 14 | -------------------------------------------------------------------------------- /deny.toml: -------------------------------------------------------------------------------- 1 | [advisories] 2 | ignore = [ 3 | "RUSTSEC-2024-0370", 4 | "RUSTSEC-2024-0384", 5 | "RUSTSEC-2024-0436", 6 | "RUSTSEC-2023-0089", 7 | ] 8 | 9 | [bans] 10 | deny = [ 11 | "aws-lc", 12 | "aws-lc-rs", 13 | "aws-lc-sys", 14 | "native-tls", 15 | "openssl", 16 | ] 17 | multiple-versions = "allow" 18 | 19 | [licenses] 20 | allow = [ 21 | "Apache-2.0", 22 | "Apache-2.0 WITH LLVM-exception", 23 | "BSD-2-Clause", 24 | "BSD-3-Clause", 25 | "BSL-1.0", 26 | "ISC", 27 | "MIT", 28 | "Zlib", 29 | "MPL-2.0", 30 | "Unicode-3.0", 31 | "Unlicense", 32 | ] 33 | 34 | [[licenses.clarify]] 35 | expression = "MIT AND ISC AND OpenSSL" 36 | name = "ring" 37 | 38 | [[licenses.clarify.license-files]] 39 | hash = 3171872035 40 | path = "LICENSE" 41 | 42 | [sources] 43 | allow-git = [] 44 | -------------------------------------------------------------------------------- /examples/discovery-local-network.rs: -------------------------------------------------------------------------------- 1 | //! Example that runs and iroh node with local node discovery and no relay server 2 | //! 3 | //! Run the follow command to run the "accept" side, that hosts the content: 4 | //! $ cargo run --example discovery_local_network --features="discovery-local-network" -- accept [FILE_PATH] 5 | //! Wait for output that looks like the following: 6 | //! $ cargo run --example discovery_local_network --features="discovery-local-network" -- connect [NODE_ID] [HASH] -o [FILE_PATH] 7 | //! Run that command on another machine in the same local network, replacing [FILE_PATH] to the path on which you want to save the transferred content. 8 | use std::path::PathBuf; 9 | 10 | use anyhow::ensure; 11 | use clap::{Parser, Subcommand}; 12 | use iroh::{ 13 | discovery::mdns::MdnsDiscovery, protocol::Router, Endpoint, NodeAddr, PublicKey, RelayMode, 14 | SecretKey, 15 | }; 16 | use iroh_blobs::{net_protocol::Blobs, rpc::client::blobs::WrapOption, Hash}; 17 | use tracing_subscriber::{prelude::*, EnvFilter}; 18 | 19 | use self::progress::show_download_progress; 20 | 21 | // set the RUST_LOG env var to one of {debug,info,warn} to see logging info 22 | pub fn setup_logging() { 23 | tracing_subscriber::registry() 24 | .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) 25 | .with(EnvFilter::from_default_env()) 26 | .try_init() 27 | .ok(); 28 | } 29 | 30 | #[derive(Debug, Parser)] 31 | #[command(version, about)] 32 | pub struct Cli { 33 | #[clap(subcommand)] 34 | command: Commands, 35 | } 36 | 37 | #[derive(Subcommand, Clone, Debug)] 38 | pub enum Commands { 39 | /// Launch an iroh node and provide the content at the given path 40 | Accept { 41 | /// path to the file you want to provide 42 | path: PathBuf, 43 | }, 44 | /// Get the node_id and hash string from a node running accept in the local network 45 | /// Download the content from that node. 46 | Connect { 47 | /// Node ID of a node on the local network 48 | node_id: PublicKey, 49 | /// Hash of content you want to download from the node 50 | hash: Hash, 51 | /// save the content to a file 52 | #[clap(long, short)] 53 | out: Option, 54 | }, 55 | } 56 | 57 | #[tokio::main] 58 | async fn main() -> anyhow::Result<()> { 59 | setup_logging(); 60 | let cli = Cli::parse(); 61 | 62 | let key = SecretKey::generate(rand::rngs::OsRng); 63 | let discovery = MdnsDiscovery::new(key.public())?; 64 | 65 | println!("Starting iroh node with mdns discovery..."); 66 | // create a new node 67 | let endpoint = Endpoint::builder() 68 | .secret_key(key) 69 | .discovery(Box::new(discovery)) 70 | .relay_mode(RelayMode::Disabled) 71 | .bind() 72 | .await?; 73 | let builder = Router::builder(endpoint); 74 | let blobs = Blobs::memory().build(builder.endpoint()); 75 | let builder = builder.accept(iroh_blobs::ALPN, blobs.clone()); 76 | let node = builder.spawn(); 77 | let blobs_client = blobs.client(); 78 | 79 | match &cli.command { 80 | Commands::Accept { path } => { 81 | if !path.is_file() { 82 | println!("Content must be a file."); 83 | node.shutdown().await?; 84 | return Ok(()); 85 | } 86 | let absolute = path.canonicalize()?; 87 | println!("Adding {} as {}...", path.display(), absolute.display()); 88 | let stream = blobs_client 89 | .add_from_path( 90 | absolute, 91 | true, 92 | iroh_blobs::util::SetTagOption::Auto, 93 | WrapOption::NoWrap, 94 | ) 95 | .await?; 96 | let outcome = stream.finish().await?; 97 | println!("To fetch the blob:\n\tcargo run --example discovery_local_network --features=\"discovery-local-network\" -- connect {} {} -o [FILE_PATH]", node.endpoint().node_id(), outcome.hash); 98 | tokio::signal::ctrl_c().await?; 99 | node.shutdown().await?; 100 | std::process::exit(0); 101 | } 102 | Commands::Connect { node_id, hash, out } => { 103 | println!("NodeID: {}", node.endpoint().node_id()); 104 | let mut stream = blobs_client 105 | .download(*hash, NodeAddr::new(*node_id)) 106 | .await?; 107 | show_download_progress(*hash, &mut stream).await?; 108 | if let Some(path) = out { 109 | let absolute = std::env::current_dir()?.join(path); 110 | ensure!(!absolute.is_dir(), "output must not be a directory"); 111 | tracing::info!( 112 | "exporting {hash} to {} -> {}", 113 | path.display(), 114 | absolute.display() 115 | ); 116 | let stream = blobs_client 117 | .export( 118 | *hash, 119 | absolute, 120 | iroh_blobs::store::ExportFormat::Blob, 121 | iroh_blobs::store::ExportMode::Copy, 122 | ) 123 | .await?; 124 | stream.await?; 125 | } 126 | } 127 | } 128 | Ok(()) 129 | } 130 | 131 | mod progress { 132 | use anyhow::{bail, Result}; 133 | use console::style; 134 | use futures_lite::{Stream, StreamExt}; 135 | use indicatif::{ 136 | HumanBytes, HumanDuration, MultiProgress, ProgressBar, ProgressDrawTarget, ProgressState, 137 | ProgressStyle, 138 | }; 139 | use iroh_blobs::{ 140 | get::{db::DownloadProgress, progress::BlobProgress, Stats}, 141 | Hash, 142 | }; 143 | 144 | pub async fn show_download_progress( 145 | hash: Hash, 146 | mut stream: impl Stream> + Unpin, 147 | ) -> Result<()> { 148 | eprintln!("Fetching: {}", hash); 149 | let mp = MultiProgress::new(); 150 | mp.set_draw_target(ProgressDrawTarget::stderr()); 151 | let op = mp.add(make_overall_progress()); 152 | let ip = mp.add(make_individual_progress()); 153 | op.set_message(format!("{} Connecting ...\n", style("[1/3]").bold().dim())); 154 | let mut seq = false; 155 | while let Some(x) = stream.next().await { 156 | match x? { 157 | DownloadProgress::InitialState(state) => { 158 | if state.connected { 159 | op.set_message(format!("{} Requesting ...\n", style("[2/3]").bold().dim())); 160 | } 161 | if let Some(count) = state.root.child_count { 162 | op.set_message(format!( 163 | "{} Downloading {} blob(s)\n", 164 | style("[3/3]").bold().dim(), 165 | count + 1, 166 | )); 167 | op.set_length(count + 1); 168 | op.reset(); 169 | op.set_position(state.current.map(u64::from).unwrap_or(0)); 170 | seq = true; 171 | } 172 | if let Some(blob) = state.get_current() { 173 | if let Some(size) = blob.size { 174 | ip.set_length(size.value()); 175 | ip.reset(); 176 | match blob.progress { 177 | BlobProgress::Pending => {} 178 | BlobProgress::Progressing(offset) => ip.set_position(offset), 179 | BlobProgress::Done => ip.finish_and_clear(), 180 | } 181 | if !seq { 182 | op.finish_and_clear(); 183 | } 184 | } 185 | } 186 | } 187 | DownloadProgress::FoundLocal { .. } => {} 188 | DownloadProgress::Connected => { 189 | op.set_message(format!("{} Requesting ...\n", style("[2/3]").bold().dim())); 190 | } 191 | DownloadProgress::FoundHashSeq { children, .. } => { 192 | op.set_message(format!( 193 | "{} Downloading {} blob(s)\n", 194 | style("[3/3]").bold().dim(), 195 | children + 1, 196 | )); 197 | op.set_length(children + 1); 198 | op.reset(); 199 | seq = true; 200 | } 201 | DownloadProgress::Found { size, child, .. } => { 202 | if seq { 203 | op.set_position(child.into()); 204 | } else { 205 | op.finish_and_clear(); 206 | } 207 | ip.set_length(size); 208 | ip.reset(); 209 | } 210 | DownloadProgress::Progress { offset, .. } => { 211 | ip.set_position(offset); 212 | } 213 | DownloadProgress::Done { .. } => { 214 | ip.finish_and_clear(); 215 | } 216 | DownloadProgress::AllDone(Stats { 217 | bytes_read, 218 | elapsed, 219 | .. 220 | }) => { 221 | op.finish_and_clear(); 222 | eprintln!( 223 | "Transferred {} in {}, {}/s", 224 | HumanBytes(bytes_read), 225 | HumanDuration(elapsed), 226 | HumanBytes((bytes_read as f64 / elapsed.as_secs_f64()) as u64) 227 | ); 228 | break; 229 | } 230 | DownloadProgress::Abort(e) => { 231 | bail!("download aborted: {}", e); 232 | } 233 | } 234 | } 235 | Ok(()) 236 | } 237 | fn make_overall_progress() -> ProgressBar { 238 | let pb = ProgressBar::hidden(); 239 | pb.enable_steady_tick(std::time::Duration::from_millis(100)); 240 | pb.set_style( 241 | ProgressStyle::with_template( 242 | "{msg}{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len}", 243 | ) 244 | .unwrap() 245 | .progress_chars("#>-"), 246 | ); 247 | pb 248 | } 249 | 250 | fn make_individual_progress() -> ProgressBar { 251 | let pb = ProgressBar::hidden(); 252 | pb.enable_steady_tick(std::time::Duration::from_millis(100)); 253 | pb.set_style( 254 | ProgressStyle::with_template("{msg}{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({eta})") 255 | .unwrap() 256 | .with_key( 257 | "eta", 258 | |state: &ProgressState, w: &mut dyn std::fmt::Write| { 259 | write!(w, "{:.1}s", state.eta().as_secs_f64()).unwrap() 260 | }, 261 | ) 262 | .progress_chars("#>-"), 263 | ); 264 | pb 265 | } 266 | } 267 | -------------------------------------------------------------------------------- /examples/fetch-fsm.rs: -------------------------------------------------------------------------------- 1 | //! An example how to download a single blob or collection from a node and write it to stdout using the `get` finite state machine directly. 2 | //! 3 | //! Since this example does not use [`iroh-net::Endpoint`], it does not do any holepunching, and so will only work locally or between two processes that have public IP addresses. 4 | //! 5 | //! Run the provide-bytes example first. It will give instructions on how to run this example properly. 6 | use std::str::FromStr; 7 | 8 | use anyhow::{Context, Result}; 9 | use iroh_blobs::{ 10 | get::fsm::{AtInitial, ConnectedNext, EndBlobNext}, 11 | hashseq::HashSeq, 12 | protocol::GetRequest, 13 | BlobFormat, 14 | }; 15 | use iroh_io::ConcatenateSliceWriter; 16 | use tracing_subscriber::{prelude::*, EnvFilter}; 17 | 18 | const EXAMPLE_ALPN: &[u8] = b"n0/iroh/examples/bytes/0"; 19 | 20 | // set the RUST_LOG env var to one of {debug,info,warn} to see logging info 21 | pub fn setup_logging() { 22 | tracing_subscriber::registry() 23 | .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) 24 | .with(EnvFilter::from_default_env()) 25 | .try_init() 26 | .ok(); 27 | } 28 | 29 | #[tokio::main] 30 | async fn main() -> Result<()> { 31 | println!("\nfetch fsm example!"); 32 | setup_logging(); 33 | let args: Vec<_> = std::env::args().collect(); 34 | if args.len() != 2 { 35 | anyhow::bail!("usage: fetch-fsm [TICKET]"); 36 | } 37 | let ticket = 38 | iroh_blobs::ticket::BlobTicket::from_str(&args[1]).context("unable to parse [TICKET]")?; 39 | 40 | let (node, hash, format) = ticket.into_parts(); 41 | 42 | // create an endpoint to listen for incoming connections 43 | let endpoint = iroh::Endpoint::builder() 44 | .relay_mode(iroh::RelayMode::Disabled) 45 | .alpns(vec![EXAMPLE_ALPN.into()]) 46 | .bind() 47 | .await?; 48 | println!( 49 | "\nlistening on {:?}", 50 | endpoint.node_addr().await?.direct_addresses 51 | ); 52 | println!("fetching hash {hash} from {:?}", node.node_id); 53 | 54 | // connect 55 | let connection = endpoint.connect(node, EXAMPLE_ALPN).await?; 56 | 57 | match format { 58 | BlobFormat::HashSeq => { 59 | // create a request for a collection 60 | let request = GetRequest::all(hash); 61 | // create the initial state of the finite state machine 62 | let initial = iroh_blobs::get::fsm::start(connection, request); 63 | 64 | write_collection(initial).await 65 | } 66 | BlobFormat::Raw => { 67 | // create a request for a single blob 68 | let request = GetRequest::single(hash); 69 | // create the initial state of the finite state machine 70 | let initial = iroh_blobs::get::fsm::start(connection, request); 71 | 72 | write_blob(initial).await 73 | } 74 | } 75 | } 76 | 77 | async fn write_blob(initial: AtInitial) -> Result<()> { 78 | // connect (create a stream pair) 79 | let connected = initial.next().await?; 80 | 81 | // we expect a start root message, since we requested a single blob 82 | let ConnectedNext::StartRoot(start_root) = connected.next().await? else { 83 | panic!("expected start root") 84 | }; 85 | // we can just call next to proceed to the header, since we know the root hash 86 | let header = start_root.next(); 87 | 88 | // we need to wrap stdout in a struct that implements AsyncSliceWriter. Since we can not 89 | // seek in stdout we use ConcatenateSliceWriter which just concatenates all the writes. 90 | let writer = ConcatenateSliceWriter::new(tokio::io::stdout()); 91 | 92 | // make the spacing nicer in the terminal 93 | println!(); 94 | // use the utility function write_all to write the entire blob 95 | let end = header.write_all(writer).await?; 96 | 97 | // we requested a single blob, so we expect to enter the closing state 98 | let EndBlobNext::Closing(closing) = end.next() else { 99 | panic!("expected closing") 100 | }; 101 | 102 | // close the connection and get the stats 103 | let _stats = closing.next().await?; 104 | Ok(()) 105 | } 106 | 107 | async fn write_collection(initial: AtInitial) -> Result<()> { 108 | // connect 109 | let connected = initial.next().await?; 110 | // read the first bytes 111 | let ConnectedNext::StartRoot(start_root) = connected.next().await? else { 112 | anyhow::bail!("failed to parse collection"); 113 | }; 114 | // check that we requested the whole collection 115 | if !start_root.ranges().is_all() { 116 | anyhow::bail!("collection was not requested completely"); 117 | } 118 | 119 | // move to the header 120 | let header: iroh_blobs::get::fsm::AtBlobHeader = start_root.next(); 121 | let (root_end, hashes_bytes) = header.concatenate_into_vec().await?; 122 | let next = root_end.next(); 123 | let EndBlobNext::MoreChildren(at_meta) = next else { 124 | anyhow::bail!("missing meta blob, got {next:?}"); 125 | }; 126 | // parse the hashes from the hash sequence bytes 127 | let hashes = HashSeq::try_from(bytes::Bytes::from(hashes_bytes)) 128 | .context("failed to parse hashes")? 129 | .into_iter() 130 | .collect::>(); 131 | let meta_hash = hashes.first().context("missing meta hash")?; 132 | 133 | let (meta_end, _meta_bytes) = at_meta.next(*meta_hash).concatenate_into_vec().await?; 134 | let mut curr = meta_end.next(); 135 | let closing = loop { 136 | match curr { 137 | EndBlobNext::MoreChildren(more) => { 138 | let Some(hash) = hashes.get(more.child_offset() as usize) else { 139 | break more.finish(); 140 | }; 141 | let header = more.next(*hash); 142 | 143 | // we need to wrap stdout in a struct that implements AsyncSliceWriter. Since we can not 144 | // seek in stdout we use ConcatenateSliceWriter which just concatenates all the writes. 145 | let writer = ConcatenateSliceWriter::new(tokio::io::stdout()); 146 | 147 | // use the utility function write_all to write the entire blob 148 | let end = header.write_all(writer).await?; 149 | println!(); 150 | curr = end.next(); 151 | } 152 | EndBlobNext::Closing(closing) => { 153 | break closing; 154 | } 155 | } 156 | }; 157 | // close the connection 158 | let _stats = closing.next().await?; 159 | Ok(()) 160 | } 161 | -------------------------------------------------------------------------------- /examples/fetch-stream.rs: -------------------------------------------------------------------------------- 1 | //! An example how to download a single blob or collection from a node and write it to stdout, using a helper method to turn the `get` finite state machine into a stream. 2 | //! 3 | //! Since this example does not use [`iroh-net::Endpoint`], it does not do any holepunching, and so will only work locally or between two processes that have public IP addresses. 4 | //! 5 | //! Run the provide-bytes example first. It will give instructions on how to run this example properly. 6 | use std::{io, str::FromStr}; 7 | 8 | use anyhow::{Context, Result}; 9 | use bao_tree::io::fsm::BaoContentItem; 10 | use bytes::Bytes; 11 | use futures_lite::{Stream, StreamExt}; 12 | use genawaiter::sync::{Co, Gen}; 13 | use iroh_blobs::{ 14 | get::fsm::{AtInitial, BlobContentNext, ConnectedNext, EndBlobNext}, 15 | hashseq::HashSeq, 16 | protocol::GetRequest, 17 | BlobFormat, 18 | }; 19 | use tokio::io::AsyncWriteExt; 20 | use tracing_subscriber::{prelude::*, EnvFilter}; 21 | 22 | const EXAMPLE_ALPN: &[u8] = b"n0/iroh/examples/bytes/0"; 23 | 24 | // set the RUST_LOG env var to one of {debug,info,warn} to see logging info 25 | pub fn setup_logging() { 26 | tracing_subscriber::registry() 27 | .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) 28 | .with(EnvFilter::from_default_env()) 29 | .try_init() 30 | .ok(); 31 | } 32 | 33 | #[tokio::main] 34 | async fn main() -> Result<()> { 35 | println!("\nfetch stream example!"); 36 | setup_logging(); 37 | let args: Vec<_> = std::env::args().collect(); 38 | if args.len() != 2 { 39 | anyhow::bail!("usage: fetch-stream [TICKET]"); 40 | } 41 | let ticket = 42 | iroh_blobs::ticket::BlobTicket::from_str(&args[1]).context("unable to parse [TICKET]")?; 43 | 44 | let (node, hash, format) = ticket.into_parts(); 45 | 46 | // create an endpoint to listen for incoming connections 47 | let endpoint = iroh::Endpoint::builder() 48 | .relay_mode(iroh::RelayMode::Disabled) 49 | .alpns(vec![EXAMPLE_ALPN.into()]) 50 | .bind() 51 | .await?; 52 | println!( 53 | "\nlistening on {:?}", 54 | endpoint.node_addr().await?.direct_addresses 55 | ); 56 | println!("fetching hash {hash} from {:?}", node.node_id); 57 | 58 | // connect 59 | let connection = endpoint.connect(node, EXAMPLE_ALPN).await?; 60 | 61 | let mut stream = match format { 62 | BlobFormat::HashSeq => { 63 | // create a request for a collection 64 | let request = GetRequest::all(hash); 65 | 66 | // create the initial state of the finite state machine 67 | let initial = iroh_blobs::get::fsm::start(connection, request); 68 | 69 | // create a stream that yields all the data of the blob 70 | stream_children(initial).boxed_local() 71 | } 72 | BlobFormat::Raw => { 73 | // create a request for a single blob 74 | let request = GetRequest::single(hash); 75 | 76 | // create the initial state of the finite state machine 77 | let initial = iroh_blobs::get::fsm::start(connection, request); 78 | 79 | // create a stream that yields all the data of the blob 80 | stream_blob(initial).boxed_local() 81 | } 82 | }; 83 | while let Some(item) = stream.next().await { 84 | let item = item?; 85 | tokio::io::stdout().write_all(&item).await?; 86 | println!(); 87 | } 88 | Ok(()) 89 | } 90 | 91 | /// Stream the response for a request for a single blob. 92 | /// 93 | /// If the request was for a part of the blob, this will stream just the requested 94 | /// blocks. 95 | /// 96 | /// This will stream the root blob and close the connection. 97 | fn stream_blob(initial: AtInitial) -> impl Stream> + 'static { 98 | async fn inner(initial: AtInitial, co: &Co>) -> io::Result<()> { 99 | // connect 100 | let connected = initial.next().await?; 101 | // read the first bytes 102 | let ConnectedNext::StartRoot(start_root) = connected.next().await? else { 103 | return Err(io::Error::new(io::ErrorKind::Other, "expected start root")); 104 | }; 105 | // move to the header 106 | let header = start_root.next(); 107 | // get the size of the content 108 | let (mut content, _size) = header.next().await?; 109 | // manually loop over the content and yield all data 110 | let done = loop { 111 | match content.next().await { 112 | BlobContentNext::More((next, data)) => { 113 | if let BaoContentItem::Leaf(leaf) = data? { 114 | // yield the data 115 | co.yield_(Ok(leaf.data)).await; 116 | } 117 | content = next; 118 | } 119 | BlobContentNext::Done(done) => { 120 | // we are done with the root blob 121 | break done; 122 | } 123 | } 124 | }; 125 | // close the connection even if there is more data 126 | let closing = match done.next() { 127 | EndBlobNext::Closing(closing) => closing, 128 | EndBlobNext::MoreChildren(more) => more.finish(), 129 | }; 130 | // close the connection 131 | let _stats = closing.next().await?; 132 | Ok(()) 133 | } 134 | 135 | Gen::new(|co| async move { 136 | if let Err(e) = inner(initial, &co).await { 137 | co.yield_(Err(e)).await; 138 | } 139 | }) 140 | } 141 | 142 | /// Stream the response for a request for an iroh collection and its children. 143 | /// 144 | /// If the request was for a part of the children, this will stream just the requested 145 | /// blocks. 146 | /// 147 | /// The root blob is not streamed. It must be fully included in the response. 148 | fn stream_children(initial: AtInitial) -> impl Stream> + 'static { 149 | async fn inner(initial: AtInitial, co: &Co>) -> io::Result<()> { 150 | // connect 151 | let connected = initial.next().await?; 152 | // read the first bytes 153 | let ConnectedNext::StartRoot(start_root) = connected.next().await? else { 154 | return Err(io::Error::new( 155 | io::ErrorKind::Other, 156 | "failed to parse collection", 157 | )); 158 | }; 159 | // check that we requested the whole collection 160 | if !start_root.ranges().is_all() { 161 | return Err(io::Error::new( 162 | io::ErrorKind::Other, 163 | "collection was not requested completely", 164 | )); 165 | } 166 | // move to the header 167 | let header: iroh_blobs::get::fsm::AtBlobHeader = start_root.next(); 168 | let (root_end, hashes_bytes) = header.concatenate_into_vec().await?; 169 | 170 | // parse the hashes from the hash sequence bytes 171 | let hashes = HashSeq::try_from(bytes::Bytes::from(hashes_bytes)) 172 | .map_err(|e| { 173 | io::Error::new(io::ErrorKind::Other, format!("failed to parse hashes: {e}")) 174 | })? 175 | .into_iter() 176 | .collect::>(); 177 | 178 | let next = root_end.next(); 179 | let EndBlobNext::MoreChildren(at_meta) = next else { 180 | return Err(io::Error::new(io::ErrorKind::Other, "missing meta blob")); 181 | }; 182 | let meta_hash = hashes 183 | .first() 184 | .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "missing meta link"))?; 185 | let (meta_end, _meta_bytes) = at_meta.next(*meta_hash).concatenate_into_vec().await?; 186 | let mut curr = meta_end.next(); 187 | let closing = loop { 188 | match curr { 189 | EndBlobNext::MoreChildren(more) => { 190 | let Some(hash) = hashes.get(more.child_offset() as usize) else { 191 | break more.finish(); 192 | }; 193 | let header = more.next(*hash); 194 | let (mut content, _size) = header.next().await?; 195 | // manually loop over the content and yield all data 196 | let done = loop { 197 | match content.next().await { 198 | BlobContentNext::More((next, data)) => { 199 | if let BaoContentItem::Leaf(leaf) = data? { 200 | // yield the data 201 | co.yield_(Ok(leaf.data)).await; 202 | } 203 | content = next; 204 | } 205 | BlobContentNext::Done(done) => { 206 | // we are done with the root blob 207 | break done; 208 | } 209 | } 210 | }; 211 | curr = done.next(); 212 | } 213 | EndBlobNext::Closing(closing) => { 214 | break closing; 215 | } 216 | } 217 | }; 218 | // close the connection 219 | let _stats = closing.next().await?; 220 | Ok(()) 221 | } 222 | 223 | Gen::new(|co| async move { 224 | if let Err(e) = inner(initial, &co).await { 225 | co.yield_(Err(e)).await; 226 | } 227 | }) 228 | } 229 | -------------------------------------------------------------------------------- /examples/hello-world-fetch.rs: -------------------------------------------------------------------------------- 1 | //! An example that fetches an iroh blob and prints the contents. 2 | //! Will only work with blobs and collections that contain text, and is meant as a companion to the `hello-world-get` examples. 3 | //! 4 | //! This is using an in memory database and a random node id. 5 | //! Run the `provide` example, which will give you instructions on how to run this example. 6 | use std::{env, str::FromStr}; 7 | 8 | use anyhow::{bail, ensure, Context, Result}; 9 | use iroh::{protocol::Router, Endpoint}; 10 | use iroh_blobs::{net_protocol::Blobs, ticket::BlobTicket, BlobFormat}; 11 | use tracing_subscriber::{prelude::*, EnvFilter}; 12 | 13 | // set the RUST_LOG env var to one of {debug,info,warn} to see logging info 14 | pub fn setup_logging() { 15 | tracing_subscriber::registry() 16 | .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) 17 | .with(EnvFilter::from_default_env()) 18 | .try_init() 19 | .ok(); 20 | } 21 | 22 | #[tokio::main] 23 | async fn main() -> Result<()> { 24 | setup_logging(); 25 | println!("\n'Hello World' fetch example!"); 26 | // get the ticket 27 | let args: Vec = env::args().collect(); 28 | 29 | if args.len() != 2 { 30 | bail!("expected one argument [BLOB_TICKET]\n\nGet a ticket by running the follow command in a separate terminal:\n\n`cargo run --example hello-world-provide`"); 31 | } 32 | 33 | // deserialize ticket string into a ticket 34 | let ticket = 35 | BlobTicket::from_str(&args[1]).context("failed parsing blob ticket\n\nGet a ticket by running the follow command in a separate terminal:\n\n`cargo run --example hello-world-provide`")?; 36 | 37 | // create a new node 38 | let endpoint = Endpoint::builder().bind().await?; 39 | let builder = Router::builder(endpoint); 40 | let blobs = Blobs::memory().build(builder.endpoint()); 41 | let builder = builder.accept(iroh_blobs::ALPN, blobs.clone()); 42 | let node = builder.spawn(); 43 | let blobs_client = blobs.client(); 44 | 45 | println!("fetching hash: {}", ticket.hash()); 46 | println!("node id: {}", node.endpoint().node_id()); 47 | println!("node listening addresses:"); 48 | let addrs = node.endpoint().node_addr().await?; 49 | for addr in addrs.direct_addresses() { 50 | println!("\t{:?}", addr); 51 | } 52 | println!( 53 | "node relay server url: {:?}", 54 | node.endpoint() 55 | .home_relay() 56 | .get()? 57 | .expect("a default relay url should be provided") 58 | .to_string() 59 | ); 60 | 61 | // If the `BlobFormat` is `Raw`, we have the hash for a single blob, and simply need to read the blob using the `blobs` API on the client to get the content. 62 | ensure!( 63 | ticket.format() == BlobFormat::Raw, 64 | "'Hello World' example expects to fetch a single blob, but the ticket indicates a collection.", 65 | ); 66 | 67 | // `download` returns a stream of `DownloadProgress` events. You can iterate through these updates to get progress 68 | // on the state of your download. 69 | let download_stream = blobs_client 70 | .download(ticket.hash(), ticket.node_addr().clone()) 71 | .await?; 72 | 73 | // You can also just `await` the stream, which will poll the `DownloadProgress` stream for you. 74 | let outcome = download_stream.await.context("unable to download hash")?; 75 | 76 | println!( 77 | "\ndownloaded {} bytes from node {}", 78 | outcome.downloaded_size, 79 | ticket.node_addr().node_id 80 | ); 81 | 82 | // Get the content we have just fetched from the iroh database. 83 | 84 | let bytes = blobs_client.read_to_bytes(ticket.hash()).await?; 85 | let s = std::str::from_utf8(&bytes).context("unable to parse blob as as utf-8 string")?; 86 | println!("{s}"); 87 | 88 | Ok(()) 89 | } 90 | -------------------------------------------------------------------------------- /examples/hello-world-provide.rs: -------------------------------------------------------------------------------- 1 | //! The smallest possible example to spin up a node and serve a single blob. 2 | //! 3 | //! This is using an in memory database and a random node id. 4 | //! run this example from the project root: 5 | //! $ cargo run --example hello-world-provide 6 | use iroh::{protocol::Router, Endpoint}; 7 | use iroh_blobs::{net_protocol::Blobs, ticket::BlobTicket}; 8 | use tracing_subscriber::{prelude::*, EnvFilter}; 9 | 10 | // set the RUST_LOG env var to one of {debug,info,warn} to see logging info 11 | pub fn setup_logging() { 12 | tracing_subscriber::registry() 13 | .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) 14 | .with(EnvFilter::from_default_env()) 15 | .try_init() 16 | .ok(); 17 | } 18 | 19 | #[tokio::main] 20 | async fn main() -> anyhow::Result<()> { 21 | setup_logging(); 22 | println!("'Hello World' provide example!"); 23 | 24 | // create a new node 25 | let endpoint = Endpoint::builder().bind().await?; 26 | let builder = Router::builder(endpoint); 27 | let blobs = Blobs::memory().build(builder.endpoint()); 28 | let builder = builder.accept(iroh_blobs::ALPN, blobs.clone()); 29 | let blobs_client = blobs.client(); 30 | let node = builder.spawn(); 31 | 32 | // add some data and remember the hash 33 | let res = blobs_client.add_bytes("Hello, world!").await?; 34 | 35 | // create a ticket 36 | let addr = node.endpoint().node_addr().await?; 37 | let ticket = BlobTicket::new(addr, res.hash, res.format)?; 38 | 39 | // print some info about the node 40 | println!("serving hash: {}", ticket.hash()); 41 | println!("node id: {}", ticket.node_addr().node_id); 42 | println!("node listening addresses:"); 43 | for addr in ticket.node_addr().direct_addresses() { 44 | println!("\t{:?}", addr); 45 | } 46 | println!( 47 | "node relay server url: {:?}", 48 | ticket 49 | .node_addr() 50 | .relay_url() 51 | .expect("a default relay url should be provided") 52 | .to_string() 53 | ); 54 | // print the ticket, containing all the above information 55 | println!("\nin another terminal, run:"); 56 | println!("\t cargo run --example hello-world-fetch {}", ticket); 57 | // block until SIGINT is received (ctrl+c) 58 | tokio::signal::ctrl_c().await?; 59 | node.shutdown().await?; 60 | Ok(()) 61 | } 62 | -------------------------------------------------------------------------------- /examples/provide-bytes.rs: -------------------------------------------------------------------------------- 1 | //! An example that provides a blob or a collection over a Quinn connection. 2 | //! 3 | //! Since this example does not use [`iroh-net::Endpoint`], it does not do any holepunching, and so will only work locally or between two processes that have public IP addresses. 4 | //! 5 | //! Run this example with 6 | //! cargo run --example provide-bytes blob 7 | //! To provide a blob (single file) 8 | //! 9 | //! Run this example with 10 | //! cargo run --example provide-bytes collection 11 | //! To provide a collection (multiple blobs) 12 | use anyhow::Result; 13 | use iroh_blobs::{format::collection::Collection, util::local_pool::LocalPool, BlobFormat, Hash}; 14 | use tracing::warn; 15 | use tracing_subscriber::{prelude::*, EnvFilter}; 16 | 17 | const EXAMPLE_ALPN: &[u8] = b"n0/iroh/examples/bytes/0"; 18 | 19 | // set the RUST_LOG env var to one of {debug,info,warn} to see logging info 20 | pub fn setup_logging() { 21 | tracing_subscriber::registry() 22 | .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) 23 | .with(EnvFilter::from_default_env()) 24 | .try_init() 25 | .ok(); 26 | } 27 | 28 | #[tokio::main] 29 | async fn main() -> Result<()> { 30 | let args: Vec<_> = std::env::args().collect(); 31 | if args.len() != 2 { 32 | anyhow::bail!( 33 | "usage: provide-bytes [FORMAT], where [FORMAT] is either 'blob' or 'collection'\n\nThe 'blob' example demonstrates sending a single blob of bytes. The 'collection' example demonstrates sending multiple blobs of bytes, grouped together in a 'collection'." 34 | ); 35 | } 36 | let format = { 37 | if args[1] != "blob" && args[1] != "collection" { 38 | anyhow::bail!( 39 | "expected either 'blob' or 'collection' for FORMAT argument, got {}", 40 | args[1] 41 | ); 42 | } 43 | args[1].clone() 44 | }; 45 | println!("\nprovide bytes {format} example!"); 46 | 47 | let (db, hash, format) = if format == "collection" { 48 | let (mut db, names) = iroh_blobs::store::readonly_mem::Store::new([ 49 | ("blob1", b"the first blob of bytes".to_vec()), 50 | ("blob2", b"the second blob of bytes".to_vec()), 51 | ]); // create a collection 52 | let collection: Collection = names 53 | .into_iter() 54 | .map(|(name, hash)| (name, Hash::from(hash))) 55 | .collect(); 56 | // add it to the db 57 | let hash = db.insert_many(collection.to_blobs()).unwrap(); 58 | (db, hash, BlobFormat::HashSeq) 59 | } else { 60 | // create a new database and add a blob 61 | let (db, names) = 62 | iroh_blobs::store::readonly_mem::Store::new([("hello", b"Hello World!".to_vec())]); 63 | 64 | // get the hash of the content 65 | let hash = names.get("hello").unwrap(); 66 | (db, Hash::from(hash.as_bytes()), BlobFormat::Raw) 67 | }; 68 | 69 | // create an endpoint to listen for incoming connections 70 | let endpoint = iroh::Endpoint::builder() 71 | .relay_mode(iroh::RelayMode::Disabled) 72 | .alpns(vec![EXAMPLE_ALPN.into()]) 73 | .bind() 74 | .await?; 75 | let addr = endpoint.node_addr().await?; 76 | println!("\nlistening on {:?}", addr.direct_addresses); 77 | println!("providing hash {hash}"); 78 | 79 | let ticket = iroh_blobs::ticket::BlobTicket::new(addr, hash, format)?; 80 | 81 | println!("\nfetch the content using a finite state machine by running the following example:\n\ncargo run --example fetch-fsm {ticket}"); 82 | println!("\nfetch the content using a stream by running the following example:\n\ncargo run --example fetch-stream {ticket}\n"); 83 | 84 | // create a new local pool handle with 1 worker thread 85 | let lp = LocalPool::single(); 86 | 87 | let accept_task = tokio::spawn(async move { 88 | while let Some(incoming) = endpoint.accept().await { 89 | println!("connection incoming"); 90 | 91 | let conn = match incoming.accept() { 92 | Ok(conn) => conn, 93 | Err(err) => { 94 | warn!("incoming connection failed: {err:#}"); 95 | // we can carry on in these cases: 96 | // this can be caused by retransmitted datagrams 97 | continue; 98 | } 99 | }; 100 | let db = db.clone(); 101 | let lp = lp.clone(); 102 | 103 | // spawn a task to handle the connection 104 | tokio::spawn(async move { 105 | let conn = match conn.await { 106 | Ok(conn) => conn, 107 | Err(err) => { 108 | warn!("Error connecting: {err:#}"); 109 | return; 110 | } 111 | }; 112 | iroh_blobs::provider::handle_connection(conn, db, Default::default(), lp).await 113 | }); 114 | } 115 | }); 116 | 117 | match tokio::signal::ctrl_c().await { 118 | Ok(()) => { 119 | accept_task.abort(); 120 | Ok(()) 121 | } 122 | Err(e) => Err(anyhow::anyhow!("unable to listen for ctrl-c: {e}")), 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /examples/transfer.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | 3 | use anyhow::Result; 4 | use iroh::{protocol::Router, Endpoint}; 5 | use iroh_blobs::{ 6 | net_protocol::Blobs, 7 | rpc::client::blobs::WrapOption, 8 | store::{ExportFormat, ExportMode}, 9 | ticket::BlobTicket, 10 | util::SetTagOption, 11 | }; 12 | 13 | #[tokio::main] 14 | async fn main() -> Result<()> { 15 | // Create an endpoint, it allows creating and accepting 16 | // connections in the iroh p2p world 17 | let endpoint = Endpoint::builder().discovery_n0().bind().await?; 18 | // We initialize the Blobs protocol in-memory 19 | let blobs = Blobs::memory().build(&endpoint); 20 | 21 | // Now we build a router that accepts blobs connections & routes them 22 | // to the blobs protocol. 23 | let router = Router::builder(endpoint) 24 | .accept(iroh_blobs::ALPN, blobs.clone()) 25 | .spawn(); 26 | 27 | // We use a blobs client to interact with the blobs protocol we're running locally: 28 | let blobs_client = blobs.client(); 29 | 30 | // Grab all passed in arguments, the first one is the binary itself, so we skip it. 31 | let args: Vec = std::env::args().skip(1).collect(); 32 | // Convert to &str, so we can pattern-match easily: 33 | let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect(); 34 | 35 | match arg_refs.as_slice() { 36 | ["send", filename] => { 37 | let filename: PathBuf = filename.parse()?; 38 | let abs_path = std::path::absolute(&filename)?; 39 | 40 | println!("Hashing file."); 41 | 42 | // keep the file in place and link it, instead of copying it into the in-memory blobs database 43 | let in_place = true; 44 | let blob = blobs_client 45 | .add_from_path(abs_path, in_place, SetTagOption::Auto, WrapOption::NoWrap) 46 | .await? 47 | .finish() 48 | .await?; 49 | 50 | let node_id = router.endpoint().node_id(); 51 | let ticket = BlobTicket::new(node_id.into(), blob.hash, blob.format)?; 52 | 53 | println!("File hashed. Fetch this file by running:"); 54 | println!( 55 | "cargo run --example transfer -- receive {ticket} {}", 56 | filename.display() 57 | ); 58 | 59 | tokio::signal::ctrl_c().await?; 60 | } 61 | ["receive", ticket, filename] => { 62 | let filename: PathBuf = filename.parse()?; 63 | let abs_path = std::path::absolute(filename)?; 64 | let ticket: BlobTicket = ticket.parse()?; 65 | 66 | println!("Starting download."); 67 | 68 | blobs_client 69 | .download(ticket.hash(), ticket.node_addr().clone()) 70 | .await? 71 | .finish() 72 | .await?; 73 | 74 | println!("Finished download."); 75 | println!("Copying to destination."); 76 | 77 | blobs_client 78 | .export( 79 | ticket.hash(), 80 | abs_path, 81 | ExportFormat::Blob, 82 | ExportMode::Copy, 83 | ) 84 | .await? 85 | .finish() 86 | .await?; 87 | 88 | println!("Finished copying."); 89 | } 90 | _ => { 91 | println!("Couldn't parse command line arguments: {args:?}"); 92 | println!("Usage:"); 93 | println!(" # to send:"); 94 | println!(" cargo run --example transfer -- send [FILE]"); 95 | println!(" # this will print a ticket."); 96 | println!(); 97 | println!(" # to receive:"); 98 | println!(" cargo run --example transfer -- receive [TICKET] [FILE]"); 99 | } 100 | } 101 | 102 | // Gracefully shut down the node 103 | println!("Shutting down."); 104 | router.shutdown().await?; 105 | 106 | Ok(()) 107 | } 108 | -------------------------------------------------------------------------------- /proptest-regressions/protocol/range_spec.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 7375b003a63bfe725eb4bcb2f266fae6afd9b3c921f9c2018f97daf6ef05a364 # shrinks to ranges = [RangeSet{ChunkNum(0)..ChunkNum(1)}, RangeSet{}] 8 | cc 23322efa46881646f1468137a688e66aee7ec2a3d01895ccad851d442a7828af # shrinks to ranges = [RangeSet{}, RangeSet{ChunkNum(0)..ChunkNum(1)}] 9 | -------------------------------------------------------------------------------- /proptest-regressions/provider.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 25ec044e2b84054195984d7e04b93d9b39e2cc25eaee4037dc1be9398f9fd4b4 # shrinks to db = Database(RwLock { data: {}, poisoned: false, .. }) 8 | -------------------------------------------------------------------------------- /release.toml: -------------------------------------------------------------------------------- 1 | pre-release-hook = ["git", "cliff", "--prepend", "CHANGELOG.md", "--tag", "{{version}}", "--unreleased" ] 2 | -------------------------------------------------------------------------------- /src/cli/tags.rs: -------------------------------------------------------------------------------- 1 | //! Define the tags subcommand. 2 | 3 | use anyhow::Result; 4 | use bytes::Bytes; 5 | use clap::Subcommand; 6 | use futures_lite::StreamExt; 7 | 8 | use crate::{rpc::client::tags, Tag}; 9 | 10 | /// Commands to manage tags. 11 | #[derive(Subcommand, Debug, Clone)] 12 | #[allow(clippy::large_enum_variant)] 13 | pub enum TagCommands { 14 | /// List all tags 15 | List, 16 | /// Delete a tag 17 | Delete { 18 | tag: String, 19 | #[clap(long, default_value_t = false)] 20 | hex: bool, 21 | }, 22 | } 23 | 24 | impl TagCommands { 25 | /// Runs the tag command given the iroh client. 26 | pub async fn run(self, tags: &tags::Client) -> Result<()> { 27 | match self { 28 | Self::List => { 29 | let mut response = tags.list().await?; 30 | while let Some(res) = response.next().await { 31 | let res = res?; 32 | println!("{}: {} ({:?})", res.name, res.hash, res.format); 33 | } 34 | } 35 | Self::Delete { tag, hex } => { 36 | let tag = if hex { 37 | Tag::from(Bytes::from(hex::decode(tag)?)) 38 | } else { 39 | Tag::from(tag) 40 | }; 41 | tags.delete(tag).await?; 42 | } 43 | } 44 | Ok(()) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/downloader/get.rs: -------------------------------------------------------------------------------- 1 | //! [`Getter`] implementation that performs requests over [`Connection`]s. 2 | //! 3 | //! [`Connection`]: iroh::endpoint::Connection 4 | 5 | use futures_lite::FutureExt; 6 | use iroh::endpoint; 7 | 8 | use super::{progress::BroadcastProgressSender, DownloadKind, FailureAction, GetStartFut, Getter}; 9 | use crate::{ 10 | get::{db::get_to_db_in_steps, error::GetError}, 11 | store::Store, 12 | }; 13 | 14 | impl From for FailureAction { 15 | fn from(e: GetError) -> Self { 16 | match e { 17 | e @ GetError::NotFound(_) => FailureAction::AbortRequest(e), 18 | e @ GetError::RemoteReset(_) => FailureAction::RetryLater(e.into()), 19 | e @ GetError::NoncompliantNode(_) => FailureAction::DropPeer(e.into()), 20 | e @ GetError::Io(_) => FailureAction::RetryLater(e.into()), 21 | e @ GetError::BadRequest(_) => FailureAction::AbortRequest(e), 22 | // TODO: what do we want to do on local failures? 23 | e @ GetError::LocalFailure(_) => FailureAction::AbortRequest(e), 24 | } 25 | } 26 | } 27 | 28 | /// [`Getter`] implementation that performs requests over [`Connection`]s. 29 | /// 30 | /// [`Connection`]: iroh::endpoint::Connection 31 | pub(crate) struct IoGetter { 32 | pub store: S, 33 | } 34 | 35 | impl Getter for IoGetter { 36 | type Connection = endpoint::Connection; 37 | type NeedsConn = crate::get::db::GetStateNeedsConn; 38 | 39 | fn get( 40 | &mut self, 41 | kind: DownloadKind, 42 | progress_sender: BroadcastProgressSender, 43 | ) -> GetStartFut { 44 | let store = self.store.clone(); 45 | async move { 46 | match get_to_db_in_steps(store, kind.hash_and_format(), progress_sender).await { 47 | Err(err) => Err(err.into()), 48 | Ok(crate::get::db::GetState::Complete(stats)) => { 49 | Ok(super::GetOutput::Complete(stats)) 50 | } 51 | Ok(crate::get::db::GetState::NeedsConn(needs_conn)) => { 52 | Ok(super::GetOutput::NeedsConn(needs_conn)) 53 | } 54 | } 55 | } 56 | .boxed_local() 57 | } 58 | } 59 | 60 | impl super::NeedsConn for crate::get::db::GetStateNeedsConn { 61 | fn proceed(self, conn: endpoint::Connection) -> super::GetProceedFut { 62 | async move { 63 | let res = self.proceed(conn).await; 64 | match res { 65 | Ok(stats) => Ok(stats), 66 | Err(err) => Err(err.into()), 67 | } 68 | } 69 | .boxed_local() 70 | } 71 | } 72 | 73 | pub(super) fn track_metrics( 74 | res: &Result, 75 | metrics: &crate::metrics::Metrics, 76 | ) { 77 | match res { 78 | Ok(stats) => { 79 | let crate::get::Stats { 80 | bytes_written, 81 | bytes_read: _, 82 | elapsed, 83 | } = stats; 84 | 85 | metrics.downloads_success.inc(); 86 | metrics.download_bytes_total.inc_by(*bytes_written); 87 | metrics 88 | .download_time_total 89 | .inc_by(elapsed.as_millis() as u64); 90 | } 91 | Err(e) => match &e { 92 | FailureAction::AbortRequest(GetError::NotFound(_)) => { 93 | metrics.downloads_notfound.inc(); 94 | } 95 | _ => { 96 | metrics.downloads_error.inc(); 97 | } 98 | }, 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/downloader/invariants.rs: -------------------------------------------------------------------------------- 1 | //! Invariants for the service. 2 | 3 | #![cfg(any(test, debug_assertions))] 4 | 5 | use super::*; 6 | 7 | /// invariants for the service. 8 | impl, D: DialerT> Service { 9 | /// Checks the various invariants the service must maintain 10 | #[track_caller] 11 | pub(in crate::downloader) fn check_invariants(&self) { 12 | self.check_active_request_count(); 13 | self.check_queued_requests_consistency(); 14 | self.check_idle_peer_consistency(); 15 | self.check_concurrency_limits(); 16 | self.check_provider_map_prunning(); 17 | } 18 | 19 | /// Checks concurrency limits are maintained. 20 | #[track_caller] 21 | fn check_concurrency_limits(&self) { 22 | let ConcurrencyLimits { 23 | max_concurrent_requests, 24 | max_concurrent_requests_per_node, 25 | max_open_connections, 26 | max_concurrent_dials_per_hash, 27 | } = &self.concurrency_limits; 28 | 29 | // check the total number of active requests to ensure it stays within the limit 30 | assert!( 31 | self.in_progress_downloads.len() <= *max_concurrent_requests, 32 | "max_concurrent_requests exceeded" 33 | ); 34 | 35 | // check that the open and dialing peers don't exceed the connection capacity 36 | tracing::trace!( 37 | "limits: conns: {}/{} | reqs: {}/{}", 38 | self.connections_count(), 39 | max_open_connections, 40 | self.in_progress_downloads.len(), 41 | max_concurrent_requests 42 | ); 43 | assert!( 44 | self.connections_count() <= *max_open_connections, 45 | "max_open_connections exceeded" 46 | ); 47 | 48 | // check the active requests per peer don't exceed the limit 49 | for (node, info) in self.connected_nodes.iter() { 50 | assert!( 51 | info.active_requests() <= *max_concurrent_requests_per_node, 52 | "max_concurrent_requests_per_node exceeded for {node}" 53 | ) 54 | } 55 | 56 | // check that we do not dial more nodes than allowed for the next pending hashes 57 | if let Some(kind) = self.queue.front() { 58 | let hash = kind.hash(); 59 | let nodes = self.providers.get_candidates(&hash); 60 | let mut dialing = 0; 61 | for node in nodes { 62 | if self.dialer.is_pending(node) { 63 | dialing += 1; 64 | } 65 | } 66 | assert!( 67 | dialing <= *max_concurrent_dials_per_hash, 68 | "max_concurrent_dials_per_hash exceeded for {hash}" 69 | ) 70 | } 71 | } 72 | 73 | /// Checks that the count of active requests per peer is consistent with the active requests, 74 | /// and that active request are consistent with download futures 75 | #[track_caller] 76 | fn check_active_request_count(&self) { 77 | // check that the count of futures we are polling for downloads is consistent with the 78 | // number of requests 79 | assert_eq!( 80 | self.active_requests.len(), 81 | self.in_progress_downloads.len(), 82 | "active_requests and in_progress_downloads are out of sync" 83 | ); 84 | // check that the count of requests per peer matches the number of requests that have that 85 | // peer as active 86 | let mut real_count: HashMap = 87 | HashMap::with_capacity(self.connected_nodes.len()); 88 | for req_info in self.active_requests.values() { 89 | // nothing like some classic word count 90 | *real_count.entry(req_info.node).or_default() += 1; 91 | } 92 | for (peer, info) in self.connected_nodes.iter() { 93 | assert_eq!( 94 | info.active_requests(), 95 | real_count.get(peer).copied().unwrap_or_default(), 96 | "mismatched count of active requests for {peer}" 97 | ) 98 | } 99 | } 100 | 101 | /// Checks that the queued requests all appear in the provider map and request map. 102 | #[track_caller] 103 | fn check_queued_requests_consistency(&self) { 104 | // check that all hashes in the queue have candidates 105 | for entry in self.queue.iter() { 106 | assert!( 107 | self.providers 108 | .get_candidates(&entry.hash()) 109 | .next() 110 | .is_some(), 111 | "all queued requests have providers" 112 | ); 113 | assert!( 114 | self.requests.contains_key(entry), 115 | "all queued requests have request info" 116 | ); 117 | } 118 | 119 | // check that all parked hashes should be parked 120 | for entry in self.queue.iter_parked() { 121 | assert!( 122 | matches!(self.next_step(entry), NextStep::Park), 123 | "all parked downloads evaluate to the correct next step" 124 | ); 125 | assert!( 126 | self.providers 127 | .get_candidates(&entry.hash()) 128 | .all(|node| matches!(self.node_state(node), NodeState::WaitForRetry)), 129 | "all parked downloads have only retrying nodes" 130 | ); 131 | } 132 | } 133 | 134 | /// Check that peers queued to be disconnected are consistent with peers considered idle. 135 | #[track_caller] 136 | fn check_idle_peer_consistency(&self) { 137 | let idle_peers = self 138 | .connected_nodes 139 | .values() 140 | .filter(|info| info.active_requests() == 0) 141 | .count(); 142 | assert_eq!( 143 | self.goodbye_nodes_queue.len(), 144 | idle_peers, 145 | "inconsistent count of idle peers" 146 | ); 147 | } 148 | 149 | /// Check that every hash in the provider map is needed. 150 | #[track_caller] 151 | fn check_provider_map_prunning(&self) { 152 | for hash in self.providers.hash_node.keys() { 153 | let as_raw = DownloadKind(HashAndFormat::raw(*hash)); 154 | let as_hash_seq = DownloadKind(HashAndFormat::hash_seq(*hash)); 155 | assert!( 156 | self.queue.contains_hash(*hash) 157 | || self.active_requests.contains_key(&as_raw) 158 | || self.active_requests.contains_key(&as_hash_seq), 159 | "all hashes in the provider map are in the queue or active" 160 | ) 161 | } 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /src/downloader/progress.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::HashMap, 3 | sync::{ 4 | atomic::{AtomicU64, Ordering}, 5 | Arc, 6 | }, 7 | }; 8 | 9 | use anyhow::anyhow; 10 | use parking_lot::Mutex; 11 | 12 | use super::DownloadKind; 13 | use crate::{ 14 | get::{db::DownloadProgress, progress::TransferState}, 15 | util::progress::{AsyncChannelProgressSender, IdGenerator, ProgressSendError, ProgressSender}, 16 | }; 17 | 18 | /// The channel that can be used to subscribe to progress updates. 19 | pub type ProgressSubscriber = AsyncChannelProgressSender; 20 | 21 | /// Track the progress of downloads. 22 | /// 23 | /// This struct allows to create [`ProgressSender`] structs to be passed to 24 | /// [`crate::get::db::get_to_db`]. Each progress sender can be subscribed to by any number of 25 | /// [`ProgressSubscriber`] channel senders, which will receive each progress update (if they have 26 | /// capacity). Additionally, the [`ProgressTracker`] maintains a [`TransferState`] for each 27 | /// transfer, applying each progress update to update this state. When subscribing to an already 28 | /// running transfer, the subscriber will receive a [`DownloadProgress::InitialState`] message 29 | /// containing the state at the time of the subscription, and then receive all further progress 30 | /// events directly. 31 | #[derive(Debug, Default)] 32 | pub struct ProgressTracker { 33 | /// Map of shared state for each tracked download. 34 | running: HashMap, 35 | /// Shared [`IdGenerator`] for all progress senders created by the tracker. 36 | id_gen: Arc, 37 | } 38 | 39 | impl ProgressTracker { 40 | pub fn new() -> Self { 41 | Self::default() 42 | } 43 | 44 | /// Track a new download with a list of initial subscribers. 45 | /// 46 | /// Note that this should only be called for *new* downloads. If a download for the `kind` is 47 | /// already tracked in this [`ProgressTracker`], calling `track` will replace all existing 48 | /// state and subscribers (equal to calling [`Self::remove`] first). 49 | pub fn track( 50 | &mut self, 51 | kind: DownloadKind, 52 | subscribers: impl IntoIterator, 53 | ) -> BroadcastProgressSender { 54 | let inner = Inner { 55 | subscribers: subscribers.into_iter().collect(), 56 | state: TransferState::new(kind.hash()), 57 | }; 58 | let shared = Arc::new(Mutex::new(inner)); 59 | self.running.insert(kind, Arc::clone(&shared)); 60 | let id_gen = Arc::clone(&self.id_gen); 61 | BroadcastProgressSender { shared, id_gen } 62 | } 63 | 64 | /// Subscribe to a tracked download. 65 | /// 66 | /// Will return an error if `kind` is not yet tracked. 67 | pub async fn subscribe( 68 | &mut self, 69 | kind: DownloadKind, 70 | sender: ProgressSubscriber, 71 | ) -> anyhow::Result<()> { 72 | let initial_msg = self 73 | .running 74 | .get_mut(&kind) 75 | .ok_or_else(|| anyhow!("state for download {kind:?} not found"))? 76 | .lock() 77 | .subscribe(sender.clone()); 78 | sender.send(initial_msg).await?; 79 | Ok(()) 80 | } 81 | 82 | /// Unsubscribe `sender` from `kind`. 83 | pub fn unsubscribe(&mut self, kind: &DownloadKind, sender: &ProgressSubscriber) { 84 | if let Some(shared) = self.running.get_mut(kind) { 85 | shared.lock().unsubscribe(sender) 86 | } 87 | } 88 | 89 | /// Remove all state for a download. 90 | pub fn remove(&mut self, kind: &DownloadKind) { 91 | self.running.remove(kind); 92 | } 93 | } 94 | 95 | type Shared = Arc>; 96 | 97 | #[derive(Debug)] 98 | struct Inner { 99 | subscribers: Vec, 100 | state: TransferState, 101 | } 102 | 103 | impl Inner { 104 | fn subscribe(&mut self, subscriber: ProgressSubscriber) -> DownloadProgress { 105 | let msg = DownloadProgress::InitialState(self.state.clone()); 106 | self.subscribers.push(subscriber); 107 | msg 108 | } 109 | 110 | fn unsubscribe(&mut self, sender: &ProgressSubscriber) { 111 | self.subscribers.retain(|s| !s.same_channel(sender)); 112 | } 113 | 114 | fn on_progress(&mut self, progress: DownloadProgress) { 115 | self.state.on_progress(progress); 116 | } 117 | } 118 | 119 | #[derive(Debug, Clone)] 120 | pub struct BroadcastProgressSender { 121 | shared: Shared, 122 | id_gen: Arc, 123 | } 124 | 125 | impl IdGenerator for BroadcastProgressSender { 126 | fn new_id(&self) -> u64 { 127 | self.id_gen.fetch_add(1, Ordering::SeqCst) 128 | } 129 | } 130 | 131 | impl ProgressSender for BroadcastProgressSender { 132 | type Msg = DownloadProgress; 133 | 134 | async fn send(&self, msg: Self::Msg) -> Result<(), ProgressSendError> { 135 | // making sure that the lock is not held across an await point. 136 | let futs = { 137 | let mut inner = self.shared.lock(); 138 | inner.on_progress(msg.clone()); 139 | let futs = inner 140 | .subscribers 141 | .iter_mut() 142 | .map(|sender| { 143 | let sender = sender.clone(); 144 | let msg = msg.clone(); 145 | async move { 146 | match sender.send(msg).await { 147 | Ok(()) => None, 148 | Err(ProgressSendError::ReceiverDropped) => Some(sender), 149 | } 150 | } 151 | }) 152 | .collect::>(); 153 | drop(inner); 154 | futs 155 | }; 156 | 157 | let failed_senders = futures_buffered::join_all(futs).await; 158 | // remove senders where the receiver is dropped 159 | if failed_senders.iter().any(|s| s.is_some()) { 160 | let mut inner = self.shared.lock(); 161 | for sender in failed_senders.into_iter().flatten() { 162 | inner.unsubscribe(&sender); 163 | } 164 | drop(inner); 165 | } 166 | Ok(()) 167 | } 168 | 169 | fn try_send(&self, msg: Self::Msg) -> Result<(), ProgressSendError> { 170 | let mut inner = self.shared.lock(); 171 | inner.on_progress(msg.clone()); 172 | // remove senders where the receiver is dropped 173 | inner 174 | .subscribers 175 | .retain_mut(|sender| match sender.try_send(msg.clone()) { 176 | Err(ProgressSendError::ReceiverDropped) => false, 177 | Ok(()) => true, 178 | }); 179 | Ok(()) 180 | } 181 | 182 | fn blocking_send(&self, msg: Self::Msg) -> Result<(), ProgressSendError> { 183 | let mut inner = self.shared.lock(); 184 | inner.on_progress(msg.clone()); 185 | // remove senders where the receiver is dropped 186 | inner 187 | .subscribers 188 | .retain_mut(|sender| match sender.blocking_send(msg.clone()) { 189 | Err(ProgressSendError::ReceiverDropped) => false, 190 | Ok(()) => true, 191 | }); 192 | Ok(()) 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /src/downloader/test/dialer.rs: -------------------------------------------------------------------------------- 1 | //! Implementation of [`super::Dialer`] used for testing. 2 | 3 | use std::task::{Context, Poll}; 4 | 5 | use parking_lot::RwLock; 6 | 7 | use super::*; 8 | 9 | /// Dialer for testing that keeps track of the dialing history. 10 | #[derive(Default, Clone)] 11 | pub(super) struct TestingDialer(Arc>); 12 | 13 | struct TestingDialerInner { 14 | /// Peers that are being dialed. 15 | dialing: HashSet, 16 | /// Queue of dials. 17 | dial_futs: delay_queue::DelayQueue, 18 | /// History of attempted dials. 19 | dial_history: Vec, 20 | /// How long does a dial last. 21 | dial_duration: Duration, 22 | /// Fn deciding if a dial is successful. 23 | dial_outcome: Box bool + Send + Sync + 'static>, 24 | /// Our own node id 25 | node_id: NodeId, 26 | } 27 | 28 | impl Default for TestingDialerInner { 29 | fn default() -> Self { 30 | TestingDialerInner { 31 | dialing: HashSet::default(), 32 | dial_futs: delay_queue::DelayQueue::default(), 33 | dial_history: Vec::default(), 34 | dial_duration: Duration::from_millis(10), 35 | dial_outcome: Box::new(|_| true), 36 | node_id: NodeId::from_bytes(&[0u8; 32]).unwrap(), 37 | } 38 | } 39 | } 40 | 41 | impl DialerT for TestingDialer { 42 | type Connection = NodeId; 43 | 44 | fn queue_dial(&mut self, node_id: NodeId) { 45 | let mut inner = self.0.write(); 46 | inner.dial_history.push(node_id); 47 | // for now assume every dial works 48 | let dial_duration = inner.dial_duration; 49 | if inner.dialing.insert(node_id) { 50 | inner.dial_futs.insert(node_id, dial_duration); 51 | } 52 | } 53 | 54 | fn pending_count(&self) -> usize { 55 | self.0.read().dialing.len() 56 | } 57 | 58 | fn is_pending(&self, node: NodeId) -> bool { 59 | self.0.read().dialing.contains(&node) 60 | } 61 | 62 | fn node_id(&self) -> NodeId { 63 | self.0.read().node_id 64 | } 65 | } 66 | 67 | impl Stream for TestingDialer { 68 | type Item = (NodeId, anyhow::Result); 69 | 70 | fn poll_next(self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 71 | let mut inner = self.0.write(); 72 | match inner.dial_futs.poll_expired(cx) { 73 | Poll::Ready(Some(expired)) => { 74 | let node = expired.into_inner(); 75 | let report_ok = (inner.dial_outcome)(node); 76 | let result = report_ok 77 | .then_some(node) 78 | .ok_or_else(|| anyhow::anyhow!("dialing test set to fail")); 79 | inner.dialing.remove(&node); 80 | Poll::Ready(Some((node, result))) 81 | } 82 | _ => Poll::Pending, 83 | } 84 | } 85 | } 86 | 87 | impl TestingDialer { 88 | #[track_caller] 89 | pub(super) fn assert_history(&self, history: &[NodeId]) { 90 | assert_eq!(self.0.read().dial_history, history) 91 | } 92 | 93 | pub(super) fn set_dial_outcome( 94 | &self, 95 | dial_outcome: impl Fn(NodeId) -> bool + Send + Sync + 'static, 96 | ) { 97 | let mut inner = self.0.write(); 98 | inner.dial_outcome = Box::new(dial_outcome); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/downloader/test/getter.rs: -------------------------------------------------------------------------------- 1 | //! Implementation of [`super::Getter`] used for testing. 2 | 3 | use futures_lite::{future::Boxed as BoxFuture, FutureExt}; 4 | use parking_lot::RwLock; 5 | 6 | use super::*; 7 | use crate::downloader; 8 | 9 | #[derive(Default, Clone, derive_more::Debug)] 10 | #[debug("TestingGetter")] 11 | pub(super) struct TestingGetter(Arc>); 12 | 13 | pub(super) type RequestHandlerFn = Arc< 14 | dyn Fn( 15 | DownloadKind, 16 | NodeId, 17 | BroadcastProgressSender, 18 | Duration, 19 | ) -> BoxFuture 20 | + Send 21 | + Sync 22 | + 'static, 23 | >; 24 | 25 | #[derive(Default)] 26 | struct TestingGetterInner { 27 | /// How long requests take. 28 | request_duration: Duration, 29 | /// History of requests performed by the [`Getter`] and if they were successful. 30 | request_history: Vec<(DownloadKind, NodeId)>, 31 | /// Set a handler function which actually handles the requests. 32 | request_handler: Option, 33 | } 34 | 35 | impl Getter for TestingGetter { 36 | // since for testing we don't need a real connection, just keep track of what peer is the 37 | // request being sent to 38 | type Connection = NodeId; 39 | type NeedsConn = GetStateNeedsConn; 40 | 41 | fn get( 42 | &mut self, 43 | kind: DownloadKind, 44 | progress_sender: BroadcastProgressSender, 45 | ) -> GetStartFut { 46 | std::future::ready(Ok(downloader::GetOutput::NeedsConn(GetStateNeedsConn( 47 | self.clone(), 48 | kind, 49 | progress_sender, 50 | )))) 51 | .boxed_local() 52 | } 53 | } 54 | 55 | #[derive(Debug)] 56 | pub(super) struct GetStateNeedsConn(TestingGetter, DownloadKind, BroadcastProgressSender); 57 | 58 | impl downloader::NeedsConn for GetStateNeedsConn { 59 | fn proceed(self, peer: NodeId) -> super::GetProceedFut { 60 | let GetStateNeedsConn(getter, kind, progress_sender) = self; 61 | let mut inner = getter.0.write(); 62 | inner.request_history.push((kind, peer)); 63 | let request_duration = inner.request_duration; 64 | let handler = inner.request_handler.clone(); 65 | async move { 66 | if let Some(f) = handler { 67 | f(kind, peer, progress_sender, request_duration).await 68 | } else { 69 | tokio::time::sleep(request_duration).await; 70 | Ok(Stats::default()) 71 | } 72 | } 73 | .boxed_local() 74 | } 75 | } 76 | 77 | impl TestingGetter { 78 | pub(super) fn set_handler(&self, handler: RequestHandlerFn) { 79 | self.0.write().request_handler = Some(handler); 80 | } 81 | pub(super) fn set_request_duration(&self, request_duration: Duration) { 82 | self.0.write().request_duration = request_duration; 83 | } 84 | /// Verify that the request history is as expected 85 | #[track_caller] 86 | pub(super) fn assert_history(&self, history: &[(DownloadKind, NodeId)]) { 87 | assert_eq!(self.0.read().request_history, history); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/export.rs: -------------------------------------------------------------------------------- 1 | //! Functions to export data from a store 2 | 3 | use std::path::PathBuf; 4 | 5 | use anyhow::Context; 6 | use bytes::Bytes; 7 | use serde::{Deserialize, Serialize}; 8 | use tracing::trace; 9 | 10 | use crate::{ 11 | format::collection::Collection, 12 | store::{BaoBlobSize, ExportFormat, ExportMode, MapEntry, Store as BaoStore}, 13 | util::progress::{IdGenerator, ProgressSender}, 14 | Hash, 15 | }; 16 | 17 | /// Export a hash to the local file system. 18 | /// 19 | /// This exports a single hash, or a collection `recursive` is true, from the `db` store to the 20 | /// local filesystem. Depending on `mode` the data is either copied or reflinked (if possible). 21 | /// 22 | /// Progress is reported as [`ExportProgress`] through a [`ProgressSender`]. Note that the 23 | /// [`ExportProgress::AllDone`] event is not emitted from here, but left to an upper layer to send, 24 | /// if desired. 25 | pub async fn export( 26 | db: &D, 27 | hash: Hash, 28 | outpath: PathBuf, 29 | format: ExportFormat, 30 | mode: ExportMode, 31 | progress: impl ProgressSender + IdGenerator, 32 | ) -> anyhow::Result<()> { 33 | match format { 34 | ExportFormat::Blob => export_blob(db, hash, outpath, mode, progress).await, 35 | ExportFormat::Collection => export_collection(db, hash, outpath, mode, progress).await, 36 | } 37 | } 38 | 39 | /// Export all entries of a collection, recursively, to files on the local filesystem. 40 | pub async fn export_collection( 41 | db: &D, 42 | hash: Hash, 43 | outpath: PathBuf, 44 | mode: ExportMode, 45 | progress: impl ProgressSender + IdGenerator, 46 | ) -> anyhow::Result<()> { 47 | tokio::fs::create_dir_all(&outpath).await?; 48 | let collection = Collection::load_db(db, &hash).await?; 49 | for (name, hash) in collection.into_iter() { 50 | #[allow(clippy::needless_borrow)] 51 | let path = outpath.join(pathbuf_from_name(&name)); 52 | export_blob(db, hash, path, mode, progress.clone()).await?; 53 | } 54 | Ok(()) 55 | } 56 | 57 | /// Export a single blob to a file on the local filesystem. 58 | pub async fn export_blob( 59 | db: &D, 60 | hash: Hash, 61 | outpath: PathBuf, 62 | mode: ExportMode, 63 | progress: impl ProgressSender + IdGenerator, 64 | ) -> anyhow::Result<()> { 65 | if let Some(parent) = outpath.parent() { 66 | tokio::fs::create_dir_all(parent).await?; 67 | } 68 | trace!("exporting blob {} to {}", hash, outpath.display()); 69 | let id = progress.new_id(); 70 | let entry = db.get(&hash).await?.context("entry not there")?; 71 | progress 72 | .send(ExportProgress::Found { 73 | id, 74 | hash, 75 | outpath: outpath.clone(), 76 | size: entry.size(), 77 | meta: None, 78 | }) 79 | .await?; 80 | let progress1 = progress.clone(); 81 | db.export( 82 | hash, 83 | outpath, 84 | mode, 85 | Box::new(move |offset| Ok(progress1.try_send(ExportProgress::Progress { id, offset })?)), 86 | ) 87 | .await?; 88 | progress.send(ExportProgress::Done { id }).await?; 89 | Ok(()) 90 | } 91 | 92 | /// Progress events for an export operation 93 | #[derive(Debug, Clone, Serialize, Deserialize)] 94 | pub enum ExportProgress { 95 | /// The download part is done for this id, we are now exporting the data 96 | /// to the specified out path. 97 | Found { 98 | /// Unique id of the entry. 99 | id: u64, 100 | /// The hash of the entry. 101 | hash: Hash, 102 | /// The size of the entry in bytes. 103 | size: BaoBlobSize, 104 | /// The path to the file where the data is exported. 105 | outpath: PathBuf, 106 | /// Operation-specific metadata. 107 | meta: Option, 108 | }, 109 | /// We have made progress exporting the data. 110 | /// 111 | /// This is only sent for large blobs. 112 | Progress { 113 | /// Unique id of the entry that is being exported. 114 | id: u64, 115 | /// The offset of the progress, in bytes. 116 | offset: u64, 117 | }, 118 | /// We finished exporting a blob 119 | Done { 120 | /// Unique id of the entry that is being exported. 121 | id: u64, 122 | }, 123 | /// We are done with the whole operation. 124 | AllDone, 125 | /// We got an error and need to abort. 126 | Abort(serde_error::Error), 127 | } 128 | 129 | fn pathbuf_from_name(name: &str) -> PathBuf { 130 | let mut path = PathBuf::new(); 131 | for part in name.split('/') { 132 | path.push(part); 133 | } 134 | path 135 | } 136 | -------------------------------------------------------------------------------- /src/format.rs: -------------------------------------------------------------------------------- 1 | //! Defines data formats for HashSeq. 2 | //! 3 | //! The exact details how to use a HashSeq for specific purposes is up to the 4 | //! user. However, the following approach is used by iroh formats: 5 | //! 6 | //! The first child blob is a metadata blob. It starts with a header, followed 7 | //! by serialized metadata. We mostly use [postcard] for serialization. The 8 | //! metadata either implicitly or explicitly refers to the other blobs in the 9 | //! HashSeq by index. 10 | //! 11 | //! In a very simple case, the metadata just an array of items, where each item 12 | //! is the metadata for the corresponding blob. The metadata array will have 13 | //! n-1 items, where n is the number of blobs in the HashSeq. 14 | //! 15 | //! [postcard]: https://docs.rs/postcard/latest/postcard/ 16 | pub mod collection; 17 | -------------------------------------------------------------------------------- /src/get/error.rs: -------------------------------------------------------------------------------- 1 | //! Error returned from get operations 2 | 3 | use iroh::endpoint::{self, ClosedStream}; 4 | 5 | use crate::util::progress::ProgressSendError; 6 | 7 | /// Failures for a get operation 8 | #[derive(Debug, thiserror::Error)] 9 | pub enum GetError { 10 | /// Hash not found. 11 | #[error("Hash not found")] 12 | NotFound(#[source] anyhow::Error), 13 | /// Remote has reset the connection. 14 | #[error("Remote has reset the connection")] 15 | RemoteReset(#[source] anyhow::Error), 16 | /// Remote behaved in a non-compliant way. 17 | #[error("Remote behaved in a non-compliant way")] 18 | NoncompliantNode(#[source] anyhow::Error), 19 | 20 | /// Network or IO operation failed. 21 | #[error("A network or IO operation failed")] 22 | Io(#[source] anyhow::Error), 23 | 24 | /// Our download request is invalid. 25 | #[error("Our download request is invalid")] 26 | BadRequest(#[source] anyhow::Error), 27 | /// Operation failed on the local node. 28 | #[error("Operation failed on the local node")] 29 | LocalFailure(#[source] anyhow::Error), 30 | } 31 | 32 | impl From for GetError { 33 | fn from(value: ProgressSendError) -> Self { 34 | Self::LocalFailure(value.into()) 35 | } 36 | } 37 | 38 | impl From for GetError { 39 | fn from(value: endpoint::ConnectionError) -> Self { 40 | // explicit match just to be sure we are taking everything into account 41 | use endpoint::ConnectionError; 42 | match value { 43 | e @ ConnectionError::VersionMismatch => { 44 | // > The peer doesn't implement any supported version 45 | // unsupported version is likely a long time error, so this peer is not usable 46 | GetError::NoncompliantNode(e.into()) 47 | } 48 | e @ ConnectionError::TransportError(_) => { 49 | // > The peer violated the QUIC specification as understood by this implementation 50 | // bad peer we don't want to keep around 51 | GetError::NoncompliantNode(e.into()) 52 | } 53 | e @ ConnectionError::ConnectionClosed(_) => { 54 | // > The peer's QUIC stack aborted the connection automatically 55 | // peer might be disconnecting or otherwise unavailable, drop it 56 | GetError::Io(e.into()) 57 | } 58 | e @ ConnectionError::ApplicationClosed(_) => { 59 | // > The peer closed the connection 60 | // peer might be disconnecting or otherwise unavailable, drop it 61 | GetError::Io(e.into()) 62 | } 63 | e @ ConnectionError::Reset => { 64 | // > The peer is unable to continue processing this connection, usually due to having restarted 65 | GetError::RemoteReset(e.into()) 66 | } 67 | e @ ConnectionError::TimedOut => { 68 | // > Communication with the peer has lapsed for longer than the negotiated idle timeout 69 | GetError::Io(e.into()) 70 | } 71 | e @ ConnectionError::LocallyClosed => { 72 | // > The local application closed the connection 73 | // TODO(@divma): don't see how this is reachable but let's just not use the peer 74 | GetError::Io(e.into()) 75 | } 76 | e @ ConnectionError::CidsExhausted => { 77 | // > The connection could not be created because not enough of the CID space 78 | // > is available 79 | GetError::Io(e.into()) 80 | } 81 | } 82 | } 83 | } 84 | 85 | impl From for GetError { 86 | fn from(value: endpoint::ReadError) -> Self { 87 | use endpoint::ReadError; 88 | match value { 89 | e @ ReadError::Reset(_) => GetError::RemoteReset(e.into()), 90 | ReadError::ConnectionLost(conn_error) => conn_error.into(), 91 | ReadError::ClosedStream 92 | | ReadError::IllegalOrderedRead 93 | | ReadError::ZeroRttRejected => { 94 | // all these errors indicate the peer is not usable at this moment 95 | GetError::Io(value.into()) 96 | } 97 | } 98 | } 99 | } 100 | impl From for GetError { 101 | fn from(value: ClosedStream) -> Self { 102 | GetError::Io(value.into()) 103 | } 104 | } 105 | 106 | impl From for GetError { 107 | fn from(value: endpoint::WriteError) -> Self { 108 | use endpoint::WriteError; 109 | match value { 110 | e @ WriteError::Stopped(_) => GetError::RemoteReset(e.into()), 111 | WriteError::ConnectionLost(conn_error) => conn_error.into(), 112 | WriteError::ClosedStream | WriteError::ZeroRttRejected => { 113 | // all these errors indicate the peer is not usable at this moment 114 | GetError::Io(value.into()) 115 | } 116 | } 117 | } 118 | } 119 | 120 | impl From for GetError { 121 | fn from(value: crate::get::fsm::ConnectedNextError) -> Self { 122 | use crate::get::fsm::ConnectedNextError::*; 123 | match value { 124 | e @ PostcardSer(_) => { 125 | // serialization errors indicate something wrong with the request itself 126 | GetError::BadRequest(e.into()) 127 | } 128 | e @ RequestTooBig => { 129 | // request will never be sent, drop it 130 | GetError::BadRequest(e.into()) 131 | } 132 | Write(e) => e.into(), 133 | Closed(e) => e.into(), 134 | e @ Io(_) => { 135 | // io errors are likely recoverable 136 | GetError::Io(e.into()) 137 | } 138 | } 139 | } 140 | } 141 | 142 | impl From for GetError { 143 | fn from(value: crate::get::fsm::AtBlobHeaderNextError) -> Self { 144 | use crate::get::fsm::AtBlobHeaderNextError::*; 145 | match value { 146 | e @ NotFound => { 147 | // > This indicates that the provider does not have the requested data. 148 | // peer might have the data later, simply retry it 149 | GetError::NotFound(e.into()) 150 | } 151 | Read(e) => e.into(), 152 | e @ Io(_) => { 153 | // io errors are likely recoverable 154 | GetError::Io(e.into()) 155 | } 156 | } 157 | } 158 | } 159 | 160 | impl From for GetError { 161 | fn from(value: crate::get::fsm::DecodeError) -> Self { 162 | use crate::get::fsm::DecodeError::*; 163 | 164 | match value { 165 | e @ NotFound => GetError::NotFound(e.into()), 166 | e @ ParentNotFound(_) => GetError::NotFound(e.into()), 167 | e @ LeafNotFound(_) => GetError::NotFound(e.into()), 168 | e @ ParentHashMismatch(_) => { 169 | // TODO(@divma): did the peer sent wrong data? is it corrupted? did we sent a wrong 170 | // request? 171 | GetError::NoncompliantNode(e.into()) 172 | } 173 | e @ LeafHashMismatch(_) => { 174 | // TODO(@divma): did the peer sent wrong data? is it corrupted? did we sent a wrong 175 | // request? 176 | GetError::NoncompliantNode(e.into()) 177 | } 178 | Read(e) => e.into(), 179 | Io(e) => e.into(), 180 | } 181 | } 182 | } 183 | 184 | impl From for GetError { 185 | fn from(value: std::io::Error) -> Self { 186 | // generally consider io errors recoverable 187 | // we might want to revisit this at some point 188 | GetError::Io(value.into()) 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /src/get/progress.rs: -------------------------------------------------------------------------------- 1 | //! Types for get progress state management. 2 | 3 | use std::{collections::HashMap, num::NonZeroU64}; 4 | 5 | use serde::{Deserialize, Serialize}; 6 | use tracing::warn; 7 | 8 | use super::db::{BlobId, DownloadProgress}; 9 | use crate::{protocol::RangeSpec, store::BaoBlobSize, Hash}; 10 | 11 | /// The identifier for progress events. 12 | pub type ProgressId = u64; 13 | 14 | /// Accumulated progress state of a transfer. 15 | #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] 16 | pub struct TransferState { 17 | /// The root blob of this transfer (may be a hash seq), 18 | pub root: BlobState, 19 | /// Whether we are connected to a node 20 | pub connected: bool, 21 | /// Children if the root blob is a hash seq, empty for raw blobs 22 | pub children: HashMap, 23 | /// Child being transferred at the moment. 24 | pub current: Option, 25 | /// Progress ids for individual blobs. 26 | pub progress_id_to_blob: HashMap, 27 | } 28 | 29 | impl TransferState { 30 | /// Create a new, empty transfer state. 31 | pub fn new(root_hash: Hash) -> Self { 32 | Self { 33 | root: BlobState::new(root_hash), 34 | connected: false, 35 | children: Default::default(), 36 | current: None, 37 | progress_id_to_blob: Default::default(), 38 | } 39 | } 40 | } 41 | 42 | /// State of a single blob in transfer 43 | #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] 44 | pub struct BlobState { 45 | /// The hash of this blob. 46 | pub hash: Hash, 47 | /// The size of this blob. Only known if the blob is partially present locally, or after having 48 | /// received the size from the remote. 49 | pub size: Option, 50 | /// The current state of the blob transfer. 51 | pub progress: BlobProgress, 52 | /// Ranges already available locally at the time of starting the transfer. 53 | pub local_ranges: Option, 54 | /// Number of children (only applies to hashseqs, None for raw blobs). 55 | pub child_count: Option, 56 | } 57 | 58 | /// Progress state for a single blob 59 | #[derive(Debug, Default, Clone, Serialize, Deserialize, Eq, PartialEq)] 60 | pub enum BlobProgress { 61 | /// Download is pending 62 | #[default] 63 | Pending, 64 | /// Download is in progress 65 | Progressing(u64), 66 | /// Download has finished 67 | Done, 68 | } 69 | 70 | impl BlobState { 71 | /// Create a new [`BlobState`]. 72 | pub fn new(hash: Hash) -> Self { 73 | Self { 74 | hash, 75 | size: None, 76 | local_ranges: None, 77 | child_count: None, 78 | progress: BlobProgress::default(), 79 | } 80 | } 81 | } 82 | 83 | impl TransferState { 84 | /// Get state of the root blob of this transfer. 85 | pub fn root(&self) -> &BlobState { 86 | &self.root 87 | } 88 | 89 | /// Get a blob state by its [`BlobId`] in this transfer. 90 | pub fn get_blob(&self, blob_id: &BlobId) -> Option<&BlobState> { 91 | match blob_id { 92 | BlobId::Root => Some(&self.root), 93 | BlobId::Child(id) => self.children.get(id), 94 | } 95 | } 96 | 97 | /// Get the blob state currently being transferred. 98 | pub fn get_current(&self) -> Option<&BlobState> { 99 | self.current.as_ref().and_then(|id| self.get_blob(id)) 100 | } 101 | 102 | fn get_or_insert_blob(&mut self, blob_id: BlobId, hash: Hash) -> &mut BlobState { 103 | match blob_id { 104 | BlobId::Root => &mut self.root, 105 | BlobId::Child(id) => self 106 | .children 107 | .entry(id) 108 | .or_insert_with(|| BlobState::new(hash)), 109 | } 110 | } 111 | fn get_blob_mut(&mut self, blob_id: &BlobId) -> Option<&mut BlobState> { 112 | match blob_id { 113 | BlobId::Root => Some(&mut self.root), 114 | BlobId::Child(id) => self.children.get_mut(id), 115 | } 116 | } 117 | 118 | fn get_by_progress_id(&mut self, progress_id: ProgressId) -> Option<&mut BlobState> { 119 | let blob_id = *self.progress_id_to_blob.get(&progress_id)?; 120 | self.get_blob_mut(&blob_id) 121 | } 122 | 123 | /// Update the state with a new [`DownloadProgress`] event for this transfer. 124 | pub fn on_progress(&mut self, event: DownloadProgress) { 125 | match event { 126 | DownloadProgress::InitialState(s) => { 127 | *self = s; 128 | } 129 | DownloadProgress::FoundLocal { 130 | child, 131 | hash, 132 | size, 133 | valid_ranges, 134 | } => { 135 | let blob = self.get_or_insert_blob(child, hash); 136 | blob.size = Some(size); 137 | blob.local_ranges = Some(valid_ranges); 138 | } 139 | DownloadProgress::Connected => self.connected = true, 140 | DownloadProgress::Found { 141 | id: progress_id, 142 | child: blob_id, 143 | hash, 144 | size, 145 | } => { 146 | let blob = self.get_or_insert_blob(blob_id, hash); 147 | blob.size = match blob.size { 148 | // If we don't have a verified size for this blob yet: Use the size as reported 149 | // by the remote. 150 | None | Some(BaoBlobSize::Unverified(_)) => Some(BaoBlobSize::Unverified(size)), 151 | // Otherwise, keep the existing verified size. 152 | value @ Some(BaoBlobSize::Verified(_)) => value, 153 | }; 154 | blob.progress = BlobProgress::Progressing(0); 155 | self.progress_id_to_blob.insert(progress_id, blob_id); 156 | self.current = Some(blob_id); 157 | } 158 | DownloadProgress::FoundHashSeq { hash, children } => { 159 | if hash == self.root.hash { 160 | self.root.child_count = Some(children); 161 | } else { 162 | // I think it is an invariant of the protocol that `FoundHashSeq` is only 163 | // triggered for the root hash. 164 | warn!("Received `FoundHashSeq` event for a hash which is not the download's root hash.") 165 | } 166 | } 167 | DownloadProgress::Progress { id, offset } => { 168 | if let Some(blob) = self.get_by_progress_id(id) { 169 | blob.progress = BlobProgress::Progressing(offset); 170 | } else { 171 | warn!(%id, "Received `Progress` event for unknown progress id.") 172 | } 173 | } 174 | DownloadProgress::Done { id } => { 175 | if let Some(blob) = self.get_by_progress_id(id) { 176 | blob.progress = BlobProgress::Done; 177 | self.progress_id_to_blob.remove(&id); 178 | } else { 179 | warn!(%id, "Received `Done` event for unknown progress id.") 180 | } 181 | } 182 | DownloadProgress::AllDone(_) | DownloadProgress::Abort(_) => {} 183 | } 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /src/get/request.rs: -------------------------------------------------------------------------------- 1 | //! Utilities for complex get requests. 2 | use std::sync::Arc; 3 | 4 | use bao_tree::{ChunkNum, ChunkRanges}; 5 | use bytes::Bytes; 6 | use iroh::endpoint::Connection; 7 | use rand::Rng; 8 | 9 | use super::{fsm, Stats}; 10 | use crate::{ 11 | hashseq::HashSeq, 12 | protocol::{GetRequest, RangeSpecSeq}, 13 | Hash, HashAndFormat, 14 | }; 15 | 16 | /// Get the claimed size of a blob from a peer. 17 | /// 18 | /// This is just reading the size header and then immediately closing the connection. 19 | /// It can be used to check if a peer has any data at all. 20 | pub async fn get_unverified_size( 21 | connection: &Connection, 22 | hash: &Hash, 23 | ) -> anyhow::Result<(u64, Stats)> { 24 | let request = GetRequest::new( 25 | *hash, 26 | RangeSpecSeq::from_ranges(vec![ChunkRanges::from(ChunkNum(u64::MAX)..)]), 27 | ); 28 | let request = fsm::start(connection.clone(), request); 29 | let connected = request.next().await?; 30 | let fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { 31 | unreachable!("expected start root"); 32 | }; 33 | let at_blob_header = start.next(); 34 | let (curr, size) = at_blob_header.next().await?; 35 | let stats = curr.finish().next().await?; 36 | Ok((size, stats)) 37 | } 38 | 39 | /// Get the verified size of a blob from a peer. 40 | /// 41 | /// This asks for the last chunk of the blob and validates the response. 42 | /// Note that this does not validate that the peer has all the data. 43 | pub async fn get_verified_size( 44 | connection: &Connection, 45 | hash: &Hash, 46 | ) -> anyhow::Result<(u64, Stats)> { 47 | tracing::trace!("Getting verified size of {}", hash.to_hex()); 48 | let request = GetRequest::new( 49 | *hash, 50 | RangeSpecSeq::from_ranges(vec![ChunkRanges::from(ChunkNum(u64::MAX)..)]), 51 | ); 52 | let request = fsm::start(connection.clone(), request); 53 | let connected = request.next().await?; 54 | let fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { 55 | unreachable!("expected start root"); 56 | }; 57 | let header = start.next(); 58 | let (mut curr, size) = header.next().await?; 59 | let end = loop { 60 | match curr.next().await { 61 | fsm::BlobContentNext::More((next, res)) => { 62 | let _ = res?; 63 | curr = next; 64 | } 65 | fsm::BlobContentNext::Done(end) => { 66 | break end; 67 | } 68 | } 69 | }; 70 | let fsm::EndBlobNext::Closing(closing) = end.next() else { 71 | unreachable!("expected closing"); 72 | }; 73 | let stats = closing.next().await?; 74 | tracing::trace!( 75 | "Got verified size of {}, {:.6}s", 76 | hash.to_hex(), 77 | stats.elapsed.as_secs_f64() 78 | ); 79 | Ok((size, stats)) 80 | } 81 | 82 | /// Given a hash of a hash seq, get the hash seq and the verified sizes of its 83 | /// children. 84 | /// 85 | /// This can be used to compute the total size when requesting a hash seq. 86 | pub async fn get_hash_seq_and_sizes( 87 | connection: &Connection, 88 | hash: &Hash, 89 | max_size: u64, 90 | ) -> anyhow::Result<(HashSeq, Arc<[u64]>)> { 91 | let content = HashAndFormat::hash_seq(*hash); 92 | tracing::debug!("Getting hash seq and children sizes of {}", content); 93 | let request = GetRequest::new( 94 | *hash, 95 | RangeSpecSeq::from_ranges_infinite([ 96 | ChunkRanges::all(), 97 | ChunkRanges::from(ChunkNum(u64::MAX)..), 98 | ]), 99 | ); 100 | let at_start = fsm::start(connection.clone(), request); 101 | let at_connected = at_start.next().await?; 102 | let fsm::ConnectedNext::StartRoot(start) = at_connected.next().await? else { 103 | unreachable!("query includes root"); 104 | }; 105 | let at_start_root = start.next(); 106 | let (at_blob_content, size) = at_start_root.next().await?; 107 | // check the size to avoid parsing a maliciously large hash seq 108 | if size > max_size { 109 | anyhow::bail!("size too large"); 110 | } 111 | let (mut curr, hash_seq) = at_blob_content.concatenate_into_vec().await?; 112 | let hash_seq = HashSeq::try_from(Bytes::from(hash_seq))?; 113 | let mut sizes = Vec::with_capacity(hash_seq.len()); 114 | let closing = loop { 115 | match curr.next() { 116 | fsm::EndBlobNext::MoreChildren(more) => { 117 | let hash = match hash_seq.get(sizes.len()) { 118 | Some(hash) => hash, 119 | None => break more.finish(), 120 | }; 121 | let at_header = more.next(hash); 122 | let (at_content, size) = at_header.next().await?; 123 | let next = at_content.drain().await?; 124 | sizes.push(size); 125 | curr = next; 126 | } 127 | fsm::EndBlobNext::Closing(closing) => break closing, 128 | } 129 | }; 130 | let _stats = closing.next().await?; 131 | tracing::debug!( 132 | "Got hash seq and children sizes of {}: {:?}", 133 | content, 134 | sizes 135 | ); 136 | Ok((hash_seq, sizes.into())) 137 | } 138 | 139 | /// Probe for a single chunk of a blob. 140 | /// 141 | /// This is used to check if a peer has a specific chunk. 142 | pub async fn get_chunk_probe( 143 | connection: &Connection, 144 | hash: &Hash, 145 | chunk: ChunkNum, 146 | ) -> anyhow::Result { 147 | let ranges = ChunkRanges::from(chunk..chunk + 1); 148 | let ranges = RangeSpecSeq::from_ranges([ranges]); 149 | let request = GetRequest::new(*hash, ranges); 150 | let request = fsm::start(connection.clone(), request); 151 | let connected = request.next().await?; 152 | let fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { 153 | unreachable!("query includes root"); 154 | }; 155 | let header = start.next(); 156 | let (mut curr, _size) = header.next().await?; 157 | let end = loop { 158 | match curr.next().await { 159 | fsm::BlobContentNext::More((next, res)) => { 160 | res?; 161 | curr = next; 162 | } 163 | fsm::BlobContentNext::Done(end) => { 164 | break end; 165 | } 166 | } 167 | }; 168 | let fsm::EndBlobNext::Closing(closing) = end.next() else { 169 | unreachable!("query contains only one blob"); 170 | }; 171 | let stats = closing.next().await?; 172 | Ok(stats) 173 | } 174 | 175 | /// Given a sequence of sizes of children, generate a range spec that selects a 176 | /// random chunk of a random child. 177 | /// 178 | /// The random chunk is chosen uniformly from the chunks of the children, so 179 | /// larger children are more likely to be selected. 180 | pub fn random_hash_seq_ranges(sizes: &[u64], mut rng: impl Rng) -> RangeSpecSeq { 181 | let total_chunks = sizes 182 | .iter() 183 | .map(|size| ChunkNum::full_chunks(*size).0) 184 | .sum::(); 185 | let random_chunk = rng.gen_range(0..total_chunks); 186 | let mut remaining = random_chunk; 187 | let mut ranges = vec![]; 188 | ranges.push(ChunkRanges::empty()); 189 | for size in sizes.iter() { 190 | let chunks = ChunkNum::full_chunks(*size).0; 191 | if remaining < chunks { 192 | ranges.push(ChunkRanges::from( 193 | ChunkNum(remaining)..ChunkNum(remaining + 1), 194 | )); 195 | break; 196 | } else { 197 | remaining -= chunks; 198 | ranges.push(ChunkRanges::empty()); 199 | } 200 | } 201 | RangeSpecSeq::from_ranges(ranges) 202 | } 203 | -------------------------------------------------------------------------------- /src/hashseq.rs: -------------------------------------------------------------------------------- 1 | //! traits related to collections of blobs 2 | use std::{fmt::Debug, io}; 3 | 4 | use bytes::Bytes; 5 | use iroh_io::{AsyncSliceReader, AsyncSliceReaderExt}; 6 | 7 | use crate::Hash; 8 | 9 | /// A sequence of links, backed by a [`Bytes`] object. 10 | #[derive(Debug, Clone, derive_more::Into)] 11 | pub struct HashSeq(Bytes); 12 | 13 | impl FromIterator for HashSeq { 14 | fn from_iter>(iter: T) -> Self { 15 | let iter = iter.into_iter(); 16 | let (lower, _upper) = iter.size_hint(); 17 | let mut bytes = Vec::with_capacity(lower * 32); 18 | for hash in iter { 19 | bytes.extend_from_slice(hash.as_ref()); 20 | } 21 | Self(bytes.into()) 22 | } 23 | } 24 | 25 | impl TryFrom for HashSeq { 26 | type Error = anyhow::Error; 27 | 28 | fn try_from(bytes: Bytes) -> Result { 29 | Self::new(bytes).ok_or_else(|| anyhow::anyhow!("invalid hash sequence")) 30 | } 31 | } 32 | 33 | impl IntoIterator for HashSeq { 34 | type Item = Hash; 35 | type IntoIter = HashSeqIter; 36 | 37 | fn into_iter(self) -> Self::IntoIter { 38 | HashSeqIter(self) 39 | } 40 | } 41 | 42 | /// Stream over the hashes in a [`HashSeq`]. 43 | /// 44 | /// todo: make this wrap a reader instead of a [`HashSeq`]. 45 | #[derive(Debug, Clone)] 46 | pub struct HashSeqStream(HashSeq); 47 | 48 | impl HashSeqStream { 49 | /// Get the next hash in the sequence. 50 | #[allow(clippy::should_implement_trait, clippy::unused_async)] 51 | pub async fn next(&mut self) -> io::Result> { 52 | Ok(self.0.pop_front()) 53 | } 54 | 55 | /// Skip a number of hashes in the sequence. 56 | #[allow(clippy::unused_async)] 57 | pub async fn skip(&mut self, n: u64) -> io::Result<()> { 58 | let ok = self.0.drop_front(n as usize); 59 | if !ok { 60 | Err(io::Error::new( 61 | io::ErrorKind::UnexpectedEof, 62 | "end of sequence", 63 | )) 64 | } else { 65 | Ok(()) 66 | } 67 | } 68 | } 69 | 70 | impl HashSeq { 71 | /// Create a new sequence of hashes. 72 | pub fn new(bytes: Bytes) -> Option { 73 | if bytes.len() % 32 == 0 { 74 | Some(Self(bytes)) 75 | } else { 76 | None 77 | } 78 | } 79 | 80 | fn drop_front(&mut self, n: usize) -> bool { 81 | let start = n * 32; 82 | if start > self.0.len() { 83 | false 84 | } else { 85 | self.0 = self.0.slice(start..); 86 | true 87 | } 88 | } 89 | 90 | /// Iterate over the hashes in this sequence. 91 | pub fn iter(&self) -> impl Iterator + '_ { 92 | self.0.chunks_exact(32).map(|chunk| { 93 | let hash: [u8; 32] = chunk.try_into().unwrap(); 94 | hash.into() 95 | }) 96 | } 97 | 98 | /// Get the number of hashes in this sequence. 99 | pub fn len(&self) -> usize { 100 | self.0.len() / 32 101 | } 102 | 103 | /// Check if this sequence is empty. 104 | pub fn is_empty(&self) -> bool { 105 | self.0.is_empty() 106 | } 107 | 108 | /// Get the hash at the given index. 109 | pub fn get(&self, index: usize) -> Option { 110 | if index < self.len() { 111 | let hash: [u8; 32] = self.0[index * 32..(index + 1) * 32].try_into().unwrap(); 112 | Some(hash.into()) 113 | } else { 114 | None 115 | } 116 | } 117 | 118 | /// Get and remove the first hash in this sequence. 119 | pub fn pop_front(&mut self) -> Option { 120 | if self.is_empty() { 121 | None 122 | } else { 123 | let hash = self.get(0).unwrap(); 124 | self.0 = self.0.slice(32..); 125 | Some(hash) 126 | } 127 | } 128 | 129 | /// Get the underlying bytes. 130 | pub fn into_inner(self) -> Bytes { 131 | self.0 132 | } 133 | } 134 | 135 | /// Iterator over the hashes in a [`HashSeq`]. 136 | #[derive(Debug, Clone)] 137 | pub struct HashSeqIter(HashSeq); 138 | 139 | impl Iterator for HashSeqIter { 140 | type Item = Hash; 141 | 142 | fn next(&mut self) -> Option { 143 | self.0.pop_front() 144 | } 145 | } 146 | 147 | /// Parse a sequence of hashes. 148 | pub async fn parse_hash_seq<'a, R: AsyncSliceReader + 'a>( 149 | mut reader: R, 150 | ) -> anyhow::Result<(HashSeqStream, u64)> { 151 | let bytes = reader.read_to_end().await?; 152 | let hashes = HashSeq::try_from(bytes)?; 153 | let num_hashes = hashes.len() as u64; 154 | let stream = HashSeqStream(hashes); 155 | Ok((stream, num_hashes)) 156 | } 157 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("../README.md")] 2 | //! Blobs layer for iroh. 3 | //! 4 | //! The crate is designed to be used from the [iroh] crate, which provides a 5 | //! [high level interface](https://docs.rs/iroh/latest/iroh/client/blobs/index.html), 6 | //! but can also be used standalone. 7 | //! 8 | //! It implements a [protocol] for streaming content-addressed data transfer using 9 | //! [BLAKE3] verified streaming. 10 | //! 11 | //! It also provides a [store] interface for storage of blobs and outboards, 12 | //! as well as a [persistent](crate::store::fs) and a [memory](crate::store::mem) 13 | //! store implementation. 14 | //! 15 | //! To implement a server, the [provider] module provides helpers for handling 16 | //! connections and individual requests given a store. 17 | //! 18 | //! To perform get requests, the [get] module provides utilities to perform 19 | //! requests and store the result in a store, as well as a low level state 20 | //! machine for executing requests. 21 | //! 22 | //! The [downloader] module provides a component to download blobs from 23 | //! multiple sources and store them in a store. 24 | //! 25 | //! # Feature flags 26 | //! 27 | //! - rpc: Enable the rpc server and client. Enabled by default. 28 | //! - net_protocol: Enable the network protocol. Enabled by default. 29 | //! - downloader: Enable the downloader. Enabled by default. 30 | //! - fs-store: Enable the filesystem store. Enabled by default. 31 | //! 32 | //! - cli: Enable the cli. Disabled by default. 33 | //! - example-iroh: dependencies for examples in this crate. Disabled by default. 34 | //! - test: test utilities. Disabled by default. 35 | //! 36 | //! [BLAKE3]: https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf 37 | //! [iroh]: https://docs.rs/iroh 38 | #![deny(missing_docs, rustdoc::broken_intra_doc_links)] 39 | #![recursion_limit = "256"] 40 | #![cfg_attr(iroh_docsrs, feature(doc_auto_cfg))] 41 | 42 | #[cfg(feature = "cli")] 43 | pub mod cli; 44 | #[cfg(feature = "downloader")] 45 | pub mod downloader; 46 | pub mod export; 47 | pub mod format; 48 | pub mod get; 49 | pub mod hashseq; 50 | pub mod metrics; 51 | #[cfg(feature = "net_protocol")] 52 | pub mod net_protocol; 53 | pub mod protocol; 54 | pub mod provider; 55 | #[cfg(feature = "rpc")] 56 | pub mod rpc; 57 | pub mod store; 58 | pub mod ticket; 59 | pub mod util; 60 | 61 | mod hash; 62 | 63 | use bao_tree::BlockSize; 64 | 65 | #[doc(inline)] 66 | pub use crate::protocol::ALPN; 67 | pub use crate::{ 68 | hash::{BlobFormat, Hash, HashAndFormat}, 69 | util::{Tag, TempTag}, 70 | }; 71 | 72 | /// Block size used by iroh, 2^4*1024 = 16KiB 73 | pub const IROH_BLOCK_SIZE: BlockSize = BlockSize::from_chunk_log(4); 74 | -------------------------------------------------------------------------------- /src/metrics.rs: -------------------------------------------------------------------------------- 1 | //! Metrics for iroh-blobs 2 | 3 | use iroh_metrics::{Counter, MetricsGroup}; 4 | 5 | /// Enum of metrics for the module 6 | #[derive(Debug, MetricsGroup, Default)] 7 | #[metrics(name = "iroh-blobs")] 8 | pub struct Metrics { 9 | /// Total number of content bytes downloaded 10 | pub download_bytes_total: Counter, 11 | /// Total time in ms spent downloading content bytes 12 | pub download_time_total: Counter, 13 | /// Total number of successful downloads 14 | pub downloads_success: Counter, 15 | /// Total number of downloads failed with error 16 | pub downloads_error: Counter, 17 | /// Total number of downloads failed with not found 18 | pub downloads_notfound: Counter, 19 | 20 | /// Number of times the main pub downloader actor loop ticked 21 | pub downloader_tick_main: Counter, 22 | 23 | /// Number of times the pub downloader actor ticked for a connection ready 24 | pub downloader_tick_connection_ready: Counter, 25 | 26 | /// Number of times the pub downloader actor ticked for a message received 27 | pub downloader_tick_message_received: Counter, 28 | 29 | /// Number of times the pub downloader actor ticked for a transfer completed 30 | pub downloader_tick_transfer_completed: Counter, 31 | 32 | /// Number of times the pub downloader actor ticked for a transfer failed 33 | pub downloader_tick_transfer_failed: Counter, 34 | 35 | /// Number of times the pub downloader actor ticked for a retry node 36 | pub downloader_tick_retry_node: Counter, 37 | 38 | /// Number of times the pub downloader actor ticked for a goodbye node 39 | pub downloader_tick_goodbye_node: Counter, 40 | } 41 | -------------------------------------------------------------------------------- /src/rpc/client.rs: -------------------------------------------------------------------------------- 1 | //! Iroh blobs and tags client 2 | use anyhow::Result; 3 | use futures_util::{Stream, StreamExt}; 4 | use quic_rpc::transport::flume::FlumeConnector; 5 | 6 | pub mod blobs; 7 | pub mod tags; 8 | 9 | /// Type alias for a memory-backed client. 10 | pub(crate) type MemConnector = 11 | FlumeConnector; 12 | 13 | fn flatten( 14 | s: impl Stream, E2>>, 15 | ) -> impl Stream> 16 | where 17 | E1: std::error::Error + Send + Sync + 'static, 18 | E2: std::error::Error + Send + Sync + 'static, 19 | { 20 | s.map(|res| match res { 21 | Ok(Ok(res)) => Ok(res), 22 | Ok(Err(err)) => Err(err.into()), 23 | Err(err) => Err(err.into()), 24 | }) 25 | } 26 | -------------------------------------------------------------------------------- /src/rpc/client/tags.rs: -------------------------------------------------------------------------------- 1 | //! API for tag management. 2 | //! 3 | //! The purpose of tags is to mark information as important to prevent it 4 | //! from being garbage-collected (if the garbage collector is turned on). 5 | //! 6 | //! A tag has a name that is an arbitrary byte string. In many cases this will be 7 | //! a valid UTF8 string, but there are also use cases where it is useful to have 8 | //! non string data like integer ids in the tag name. 9 | //! 10 | //! Tags point to a [`HashAndFormat`]. 11 | //! 12 | //! A tag can point to a hash with format [`BlobFormat::Raw`]. In that case it will 13 | //! protect *just this blob* from being garbage-collected. 14 | //! 15 | //! It can also point to a hash in format [`BlobFormat::HashSeq`]. In that case it will 16 | //! protect the blob itself and all hashes in the blob (the blob must be just a sequence of hashes). 17 | //! Using this format it is possible to protect a large number of blobs with a single tag. 18 | //! 19 | //! Tags can be created, read, renamed and deleted. Tags *do not* have to correspond to 20 | //! already existing data. It is perfectly valid to create a tag for data you don't have yet. 21 | //! 22 | //! The main entry point is the [`Client`]. 23 | use std::ops::{Bound, RangeBounds}; 24 | 25 | use anyhow::Result; 26 | use futures_lite::{Stream, StreamExt}; 27 | use quic_rpc::{client::BoxedConnector, Connector, RpcClient}; 28 | use serde::{Deserialize, Serialize}; 29 | 30 | use crate::{ 31 | rpc::proto::{ 32 | tags::{DeleteRequest, ListRequest, RenameRequest, SetRequest, SyncMode}, 33 | RpcService, 34 | }, 35 | BlobFormat, Hash, HashAndFormat, Tag, 36 | }; 37 | 38 | /// Iroh tags client. 39 | #[derive(Debug, Clone)] 40 | #[repr(transparent)] 41 | pub struct Client> { 42 | pub(super) rpc: RpcClient, 43 | } 44 | 45 | /// Options for a list operation. 46 | #[derive(Debug, Clone)] 47 | pub struct ListOptions { 48 | /// List tags to hash seqs 49 | pub hash_seq: bool, 50 | /// List tags to raw blobs 51 | pub raw: bool, 52 | /// Optional from tag (inclusive) 53 | pub from: Option, 54 | /// Optional to tag (exclusive) 55 | pub to: Option, 56 | } 57 | 58 | fn tags_from_range(range: R) -> (Option, Option) 59 | where 60 | R: RangeBounds, 61 | E: AsRef<[u8]>, 62 | { 63 | let from = match range.start_bound() { 64 | Bound::Included(start) => Some(Tag::from(start.as_ref())), 65 | Bound::Excluded(start) => Some(Tag::from(start.as_ref()).successor()), 66 | Bound::Unbounded => None, 67 | }; 68 | let to = match range.end_bound() { 69 | Bound::Included(end) => Some(Tag::from(end.as_ref()).successor()), 70 | Bound::Excluded(end) => Some(Tag::from(end.as_ref())), 71 | Bound::Unbounded => None, 72 | }; 73 | (from, to) 74 | } 75 | 76 | impl ListOptions { 77 | /// List a range of tags 78 | pub fn range(range: R) -> Self 79 | where 80 | R: RangeBounds, 81 | E: AsRef<[u8]>, 82 | { 83 | let (from, to) = tags_from_range(range); 84 | Self { 85 | from, 86 | to, 87 | raw: true, 88 | hash_seq: true, 89 | } 90 | } 91 | 92 | /// List tags with a prefix 93 | pub fn prefix(prefix: &[u8]) -> Self { 94 | let from = Tag::from(prefix); 95 | let to = from.next_prefix(); 96 | Self { 97 | raw: true, 98 | hash_seq: true, 99 | from: Some(from), 100 | to, 101 | } 102 | } 103 | 104 | /// List a single tag 105 | pub fn single(name: &[u8]) -> Self { 106 | let from = Tag::from(name); 107 | Self { 108 | to: Some(from.successor()), 109 | from: Some(from), 110 | raw: true, 111 | hash_seq: true, 112 | } 113 | } 114 | 115 | /// List all tags 116 | pub fn all() -> Self { 117 | Self { 118 | raw: true, 119 | hash_seq: true, 120 | from: None, 121 | to: None, 122 | } 123 | } 124 | 125 | /// List raw tags 126 | pub fn raw() -> Self { 127 | Self { 128 | raw: true, 129 | hash_seq: false, 130 | from: None, 131 | to: None, 132 | } 133 | } 134 | 135 | /// List hash seq tags 136 | pub fn hash_seq() -> Self { 137 | Self { 138 | raw: false, 139 | hash_seq: true, 140 | from: None, 141 | to: None, 142 | } 143 | } 144 | } 145 | 146 | /// Options for a delete operation. 147 | #[derive(Debug, Clone)] 148 | pub struct DeleteOptions { 149 | /// Optional from tag (inclusive) 150 | pub from: Option, 151 | /// Optional to tag (exclusive) 152 | pub to: Option, 153 | } 154 | 155 | impl DeleteOptions { 156 | /// Delete a single tag 157 | pub fn single(name: &[u8]) -> Self { 158 | let name = Tag::from(name); 159 | Self { 160 | to: Some(name.successor()), 161 | from: Some(name), 162 | } 163 | } 164 | 165 | /// Delete a range of tags 166 | pub fn range(range: R) -> Self 167 | where 168 | R: RangeBounds, 169 | E: AsRef<[u8]>, 170 | { 171 | let (from, to) = tags_from_range(range); 172 | Self { from, to } 173 | } 174 | 175 | /// Delete tags with a prefix 176 | pub fn prefix(prefix: &[u8]) -> Self { 177 | let from = Tag::from(prefix); 178 | let to = from.next_prefix(); 179 | Self { 180 | from: Some(from), 181 | to, 182 | } 183 | } 184 | } 185 | 186 | /// A client that uses the memory connector. 187 | pub type MemClient = Client; 188 | 189 | impl Client 190 | where 191 | C: Connector, 192 | { 193 | /// Creates a new client 194 | pub fn new(rpc: RpcClient) -> Self { 195 | Self { rpc } 196 | } 197 | 198 | /// List all tags with options. 199 | /// 200 | /// This is the most flexible way to list tags. All the other list methods are just convenience 201 | /// methods that call this one with the appropriate options. 202 | pub async fn list_with_opts( 203 | &self, 204 | options: ListOptions, 205 | ) -> Result>> { 206 | let stream = self 207 | .rpc 208 | .server_streaming(ListRequest::from(options)) 209 | .await?; 210 | Ok(stream.map(|res| res.map_err(anyhow::Error::from))) 211 | } 212 | 213 | /// Set the value for a single tag 214 | pub async fn set(&self, name: impl AsRef<[u8]>, value: impl Into) -> Result<()> { 215 | self.rpc 216 | .rpc(SetRequest { 217 | name: Tag::from(name.as_ref()), 218 | value: value.into(), 219 | batch: None, 220 | sync: SyncMode::Full, 221 | }) 222 | .await??; 223 | Ok(()) 224 | } 225 | 226 | /// Get the value of a single tag 227 | pub async fn get(&self, name: impl AsRef<[u8]>) -> Result> { 228 | let mut stream = self 229 | .list_with_opts(ListOptions::single(name.as_ref())) 230 | .await?; 231 | stream.next().await.transpose() 232 | } 233 | 234 | /// Rename a tag atomically 235 | /// 236 | /// If the tag does not exist, this will return an error. 237 | pub async fn rename(&self, from: impl AsRef<[u8]>, to: impl AsRef<[u8]>) -> Result<()> { 238 | self.rpc 239 | .rpc(RenameRequest { 240 | from: Tag::from(from.as_ref()), 241 | to: Tag::from(to.as_ref()), 242 | }) 243 | .await??; 244 | Ok(()) 245 | } 246 | 247 | /// List a range of tags 248 | pub async fn list_range(&self, range: R) -> Result>> 249 | where 250 | R: RangeBounds, 251 | E: AsRef<[u8]>, 252 | { 253 | self.list_with_opts(ListOptions::range(range)).await 254 | } 255 | 256 | /// Lists all tags with the given prefix. 257 | pub async fn list_prefix( 258 | &self, 259 | prefix: impl AsRef<[u8]>, 260 | ) -> Result>> { 261 | self.list_with_opts(ListOptions::prefix(prefix.as_ref())) 262 | .await 263 | } 264 | 265 | /// Lists all tags. 266 | pub async fn list(&self) -> Result>> { 267 | self.list_with_opts(ListOptions::all()).await 268 | } 269 | 270 | /// Lists all tags with a hash_seq format. 271 | pub async fn list_hash_seq(&self) -> Result>> { 272 | self.list_with_opts(ListOptions::hash_seq()).await 273 | } 274 | 275 | /// Deletes a tag. 276 | pub async fn delete_with_opts(&self, options: DeleteOptions) -> Result<()> { 277 | self.rpc.rpc(DeleteRequest::from(options)).await??; 278 | Ok(()) 279 | } 280 | 281 | /// Deletes a tag. 282 | pub async fn delete(&self, name: impl AsRef<[u8]>) -> Result<()> { 283 | self.delete_with_opts(DeleteOptions::single(name.as_ref())) 284 | .await 285 | } 286 | 287 | /// Deletes a range of tags. 288 | pub async fn delete_range(&self, range: R) -> Result<()> 289 | where 290 | R: RangeBounds, 291 | E: AsRef<[u8]>, 292 | { 293 | self.delete_with_opts(DeleteOptions::range(range)).await 294 | } 295 | 296 | /// Delete all tags with the given prefix. 297 | pub async fn delete_prefix(&self, prefix: impl AsRef<[u8]>) -> Result<()> { 298 | self.delete_with_opts(DeleteOptions::prefix(prefix.as_ref())) 299 | .await 300 | } 301 | 302 | /// Delete all tags. Use with care. After this, all data will be garbage collected. 303 | pub async fn delete_all(&self) -> Result<()> { 304 | self.delete_with_opts(DeleteOptions { 305 | from: None, 306 | to: None, 307 | }) 308 | .await 309 | } 310 | } 311 | 312 | /// Information about a tag. 313 | #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] 314 | pub struct TagInfo { 315 | /// Name of the tag 316 | pub name: Tag, 317 | /// Format of the data 318 | pub format: BlobFormat, 319 | /// Hash of the data 320 | pub hash: Hash, 321 | } 322 | 323 | impl TagInfo { 324 | /// Create a new tag info. 325 | pub fn new(name: impl AsRef<[u8]>, value: impl Into) -> Self { 326 | let name = name.as_ref(); 327 | let value = value.into(); 328 | Self { 329 | name: Tag::from(name), 330 | hash: value.hash, 331 | format: value.format, 332 | } 333 | } 334 | 335 | /// Get the hash and format of the tag. 336 | pub fn hash_and_format(&self) -> HashAndFormat { 337 | HashAndFormat { 338 | hash: self.hash, 339 | format: self.format, 340 | } 341 | } 342 | } 343 | -------------------------------------------------------------------------------- /src/rpc/proto.rs: -------------------------------------------------------------------------------- 1 | //! RPC protocol for the iroh-blobs service 2 | use nested_enum_utils::enum_conversions; 3 | use serde::{Deserialize, Serialize}; 4 | 5 | pub mod blobs; 6 | pub mod tags; 7 | 8 | /// quic-rpc service for iroh blobs 9 | #[derive(Debug, Clone)] 10 | pub struct RpcService; 11 | 12 | impl quic_rpc::Service for RpcService { 13 | type Req = Request; 14 | type Res = Response; 15 | } 16 | 17 | #[allow(missing_docs)] 18 | #[enum_conversions] 19 | #[derive(Debug, Serialize, Deserialize)] 20 | pub enum Request { 21 | Blobs(blobs::Request), 22 | Tags(tags::Request), 23 | } 24 | 25 | #[allow(missing_docs)] 26 | #[enum_conversions] 27 | #[derive(Debug, Serialize, Deserialize)] 28 | pub enum Response { 29 | Blobs(blobs::Response), 30 | Tags(tags::Response), 31 | } 32 | 33 | /// Error type for RPC operations 34 | pub type RpcError = serde_error::Error; 35 | /// Result type for RPC operations 36 | pub type RpcResult = Result; 37 | -------------------------------------------------------------------------------- /src/rpc/proto/tags.rs: -------------------------------------------------------------------------------- 1 | //! Tags RPC protocol 2 | use nested_enum_utils::enum_conversions; 3 | use quic_rpc_derive::rpc_requests; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | use super::{RpcResult, RpcService}; 7 | use crate::{ 8 | net_protocol::BatchId, 9 | rpc::client::tags::{DeleteOptions, ListOptions, TagInfo}, 10 | HashAndFormat, Tag, 11 | }; 12 | 13 | #[allow(missing_docs)] 14 | #[derive(strum::Display, Debug, Serialize, Deserialize)] 15 | #[enum_conversions(super::Request)] 16 | #[rpc_requests(RpcService)] 17 | pub enum Request { 18 | #[rpc(response = RpcResult)] 19 | Create(CreateRequest), 20 | #[rpc(response = RpcResult<()>)] 21 | Set(SetRequest), 22 | #[rpc(response = RpcResult<()>)] 23 | Rename(RenameRequest), 24 | #[rpc(response = RpcResult<()>)] 25 | DeleteTag(DeleteRequest), 26 | #[server_streaming(response = TagInfo)] 27 | ListTags(ListRequest), 28 | } 29 | 30 | #[allow(missing_docs)] 31 | #[derive(strum::Display, Debug, Serialize, Deserialize)] 32 | #[enum_conversions(super::Response)] 33 | pub enum Response { 34 | Create(RpcResult), 35 | ListTags(TagInfo), 36 | DeleteTag(RpcResult<()>), 37 | } 38 | 39 | /// Determine how to sync the db after a modification operation 40 | #[derive(Debug, Serialize, Deserialize, Default)] 41 | pub enum SyncMode { 42 | /// Fully sync the db 43 | #[default] 44 | Full, 45 | /// Do not sync the db 46 | None, 47 | } 48 | 49 | /// Create a tag 50 | #[derive(Debug, Serialize, Deserialize)] 51 | pub struct CreateRequest { 52 | /// Value of the tag 53 | pub value: HashAndFormat, 54 | /// Batch to use, none for global 55 | pub batch: Option, 56 | /// Sync mode 57 | pub sync: SyncMode, 58 | } 59 | 60 | /// Set or delete a tag 61 | #[derive(Debug, Serialize, Deserialize)] 62 | pub struct SetRequest { 63 | /// Name of the tag 64 | pub name: Tag, 65 | /// Value of the tag 66 | pub value: HashAndFormat, 67 | /// Batch to use, none for global 68 | pub batch: Option, 69 | /// Sync mode 70 | pub sync: SyncMode, 71 | } 72 | 73 | /// List all collections 74 | /// 75 | /// Lists all collections that have been explicitly added to the database. 76 | #[derive(Debug, Serialize, Deserialize)] 77 | pub struct ListRequest { 78 | /// List raw tags 79 | pub raw: bool, 80 | /// List hash seq tags 81 | pub hash_seq: bool, 82 | /// From tag (inclusive) 83 | pub from: Option, 84 | /// To tag (exclusive) 85 | pub to: Option, 86 | } 87 | 88 | impl From for ListRequest { 89 | fn from(options: ListOptions) -> Self { 90 | Self { 91 | raw: options.raw, 92 | hash_seq: options.hash_seq, 93 | from: options.from, 94 | to: options.to, 95 | } 96 | } 97 | } 98 | 99 | /// Delete a tag 100 | #[derive(Debug, Serialize, Deserialize)] 101 | pub struct DeleteRequest { 102 | /// From tag (inclusive) 103 | pub from: Option, 104 | /// To tag (exclusive) 105 | pub to: Option, 106 | } 107 | 108 | impl From for DeleteRequest { 109 | fn from(options: DeleteOptions) -> Self { 110 | Self { 111 | from: options.from, 112 | to: options.to, 113 | } 114 | } 115 | } 116 | 117 | /// Rename a tag atomically 118 | #[derive(Debug, Serialize, Deserialize)] 119 | pub struct RenameRequest { 120 | /// Old tag name 121 | pub from: Tag, 122 | /// New tag name 123 | pub to: Tag, 124 | } 125 | -------------------------------------------------------------------------------- /src/store.rs: -------------------------------------------------------------------------------- 1 | //! Implementations of blob stores 2 | use crate::{BlobFormat, Hash, HashAndFormat}; 3 | 4 | #[cfg(feature = "fs-store")] 5 | mod bao_file; 6 | pub mod mem; 7 | mod mutable_mem_storage; 8 | pub mod readonly_mem; 9 | 10 | #[cfg(feature = "fs-store")] 11 | pub mod fs; 12 | 13 | mod traits; 14 | use tracing::warn; 15 | pub use traits::*; 16 | 17 | /// Create a 16 byte unique ID. 18 | fn new_uuid() -> [u8; 16] { 19 | use rand::Rng; 20 | rand::thread_rng().gen::<[u8; 16]>() 21 | } 22 | 23 | /// Create temp file name based on a 16 byte UUID. 24 | fn temp_name() -> String { 25 | format!("{}.temp", hex::encode(new_uuid())) 26 | } 27 | 28 | #[derive(Debug, Default, Clone)] 29 | struct TempCounters { 30 | /// number of raw temp tags for a hash 31 | raw: u64, 32 | /// number of hash seq temp tags for a hash 33 | hash_seq: u64, 34 | } 35 | 36 | impl TempCounters { 37 | fn counter(&mut self, format: BlobFormat) -> &mut u64 { 38 | match format { 39 | BlobFormat::Raw => &mut self.raw, 40 | BlobFormat::HashSeq => &mut self.hash_seq, 41 | } 42 | } 43 | 44 | fn inc(&mut self, format: BlobFormat) { 45 | let counter = self.counter(format); 46 | *counter = counter.checked_add(1).unwrap(); 47 | } 48 | 49 | fn dec(&mut self, format: BlobFormat) { 50 | let counter = self.counter(format); 51 | *counter = counter.saturating_sub(1); 52 | } 53 | 54 | fn is_empty(&self) -> bool { 55 | self.raw == 0 && self.hash_seq == 0 56 | } 57 | } 58 | 59 | #[derive(Debug, Clone, Default)] 60 | struct TempCounterMap(std::collections::BTreeMap); 61 | 62 | impl TempCounterMap { 63 | fn inc(&mut self, value: &HashAndFormat) { 64 | let HashAndFormat { hash, format } = value; 65 | self.0.entry(*hash).or_default().inc(*format) 66 | } 67 | 68 | fn dec(&mut self, value: &HashAndFormat) { 69 | let HashAndFormat { hash, format } = value; 70 | let Some(counters) = self.0.get_mut(hash) else { 71 | warn!("Decrementing non-existent temp tag"); 72 | return; 73 | }; 74 | counters.dec(*format); 75 | if counters.is_empty() { 76 | self.0.remove(hash); 77 | } 78 | } 79 | 80 | fn contains(&self, hash: &Hash) -> bool { 81 | self.0.contains_key(hash) 82 | } 83 | 84 | fn keys(&self) -> impl Iterator { 85 | let mut res = Vec::new(); 86 | for (k, v) in self.0.iter() { 87 | if v.raw > 0 { 88 | res.push(HashAndFormat::raw(*k)); 89 | } 90 | if v.hash_seq > 0 { 91 | res.push(HashAndFormat::hash_seq(*k)); 92 | } 93 | } 94 | res.into_iter() 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/store/fs/tables.rs: -------------------------------------------------------------------------------- 1 | //! Table definitions and accessors for the redb database. 2 | use std::collections::BTreeSet; 3 | 4 | use redb::{ReadableTable, TableDefinition, TableError}; 5 | 6 | use super::{EntryState, PathOptions}; 7 | use crate::{util::Tag, Hash, HashAndFormat}; 8 | 9 | pub(super) const BLOBS_TABLE: TableDefinition = TableDefinition::new("blobs-0"); 10 | 11 | pub(super) const TAGS_TABLE: TableDefinition = TableDefinition::new("tags-0"); 12 | 13 | pub(super) const INLINE_DATA_TABLE: TableDefinition = 14 | TableDefinition::new("inline-data-0"); 15 | 16 | pub(super) const INLINE_OUTBOARD_TABLE: TableDefinition = 17 | TableDefinition::new("inline-outboard-0"); 18 | 19 | /// A trait similar to [`redb::ReadableTable`] but for all tables that make up 20 | /// the blob store. This can be used in places where either a readonly or 21 | /// mutable table is needed. 22 | pub(super) trait ReadableTables { 23 | fn blobs(&self) -> &impl ReadableTable; 24 | fn tags(&self) -> &impl ReadableTable; 25 | fn inline_data(&self) -> &impl ReadableTable; 26 | fn inline_outboard(&self) -> &impl ReadableTable; 27 | } 28 | 29 | /// A struct similar to [`redb::Table`] but for all tables that make up the 30 | /// blob store. 31 | pub(super) struct Tables<'a> { 32 | pub blobs: redb::Table<'a, Hash, EntryState>, 33 | pub tags: redb::Table<'a, Tag, HashAndFormat>, 34 | pub inline_data: redb::Table<'a, Hash, &'static [u8]>, 35 | pub inline_outboard: redb::Table<'a, Hash, &'static [u8]>, 36 | pub delete_after_commit: &'a mut DeleteSet, 37 | } 38 | 39 | #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] 40 | pub(super) enum BaoFilePart { 41 | Outboard, 42 | Data, 43 | Sizes, 44 | } 45 | 46 | impl<'txn> Tables<'txn> { 47 | pub fn new( 48 | tx: &'txn redb::WriteTransaction, 49 | delete_after_commit: &'txn mut DeleteSet, 50 | ) -> std::result::Result { 51 | Ok(Self { 52 | blobs: tx.open_table(BLOBS_TABLE)?, 53 | tags: tx.open_table(TAGS_TABLE)?, 54 | inline_data: tx.open_table(INLINE_DATA_TABLE)?, 55 | inline_outboard: tx.open_table(INLINE_OUTBOARD_TABLE)?, 56 | delete_after_commit, 57 | }) 58 | } 59 | } 60 | 61 | impl ReadableTables for Tables<'_> { 62 | fn blobs(&self) -> &impl ReadableTable { 63 | &self.blobs 64 | } 65 | fn tags(&self) -> &impl ReadableTable { 66 | &self.tags 67 | } 68 | fn inline_data(&self) -> &impl ReadableTable { 69 | &self.inline_data 70 | } 71 | fn inline_outboard(&self) -> &impl ReadableTable { 72 | &self.inline_outboard 73 | } 74 | } 75 | 76 | /// A struct similar to [`redb::ReadOnlyTable`] but for all tables that make up 77 | /// the blob store. 78 | pub(super) struct ReadOnlyTables { 79 | pub blobs: redb::ReadOnlyTable, 80 | pub tags: redb::ReadOnlyTable, 81 | pub inline_data: redb::ReadOnlyTable, 82 | pub inline_outboard: redb::ReadOnlyTable, 83 | } 84 | 85 | impl ReadOnlyTables { 86 | pub fn new(tx: &redb::ReadTransaction) -> std::result::Result { 87 | Ok(Self { 88 | blobs: tx.open_table(BLOBS_TABLE)?, 89 | tags: tx.open_table(TAGS_TABLE)?, 90 | inline_data: tx.open_table(INLINE_DATA_TABLE)?, 91 | inline_outboard: tx.open_table(INLINE_OUTBOARD_TABLE)?, 92 | }) 93 | } 94 | } 95 | 96 | impl ReadableTables for ReadOnlyTables { 97 | fn blobs(&self) -> &impl ReadableTable { 98 | &self.blobs 99 | } 100 | fn tags(&self) -> &impl ReadableTable { 101 | &self.tags 102 | } 103 | fn inline_data(&self) -> &impl ReadableTable { 104 | &self.inline_data 105 | } 106 | fn inline_outboard(&self) -> &impl ReadableTable { 107 | &self.inline_outboard 108 | } 109 | } 110 | 111 | /// Helper to keep track of files to delete after a transaction is committed. 112 | #[derive(Debug, Default)] 113 | pub(super) struct DeleteSet(BTreeSet<(Hash, BaoFilePart)>); 114 | 115 | impl DeleteSet { 116 | /// Mark a file as to be deleted after the transaction is committed. 117 | pub fn insert(&mut self, hash: Hash, parts: impl IntoIterator) { 118 | for part in parts { 119 | self.0.insert((hash, part)); 120 | } 121 | } 122 | 123 | /// Mark a file as to be kept after the transaction is committed. 124 | /// 125 | /// This will cancel any previous delete for the same file in the same transaction. 126 | pub fn remove(&mut self, hash: Hash, parts: impl IntoIterator) { 127 | for part in parts { 128 | self.0.remove(&(hash, part)); 129 | } 130 | } 131 | 132 | /// Get the inner set of files to delete. 133 | pub fn into_inner(self) -> BTreeSet<(Hash, BaoFilePart)> { 134 | self.0 135 | } 136 | 137 | /// Apply the delete set and clear it. 138 | /// 139 | /// This will delete all files marked for deletion and then clear the set. 140 | /// Errors will just be logged. 141 | pub fn apply_and_clear(&mut self, options: &PathOptions) { 142 | for (hash, to_delete) in &self.0 { 143 | tracing::debug!("deleting {:?} for {hash}", to_delete); 144 | let path = match to_delete { 145 | BaoFilePart::Data => options.owned_data_path(hash), 146 | BaoFilePart::Outboard => options.owned_outboard_path(hash), 147 | BaoFilePart::Sizes => options.owned_sizes_path(hash), 148 | }; 149 | if let Err(cause) = std::fs::remove_file(&path) { 150 | // Ignore NotFound errors, if the file is already gone that's fine. 151 | if cause.kind() != std::io::ErrorKind::NotFound { 152 | tracing::warn!( 153 | "failed to delete {:?} {}: {}", 154 | to_delete, 155 | path.display(), 156 | cause 157 | ); 158 | } 159 | } 160 | } 161 | self.0.clear(); 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /src/store/fs/util.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fs::OpenOptions, 3 | io::{self, Write}, 4 | path::Path, 5 | }; 6 | 7 | /// overwrite a file with the given data. 8 | /// 9 | /// This is almost like `std::fs::write`, but it does not truncate the file. 10 | /// 11 | /// So if you overwrite a file with less data than it had before, the file will 12 | /// still have the same size as before. 13 | /// 14 | /// Also, if you overwrite a file with the same data as it had before, the 15 | /// file will be unchanged even if the overwrite operation is interrupted. 16 | pub fn overwrite_and_sync(path: &Path, data: &[u8]) -> io::Result { 17 | tracing::trace!( 18 | "overwriting file {} with {} bytes", 19 | path.display(), 20 | data.len() 21 | ); 22 | // std::fs::create_dir_all(path.parent().unwrap()).unwrap(); 23 | // tracing::error!("{}", path.parent().unwrap().display()); 24 | // tracing::error!("{}", path.parent().unwrap().metadata().unwrap().is_dir()); 25 | let mut file = OpenOptions::new() 26 | .write(true) 27 | .create(true) 28 | .truncate(false) 29 | .open(path)?; 30 | file.write_all(data)?; 31 | // todo: figure out if it is safe to not sync here 32 | file.sync_all()?; 33 | Ok(file) 34 | } 35 | 36 | /// Read a file into memory and then delete it. 37 | pub fn read_and_remove(path: &Path) -> io::Result> { 38 | let data = std::fs::read(path)?; 39 | // todo: should we fail here or just log a warning? 40 | // remove could fail e.g. on windows if the file is still open 41 | std::fs::remove_file(path)?; 42 | Ok(data) 43 | } 44 | 45 | /// A wrapper for a flume receiver that allows peeking at the next message. 46 | #[derive(Debug)] 47 | pub(super) struct PeekableFlumeReceiver { 48 | msg: Option, 49 | recv: async_channel::Receiver, 50 | } 51 | 52 | #[allow(dead_code)] 53 | impl PeekableFlumeReceiver { 54 | pub fn new(recv: async_channel::Receiver) -> Self { 55 | Self { msg: None, recv } 56 | } 57 | 58 | /// Receive the next message. 59 | /// 60 | /// Will block if there are no messages. 61 | /// Returns None only if there are no more messages (sender is dropped). 62 | pub async fn recv(&mut self) -> Option { 63 | if let Some(msg) = self.msg.take() { 64 | return Some(msg); 65 | } 66 | self.recv.recv().await.ok() 67 | } 68 | 69 | /// Push back a message. This will only work if there is room for it. 70 | /// Otherwise, it will fail and return the message. 71 | pub fn push_back(&mut self, msg: T) -> std::result::Result<(), T> { 72 | if self.msg.is_none() { 73 | self.msg = Some(msg); 74 | Ok(()) 75 | } else { 76 | Err(msg) 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/store/mutable_mem_storage.rs: -------------------------------------------------------------------------------- 1 | use bao_tree::{ 2 | io::{fsm::BaoContentItem, sync::WriteAt}, 3 | BaoTree, 4 | }; 5 | use bytes::Bytes; 6 | 7 | use crate::{ 8 | util::{compute_outboard, copy_limited_slice, SparseMemFile}, 9 | IROH_BLOCK_SIZE, 10 | }; 11 | 12 | /// Mutable in memory storage for a bao file. 13 | /// 14 | /// This is used for incomplete files if they are not big enough to warrant 15 | /// writing to disk. We must keep track of ranges in both data and outboard 16 | /// that have been written to, and track the most precise known size. 17 | #[derive(Debug, Default)] 18 | pub struct MutableMemStorage { 19 | /// Data file, can be any size. 20 | pub data: SparseMemFile, 21 | /// Outboard file, must be a multiple of 64 bytes. 22 | pub outboard: SparseMemFile, 23 | /// Size that was announced as we wrote that chunk 24 | pub sizes: SizeInfo, 25 | } 26 | 27 | /// Keep track of the most precise size we know of. 28 | /// 29 | /// When in memory, we don't have to write the size for every chunk to a separate 30 | /// slot, but can just keep the best one. 31 | #[derive(Debug, Default)] 32 | pub struct SizeInfo { 33 | pub offset: u64, 34 | pub size: u64, 35 | } 36 | 37 | impl SizeInfo { 38 | /// Create a new size info for a complete file of size `size`. 39 | pub(crate) fn complete(size: u64) -> Self { 40 | let mask = (1 << IROH_BLOCK_SIZE.chunk_log()) - 1; 41 | // offset of the last bao chunk in a file of size `size` 42 | let last_chunk_offset = size & mask; 43 | Self { 44 | offset: last_chunk_offset, 45 | size, 46 | } 47 | } 48 | 49 | /// Write a size at the given offset. The size at the highest offset is going to be kept. 50 | fn write(&mut self, offset: u64, size: u64) { 51 | // >= instead of > because we want to be able to update size 0, the initial value. 52 | if offset >= self.offset { 53 | self.offset = offset; 54 | self.size = size; 55 | } 56 | } 57 | 58 | /// The current size, representing the most correct size we know. 59 | pub fn current_size(&self) -> u64 { 60 | self.size 61 | } 62 | } 63 | 64 | impl MutableMemStorage { 65 | /// Create a new mutable mem storage from the given data 66 | pub fn complete(bytes: Bytes, cb: impl Fn(u64) + Send + Sync + 'static) -> (Self, crate::Hash) { 67 | let (hash, outboard) = compute_outboard(&bytes[..], bytes.len() as u64, move |offset| { 68 | cb(offset); 69 | Ok(()) 70 | }) 71 | .unwrap(); 72 | let outboard = outboard.unwrap_or_default(); 73 | let res = Self { 74 | data: bytes.to_vec().into(), 75 | outboard: outboard.into(), 76 | sizes: SizeInfo::complete(bytes.len() as u64), 77 | }; 78 | (res, hash) 79 | } 80 | 81 | pub(super) fn current_size(&self) -> u64 { 82 | self.sizes.current_size() 83 | } 84 | 85 | pub(super) fn read_data_at(&self, offset: u64, len: usize) -> Bytes { 86 | copy_limited_slice(&self.data, offset, len) 87 | } 88 | 89 | pub(super) fn data_len(&self) -> u64 { 90 | self.data.len() as u64 91 | } 92 | 93 | pub(super) fn read_outboard_at(&self, offset: u64, len: usize) -> Bytes { 94 | copy_limited_slice(&self.outboard, offset, len) 95 | } 96 | 97 | pub(super) fn outboard_len(&self) -> u64 { 98 | self.outboard.len() as u64 99 | } 100 | 101 | pub(super) fn write_batch( 102 | &mut self, 103 | size: u64, 104 | batch: &[BaoContentItem], 105 | ) -> std::io::Result<()> { 106 | let tree = BaoTree::new(size, IROH_BLOCK_SIZE); 107 | for item in batch { 108 | match item { 109 | BaoContentItem::Parent(parent) => { 110 | if let Some(offset) = tree.pre_order_offset(parent.node) { 111 | let o0 = offset 112 | .checked_mul(64) 113 | .expect("u64 overflow multiplying to hash pair offset"); 114 | let o1 = o0.checked_add(32).expect("u64 overflow"); 115 | let outboard = &mut self.outboard; 116 | outboard.write_all_at(o0, parent.pair.0.as_bytes().as_slice())?; 117 | outboard.write_all_at(o1, parent.pair.1.as_bytes().as_slice())?; 118 | } 119 | } 120 | BaoContentItem::Leaf(leaf) => { 121 | self.sizes.write(leaf.offset, size); 122 | self.data.write_all_at(leaf.offset, leaf.data.as_ref())?; 123 | } 124 | } 125 | } 126 | Ok(()) 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/ticket.rs: -------------------------------------------------------------------------------- 1 | //! Tickets for blobs. 2 | use std::{collections::BTreeSet, net::SocketAddr, str::FromStr}; 3 | 4 | use anyhow::Result; 5 | use iroh::{NodeAddr, NodeId, RelayUrl}; 6 | use iroh_base::ticket::{self, Ticket}; 7 | use serde::{Deserialize, Serialize}; 8 | 9 | use crate::{BlobFormat, Hash}; 10 | 11 | /// A token containing everything to get a file from the provider. 12 | /// 13 | /// It is a single item which can be easily serialized and deserialized. 14 | #[derive(Debug, Clone, PartialEq, Eq, derive_more::Display)] 15 | #[display("{}", Ticket::serialize(self))] 16 | pub struct BlobTicket { 17 | /// The provider to get a file from. 18 | node: NodeAddr, 19 | /// The format of the blob. 20 | format: BlobFormat, 21 | /// The hash to retrieve. 22 | hash: Hash, 23 | } 24 | 25 | /// Wire format for [`BlobTicket`]. 26 | /// 27 | /// In the future we might have multiple variants (not versions, since they 28 | /// might be both equally valid), so this is a single variant enum to force 29 | /// postcard to add a discriminator. 30 | #[derive(Serialize, Deserialize)] 31 | enum TicketWireFormat { 32 | Variant0(Variant0BlobTicket), 33 | } 34 | 35 | // Legacy 36 | #[derive(Serialize, Deserialize)] 37 | struct Variant0BlobTicket { 38 | node: Variant0NodeAddr, 39 | format: BlobFormat, 40 | hash: Hash, 41 | } 42 | 43 | #[derive(Serialize, Deserialize)] 44 | struct Variant0NodeAddr { 45 | node_id: NodeId, 46 | info: Variant0AddrInfo, 47 | } 48 | 49 | #[derive(Serialize, Deserialize)] 50 | struct Variant0AddrInfo { 51 | relay_url: Option, 52 | direct_addresses: BTreeSet, 53 | } 54 | 55 | impl Ticket for BlobTicket { 56 | const KIND: &'static str = "blob"; 57 | 58 | fn to_bytes(&self) -> Vec { 59 | let data = TicketWireFormat::Variant0(Variant0BlobTicket { 60 | node: Variant0NodeAddr { 61 | node_id: self.node.node_id, 62 | info: Variant0AddrInfo { 63 | relay_url: self.node.relay_url.clone(), 64 | direct_addresses: self.node.direct_addresses.clone(), 65 | }, 66 | }, 67 | format: self.format, 68 | hash: self.hash, 69 | }); 70 | postcard::to_stdvec(&data).expect("postcard serialization failed") 71 | } 72 | 73 | fn from_bytes(bytes: &[u8]) -> std::result::Result { 74 | let res: TicketWireFormat = postcard::from_bytes(bytes).map_err(ticket::Error::Postcard)?; 75 | let TicketWireFormat::Variant0(Variant0BlobTicket { node, format, hash }) = res; 76 | Ok(Self { 77 | node: NodeAddr { 78 | node_id: node.node_id, 79 | relay_url: node.info.relay_url, 80 | direct_addresses: node.info.direct_addresses, 81 | }, 82 | format, 83 | hash, 84 | }) 85 | } 86 | } 87 | 88 | impl FromStr for BlobTicket { 89 | type Err = ticket::Error; 90 | 91 | fn from_str(s: &str) -> Result { 92 | Ticket::deserialize(s) 93 | } 94 | } 95 | 96 | impl BlobTicket { 97 | /// Creates a new ticket. 98 | pub fn new(node: NodeAddr, hash: Hash, format: BlobFormat) -> Result { 99 | Ok(Self { hash, format, node }) 100 | } 101 | 102 | /// The hash of the item this ticket can retrieve. 103 | pub fn hash(&self) -> Hash { 104 | self.hash 105 | } 106 | 107 | /// The [`NodeAddr`] of the provider for this ticket. 108 | pub fn node_addr(&self) -> &NodeAddr { 109 | &self.node 110 | } 111 | 112 | /// The [`BlobFormat`] for this ticket. 113 | pub fn format(&self) -> BlobFormat { 114 | self.format 115 | } 116 | 117 | /// True if the ticket is for a collection and should retrieve all blobs in it. 118 | pub fn recursive(&self) -> bool { 119 | self.format.is_hash_seq() 120 | } 121 | 122 | /// Get the contents of the ticket, consuming it. 123 | pub fn into_parts(self) -> (NodeAddr, Hash, BlobFormat) { 124 | let BlobTicket { node, hash, format } = self; 125 | (node, hash, format) 126 | } 127 | } 128 | 129 | impl Serialize for BlobTicket { 130 | fn serialize(&self, serializer: S) -> Result { 131 | if serializer.is_human_readable() { 132 | serializer.serialize_str(&self.to_string()) 133 | } else { 134 | let BlobTicket { node, format, hash } = self; 135 | (node, format, hash).serialize(serializer) 136 | } 137 | } 138 | } 139 | 140 | impl<'de> Deserialize<'de> for BlobTicket { 141 | fn deserialize>(deserializer: D) -> Result { 142 | if deserializer.is_human_readable() { 143 | let s = String::deserialize(deserializer)?; 144 | Self::from_str(&s).map_err(serde::de::Error::custom) 145 | } else { 146 | let (peer, format, hash) = Deserialize::deserialize(deserializer)?; 147 | Self::new(peer, hash, format).map_err(serde::de::Error::custom) 148 | } 149 | } 150 | } 151 | 152 | #[cfg(test)] 153 | mod tests { 154 | use std::net::SocketAddr; 155 | 156 | use iroh::{PublicKey, SecretKey}; 157 | 158 | use super::*; 159 | use crate::{assert_eq_hex, util::hexdump::parse_hexdump}; 160 | 161 | fn make_ticket() -> BlobTicket { 162 | let hash = Hash::new(b"hi there"); 163 | let peer = SecretKey::generate(rand::thread_rng()).public(); 164 | let addr = SocketAddr::from_str("127.0.0.1:1234").unwrap(); 165 | let relay_url = None; 166 | BlobTicket { 167 | hash, 168 | node: NodeAddr::from_parts(peer, relay_url, [addr]), 169 | format: BlobFormat::HashSeq, 170 | } 171 | } 172 | 173 | #[test] 174 | fn test_ticket_postcard() { 175 | let ticket = make_ticket(); 176 | let bytes = postcard::to_stdvec(&ticket).unwrap(); 177 | let ticket2: BlobTicket = postcard::from_bytes(&bytes).unwrap(); 178 | assert_eq!(ticket2, ticket); 179 | } 180 | 181 | #[test] 182 | fn test_ticket_json() { 183 | let ticket = make_ticket(); 184 | let json = serde_json::to_string(&ticket).unwrap(); 185 | let ticket2: BlobTicket = serde_json::from_str(&json).unwrap(); 186 | assert_eq!(ticket2, ticket); 187 | } 188 | 189 | #[test] 190 | fn test_ticket_base32() { 191 | let hash = 192 | Hash::from_str("0b84d358e4c8be6c38626b2182ff575818ba6bd3f4b90464994be14cb354a072") 193 | .unwrap(); 194 | let node_id = 195 | PublicKey::from_str("ae58ff8833241ac82d6ff7611046ed67b5072d142c588d0063e942d9a75502b6") 196 | .unwrap(); 197 | 198 | let ticket = BlobTicket { 199 | node: NodeAddr::from_parts(node_id, None, []), 200 | format: BlobFormat::Raw, 201 | hash, 202 | }; 203 | let encoded = ticket.to_string(); 204 | let stripped = encoded.strip_prefix("blob").unwrap(); 205 | let base32 = data_encoding::BASE32_NOPAD 206 | .decode(stripped.to_ascii_uppercase().as_bytes()) 207 | .unwrap(); 208 | let expected = parse_hexdump(" 209 | 00 # discriminator for variant 0 210 | ae58ff8833241ac82d6ff7611046ed67b5072d142c588d0063e942d9a75502b6 # node id, 32 bytes, see above 211 | 00 # relay url 212 | 00 # number of addresses (0) 213 | 00 # format (raw) 214 | 0b84d358e4c8be6c38626b2182ff575818ba6bd3f4b90464994be14cb354a072 # hash, 32 bytes, see above 215 | ").unwrap(); 216 | assert_eq_hex!(base32, expected); 217 | } 218 | } 219 | -------------------------------------------------------------------------------- /src/util/hexdump.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{ensure, Context, Result}; 2 | 3 | /// Parses a commented multi line hexdump into a vector of bytes. 4 | /// 5 | /// This is useful to write wire level protocol tests. 6 | pub fn parse_hexdump(s: &str) -> Result> { 7 | let mut result = Vec::new(); 8 | 9 | for (line_number, line) in s.lines().enumerate() { 10 | let data_part = line.split('#').next().unwrap_or(""); 11 | let cleaned: String = data_part.chars().filter(|c| !c.is_whitespace()).collect(); 12 | 13 | ensure!( 14 | cleaned.len() % 2 == 0, 15 | "Non-even number of hex chars detected on line {}.", 16 | line_number + 1 17 | ); 18 | 19 | for i in (0..cleaned.len()).step_by(2) { 20 | let byte_str = &cleaned[i..i + 2]; 21 | let byte = u8::from_str_radix(byte_str, 16) 22 | .with_context(|| format!("Invalid hex data on line {}.", line_number + 1))?; 23 | 24 | result.push(byte); 25 | } 26 | } 27 | 28 | Ok(result) 29 | } 30 | 31 | /// Returns a hexdump of the given bytes in multiple lines as a String. 32 | pub fn print_hexdump(bytes: impl AsRef<[u8]>, line_lengths: impl AsRef<[usize]>) -> String { 33 | let line_lengths = line_lengths.as_ref(); 34 | let mut bytes_iter = bytes.as_ref().iter(); 35 | let default_line_length = line_lengths 36 | .last() 37 | .filter(|x| **x != 0) 38 | .copied() 39 | .unwrap_or(16); 40 | let mut line_lengths_iter = line_lengths.iter(); 41 | let mut output = String::new(); 42 | 43 | loop { 44 | let line_length = line_lengths_iter 45 | .next() 46 | .copied() 47 | .unwrap_or(default_line_length); 48 | if line_length == 0 { 49 | output.push('\n'); 50 | } else { 51 | let line: Vec<_> = bytes_iter.by_ref().take(line_length).collect(); 52 | 53 | if line.is_empty() { 54 | break; 55 | } 56 | 57 | for byte in &line { 58 | output.push_str(&format!("{:02x} ", byte)); 59 | } 60 | output.pop(); // Remove the trailing space 61 | output.push('\n'); 62 | } 63 | } 64 | 65 | output 66 | } 67 | 68 | /// This is a macro to assert that two byte slices are equal. 69 | /// 70 | /// It is like assert_eq!, but it will print a nicely formatted hexdump of the 71 | /// two slices if they are not equal. This makes it much easier to track down 72 | /// a difference in a large byte slice. 73 | #[macro_export] 74 | macro_rules! assert_eq_hex { 75 | ($a:expr, $b:expr) => { 76 | assert_eq_hex!($a, $b, []) 77 | }; 78 | ($a:expr, $b:expr, $hint:expr) => { 79 | let a = $a; 80 | let b = $b; 81 | let hint = $hint; 82 | let ar: &[u8] = a.as_ref(); 83 | let br: &[u8] = b.as_ref(); 84 | let hintr: &[usize] = hint.as_ref(); 85 | if ar != br { 86 | use $crate::util::hexdump::print_hexdump; 87 | panic!( 88 | "assertion failed: `(left == right)`\nleft:\n{}\nright:\n{}\n", 89 | print_hexdump(ar, hintr), 90 | print_hexdump(br, hintr), 91 | ) 92 | } 93 | }; 94 | } 95 | 96 | #[cfg(test)] 97 | mod tests { 98 | use super::{parse_hexdump, print_hexdump}; 99 | 100 | #[test] 101 | fn test_basic() { 102 | let input = r" 103 | a1b2 # comment 104 | 3c4d 105 | "; 106 | let result = parse_hexdump(input).unwrap(); 107 | assert_eq!(result, vec![0xa1, 0xb2, 0x3c, 0x4d]); 108 | } 109 | 110 | #[test] 111 | fn test_upper_case() { 112 | let input = r" 113 | A1B2 # comment 114 | 3C4D 115 | "; 116 | let result = parse_hexdump(input).unwrap(); 117 | assert_eq!(result, vec![0xa1, 0xb2, 0x3c, 0x4d]); 118 | } 119 | 120 | #[test] 121 | fn test_mixed_case() { 122 | let input = r" 123 | a1B2 # comment 124 | 3C4d 125 | "; 126 | let result = parse_hexdump(input).unwrap(); 127 | assert_eq!(result, vec![0xa1, 0xb2, 0x3c, 0x4d]); 128 | } 129 | 130 | #[test] 131 | fn test_odd_characters() { 132 | let input = r" 133 | a1b 134 | "; 135 | let result = parse_hexdump(input); 136 | assert!(result.is_err()); 137 | } 138 | 139 | #[test] 140 | fn test_invalid_characters() { 141 | let input = r" 142 | a1g2 # 'g' is not valid in hex 143 | "; 144 | let result = parse_hexdump(input); 145 | assert!(result.is_err()); 146 | } 147 | #[test] 148 | fn test_basic_hexdump() { 149 | let data: &[u8] = &[0x1, 0x2, 0x3, 0x4, 0x5]; 150 | let output = print_hexdump(data, [1, 2]); 151 | assert_eq!(output, "01\n02 03\n04 05\n"); 152 | } 153 | 154 | #[test] 155 | fn test_newline_insertion() { 156 | let data: &[u8] = &[0x1, 0x2, 0x3, 0x4]; 157 | let output = print_hexdump(data, [1, 0, 2]); 158 | assert_eq!(output, "01\n\n02 03\n04\n"); 159 | } 160 | 161 | #[test] 162 | fn test_indefinite_line_length() { 163 | let data: &[u8] = &[0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8]; 164 | let output = print_hexdump(data, [2, 4]); 165 | assert_eq!(output, "01 02\n03 04 05 06\n07 08\n"); 166 | } 167 | 168 | #[test] 169 | fn test_empty_data() { 170 | let data: &[u8] = &[]; 171 | let output = print_hexdump(data, [1, 2]); 172 | assert_eq!(output, ""); 173 | } 174 | 175 | #[test] 176 | fn test_zeros_then_default() { 177 | let data: &[u8] = &[0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8]; 178 | let output = print_hexdump(data, [1, 0, 0, 2]); 179 | assert_eq!(output, "01\n\n\n02 03\n04 05\n06 07\n08\n"); 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /src/util/io.rs: -------------------------------------------------------------------------------- 1 | //! Utilities for working with tokio io 2 | 3 | use std::{io, pin::Pin, task::Poll}; 4 | 5 | use iroh_io::AsyncStreamReader; 6 | use tokio::io::AsyncWrite; 7 | 8 | /// A reader that tracks the number of bytes read 9 | #[derive(Debug)] 10 | pub struct TrackingReader { 11 | inner: R, 12 | read: u64, 13 | } 14 | 15 | impl TrackingReader { 16 | /// Wrap a reader in a tracking reader 17 | pub fn new(inner: R) -> Self { 18 | Self { inner, read: 0 } 19 | } 20 | 21 | /// Get the number of bytes read 22 | #[allow(dead_code)] 23 | pub fn bytes_read(&self) -> u64 { 24 | self.read 25 | } 26 | 27 | /// Get the inner reader 28 | pub fn into_parts(self) -> (R, u64) { 29 | (self.inner, self.read) 30 | } 31 | } 32 | 33 | impl AsyncStreamReader for TrackingReader 34 | where 35 | R: AsyncStreamReader, 36 | { 37 | async fn read_bytes(&mut self, len: usize) -> io::Result { 38 | let bytes = self.inner.read_bytes(len).await?; 39 | self.read = self.read.saturating_add(bytes.len() as u64); 40 | Ok(bytes) 41 | } 42 | 43 | async fn read(&mut self) -> io::Result<[u8; L]> { 44 | let res = self.inner.read::().await?; 45 | self.read = self.read.saturating_add(L as u64); 46 | Ok(res) 47 | } 48 | } 49 | 50 | /// A writer that tracks the number of bytes written 51 | #[derive(Debug)] 52 | pub struct TrackingWriter { 53 | inner: W, 54 | written: u64, 55 | } 56 | 57 | impl TrackingWriter { 58 | /// Wrap a writer in a tracking writer 59 | pub fn new(inner: W) -> Self { 60 | Self { inner, written: 0 } 61 | } 62 | 63 | /// Get the number of bytes written 64 | #[allow(dead_code)] 65 | pub fn bytes_written(&self) -> u64 { 66 | self.written 67 | } 68 | 69 | /// Get the inner writer 70 | pub fn into_parts(self) -> (W, u64) { 71 | (self.inner, self.written) 72 | } 73 | } 74 | 75 | impl AsyncWrite for TrackingWriter { 76 | fn poll_write( 77 | mut self: Pin<&mut Self>, 78 | cx: &mut std::task::Context<'_>, 79 | buf: &[u8], 80 | ) -> Poll> { 81 | let this = &mut *self; 82 | let res = Pin::new(&mut this.inner).poll_write(cx, buf); 83 | if let Poll::Ready(Ok(size)) = res { 84 | this.written = this.written.saturating_add(size as u64); 85 | } 86 | res 87 | } 88 | 89 | fn poll_flush( 90 | mut self: Pin<&mut Self>, 91 | cx: &mut std::task::Context<'_>, 92 | ) -> Poll> { 93 | Pin::new(&mut self.inner).poll_flush(cx) 94 | } 95 | 96 | fn poll_shutdown( 97 | mut self: Pin<&mut Self>, 98 | cx: &mut std::task::Context<'_>, 99 | ) -> Poll> { 100 | Pin::new(&mut self.inner).poll_shutdown(cx) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/util/mem_or_file.rs: -------------------------------------------------------------------------------- 1 | use std::{fs::File, io}; 2 | 3 | use bao_tree::io::sync::{ReadAt, Size}; 4 | use bytes::Bytes; 5 | 6 | /// This is a general purpose Either, just like Result, except that the two cases 7 | /// are Mem for something that is in memory, and File for something that is somewhere 8 | /// external and only available via io. 9 | #[derive(Debug)] 10 | pub enum MemOrFile { 11 | /// We got it all in memory 12 | Mem(M), 13 | /// A file 14 | File(F), 15 | } 16 | 17 | /// Helper methods for a common way to use MemOrFile, where the memory part is something 18 | /// like a slice, and the file part is a tuple consisiting of path or file and size. 19 | impl MemOrFile 20 | where 21 | M: AsRef<[u8]>, 22 | { 23 | /// Get the size of the MemOrFile 24 | pub fn size(&self) -> u64 { 25 | match self { 26 | MemOrFile::Mem(mem) => mem.as_ref().len() as u64, 27 | MemOrFile::File((_, size)) => *size, 28 | } 29 | } 30 | } 31 | 32 | impl ReadAt for MemOrFile { 33 | fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result { 34 | match self { 35 | MemOrFile::Mem(mem) => mem.as_ref().read_at(offset, buf), 36 | MemOrFile::File(file) => file.read_at(offset, buf), 37 | } 38 | } 39 | } 40 | 41 | impl Size for MemOrFile { 42 | fn size(&self) -> io::Result> { 43 | match self { 44 | MemOrFile::Mem(mem) => Ok(Some(mem.len() as u64)), 45 | MemOrFile::File(file) => file.size(), 46 | } 47 | } 48 | } 49 | 50 | impl Default for MemOrFile { 51 | fn default() -> Self { 52 | MemOrFile::Mem(Default::default()) 53 | } 54 | } 55 | 56 | impl MemOrFile { 57 | /// Turn a reference to a MemOrFile into a MemOrFile of references 58 | pub fn as_ref(&self) -> MemOrFile<&M, &F> { 59 | match self { 60 | MemOrFile::Mem(mem) => MemOrFile::Mem(mem), 61 | MemOrFile::File(file) => MemOrFile::File(file), 62 | } 63 | } 64 | 65 | /// True if this is a Mem 66 | pub fn is_mem(&self) -> bool { 67 | matches!(self, MemOrFile::Mem(_)) 68 | } 69 | 70 | /// Get the mem part 71 | pub fn mem(&self) -> Option<&M> { 72 | match self { 73 | MemOrFile::Mem(mem) => Some(mem), 74 | MemOrFile::File(_) => None, 75 | } 76 | } 77 | 78 | /// Map the file part of this MemOrFile 79 | pub fn map_file(self, f: impl FnOnce(F) -> F2) -> MemOrFile { 80 | match self { 81 | MemOrFile::Mem(mem) => MemOrFile::Mem(mem), 82 | MemOrFile::File(file) => MemOrFile::File(f(file)), 83 | } 84 | } 85 | 86 | /// Try to map the file part of this MemOrFile 87 | pub fn try_map_file( 88 | self, 89 | f: impl FnOnce(F) -> Result, 90 | ) -> Result, E> { 91 | match self { 92 | MemOrFile::Mem(mem) => Ok(MemOrFile::Mem(mem)), 93 | MemOrFile::File(file) => f(file).map(MemOrFile::File), 94 | } 95 | } 96 | 97 | /// Map the memory part of this MemOrFile 98 | pub fn map_mem(self, f: impl FnOnce(M) -> M2) -> MemOrFile { 99 | match self { 100 | MemOrFile::Mem(mem) => MemOrFile::Mem(f(mem)), 101 | MemOrFile::File(file) => MemOrFile::File(file), 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/util/sparse_mem_file.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | 3 | use bao_tree::io::sync::{ReadAt, Size, WriteAt}; 4 | use derive_more::Deref; 5 | use range_collections::{range_set::RangeSetRange, RangeSet2}; 6 | 7 | /// A file that is sparse in memory 8 | /// 9 | /// It is not actually using sparse storage to make reading faster, so it will 10 | /// not conserve memory. It is just a way to remember the gaps so we can 11 | /// write it to a file in a sparse way later. 12 | #[derive(derive_more::Debug)] 13 | pub struct SparseMemFile { 14 | /// The data, with gaps filled with zeros 15 | #[debug("{} bytes", data.len())] 16 | data: Vec, 17 | /// The ranges that are not zeros, so we can distinguish between zeros and gaps 18 | ranges: RangeSet2, 19 | } 20 | 21 | impl Default for SparseMemFile { 22 | fn default() -> Self { 23 | Self::new() 24 | } 25 | } 26 | 27 | impl From> for SparseMemFile { 28 | fn from(data: Vec) -> Self { 29 | let ranges = RangeSet2::from(0..data.len()); 30 | Self { data, ranges } 31 | } 32 | } 33 | 34 | impl TryInto> for SparseMemFile { 35 | type Error = io::Error; 36 | 37 | fn try_into(self) -> Result, Self::Error> { 38 | let (data, ranges) = self.into_parts(); 39 | if ranges == RangeSet2::from(0..data.len()) { 40 | Ok(data) 41 | } else { 42 | Err(io::Error::new( 43 | io::ErrorKind::InvalidData, 44 | "SparseMemFile has gaps", 45 | )) 46 | } 47 | } 48 | } 49 | 50 | impl SparseMemFile { 51 | /// Create a new, empty SparseMemFile 52 | pub fn new() -> Self { 53 | Self { 54 | data: Vec::new(), 55 | ranges: RangeSet2::empty(), 56 | } 57 | } 58 | 59 | /// Get the data and the valid ranges 60 | pub fn into_parts(self) -> (Vec, RangeSet2) { 61 | (self.data, self.ranges) 62 | } 63 | 64 | /// Persist the SparseMemFile to a WriteAt 65 | /// 66 | /// This will not persist the gaps, only the data that was written. 67 | pub fn persist(&self, mut target: impl WriteAt) -> io::Result<()> { 68 | let size = self.data.len(); 69 | for range in self.ranges.iter() { 70 | let range = match range { 71 | RangeSetRange::Range(range) => *range.start..*range.end, 72 | RangeSetRange::RangeFrom(range) => *range.start..size, 73 | }; 74 | let start = range.start.try_into().unwrap(); 75 | let buf = &self.data[range]; 76 | target.write_at(start, buf)?; 77 | } 78 | Ok(()) 79 | } 80 | } 81 | 82 | impl AsRef<[u8]> for SparseMemFile { 83 | fn as_ref(&self) -> &[u8] { 84 | &self.data 85 | } 86 | } 87 | 88 | impl Deref for SparseMemFile { 89 | type Target = [u8]; 90 | 91 | fn deref(&self) -> &Self::Target { 92 | &self.data 93 | } 94 | } 95 | 96 | impl ReadAt for SparseMemFile { 97 | fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result { 98 | self.data.read_at(offset, buf) 99 | } 100 | } 101 | 102 | impl WriteAt for SparseMemFile { 103 | fn write_at(&mut self, offset: u64, buf: &[u8]) -> io::Result { 104 | let start: usize = offset.try_into().map_err(|_| io::ErrorKind::InvalidInput)?; 105 | let end = start 106 | .checked_add(buf.len()) 107 | .ok_or(io::ErrorKind::InvalidInput)?; 108 | let n = self.data.write_at(offset, buf)?; 109 | self.ranges |= RangeSet2::from(start..end); 110 | Ok(n) 111 | } 112 | 113 | fn flush(&mut self) -> io::Result<()> { 114 | Ok(()) 115 | } 116 | } 117 | 118 | impl Size for SparseMemFile { 119 | fn size(&self) -> io::Result> { 120 | Ok(Some(self.data.len() as u64)) 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /tests/blobs.rs: -------------------------------------------------------------------------------- 1 | #![cfg(all(feature = "net_protocol", feature = "rpc"))] 2 | use std::{ 3 | sync::{Arc, Mutex}, 4 | time::Duration, 5 | }; 6 | 7 | use iroh::Endpoint; 8 | use iroh_blobs::{net_protocol::Blobs, store::GcConfig}; 9 | use testresult::TestResult; 10 | 11 | #[tokio::test] 12 | async fn blobs_gc_smoke() -> TestResult<()> { 13 | let endpoint = Endpoint::builder().bind().await?; 14 | let blobs = Blobs::memory().build(&endpoint); 15 | let client = blobs.client(); 16 | blobs.start_gc(GcConfig { 17 | period: Duration::from_millis(1), 18 | done_callback: None, 19 | })?; 20 | let h1 = client.add_bytes(b"test".to_vec()).await?; 21 | tokio::time::sleep(Duration::from_millis(100)).await; 22 | assert!(client.has(h1.hash).await?); 23 | client.tags().delete(h1.tag).await?; 24 | tokio::time::sleep(Duration::from_millis(100)).await; 25 | assert!(!client.has(h1.hash).await?); 26 | Ok(()) 27 | } 28 | 29 | #[tokio::test] 30 | async fn blobs_gc_protected() -> TestResult<()> { 31 | let endpoint = Endpoint::builder().bind().await?; 32 | let blobs = Blobs::memory().build(&endpoint); 33 | let client = blobs.client(); 34 | let h1 = client.add_bytes(b"test".to_vec()).await?; 35 | let protected = Arc::new(Mutex::new(Vec::new())); 36 | blobs.add_protected(Box::new({ 37 | let protected = protected.clone(); 38 | move |x| { 39 | let protected = protected.clone(); 40 | Box::pin(async move { 41 | let protected = protected.lock().unwrap(); 42 | for h in protected.as_slice() { 43 | x.insert(*h); 44 | } 45 | }) 46 | } 47 | }))?; 48 | blobs.start_gc(GcConfig { 49 | period: Duration::from_millis(1), 50 | done_callback: None, 51 | })?; 52 | tokio::time::sleep(Duration::from_millis(100)).await; 53 | // protected from gc due to tag 54 | assert!(client.has(h1.hash).await?); 55 | protected.lock().unwrap().push(h1.hash); 56 | client.tags().delete(h1.tag).await?; 57 | tokio::time::sleep(Duration::from_millis(100)).await; 58 | // protected from gc due to being in protected set 59 | assert!(client.has(h1.hash).await?); 60 | protected.lock().unwrap().clear(); 61 | tokio::time::sleep(Duration::from_millis(100)).await; 62 | // not protected, must be gone 63 | assert!(!client.has(h1.hash).await?); 64 | Ok(()) 65 | } 66 | -------------------------------------------------------------------------------- /tests/rpc.rs: -------------------------------------------------------------------------------- 1 | #![cfg(feature = "test")] 2 | use std::{net::SocketAddr, path::PathBuf, vec}; 3 | 4 | use iroh_blobs::{downloader, net_protocol::Blobs}; 5 | use quic_rpc::client::QuinnConnector; 6 | use tempfile::TempDir; 7 | use testresult::TestResult; 8 | use tokio_util::task::AbortOnDropHandle; 9 | 10 | type QC = QuinnConnector; 11 | type BlobsClient = iroh_blobs::rpc::client::blobs::Client; 12 | 13 | /// An iroh node that just has the blobs transport 14 | #[derive(Debug)] 15 | pub struct Node { 16 | pub router: iroh::protocol::Router, 17 | pub blobs: Blobs, 18 | pub rpc_task: AbortOnDropHandle<()>, 19 | } 20 | 21 | impl Node { 22 | pub async fn new(path: PathBuf) -> anyhow::Result<(Self, SocketAddr, Vec)> { 23 | let store = iroh_blobs::store::fs::Store::load(path).await?; 24 | let endpoint = iroh::Endpoint::builder().bind().await?; 25 | let blobs = Blobs::builder(store).build(&endpoint); 26 | let router = iroh::protocol::Router::builder(endpoint) 27 | .accept(iroh_blobs::ALPN, blobs.clone()) 28 | .spawn(); 29 | let (config, key) = quic_rpc::transport::quinn::configure_server()?; 30 | let endpoint = quinn::Endpoint::server(config, "127.0.0.1:0".parse().unwrap())?; 31 | let local_addr = endpoint.local_addr()?; 32 | let rpc_server = quic_rpc::transport::quinn::QuinnListener::new(endpoint)?; 33 | let rpc_server = 34 | quic_rpc::RpcServer::::new(rpc_server); 35 | let blobs2 = blobs.clone(); 36 | let rpc_task = rpc_server 37 | .spawn_accept_loop(move |msg, chan| blobs2.clone().handle_rpc_request(msg, chan)); 38 | let node = Self { 39 | router, 40 | blobs, 41 | rpc_task, 42 | }; 43 | Ok((node, local_addr, key)) 44 | } 45 | } 46 | 47 | async fn node_and_client() -> TestResult<(Node, BlobsClient, TempDir)> { 48 | let testdir = tempfile::tempdir()?; 49 | let (node, addr, key) = Node::new(testdir.path().join("blobs")).await?; 50 | let client = quic_rpc::transport::quinn::make_client_endpoint( 51 | "127.0.0.1:0".parse().unwrap(), 52 | &[key.as_slice()], 53 | )?; 54 | let client = QuinnConnector::::new( 55 | client, 56 | addr, 57 | "localhost".to_string(), 58 | ); 59 | let client = quic_rpc::RpcClient::::new(client); 60 | let client = iroh_blobs::rpc::client::blobs::Client::new(client); 61 | Ok((node, client, testdir)) 62 | } 63 | 64 | #[tokio::test] 65 | async fn quinn_rpc_smoke() -> TestResult<()> { 66 | let _ = tracing_subscriber::fmt::try_init(); 67 | let (_node, client, _testdir) = node_and_client().await?; 68 | let data = b"hello"; 69 | let hash = client.add_bytes(data.to_vec()).await?.hash; 70 | assert_eq!(hash, iroh_blobs::Hash::new(data)); 71 | let data2 = client.read_to_bytes(hash).await?; 72 | assert_eq!(data, &data2[..]); 73 | Ok(()) 74 | } 75 | 76 | #[tokio::test] 77 | async fn quinn_rpc_large() -> TestResult<()> { 78 | let _ = tracing_subscriber::fmt::try_init(); 79 | let (_node, client, _testdir) = node_and_client().await?; 80 | let data = vec![0; 1024 * 1024 * 16]; 81 | let hash = client.add_bytes(data.clone()).await?.hash; 82 | assert_eq!(hash, iroh_blobs::Hash::new(&data)); 83 | let data2 = client.read_to_bytes(hash).await?; 84 | assert_eq!(data, &data2[..]); 85 | Ok(()) 86 | } 87 | 88 | #[tokio::test] 89 | async fn downloader_config() -> TestResult<()> { 90 | let _ = tracing_subscriber::fmt::try_init(); 91 | let endpoint = iroh::Endpoint::builder().bind().await?; 92 | let store = iroh_blobs::store::mem::Store::default(); 93 | let expected = downloader::Config { 94 | concurrency: downloader::ConcurrencyLimits { 95 | max_concurrent_requests: usize::MAX, 96 | max_concurrent_requests_per_node: usize::MAX, 97 | max_open_connections: usize::MAX, 98 | max_concurrent_dials_per_hash: usize::MAX, 99 | }, 100 | retry: downloader::RetryConfig { 101 | max_retries_per_node: u32::MAX, 102 | initial_retry_delay: std::time::Duration::from_secs(1), 103 | }, 104 | }; 105 | let blobs = Blobs::builder(store) 106 | .downloader_config(expected) 107 | .build(&endpoint); 108 | let actual = blobs.downloader().config(); 109 | assert_eq!(&expected, actual); 110 | Ok(()) 111 | } 112 | -------------------------------------------------------------------------------- /tests/tags.rs: -------------------------------------------------------------------------------- 1 | #![cfg(all(feature = "net_protocol", feature = "rpc"))] 2 | use futures_lite::StreamExt; 3 | use futures_util::Stream; 4 | use iroh::Endpoint; 5 | use iroh_blobs::{ 6 | net_protocol::Blobs, 7 | rpc::{ 8 | client::tags::{self, TagInfo}, 9 | proto::RpcService, 10 | }, 11 | Hash, HashAndFormat, 12 | }; 13 | use testresult::TestResult; 14 | 15 | async fn to_vec(stream: impl Stream>) -> anyhow::Result> { 16 | let res = stream.collect::>().await; 17 | res.into_iter().collect::>>() 18 | } 19 | 20 | fn expected(tags: impl IntoIterator) -> Vec { 21 | tags.into_iter() 22 | .map(|tag| TagInfo::new(tag, Hash::new(tag))) 23 | .collect() 24 | } 25 | 26 | async fn set>( 27 | tags: &tags::Client, 28 | names: impl IntoIterator, 29 | ) -> TestResult<()> { 30 | for name in names { 31 | tags.set(name, Hash::new(name)).await?; 32 | } 33 | Ok(()) 34 | } 35 | 36 | async fn tags_smoke>(tags: tags::Client) -> TestResult<()> { 37 | set(&tags, ["a", "b", "c", "d", "e"]).await?; 38 | let stream = tags.list().await?; 39 | let res = to_vec(stream).await?; 40 | assert_eq!(res, expected(["a", "b", "c", "d", "e"])); 41 | 42 | let stream = tags.list_range("b".."d").await?; 43 | let res = to_vec(stream).await?; 44 | assert_eq!(res, expected(["b", "c"])); 45 | 46 | let stream = tags.list_range("b"..).await?; 47 | let res = to_vec(stream).await?; 48 | assert_eq!(res, expected(["b", "c", "d", "e"])); 49 | 50 | let stream = tags.list_range(.."d").await?; 51 | let res = to_vec(stream).await?; 52 | assert_eq!(res, expected(["a", "b", "c"])); 53 | 54 | let stream = tags.list_range(..="d").await?; 55 | let res = to_vec(stream).await?; 56 | assert_eq!(res, expected(["a", "b", "c", "d"])); 57 | 58 | tags.delete_range("b"..).await?; 59 | let stream = tags.list().await?; 60 | let res = to_vec(stream).await?; 61 | assert_eq!(res, expected(["a"])); 62 | 63 | tags.delete_range(..="a").await?; 64 | let stream = tags.list().await?; 65 | let res = to_vec(stream).await?; 66 | assert_eq!(res, expected([])); 67 | 68 | set(&tags, ["a", "aa", "aaa", "aab", "b"]).await?; 69 | 70 | let stream = tags.list_prefix("aa").await?; 71 | let res = to_vec(stream).await?; 72 | assert_eq!(res, expected(["aa", "aaa", "aab"])); 73 | 74 | tags.delete_prefix("aa").await?; 75 | let stream = tags.list().await?; 76 | let res = to_vec(stream).await?; 77 | assert_eq!(res, expected(["a", "b"])); 78 | 79 | tags.delete_prefix("").await?; 80 | let stream = tags.list().await?; 81 | let res = to_vec(stream).await?; 82 | assert_eq!(res, expected([])); 83 | 84 | set(&tags, ["a", "b", "c"]).await?; 85 | 86 | assert_eq!( 87 | tags.get("b").await?, 88 | Some(TagInfo::new("b", Hash::new("b"))) 89 | ); 90 | 91 | tags.delete("b").await?; 92 | let stream = tags.list().await?; 93 | let res = to_vec(stream).await?; 94 | assert_eq!(res, expected(["a", "c"])); 95 | 96 | assert_eq!(tags.get("b").await?, None); 97 | 98 | tags.delete_all().await?; 99 | 100 | tags.set("a", HashAndFormat::hash_seq(Hash::new("a"))) 101 | .await?; 102 | tags.set("b", HashAndFormat::raw(Hash::new("b"))).await?; 103 | let stream = tags.list_hash_seq().await?; 104 | let res = to_vec(stream).await?; 105 | assert_eq!( 106 | res, 107 | vec![TagInfo { 108 | name: "a".into(), 109 | hash: Hash::new("a"), 110 | format: iroh_blobs::BlobFormat::HashSeq, 111 | }] 112 | ); 113 | 114 | tags.delete_all().await?; 115 | set(&tags, ["c"]).await?; 116 | tags.rename("c", "f").await?; 117 | let stream = tags.list().await?; 118 | let res = to_vec(stream).await?; 119 | assert_eq!( 120 | res, 121 | vec![TagInfo { 122 | name: "f".into(), 123 | hash: Hash::new("c"), 124 | format: iroh_blobs::BlobFormat::Raw, 125 | }] 126 | ); 127 | 128 | let res = tags.rename("y", "z").await; 129 | assert!(res.is_err()); 130 | Ok(()) 131 | } 132 | 133 | #[tokio::test] 134 | async fn tags_smoke_mem() -> TestResult<()> { 135 | let endpoint = Endpoint::builder().bind().await?; 136 | let blobs = Blobs::memory().build(&endpoint); 137 | let client = blobs.client(); 138 | tags_smoke(client.tags()).await 139 | } 140 | 141 | #[tokio::test] 142 | async fn tags_smoke_fs() -> TestResult<()> { 143 | let td = tempfile::tempdir()?; 144 | let endpoint = Endpoint::builder().bind().await?; 145 | let blobs = Blobs::persistent(td.path().join("blobs.db")) 146 | .await? 147 | .build(&endpoint); 148 | let client = blobs.client(); 149 | tags_smoke(client.tags()).await 150 | } 151 | --------------------------------------------------------------------------------