├── .github ├── ISSUE_TEMPLATE │ ├── config.yml │ └── open_an_issue.md ├── config.yml └── workflows │ ├── automerge.yml │ ├── go-check.yml │ ├── go-test.yml │ ├── release-check.yml │ ├── releaser.yml │ ├── stale.yml │ └── tagpush.yml ├── .gx └── lastpubver ├── LICENSE ├── README.md ├── arc_cache.go ├── arc_cache_test.go ├── blockstore.go ├── blockstore_test.go ├── bloom_cache.go ├── bloom_cache_test.go ├── caching.go ├── caching_test.go ├── go.mod ├── go.sum ├── idstore.go ├── idstore_test.go └── version.json /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: Getting Help on IPFS 4 | url: https://ipfs.io/help 5 | about: All information about how and where to get help on IPFS. 6 | - name: IPFS Official Forum 7 | url: https://discuss.ipfs.io 8 | about: Please post general questions, support requests, and discussions here. 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/open_an_issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Open an issue 3 | about: Only for actionable issues relevant to this repository. 4 | title: '' 5 | labels: need/triage 6 | assignees: '' 7 | 8 | --- 9 | 20 | -------------------------------------------------------------------------------- /.github/config.yml: -------------------------------------------------------------------------------- 1 | # Configuration for welcome - https://github.com/behaviorbot/welcome 2 | 3 | # Configuration for new-issue-welcome - https://github.com/behaviorbot/new-issue-welcome 4 | # Comment to be posted to on first time issues 5 | newIssueWelcomeComment: > 6 | Thank you for submitting your first issue to this repository! A maintainer 7 | will be here shortly to triage and review. 8 | 9 | In the meantime, please double-check that you have provided all the 10 | necessary information to make this process easy! Any information that can 11 | help save additional round trips is useful! We currently aim to give 12 | initial feedback within **two business days**. If this does not happen, feel 13 | free to leave a comment. 14 | 15 | Please keep an eye on how this issue will be labeled, as labels give an 16 | overview of priorities, assignments and additional actions requested by the 17 | maintainers: 18 | 19 | - "Priority" labels will show how urgent this is for the team. 20 | - "Status" labels will show if this is ready to be worked on, blocked, or in progress. 21 | - "Need" labels will indicate if additional input or analysis is required. 22 | 23 | Finally, remember to use https://discuss.ipfs.io if you just need general 24 | support. 25 | 26 | # Configuration for new-pr-welcome - https://github.com/behaviorbot/new-pr-welcome 27 | # Comment to be posted to on PRs from first time contributors in your repository 28 | newPRWelcomeComment: > 29 | Thank you for submitting this PR! 30 | 31 | A maintainer will be here shortly to review it. 32 | 33 | We are super grateful, but we are also overloaded! Help us by making sure 34 | that: 35 | 36 | * The context for this PR is clear, with relevant discussion, decisions 37 | and stakeholders linked/mentioned. 38 | 39 | * Your contribution itself is clear (code comments, self-review for the 40 | rest) and in its best form. Follow the [code contribution 41 | guidelines](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md#code-contribution-guidelines) 42 | if they apply. 43 | 44 | Getting other community members to do a review would be great help too on 45 | complex PRs (you can ask in the chats/forums). If you are unsure about 46 | something, just leave us a comment. 47 | 48 | Next steps: 49 | 50 | * A maintainer will triage and assign priority to this PR, commenting on 51 | any missing things and potentially assigning a reviewer for high 52 | priority items. 53 | 54 | * The PR gets reviews, discussed and approvals as needed. 55 | 56 | * The PR is merged by maintainers when it has been approved and comments addressed. 57 | 58 | We currently aim to provide initial feedback/triaging within **two business 59 | days**. Please keep an eye on any labelling actions, as these will indicate 60 | priorities and status of your contribution. 61 | 62 | We are very grateful for your contribution! 63 | 64 | 65 | # Configuration for first-pr-merge - https://github.com/behaviorbot/first-pr-merge 66 | # Comment to be posted to on pull requests merged by a first time user 67 | # Currently disabled 68 | #firstPRMergeComment: "" 69 | -------------------------------------------------------------------------------- /.github/workflows/automerge.yml: -------------------------------------------------------------------------------- 1 | # File managed by web3-bot. DO NOT EDIT. 2 | # See https://github.com/protocol/.github/ for details. 3 | 4 | name: Automerge 5 | on: [ pull_request ] 6 | 7 | jobs: 8 | automerge: 9 | uses: protocol/.github/.github/workflows/automerge.yml@master 10 | with: 11 | job: 'automerge' 12 | -------------------------------------------------------------------------------- /.github/workflows/go-check.yml: -------------------------------------------------------------------------------- 1 | # File managed by web3-bot. DO NOT EDIT. 2 | # See https://github.com/protocol/.github/ for details. 3 | 4 | on: [push, pull_request] 5 | name: Go Checks 6 | 7 | jobs: 8 | unit: 9 | runs-on: ubuntu-latest 10 | name: All 11 | steps: 12 | - uses: actions/checkout@v3 13 | with: 14 | submodules: recursive 15 | - id: config 16 | uses: protocol/.github/.github/actions/read-config@master 17 | - uses: actions/setup-go@v3 18 | with: 19 | go-version: 1.20.x 20 | - name: Run repo-specific setup 21 | uses: ./.github/actions/go-check-setup 22 | if: hashFiles('./.github/actions/go-check-setup') != '' 23 | - name: Install staticcheck 24 | run: go install honnef.co/go/tools/cmd/staticcheck@4970552d932f48b71485287748246cf3237cebdf # 2023.1 (v0.4.0) 25 | - name: Check that go.mod is tidy 26 | uses: protocol/multiple-go-modules@v1.2 27 | with: 28 | run: | 29 | go mod tidy 30 | if [[ -n $(git ls-files --other --exclude-standard --directory -- go.sum) ]]; then 31 | echo "go.sum was added by go mod tidy" 32 | exit 1 33 | fi 34 | git diff --exit-code -- go.sum go.mod 35 | - name: gofmt 36 | if: success() || failure() # run this step even if the previous one failed 37 | run: | 38 | out=$(gofmt -s -l .) 39 | if [[ -n "$out" ]]; then 40 | echo $out | awk '{print "::error file=" $0 ",line=0,col=0::File is not gofmt-ed."}' 41 | exit 1 42 | fi 43 | - name: go vet 44 | if: success() || failure() # run this step even if the previous one failed 45 | uses: protocol/multiple-go-modules@v1.2 46 | with: 47 | run: go vet ./... 48 | - name: staticcheck 49 | if: success() || failure() # run this step even if the previous one failed 50 | uses: protocol/multiple-go-modules@v1.2 51 | with: 52 | run: | 53 | set -o pipefail 54 | staticcheck ./... | sed -e 's@\(.*\)\.go@./\1.go@g' 55 | - name: go generate 56 | uses: protocol/multiple-go-modules@v1.2 57 | if: (success() || failure()) && fromJSON(steps.config.outputs.json).gogenerate == true 58 | with: 59 | run: | 60 | git clean -fd # make sure there aren't untracked files / directories 61 | go generate -x ./... 62 | # check if go generate modified or added any files 63 | if ! $(git add . && git diff-index HEAD --exit-code --quiet); then 64 | echo "go generated caused changes to the repository:" 65 | git status --short 66 | exit 1 67 | fi 68 | -------------------------------------------------------------------------------- /.github/workflows/go-test.yml: -------------------------------------------------------------------------------- 1 | # File managed by web3-bot. DO NOT EDIT. 2 | # See https://github.com/protocol/.github/ for details. 3 | 4 | on: [push, pull_request] 5 | name: Go Test 6 | 7 | jobs: 8 | unit: 9 | strategy: 10 | fail-fast: false 11 | matrix: 12 | os: [ "ubuntu", "windows", "macos" ] 13 | go: ["1.19.x","1.20.x"] 14 | env: 15 | COVERAGES: "" 16 | runs-on: ${{ fromJSON(vars[format('UCI_GO_TEST_RUNNER_{0}', matrix.os)] || format('"{0}-latest"', matrix.os)) }} 17 | name: ${{ matrix.os }} (go ${{ matrix.go }}) 18 | steps: 19 | - uses: actions/checkout@v3 20 | with: 21 | submodules: recursive 22 | - id: config 23 | uses: protocol/.github/.github/actions/read-config@master 24 | - uses: actions/setup-go@v3 25 | with: 26 | go-version: ${{ matrix.go }} 27 | - name: Go information 28 | run: | 29 | go version 30 | go env 31 | - name: Use msys2 on windows 32 | if: matrix.os == 'windows' 33 | shell: bash 34 | # The executable for msys2 is also called bash.cmd 35 | # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md#shells 36 | # If we prepend its location to the PATH 37 | # subsequent 'shell: bash' steps will use msys2 instead of gitbash 38 | run: echo "C:/msys64/usr/bin" >> $GITHUB_PATH 39 | - name: Run repo-specific setup 40 | uses: ./.github/actions/go-test-setup 41 | if: hashFiles('./.github/actions/go-test-setup') != '' 42 | - name: Run tests 43 | if: contains(fromJSON(steps.config.outputs.json).skipOSes, matrix.os) == false 44 | uses: protocol/multiple-go-modules@v1.2 45 | with: 46 | # Use -coverpkg=./..., so that we include cross-package coverage. 47 | # If package ./A imports ./B, and ./A's tests also cover ./B, 48 | # this means ./B's coverage will be significantly higher than 0%. 49 | run: go test -v -shuffle=on -coverprofile=module-coverage.txt -coverpkg=./... ./... 50 | - name: Run tests (32 bit) 51 | # can't run 32 bit tests on OSX. 52 | if: matrix.os != 'macos' && 53 | fromJSON(steps.config.outputs.json).skip32bit != true && 54 | contains(fromJSON(steps.config.outputs.json).skipOSes, matrix.os) == false 55 | uses: protocol/multiple-go-modules@v1.2 56 | env: 57 | GOARCH: 386 58 | with: 59 | run: | 60 | export "PATH=$PATH_386:$PATH" 61 | go test -v -shuffle=on ./... 62 | - name: Run tests with race detector 63 | # speed things up. Windows and OSX VMs are slow 64 | if: matrix.os == 'ubuntu' && 65 | contains(fromJSON(steps.config.outputs.json).skipOSes, matrix.os) == false 66 | uses: protocol/multiple-go-modules@v1.2 67 | with: 68 | run: go test -v -race ./... 69 | - name: Collect coverage files 70 | shell: bash 71 | run: echo "COVERAGES=$(find . -type f -name 'module-coverage.txt' | tr -s '\n' ',' | sed 's/,$//')" >> $GITHUB_ENV 72 | - name: Upload coverage to Codecov 73 | uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 74 | with: 75 | files: '${{ env.COVERAGES }}' 76 | env_vars: OS=${{ matrix.os }}, GO=${{ matrix.go }} 77 | -------------------------------------------------------------------------------- /.github/workflows/release-check.yml: -------------------------------------------------------------------------------- 1 | # File managed by web3-bot. DO NOT EDIT. 2 | # See https://github.com/protocol/.github/ for details. 3 | 4 | name: Release Checker 5 | on: 6 | pull_request_target: 7 | paths: [ 'version.json' ] 8 | 9 | jobs: 10 | release-check: 11 | uses: protocol/.github/.github/workflows/release-check.yml@master 12 | with: 13 | go-version: 1.20.x 14 | -------------------------------------------------------------------------------- /.github/workflows/releaser.yml: -------------------------------------------------------------------------------- 1 | # File managed by web3-bot. DO NOT EDIT. 2 | # See https://github.com/protocol/.github/ for details. 3 | 4 | name: Releaser 5 | on: 6 | push: 7 | paths: [ 'version.json' ] 8 | 9 | jobs: 10 | releaser: 11 | uses: protocol/.github/.github/workflows/releaser.yml@master 12 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Close and mark stale issue 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' 6 | 7 | jobs: 8 | stale: 9 | uses: pl-strflt/.github/.github/workflows/reusable-stale-issue.yml@v0.3 10 | -------------------------------------------------------------------------------- /.github/workflows/tagpush.yml: -------------------------------------------------------------------------------- 1 | # File managed by web3-bot. DO NOT EDIT. 2 | # See https://github.com/protocol/.github/ for details. 3 | 4 | name: Tag Push Checker 5 | on: 6 | push: 7 | tags: 8 | - v* 9 | 10 | jobs: 11 | releaser: 12 | uses: protocol/.github/.github/workflows/tagpush.yml@master 13 | -------------------------------------------------------------------------------- /.gx/lastpubver: -------------------------------------------------------------------------------- 1 | 0.1.8: QmXjKkjMDTtXAiLBwstVexofB8LeruZmE2eBd85GwGFFLA 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Protocol Labs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-ipfs-blockstore 2 | 3 | > go-ipfs-blockstore implements a thin wrapper over a datastore, giving a clean interface for Getting and Putting block objects. 4 | 5 | [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) 6 | [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) 7 | [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) 8 | [![GoDoc](https://godoc.org/github.com/ipfs/go-ipfs-blockstore?status.svg)](https://godoc.org/github.com/ipfs/go-ipfs-blockstore) 9 | [![Build Status](https://travis-ci.com/ipfs/go-ipfs-blockstore.svg?branch=master)](https://travis-ci.com/ipfs/go-ipfs-blockstore) 10 | 11 | ## ❗ This repo is no longer maintained. 12 | 👉 We highly recommend switching to the maintained version at https://github.com/ipfs/boxo/tree/main/blockstore. 13 | 🏎️ Good news! There is [tooling and documentation](https://github.com/ipfs/boxo#migrating-to-boxo) to expedite a switch in your repo. 14 | 15 | ⚠️ If you continue using this repo, please note that security fixes will not be provided (unless someone steps in to maintain it). 16 | 17 | 📚 Learn more, including how to take the maintainership mantle or ask questions, [here](https://github.com/ipfs/boxo/wiki/Copied-or-Migrated-Repos-FAQ). 18 | 19 | ## Table of Contents 20 | 21 | - [Install](#install) 22 | - [Usage](#usage) 23 | - [License](#license) 24 | 25 | ## Install 26 | 27 | `go-ipfs-blockstore` works like a regular Go module: 28 | 29 | ``` 30 | > go get github.com/ipfs/go-ipfs-blockstore 31 | ``` 32 | 33 | ## Usage 34 | 35 | ``` 36 | import "github.com/ipfs/go-ipfs-blockstore" 37 | ``` 38 | 39 | Check the [GoDoc documentation](https://godoc.org/github.com/ipfs/go-ipfs-blockstore) 40 | 41 | ## License 42 | 43 | MIT © Protocol Labs, Inc. 44 | -------------------------------------------------------------------------------- /arc_cache.go: -------------------------------------------------------------------------------- 1 | package blockstore 2 | 3 | import ( 4 | "context" 5 | "sort" 6 | "sync" 7 | 8 | lru "github.com/hashicorp/golang-lru" 9 | blocks "github.com/ipfs/go-block-format" 10 | cid "github.com/ipfs/go-cid" 11 | ipld "github.com/ipfs/go-ipld-format" 12 | metrics "github.com/ipfs/go-metrics-interface" 13 | ) 14 | 15 | type cacheHave bool 16 | type cacheSize int 17 | 18 | type lock struct { 19 | mu sync.RWMutex 20 | refcnt int 21 | } 22 | 23 | // arccache wraps a BlockStore with an Adaptive Replacement Cache (ARC) that 24 | // does not store the actual blocks, just metadata about them: existence and 25 | // size. This provides block access-time improvements, allowing 26 | // to short-cut many searches without querying the underlying datastore. 27 | type arccache struct { 28 | lklk sync.Mutex 29 | lks map[string]*lock 30 | 31 | cache *lru.TwoQueueCache 32 | 33 | blockstore Blockstore 34 | viewer Viewer 35 | 36 | hits metrics.Counter 37 | total metrics.Counter 38 | } 39 | 40 | var _ Blockstore = (*arccache)(nil) 41 | var _ Viewer = (*arccache)(nil) 42 | 43 | func newARCCachedBS(ctx context.Context, bs Blockstore, lruSize int) (*arccache, error) { 44 | cache, err := lru.New2Q(lruSize) 45 | if err != nil { 46 | return nil, err 47 | } 48 | c := &arccache{cache: cache, blockstore: bs, lks: make(map[string]*lock)} 49 | c.hits = metrics.NewCtx(ctx, "arc.hits_total", "Number of ARC cache hits").Counter() 50 | c.total = metrics.NewCtx(ctx, "arc_total", "Total number of ARC cache requests").Counter() 51 | if v, ok := bs.(Viewer); ok { 52 | c.viewer = v 53 | } 54 | return c, nil 55 | } 56 | 57 | func (b *arccache) lock(k string, write bool) { 58 | b.lklk.Lock() 59 | lk, ok := b.lks[k] 60 | if !ok { 61 | lk = new(lock) 62 | b.lks[k] = lk 63 | } 64 | lk.refcnt++ 65 | b.lklk.Unlock() 66 | if write { 67 | lk.mu.Lock() 68 | } else { 69 | lk.mu.RLock() 70 | } 71 | } 72 | 73 | func (b *arccache) unlock(key string, write bool) { 74 | b.lklk.Lock() 75 | lk := b.lks[key] 76 | lk.refcnt-- 77 | if lk.refcnt == 0 { 78 | delete(b.lks, key) 79 | } 80 | b.lklk.Unlock() 81 | if write { 82 | lk.mu.Unlock() 83 | } else { 84 | lk.mu.RUnlock() 85 | } 86 | } 87 | 88 | func cacheKey(k cid.Cid) string { 89 | return string(k.Hash()) 90 | } 91 | 92 | func (b *arccache) DeleteBlock(ctx context.Context, k cid.Cid) error { 93 | if !k.Defined() { 94 | return nil 95 | } 96 | 97 | key := cacheKey(k) 98 | 99 | if has, _, ok := b.queryCache(key); ok && !has { 100 | return nil 101 | } 102 | 103 | b.lock(key, true) 104 | defer b.unlock(key, true) 105 | 106 | err := b.blockstore.DeleteBlock(ctx, k) 107 | if err == nil { 108 | b.cacheHave(key, false) 109 | } else { 110 | b.cacheInvalidate(key) 111 | } 112 | return err 113 | } 114 | 115 | func (b *arccache) Has(ctx context.Context, k cid.Cid) (bool, error) { 116 | if !k.Defined() { 117 | logger.Error("undefined cid in arccache") 118 | // Return cache invalid so the call to blockstore happens 119 | // in case of invalid key and correct error is created. 120 | return false, nil 121 | } 122 | 123 | key := cacheKey(k) 124 | 125 | if has, _, ok := b.queryCache(key); ok { 126 | return has, nil 127 | } 128 | 129 | b.lock(key, false) 130 | defer b.unlock(key, false) 131 | 132 | has, err := b.blockstore.Has(ctx, k) 133 | if err != nil { 134 | return false, err 135 | } 136 | b.cacheHave(key, has) 137 | return has, nil 138 | } 139 | 140 | func (b *arccache) GetSize(ctx context.Context, k cid.Cid) (int, error) { 141 | if !k.Defined() { 142 | return -1, ipld.ErrNotFound{Cid: k} 143 | } 144 | 145 | key := cacheKey(k) 146 | 147 | if has, blockSize, ok := b.queryCache(key); ok { 148 | if !has { 149 | // don't have it, return 150 | return -1, ipld.ErrNotFound{Cid: k} 151 | } 152 | if blockSize >= 0 { 153 | // have it and we know the size 154 | return blockSize, nil 155 | } 156 | // we have it but don't know the size, ask the datastore. 157 | } 158 | 159 | b.lock(key, false) 160 | defer b.unlock(key, false) 161 | 162 | blockSize, err := b.blockstore.GetSize(ctx, k) 163 | if ipld.IsNotFound(err) { 164 | b.cacheHave(key, false) 165 | } else if err == nil { 166 | b.cacheSize(key, blockSize) 167 | } 168 | return blockSize, err 169 | } 170 | 171 | func (b *arccache) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error { 172 | // shortcircuit and fall back to Get if the underlying store 173 | // doesn't support Viewer. 174 | if b.viewer == nil { 175 | blk, err := b.Get(ctx, k) 176 | if err != nil { 177 | return err 178 | } 179 | return callback(blk.RawData()) 180 | } 181 | 182 | if !k.Defined() { 183 | return ipld.ErrNotFound{Cid: k} 184 | } 185 | 186 | key := cacheKey(k) 187 | 188 | if has, _, ok := b.queryCache(key); ok && !has { 189 | // short circuit if the cache deterministically tells us the item 190 | // doesn't exist. 191 | return ipld.ErrNotFound{Cid: k} 192 | } 193 | 194 | b.lock(key, false) 195 | defer b.unlock(key, false) 196 | 197 | var cberr error 198 | var size int 199 | if err := b.viewer.View(ctx, k, func(buf []byte) error { 200 | size = len(buf) 201 | cberr = callback(buf) 202 | return nil 203 | }); err != nil { 204 | if ipld.IsNotFound(err) { 205 | b.cacheHave(key, false) 206 | } 207 | return err 208 | } 209 | 210 | b.cacheSize(key, size) 211 | 212 | return cberr 213 | } 214 | 215 | func (b *arccache) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) { 216 | if !k.Defined() { 217 | return nil, ipld.ErrNotFound{Cid: k} 218 | } 219 | 220 | key := cacheKey(k) 221 | 222 | if has, _, ok := b.queryCache(key); ok && !has { 223 | return nil, ipld.ErrNotFound{Cid: k} 224 | } 225 | 226 | b.lock(key, false) 227 | defer b.unlock(key, false) 228 | 229 | bl, err := b.blockstore.Get(ctx, k) 230 | if bl == nil && ipld.IsNotFound(err) { 231 | b.cacheHave(key, false) 232 | } else if bl != nil { 233 | b.cacheSize(key, len(bl.RawData())) 234 | } 235 | return bl, err 236 | } 237 | 238 | func (b *arccache) Put(ctx context.Context, bl blocks.Block) error { 239 | key := cacheKey(bl.Cid()) 240 | 241 | if has, _, ok := b.queryCache(key); ok && has { 242 | return nil 243 | } 244 | 245 | b.lock(key, true) 246 | defer b.unlock(key, true) 247 | 248 | err := b.blockstore.Put(ctx, bl) 249 | if err == nil { 250 | b.cacheSize(key, len(bl.RawData())) 251 | } else { 252 | b.cacheInvalidate(key) 253 | } 254 | return err 255 | } 256 | 257 | type keyedBlocks struct { 258 | keys []string 259 | blocks []blocks.Block 260 | } 261 | 262 | func (b *keyedBlocks) Len() int { 263 | return len(b.keys) 264 | } 265 | 266 | func (b *keyedBlocks) Less(i, j int) bool { 267 | return b.keys[i] < b.keys[j] 268 | } 269 | 270 | func (b *keyedBlocks) Swap(i, j int) { 271 | b.keys[i], b.keys[j] = b.keys[j], b.keys[i] 272 | b.blocks[i], b.blocks[j] = b.blocks[j], b.blocks[i] 273 | } 274 | 275 | func (b *keyedBlocks) append(key string, blk blocks.Block) { 276 | b.keys = append(b.keys, key) 277 | b.blocks = append(b.blocks, blk) 278 | } 279 | 280 | func (b *keyedBlocks) isEmpty() bool { 281 | return len(b.keys) == 0 282 | } 283 | 284 | func (b *keyedBlocks) sortAndDedup() { 285 | if b.isEmpty() { 286 | return 287 | } 288 | 289 | sort.Sort(b) 290 | 291 | // https://github.com/golang/go/wiki/SliceTricks#in-place-deduplicate-comparable 292 | j := 0 293 | for i := 1; i < len(b.keys); i++ { 294 | if b.keys[j] == b.keys[i] { 295 | continue 296 | } 297 | j++ 298 | b.keys[j] = b.keys[i] 299 | b.blocks[j] = b.blocks[i] 300 | } 301 | 302 | b.keys = b.keys[:j+1] 303 | b.blocks = b.blocks[:j+1] 304 | } 305 | 306 | func newKeyedBlocks(cap int) *keyedBlocks { 307 | return &keyedBlocks{ 308 | keys: make([]string, 0, cap), 309 | blocks: make([]blocks.Block, 0, cap), 310 | } 311 | } 312 | 313 | func (b *arccache) PutMany(ctx context.Context, bs []blocks.Block) error { 314 | good := newKeyedBlocks(len(bs)) 315 | for _, blk := range bs { 316 | // call put on block if result is inconclusive or we are sure that 317 | // the block isn't in storage 318 | key := cacheKey(blk.Cid()) 319 | if has, _, ok := b.queryCache(key); !ok || (ok && !has) { 320 | good.append(key, blk) 321 | } 322 | } 323 | 324 | if good.isEmpty() { 325 | return nil 326 | } 327 | 328 | good.sortAndDedup() 329 | 330 | for _, key := range good.keys { 331 | b.lock(key, true) 332 | } 333 | 334 | defer func() { 335 | for _, key := range good.keys { 336 | b.unlock(key, true) 337 | } 338 | }() 339 | 340 | err := b.blockstore.PutMany(ctx, good.blocks) 341 | if err != nil { 342 | return err 343 | } 344 | for i, key := range good.keys { 345 | b.cacheSize(key, len(good.blocks[i].RawData())) 346 | } 347 | 348 | return nil 349 | } 350 | 351 | func (b *arccache) HashOnRead(enabled bool) { 352 | b.blockstore.HashOnRead(enabled) 353 | } 354 | 355 | func (b *arccache) cacheHave(key string, have bool) { 356 | b.cache.Add(key, cacheHave(have)) 357 | } 358 | 359 | func (b *arccache) cacheSize(key string, blockSize int) { 360 | b.cache.Add(key, cacheSize(blockSize)) 361 | } 362 | 363 | func (b *arccache) cacheInvalidate(key string) { 364 | b.cache.Remove(key) 365 | } 366 | 367 | // queryCache checks if the CID is in the cache. If so, it returns: 368 | // 369 | // - exists (bool): whether the CID is known to exist or not. 370 | // - size (int): the size if cached, or -1 if not cached. 371 | // - ok (bool): whether present in the cache. 372 | // 373 | // When ok is false, the answer in inconclusive and the caller must ignore the 374 | // other two return values. Querying the underying store is necessary. 375 | // 376 | // When ok is true, exists carries the correct answer, and size carries the 377 | // size, if known, or -1 if not. 378 | func (b *arccache) queryCache(k string) (exists bool, size int, ok bool) { 379 | b.total.Inc() 380 | 381 | h, ok := b.cache.Get(k) 382 | if ok { 383 | b.hits.Inc() 384 | switch h := h.(type) { 385 | case cacheHave: 386 | return bool(h), -1, true 387 | case cacheSize: 388 | return true, int(h), true 389 | } 390 | } 391 | return false, -1, false 392 | } 393 | 394 | func (b *arccache) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { 395 | return b.blockstore.AllKeysChan(ctx) 396 | } 397 | 398 | func (b *arccache) GCLock(ctx context.Context) Unlocker { 399 | return b.blockstore.(GCBlockstore).GCLock(ctx) 400 | } 401 | 402 | func (b *arccache) PinLock(ctx context.Context) Unlocker { 403 | return b.blockstore.(GCBlockstore).PinLock(ctx) 404 | } 405 | 406 | func (b *arccache) GCRequested(ctx context.Context) bool { 407 | return b.blockstore.(GCBlockstore).GCRequested(ctx) 408 | } 409 | -------------------------------------------------------------------------------- /arc_cache_test.go: -------------------------------------------------------------------------------- 1 | package blockstore 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "math/rand" 7 | "sync/atomic" 8 | "testing" 9 | "time" 10 | 11 | blocks "github.com/ipfs/go-block-format" 12 | cid "github.com/ipfs/go-cid" 13 | ds "github.com/ipfs/go-datastore" 14 | syncds "github.com/ipfs/go-datastore/sync" 15 | ipld "github.com/ipfs/go-ipld-format" 16 | ) 17 | 18 | var exampleBlock = blocks.NewBlock([]byte("foo")) 19 | 20 | func testArcCached(ctx context.Context, bs Blockstore) (*arccache, error) { 21 | if ctx == nil { 22 | ctx = context.TODO() 23 | } 24 | opts := DefaultCacheOpts() 25 | opts.HasBloomFilterSize = 0 26 | opts.HasBloomFilterHashes = 0 27 | bbs, err := CachedBlockstore(ctx, bs, opts) 28 | if err == nil { 29 | return bbs.(*arccache), nil 30 | } 31 | return nil, err 32 | } 33 | 34 | func createStores(t testing.TB) (*arccache, Blockstore, *callbackDatastore) { 35 | cd := &callbackDatastore{f: func() {}, ds: ds.NewMapDatastore()} 36 | bs := NewBlockstore(syncds.MutexWrap(cd)) 37 | arc, err := testArcCached(context.TODO(), bs) 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | return arc, bs, cd 42 | } 43 | 44 | func trap(message string, cd *callbackDatastore, t *testing.T) { 45 | cd.SetFunc(func() { 46 | t.Fatal(message) 47 | }) 48 | } 49 | func untrap(cd *callbackDatastore) { 50 | cd.SetFunc(func() {}) 51 | } 52 | 53 | func TestRemoveCacheEntryOnDelete(t *testing.T) { 54 | arc, _, cd := createStores(t) 55 | 56 | arc.Put(bg, exampleBlock) 57 | 58 | cd.Lock() 59 | writeHitTheDatastore := false 60 | cd.Unlock() 61 | 62 | cd.SetFunc(func() { 63 | writeHitTheDatastore = true 64 | }) 65 | 66 | arc.DeleteBlock(bg, exampleBlock.Cid()) 67 | arc.Put(bg, exampleBlock) 68 | if !writeHitTheDatastore { 69 | t.Fail() 70 | } 71 | } 72 | 73 | func TestElideDuplicateWrite(t *testing.T) { 74 | arc, _, cd := createStores(t) 75 | 76 | arc.Put(bg, exampleBlock) 77 | trap("write hit datastore", cd, t) 78 | arc.Put(bg, exampleBlock) 79 | } 80 | 81 | func TestHasRequestTriggersCache(t *testing.T) { 82 | arc, _, cd := createStores(t) 83 | 84 | arc.Has(bg, exampleBlock.Cid()) 85 | trap("has hit datastore", cd, t) 86 | if has, err := arc.Has(bg, exampleBlock.Cid()); has || err != nil { 87 | t.Fatal("has was true but there is no such block") 88 | } 89 | 90 | untrap(cd) 91 | err := arc.Put(bg, exampleBlock) 92 | if err != nil { 93 | t.Fatal(err) 94 | } 95 | 96 | trap("has hit datastore", cd, t) 97 | 98 | if has, err := arc.Has(bg, exampleBlock.Cid()); !has || err != nil { 99 | t.Fatal("has returned invalid result") 100 | } 101 | } 102 | 103 | func TestGetFillsCache(t *testing.T) { 104 | arc, _, cd := createStores(t) 105 | 106 | if bl, err := arc.Get(bg, exampleBlock.Cid()); bl != nil || err == nil { 107 | t.Fatal("block was found or there was no error") 108 | } 109 | 110 | trap("has hit datastore", cd, t) 111 | 112 | if has, err := arc.Has(bg, exampleBlock.Cid()); has || err != nil { 113 | t.Fatal("has was true but there is no such block") 114 | } 115 | if _, err := arc.GetSize(bg, exampleBlock.Cid()); !ipld.IsNotFound(err) { 116 | t.Fatal("getsize was true but there is no such block") 117 | } 118 | 119 | untrap(cd) 120 | 121 | if err := arc.Put(bg, exampleBlock); err != nil { 122 | t.Fatal(err) 123 | } 124 | 125 | trap("has hit datastore", cd, t) 126 | 127 | if has, err := arc.Has(bg, exampleBlock.Cid()); !has || err != nil { 128 | t.Fatal("has returned invalid result") 129 | } 130 | if blockSize, err := arc.GetSize(bg, exampleBlock.Cid()); blockSize == -1 || err != nil { 131 | t.Fatal("getsize returned invalid result", blockSize, err) 132 | } 133 | } 134 | 135 | func TestGetAndDeleteFalseShortCircuit(t *testing.T) { 136 | arc, _, cd := createStores(t) 137 | 138 | arc.Has(bg, exampleBlock.Cid()) 139 | arc.GetSize(bg, exampleBlock.Cid()) 140 | 141 | trap("get hit datastore", cd, t) 142 | 143 | if bl, err := arc.Get(bg, exampleBlock.Cid()); bl != nil || !ipld.IsNotFound(err) { 144 | t.Fatal("get returned invalid result") 145 | } 146 | 147 | if arc.DeleteBlock(bg, exampleBlock.Cid()) != nil { 148 | t.Fatal("expected deletes to be idempotent") 149 | } 150 | } 151 | 152 | func TestArcCreationFailure(t *testing.T) { 153 | if arc, err := newARCCachedBS(context.TODO(), nil, -1); arc != nil || err == nil { 154 | t.Fatal("expected error and no cache") 155 | } 156 | } 157 | 158 | func TestInvalidKey(t *testing.T) { 159 | arc, _, _ := createStores(t) 160 | 161 | bl, err := arc.Get(bg, cid.Cid{}) 162 | 163 | if bl != nil { 164 | t.Fatal("blocks should be nil") 165 | } 166 | if err == nil { 167 | t.Fatal("expected error") 168 | } 169 | } 170 | 171 | func TestHasAfterSucessfulGetIsCached(t *testing.T) { 172 | arc, bs, cd := createStores(t) 173 | 174 | bs.Put(bg, exampleBlock) 175 | 176 | arc.Get(bg, exampleBlock.Cid()) 177 | 178 | trap("has hit datastore", cd, t) 179 | arc.Has(bg, exampleBlock.Cid()) 180 | } 181 | 182 | func TestGetSizeAfterSucessfulGetIsCached(t *testing.T) { 183 | arc, bs, cd := createStores(t) 184 | 185 | bs.Put(bg, exampleBlock) 186 | 187 | arc.Get(bg, exampleBlock.Cid()) 188 | 189 | trap("has hit datastore", cd, t) 190 | arc.GetSize(bg, exampleBlock.Cid()) 191 | } 192 | 193 | func TestGetSizeAfterSucessfulHas(t *testing.T) { 194 | arc, bs, _ := createStores(t) 195 | 196 | bs.Put(bg, exampleBlock) 197 | has, err := arc.Has(bg, exampleBlock.Cid()) 198 | if err != nil { 199 | t.Fatal(err) 200 | } 201 | if !has { 202 | t.Fatal("expected to have block") 203 | } 204 | 205 | if size, err := arc.GetSize(bg, exampleBlock.Cid()); err != nil { 206 | t.Fatal(err) 207 | } else if size != len(exampleBlock.RawData()) { 208 | t.Fatalf("expected size %d, got %d", len(exampleBlock.RawData()), size) 209 | } 210 | } 211 | 212 | func TestGetSizeMissingZeroSizeBlock(t *testing.T) { 213 | arc, bs, cd := createStores(t) 214 | emptyBlock := blocks.NewBlock([]byte{}) 215 | missingBlock := blocks.NewBlock([]byte("missingBlock")) 216 | 217 | bs.Put(bg, emptyBlock) 218 | 219 | arc.Get(bg, emptyBlock.Cid()) 220 | 221 | trap("has hit datastore", cd, t) 222 | if blockSize, err := arc.GetSize(bg, emptyBlock.Cid()); blockSize != 0 || err != nil { 223 | t.Fatal("getsize returned invalid result") 224 | } 225 | untrap(cd) 226 | 227 | arc.Get(bg, missingBlock.Cid()) 228 | 229 | trap("has hit datastore", cd, t) 230 | if _, err := arc.GetSize(bg, missingBlock.Cid()); !ipld.IsNotFound(err) { 231 | t.Fatal("getsize returned invalid result") 232 | } 233 | } 234 | 235 | func TestDifferentKeyObjectsWork(t *testing.T) { 236 | arc, bs, cd := createStores(t) 237 | 238 | bs.Put(bg, exampleBlock) 239 | 240 | arc.Get(bg, exampleBlock.Cid()) 241 | 242 | trap("has hit datastore", cd, t) 243 | cidstr := exampleBlock.Cid().String() 244 | 245 | ncid, err := cid.Decode(cidstr) 246 | if err != nil { 247 | t.Fatal(err) 248 | } 249 | 250 | arc.Has(bg, ncid) 251 | } 252 | 253 | func TestPutManyCaches(t *testing.T) { 254 | t.Run("happy path PutMany", func(t *testing.T) { 255 | arc, _, cd := createStores(t) 256 | arc.PutMany(bg, []blocks.Block{exampleBlock}) 257 | 258 | trap("has hit datastore", cd, t) 259 | arc.Has(bg, exampleBlock.Cid()) 260 | arc.GetSize(bg, exampleBlock.Cid()) 261 | untrap(cd) 262 | arc.DeleteBlock(bg, exampleBlock.Cid()) 263 | 264 | arc.Put(bg, exampleBlock) 265 | trap("PunMany has hit datastore", cd, t) 266 | arc.PutMany(bg, []blocks.Block{exampleBlock}) 267 | }) 268 | 269 | t.Run("PutMany with duplicates", func(t *testing.T) { 270 | arc, _, cd := createStores(t) 271 | arc.PutMany(bg, []blocks.Block{exampleBlock, exampleBlock}) 272 | 273 | trap("has hit datastore", cd, t) 274 | arc.Has(bg, exampleBlock.Cid()) 275 | arc.GetSize(bg, exampleBlock.Cid()) 276 | untrap(cd) 277 | arc.DeleteBlock(bg, exampleBlock.Cid()) 278 | 279 | arc.Put(bg, exampleBlock) 280 | trap("PunMany has hit datastore", cd, t) 281 | arc.PutMany(bg, []blocks.Block{exampleBlock}) 282 | }) 283 | } 284 | 285 | func BenchmarkARCCacheConcurrentOps(b *testing.B) { 286 | // ~4k blocks seems high enough to be realistic, 287 | // but low enough to cause collisions. 288 | // Keep it as a power of 2, to simplify code below. 289 | const numBlocks = 4 << 10 290 | 291 | dummyBlocks := make([]blocks.Block, numBlocks) 292 | 293 | { 294 | // scope dummyRand to prevent its unsafe concurrent use below 295 | dummyRand := rand.New(rand.NewSource(time.Now().UnixNano())) 296 | for i := range dummyBlocks { 297 | dummy := make([]byte, 32) 298 | if _, err := io.ReadFull(dummyRand, dummy); err != nil { 299 | b.Fatal(err) 300 | } 301 | dummyBlocks[i] = blocks.NewBlock(dummy) 302 | } 303 | } 304 | 305 | // Each test begins with half the blocks present in the cache. 306 | // This allows test cases to have both hits and misses, 307 | // regardless of whether or not they do Puts. 308 | putHalfBlocks := func(arc *arccache) { 309 | for i, block := range dummyBlocks { 310 | if i%2 == 0 { 311 | if err := arc.Put(bg, block); err != nil { 312 | b.Fatal(err) 313 | } 314 | } 315 | } 316 | } 317 | 318 | // We always mix just two operations at a time. 319 | const numOps = 2 320 | var testOps = []struct { 321 | name string 322 | ops [numOps]func(*arccache, blocks.Block) 323 | }{ 324 | {"PutDelete", [...]func(*arccache, blocks.Block){ 325 | func(arc *arccache, block blocks.Block) { 326 | arc.Put(bg, block) 327 | }, 328 | func(arc *arccache, block blocks.Block) { 329 | arc.DeleteBlock(bg, block.Cid()) 330 | }, 331 | }}, 332 | {"GetDelete", [...]func(*arccache, blocks.Block){ 333 | func(arc *arccache, block blocks.Block) { 334 | arc.Get(bg, block.Cid()) 335 | }, 336 | func(arc *arccache, block blocks.Block) { 337 | arc.DeleteBlock(bg, block.Cid()) 338 | }, 339 | }}, 340 | {"GetPut", [...]func(*arccache, blocks.Block){ 341 | func(arc *arccache, block blocks.Block) { 342 | arc.Get(bg, block.Cid()) 343 | }, 344 | func(arc *arccache, block blocks.Block) { 345 | arc.Put(bg, block) 346 | }, 347 | }}, 348 | } 349 | 350 | for _, test := range testOps { 351 | test := test // prevent reuse of the range var 352 | b.Run(test.name, func(b *testing.B) { 353 | arc, _, _ := createStores(b) 354 | putHalfBlocks(arc) 355 | var opCounts [numOps]uint64 356 | 357 | b.ResetTimer() 358 | b.ReportAllocs() 359 | 360 | b.RunParallel(func(pb *testing.PB) { 361 | rnd := rand.New(rand.NewSource(time.Now().UnixNano())) 362 | for pb.Next() { 363 | n := rnd.Int63() 364 | blockIdx := n % numBlocks // lower bits decide the block 365 | opIdx := (n / numBlocks) % numOps // higher bits decide what operation 366 | 367 | block := dummyBlocks[blockIdx] 368 | op := test.ops[opIdx] 369 | op(arc, block) 370 | 371 | atomic.AddUint64(&opCounts[opIdx], 1) 372 | } 373 | }) 374 | 375 | // We expect each op to fire roughly an equal amount of times. 376 | // Error otherwise, as that likely means the logic is wrong. 377 | var minIdx, maxIdx int 378 | var minCount, maxCount uint64 379 | for opIdx, count := range opCounts { 380 | if minCount == 0 || count < minCount { 381 | minIdx = opIdx 382 | minCount = count 383 | } 384 | if maxCount == 0 || count > maxCount { 385 | maxIdx = opIdx 386 | maxCount = count 387 | } 388 | } 389 | // Skip this check if we ran few times, to avoid false positives. 390 | if maxCount > 100 { 391 | ratio := float64(maxCount) / float64(minCount) 392 | if maxRatio := 2.0; ratio > maxRatio { 393 | b.Fatalf("op %d ran %fx as many times as %d", maxIdx, ratio, minIdx) 394 | } 395 | } 396 | 397 | }) 398 | } 399 | } 400 | -------------------------------------------------------------------------------- /blockstore.go: -------------------------------------------------------------------------------- 1 | // Package blockstore implements a thin wrapper over a datastore, giving a 2 | // clean interface for Getting and Putting block objects. 3 | package blockstore 4 | 5 | import ( 6 | "context" 7 | "errors" 8 | "sync" 9 | "sync/atomic" 10 | 11 | blocks "github.com/ipfs/go-block-format" 12 | cid "github.com/ipfs/go-cid" 13 | ds "github.com/ipfs/go-datastore" 14 | dsns "github.com/ipfs/go-datastore/namespace" 15 | dsq "github.com/ipfs/go-datastore/query" 16 | dshelp "github.com/ipfs/go-ipfs-ds-help" 17 | ipld "github.com/ipfs/go-ipld-format" 18 | logging "github.com/ipfs/go-log" 19 | uatomic "go.uber.org/atomic" 20 | ) 21 | 22 | var logger = logging.Logger("blockstore") 23 | 24 | // BlockPrefix namespaces blockstore datastores 25 | // 26 | // Deprecated: use github.com/ipfs/boxo/blockstore.BlockPrefix 27 | var BlockPrefix = ds.NewKey("blocks") 28 | 29 | // ErrHashMismatch is an error returned when the hash of a block 30 | // is different than expected. 31 | // 32 | // Deprecated: use github.com/ipfs/boxo/blockstore.ErrHashMismatch 33 | var ErrHashMismatch = errors.New("block in storage has different hash than requested") 34 | 35 | // Blockstore wraps a Datastore block-centered methods and provides a layer 36 | // of abstraction which allows to add different caching strategies. 37 | // 38 | // Deprecated: use github.com/ipfs/boxo/blockstore.Blockstore 39 | type Blockstore interface { 40 | DeleteBlock(context.Context, cid.Cid) error 41 | Has(context.Context, cid.Cid) (bool, error) 42 | Get(context.Context, cid.Cid) (blocks.Block, error) 43 | 44 | // GetSize returns the CIDs mapped BlockSize 45 | GetSize(context.Context, cid.Cid) (int, error) 46 | 47 | // Put puts a given block to the underlying datastore 48 | Put(context.Context, blocks.Block) error 49 | 50 | // PutMany puts a slice of blocks at the same time using batching 51 | // capabilities of the underlying datastore whenever possible. 52 | PutMany(context.Context, []blocks.Block) error 53 | 54 | // AllKeysChan returns a channel from which 55 | // the CIDs in the Blockstore can be read. It should respect 56 | // the given context, closing the channel if it becomes Done. 57 | AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) 58 | 59 | // HashOnRead specifies if every read block should be 60 | // rehashed to make sure it matches its CID. 61 | HashOnRead(enabled bool) 62 | } 63 | 64 | // Viewer can be implemented by blockstores that offer zero-copy access to 65 | // values. 66 | // 67 | // Callers of View must not mutate or retain the byte slice, as it could be 68 | // an mmapped memory region, or a pooled byte buffer. 69 | // 70 | // View is especially suitable for deserialising in place. 71 | // 72 | // The callback will only be called iff the query operation is successful (and 73 | // the block is found); otherwise, the error will be propagated. Errors returned 74 | // by the callback will be propagated as well. 75 | // 76 | // Deprecated: use github.com/ipfs/boxo/blockstore.Viewer 77 | type Viewer interface { 78 | View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error 79 | } 80 | 81 | // GCLocker abstract functionality to lock a blockstore when performing 82 | // garbage-collection operations. 83 | // 84 | // Deprecated: use github.com/ipfs/boxo/blockstore.GCLocker 85 | type GCLocker interface { 86 | // GCLock locks the blockstore for garbage collection. No operations 87 | // that expect to finish with a pin should ocurr simultaneously. 88 | // Reading during GC is safe, and requires no lock. 89 | GCLock(context.Context) Unlocker 90 | 91 | // PinLock locks the blockstore for sequences of puts expected to finish 92 | // with a pin (before GC). Multiple put->pin sequences can write through 93 | // at the same time, but no GC should happen simulatenously. 94 | // Reading during Pinning is safe, and requires no lock. 95 | PinLock(context.Context) Unlocker 96 | 97 | // GcRequested returns true if GCLock has been called and is waiting to 98 | // take the lock 99 | GCRequested(context.Context) bool 100 | } 101 | 102 | // GCBlockstore is a blockstore that can safely run garbage-collection 103 | // operations. 104 | // 105 | // Deprecated: use github.com/ipfs/boxo/blockstore.GCBlockstore 106 | type GCBlockstore interface { 107 | Blockstore 108 | GCLocker 109 | } 110 | 111 | // NewGCBlockstore returns a default implementation of GCBlockstore 112 | // using the given Blockstore and GCLocker. 113 | // 114 | // Deprecated: use github.com/ipfs/boxo/blockstore.NewGCBlockstore 115 | func NewGCBlockstore(bs Blockstore, gcl GCLocker) GCBlockstore { 116 | return gcBlockstore{bs, gcl} 117 | } 118 | 119 | type gcBlockstore struct { 120 | Blockstore 121 | GCLocker 122 | } 123 | 124 | // Option is a default implementation Blockstore option 125 | // 126 | // Deprecated: use github.com/ipfs/boxo/blockstore.Option 127 | type Option struct { 128 | f func(bs *blockstore) 129 | } 130 | 131 | // WriteThrough skips checking if the blockstore already has a block before 132 | // writing it. 133 | // 134 | // Deprecated: use github.com/ipfs/boxo/blockstore.WriteThrough 135 | func WriteThrough() Option { 136 | return Option{ 137 | func(bs *blockstore) { 138 | bs.writeThrough = true 139 | }, 140 | } 141 | } 142 | 143 | // NoPrefix avoids wrapping the blockstore into the BlockPrefix namespace 144 | // ("/blocks"), so keys will not be modified in any way. 145 | // 146 | // Deprecated: use github.com/ipfs/boxo/blockstore.NoPrefix 147 | func NoPrefix() Option { 148 | return Option{ 149 | func(bs *blockstore) { 150 | bs.noPrefix = true 151 | }, 152 | } 153 | } 154 | 155 | // NewBlockstore returns a default Blockstore implementation 156 | // using the provided datastore.Batching backend. 157 | // 158 | // Deprecated: use github.com/ipfs/boxo/blockstore.NewBlockstore 159 | func NewBlockstore(d ds.Batching, opts ...Option) Blockstore { 160 | bs := &blockstore{ 161 | datastore: d, 162 | rehash: uatomic.NewBool(false), 163 | } 164 | 165 | for _, o := range opts { 166 | o.f(bs) 167 | } 168 | 169 | if !bs.noPrefix { 170 | bs.datastore = dsns.Wrap(bs.datastore, BlockPrefix) 171 | } 172 | return bs 173 | } 174 | 175 | // NewBlockstoreNoPrefix returns a default Blockstore implementation 176 | // using the provided datastore.Batching backend. 177 | // This constructor does not modify input keys in any way 178 | // 179 | // Deprecated: Use NewBlockstore with the NoPrefix option instead. 180 | // 181 | // Deprecated: use github.com/ipfs/boxo/blockstore.NewBlockstoreNoPrefix 182 | func NewBlockstoreNoPrefix(d ds.Batching) Blockstore { 183 | return NewBlockstore(d, NoPrefix()) 184 | } 185 | 186 | type blockstore struct { 187 | datastore ds.Batching 188 | 189 | rehash *uatomic.Bool 190 | writeThrough bool 191 | noPrefix bool 192 | } 193 | 194 | func (bs *blockstore) HashOnRead(enabled bool) { 195 | bs.rehash.Store(enabled) 196 | } 197 | 198 | func (bs *blockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) { 199 | if !k.Defined() { 200 | logger.Error("undefined cid in blockstore") 201 | return nil, ipld.ErrNotFound{Cid: k} 202 | } 203 | bdata, err := bs.datastore.Get(ctx, dshelp.MultihashToDsKey(k.Hash())) 204 | if err == ds.ErrNotFound { 205 | return nil, ipld.ErrNotFound{Cid: k} 206 | } 207 | if err != nil { 208 | return nil, err 209 | } 210 | if bs.rehash.Load() { 211 | rbcid, err := k.Prefix().Sum(bdata) 212 | if err != nil { 213 | return nil, err 214 | } 215 | 216 | if !rbcid.Equals(k) { 217 | return nil, ErrHashMismatch 218 | } 219 | 220 | return blocks.NewBlockWithCid(bdata, rbcid) 221 | } 222 | return blocks.NewBlockWithCid(bdata, k) 223 | } 224 | 225 | func (bs *blockstore) Put(ctx context.Context, block blocks.Block) error { 226 | k := dshelp.MultihashToDsKey(block.Cid().Hash()) 227 | 228 | // Has is cheaper than Put, so see if we already have it 229 | if !bs.writeThrough { 230 | exists, err := bs.datastore.Has(ctx, k) 231 | if err == nil && exists { 232 | return nil // already stored. 233 | } 234 | } 235 | return bs.datastore.Put(ctx, k, block.RawData()) 236 | } 237 | 238 | func (bs *blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error { 239 | if len(blocks) == 1 { 240 | // performance fast-path 241 | return bs.Put(ctx, blocks[0]) 242 | } 243 | 244 | t, err := bs.datastore.Batch(ctx) 245 | if err != nil { 246 | return err 247 | } 248 | for _, b := range blocks { 249 | k := dshelp.MultihashToDsKey(b.Cid().Hash()) 250 | 251 | if !bs.writeThrough { 252 | exists, err := bs.datastore.Has(ctx, k) 253 | if err == nil && exists { 254 | continue 255 | } 256 | } 257 | 258 | err = t.Put(ctx, k, b.RawData()) 259 | if err != nil { 260 | return err 261 | } 262 | } 263 | return t.Commit(ctx) 264 | } 265 | 266 | func (bs *blockstore) Has(ctx context.Context, k cid.Cid) (bool, error) { 267 | return bs.datastore.Has(ctx, dshelp.MultihashToDsKey(k.Hash())) 268 | } 269 | 270 | func (bs *blockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) { 271 | size, err := bs.datastore.GetSize(ctx, dshelp.MultihashToDsKey(k.Hash())) 272 | if err == ds.ErrNotFound { 273 | return -1, ipld.ErrNotFound{Cid: k} 274 | } 275 | return size, err 276 | } 277 | 278 | func (bs *blockstore) DeleteBlock(ctx context.Context, k cid.Cid) error { 279 | return bs.datastore.Delete(ctx, dshelp.MultihashToDsKey(k.Hash())) 280 | } 281 | 282 | // AllKeysChan runs a query for keys from the blockstore. 283 | // this is very simplistic, in the future, take dsq.Query as a param? 284 | // 285 | // AllKeysChan respects context. 286 | func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { 287 | 288 | // KeysOnly, because that would be _a lot_ of data. 289 | q := dsq.Query{KeysOnly: true} 290 | res, err := bs.datastore.Query(ctx, q) 291 | if err != nil { 292 | return nil, err 293 | } 294 | 295 | output := make(chan cid.Cid, dsq.KeysOnlyBufSize) 296 | go func() { 297 | defer func() { 298 | res.Close() // ensure exit (signals early exit, too) 299 | close(output) 300 | }() 301 | 302 | for { 303 | e, ok := res.NextSync() 304 | if !ok { 305 | return 306 | } 307 | if e.Error != nil { 308 | logger.Errorf("blockstore.AllKeysChan got err: %s", e.Error) 309 | return 310 | } 311 | 312 | // need to convert to key.Key using key.KeyFromDsKey. 313 | bk, err := dshelp.BinaryFromDsKey(ds.RawKey(e.Key)) 314 | if err != nil { 315 | logger.Warningf("error parsing key from binary: %s", err) 316 | continue 317 | } 318 | k := cid.NewCidV1(cid.Raw, bk) 319 | select { 320 | case <-ctx.Done(): 321 | return 322 | case output <- k: 323 | } 324 | } 325 | }() 326 | 327 | return output, nil 328 | } 329 | 330 | // NewGCLocker returns a default implementation of 331 | // GCLocker using standard [RW] mutexes. 332 | // 333 | // Deprecated: use github.com/ipfs/boxo/blockstore.NewGCLocker 334 | func NewGCLocker() GCLocker { 335 | return &gclocker{} 336 | } 337 | 338 | type gclocker struct { 339 | lk sync.RWMutex 340 | gcreq int32 341 | } 342 | 343 | // Unlocker represents an object which can Unlock 344 | // something. 345 | // 346 | // Deprecated: use github.com/ipfs/boxo/blockstore.Unlocker 347 | type Unlocker interface { 348 | Unlock(context.Context) 349 | } 350 | 351 | type unlocker struct { 352 | unlock func() 353 | } 354 | 355 | func (u *unlocker) Unlock(_ context.Context) { 356 | u.unlock() 357 | u.unlock = nil // ensure its not called twice 358 | } 359 | 360 | func (bs *gclocker) GCLock(_ context.Context) Unlocker { 361 | atomic.AddInt32(&bs.gcreq, 1) 362 | bs.lk.Lock() 363 | atomic.AddInt32(&bs.gcreq, -1) 364 | return &unlocker{bs.lk.Unlock} 365 | } 366 | 367 | func (bs *gclocker) PinLock(_ context.Context) Unlocker { 368 | bs.lk.RLock() 369 | return &unlocker{bs.lk.RUnlock} 370 | } 371 | 372 | func (bs *gclocker) GCRequested(_ context.Context) bool { 373 | return atomic.LoadInt32(&bs.gcreq) > 0 374 | } 375 | -------------------------------------------------------------------------------- /blockstore_test.go: -------------------------------------------------------------------------------- 1 | package blockstore 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "testing" 8 | 9 | blocks "github.com/ipfs/go-block-format" 10 | cid "github.com/ipfs/go-cid" 11 | ds "github.com/ipfs/go-datastore" 12 | dsq "github.com/ipfs/go-datastore/query" 13 | ds_sync "github.com/ipfs/go-datastore/sync" 14 | u "github.com/ipfs/go-ipfs-util" 15 | ipld "github.com/ipfs/go-ipld-format" 16 | ) 17 | 18 | func TestGetWhenKeyNotPresent(t *testing.T) { 19 | bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) 20 | c := cid.NewCidV0(u.Hash([]byte("stuff"))) 21 | bl, err := bs.Get(bg, c) 22 | 23 | if bl != nil { 24 | t.Error("nil block expected") 25 | } 26 | if err == nil { 27 | t.Error("error expected, got nil") 28 | } 29 | } 30 | 31 | func TestGetWhenKeyIsNil(t *testing.T) { 32 | bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) 33 | _, err := bs.Get(bg, cid.Cid{}) 34 | if !ipld.IsNotFound(err) { 35 | t.Fail() 36 | } 37 | } 38 | 39 | func TestPutThenGetBlock(t *testing.T) { 40 | bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) 41 | block := blocks.NewBlock([]byte("some data")) 42 | 43 | err := bs.Put(bg, block) 44 | if err != nil { 45 | t.Fatal(err) 46 | } 47 | 48 | blockFromBlockstore, err := bs.Get(bg, block.Cid()) 49 | if err != nil { 50 | t.Fatal(err) 51 | } 52 | if !bytes.Equal(block.RawData(), blockFromBlockstore.RawData()) { 53 | t.Fail() 54 | } 55 | } 56 | 57 | func TestCidv0v1(t *testing.T) { 58 | bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) 59 | block := blocks.NewBlock([]byte("some data")) 60 | 61 | err := bs.Put(bg, block) 62 | if err != nil { 63 | t.Fatal(err) 64 | } 65 | 66 | blockFromBlockstore, err := bs.Get(bg, cid.NewCidV1(cid.DagProtobuf, block.Cid().Hash())) 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | if !bytes.Equal(block.RawData(), blockFromBlockstore.RawData()) { 71 | t.Fail() 72 | } 73 | } 74 | 75 | func TestPutThenGetSizeBlock(t *testing.T) { 76 | bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) 77 | block := blocks.NewBlock([]byte("some data")) 78 | missingBlock := blocks.NewBlock([]byte("missingBlock")) 79 | emptyBlock := blocks.NewBlock([]byte{}) 80 | 81 | err := bs.Put(bg, block) 82 | if err != nil { 83 | t.Fatal(err) 84 | } 85 | 86 | blockSize, err := bs.GetSize(bg, block.Cid()) 87 | if err != nil { 88 | t.Fatal(err) 89 | } 90 | if len(block.RawData()) != blockSize { 91 | t.Fail() 92 | } 93 | 94 | err = bs.Put(bg, emptyBlock) 95 | if err != nil { 96 | t.Fatal(err) 97 | } 98 | 99 | if blockSize, err := bs.GetSize(bg, emptyBlock.Cid()); blockSize != 0 || err != nil { 100 | t.Fatal(err) 101 | } 102 | 103 | if blockSize, err := bs.GetSize(bg, missingBlock.Cid()); blockSize != -1 || err == nil { 104 | t.Fatal("getsize returned invalid result") 105 | } 106 | } 107 | 108 | type countHasDS struct { 109 | ds.Datastore 110 | hasCount int 111 | } 112 | 113 | func (ds *countHasDS) Has(ctx context.Context, key ds.Key) (exists bool, err error) { 114 | ds.hasCount += 1 115 | return ds.Datastore.Has(ctx, key) 116 | } 117 | 118 | func TestPutUsesHas(t *testing.T) { 119 | // Some datastores rely on the implementation detail that Put checks Has 120 | // first, to avoid overriding existing objects' metadata. This test ensures 121 | // that Blockstore continues to behave this way. 122 | // Please ping https://github.com/ipfs/go-ipfs-blockstore/pull/47 if this 123 | // behavior is being removed. 124 | ds := &countHasDS{ 125 | Datastore: ds.NewMapDatastore(), 126 | } 127 | bs := NewBlockstore(ds_sync.MutexWrap(ds)) 128 | bl := blocks.NewBlock([]byte("some data")) 129 | if err := bs.Put(bg, bl); err != nil { 130 | t.Fatal(err) 131 | } 132 | if err := bs.Put(bg, bl); err != nil { 133 | t.Fatal(err) 134 | } 135 | if ds.hasCount != 2 { 136 | t.Errorf("Blockstore did not call Has before attempting Put, this breaks compatibility") 137 | } 138 | } 139 | 140 | func TestHashOnRead(t *testing.T) { 141 | orginalDebug := u.Debug 142 | defer (func() { 143 | u.Debug = orginalDebug 144 | })() 145 | u.Debug = false 146 | 147 | bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) 148 | bl := blocks.NewBlock([]byte("some data")) 149 | blBad, err := blocks.NewBlockWithCid([]byte("some other data"), bl.Cid()) 150 | if err != nil { 151 | t.Fatal("debug is off, still got an error") 152 | } 153 | bl2 := blocks.NewBlock([]byte("some other data")) 154 | bs.Put(bg, blBad) 155 | bs.Put(bg, bl2) 156 | bs.HashOnRead(true) 157 | 158 | if _, err := bs.Get(bg, bl.Cid()); err != ErrHashMismatch { 159 | t.Fatalf("expected '%v' got '%v'\n", ErrHashMismatch, err) 160 | } 161 | 162 | if b, err := bs.Get(bg, bl2.Cid()); err != nil || b.String() != bl2.String() { 163 | t.Fatal("got wrong blocks") 164 | } 165 | } 166 | 167 | func newBlockStoreWithKeys(t *testing.T, d ds.Datastore, N int) (Blockstore, []cid.Cid) { 168 | if d == nil { 169 | d = ds.NewMapDatastore() 170 | } 171 | bs := NewBlockstore(ds_sync.MutexWrap(d)) 172 | 173 | keys := make([]cid.Cid, N) 174 | for i := 0; i < N; i++ { 175 | block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i))) 176 | err := bs.Put(bg, block) 177 | if err != nil { 178 | t.Fatal(err) 179 | } 180 | keys[i] = block.Cid() 181 | } 182 | return bs, keys 183 | } 184 | 185 | func collect(ch <-chan cid.Cid) []cid.Cid { 186 | var keys []cid.Cid 187 | for k := range ch { 188 | keys = append(keys, k) 189 | } 190 | return keys 191 | } 192 | 193 | func TestAllKeysSimple(t *testing.T) { 194 | bs, keys := newBlockStoreWithKeys(t, nil, 100) 195 | 196 | ctx := context.Background() 197 | ch, err := bs.AllKeysChan(ctx) 198 | if err != nil { 199 | t.Fatal(err) 200 | } 201 | keys2 := collect(ch) 202 | 203 | // for _, k2 := range keys2 { 204 | // t.Log("found ", k2.B58String()) 205 | // } 206 | 207 | expectMatches(t, keys, keys2) 208 | } 209 | 210 | func TestAllKeysRespectsContext(t *testing.T) { 211 | N := 100 212 | 213 | d := &queryTestDS{ds: ds.NewMapDatastore()} 214 | bs, _ := newBlockStoreWithKeys(t, d, N) 215 | 216 | started := make(chan struct{}, 1) 217 | done := make(chan struct{}, 1) 218 | errors := make(chan error, 100) 219 | 220 | getKeys := func(ctx context.Context) { 221 | started <- struct{}{} 222 | ch, err := bs.AllKeysChan(ctx) // once without cancelling 223 | if err != nil { 224 | errors <- err 225 | } 226 | _ = collect(ch) 227 | done <- struct{}{} 228 | errors <- nil // a nil one to signal break 229 | } 230 | 231 | var results dsq.Results 232 | var resultsmu = make(chan struct{}) 233 | resultChan := make(chan dsq.Result) 234 | d.SetFunc(func(q dsq.Query) (dsq.Results, error) { 235 | results = dsq.ResultsWithChan(q, resultChan) 236 | resultsmu <- struct{}{} 237 | return results, nil 238 | }) 239 | 240 | go getKeys(context.Background()) 241 | 242 | // make sure it's waiting. 243 | <-started 244 | <-resultsmu 245 | select { 246 | case <-done: 247 | t.Fatal("sync is wrong") 248 | case <-results.Process().Closing(): 249 | t.Fatal("should not be closing") 250 | case <-results.Process().Closed(): 251 | t.Fatal("should not be closed") 252 | default: 253 | } 254 | 255 | e := dsq.Entry{Key: BlockPrefix.ChildString("foo").String()} 256 | resultChan <- dsq.Result{Entry: e} // let it go. 257 | close(resultChan) 258 | <-done // should be done now. 259 | <-results.Process().Closed() // should be closed now 260 | 261 | // print any errors 262 | for err := range errors { 263 | if err == nil { 264 | break 265 | } 266 | t.Error(err) 267 | } 268 | 269 | } 270 | 271 | func expectMatches(t *testing.T, expect, actual []cid.Cid) { 272 | t.Helper() 273 | 274 | if len(expect) != len(actual) { 275 | t.Errorf("expect and actual differ: %d != %d", len(expect), len(actual)) 276 | } 277 | 278 | actualSet := make(map[string]bool, len(actual)) 279 | for _, k := range actual { 280 | actualSet[string(k.Hash())] = true 281 | } 282 | 283 | for _, ek := range expect { 284 | if !actualSet[string(ek.Hash())] { 285 | t.Error("expected key not found: ", ek) 286 | } 287 | } 288 | } 289 | 290 | type queryTestDS struct { 291 | cb func(q dsq.Query) (dsq.Results, error) 292 | ds ds.Datastore 293 | } 294 | 295 | func (c *queryTestDS) SetFunc(f func(dsq.Query) (dsq.Results, error)) { c.cb = f } 296 | 297 | func (c *queryTestDS) Put(ctx context.Context, key ds.Key, value []byte) (err error) { 298 | return c.ds.Put(ctx, key, value) 299 | } 300 | 301 | func (c *queryTestDS) Get(ctx context.Context, key ds.Key) (value []byte, err error) { 302 | return c.ds.Get(ctx, key) 303 | } 304 | 305 | func (c *queryTestDS) Has(ctx context.Context, key ds.Key) (exists bool, err error) { 306 | return c.ds.Has(ctx, key) 307 | } 308 | 309 | func (c *queryTestDS) GetSize(ctx context.Context, key ds.Key) (size int, err error) { 310 | return c.ds.GetSize(ctx, key) 311 | } 312 | 313 | func (c *queryTestDS) Delete(ctx context.Context, key ds.Key) (err error) { 314 | return c.ds.Delete(ctx, key) 315 | } 316 | 317 | func (c *queryTestDS) Query(ctx context.Context, q dsq.Query) (dsq.Results, error) { 318 | if c.cb != nil { 319 | return c.cb(q) 320 | } 321 | return c.ds.Query(ctx, q) 322 | } 323 | 324 | func (c *queryTestDS) Sync(ctx context.Context, key ds.Key) error { 325 | return c.ds.Sync(ctx, key) 326 | } 327 | 328 | func (c *queryTestDS) Batch(_ context.Context) (ds.Batch, error) { 329 | return ds.NewBasicBatch(c), nil 330 | } 331 | func (c *queryTestDS) Close() error { 332 | return nil 333 | } 334 | -------------------------------------------------------------------------------- /bloom_cache.go: -------------------------------------------------------------------------------- 1 | package blockstore 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync/atomic" 7 | "time" 8 | 9 | bloom "github.com/ipfs/bbloom" 10 | blocks "github.com/ipfs/go-block-format" 11 | cid "github.com/ipfs/go-cid" 12 | ipld "github.com/ipfs/go-ipld-format" 13 | metrics "github.com/ipfs/go-metrics-interface" 14 | ) 15 | 16 | // bloomCached returns a Blockstore that caches Has requests using a Bloom 17 | // filter. bloomSize is size of bloom filter in bytes. hashCount specifies the 18 | // number of hashing functions in the bloom filter (usually known as k). 19 | func bloomCached(ctx context.Context, bs Blockstore, bloomSize, hashCount int) (*bloomcache, error) { 20 | bl, err := bloom.New(float64(bloomSize), float64(hashCount)) 21 | if err != nil { 22 | return nil, err 23 | } 24 | bc := &bloomcache{ 25 | blockstore: bs, 26 | bloom: bl, 27 | hits: metrics.NewCtx(ctx, "bloom.hits_total", 28 | "Number of cache hits in bloom cache").Counter(), 29 | total: metrics.NewCtx(ctx, "bloom_total", 30 | "Total number of requests to bloom cache").Counter(), 31 | buildChan: make(chan struct{}), 32 | } 33 | if v, ok := bs.(Viewer); ok { 34 | bc.viewer = v 35 | } 36 | go func() { 37 | err := bc.build(ctx) 38 | if err != nil { 39 | select { 40 | case <-ctx.Done(): 41 | logger.Warning("Cache rebuild closed by context finishing: ", err) 42 | default: 43 | logger.Error(err) 44 | } 45 | return 46 | } 47 | if metrics.Active() { 48 | fill := metrics.NewCtx(ctx, "bloom_fill_ratio", 49 | "Ratio of bloom filter fullnes, (updated once a minute)").Gauge() 50 | 51 | t := time.NewTicker(1 * time.Minute) 52 | defer t.Stop() 53 | for { 54 | select { 55 | case <-ctx.Done(): 56 | return 57 | case <-t.C: 58 | fill.Set(bc.bloom.FillRatioTS()) 59 | } 60 | } 61 | } 62 | }() 63 | return bc, nil 64 | } 65 | 66 | type bloomcache struct { 67 | active int32 68 | 69 | bloom *bloom.Bloom 70 | buildErr error 71 | 72 | buildChan chan struct{} 73 | blockstore Blockstore 74 | viewer Viewer 75 | 76 | // Statistics 77 | hits metrics.Counter 78 | total metrics.Counter 79 | } 80 | 81 | var _ Blockstore = (*bloomcache)(nil) 82 | var _ Viewer = (*bloomcache)(nil) 83 | 84 | func (b *bloomcache) BloomActive() bool { 85 | return atomic.LoadInt32(&b.active) != 0 86 | } 87 | 88 | func (b *bloomcache) Wait(ctx context.Context) error { 89 | select { 90 | case <-ctx.Done(): 91 | return ctx.Err() 92 | case <-b.buildChan: 93 | return b.buildErr 94 | } 95 | } 96 | 97 | func (b *bloomcache) build(ctx context.Context) error { 98 | evt := logger.EventBegin(ctx, "bloomcache.build") 99 | defer evt.Done() 100 | defer close(b.buildChan) 101 | 102 | ch, err := b.blockstore.AllKeysChan(ctx) 103 | if err != nil { 104 | b.buildErr = fmt.Errorf("AllKeysChan failed in bloomcache rebuild with: %v", err) 105 | return b.buildErr 106 | } 107 | for { 108 | select { 109 | case key, ok := <-ch: 110 | if !ok { 111 | atomic.StoreInt32(&b.active, 1) 112 | return nil 113 | } 114 | b.bloom.AddTS(key.Hash()) // Use binary key, the more compact the better 115 | case <-ctx.Done(): 116 | b.buildErr = ctx.Err() 117 | return b.buildErr 118 | } 119 | } 120 | } 121 | 122 | func (b *bloomcache) DeleteBlock(ctx context.Context, k cid.Cid) error { 123 | if has, ok := b.hasCached(k); ok && !has { 124 | return nil 125 | } 126 | 127 | return b.blockstore.DeleteBlock(ctx, k) 128 | } 129 | 130 | // if ok == false has is inconclusive 131 | // if ok == true then has respons to question: is it contained 132 | func (b *bloomcache) hasCached(k cid.Cid) (has bool, ok bool) { 133 | b.total.Inc() 134 | if !k.Defined() { 135 | logger.Error("undefined in bloom cache") 136 | // Return cache invalid so call to blockstore 137 | // in case of invalid key is forwarded deeper 138 | return false, false 139 | } 140 | if b.BloomActive() { 141 | blr := b.bloom.HasTS(k.Hash()) 142 | if !blr { // not contained in bloom is only conclusive answer bloom gives 143 | b.hits.Inc() 144 | return false, true 145 | } 146 | } 147 | return false, false 148 | } 149 | 150 | func (b *bloomcache) Has(ctx context.Context, k cid.Cid) (bool, error) { 151 | if has, ok := b.hasCached(k); ok { 152 | return has, nil 153 | } 154 | 155 | return b.blockstore.Has(ctx, k) 156 | } 157 | 158 | func (b *bloomcache) GetSize(ctx context.Context, k cid.Cid) (int, error) { 159 | if has, ok := b.hasCached(k); ok && !has { 160 | return -1, ipld.ErrNotFound{Cid: k} 161 | } 162 | 163 | return b.blockstore.GetSize(ctx, k) 164 | } 165 | 166 | func (b *bloomcache) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error { 167 | if b.viewer == nil { 168 | blk, err := b.Get(ctx, k) 169 | if err != nil { 170 | return err 171 | } 172 | return callback(blk.RawData()) 173 | } 174 | 175 | if has, ok := b.hasCached(k); ok && !has { 176 | return ipld.ErrNotFound{Cid: k} 177 | } 178 | return b.viewer.View(ctx, k, callback) 179 | } 180 | 181 | func (b *bloomcache) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) { 182 | if has, ok := b.hasCached(k); ok && !has { 183 | return nil, ipld.ErrNotFound{Cid: k} 184 | } 185 | 186 | return b.blockstore.Get(ctx, k) 187 | } 188 | 189 | func (b *bloomcache) Put(ctx context.Context, bl blocks.Block) error { 190 | // See comment in PutMany 191 | err := b.blockstore.Put(ctx, bl) 192 | if err == nil { 193 | b.bloom.AddTS(bl.Cid().Hash()) 194 | } 195 | return err 196 | } 197 | 198 | func (b *bloomcache) PutMany(ctx context.Context, bs []blocks.Block) error { 199 | // bloom cache gives only conclusive resulty if key is not contained 200 | // to reduce number of puts we need conclusive information if block is contained 201 | // this means that PutMany can't be improved with bloom cache so we just 202 | // just do a passthrough. 203 | err := b.blockstore.PutMany(ctx, bs) 204 | if err != nil { 205 | return err 206 | } 207 | for _, bl := range bs { 208 | b.bloom.AddTS(bl.Cid().Hash()) 209 | } 210 | return nil 211 | } 212 | 213 | func (b *bloomcache) HashOnRead(enabled bool) { 214 | b.blockstore.HashOnRead(enabled) 215 | } 216 | 217 | func (b *bloomcache) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { 218 | return b.blockstore.AllKeysChan(ctx) 219 | } 220 | 221 | func (b *bloomcache) GCLock(ctx context.Context) Unlocker { 222 | return b.blockstore.(GCBlockstore).GCLock(ctx) 223 | } 224 | 225 | func (b *bloomcache) PinLock(ctx context.Context) Unlocker { 226 | return b.blockstore.(GCBlockstore).PinLock(ctx) 227 | } 228 | 229 | func (b *bloomcache) GCRequested(ctx context.Context) bool { 230 | return b.blockstore.(GCBlockstore).GCRequested(ctx) 231 | } 232 | -------------------------------------------------------------------------------- /bloom_cache_test.go: -------------------------------------------------------------------------------- 1 | package blockstore 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "testing" 8 | "time" 9 | 10 | blocks "github.com/ipfs/go-block-format" 11 | ds "github.com/ipfs/go-datastore" 12 | dsq "github.com/ipfs/go-datastore/query" 13 | syncds "github.com/ipfs/go-datastore/sync" 14 | ipld "github.com/ipfs/go-ipld-format" 15 | ) 16 | 17 | var bg = context.Background() 18 | 19 | func testBloomCached(ctx context.Context, bs Blockstore) (*bloomcache, error) { 20 | if ctx == nil { 21 | ctx = context.Background() 22 | } 23 | opts := DefaultCacheOpts() 24 | opts.HasARCCacheSize = 0 25 | bbs, err := CachedBlockstore(ctx, bs, opts) 26 | if err == nil { 27 | return bbs.(*bloomcache), nil 28 | } 29 | return nil, err 30 | } 31 | 32 | func TestPutManyAddsToBloom(t *testing.T) { 33 | bs := NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore())) 34 | 35 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) 36 | defer cancel() 37 | 38 | cachedbs, err := testBloomCached(ctx, bs) 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | 43 | if err := cachedbs.Wait(ctx); err != nil { 44 | t.Fatalf("Failed while waiting for the filter to build: %d", cachedbs.bloom.ElementsAdded()) 45 | } 46 | 47 | block1 := blocks.NewBlock([]byte("foo")) 48 | block2 := blocks.NewBlock([]byte("bar")) 49 | emptyBlock := blocks.NewBlock([]byte{}) 50 | 51 | cachedbs.PutMany(bg, []blocks.Block{block1, emptyBlock}) 52 | has, err := cachedbs.Has(bg, block1.Cid()) 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | blockSize, err := cachedbs.GetSize(bg, block1.Cid()) 57 | if err != nil { 58 | t.Fatal(err) 59 | } 60 | if blockSize == -1 || !has { 61 | t.Fatal("added block is reported missing") 62 | } 63 | 64 | has, err = cachedbs.Has(bg, block2.Cid()) 65 | if err != nil { 66 | t.Fatal(err) 67 | } 68 | blockSize, err = cachedbs.GetSize(bg, block2.Cid()) 69 | if err != nil && !ipld.IsNotFound(err) { 70 | t.Fatal(err) 71 | } 72 | if blockSize > -1 || has { 73 | t.Fatal("not added block is reported to be in blockstore") 74 | } 75 | 76 | has, err = cachedbs.Has(bg, emptyBlock.Cid()) 77 | if err != nil { 78 | t.Fatal(err) 79 | } 80 | blockSize, err = cachedbs.GetSize(bg, emptyBlock.Cid()) 81 | if err != nil { 82 | t.Fatal(err) 83 | } 84 | if blockSize != 0 || !has { 85 | t.Fatal("added block is reported missing") 86 | } 87 | } 88 | 89 | func TestReturnsErrorWhenSizeNegative(t *testing.T) { 90 | bs := NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore())) 91 | _, err := bloomCached(context.Background(), bs, -1, 1) 92 | if err == nil { 93 | t.Fail() 94 | } 95 | } 96 | func TestHasIsBloomCached(t *testing.T) { 97 | cd := &callbackDatastore{f: func() {}, ds: ds.NewMapDatastore()} 98 | bs := NewBlockstore(syncds.MutexWrap(cd)) 99 | 100 | for i := 0; i < 1000; i++ { 101 | bs.Put(bg, blocks.NewBlock([]byte(fmt.Sprintf("data: %d", i)))) 102 | } 103 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) 104 | defer cancel() 105 | 106 | cachedbs, err := testBloomCached(ctx, bs) 107 | if err != nil { 108 | t.Fatal(err) 109 | } 110 | 111 | if err := cachedbs.Wait(ctx); err != nil { 112 | t.Fatalf("Failed while waiting for the filter to build: %d", cachedbs.bloom.ElementsAdded()) 113 | } 114 | 115 | cacheFails := 0 116 | cd.SetFunc(func() { 117 | cacheFails++ 118 | }) 119 | 120 | for i := 0; i < 1000; i++ { 121 | cachedbs.Has(bg, blocks.NewBlock([]byte(fmt.Sprintf("data: %d", i+2000))).Cid()) 122 | } 123 | 124 | if float64(cacheFails)/float64(1000) > float64(0.05) { 125 | t.Fatalf("Bloom filter has cache miss rate of more than 5%%") 126 | } 127 | 128 | cacheFails = 0 129 | block := blocks.NewBlock([]byte("newBlock")) 130 | 131 | cachedbs.PutMany(bg, []blocks.Block{block}) 132 | if cacheFails != 2 { 133 | t.Fatalf("expected two datastore hits: %d", cacheFails) 134 | } 135 | cachedbs.Put(bg, block) 136 | if cacheFails != 3 { 137 | t.Fatalf("expected datastore hit: %d", cacheFails) 138 | } 139 | 140 | if has, err := cachedbs.Has(bg, block.Cid()); !has || err != nil { 141 | t.Fatal("has gave wrong response") 142 | } 143 | 144 | bl, err := cachedbs.Get(bg, block.Cid()) 145 | if bl.String() != block.String() { 146 | t.Fatal("block data doesn't match") 147 | } 148 | 149 | if err != nil { 150 | t.Fatal("there should't be an error") 151 | } 152 | } 153 | 154 | var _ ds.Batching = (*callbackDatastore)(nil) 155 | 156 | type callbackDatastore struct { 157 | sync.Mutex 158 | f func() 159 | ds ds.Datastore 160 | } 161 | 162 | func (c *callbackDatastore) SetFunc(f func()) { 163 | c.Lock() 164 | defer c.Unlock() 165 | c.f = f 166 | } 167 | 168 | func (c *callbackDatastore) CallF() { 169 | c.Lock() 170 | defer c.Unlock() 171 | c.f() 172 | } 173 | 174 | func (c *callbackDatastore) Put(ctx context.Context, key ds.Key, value []byte) (err error) { 175 | c.CallF() 176 | return c.ds.Put(ctx, key, value) 177 | } 178 | 179 | func (c *callbackDatastore) Get(ctx context.Context, key ds.Key) (value []byte, err error) { 180 | c.CallF() 181 | return c.ds.Get(ctx, key) 182 | } 183 | 184 | func (c *callbackDatastore) Has(ctx context.Context, key ds.Key) (exists bool, err error) { 185 | c.CallF() 186 | return c.ds.Has(ctx, key) 187 | } 188 | 189 | func (c *callbackDatastore) GetSize(ctx context.Context, key ds.Key) (size int, err error) { 190 | c.CallF() 191 | return c.ds.GetSize(ctx, key) 192 | } 193 | 194 | func (c *callbackDatastore) Close() error { 195 | return nil 196 | } 197 | 198 | func (c *callbackDatastore) Delete(ctx context.Context, key ds.Key) (err error) { 199 | c.CallF() 200 | return c.ds.Delete(ctx, key) 201 | } 202 | 203 | func (c *callbackDatastore) Query(ctx context.Context, q dsq.Query) (dsq.Results, error) { 204 | c.CallF() 205 | return c.ds.Query(ctx, q) 206 | } 207 | 208 | func (c *callbackDatastore) Sync(ctx context.Context, key ds.Key) error { 209 | c.CallF() 210 | return c.ds.Sync(ctx, key) 211 | } 212 | 213 | func (c *callbackDatastore) Batch(_ context.Context) (ds.Batch, error) { 214 | return ds.NewBasicBatch(c), nil 215 | } 216 | -------------------------------------------------------------------------------- /caching.go: -------------------------------------------------------------------------------- 1 | package blockstore 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | metrics "github.com/ipfs/go-metrics-interface" 8 | ) 9 | 10 | // CacheOpts wraps options for CachedBlockStore(). 11 | // Next to each option is it aproximate memory usage per unit 12 | // 13 | // Deprecated: use github.com/ipfs/boxo/blockstore.CacheOpts 14 | type CacheOpts struct { 15 | HasBloomFilterSize int // 1 byte 16 | HasBloomFilterHashes int // No size, 7 is usually best, consult bloom papers 17 | HasARCCacheSize int // 32 bytes 18 | } 19 | 20 | // DefaultCacheOpts returns a CacheOpts initialized with default values. 21 | // 22 | // Deprecated: use github.com/ipfs/boxo/blockstore.DefaultCacheOpts 23 | func DefaultCacheOpts() CacheOpts { 24 | return CacheOpts{ 25 | HasBloomFilterSize: 512 << 10, 26 | HasBloomFilterHashes: 7, 27 | HasARCCacheSize: 64 << 10, 28 | } 29 | } 30 | 31 | // CachedBlockstore returns a blockstore wrapped in an ARCCache and 32 | // then in a bloom filter cache, if the options indicate it. 33 | // 34 | // Deprecated: use github.com/ipfs/boxo/blockstore.CachedBlockstore 35 | func CachedBlockstore( 36 | ctx context.Context, 37 | bs Blockstore, 38 | opts CacheOpts) (cbs Blockstore, err error) { 39 | cbs = bs 40 | 41 | if opts.HasBloomFilterSize < 0 || opts.HasBloomFilterHashes < 0 || 42 | opts.HasARCCacheSize < 0 { 43 | return nil, errors.New("all options for cache need to be greater than zero") 44 | } 45 | 46 | if opts.HasBloomFilterSize != 0 && opts.HasBloomFilterHashes == 0 { 47 | return nil, errors.New("bloom filter hash count can't be 0 when there is size set") 48 | } 49 | 50 | ctx = metrics.CtxSubScope(ctx, "bs.cache") 51 | 52 | if opts.HasARCCacheSize > 0 { 53 | cbs, err = newARCCachedBS(ctx, cbs, opts.HasARCCacheSize) 54 | } 55 | if opts.HasBloomFilterSize != 0 { 56 | // *8 because of bytes to bits conversion 57 | cbs, err = bloomCached(ctx, cbs, opts.HasBloomFilterSize*8, opts.HasBloomFilterHashes) 58 | } 59 | 60 | return cbs, err 61 | } 62 | -------------------------------------------------------------------------------- /caching_test.go: -------------------------------------------------------------------------------- 1 | package blockstore 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | ) 7 | 8 | func TestCachingOptsLessThanZero(t *testing.T) { 9 | opts := DefaultCacheOpts() 10 | opts.HasARCCacheSize = -1 11 | 12 | if _, err := CachedBlockstore(context.TODO(), nil, opts); err == nil { 13 | t.Error("wrong ARC setting was not detected") 14 | } 15 | 16 | opts = DefaultCacheOpts() 17 | opts.HasBloomFilterSize = -1 18 | 19 | if _, err := CachedBlockstore(context.TODO(), nil, opts); err == nil { 20 | t.Error("negative bloom size was not detected") 21 | } 22 | 23 | opts = DefaultCacheOpts() 24 | opts.HasBloomFilterHashes = -1 25 | 26 | if _, err := CachedBlockstore(context.TODO(), nil, opts); err == nil { 27 | t.Error("negative hashes setting was not detected") 28 | } 29 | } 30 | 31 | func TestBloomHashesAtZero(t *testing.T) { 32 | opts := DefaultCacheOpts() 33 | opts.HasBloomFilterHashes = 0 34 | 35 | if _, err := CachedBlockstore(context.TODO(), nil, opts); err == nil { 36 | t.Error("zero hashes setting with positive size was not detected") 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ipfs/go-ipfs-blockstore 2 | 3 | require ( 4 | github.com/hashicorp/golang-lru v0.5.4 5 | github.com/ipfs/bbloom v0.0.4 6 | github.com/ipfs/go-block-format v0.0.3 7 | github.com/ipfs/go-cid v0.0.7 8 | github.com/ipfs/go-datastore v0.5.0 9 | github.com/ipfs/go-ipfs-ds-help v1.1.0 10 | github.com/ipfs/go-ipfs-util v0.0.2 11 | github.com/ipfs/go-ipld-format v0.3.0 12 | github.com/ipfs/go-log v0.0.1 13 | github.com/ipfs/go-metrics-interface v0.0.1 14 | github.com/multiformats/go-multihash v0.0.14 15 | go.uber.org/atomic v1.6.0 16 | ) 17 | 18 | require ( 19 | github.com/gogo/protobuf v1.2.1 // indirect 20 | github.com/google/uuid v1.1.1 // indirect 21 | github.com/jbenet/goprocess v0.1.4 // indirect 22 | github.com/mattn/go-colorable v0.1.1 // indirect 23 | github.com/mattn/go-isatty v0.0.5 // indirect 24 | github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect 25 | github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 // indirect 26 | github.com/mr-tron/base58 v1.1.3 // indirect 27 | github.com/multiformats/go-base32 v0.0.3 // indirect 28 | github.com/multiformats/go-base36 v0.1.0 // indirect 29 | github.com/multiformats/go-multibase v0.0.3 // indirect 30 | github.com/multiformats/go-varint v0.0.5 // indirect 31 | github.com/opentracing/opentracing-go v1.0.2 // indirect 32 | github.com/spaolacci/murmur3 v1.1.0 // indirect 33 | github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc // indirect 34 | golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 // indirect 35 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect 36 | golang.org/x/sys v0.0.0-20190412213103-97732733099d // indirect 37 | ) 38 | 39 | go 1.19 40 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= 6 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 7 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 8 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 9 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 10 | github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= 11 | github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= 12 | github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= 13 | github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 14 | github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= 15 | github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= 16 | github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= 17 | github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/dDTpMc= 18 | github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= 19 | github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= 20 | github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= 21 | github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= 22 | github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= 23 | github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= 24 | github.com/ipfs/go-datastore v0.5.0 h1:rQicVCEacWyk4JZ6G5bD9TKR7lZEG1MWcG7UdWYrFAU= 25 | github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= 26 | github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= 27 | github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= 28 | github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= 29 | github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= 30 | github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= 31 | github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= 32 | github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= 33 | github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= 34 | github.com/ipfs/go-ipld-format v0.3.0 h1:Mwm2oRLzIuUwEPewWAWyMuuBQUsn3awfFEYVb8akMOQ= 35 | github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= 36 | github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= 37 | github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= 38 | github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= 39 | github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= 40 | github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= 41 | github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= 42 | github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= 43 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 44 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 45 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 46 | github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= 47 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 48 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 49 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 50 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 51 | github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= 52 | github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= 53 | github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= 54 | github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 55 | github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= 56 | github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= 57 | github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= 58 | github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo= 59 | github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= 60 | github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= 61 | github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= 62 | github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= 63 | github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= 64 | github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= 65 | github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= 66 | github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= 67 | github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= 68 | github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= 69 | github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= 70 | github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= 71 | github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= 72 | github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I= 73 | github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= 74 | github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= 75 | github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= 76 | github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= 77 | github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 78 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 79 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 80 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 81 | github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= 82 | github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 83 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 84 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 85 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 86 | github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= 87 | github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= 88 | go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= 89 | go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= 90 | go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= 91 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= 92 | golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 93 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 94 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 95 | golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= 96 | golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 97 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= 98 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 99 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 100 | golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 101 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 102 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 103 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= 104 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 105 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 106 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 107 | golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 108 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 109 | golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= 110 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 111 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 112 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 113 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 114 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 115 | golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 116 | golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= 117 | golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 118 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 119 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 120 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 121 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 122 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 123 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 124 | -------------------------------------------------------------------------------- /idstore.go: -------------------------------------------------------------------------------- 1 | package blockstore 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | blocks "github.com/ipfs/go-block-format" 8 | cid "github.com/ipfs/go-cid" 9 | mh "github.com/multiformats/go-multihash" 10 | ) 11 | 12 | // idstore wraps a BlockStore to add support for identity hashes 13 | type idstore struct { 14 | bs Blockstore 15 | viewer Viewer 16 | } 17 | 18 | var _ Blockstore = (*idstore)(nil) 19 | var _ Viewer = (*idstore)(nil) 20 | var _ io.Closer = (*idstore)(nil) 21 | 22 | // Deprecated: use github.com/ipfs/boxo/blockstore.NewIdStore 23 | func NewIdStore(bs Blockstore) Blockstore { 24 | ids := &idstore{bs: bs} 25 | if v, ok := bs.(Viewer); ok { 26 | ids.viewer = v 27 | } 28 | return ids 29 | } 30 | 31 | func extractContents(k cid.Cid) (bool, []byte) { 32 | // Pre-check by calling Prefix(), this much faster than extracting the hash. 33 | if k.Prefix().MhType != mh.IDENTITY { 34 | return false, nil 35 | } 36 | 37 | dmh, err := mh.Decode(k.Hash()) 38 | if err != nil || dmh.Code != mh.IDENTITY { 39 | return false, nil 40 | } 41 | return true, dmh.Digest 42 | } 43 | 44 | func (b *idstore) DeleteBlock(ctx context.Context, k cid.Cid) error { 45 | isId, _ := extractContents(k) 46 | if isId { 47 | return nil 48 | } 49 | return b.bs.DeleteBlock(ctx, k) 50 | } 51 | 52 | func (b *idstore) Has(ctx context.Context, k cid.Cid) (bool, error) { 53 | isId, _ := extractContents(k) 54 | if isId { 55 | return true, nil 56 | } 57 | return b.bs.Has(ctx, k) 58 | } 59 | 60 | func (b *idstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error { 61 | if b.viewer == nil { 62 | blk, err := b.Get(ctx, k) 63 | if err != nil { 64 | return err 65 | } 66 | return callback(blk.RawData()) 67 | } 68 | isId, bdata := extractContents(k) 69 | if isId { 70 | return callback(bdata) 71 | } 72 | return b.viewer.View(ctx, k, callback) 73 | } 74 | 75 | func (b *idstore) GetSize(ctx context.Context, k cid.Cid) (int, error) { 76 | isId, bdata := extractContents(k) 77 | if isId { 78 | return len(bdata), nil 79 | } 80 | return b.bs.GetSize(ctx, k) 81 | } 82 | 83 | func (b *idstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) { 84 | isId, bdata := extractContents(k) 85 | if isId { 86 | return blocks.NewBlockWithCid(bdata, k) 87 | } 88 | return b.bs.Get(ctx, k) 89 | } 90 | 91 | func (b *idstore) Put(ctx context.Context, bl blocks.Block) error { 92 | isId, _ := extractContents(bl.Cid()) 93 | if isId { 94 | return nil 95 | } 96 | return b.bs.Put(ctx, bl) 97 | } 98 | 99 | func (b *idstore) PutMany(ctx context.Context, bs []blocks.Block) error { 100 | toPut := make([]blocks.Block, 0, len(bs)) 101 | for _, bl := range bs { 102 | isId, _ := extractContents(bl.Cid()) 103 | if isId { 104 | continue 105 | } 106 | toPut = append(toPut, bl) 107 | } 108 | return b.bs.PutMany(ctx, toPut) 109 | } 110 | 111 | func (b *idstore) HashOnRead(enabled bool) { 112 | b.bs.HashOnRead(enabled) 113 | } 114 | 115 | func (b *idstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { 116 | return b.bs.AllKeysChan(ctx) 117 | } 118 | 119 | func (b *idstore) Close() error { 120 | if c, ok := b.bs.(io.Closer); ok { 121 | return c.Close() 122 | } 123 | return nil 124 | } 125 | -------------------------------------------------------------------------------- /idstore_test.go: -------------------------------------------------------------------------------- 1 | package blockstore 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | blk "github.com/ipfs/go-block-format" 8 | cid "github.com/ipfs/go-cid" 9 | ds "github.com/ipfs/go-datastore" 10 | mh "github.com/multiformats/go-multihash" 11 | ) 12 | 13 | func createTestStores() (Blockstore, *callbackDatastore) { 14 | cd := &callbackDatastore{f: func() {}, ds: ds.NewMapDatastore()} 15 | ids := NewIdStore(NewBlockstore(cd)) 16 | return ids, cd 17 | } 18 | 19 | func TestIdStore(t *testing.T) { 20 | idhash1, _ := cid.NewPrefixV1(cid.Raw, mh.IDENTITY).Sum([]byte("idhash1")) 21 | idblock1, _ := blk.NewBlockWithCid([]byte("idhash1"), idhash1) 22 | hash1, _ := cid.NewPrefixV1(cid.Raw, mh.SHA2_256).Sum([]byte("hash1")) 23 | block1, _ := blk.NewBlockWithCid([]byte("hash1"), hash1) 24 | emptyHash, _ := cid.NewPrefixV1(cid.Raw, mh.SHA2_256).Sum([]byte("emptyHash")) 25 | emptyBlock, _ := blk.NewBlockWithCid([]byte{}, emptyHash) 26 | 27 | ids, cb := createTestStores() 28 | 29 | have, _ := ids.Has(bg, idhash1) 30 | if !have { 31 | t.Fatal("Has() failed on idhash") 32 | } 33 | 34 | _, err := ids.Get(bg, idhash1) 35 | if err != nil { 36 | t.Fatalf("Get() failed on idhash: %v", err) 37 | } 38 | 39 | noop := func() {} 40 | failIfPassThough := func() { 41 | t.Fatal("operation on identity hash passed though to datastore") 42 | } 43 | 44 | cb.f = failIfPassThough 45 | err = ids.Put(bg, idblock1) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | 50 | cb.f = noop 51 | err = ids.Put(bg, block1) 52 | if err != nil { 53 | t.Fatalf("Put() failed on normal block: %v", err) 54 | } 55 | 56 | have, _ = ids.Has(bg, hash1) 57 | if !have { 58 | t.Fatal("normal block not added to datastore") 59 | } 60 | 61 | blockSize, _ := ids.GetSize(bg, hash1) 62 | if blockSize == -1 { 63 | t.Fatal("normal block not added to datastore") 64 | } 65 | 66 | _, err = ids.Get(bg, hash1) 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | 71 | err = ids.Put(bg, emptyBlock) 72 | if err != nil { 73 | t.Fatalf("Put() failed on normal block: %v", err) 74 | } 75 | 76 | have, _ = ids.Has(bg, emptyHash) 77 | if !have { 78 | t.Fatal("normal block not added to datastore") 79 | } 80 | 81 | blockSize, _ = ids.GetSize(bg, emptyHash) 82 | if blockSize != 0 { 83 | t.Fatal("normal block not added to datastore") 84 | } 85 | 86 | cb.f = failIfPassThough 87 | err = ids.DeleteBlock(bg, idhash1) 88 | if err != nil { 89 | t.Fatal(err) 90 | } 91 | 92 | cb.f = noop 93 | err = ids.DeleteBlock(bg, hash1) 94 | if err != nil { 95 | t.Fatal(err) 96 | } 97 | 98 | have, _ = ids.Has(bg, hash1) 99 | if have { 100 | t.Fatal("normal block not deleted from datastore") 101 | } 102 | 103 | blockSize, _ = ids.GetSize(bg, hash1) 104 | if blockSize > -1 { 105 | t.Fatal("normal block not deleted from datastore") 106 | } 107 | 108 | err = ids.DeleteBlock(bg, emptyHash) 109 | if err != nil { 110 | t.Fatal(err) 111 | } 112 | 113 | idhash2, _ := cid.NewPrefixV1(cid.Raw, mh.IDENTITY).Sum([]byte("idhash2")) 114 | idblock2, _ := blk.NewBlockWithCid([]byte("idhash2"), idhash2) 115 | hash2, _ := cid.NewPrefixV1(cid.Raw, mh.SHA2_256).Sum([]byte("hash2")) 116 | block2, _ := blk.NewBlockWithCid([]byte("hash2"), hash2) 117 | 118 | cb.f = failIfPassThough 119 | err = ids.PutMany(bg, []blk.Block{idblock1, idblock2}) 120 | if err != nil { 121 | t.Fatal(err) 122 | } 123 | 124 | opCount := 0 125 | cb.f = func() { 126 | opCount++ 127 | } 128 | 129 | err = ids.PutMany(bg, []blk.Block{block1, block2}) 130 | if err != nil { 131 | t.Fatal(err) 132 | } 133 | if opCount != 4 { 134 | // one call to Has and Put for each Cid 135 | t.Fatalf("expected exactly 4 operations got %d", opCount) 136 | } 137 | 138 | opCount = 0 139 | err = ids.PutMany(bg, []blk.Block{idblock1, block1}) 140 | if err != nil { 141 | t.Fatal(err) 142 | } 143 | if opCount != 1 { 144 | // just one call to Put from the normal (non-id) block 145 | t.Fatalf("expected exactly 1 operations got %d", opCount) 146 | } 147 | 148 | ch, err := ids.AllKeysChan(context.TODO()) 149 | if err != nil { 150 | t.Fatal(err) 151 | } 152 | cnt := 0 153 | for c := range ch { 154 | cnt++ 155 | if c.Prefix().MhType == mh.IDENTITY { 156 | t.Fatalf("block with identity hash found in blockstore") 157 | } 158 | } 159 | if cnt != 2 { 160 | t.Fatalf("expected exactly two keys returned by AllKeysChan got %d", cnt) 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /version.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "v1.3.1" 3 | } 4 | --------------------------------------------------------------------------------