├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ ├── benchmark-pr.yaml │ ├── benchmark-releases.yaml │ ├── benchmark-template.yaml │ ├── failpoint_test.yaml │ ├── gh-workflow-approve.yaml │ ├── robustness_nightly.yaml │ ├── robustness_template.yaml │ ├── robustness_test.yaml │ ├── stale.yaml │ ├── tests-template.yml │ ├── tests_amd64.yaml │ ├── tests_arm64.yaml │ └── tests_windows.yml ├── .gitignore ├── .go-version ├── .golangci.yaml ├── CHANGELOG ├── CHANGELOG-1.3.md └── CHANGELOG-1.4.md ├── LICENSE ├── Makefile ├── OWNERS ├── README.md ├── allocate_test.go ├── bolt_386.go ├── bolt_aix.go ├── bolt_amd64.go ├── bolt_android.go ├── bolt_arm.go ├── bolt_arm64.go ├── bolt_linux.go ├── bolt_loong64.go ├── bolt_mips64x.go ├── bolt_mipsx.go ├── bolt_openbsd.go ├── bolt_ppc.go ├── bolt_ppc64.go ├── bolt_ppc64le.go ├── bolt_riscv64.go ├── bolt_s390x.go ├── bolt_solaris.go ├── bolt_unix.go ├── bolt_windows.go ├── boltsync_unix.go ├── bucket.go ├── bucket_test.go ├── cmd └── bbolt │ ├── OWNERS │ ├── README.md │ ├── command_check.go │ ├── command_check_test.go │ ├── command_inspect.go │ ├── command_inspect_test.go │ ├── command_root.go │ ├── command_surgery.go │ ├── command_surgery_freelist.go │ ├── command_surgery_freelist_test.go │ ├── command_surgery_meta.go │ ├── command_surgery_meta_test.go │ ├── command_surgery_test.go │ ├── command_version.go │ ├── main.go │ ├── main_test.go │ ├── page_command.go │ ├── utils.go │ └── utils_test.go ├── code-of-conduct.md ├── compact.go ├── concurrent_test.go ├── cursor.go ├── cursor_test.go ├── db.go ├── db_test.go ├── db_whitebox_test.go ├── doc.go ├── errors.go ├── errors └── errors.go ├── go.mod ├── go.sum ├── internal ├── btesting │ └── btesting.go ├── common │ ├── bucket.go │ ├── inode.go │ ├── meta.go │ ├── page.go │ ├── page_test.go │ ├── types.go │ ├── unsafe.go │ ├── utils.go │ └── verify.go ├── freelist │ ├── array.go │ ├── array_test.go │ ├── freelist.go │ ├── freelist_test.go │ ├── hashmap.go │ ├── hashmap_test.go │ └── shared.go ├── guts_cli │ └── guts_cli.go ├── surgeon │ ├── surgeon.go │ ├── surgeon_test.go │ ├── xray.go │ └── xray_test.go └── tests │ └── tx_check_test.go ├── logger.go ├── manydbs_test.go ├── mlock_unix.go ├── mlock_windows.go ├── movebucket_test.go ├── node.go ├── node_test.go ├── quick_test.go ├── scripts ├── compare_benchmarks.sh └── fix.sh ├── simulation_no_freelist_sync_test.go ├── simulation_test.go ├── tests ├── dmflakey │ ├── dmflakey.go │ ├── dmflakey_test.go │ ├── dmsetup.go │ └── loopback.go ├── failpoint │ └── db_failpoint_test.go ├── robustness │ ├── main_test.go │ └── powerfailure_test.go └── utils │ └── helpers.go ├── tx.go ├── tx_check.go ├── tx_check_test.go ├── tx_stats_test.go ├── tx_test.go ├── unix_test.go ├── utils_test.go └── version └── version.go /.gitattributes: -------------------------------------------------------------------------------- 1 | # ensure that line endings for Windows builds are properly formatted 2 | # see https://github.com/golangci/golangci-lint-action?tab=readme-ov-file#how-to-use 3 | # at "Multiple OS Example" section 4 | *.go text eol=lf 5 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: github-actions 4 | directory: / 5 | schedule: 6 | interval: weekly 7 | 8 | - package-ecosystem: gomod 9 | directory: / 10 | schedule: 11 | interval: weekly 12 | -------------------------------------------------------------------------------- /.github/workflows/benchmark-pr.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Benchmarks on PRs (AMD64) 3 | permissions: read-all 4 | on: [pull_request] 5 | jobs: 6 | amd64: 7 | uses: ./.github/workflows/benchmark-template.yaml 8 | with: 9 | benchGitRef: ${{ github.event.pull_request.base.sha }} 10 | -------------------------------------------------------------------------------- /.github/workflows/benchmark-releases.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Nightly Benchmarks against last release (AMD64) 3 | permissions: read-all 4 | on: 5 | schedule: 6 | - cron: '10 5 * * *' # runs every day at 05:10 UTC 7 | # workflow_dispatch enables manual testing of this job by maintainers 8 | workflow_dispatch: 9 | jobs: 10 | amd64: 11 | uses: ./.github/workflows/benchmark-template.yaml 12 | with: 13 | benchGitRef: release-1.3 14 | -------------------------------------------------------------------------------- /.github/workflows/benchmark-template.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Reusable Benchmark Template 3 | on: 4 | workflow_call: 5 | inputs: 6 | # which git reference to benchmark against 7 | benchGitRef: 8 | required: true 9 | type: string 10 | maxAcceptableDifferencePercent: 11 | required: false 12 | type: number 13 | default: 5 14 | runs-on: 15 | required: false 16 | type: string 17 | default: "['ubuntu-latest']" 18 | permissions: read-all 19 | 20 | jobs: 21 | benchmark: 22 | runs-on: ${{ fromJson(inputs.runs-on) }} 23 | steps: 24 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 25 | with: 26 | fetch-depth: 0 27 | - id: goversion 28 | run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" 29 | - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 30 | with: 31 | go-version: ${{ steps.goversion.outputs.goversion }} 32 | - name: Run Benchmarks 33 | run: | 34 | BENCHSTAT_OUTPUT_FILE=result.txt make test-benchmark-compare REF=${{ inputs.benchGitRef }} 35 | - run: | 36 | echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" 37 | cat result.txt >> "$GITHUB_STEP_SUMMARY" 38 | echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" 39 | cat <> "$GITHUB_STEP_SUMMARY" 40 |
41 | The table shows the median and 90% confidence interval (CI) summaries for each benchmark comparing the HEAD and the BASE, and an A/B comparison under "vs base". The last column shows the statistical p-value with ten runs (n=10). 42 | The last row has the Geometric Mean (geomean) for the given rows in the table. 43 | Refer to [benchstat's documentation](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat) for more help. 44 | EOL 45 | - name: Validate results under acceptable limit 46 | run: | 47 | export MAX_ACCEPTABLE_DIFFERENCE=${{ inputs.maxAcceptableDifferencePercent }} 48 | while IFS= read -r line; do 49 | # Get fourth value, which is the comparison with the base. 50 | value="$(echo "$line" | awk '{print $4}')" 51 | if [[ "$value" = +* ]] || [[ "$value" = -* ]]; then 52 | if (( $(echo "${value//[^0-9.]/}"'>'"$MAX_ACCEPTABLE_DIFFERENCE" | bc -l) )); then 53 | echo "::error::$value is above the maximum acceptable difference ($MAX_ACCEPTABLE_DIFFERENCE)" 54 | exit 1 55 | fi 56 | fi 57 | done < <(grep geomean result.txt) 58 | -------------------------------------------------------------------------------- /.github/workflows/failpoint_test.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Failpoint test 3 | on: [push, pull_request] 4 | permissions: read-all 5 | jobs: 6 | test: 7 | strategy: 8 | matrix: 9 | os: [ubuntu-latest] 10 | runs-on: ${{ matrix.os }} 11 | steps: 12 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 13 | - id: goversion 14 | run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" 15 | - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 16 | with: 17 | go-version: ${{ steps.goversion.outputs.goversion }} 18 | - run: | 19 | make gofail-enable 20 | make test-failpoint 21 | -------------------------------------------------------------------------------- /.github/workflows/gh-workflow-approve.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Approve GitHub Workflows 3 | permissions: read-all 4 | on: 5 | pull_request_target: 6 | types: 7 | - labeled 8 | - synchronize 9 | branches: 10 | - main 11 | - release-1.3 12 | 13 | jobs: 14 | approve: 15 | name: Approve ok-to-test 16 | if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') 17 | runs-on: ubuntu-latest 18 | permissions: 19 | actions: write 20 | steps: 21 | - name: Update PR 22 | uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 23 | continue-on-error: true 24 | with: 25 | github-token: ${{ secrets.GITHUB_TOKEN }} 26 | debug: ${{ secrets.ACTIONS_RUNNER_DEBUG == 'true' }} 27 | script: | 28 | const result = await github.rest.actions.listWorkflowRunsForRepo({ 29 | owner: context.repo.owner, 30 | repo: context.repo.repo, 31 | event: "pull_request", 32 | status: "action_required", 33 | head_sha: context.payload.pull_request.head.sha, 34 | per_page: 100 35 | }); 36 | for (var run of result.data.workflow_runs) { 37 | await github.rest.actions.approveWorkflowRun({ 38 | owner: context.repo.owner, 39 | repo: context.repo.repo, 40 | run_id: run.id 41 | }); 42 | } 43 | -------------------------------------------------------------------------------- /.github/workflows/robustness_nightly.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Robustness Nightly 3 | permissions: read-all 4 | on: 5 | schedule: 6 | - cron: '25 9 * * *' # runs every day at 09:25 UTC 7 | # workflow_dispatch enables manual testing of this job by maintainers 8 | workflow_dispatch: 9 | 10 | jobs: 11 | amd64: 12 | # GHA has a maximum amount of 6h execution time, we try to get done within 3h 13 | uses: ./.github/workflows/robustness_template.yaml 14 | with: 15 | count: 100 16 | testTimeout: 200m 17 | runs-on: "['ubuntu-latest']" 18 | -------------------------------------------------------------------------------- /.github/workflows/robustness_template.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Reusable Robustness Workflow 3 | on: 4 | workflow_call: 5 | inputs: 6 | count: 7 | required: true 8 | type: number 9 | testTimeout: 10 | required: false 11 | type: string 12 | default: '30m' 13 | runs-on: 14 | required: false 15 | type: string 16 | default: "['ubuntu-latest']" 17 | permissions: read-all 18 | 19 | jobs: 20 | test: 21 | # this is to prevent the job to run at forked projects 22 | if: github.repository == 'etcd-io/bbolt' 23 | timeout-minutes: 210 24 | runs-on: ${{ fromJson(inputs.runs-on) }} 25 | steps: 26 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 27 | - id: goversion 28 | run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" 29 | - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 30 | with: 31 | go-version: ${{ steps.goversion.outputs.goversion }} 32 | - name: test-robustness 33 | run: | 34 | set -euo pipefail 35 | sudo apt-get install -y dmsetup xfsprogs 36 | 37 | ROBUSTNESS_TESTFLAGS="--count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} -failfast" make test-robustness 38 | 39 | - name: Host Status 40 | if: always() 41 | run: | 42 | set -x 43 | mount 44 | df 45 | losetup -l 46 | - name: Kernel Message 47 | if: failure() 48 | run: | 49 | sudo lsmod 50 | sudo dmesg -T -f kern 51 | -------------------------------------------------------------------------------- /.github/workflows/robustness_test.yaml: -------------------------------------------------------------------------------- 1 | name: Robustness Test 2 | on: [push, pull_request] 3 | permissions: read-all 4 | jobs: 5 | amd64: 6 | uses: ./.github/workflows/robustness_template.yaml 7 | with: 8 | count: 10 9 | testTimeout: 30m 10 | runs-on: "['ubuntu-latest']" 11 | arm64: 12 | uses: ./.github/workflows/robustness_template.yaml 13 | with: 14 | count: 10 15 | testTimeout: 30m 16 | runs-on: "['ubuntu-24.04-arm']" 17 | -------------------------------------------------------------------------------- /.github/workflows/stale.yaml: -------------------------------------------------------------------------------- 1 | name: 'Close stale issues and PRs' 2 | on: 3 | schedule: 4 | - cron: '0 0 * * *' # every day at 00:00 UTC 5 | 6 | permissions: 7 | issues: write 8 | pull-requests: write 9 | 10 | jobs: 11 | stale: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 15 | with: 16 | days-before-stale: 90 17 | days-before-close: 21 18 | stale-issue-label: stale 19 | stale-pr-label: stale 20 | -------------------------------------------------------------------------------- /.github/workflows/tests-template.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Reusable unit test Workflow 3 | on: 4 | workflow_call: 5 | inputs: 6 | runs-on: 7 | required: false 8 | type: string 9 | default: ubuntu-latest 10 | targets: 11 | required: false 12 | type: string 13 | default: "['linux-unit-test-1-cpu','linux-unit-test-2-cpu','linux-unit-test-4-cpu']" 14 | permissions: read-all 15 | 16 | jobs: 17 | test-linux: 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | target: ${{ fromJSON(inputs.targets) }} 22 | runs-on: ${{ inputs.runs-on }} 23 | steps: 24 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 25 | - id: goversion 26 | run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" 27 | - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 28 | with: 29 | go-version: ${{ steps.goversion.outputs.goversion }} 30 | - run: make fmt 31 | - env: 32 | TARGET: ${{ matrix.target }} 33 | run: | 34 | case "${TARGET}" in 35 | linux-unit-test-1-cpu) 36 | CPU=1 make test 37 | ;; 38 | linux-unit-test-2-cpu) 39 | CPU=2 make test 40 | ;; 41 | linux-unit-test-4-cpu) 42 | CPU=4 make test 43 | ;; 44 | linux-unit-test-4-cpu-race) 45 | CPU=4 ENABLE_RACE=true make test 46 | ;; 47 | *) 48 | echo "Failed to find target" 49 | exit 1 50 | ;; 51 | esac 52 | - name: golangci-lint 53 | uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0 54 | with: 55 | version: v2.1.6 56 | -------------------------------------------------------------------------------- /.github/workflows/tests_amd64.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Tests AMD64 3 | permissions: read-all 4 | on: [push, pull_request] 5 | jobs: 6 | test-linux-amd64: 7 | uses: ./.github/workflows/tests-template.yml 8 | test-linux-amd64-race: 9 | uses: ./.github/workflows/tests-template.yml 10 | with: 11 | runs-on: ubuntu-latest 12 | targets: "['linux-unit-test-4-cpu-race']" 13 | 14 | coverage: 15 | needs: 16 | - test-linux-amd64 17 | - test-linux-amd64-race 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 21 | - id: goversion 22 | run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" 23 | - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 24 | with: 25 | go-version: ${{ steps.goversion.outputs.goversion }} 26 | - run: make coverage 27 | -------------------------------------------------------------------------------- /.github/workflows/tests_arm64.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Tests ARM64 3 | permissions: read-all 4 | on: [push, pull_request] 5 | jobs: 6 | test-linux-arm64: 7 | uses: ./.github/workflows/tests-template.yml 8 | test-linux-arm64-race: 9 | uses: ./.github/workflows/tests-template.yml 10 | with: 11 | runs-on: ubuntu-24.04-arm 12 | targets: "['linux-unit-test-4-cpu-race']" 13 | 14 | coverage: 15 | needs: 16 | - test-linux-arm64 17 | - test-linux-arm64-race 18 | runs-on: ubuntu-24.04-arm 19 | steps: 20 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 21 | - id: goversion 22 | run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" 23 | - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 24 | with: 25 | go-version: ${{ steps.goversion.outputs.goversion }} 26 | - run: make coverage 27 | -------------------------------------------------------------------------------- /.github/workflows/tests_windows.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Tests 3 | on: [push, pull_request] 4 | permissions: read-all 5 | jobs: 6 | test-windows: 7 | strategy: 8 | fail-fast: false 9 | matrix: 10 | target: 11 | - windows-amd64-unit-test-4-cpu 12 | # FIXME(fuweid): 13 | # 14 | # The windows will throws the following error when enable race. 15 | # We skip it until we have solution. 16 | # 17 | # ThreadSanitizer failed to allocate 0x000200000000 (8589934592) bytes at 0x0400c0000000 (error code: 1455) 18 | # 19 | # - windows-amd64-unit-test-4-cpu-race 20 | runs-on: windows-latest 21 | steps: 22 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 23 | - id: goversion 24 | run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" 25 | - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 26 | with: 27 | go-version: ${{ steps.goversion.outputs.goversion }} 28 | - run: make fmt 29 | - env: 30 | TARGET: ${{ matrix.target }} 31 | run: | 32 | case "${TARGET}" in 33 | windows-amd64-unit-test-4-cpu) 34 | CPU=4 make test 35 | ;; 36 | *) 37 | echo "Failed to find target" 38 | exit 1 39 | ;; 40 | esac 41 | shell: bash 42 | - name: golangci-lint 43 | uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0 44 | with: 45 | version: v2.1.6 46 | 47 | coverage: 48 | needs: ["test-windows"] 49 | runs-on: windows-latest 50 | steps: 51 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 52 | - id: goversion 53 | run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" 54 | - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 55 | with: 56 | go-version: ${{ steps.goversion.outputs.goversion }} 57 | - run: make coverage 58 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.prof 2 | *.test 3 | *.swp 4 | /bin/ 5 | cover.out 6 | cover-*.out 7 | /.idea 8 | *.iml 9 | /bbolt 10 | /cmd/bbolt/bbolt 11 | .DS_Store 12 | 13 | -------------------------------------------------------------------------------- /.go-version: -------------------------------------------------------------------------------- 1 | 1.24.3 2 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | formatters: 2 | enable: 3 | - gofmt 4 | - goimports 5 | settings: # please keep this alphabetized 6 | goimports: 7 | local-prefixes: 8 | - go.etcd.io # Put imports beginning with prefix after 3rd-party packages. 9 | issues: 10 | max-same-issues: 0 11 | linters: 12 | default: none 13 | enable: # please keep this alphabetized 14 | - errcheck 15 | - govet 16 | - ineffassign 17 | - staticcheck 18 | - unused 19 | exclusions: 20 | presets: 21 | - comments 22 | - common-false-positives 23 | - legacy 24 | - std-error-handling 25 | settings: # please keep this alphabetized 26 | staticcheck: 27 | checks: 28 | - all 29 | - -QF1003 # Convert if/else-if chain to tagged switch 30 | - -QF1010 # Convert slice of bytes to string when printing it 31 | - -ST1003 # Poorly chosen identifier 32 | - -ST1005 # Incorrectly formatted error string 33 | - -ST1012 # Poorly chosen name for error variable 34 | version: "2" 35 | -------------------------------------------------------------------------------- /CHANGELOG/CHANGELOG-1.3.md: -------------------------------------------------------------------------------- 1 | Note that we start to track changes starting from v1.3.7. 2 | 3 |
4 | 5 | ## v1.3.11(2024-08-21) 6 | 7 | ### BoltDB 8 | - Fix [the `freelist.allocs` isn't rollbacked when a tx is rollbacked](https://github.com/etcd-io/bbolt/pull/823). 9 | 10 | ### CMD 11 | - Add [`-gobench-output` option for bench command to adapt to benchstat](https://github.com/etcd-io/bbolt/pull/802). 12 | 13 | ### Other 14 | - [Bump go version to 1.22.x](https://github.com/etcd-io/bbolt/pull/822). 15 | - This patch also added `dmflakey` package, which can be reused by other projects. See https://github.com/etcd-io/bbolt/pull/812. 16 | 17 |
18 | 19 | ## v1.3.10(2024-05-06) 20 | 21 | ### BoltDB 22 | - [Remove deprecated `UnsafeSlice` and use `unsafe.Slice`](https://github.com/etcd-io/bbolt/pull/717) 23 | - [Stabilize the behaviour of Prev when the cursor already points to the first element](https://github.com/etcd-io/bbolt/pull/744) 24 | 25 | ### Other 26 | - [Bump go version to 1.21.9](https://github.com/etcd-io/bbolt/pull/713) 27 | 28 |
29 | 30 | ## v1.3.9(2024-02-24) 31 | 32 | ### BoltDB 33 | - [Clone the key before operating data in bucket against the key](https://github.com/etcd-io/bbolt/pull/639) 34 | 35 | ### CMD 36 | - [Fix `bbolt keys` and `bbolt get` to prevent them from panicking when no parameter provided](https://github.com/etcd-io/bbolt/pull/683) 37 | 38 |
39 | 40 | ## v1.3.8(2023-10-26) 41 | 42 | ### BoltDB 43 | - Fix [db.close() doesn't unlock the db file if db.munnmap() fails](https://github.com/etcd-io/bbolt/pull/439). 44 | - [Avoid syscall.Syscall use on OpenBSD](https://github.com/etcd-io/bbolt/pull/406). 45 | - Fix [rollback panicking after mlock failed or both meta pages corrupted](https://github.com/etcd-io/bbolt/pull/444). 46 | - Fix [bbolt panicking due to 64bit unaligned on arm32](https://github.com/etcd-io/bbolt/pull/584). 47 | 48 | ### CMD 49 | - [Update the usage of surgery command](https://github.com/etcd-io/bbolt/pull/411). 50 | 51 |
52 | 53 | ## v1.3.7(2023-01-31) 54 | 55 | ### BoltDB 56 | - Add [recursive checker to confirm database consistency](https://github.com/etcd-io/bbolt/pull/225). 57 | - Add [support to get the page size from the second meta page if the first one is invalid](https://github.com/etcd-io/bbolt/pull/294). 58 | - Add [support for loong64 arch](https://github.com/etcd-io/bbolt/pull/303). 59 | - Add [internal iterator to Bucket that goes over buckets](https://github.com/etcd-io/bbolt/pull/356). 60 | - Add [validation on page read and write](https://github.com/etcd-io/bbolt/pull/358). 61 | - Add [PreLoadFreelist option to support loading free pages in readonly mode](https://github.com/etcd-io/bbolt/pull/381). 62 | - Add [(*Tx) CheckWithOption to support generating human-readable diagnostic messages](https://github.com/etcd-io/bbolt/pull/395). 63 | - Fix [Use `golang.org/x/sys/windows` for `FileLockEx`/`UnlockFileEx`](https://github.com/etcd-io/bbolt/pull/283). 64 | - Fix [readonly file mapping on windows](https://github.com/etcd-io/bbolt/pull/307). 65 | - Fix [the "Last" method might return no data due to not skipping the empty pages](https://github.com/etcd-io/bbolt/pull/341). 66 | - Fix [panic on db.meta when rollback](https://github.com/etcd-io/bbolt/pull/362). 67 | 68 | ### CMD 69 | - Add [support for get keys in sub buckets in `bbolt get` command](https://github.com/etcd-io/bbolt/pull/295). 70 | - Add [support for `--format` flag for `bbolt keys` command](https://github.com/etcd-io/bbolt/pull/306). 71 | - Add [safeguards to bbolt CLI commands](https://github.com/etcd-io/bbolt/pull/354). 72 | - Add [`bbolt page` supports --all and --value-format=redacted formats](https://github.com/etcd-io/bbolt/pull/359). 73 | - Add [`bbolt surgery` commands](https://github.com/etcd-io/bbolt/issues/370). 74 | - Fix [open db file readonly mode for commands which shouldn't update the db file](https://github.com/etcd-io/bbolt/pull/365), see also [pull/292](https://github.com/etcd-io/bbolt/pull/292). 75 | 76 | ### Other 77 | - [Build bbolt CLI tool, test and format the source code using golang 1.17.13](https://github.com/etcd-io/bbolt/pull/297). 78 | - [Bump golang.org/x/sys to v0.4.0](https://github.com/etcd-io/bbolt/pull/397). 79 | 80 | ### Summary 81 | Release v1.3.7 contains following critical fixes: 82 | - fix to problem that `Last` method might return incorrect value ([#341](https://github.com/etcd-io/bbolt/pull/341)) 83 | - fix of potential panic when performing transaction's rollback ([#362](https://github.com/etcd-io/bbolt/pull/362)) 84 | 85 | Other changes focused on defense-in-depth ([#358](https://github.com/etcd-io/bbolt/pull/358), [#294](https://github.com/etcd-io/bbolt/pull/294), [#225](https://github.com/etcd-io/bbolt/pull/225), [#395](https://github.com/etcd-io/bbolt/pull/395)) 86 | 87 | `bbolt` command line tool was expanded to: 88 | - allow fixing simple corruptions by `bbolt surgery` ([#370](https://github.com/etcd-io/bbolt/pull/370)) 89 | - be flexible about output formatting ([#306](https://github.com/etcd-io/bbolt/pull/306), [#359](https://github.com/etcd-io/bbolt/pull/359)) 90 | - allow accessing data in subbuckets ([#295](https://github.com/etcd-io/bbolt/pull/295)) 91 | -------------------------------------------------------------------------------- /CHANGELOG/CHANGELOG-1.4.md: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 | ## v1.4.0(2025-02-05) 5 | There isn't any production code change since v1.4.0-beta.0. Only some dependencies 6 | are bumped, also updated some typos in comment and readme, and removed the legacy 7 | build tag `// +build` in https://github.com/etcd-io/bbolt/pull/879. 8 | 9 |
10 | 11 | ## v1.4.0-beta.0(2024-11-04) 12 | 13 | ### BoltDB 14 | - Reorganized the directory structure of freelist source code 15 | - [Move array related freelist source code into a separate file](https://github.com/etcd-io/bbolt/pull/777) 16 | - [Move method `freePages` into freelist.go](https://github.com/etcd-io/bbolt/pull/783) 17 | - [Add an interface for freelist](https://github.com/etcd-io/bbolt/pull/775) 18 | - [Rollback alloc map when a transaction is rollbacked](https://github.com/etcd-io/bbolt/pull/819) 19 | - [No handling freelist as a special case when freeing a page](https://github.com/etcd-io/bbolt/pull/788) 20 | - [Ensure hashmap init method clears the data structures](https://github.com/etcd-io/bbolt/pull/794) 21 | - [Panicking when a write transaction tries to free a page allocated by itself](https://github.com/etcd-io/bbolt/pull/792) 22 | 23 | ### CMD 24 | - [Add `-gobench-output` flag for `bbolt bench` command](https://github.com/etcd-io/bbolt/pull/765) 25 | 26 | ### Other 27 | - [Bump go version to 1.23.x](https://github.com/etcd-io/bbolt/pull/821) 28 | 29 |
30 | 31 | ## v1.4.0-alpha.1(2024-05-06) 32 | 33 | ### BoltDB 34 | - [Enhance check functionality to support checking starting from a pageId](https://github.com/etcd-io/bbolt/pull/659) 35 | - [Optimize the logger performance for frequent called methods](https://github.com/etcd-io/bbolt/pull/741) 36 | - [Stabilize the behaviour of Prev when the cursor already points to the first element](https://github.com/etcd-io/bbolt/pull/734) 37 | 38 | ### CMD 39 | - [Fix `bbolt keys` and `bbolt get` to prevent them from panicking when no parameter provided](https://github.com/etcd-io/bbolt/pull/682) 40 | - [Fix surgery freelist command in info logs](https://github.com/etcd-io/bbolt/pull/700) 41 | - [Remove txid references in surgery meta command's comment and description](https://github.com/etcd-io/bbolt/pull/703) 42 | - [Add rnd read capabilities to bbolt bench](https://github.com/etcd-io/bbolt/pull/711) 43 | - [Use `cobra.ExactArgs` to simplify the argument number check](https://github.com/etcd-io/bbolt/pull/728) 44 | - [Migrate `bbolt check` command to cobra style](https://github.com/etcd-io/bbolt/pull/723) 45 | - [Simplify the naming of cobra commands](https://github.com/etcd-io/bbolt/pull/732) 46 | - [Aggregate adding completed ops for read test of the `bbolt bench` command](https://github.com/etcd-io/bbolt/pull/721) 47 | - [Add `--from-page` flag to `bbolt check` command](https://github.com/etcd-io/bbolt/pull/737) 48 | 49 | ### Document 50 | - [Add document for a known issue on the writing a value with a length of 0](https://github.com/etcd-io/bbolt/pull/730) 51 | 52 | ### Test 53 | - [Enhance robustness test to cover XFS](https://github.com/etcd-io/bbolt/pull/707) 54 | 55 | ### Other 56 | - [Bump go toolchain version to 1.22.2](https://github.com/etcd-io/bbolt/pull/712) 57 | 58 |
59 | 60 | ## v1.4.0-alpha.0(2024-01-12) 61 | 62 | ### BoltDB 63 | - [Improve the performance of hashmapGetFreePageIDs](https://github.com/etcd-io/bbolt/pull/419) 64 | - [Improve CreateBucketIfNotExists to avoid double searching the same key](https://github.com/etcd-io/bbolt/pull/532) 65 | - [Support Android platform](https://github.com/etcd-io/bbolt/pull/571) 66 | - [Record the count of free page to improve the performance of hashmapFreeCount](https://github.com/etcd-io/bbolt/pull/585) 67 | - [Add logger to bbolt](https://github.com/etcd-io/bbolt/issues/509) 68 | - [Support moving bucket inside the same db](https://github.com/etcd-io/bbolt/pull/635) 69 | - [Support inspecting database structure](https://github.com/etcd-io/bbolt/pull/674) 70 | 71 | ### CMD 72 | - [Add `surgery clear-page-elements` command](https://github.com/etcd-io/bbolt/pull/417) 73 | - [Add `surgery abandon-freelist` command](https://github.com/etcd-io/bbolt/pull/443) 74 | - [Add `bbolt version` command](https://github.com/etcd-io/bbolt/pull/552) 75 | - [Add `bbolt inspect` command](https://github.com/etcd-io/bbolt/pull/674) 76 | - [Add `--no-sync` option to `bbolt compact` command](https://github.com/etcd-io/bbolt/pull/290) 77 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Ben Johnson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BRANCH=`git rev-parse --abbrev-ref HEAD` 2 | COMMIT=`git rev-parse --short HEAD` 3 | GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)" 4 | GOFILES = $(shell find . -name \*.go) 5 | 6 | TESTFLAGS_RACE=-race=false 7 | ifdef ENABLE_RACE 8 | TESTFLAGS_RACE=-race=true 9 | endif 10 | 11 | TESTFLAGS_CPU= 12 | ifdef CPU 13 | TESTFLAGS_CPU=-cpu=$(CPU) 14 | endif 15 | TESTFLAGS = $(TESTFLAGS_RACE) $(TESTFLAGS_CPU) $(EXTRA_TESTFLAGS) 16 | 17 | TESTFLAGS_TIMEOUT=30m 18 | ifdef TIMEOUT 19 | TESTFLAGS_TIMEOUT=$(TIMEOUT) 20 | endif 21 | 22 | TESTFLAGS_ENABLE_STRICT_MODE=false 23 | ifdef ENABLE_STRICT_MODE 24 | TESTFLAGS_ENABLE_STRICT_MODE=$(ENABLE_STRICT_MODE) 25 | endif 26 | 27 | .EXPORT_ALL_VARIABLES: 28 | TEST_ENABLE_STRICT_MODE=${TESTFLAGS_ENABLE_STRICT_MODE} 29 | 30 | .PHONY: fmt 31 | fmt: 32 | @echo "Verifying gofmt, failures can be fixed with ./scripts/fix.sh" 33 | @!(gofmt -l -s -d ${GOFILES} | grep '[a-z]') 34 | 35 | @echo "Verifying goimports, failures can be fixed with ./scripts/fix.sh" 36 | @!(go run golang.org/x/tools/cmd/goimports@latest -l -d ${GOFILES} | grep '[a-z]') 37 | 38 | .PHONY: lint 39 | lint: 40 | golangci-lint run ./... 41 | 42 | .PHONY: test 43 | test: 44 | @echo "hashmap freelist test" 45 | BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT} 46 | BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} ./internal/... 47 | BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} ./cmd/bbolt 48 | 49 | @echo "array freelist test" 50 | BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT} 51 | BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./internal/... 52 | BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./cmd/bbolt 53 | 54 | .PHONY: coverage 55 | coverage: 56 | @echo "hashmap freelist test" 57 | TEST_FREELIST_TYPE=hashmap go test -v -timeout ${TESTFLAGS_TIMEOUT} \ 58 | -coverprofile cover-freelist-hashmap.out -covermode atomic 59 | 60 | @echo "array freelist test" 61 | TEST_FREELIST_TYPE=array go test -v -timeout ${TESTFLAGS_TIMEOUT} \ 62 | -coverprofile cover-freelist-array.out -covermode atomic 63 | 64 | BOLT_CMD=bbolt 65 | 66 | build: 67 | go build -o bin/${BOLT_CMD} ./cmd/${BOLT_CMD} 68 | 69 | .PHONY: clean 70 | clean: # Clean binaries 71 | rm -f ./bin/${BOLT_CMD} 72 | 73 | .PHONY: gofail-enable 74 | gofail-enable: install-gofail 75 | gofail enable . 76 | 77 | .PHONY: gofail-disable 78 | gofail-disable: install-gofail 79 | gofail disable . 80 | 81 | .PHONY: install-gofail 82 | install-gofail: 83 | go install go.etcd.io/gofail 84 | 85 | .PHONY: test-failpoint 86 | test-failpoint: 87 | @echo "[failpoint] hashmap freelist test" 88 | BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} -timeout 30m ./tests/failpoint 89 | 90 | @echo "[failpoint] array freelist test" 91 | BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout 30m ./tests/failpoint 92 | 93 | .PHONY: test-robustness # Running robustness tests requires root permission for now 94 | # TODO: Remove sudo once we fully migrate to the prow infrastructure 95 | test-robustness: gofail-enable build 96 | sudo env PATH=$$PATH go test -v ${TESTFLAGS} ./tests/dmflakey -test.root 97 | sudo env PATH=$(PWD)/bin:$$PATH go test -v ${TESTFLAGS} ${ROBUSTNESS_TESTFLAGS} ./tests/robustness -test.root 98 | 99 | .PHONY: test-benchmark-compare 100 | # Runs benchmark tests on the current git ref and the given REF, and compares 101 | # the two. 102 | test-benchmark-compare: install-benchstat 103 | @git fetch 104 | ./scripts/compare_benchmarks.sh $(REF) 105 | 106 | .PHONY: install-benchstat 107 | install-benchstat: 108 | go install golang.org/x/perf/cmd/benchstat@latest 109 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | 3 | approvers: 4 | - ahrtr # Benjamin Wang 5 | - serathius # Marek Siarkowicz 6 | - ptabor # Piotr Tabor 7 | - spzala # Sahdev Zala 8 | reviewers: 9 | - fuweid # Wei Fu 10 | - tjungblu # Thomas Jungblut 11 | -------------------------------------------------------------------------------- /allocate_test.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | import ( 4 | "testing" 5 | 6 | "go.etcd.io/bbolt/internal/common" 7 | "go.etcd.io/bbolt/internal/freelist" 8 | ) 9 | 10 | func TestTx_allocatePageStats(t *testing.T) { 11 | for n, f := range map[string]freelist.Interface{"hashmap": freelist.NewHashMapFreelist(), "array": freelist.NewArrayFreelist()} { 12 | t.Run(n, func(t *testing.T) { 13 | ids := []common.Pgid{2, 3} 14 | f.Init(ids) 15 | 16 | tx := &Tx{ 17 | db: &DB{ 18 | freelist: f, 19 | pageSize: common.DefaultPageSize, 20 | }, 21 | meta: &common.Meta{}, 22 | pages: make(map[common.Pgid]*common.Page), 23 | } 24 | 25 | txStats := tx.Stats() 26 | prePageCnt := txStats.GetPageCount() 27 | allocateCnt := f.FreeCount() 28 | 29 | if _, err := tx.allocate(allocateCnt); err != nil { 30 | t.Fatal(err) 31 | } 32 | 33 | txStats = tx.Stats() 34 | if txStats.GetPageCount() != prePageCnt+int64(allocateCnt) { 35 | t.Errorf("Allocated %d but got %d page in stats", allocateCnt, txStats.GetPageCount()) 36 | } 37 | }) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /bolt_386.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | // maxMapSize represents the largest mmap size supported by Bolt. 4 | const maxMapSize = 0x7FFFFFFF // 2GB 5 | 6 | // maxAllocSize is the size used when creating array pointers. 7 | const maxAllocSize = 0xFFFFFFF 8 | -------------------------------------------------------------------------------- /bolt_aix.go: -------------------------------------------------------------------------------- 1 | //go:build aix 2 | 3 | package bbolt 4 | 5 | import ( 6 | "fmt" 7 | "syscall" 8 | "time" 9 | "unsafe" 10 | 11 | "golang.org/x/sys/unix" 12 | ) 13 | 14 | // flock acquires an advisory lock on a file descriptor. 15 | func flock(db *DB, exclusive bool, timeout time.Duration) error { 16 | var t time.Time 17 | if timeout != 0 { 18 | t = time.Now() 19 | } 20 | fd := db.file.Fd() 21 | var lockType int16 22 | if exclusive { 23 | lockType = syscall.F_WRLCK 24 | } else { 25 | lockType = syscall.F_RDLCK 26 | } 27 | for { 28 | // Attempt to obtain an exclusive lock. 29 | lock := syscall.Flock_t{Type: lockType} 30 | err := syscall.FcntlFlock(fd, syscall.F_SETLK, &lock) 31 | if err == nil { 32 | return nil 33 | } else if err != syscall.EAGAIN { 34 | return err 35 | } 36 | 37 | // If we timed out then return an error. 38 | if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { 39 | return ErrTimeout 40 | } 41 | 42 | // Wait for a bit and try again. 43 | time.Sleep(flockRetryTimeout) 44 | } 45 | } 46 | 47 | // funlock releases an advisory lock on a file descriptor. 48 | func funlock(db *DB) error { 49 | var lock syscall.Flock_t 50 | lock.Start = 0 51 | lock.Len = 0 52 | lock.Type = syscall.F_UNLCK 53 | lock.Whence = 0 54 | return syscall.FcntlFlock(uintptr(db.file.Fd()), syscall.F_SETLK, &lock) 55 | } 56 | 57 | // mmap memory maps a DB's data file. 58 | func mmap(db *DB, sz int) error { 59 | // Map the data file to memory. 60 | b, err := unix.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags) 61 | if err != nil { 62 | return err 63 | } 64 | 65 | // Advise the kernel that the mmap is accessed randomly. 66 | if err := unix.Madvise(b, syscall.MADV_RANDOM); err != nil { 67 | return fmt.Errorf("madvise: %s", err) 68 | } 69 | 70 | // Save the original byte slice and convert to a byte array pointer. 71 | db.dataref = b 72 | db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0])) 73 | db.datasz = sz 74 | return nil 75 | } 76 | 77 | // munmap unmaps a DB's data file from memory. 78 | func munmap(db *DB) error { 79 | // Ignore the unmap if we have no mapped data. 80 | if db.dataref == nil { 81 | return nil 82 | } 83 | 84 | // Unmap using the original byte slice. 85 | err := unix.Munmap(db.dataref) 86 | db.dataref = nil 87 | db.data = nil 88 | db.datasz = 0 89 | return err 90 | } 91 | -------------------------------------------------------------------------------- /bolt_amd64.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | // maxMapSize represents the largest mmap size supported by Bolt. 4 | const maxMapSize = 0xFFFFFFFFFFFF // 256TB 5 | 6 | // maxAllocSize is the size used when creating array pointers. 7 | const maxAllocSize = 0x7FFFFFFF 8 | -------------------------------------------------------------------------------- /bolt_android.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | import ( 4 | "fmt" 5 | "syscall" 6 | "time" 7 | "unsafe" 8 | 9 | "golang.org/x/sys/unix" 10 | ) 11 | 12 | // flock acquires an advisory lock on a file descriptor. 13 | func flock(db *DB, exclusive bool, timeout time.Duration) error { 14 | var t time.Time 15 | if timeout != 0 { 16 | t = time.Now() 17 | } 18 | fd := db.file.Fd() 19 | var lockType int16 20 | if exclusive { 21 | lockType = syscall.F_WRLCK 22 | } else { 23 | lockType = syscall.F_RDLCK 24 | } 25 | for { 26 | // Attempt to obtain an exclusive lock. 27 | lock := syscall.Flock_t{Type: lockType} 28 | err := syscall.FcntlFlock(fd, syscall.F_SETLK, &lock) 29 | if err == nil { 30 | return nil 31 | } else if err != syscall.EAGAIN { 32 | return err 33 | } 34 | 35 | // If we timed out then return an error. 36 | if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { 37 | return ErrTimeout 38 | } 39 | 40 | // Wait for a bit and try again. 41 | time.Sleep(flockRetryTimeout) 42 | } 43 | } 44 | 45 | // funlock releases an advisory lock on a file descriptor. 46 | func funlock(db *DB) error { 47 | var lock syscall.Flock_t 48 | lock.Start = 0 49 | lock.Len = 0 50 | lock.Type = syscall.F_UNLCK 51 | lock.Whence = 0 52 | return syscall.FcntlFlock(uintptr(db.file.Fd()), syscall.F_SETLK, &lock) 53 | } 54 | 55 | // mmap memory maps a DB's data file. 56 | func mmap(db *DB, sz int) error { 57 | // Map the data file to memory. 58 | b, err := unix.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | // Advise the kernel that the mmap is accessed randomly. 64 | err = unix.Madvise(b, syscall.MADV_RANDOM) 65 | if err != nil && err != syscall.ENOSYS { 66 | // Ignore not implemented error in kernel because it still works. 67 | return fmt.Errorf("madvise: %s", err) 68 | } 69 | 70 | // Save the original byte slice and convert to a byte array pointer. 71 | db.dataref = b 72 | db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0])) 73 | db.datasz = sz 74 | return nil 75 | } 76 | 77 | // munmap unmaps a DB's data file from memory. 78 | func munmap(db *DB) error { 79 | // Ignore the unmap if we have no mapped data. 80 | if db.dataref == nil { 81 | return nil 82 | } 83 | 84 | // Unmap using the original byte slice. 85 | err := unix.Munmap(db.dataref) 86 | db.dataref = nil 87 | db.data = nil 88 | db.datasz = 0 89 | return err 90 | } 91 | -------------------------------------------------------------------------------- /bolt_arm.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | // maxMapSize represents the largest mmap size supported by Bolt. 4 | const maxMapSize = 0x7FFFFFFF // 2GB 5 | 6 | // maxAllocSize is the size used when creating array pointers. 7 | const maxAllocSize = 0xFFFFFFF 8 | -------------------------------------------------------------------------------- /bolt_arm64.go: -------------------------------------------------------------------------------- 1 | //go:build arm64 2 | 3 | package bbolt 4 | 5 | // maxMapSize represents the largest mmap size supported by Bolt. 6 | const maxMapSize = 0xFFFFFFFFFFFF // 256TB 7 | 8 | // maxAllocSize is the size used when creating array pointers. 9 | const maxAllocSize = 0x7FFFFFFF 10 | -------------------------------------------------------------------------------- /bolt_linux.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | import ( 4 | "syscall" 5 | ) 6 | 7 | // fdatasync flushes written data to a file descriptor. 8 | func fdatasync(db *DB) error { 9 | return syscall.Fdatasync(int(db.file.Fd())) 10 | } 11 | -------------------------------------------------------------------------------- /bolt_loong64.go: -------------------------------------------------------------------------------- 1 | //go:build loong64 2 | 3 | package bbolt 4 | 5 | // maxMapSize represents the largest mmap size supported by Bolt. 6 | const maxMapSize = 0xFFFFFFFFFFFF // 256TB 7 | 8 | // maxAllocSize is the size used when creating array pointers. 9 | const maxAllocSize = 0x7FFFFFFF 10 | -------------------------------------------------------------------------------- /bolt_mips64x.go: -------------------------------------------------------------------------------- 1 | //go:build mips64 || mips64le 2 | 3 | package bbolt 4 | 5 | // maxMapSize represents the largest mmap size supported by Bolt. 6 | const maxMapSize = 0x8000000000 // 512GB 7 | 8 | // maxAllocSize is the size used when creating array pointers. 9 | const maxAllocSize = 0x7FFFFFFF 10 | -------------------------------------------------------------------------------- /bolt_mipsx.go: -------------------------------------------------------------------------------- 1 | //go:build mips || mipsle 2 | 3 | package bbolt 4 | 5 | // maxMapSize represents the largest mmap size supported by Bolt. 6 | const maxMapSize = 0x40000000 // 1GB 7 | 8 | // maxAllocSize is the size used when creating array pointers. 9 | const maxAllocSize = 0xFFFFFFF 10 | -------------------------------------------------------------------------------- /bolt_openbsd.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | import ( 4 | "golang.org/x/sys/unix" 5 | ) 6 | 7 | func msync(db *DB) error { 8 | return unix.Msync(db.data[:db.datasz], unix.MS_INVALIDATE) 9 | } 10 | 11 | func fdatasync(db *DB) error { 12 | if db.data != nil { 13 | return msync(db) 14 | } 15 | return db.file.Sync() 16 | } 17 | -------------------------------------------------------------------------------- /bolt_ppc.go: -------------------------------------------------------------------------------- 1 | //go:build ppc 2 | 3 | package bbolt 4 | 5 | // maxMapSize represents the largest mmap size supported by Bolt. 6 | const maxMapSize = 0x7FFFFFFF // 2GB 7 | 8 | // maxAllocSize is the size used when creating array pointers. 9 | const maxAllocSize = 0xFFFFFFF 10 | -------------------------------------------------------------------------------- /bolt_ppc64.go: -------------------------------------------------------------------------------- 1 | //go:build ppc64 2 | 3 | package bbolt 4 | 5 | // maxMapSize represents the largest mmap size supported by Bolt. 6 | const maxMapSize = 0xFFFFFFFFFFFF // 256TB 7 | 8 | // maxAllocSize is the size used when creating array pointers. 9 | const maxAllocSize = 0x7FFFFFFF 10 | -------------------------------------------------------------------------------- /bolt_ppc64le.go: -------------------------------------------------------------------------------- 1 | //go:build ppc64le 2 | 3 | package bbolt 4 | 5 | // maxMapSize represents the largest mmap size supported by Bolt. 6 | const maxMapSize = 0xFFFFFFFFFFFF // 256TB 7 | 8 | // maxAllocSize is the size used when creating array pointers. 9 | const maxAllocSize = 0x7FFFFFFF 10 | -------------------------------------------------------------------------------- /bolt_riscv64.go: -------------------------------------------------------------------------------- 1 | //go:build riscv64 2 | 3 | package bbolt 4 | 5 | // maxMapSize represents the largest mmap size supported by Bolt. 6 | const maxMapSize = 0xFFFFFFFFFFFF // 256TB 7 | 8 | // maxAllocSize is the size used when creating array pointers. 9 | const maxAllocSize = 0x7FFFFFFF 10 | -------------------------------------------------------------------------------- /bolt_s390x.go: -------------------------------------------------------------------------------- 1 | //go:build s390x 2 | 3 | package bbolt 4 | 5 | // maxMapSize represents the largest mmap size supported by Bolt. 6 | const maxMapSize = 0xFFFFFFFFFFFF // 256TB 7 | 8 | // maxAllocSize is the size used when creating array pointers. 9 | const maxAllocSize = 0x7FFFFFFF 10 | -------------------------------------------------------------------------------- /bolt_solaris.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | import ( 4 | "fmt" 5 | "syscall" 6 | "time" 7 | "unsafe" 8 | 9 | "golang.org/x/sys/unix" 10 | ) 11 | 12 | // flock acquires an advisory lock on a file descriptor. 13 | func flock(db *DB, exclusive bool, timeout time.Duration) error { 14 | var t time.Time 15 | if timeout != 0 { 16 | t = time.Now() 17 | } 18 | fd := db.file.Fd() 19 | var lockType int16 20 | if exclusive { 21 | lockType = syscall.F_WRLCK 22 | } else { 23 | lockType = syscall.F_RDLCK 24 | } 25 | for { 26 | // Attempt to obtain an exclusive lock. 27 | lock := syscall.Flock_t{Type: lockType} 28 | err := syscall.FcntlFlock(fd, syscall.F_SETLK, &lock) 29 | if err == nil { 30 | return nil 31 | } else if err != syscall.EAGAIN { 32 | return err 33 | } 34 | 35 | // If we timed out then return an error. 36 | if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { 37 | return ErrTimeout 38 | } 39 | 40 | // Wait for a bit and try again. 41 | time.Sleep(flockRetryTimeout) 42 | } 43 | } 44 | 45 | // funlock releases an advisory lock on a file descriptor. 46 | func funlock(db *DB) error { 47 | var lock syscall.Flock_t 48 | lock.Start = 0 49 | lock.Len = 0 50 | lock.Type = syscall.F_UNLCK 51 | lock.Whence = 0 52 | return syscall.FcntlFlock(uintptr(db.file.Fd()), syscall.F_SETLK, &lock) 53 | } 54 | 55 | // mmap memory maps a DB's data file. 56 | func mmap(db *DB, sz int) error { 57 | // Map the data file to memory. 58 | b, err := unix.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | // Advise the kernel that the mmap is accessed randomly. 64 | if err := unix.Madvise(b, syscall.MADV_RANDOM); err != nil { 65 | return fmt.Errorf("madvise: %s", err) 66 | } 67 | 68 | // Save the original byte slice and convert to a byte array pointer. 69 | db.dataref = b 70 | db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0])) 71 | db.datasz = sz 72 | return nil 73 | } 74 | 75 | // munmap unmaps a DB's data file from memory. 76 | func munmap(db *DB) error { 77 | // Ignore the unmap if we have no mapped data. 78 | if db.dataref == nil { 79 | return nil 80 | } 81 | 82 | // Unmap using the original byte slice. 83 | err := unix.Munmap(db.dataref) 84 | db.dataref = nil 85 | db.data = nil 86 | db.datasz = 0 87 | return err 88 | } 89 | -------------------------------------------------------------------------------- /bolt_unix.go: -------------------------------------------------------------------------------- 1 | //go:build !windows && !plan9 && !solaris && !aix && !android 2 | 3 | package bbolt 4 | 5 | import ( 6 | "fmt" 7 | "syscall" 8 | "time" 9 | "unsafe" 10 | 11 | "golang.org/x/sys/unix" 12 | 13 | "go.etcd.io/bbolt/errors" 14 | ) 15 | 16 | // flock acquires an advisory lock on a file descriptor. 17 | func flock(db *DB, exclusive bool, timeout time.Duration) error { 18 | var t time.Time 19 | if timeout != 0 { 20 | t = time.Now() 21 | } 22 | fd := db.file.Fd() 23 | flag := syscall.LOCK_NB 24 | if exclusive { 25 | flag |= syscall.LOCK_EX 26 | } else { 27 | flag |= syscall.LOCK_SH 28 | } 29 | for { 30 | // Attempt to obtain an exclusive lock. 31 | err := syscall.Flock(int(fd), flag) 32 | if err == nil { 33 | return nil 34 | } else if err != syscall.EWOULDBLOCK { 35 | return err 36 | } 37 | 38 | // If we timed out then return an error. 39 | if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { 40 | return errors.ErrTimeout 41 | } 42 | 43 | // Wait for a bit and try again. 44 | time.Sleep(flockRetryTimeout) 45 | } 46 | } 47 | 48 | // funlock releases an advisory lock on a file descriptor. 49 | func funlock(db *DB) error { 50 | return syscall.Flock(int(db.file.Fd()), syscall.LOCK_UN) 51 | } 52 | 53 | // mmap memory maps a DB's data file. 54 | func mmap(db *DB, sz int) error { 55 | // Map the data file to memory. 56 | b, err := unix.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | // Advise the kernel that the mmap is accessed randomly. 62 | err = unix.Madvise(b, syscall.MADV_RANDOM) 63 | if err != nil && err != syscall.ENOSYS { 64 | // Ignore not implemented error in kernel because it still works. 65 | return fmt.Errorf("madvise: %s", err) 66 | } 67 | 68 | // Save the original byte slice and convert to a byte array pointer. 69 | db.dataref = b 70 | db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0])) 71 | db.datasz = sz 72 | return nil 73 | } 74 | 75 | // munmap unmaps a DB's data file from memory. 76 | func munmap(db *DB) error { 77 | // Ignore the unmap if we have no mapped data. 78 | if db.dataref == nil { 79 | return nil 80 | } 81 | 82 | // Unmap using the original byte slice. 83 | err := unix.Munmap(db.dataref) 84 | db.dataref = nil 85 | db.data = nil 86 | db.datasz = 0 87 | return err 88 | } 89 | -------------------------------------------------------------------------------- /bolt_windows.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "syscall" 7 | "time" 8 | "unsafe" 9 | 10 | "golang.org/x/sys/windows" 11 | 12 | "go.etcd.io/bbolt/errors" 13 | ) 14 | 15 | // fdatasync flushes written data to a file descriptor. 16 | func fdatasync(db *DB) error { 17 | return db.file.Sync() 18 | } 19 | 20 | // flock acquires an advisory lock on a file descriptor. 21 | func flock(db *DB, exclusive bool, timeout time.Duration) error { 22 | var t time.Time 23 | if timeout != 0 { 24 | t = time.Now() 25 | } 26 | var flags uint32 = windows.LOCKFILE_FAIL_IMMEDIATELY 27 | if exclusive { 28 | flags |= windows.LOCKFILE_EXCLUSIVE_LOCK 29 | } 30 | for { 31 | // Fix for https://github.com/etcd-io/bbolt/issues/121. Use byte-range 32 | // -1..0 as the lock on the database file. 33 | var m1 uint32 = (1 << 32) - 1 // -1 in a uint32 34 | err := windows.LockFileEx(windows.Handle(db.file.Fd()), flags, 0, 1, 0, &windows.Overlapped{ 35 | Offset: m1, 36 | OffsetHigh: m1, 37 | }) 38 | 39 | if err == nil { 40 | return nil 41 | } else if err != windows.ERROR_LOCK_VIOLATION { 42 | return err 43 | } 44 | 45 | // If we timed oumercit then return an error. 46 | if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { 47 | return errors.ErrTimeout 48 | } 49 | 50 | // Wait for a bit and try again. 51 | time.Sleep(flockRetryTimeout) 52 | } 53 | } 54 | 55 | // funlock releases an advisory lock on a file descriptor. 56 | func funlock(db *DB) error { 57 | var m1 uint32 = (1 << 32) - 1 // -1 in a uint32 58 | return windows.UnlockFileEx(windows.Handle(db.file.Fd()), 0, 1, 0, &windows.Overlapped{ 59 | Offset: m1, 60 | OffsetHigh: m1, 61 | }) 62 | } 63 | 64 | // mmap memory maps a DB's data file. 65 | // Based on: https://github.com/edsrzf/mmap-go 66 | func mmap(db *DB, sz int) error { 67 | var sizelo, sizehi uint32 68 | 69 | if !db.readOnly { 70 | if db.MaxSize > 0 && sz > db.MaxSize { 71 | // The max size only limits future writes; however, we don’t block opening 72 | // and mapping the database if it already exceeds the limit. 73 | fileSize, err := db.fileSize() 74 | if err != nil { 75 | return fmt.Errorf("could not check existing db file size: %s", err) 76 | } 77 | 78 | if sz > fileSize { 79 | return errors.ErrMaxSizeReached 80 | } 81 | } 82 | 83 | // Truncate the database to the size of the mmap. 84 | if err := db.file.Truncate(int64(sz)); err != nil { 85 | return fmt.Errorf("truncate: %s", err) 86 | } 87 | sizehi = uint32(sz >> 32) 88 | sizelo = uint32(sz) 89 | } 90 | 91 | // Open a file mapping handle. 92 | h, errno := syscall.CreateFileMapping(syscall.Handle(db.file.Fd()), nil, syscall.PAGE_READONLY, sizehi, sizelo, nil) 93 | if h == 0 { 94 | return os.NewSyscallError("CreateFileMapping", errno) 95 | } 96 | 97 | // Create the memory map. 98 | addr, errno := syscall.MapViewOfFile(h, syscall.FILE_MAP_READ, 0, 0, 0) 99 | if addr == 0 { 100 | // Do our best and report error returned from MapViewOfFile. 101 | _ = syscall.CloseHandle(h) 102 | return os.NewSyscallError("MapViewOfFile", errno) 103 | } 104 | 105 | // Close mapping handle. 106 | if err := syscall.CloseHandle(syscall.Handle(h)); err != nil { 107 | return os.NewSyscallError("CloseHandle", err) 108 | } 109 | 110 | // Convert to a byte array. 111 | db.data = (*[maxMapSize]byte)(unsafe.Pointer(addr)) 112 | db.datasz = sz 113 | 114 | return nil 115 | } 116 | 117 | // munmap unmaps a pointer from a file. 118 | // Based on: https://github.com/edsrzf/mmap-go 119 | func munmap(db *DB) error { 120 | if db.data == nil { 121 | return nil 122 | } 123 | 124 | addr := (uintptr)(unsafe.Pointer(&db.data[0])) 125 | var err1 error 126 | if err := syscall.UnmapViewOfFile(addr); err != nil { 127 | err1 = os.NewSyscallError("UnmapViewOfFile", err) 128 | } 129 | db.data = nil 130 | db.datasz = 0 131 | return err1 132 | } 133 | -------------------------------------------------------------------------------- /boltsync_unix.go: -------------------------------------------------------------------------------- 1 | //go:build !windows && !plan9 && !linux && !openbsd 2 | 3 | package bbolt 4 | 5 | // fdatasync flushes written data to a file descriptor. 6 | func fdatasync(db *DB) error { 7 | return db.file.Sync() 8 | } 9 | -------------------------------------------------------------------------------- /cmd/bbolt/OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | 3 | approvers: 4 | - ahrtr # Benjamin Wang 5 | - fuweid # Wei Fu 6 | - serathius # Marek Siarkowicz 7 | - ptabor # Piotr Tabor 8 | - spzala # Sahdev Zala 9 | - tjungblu # Thomas Jungblut 10 | reviewers: 11 | - elbehery # Mustafa Elbehery 12 | - ivanvc # Ivan Valdes 13 | -------------------------------------------------------------------------------- /cmd/bbolt/command_check.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | "github.com/spf13/pflag" 8 | 9 | bolt "go.etcd.io/bbolt" 10 | "go.etcd.io/bbolt/internal/guts_cli" 11 | ) 12 | 13 | type checkOptions struct { 14 | fromPageID uint64 15 | } 16 | 17 | func (o *checkOptions) AddFlags(fs *pflag.FlagSet) { 18 | fs.Uint64VarP(&o.fromPageID, "from-page", "", o.fromPageID, "check db integrity starting from the given page ID") 19 | } 20 | 21 | func newCheckCommand() *cobra.Command { 22 | var o checkOptions 23 | checkCmd := &cobra.Command{ 24 | Use: "check ", 25 | Short: "verify integrity of bbolt database data", 26 | Args: cobra.ExactArgs(1), 27 | RunE: func(cmd *cobra.Command, args []string) error { 28 | return checkFunc(cmd, args[0], o) 29 | }, 30 | } 31 | 32 | o.AddFlags(checkCmd.Flags()) 33 | return checkCmd 34 | } 35 | 36 | func checkFunc(cmd *cobra.Command, dbPath string, cfg checkOptions) error { 37 | if _, err := checkSourceDBPath(dbPath); err != nil { 38 | return err 39 | } 40 | 41 | // Open database. 42 | db, err := bolt.Open(dbPath, 0600, &bolt.Options{ 43 | ReadOnly: true, 44 | PreLoadFreelist: true, 45 | }) 46 | if err != nil { 47 | return err 48 | } 49 | defer db.Close() 50 | 51 | opts := []bolt.CheckOption{bolt.WithKVStringer(CmdKvStringer())} 52 | if cfg.fromPageID != 0 { 53 | opts = append(opts, bolt.WithPageId(cfg.fromPageID)) 54 | } 55 | // Perform consistency check. 56 | return db.View(func(tx *bolt.Tx) error { 57 | var count int 58 | for err := range tx.Check(opts...) { 59 | fmt.Fprintln(cmd.OutOrStdout(), err) 60 | count++ 61 | } 62 | 63 | // Print summary of errors. 64 | if count > 0 { 65 | fmt.Fprintf(cmd.OutOrStdout(), "%d errors found\n", count) 66 | return guts_cli.ErrCorrupt 67 | } 68 | 69 | // Notify user that database is valid. 70 | fmt.Fprintln(cmd.OutOrStdout(), "OK") 71 | return nil 72 | }) 73 | } 74 | -------------------------------------------------------------------------------- /cmd/bbolt/command_check_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | 10 | main "go.etcd.io/bbolt/cmd/bbolt" 11 | "go.etcd.io/bbolt/internal/btesting" 12 | "go.etcd.io/bbolt/internal/guts_cli" 13 | ) 14 | 15 | func TestCheckCommand_Run(t *testing.T) { 16 | testCases := []struct { 17 | name string 18 | args []string 19 | expErr error 20 | expOutput string 21 | }{ 22 | { 23 | name: "check whole db", 24 | args: []string{"check", "path"}, 25 | expErr: nil, 26 | expOutput: "OK\n", 27 | }, 28 | { 29 | name: "check valid pageId", 30 | args: []string{"check", "path", "--from-page", "3"}, 31 | expErr: nil, 32 | expOutput: "OK\n", 33 | }, 34 | { 35 | name: "check invalid pageId", 36 | args: []string{"check", "path", "--from-page", "1"}, 37 | expErr: guts_cli.ErrCorrupt, 38 | expOutput: "page ID (1) out of range [2, 4)", 39 | }, 40 | } 41 | 42 | for _, tc := range testCases { 43 | t.Run(tc.name, func(t *testing.T) { 44 | 45 | t.Log("Creating sample DB") 46 | db := btesting.MustCreateDB(t) 47 | db.Close() 48 | defer requireDBNoChange(t, dbData(t, db.Path()), db.Path()) 49 | 50 | t.Log("Running check cmd") 51 | rootCmd := main.NewRootCommand() 52 | outputBuf := bytes.NewBufferString("") // capture output for assertion 53 | rootCmd.SetOut(outputBuf) 54 | 55 | tc.args[1] = db.Path() // path to be replaced with db.Path() 56 | rootCmd.SetArgs(tc.args) 57 | err := rootCmd.Execute() 58 | require.Equal(t, tc.expErr, err) 59 | 60 | t.Log("Checking output") 61 | output, err := io.ReadAll(outputBuf) 62 | require.NoError(t, err) 63 | require.Containsf(t, string(output), tc.expOutput, "unexpected stdout:\n\n%s", string(output)) 64 | }) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /cmd/bbolt/command_inspect.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/spf13/cobra" 9 | 10 | bolt "go.etcd.io/bbolt" 11 | ) 12 | 13 | func newInspectCommand() *cobra.Command { 14 | inspectCmd := &cobra.Command{ 15 | Use: "inspect ", 16 | Short: "inspect the structure of the database", 17 | Args: cobra.ExactArgs(1), 18 | RunE: func(cmd *cobra.Command, args []string) error { 19 | return inspectFunc(args[0]) 20 | }, 21 | } 22 | 23 | return inspectCmd 24 | } 25 | 26 | func inspectFunc(srcDBPath string) error { 27 | if _, err := checkSourceDBPath(srcDBPath); err != nil { 28 | return err 29 | } 30 | 31 | db, err := bolt.Open(srcDBPath, 0600, &bolt.Options{ReadOnly: true}) 32 | if err != nil { 33 | return err 34 | } 35 | defer db.Close() 36 | 37 | return db.View(func(tx *bolt.Tx) error { 38 | bs := tx.Inspect() 39 | out, err := json.MarshalIndent(bs, "", " ") 40 | if err != nil { 41 | return err 42 | } 43 | fmt.Fprintln(os.Stdout, string(out)) 44 | return nil 45 | }) 46 | } 47 | -------------------------------------------------------------------------------- /cmd/bbolt/command_inspect_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | bolt "go.etcd.io/bbolt" 9 | main "go.etcd.io/bbolt/cmd/bbolt" 10 | "go.etcd.io/bbolt/internal/btesting" 11 | ) 12 | 13 | func TestInspect(t *testing.T) { 14 | pageSize := 4096 15 | db := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize}) 16 | srcPath := db.Path() 17 | db.Close() 18 | 19 | defer requireDBNoChange(t, dbData(t, db.Path()), db.Path()) 20 | 21 | rootCmd := main.NewRootCommand() 22 | rootCmd.SetArgs([]string{ 23 | "inspect", srcPath, 24 | }) 25 | err := rootCmd.Execute() 26 | require.NoError(t, err) 27 | } 28 | -------------------------------------------------------------------------------- /cmd/bbolt/command_root.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | ) 6 | 7 | const ( 8 | cliName = "bbolt" 9 | cliDescription = "A simple command line tool for inspecting bbolt databases" 10 | ) 11 | 12 | func NewRootCommand() *cobra.Command { 13 | rootCmd := &cobra.Command{ 14 | Use: cliName, 15 | Short: cliDescription, 16 | Version: "dev", 17 | } 18 | 19 | rootCmd.AddCommand( 20 | newVersionCommand(), 21 | newSurgeryCommand(), 22 | newInspectCommand(), 23 | newCheckCommand(), 24 | ) 25 | 26 | return rootCmd 27 | } 28 | -------------------------------------------------------------------------------- /cmd/bbolt/command_surgery_freelist.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/spf13/cobra" 8 | 9 | bolt "go.etcd.io/bbolt" 10 | "go.etcd.io/bbolt/internal/common" 11 | "go.etcd.io/bbolt/internal/surgeon" 12 | ) 13 | 14 | func newSurgeryFreelistCommand() *cobra.Command { 15 | cmd := &cobra.Command{ 16 | Use: "freelist ", 17 | Short: "freelist related surgery commands", 18 | } 19 | 20 | cmd.AddCommand(newSurgeryFreelistAbandonCommand()) 21 | cmd.AddCommand(newSurgeryFreelistRebuildCommand()) 22 | 23 | return cmd 24 | } 25 | 26 | func newSurgeryFreelistAbandonCommand() *cobra.Command { 27 | var o surgeryBaseOptions 28 | abandonFreelistCmd := &cobra.Command{ 29 | Use: "abandon ", 30 | Short: "Abandon the freelist from both meta pages", 31 | Args: cobra.ExactArgs(1), 32 | RunE: func(cmd *cobra.Command, args []string) error { 33 | if err := o.Validate(); err != nil { 34 | return err 35 | } 36 | return surgeryFreelistAbandonFunc(args[0], o) 37 | }, 38 | } 39 | o.AddFlags(abandonFreelistCmd.Flags()) 40 | 41 | return abandonFreelistCmd 42 | } 43 | 44 | func surgeryFreelistAbandonFunc(srcDBPath string, cfg surgeryBaseOptions) error { 45 | if _, err := checkSourceDBPath(srcDBPath); err != nil { 46 | return err 47 | } 48 | 49 | if err := common.CopyFile(srcDBPath, cfg.outputDBFilePath); err != nil { 50 | return fmt.Errorf("[freelist abandon] copy file failed: %w", err) 51 | } 52 | 53 | if err := surgeon.ClearFreelist(cfg.outputDBFilePath); err != nil { 54 | return fmt.Errorf("abandom-freelist command failed: %w", err) 55 | } 56 | 57 | fmt.Fprintf(os.Stdout, "The freelist was abandoned in both meta pages.\nIt may cause some delay on next startup because bbolt needs to scan the whole db to reconstruct the free list.\n") 58 | return nil 59 | } 60 | 61 | func newSurgeryFreelistRebuildCommand() *cobra.Command { 62 | var o surgeryBaseOptions 63 | rebuildFreelistCmd := &cobra.Command{ 64 | Use: "rebuild ", 65 | Short: "Rebuild the freelist", 66 | Args: cobra.ExactArgs(1), 67 | RunE: func(cmd *cobra.Command, args []string) error { 68 | if err := o.Validate(); err != nil { 69 | return err 70 | } 71 | return surgeryFreelistRebuildFunc(args[0], o) 72 | }, 73 | } 74 | o.AddFlags(rebuildFreelistCmd.Flags()) 75 | 76 | return rebuildFreelistCmd 77 | } 78 | 79 | func surgeryFreelistRebuildFunc(srcDBPath string, cfg surgeryBaseOptions) error { 80 | // Ensure source file exists. 81 | fi, err := checkSourceDBPath(srcDBPath) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | // make sure the freelist isn't present in the file. 87 | meta, err := readMetaPage(srcDBPath) 88 | if err != nil { 89 | return err 90 | } 91 | if meta.IsFreelistPersisted() { 92 | return ErrSurgeryFreelistAlreadyExist 93 | } 94 | 95 | if err := common.CopyFile(srcDBPath, cfg.outputDBFilePath); err != nil { 96 | return fmt.Errorf("[freelist rebuild] copy file failed: %w", err) 97 | } 98 | 99 | // bboltDB automatically reconstruct & sync freelist in write mode. 100 | db, err := bolt.Open(cfg.outputDBFilePath, fi.Mode(), &bolt.Options{NoFreelistSync: false}) 101 | if err != nil { 102 | return fmt.Errorf("[freelist rebuild] open db file failed: %w", err) 103 | } 104 | err = db.Close() 105 | if err != nil { 106 | return fmt.Errorf("[freelist rebuild] close db file failed: %w", err) 107 | } 108 | 109 | fmt.Fprintf(os.Stdout, "The freelist was successfully rebuilt.\n") 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /cmd/bbolt/command_surgery_freelist_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | 10 | bolt "go.etcd.io/bbolt" 11 | main "go.etcd.io/bbolt/cmd/bbolt" 12 | "go.etcd.io/bbolt/internal/btesting" 13 | "go.etcd.io/bbolt/internal/common" 14 | ) 15 | 16 | func TestSurgery_Freelist_Abandon(t *testing.T) { 17 | pageSize := 4096 18 | db := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize}) 19 | srcPath := db.Path() 20 | 21 | defer requireDBNoChange(t, dbData(t, srcPath), srcPath) 22 | 23 | rootCmd := main.NewRootCommand() 24 | output := filepath.Join(t.TempDir(), "db") 25 | rootCmd.SetArgs([]string{ 26 | "surgery", "freelist", "abandon", srcPath, 27 | "--output", output, 28 | }) 29 | err := rootCmd.Execute() 30 | require.NoError(t, err) 31 | 32 | meta0 := loadMetaPage(t, output, 0) 33 | assert.Equal(t, common.PgidNoFreelist, meta0.Freelist()) 34 | meta1 := loadMetaPage(t, output, 1) 35 | assert.Equal(t, common.PgidNoFreelist, meta1.Freelist()) 36 | } 37 | 38 | func TestSurgery_Freelist_Rebuild(t *testing.T) { 39 | testCases := []struct { 40 | name string 41 | hasFreelist bool 42 | expectedError error 43 | }{ 44 | { 45 | name: "normal operation", 46 | hasFreelist: false, 47 | expectedError: nil, 48 | }, 49 | { 50 | name: "already has freelist", 51 | hasFreelist: true, 52 | expectedError: main.ErrSurgeryFreelistAlreadyExist, 53 | }, 54 | } 55 | 56 | for _, tc := range testCases { 57 | tc := tc 58 | t.Run(tc.name, func(t *testing.T) { 59 | pageSize := 4096 60 | db := btesting.MustCreateDBWithOption(t, &bolt.Options{ 61 | PageSize: pageSize, 62 | NoFreelistSync: !tc.hasFreelist, 63 | }) 64 | srcPath := db.Path() 65 | 66 | err := db.Update(func(tx *bolt.Tx) error { 67 | // do nothing 68 | return nil 69 | }) 70 | require.NoError(t, err) 71 | 72 | defer requireDBNoChange(t, dbData(t, srcPath), srcPath) 73 | 74 | // Verify the freelist isn't synced in the beginning 75 | meta := readMetaPage(t, srcPath) 76 | if tc.hasFreelist { 77 | if meta.Freelist() <= 1 || meta.Freelist() >= meta.Pgid() { 78 | t.Fatalf("freelist (%d) isn't in the valid range (1, %d)", meta.Freelist(), meta.Pgid()) 79 | } 80 | } else { 81 | require.Equal(t, common.PgidNoFreelist, meta.Freelist()) 82 | } 83 | 84 | // Execute `surgery freelist rebuild` command 85 | rootCmd := main.NewRootCommand() 86 | output := filepath.Join(t.TempDir(), "db") 87 | rootCmd.SetArgs([]string{ 88 | "surgery", "freelist", "rebuild", srcPath, 89 | "--output", output, 90 | }) 91 | err = rootCmd.Execute() 92 | require.Equal(t, tc.expectedError, err) 93 | 94 | if tc.expectedError == nil { 95 | // Verify the freelist has already been rebuilt. 96 | meta = readMetaPage(t, output) 97 | if meta.Freelist() <= 1 || meta.Freelist() >= meta.Pgid() { 98 | t.Fatalf("freelist (%d) isn't in the valid range (1, %d)", meta.Freelist(), meta.Pgid()) 99 | } 100 | } 101 | }) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /cmd/bbolt/command_surgery_meta.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/spf13/cobra" 11 | "github.com/spf13/pflag" 12 | 13 | "go.etcd.io/bbolt/internal/common" 14 | ) 15 | 16 | const ( 17 | metaFieldPageSize = "pageSize" 18 | metaFieldRoot = "root" 19 | metaFieldFreelist = "freelist" 20 | metaFieldPgid = "pgid" 21 | ) 22 | 23 | func newSurgeryMetaCommand() *cobra.Command { 24 | cmd := &cobra.Command{ 25 | Use: "meta ", 26 | Short: "meta page related surgery commands", 27 | } 28 | 29 | cmd.AddCommand(newSurgeryMetaValidateCommand()) 30 | cmd.AddCommand(newSurgeryMetaUpdateCommand()) 31 | 32 | return cmd 33 | } 34 | 35 | func newSurgeryMetaValidateCommand() *cobra.Command { 36 | metaValidateCmd := &cobra.Command{ 37 | Use: "validate ", 38 | Short: "Validate both meta pages", 39 | Args: cobra.ExactArgs(1), 40 | RunE: func(cmd *cobra.Command, args []string) error { 41 | return surgeryMetaValidateFunc(args[0]) 42 | }, 43 | } 44 | return metaValidateCmd 45 | } 46 | 47 | func surgeryMetaValidateFunc(srcDBPath string) error { 48 | if _, err := checkSourceDBPath(srcDBPath); err != nil { 49 | return err 50 | } 51 | 52 | var pageSize uint32 53 | 54 | for i := 0; i <= 1; i++ { 55 | m, _, err := ReadMetaPageAt(srcDBPath, uint32(i), pageSize) 56 | if err != nil { 57 | return fmt.Errorf("read meta page %d failed: %w", i, err) 58 | } 59 | if mValidateErr := m.Validate(); mValidateErr != nil { 60 | fmt.Fprintf(os.Stdout, "WARNING: The meta page %d isn't valid: %v!\n", i, mValidateErr) 61 | } else { 62 | fmt.Fprintf(os.Stdout, "The meta page %d is valid!\n", i) 63 | } 64 | 65 | pageSize = m.PageSize() 66 | } 67 | 68 | return nil 69 | } 70 | 71 | type surgeryMetaUpdateOptions struct { 72 | surgeryBaseOptions 73 | fields []string 74 | metaPageId uint32 75 | } 76 | 77 | var allowedMetaUpdateFields = map[string]struct{}{ 78 | metaFieldPageSize: {}, 79 | metaFieldRoot: {}, 80 | metaFieldFreelist: {}, 81 | metaFieldPgid: {}, 82 | } 83 | 84 | // AddFlags sets the flags for `meta update` command. 85 | // Example: --fields root:16,freelist:8 --fields pgid:128 86 | // Result: []string{"root:16", "freelist:8", "pgid:128"} 87 | func (o *surgeryMetaUpdateOptions) AddFlags(fs *pflag.FlagSet) { 88 | o.surgeryBaseOptions.AddFlags(fs) 89 | fs.StringSliceVarP(&o.fields, "fields", "", o.fields, "comma separated list of fields (supported fields: pageSize, root, freelist and pgid) to be updated, and each item is a colon-separated key-value pair") 90 | fs.Uint32VarP(&o.metaPageId, "meta-page", "", o.metaPageId, "the meta page ID to operate on, valid values are 0 and 1") 91 | } 92 | 93 | func (o *surgeryMetaUpdateOptions) Validate() error { 94 | if err := o.surgeryBaseOptions.Validate(); err != nil { 95 | return err 96 | } 97 | 98 | if o.metaPageId > 1 { 99 | return fmt.Errorf("invalid meta page id: %d", o.metaPageId) 100 | } 101 | 102 | for _, field := range o.fields { 103 | kv := strings.Split(field, ":") 104 | if len(kv) != 2 { 105 | return fmt.Errorf("invalid key-value pair: %s", field) 106 | } 107 | 108 | if _, ok := allowedMetaUpdateFields[kv[0]]; !ok { 109 | return fmt.Errorf("field %q isn't allowed to be updated", kv[0]) 110 | } 111 | 112 | if _, err := strconv.ParseUint(kv[1], 10, 64); err != nil { 113 | return fmt.Errorf("invalid value %q for field %q", kv[1], kv[0]) 114 | } 115 | } 116 | 117 | return nil 118 | } 119 | 120 | func newSurgeryMetaUpdateCommand() *cobra.Command { 121 | var o surgeryMetaUpdateOptions 122 | metaUpdateCmd := &cobra.Command{ 123 | Use: "update ", 124 | Short: "Update fields in meta pages", 125 | Args: cobra.ExactArgs(1), 126 | RunE: func(cmd *cobra.Command, args []string) error { 127 | if err := o.Validate(); err != nil { 128 | return err 129 | } 130 | return surgeryMetaUpdateFunc(args[0], o) 131 | }, 132 | } 133 | o.AddFlags(metaUpdateCmd.Flags()) 134 | return metaUpdateCmd 135 | } 136 | 137 | func surgeryMetaUpdateFunc(srcDBPath string, cfg surgeryMetaUpdateOptions) error { 138 | if _, err := checkSourceDBPath(srcDBPath); err != nil { 139 | return err 140 | } 141 | 142 | if err := common.CopyFile(srcDBPath, cfg.outputDBFilePath); err != nil { 143 | return fmt.Errorf("[meta update] copy file failed: %w", err) 144 | } 145 | 146 | // read the page size from the first meta page if we want to edit the second meta page. 147 | var pageSize uint32 148 | if cfg.metaPageId == 1 { 149 | m0, _, err := ReadMetaPageAt(cfg.outputDBFilePath, 0, pageSize) 150 | if err != nil { 151 | return fmt.Errorf("read the first meta page failed: %w", err) 152 | } 153 | pageSize = m0.PageSize() 154 | } 155 | 156 | // update the specified meta page 157 | m, buf, err := ReadMetaPageAt(cfg.outputDBFilePath, cfg.metaPageId, pageSize) 158 | if err != nil { 159 | return fmt.Errorf("read meta page %d failed: %w", cfg.metaPageId, err) 160 | } 161 | mChanged := updateMetaField(m, parseFields(cfg.fields)) 162 | if mChanged { 163 | if err := writeMetaPageAt(cfg.outputDBFilePath, buf, cfg.metaPageId, pageSize); err != nil { 164 | return fmt.Errorf("[meta update] write meta page %d failed: %w", cfg.metaPageId, err) 165 | } 166 | } 167 | 168 | if cfg.metaPageId == 1 && pageSize != m.PageSize() { 169 | fmt.Fprintf(os.Stdout, "WARNING: The page size (%d) in the first meta page doesn't match the second meta page (%d)\n", pageSize, m.PageSize()) 170 | } 171 | 172 | // Display results 173 | if !mChanged { 174 | fmt.Fprintln(os.Stdout, "Nothing changed!") 175 | } 176 | 177 | if mChanged { 178 | fmt.Fprintf(os.Stdout, "The meta page %d has been updated!\n", cfg.metaPageId) 179 | } 180 | 181 | return nil 182 | } 183 | 184 | func parseFields(fields []string) map[string]uint64 { 185 | fieldsMap := make(map[string]uint64) 186 | for _, field := range fields { 187 | kv := strings.SplitN(field, ":", 2) 188 | val, _ := strconv.ParseUint(kv[1], 10, 64) 189 | fieldsMap[kv[0]] = val 190 | } 191 | return fieldsMap 192 | } 193 | 194 | func updateMetaField(m *common.Meta, fields map[string]uint64) bool { 195 | changed := false 196 | for key, val := range fields { 197 | switch key { 198 | case metaFieldPageSize: 199 | m.SetPageSize(uint32(val)) 200 | case metaFieldRoot: 201 | m.SetRootBucket(common.NewInBucket(common.Pgid(val), 0)) 202 | case metaFieldFreelist: 203 | m.SetFreelist(common.Pgid(val)) 204 | case metaFieldPgid: 205 | m.SetPgid(common.Pgid(val)) 206 | } 207 | 208 | changed = true 209 | } 210 | 211 | if m.Magic() != common.Magic { 212 | m.SetMagic(common.Magic) 213 | changed = true 214 | } 215 | if m.Version() != common.Version { 216 | m.SetVersion(common.Version) 217 | changed = true 218 | } 219 | if m.Flags() != common.MetaPageFlag { 220 | m.SetFlags(common.MetaPageFlag) 221 | changed = true 222 | } 223 | 224 | newChecksum := m.Sum64() 225 | if m.Checksum() != newChecksum { 226 | m.SetChecksum(newChecksum) 227 | changed = true 228 | } 229 | 230 | return changed 231 | } 232 | 233 | func ReadMetaPageAt(dbPath string, metaPageId uint32, pageSize uint32) (*common.Meta, []byte, error) { 234 | if metaPageId > 1 { 235 | return nil, nil, fmt.Errorf("invalid metaPageId: %d", metaPageId) 236 | } 237 | 238 | f, err := os.OpenFile(dbPath, os.O_RDONLY, 0444) 239 | if err != nil { 240 | return nil, nil, err 241 | } 242 | defer f.Close() 243 | 244 | // The meta page is just 64 bytes, and definitely less than 1024 bytes, 245 | // so it's fine to only read 1024 bytes. Note we don't care about the 246 | // pageSize when reading the first meta page, because we always read the 247 | // file starting from offset 0. Actually the passed pageSize is 0 when 248 | // reading the first meta page in the `surgery meta update` command. 249 | buf := make([]byte, 1024) 250 | n, err := f.ReadAt(buf, int64(metaPageId*pageSize)) 251 | if n == len(buf) && (err == nil || err == io.EOF) { 252 | return common.LoadPageMeta(buf), buf, nil 253 | } 254 | 255 | return nil, nil, err 256 | } 257 | 258 | func writeMetaPageAt(dbPath string, buf []byte, metaPageId uint32, pageSize uint32) error { 259 | if metaPageId > 1 { 260 | return fmt.Errorf("invalid metaPageId: %d", metaPageId) 261 | } 262 | 263 | f, err := os.OpenFile(dbPath, os.O_RDWR, 0666) 264 | if err != nil { 265 | return err 266 | } 267 | defer f.Close() 268 | 269 | n, err := f.WriteAt(buf, int64(metaPageId*pageSize)) 270 | if n == len(buf) && (err == nil || err == io.EOF) { 271 | return nil 272 | } 273 | 274 | return err 275 | } 276 | -------------------------------------------------------------------------------- /cmd/bbolt/command_surgery_meta_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | bolt "go.etcd.io/bbolt" 12 | main "go.etcd.io/bbolt/cmd/bbolt" 13 | "go.etcd.io/bbolt/internal/btesting" 14 | "go.etcd.io/bbolt/internal/common" 15 | ) 16 | 17 | func TestSurgery_Meta_Validate(t *testing.T) { 18 | pageSize := 4096 19 | db := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize}) 20 | srcPath := db.Path() 21 | 22 | defer requireDBNoChange(t, dbData(t, db.Path()), db.Path()) 23 | 24 | // validate the meta pages 25 | rootCmd := main.NewRootCommand() 26 | rootCmd.SetArgs([]string{ 27 | "surgery", "meta", "validate", srcPath, 28 | }) 29 | err := rootCmd.Execute() 30 | require.NoError(t, err) 31 | 32 | // TODD: add one more case that the validation may fail. We need to 33 | // make the command output configurable, so that test cases can set 34 | // a customized io.Writer. 35 | } 36 | 37 | func TestSurgery_Meta_Update(t *testing.T) { 38 | testCases := []struct { 39 | name string 40 | root common.Pgid 41 | freelist common.Pgid 42 | pgid common.Pgid 43 | }{ 44 | { 45 | name: "root changed", 46 | root: 50, 47 | }, 48 | { 49 | name: "freelist changed", 50 | freelist: 40, 51 | }, 52 | { 53 | name: "pgid changed", 54 | pgid: 600, 55 | }, 56 | { 57 | name: "both root and freelist changed", 58 | root: 45, 59 | freelist: 46, 60 | }, 61 | { 62 | name: "both pgid and freelist changed", 63 | pgid: 256, 64 | freelist: 47, 65 | }, 66 | { 67 | name: "all fields changed", 68 | root: 43, 69 | freelist: 62, 70 | pgid: 256, 71 | }, 72 | } 73 | 74 | for _, tc := range testCases { 75 | for i := 0; i <= 1; i++ { 76 | tc := tc 77 | metaPageId := uint32(i) 78 | 79 | t.Run(tc.name, func(t *testing.T) { 80 | pageSize := 4096 81 | db := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize}) 82 | srcPath := db.Path() 83 | 84 | defer requireDBNoChange(t, dbData(t, db.Path()), db.Path()) 85 | 86 | var fields []string 87 | if tc.root != 0 { 88 | fields = append(fields, fmt.Sprintf("root:%d", tc.root)) 89 | } 90 | if tc.freelist != 0 { 91 | fields = append(fields, fmt.Sprintf("freelist:%d", tc.freelist)) 92 | } 93 | if tc.pgid != 0 { 94 | fields = append(fields, fmt.Sprintf("pgid:%d", tc.pgid)) 95 | } 96 | 97 | rootCmd := main.NewRootCommand() 98 | output := filepath.Join(t.TempDir(), "db") 99 | rootCmd.SetArgs([]string{ 100 | "surgery", "meta", "update", srcPath, 101 | "--output", output, 102 | "--meta-page", fmt.Sprintf("%d", metaPageId), 103 | "--fields", strings.Join(fields, ","), 104 | }) 105 | err := rootCmd.Execute() 106 | require.NoError(t, err) 107 | 108 | m, _, err := main.ReadMetaPageAt(output, metaPageId, 4096) 109 | require.NoError(t, err) 110 | 111 | require.Equal(t, common.Magic, m.Magic()) 112 | require.Equal(t, common.Version, m.Version()) 113 | 114 | if tc.root != 0 { 115 | require.Equal(t, tc.root, m.RootBucket().RootPage()) 116 | } 117 | if tc.freelist != 0 { 118 | require.Equal(t, tc.freelist, m.Freelist()) 119 | } 120 | if tc.pgid != 0 { 121 | require.Equal(t, tc.pgid, m.Pgid()) 122 | } 123 | }) 124 | } 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /cmd/bbolt/command_version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | 7 | "github.com/spf13/cobra" 8 | 9 | "go.etcd.io/bbolt/version" 10 | ) 11 | 12 | func newVersionCommand() *cobra.Command { 13 | versionCmd := &cobra.Command{ 14 | Use: "version", 15 | Short: "print the current version of bbolt", 16 | Long: "print the current version of bbolt", 17 | Run: func(cmd *cobra.Command, args []string) { 18 | fmt.Printf("bbolt Version: %s\n", version.Version) 19 | fmt.Printf("Go Version: %s\n", runtime.Version()) 20 | fmt.Printf("Go OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) 21 | }, 22 | } 23 | 24 | return versionCmd 25 | } 26 | -------------------------------------------------------------------------------- /cmd/bbolt/page_command.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "os" 9 | "strings" 10 | 11 | "go.etcd.io/bbolt/internal/common" 12 | "go.etcd.io/bbolt/internal/guts_cli" 13 | ) 14 | 15 | // pageCommand represents the "page" command execution. 16 | type pageCommand struct { 17 | baseCommand 18 | } 19 | 20 | // newPageCommand returns a pageCommand. 21 | func newPageCommand(m *Main) *pageCommand { 22 | c := &pageCommand{} 23 | c.baseCommand = m.baseCommand 24 | return c 25 | } 26 | 27 | // Run executes the command. 28 | func (cmd *pageCommand) Run(args ...string) error { 29 | // Parse flags. 30 | fs := flag.NewFlagSet("", flag.ContinueOnError) 31 | help := fs.Bool("h", false, "") 32 | all := fs.Bool("all", false, "list all pages") 33 | formatValue := fs.String("format-value", "auto", "One of: "+FORMAT_MODES+" . Applies to values on the leaf page.") 34 | 35 | if err := fs.Parse(args); err != nil { 36 | return err 37 | } else if *help { 38 | fmt.Fprintln(cmd.Stderr, cmd.Usage()) 39 | return ErrUsage 40 | } 41 | 42 | // Require database path and page id. 43 | path := fs.Arg(0) 44 | if path == "" { 45 | return ErrPathRequired 46 | } else if _, err := os.Stat(path); os.IsNotExist(err) { 47 | return ErrFileNotFound 48 | } 49 | 50 | if !*all { 51 | // Read page ids. 52 | pageIDs, err := stringToPages(fs.Args()[1:]) 53 | if err != nil { 54 | return err 55 | } else if len(pageIDs) == 0 { 56 | return ErrPageIDRequired 57 | } 58 | cmd.printPages(pageIDs, path, formatValue) 59 | } else { 60 | cmd.printAllPages(path, formatValue) 61 | } 62 | return nil 63 | } 64 | 65 | func (cmd *pageCommand) printPages(pageIDs []uint64, path string, formatValue *string) { 66 | // Print each page listed. 67 | for i, pageID := range pageIDs { 68 | // Print a separator. 69 | if i > 0 { 70 | fmt.Fprintln(cmd.Stdout, "===============================================") 71 | } 72 | _, err2 := cmd.printPage(path, pageID, *formatValue) 73 | if err2 != nil { 74 | fmt.Fprintf(cmd.Stdout, "Prining page %d failed: %s. Continuing...\n", pageID, err2) 75 | } 76 | } 77 | } 78 | 79 | func (cmd *pageCommand) printAllPages(path string, formatValue *string) { 80 | _, hwm, err := guts_cli.ReadPageAndHWMSize(path) 81 | if err != nil { 82 | fmt.Fprintf(cmd.Stdout, "cannot read number of pages: %v", err) 83 | } 84 | 85 | // Print each page listed. 86 | for pageID := uint64(0); pageID < uint64(hwm); { 87 | // Print a separator. 88 | if pageID > 0 { 89 | fmt.Fprintln(cmd.Stdout, "===============================================") 90 | } 91 | overflow, err2 := cmd.printPage(path, pageID, *formatValue) 92 | if err2 != nil { 93 | fmt.Fprintf(cmd.Stdout, "Prining page %d failed: %s. Continuing...\n", pageID, err2) 94 | pageID++ 95 | } else { 96 | pageID += uint64(overflow) + 1 97 | } 98 | } 99 | } 100 | 101 | // printPage prints given page to cmd.Stdout and returns error or number of interpreted pages. 102 | func (cmd *pageCommand) printPage(path string, pageID uint64, formatValue string) (numPages uint32, reterr error) { 103 | defer func() { 104 | if err := recover(); err != nil { 105 | reterr = fmt.Errorf("%s", err) 106 | } 107 | }() 108 | 109 | // Retrieve page info and page size. 110 | p, buf, err := guts_cli.ReadPage(path, pageID) 111 | if err != nil { 112 | return 0, err 113 | } 114 | 115 | // Print basic page info. 116 | fmt.Fprintf(cmd.Stdout, "Page ID: %d\n", p.Id()) 117 | fmt.Fprintf(cmd.Stdout, "Page Type: %s\n", p.Typ()) 118 | fmt.Fprintf(cmd.Stdout, "Total Size: %d bytes\n", len(buf)) 119 | fmt.Fprintf(cmd.Stdout, "Overflow pages: %d\n", p.Overflow()) 120 | 121 | // Print type-specific data. 122 | switch p.Typ() { 123 | case "meta": 124 | err = cmd.PrintMeta(cmd.Stdout, buf) 125 | case "leaf": 126 | err = cmd.PrintLeaf(cmd.Stdout, buf, formatValue) 127 | case "branch": 128 | err = cmd.PrintBranch(cmd.Stdout, buf) 129 | case "freelist": 130 | err = cmd.PrintFreelist(cmd.Stdout, buf) 131 | } 132 | if err != nil { 133 | return 0, err 134 | } 135 | return p.Overflow(), nil 136 | } 137 | 138 | // PrintMeta prints the data from the meta page. 139 | func (cmd *pageCommand) PrintMeta(w io.Writer, buf []byte) error { 140 | m := common.LoadPageMeta(buf) 141 | m.Print(w) 142 | return nil 143 | } 144 | 145 | // PrintLeaf prints the data for a leaf page. 146 | func (cmd *pageCommand) PrintLeaf(w io.Writer, buf []byte, formatValue string) error { 147 | p := common.LoadPage(buf) 148 | 149 | // Print number of items. 150 | fmt.Fprintf(w, "Item Count: %d\n", p.Count()) 151 | fmt.Fprintf(w, "\n") 152 | 153 | // Print each key/value. 154 | for i := uint16(0); i < p.Count(); i++ { 155 | e := p.LeafPageElement(i) 156 | 157 | // Format key as string. 158 | var k string 159 | if isPrintable(string(e.Key())) { 160 | k = fmt.Sprintf("%q", string(e.Key())) 161 | } else { 162 | k = fmt.Sprintf("%x", string(e.Key())) 163 | } 164 | 165 | // Format value as string. 166 | var v string 167 | if e.IsBucketEntry() { 168 | b := e.Bucket() 169 | v = b.String() 170 | } else { 171 | var err error 172 | v, err = formatBytes(e.Value(), formatValue) 173 | if err != nil { 174 | return err 175 | } 176 | } 177 | 178 | fmt.Fprintf(w, "%s: %s\n", k, v) 179 | } 180 | fmt.Fprintf(w, "\n") 181 | return nil 182 | } 183 | 184 | // PrintBranch prints the data for a leaf page. 185 | func (cmd *pageCommand) PrintBranch(w io.Writer, buf []byte) error { 186 | p := common.LoadPage(buf) 187 | 188 | // Print number of items. 189 | fmt.Fprintf(w, "Item Count: %d\n", p.Count()) 190 | fmt.Fprintf(w, "\n") 191 | 192 | // Print each key/value. 193 | for i := uint16(0); i < p.Count(); i++ { 194 | e := p.BranchPageElement(i) 195 | 196 | // Format key as string. 197 | var k string 198 | if isPrintable(string(e.Key())) { 199 | k = fmt.Sprintf("%q", string(e.Key())) 200 | } else { 201 | k = fmt.Sprintf("%x", string(e.Key())) 202 | } 203 | 204 | fmt.Fprintf(w, "%s: \n", k, e.Pgid()) 205 | } 206 | fmt.Fprintf(w, "\n") 207 | return nil 208 | } 209 | 210 | // PrintFreelist prints the data for a freelist page. 211 | func (cmd *pageCommand) PrintFreelist(w io.Writer, buf []byte) error { 212 | p := common.LoadPage(buf) 213 | 214 | // Print number of items. 215 | _, cnt := p.FreelistPageCount() 216 | fmt.Fprintf(w, "Item Count: %d\n", cnt) 217 | fmt.Fprintf(w, "Overflow: %d\n", p.Overflow()) 218 | 219 | fmt.Fprintf(w, "\n") 220 | 221 | // Print each page in the freelist. 222 | ids := p.FreelistPageIds() 223 | for _, ids := range ids { 224 | fmt.Fprintf(w, "%d\n", ids) 225 | } 226 | fmt.Fprintf(w, "\n") 227 | return nil 228 | } 229 | 230 | // PrintPage prints a given page as hexadecimal. 231 | func (cmd *pageCommand) PrintPage(w io.Writer, r io.ReaderAt, pageID int, pageSize int) error { 232 | const bytesPerLineN = 16 233 | 234 | // Read page into buffer. 235 | buf := make([]byte, pageSize) 236 | addr := pageID * pageSize 237 | if n, err := r.ReadAt(buf, int64(addr)); err != nil { 238 | return err 239 | } else if n != pageSize { 240 | return io.ErrUnexpectedEOF 241 | } 242 | 243 | // Write out to writer in 16-byte lines. 244 | var prev []byte 245 | var skipped bool 246 | for offset := 0; offset < pageSize; offset += bytesPerLineN { 247 | // Retrieve current 16-byte line. 248 | line := buf[offset : offset+bytesPerLineN] 249 | isLastLine := offset == (pageSize - bytesPerLineN) 250 | 251 | // If it's the same as the previous line then print a skip. 252 | if bytes.Equal(line, prev) && !isLastLine { 253 | if !skipped { 254 | fmt.Fprintf(w, "%07x *\n", addr+offset) 255 | skipped = true 256 | } 257 | } else { 258 | // Print line as hexadecimal in 2-byte groups. 259 | fmt.Fprintf(w, "%07x %04x %04x %04x %04x %04x %04x %04x %04x\n", addr+offset, 260 | line[0:2], line[2:4], line[4:6], line[6:8], 261 | line[8:10], line[10:12], line[12:14], line[14:16], 262 | ) 263 | 264 | skipped = false 265 | } 266 | 267 | // Save the previous line. 268 | prev = line 269 | } 270 | fmt.Fprint(w, "\n") 271 | 272 | return nil 273 | } 274 | 275 | // Usage returns the help message. 276 | func (cmd *pageCommand) Usage() string { 277 | return strings.TrimLeft(` 278 | usage: bolt page PATH pageid [pageid...] 279 | or: bolt page --all PATH 280 | 281 | Additional options include: 282 | 283 | --all 284 | prints all pages (only skips pages that were considered successful overflow pages) 285 | --format-value=`+FORMAT_MODES+` (default: auto) 286 | prints values (on the leaf page) using the given format. 287 | 288 | Page prints one or more pages in human readable format. 289 | `, "\n") 290 | } 291 | -------------------------------------------------------------------------------- /cmd/bbolt/utils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | func checkSourceDBPath(srcPath string) (os.FileInfo, error) { 9 | fi, err := os.Stat(srcPath) 10 | if os.IsNotExist(err) { 11 | return nil, fmt.Errorf("source database file %q doesn't exist", srcPath) 12 | } else if err != nil { 13 | return nil, fmt.Errorf("failed to open source database file %q: %v", srcPath, err) 14 | } 15 | return fi, nil 16 | } 17 | -------------------------------------------------------------------------------- /cmd/bbolt/utils_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | 9 | "go.etcd.io/bbolt/internal/common" 10 | "go.etcd.io/bbolt/internal/guts_cli" 11 | ) 12 | 13 | func loadMetaPage(t *testing.T, dbPath string, pageID uint64) *common.Meta { 14 | _, buf, err := guts_cli.ReadPage(dbPath, pageID) 15 | require.NoError(t, err) 16 | return common.LoadPageMeta(buf) 17 | } 18 | 19 | func readMetaPage(t *testing.T, path string) *common.Meta { 20 | _, activeMetaPageId, err := guts_cli.GetRootPage(path) 21 | require.NoError(t, err) 22 | _, buf, err := guts_cli.ReadPage(path, uint64(activeMetaPageId)) 23 | require.NoError(t, err) 24 | return common.LoadPageMeta(buf) 25 | } 26 | 27 | func readPage(t *testing.T, path string, pageId int, pageSize int) []byte { 28 | dbFile, err := os.Open(path) 29 | require.NoError(t, err) 30 | defer dbFile.Close() 31 | 32 | fi, err := dbFile.Stat() 33 | require.NoError(t, err) 34 | require.GreaterOrEqual(t, fi.Size(), int64((pageId+1)*pageSize)) 35 | 36 | buf := make([]byte, pageSize) 37 | byteRead, err := dbFile.ReadAt(buf, int64(pageId*pageSize)) 38 | require.NoError(t, err) 39 | require.Equal(t, pageSize, byteRead) 40 | 41 | return buf 42 | } 43 | 44 | func pageDataWithoutPageId(buf []byte) []byte { 45 | return buf[8:] 46 | } 47 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # etcd Community Code of Conduct 2 | 3 | Please refer to [etcd Community Code of Conduct](https://github.com/etcd-io/etcd/blob/main/code-of-conduct.md). 4 | -------------------------------------------------------------------------------- /compact.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | // Compact will create a copy of the source DB and in the destination DB. This may 4 | // reclaim space that the source database no longer has use for. txMaxSize can be 5 | // used to limit the transactions size of this process and may trigger intermittent 6 | // commits. A value of zero will ignore transaction sizes. 7 | // TODO: merge with: https://github.com/etcd-io/etcd/blob/b7f0f52a16dbf83f18ca1d803f7892d750366a94/mvcc/backend/backend.go#L349 8 | func Compact(dst, src *DB, txMaxSize int64) error { 9 | // commit regularly, or we'll run out of memory for large datasets if using one transaction. 10 | var size int64 11 | tx, err := dst.Begin(true) 12 | if err != nil { 13 | return err 14 | } 15 | defer func() { 16 | if tempErr := tx.Rollback(); tempErr != nil { 17 | err = tempErr 18 | } 19 | }() 20 | 21 | if err := walk(src, func(keys [][]byte, k, v []byte, seq uint64) error { 22 | // On each key/value, check if we have exceeded tx size. 23 | sz := int64(len(k) + len(v)) 24 | if size+sz > txMaxSize && txMaxSize != 0 { 25 | // Commit previous transaction. 26 | if err := tx.Commit(); err != nil { 27 | return err 28 | } 29 | 30 | // Start new transaction. 31 | tx, err = dst.Begin(true) 32 | if err != nil { 33 | return err 34 | } 35 | size = 0 36 | } 37 | size += sz 38 | 39 | // Create bucket on the root transaction if this is the first level. 40 | nk := len(keys) 41 | if nk == 0 { 42 | bkt, err := tx.CreateBucket(k) 43 | if err != nil { 44 | return err 45 | } 46 | if err := bkt.SetSequence(seq); err != nil { 47 | return err 48 | } 49 | return nil 50 | } 51 | 52 | // Create buckets on subsequent levels, if necessary. 53 | b := tx.Bucket(keys[0]) 54 | if nk > 1 { 55 | for _, k := range keys[1:] { 56 | b = b.Bucket(k) 57 | } 58 | } 59 | 60 | // Fill the entire page for best compaction. 61 | b.FillPercent = 1.0 62 | 63 | // If there is no value then this is a bucket call. 64 | if v == nil { 65 | bkt, err := b.CreateBucket(k) 66 | if err != nil { 67 | return err 68 | } 69 | if err := bkt.SetSequence(seq); err != nil { 70 | return err 71 | } 72 | return nil 73 | } 74 | 75 | // Otherwise treat it as a key/value pair. 76 | return b.Put(k, v) 77 | }); err != nil { 78 | return err 79 | } 80 | err = tx.Commit() 81 | 82 | return err 83 | } 84 | 85 | // walkFunc is the type of the function called for keys (buckets and "normal" 86 | // values) discovered by Walk. keys is the list of keys to descend to the bucket 87 | // owning the discovered key/value pair k/v. 88 | type walkFunc func(keys [][]byte, k, v []byte, seq uint64) error 89 | 90 | // walk walks recursively the bolt database db, calling walkFn for each key it finds. 91 | func walk(db *DB, walkFn walkFunc) error { 92 | return db.View(func(tx *Tx) error { 93 | return tx.ForEach(func(name []byte, b *Bucket) error { 94 | return walkBucket(b, nil, name, nil, b.Sequence(), walkFn) 95 | }) 96 | }) 97 | } 98 | 99 | func walkBucket(b *Bucket, keypath [][]byte, k, v []byte, seq uint64, fn walkFunc) error { 100 | // Execute callback. 101 | if err := fn(keypath, k, v, seq); err != nil { 102 | return err 103 | } 104 | 105 | // If this is not a bucket then stop. 106 | if v != nil { 107 | return nil 108 | } 109 | 110 | // Iterate over each child key/value. 111 | keypath = append(keypath, k) 112 | return b.ForEach(func(k, v []byte) error { 113 | if v == nil { 114 | bkt := b.Bucket(k) 115 | return walkBucket(bkt, keypath, k, nil, bkt.Sequence(), fn) 116 | } 117 | return walkBucket(b, keypath, k, v, b.Sequence(), fn) 118 | }) 119 | } 120 | -------------------------------------------------------------------------------- /db_whitebox_test.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | 10 | "go.etcd.io/bbolt/errors" 11 | ) 12 | 13 | func TestOpenWithPreLoadFreelist(t *testing.T) { 14 | testCases := []struct { 15 | name string 16 | readonly bool 17 | preLoadFreePage bool 18 | expectedFreePagesLoaded bool 19 | }{ 20 | { 21 | name: "write mode always load free pages", 22 | readonly: false, 23 | preLoadFreePage: false, 24 | expectedFreePagesLoaded: true, 25 | }, 26 | { 27 | name: "readonly mode load free pages when flag set", 28 | readonly: true, 29 | preLoadFreePage: true, 30 | expectedFreePagesLoaded: true, 31 | }, 32 | { 33 | name: "readonly mode doesn't load free pages when flag not set", 34 | readonly: true, 35 | preLoadFreePage: false, 36 | expectedFreePagesLoaded: false, 37 | }, 38 | } 39 | 40 | fileName, err := prepareData(t) 41 | require.NoError(t, err) 42 | 43 | for _, tc := range testCases { 44 | t.Run(tc.name, func(t *testing.T) { 45 | db, err := Open(fileName, 0666, &Options{ 46 | ReadOnly: tc.readonly, 47 | PreLoadFreelist: tc.preLoadFreePage, 48 | }) 49 | require.NoError(t, err) 50 | 51 | assert.Equal(t, tc.expectedFreePagesLoaded, db.freelist != nil) 52 | 53 | assert.NoError(t, db.Close()) 54 | }) 55 | } 56 | } 57 | 58 | func TestMethodPage(t *testing.T) { 59 | testCases := []struct { 60 | name string 61 | readonly bool 62 | preLoadFreePage bool 63 | expectedError error 64 | }{ 65 | { 66 | name: "write mode", 67 | readonly: false, 68 | preLoadFreePage: false, 69 | expectedError: nil, 70 | }, 71 | { 72 | name: "readonly mode with preloading free pages", 73 | readonly: true, 74 | preLoadFreePage: true, 75 | expectedError: nil, 76 | }, 77 | { 78 | name: "readonly mode without preloading free pages", 79 | readonly: true, 80 | preLoadFreePage: false, 81 | expectedError: errors.ErrFreePagesNotLoaded, 82 | }, 83 | } 84 | 85 | fileName, err := prepareData(t) 86 | require.NoError(t, err) 87 | 88 | for _, tc := range testCases { 89 | tc := tc 90 | t.Run(tc.name, func(t *testing.T) { 91 | db, err := Open(fileName, 0666, &Options{ 92 | ReadOnly: tc.readonly, 93 | PreLoadFreelist: tc.preLoadFreePage, 94 | }) 95 | require.NoError(t, err) 96 | defer db.Close() 97 | 98 | tx, err := db.Begin(!tc.readonly) 99 | require.NoError(t, err) 100 | 101 | _, err = tx.Page(0) 102 | require.Equal(t, tc.expectedError, err) 103 | 104 | if tc.readonly { 105 | require.NoError(t, tx.Rollback()) 106 | } else { 107 | require.NoError(t, tx.Commit()) 108 | } 109 | 110 | require.NoError(t, db.Close()) 111 | }) 112 | } 113 | } 114 | 115 | func prepareData(t *testing.T) (string, error) { 116 | fileName := filepath.Join(t.TempDir(), "db") 117 | db, err := Open(fileName, 0666, nil) 118 | if err != nil { 119 | return "", err 120 | } 121 | if err := db.Close(); err != nil { 122 | return "", err 123 | } 124 | 125 | return fileName, nil 126 | } 127 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | package bbolt implements a low-level key/value store in pure Go. It supports 3 | fully serializable transactions, ACID semantics, and lock-free MVCC with 4 | multiple readers and a single writer. Bolt can be used for projects that 5 | want a simple data store without the need to add large dependencies such as 6 | Postgres or MySQL. 7 | 8 | Bolt is a single-level, zero-copy, B+tree data store. This means that Bolt is 9 | optimized for fast read access and does not require recovery in the event of a 10 | system crash. Transactions which have not finished committing will simply be 11 | rolled back in the event of a crash. 12 | 13 | The design of Bolt is based on Howard Chu's LMDB database project. 14 | 15 | Bolt currently works on Windows, Mac OS X, and Linux. 16 | 17 | # Basics 18 | 19 | There are only a few types in Bolt: DB, Bucket, Tx, and Cursor. The DB is 20 | a collection of buckets and is represented by a single file on disk. A bucket is 21 | a collection of unique keys that are associated with values. 22 | 23 | Transactions provide either read-only or read-write access to the database. 24 | Read-only transactions can retrieve key/value pairs and can use Cursors to 25 | iterate over the dataset sequentially. Read-write transactions can create and 26 | delete buckets and can insert and remove keys. Only one read-write transaction 27 | is allowed at a time. 28 | 29 | # Caveats 30 | 31 | The database uses a read-only, memory-mapped data file to ensure that 32 | applications cannot corrupt the database, however, this means that keys and 33 | values returned from Bolt cannot be changed. Writing to a read-only byte slice 34 | will cause Go to panic. 35 | 36 | Keys and values retrieved from the database are only valid for the life of 37 | the transaction. When used outside the transaction, these byte slices can 38 | point to different data or can point to invalid memory which will cause a panic. 39 | */ 40 | package bbolt 41 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | import "go.etcd.io/bbolt/errors" 4 | 5 | // These errors can be returned when opening or calling methods on a DB. 6 | var ( 7 | // ErrDatabaseNotOpen is returned when a DB instance is accessed before it 8 | // is opened or after it is closed. 9 | // 10 | // Deprecated: Use the error variables defined in the bbolt/errors package. 11 | ErrDatabaseNotOpen = errors.ErrDatabaseNotOpen 12 | 13 | // ErrInvalid is returned when both meta pages on a database are invalid. 14 | // This typically occurs when a file is not a bolt database. 15 | // 16 | // Deprecated: Use the error variables defined in the bbolt/errors package. 17 | ErrInvalid = errors.ErrInvalid 18 | 19 | // ErrInvalidMapping is returned when the database file fails to get mapped. 20 | // 21 | // Deprecated: Use the error variables defined in the bbolt/errors package. 22 | ErrInvalidMapping = errors.ErrInvalidMapping 23 | 24 | // ErrVersionMismatch is returned when the data file was created with a 25 | // different version of Bolt. 26 | // 27 | // Deprecated: Use the error variables defined in the bbolt/errors package. 28 | ErrVersionMismatch = errors.ErrVersionMismatch 29 | 30 | // ErrChecksum is returned when a checksum mismatch occurs on either of the two meta pages. 31 | // 32 | // Deprecated: Use the error variables defined in the bbolt/errors package. 33 | ErrChecksum = errors.ErrChecksum 34 | 35 | // ErrTimeout is returned when a database cannot obtain an exclusive lock 36 | // on the data file after the timeout passed to Open(). 37 | // 38 | // Deprecated: Use the error variables defined in the bbolt/errors package. 39 | ErrTimeout = errors.ErrTimeout 40 | ) 41 | 42 | // These errors can occur when beginning or committing a Tx. 43 | var ( 44 | // ErrTxNotWritable is returned when performing a write operation on a 45 | // read-only transaction. 46 | // 47 | // Deprecated: Use the error variables defined in the bbolt/errors package. 48 | ErrTxNotWritable = errors.ErrTxNotWritable 49 | 50 | // ErrTxClosed is returned when committing or rolling back a transaction 51 | // that has already been committed or rolled back. 52 | // 53 | // Deprecated: Use the error variables defined in the bbolt/errors package. 54 | ErrTxClosed = errors.ErrTxClosed 55 | 56 | // ErrDatabaseReadOnly is returned when a mutating transaction is started on a 57 | // read-only database. 58 | // 59 | // Deprecated: Use the error variables defined in the bbolt/errors package. 60 | ErrDatabaseReadOnly = errors.ErrDatabaseReadOnly 61 | 62 | // ErrFreePagesNotLoaded is returned when a readonly transaction without 63 | // preloading the free pages is trying to access the free pages. 64 | // 65 | // Deprecated: Use the error variables defined in the bbolt/errors package. 66 | ErrFreePagesNotLoaded = errors.ErrFreePagesNotLoaded 67 | ) 68 | 69 | // These errors can occur when putting or deleting a value or a bucket. 70 | var ( 71 | // ErrBucketNotFound is returned when trying to access a bucket that has 72 | // not been created yet. 73 | // 74 | // Deprecated: Use the error variables defined in the bbolt/errors package. 75 | ErrBucketNotFound = errors.ErrBucketNotFound 76 | 77 | // ErrBucketExists is returned when creating a bucket that already exists. 78 | // 79 | // Deprecated: Use the error variables defined in the bbolt/errors package. 80 | ErrBucketExists = errors.ErrBucketExists 81 | 82 | // ErrBucketNameRequired is returned when creating a bucket with a blank name. 83 | // 84 | // Deprecated: Use the error variables defined in the bbolt/errors package. 85 | ErrBucketNameRequired = errors.ErrBucketNameRequired 86 | 87 | // ErrKeyRequired is returned when inserting a zero-length key. 88 | // 89 | // Deprecated: Use the error variables defined in the bbolt/errors package. 90 | ErrKeyRequired = errors.ErrKeyRequired 91 | 92 | // ErrKeyTooLarge is returned when inserting a key that is larger than MaxKeySize. 93 | // 94 | // Deprecated: Use the error variables defined in the bbolt/errors package. 95 | ErrKeyTooLarge = errors.ErrKeyTooLarge 96 | 97 | // ErrValueTooLarge is returned when inserting a value that is larger than MaxValueSize. 98 | // 99 | // Deprecated: Use the error variables defined in the bbolt/errors package. 100 | ErrValueTooLarge = errors.ErrValueTooLarge 101 | 102 | // ErrIncompatibleValue is returned when trying create or delete a bucket 103 | // on an existing non-bucket key or when trying to create or delete a 104 | // non-bucket key on an existing bucket key. 105 | // 106 | // Deprecated: Use the error variables defined in the bbolt/errors package. 107 | ErrIncompatibleValue = errors.ErrIncompatibleValue 108 | ) 109 | -------------------------------------------------------------------------------- /errors/errors.go: -------------------------------------------------------------------------------- 1 | // Package errors defines the error variables that may be returned 2 | // during bbolt operations. 3 | package errors 4 | 5 | import "errors" 6 | 7 | // These errors can be returned when opening or calling methods on a DB. 8 | var ( 9 | // ErrDatabaseNotOpen is returned when a DB instance is accessed before it 10 | // is opened or after it is closed. 11 | ErrDatabaseNotOpen = errors.New("database not open") 12 | 13 | // ErrInvalid is returned when both meta pages on a database are invalid. 14 | // This typically occurs when a file is not a bolt database. 15 | ErrInvalid = errors.New("invalid database") 16 | 17 | // ErrInvalidMapping is returned when the database file fails to get mapped. 18 | ErrInvalidMapping = errors.New("database isn't correctly mapped") 19 | 20 | // ErrVersionMismatch is returned when the data file was created with a 21 | // different version of Bolt. 22 | ErrVersionMismatch = errors.New("version mismatch") 23 | 24 | // ErrChecksum is returned when a checksum mismatch occurs on either of the two meta pages. 25 | ErrChecksum = errors.New("checksum error") 26 | 27 | // ErrTimeout is returned when a database cannot obtain an exclusive lock 28 | // on the data file after the timeout passed to Open(). 29 | ErrTimeout = errors.New("timeout") 30 | ) 31 | 32 | // These errors can occur when beginning or committing a Tx. 33 | var ( 34 | // ErrTxNotWritable is returned when performing a write operation on a 35 | // read-only transaction. 36 | ErrTxNotWritable = errors.New("tx not writable") 37 | 38 | // ErrTxClosed is returned when committing or rolling back a transaction 39 | // that has already been committed or rolled back. 40 | ErrTxClosed = errors.New("tx closed") 41 | 42 | // ErrDatabaseReadOnly is returned when a mutating transaction is started on a 43 | // read-only database. 44 | ErrDatabaseReadOnly = errors.New("database is in read-only mode") 45 | 46 | // ErrFreePagesNotLoaded is returned when a readonly transaction without 47 | // preloading the free pages is trying to access the free pages. 48 | ErrFreePagesNotLoaded = errors.New("free pages are not pre-loaded") 49 | ) 50 | 51 | // These errors can occur when putting or deleting a value or a bucket. 52 | var ( 53 | // ErrBucketNotFound is returned when trying to access a bucket that has 54 | // not been created yet. 55 | ErrBucketNotFound = errors.New("bucket not found") 56 | 57 | // ErrBucketExists is returned when creating a bucket that already exists. 58 | ErrBucketExists = errors.New("bucket already exists") 59 | 60 | // ErrBucketNameRequired is returned when creating a bucket with a blank name. 61 | ErrBucketNameRequired = errors.New("bucket name required") 62 | 63 | // ErrKeyRequired is returned when inserting a zero-length key. 64 | ErrKeyRequired = errors.New("key required") 65 | 66 | // ErrKeyTooLarge is returned when inserting a key that is larger than MaxKeySize. 67 | ErrKeyTooLarge = errors.New("key too large") 68 | 69 | // ErrValueTooLarge is returned when inserting a value that is larger than MaxValueSize. 70 | ErrValueTooLarge = errors.New("value too large") 71 | 72 | // ErrMaxSizeReached is returned when the configured maximum size of the data file is reached. 73 | ErrMaxSizeReached = errors.New("database reached maximum size") 74 | 75 | // ErrIncompatibleValue is returned when trying to create or delete a bucket 76 | // on an existing non-bucket key or when trying to create or delete a 77 | // non-bucket key on an existing bucket key. 78 | ErrIncompatibleValue = errors.New("incompatible value") 79 | 80 | // ErrSameBuckets is returned when trying to move a sub-bucket between 81 | // source and target buckets, while source and target buckets are the same. 82 | ErrSameBuckets = errors.New("the source and target are the same bucket") 83 | 84 | // ErrDifferentDB is returned when trying to move a sub-bucket between 85 | // source and target buckets, while source and target buckets are in different database files. 86 | ErrDifferentDB = errors.New("the source and target buckets are in different database files") 87 | ) 88 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module go.etcd.io/bbolt 2 | 3 | go 1.24 4 | 5 | toolchain go1.24.3 6 | 7 | require ( 8 | github.com/spf13/cobra v1.9.1 9 | github.com/spf13/pflag v1.0.6 10 | github.com/stretchr/testify v1.10.0 11 | go.etcd.io/gofail v0.2.0 12 | golang.org/x/sync v0.14.0 13 | golang.org/x/sys v0.33.0 14 | ) 15 | 16 | require ( 17 | github.com/davecgh/go-spew v1.1.1 // indirect 18 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 19 | github.com/pmezard/go-difflib v1.0.0 // indirect 20 | gopkg.in/yaml.v3 v3.0.1 // indirect 21 | ) 22 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 5 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 6 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 7 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 8 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 9 | github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= 10 | github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= 11 | github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= 12 | github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 13 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 14 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 15 | go.etcd.io/gofail v0.2.0 h1:p19drv16FKK345a09a1iubchlw/vmRuksmRzgBIGjcA= 16 | go.etcd.io/gofail v0.2.0/go.mod h1:nL3ILMGfkXTekKI3clMBNazKnjUZjYLKmBHzsVAnC1o= 17 | golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= 18 | golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 19 | golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= 20 | golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 21 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 22 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 23 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 24 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 25 | -------------------------------------------------------------------------------- /internal/btesting/btesting.go: -------------------------------------------------------------------------------- 1 | package btesting 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "regexp" 9 | "strings" 10 | "testing" 11 | "time" 12 | 13 | "github.com/stretchr/testify/require" 14 | 15 | bolt "go.etcd.io/bbolt" 16 | ) 17 | 18 | var statsFlag = flag.Bool("stats", false, "show performance stats") 19 | 20 | const ( 21 | // TestFreelistType is used as an env variable for test to indicate the backend type. 22 | TestFreelistType = "TEST_FREELIST_TYPE" 23 | // TestEnableStrictMode is used to enable strict check by default after opening each DB. 24 | TestEnableStrictMode = "TEST_ENABLE_STRICT_MODE" 25 | ) 26 | 27 | // DB is a test wrapper for bolt.DB. 28 | type DB struct { 29 | *bolt.DB 30 | f string 31 | o *bolt.Options 32 | t testing.TB 33 | } 34 | 35 | // MustCreateDB returns a new, open DB at a temporary location. 36 | func MustCreateDB(t testing.TB) *DB { 37 | return MustCreateDBWithOption(t, nil) 38 | } 39 | 40 | // MustCreateDBWithOption returns a new, open DB at a temporary location with given options. 41 | func MustCreateDBWithOption(t testing.TB, o *bolt.Options) *DB { 42 | f := filepath.Join(t.TempDir(), "db") 43 | return MustOpenDBWithOption(t, f, o) 44 | } 45 | 46 | func MustOpenDBWithOption(t testing.TB, f string, o *bolt.Options) *DB { 47 | db, err := OpenDBWithOption(t, f, o) 48 | require.NoError(t, err) 49 | require.NotNil(t, db) 50 | return db 51 | } 52 | 53 | func OpenDBWithOption(t testing.TB, f string, o *bolt.Options) (*DB, error) { 54 | t.Logf("Opening bbolt DB at: %s", f) 55 | if o == nil { 56 | o = bolt.DefaultOptions 57 | } 58 | 59 | freelistType := bolt.FreelistArrayType 60 | if env := os.Getenv(TestFreelistType); env == string(bolt.FreelistMapType) { 61 | freelistType = bolt.FreelistMapType 62 | } 63 | 64 | o.FreelistType = freelistType 65 | 66 | db, err := bolt.Open(f, 0600, o) 67 | if err != nil { 68 | return nil, err 69 | } 70 | resDB := &DB{ 71 | DB: db, 72 | f: f, 73 | o: o, 74 | t: t, 75 | } 76 | resDB.strictModeEnabledDefault() 77 | t.Cleanup(resDB.PostTestCleanup) 78 | return resDB, nil 79 | } 80 | 81 | func (db *DB) PostTestCleanup() { 82 | // Check database consistency after every test. 83 | if db.DB != nil { 84 | db.MustCheck() 85 | db.MustClose() 86 | } 87 | } 88 | 89 | // Close closes the database but does NOT delete the underlying file. 90 | func (db *DB) Close() error { 91 | if db.DB != nil { 92 | // Log statistics. 93 | if *statsFlag { 94 | db.PrintStats() 95 | } 96 | db.t.Logf("Closing bbolt DB at: %s", db.f) 97 | err := db.DB.Close() 98 | if err != nil { 99 | return err 100 | } 101 | db.DB = nil 102 | } 103 | return nil 104 | } 105 | 106 | // MustClose closes the database but does NOT delete the underlying file. 107 | func (db *DB) MustClose() { 108 | err := db.Close() 109 | require.NoError(db.t, err) 110 | } 111 | 112 | func (db *DB) MustDeleteFile() { 113 | err := os.Remove(db.Path()) 114 | require.NoError(db.t, err) 115 | } 116 | 117 | func (db *DB) SetOptions(o *bolt.Options) { 118 | db.o = o 119 | } 120 | 121 | // MustReopen reopen the database. Panic on error. 122 | func (db *DB) MustReopen() { 123 | if db.DB != nil { 124 | panic("Please call Close() before MustReopen()") 125 | } 126 | db.t.Logf("Reopening bbolt DB at: %s", db.f) 127 | indb, err := bolt.Open(db.Path(), 0600, db.o) 128 | require.NoError(db.t, err) 129 | db.DB = indb 130 | db.strictModeEnabledDefault() 131 | } 132 | 133 | // MustCheck runs a consistency check on the database and panics if any errors are found. 134 | func (db *DB) MustCheck() { 135 | err := db.View(func(tx *bolt.Tx) error { 136 | // Collect all the errors. 137 | var errors []error 138 | for err := range tx.Check() { 139 | errors = append(errors, err) 140 | if len(errors) > 10 { 141 | break 142 | } 143 | } 144 | 145 | // If errors occurred, copy the DB and print the errors. 146 | if len(errors) > 0 { 147 | var path = filepath.Join(db.t.TempDir(), "db.backup") 148 | err := tx.CopyFile(path, 0600) 149 | require.NoError(db.t, err) 150 | 151 | // Print errors. 152 | fmt.Print("\n\n") 153 | fmt.Printf("consistency check failed (%d errors)\n", len(errors)) 154 | for _, err := range errors { 155 | fmt.Println(err) 156 | } 157 | fmt.Println("") 158 | fmt.Println("db saved to:") 159 | fmt.Println(path) 160 | fmt.Print("\n\n") 161 | os.Exit(-1) 162 | } 163 | 164 | return nil 165 | }) 166 | require.NoError(db.t, err) 167 | } 168 | 169 | // Fill - fills the DB using numTx transactions and numKeysPerTx. 170 | func (db *DB) Fill(bucket []byte, numTx int, numKeysPerTx int, 171 | keyGen func(tx int, key int) []byte, 172 | valueGen func(tx int, key int) []byte) error { 173 | for tr := 0; tr < numTx; tr++ { 174 | err := db.Update(func(tx *bolt.Tx) error { 175 | b, _ := tx.CreateBucketIfNotExists(bucket) 176 | for i := 0; i < numKeysPerTx; i++ { 177 | if err := b.Put(keyGen(tr, i), valueGen(tr, i)); err != nil { 178 | return err 179 | } 180 | } 181 | return nil 182 | }) 183 | if err != nil { 184 | return err 185 | } 186 | } 187 | return nil 188 | } 189 | 190 | func (db *DB) Path() string { 191 | return db.f 192 | } 193 | 194 | // CopyTempFile copies a database to a temporary file. 195 | func (db *DB) CopyTempFile() { 196 | path := filepath.Join(db.t.TempDir(), "db.copy") 197 | err := db.View(func(tx *bolt.Tx) error { 198 | return tx.CopyFile(path, 0600) 199 | }) 200 | require.NoError(db.t, err) 201 | fmt.Println("db copied to: ", path) 202 | } 203 | 204 | // PrintStats prints the database stats 205 | func (db *DB) PrintStats() { 206 | var stats = db.Stats() 207 | fmt.Printf("[db] %-20s %-20s %-20s\n", 208 | fmt.Sprintf("pg(%d/%d)", stats.TxStats.GetPageCount(), stats.TxStats.GetPageAlloc()), 209 | fmt.Sprintf("cur(%d)", stats.TxStats.GetCursorCount()), 210 | fmt.Sprintf("node(%d/%d)", stats.TxStats.GetNodeCount(), stats.TxStats.GetNodeDeref()), 211 | ) 212 | fmt.Printf(" %-20s %-20s %-20s\n", 213 | fmt.Sprintf("rebal(%d/%v)", stats.TxStats.GetRebalance(), truncDuration(stats.TxStats.GetRebalanceTime())), 214 | fmt.Sprintf("spill(%d/%v)", stats.TxStats.GetSpill(), truncDuration(stats.TxStats.GetSpillTime())), 215 | fmt.Sprintf("w(%d/%v)", stats.TxStats.GetWrite(), truncDuration(stats.TxStats.GetWriteTime())), 216 | ) 217 | } 218 | 219 | func truncDuration(d time.Duration) string { 220 | return regexp.MustCompile(`^(\d+)(\.\d+)`).ReplaceAllString(d.String(), "$1") 221 | } 222 | 223 | func (db *DB) strictModeEnabledDefault() { 224 | strictModeEnabled := strings.ToLower(os.Getenv(TestEnableStrictMode)) 225 | db.StrictMode = strictModeEnabled == "true" 226 | } 227 | 228 | func (db *DB) ForceDisableStrictMode() { 229 | db.StrictMode = false 230 | } 231 | -------------------------------------------------------------------------------- /internal/common/bucket.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "unsafe" 6 | ) 7 | 8 | const BucketHeaderSize = int(unsafe.Sizeof(InBucket{})) 9 | 10 | // InBucket represents the on-file representation of a bucket. 11 | // This is stored as the "value" of a bucket key. If the bucket is small enough, 12 | // then its root page can be stored inline in the "value", after the bucket 13 | // header. In the case of inline buckets, the "root" will be 0. 14 | type InBucket struct { 15 | root Pgid // page id of the bucket's root-level page 16 | sequence uint64 // monotonically incrementing, used by NextSequence() 17 | } 18 | 19 | func NewInBucket(root Pgid, seq uint64) InBucket { 20 | return InBucket{ 21 | root: root, 22 | sequence: seq, 23 | } 24 | } 25 | 26 | func (b *InBucket) RootPage() Pgid { 27 | return b.root 28 | } 29 | 30 | func (b *InBucket) SetRootPage(id Pgid) { 31 | b.root = id 32 | } 33 | 34 | // InSequence returns the sequence. The reason why not naming it `Sequence` 35 | // is to avoid duplicated name as `(*Bucket) Sequence()` 36 | func (b *InBucket) InSequence() uint64 { 37 | return b.sequence 38 | } 39 | 40 | func (b *InBucket) SetInSequence(v uint64) { 41 | b.sequence = v 42 | } 43 | 44 | func (b *InBucket) IncSequence() { 45 | b.sequence++ 46 | } 47 | 48 | func (b *InBucket) InlinePage(v []byte) *Page { 49 | return (*Page)(unsafe.Pointer(&v[BucketHeaderSize])) 50 | } 51 | 52 | func (b *InBucket) String() string { 53 | return fmt.Sprintf("", b.root, b.sequence) 54 | } 55 | -------------------------------------------------------------------------------- /internal/common/inode.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "unsafe" 4 | 5 | // Inode represents an internal node inside of a node. 6 | // It can be used to point to elements in a page or point 7 | // to an element which hasn't been added to a page yet. 8 | type Inode struct { 9 | flags uint32 10 | pgid Pgid 11 | key []byte 12 | value []byte 13 | } 14 | 15 | type Inodes []Inode 16 | 17 | func (in *Inode) Flags() uint32 { 18 | return in.flags 19 | } 20 | 21 | func (in *Inode) SetFlags(flags uint32) { 22 | in.flags = flags 23 | } 24 | 25 | func (in *Inode) Pgid() Pgid { 26 | return in.pgid 27 | } 28 | 29 | func (in *Inode) SetPgid(id Pgid) { 30 | in.pgid = id 31 | } 32 | 33 | func (in *Inode) Key() []byte { 34 | return in.key 35 | } 36 | 37 | func (in *Inode) SetKey(key []byte) { 38 | in.key = key 39 | } 40 | 41 | func (in *Inode) Value() []byte { 42 | return in.value 43 | } 44 | 45 | func (in *Inode) SetValue(value []byte) { 46 | in.value = value 47 | } 48 | 49 | func ReadInodeFromPage(p *Page) Inodes { 50 | inodes := make(Inodes, int(p.Count())) 51 | isLeaf := p.IsLeafPage() 52 | for i := 0; i < int(p.Count()); i++ { 53 | inode := &inodes[i] 54 | if isLeaf { 55 | elem := p.LeafPageElement(uint16(i)) 56 | inode.SetFlags(elem.Flags()) 57 | inode.SetKey(elem.Key()) 58 | inode.SetValue(elem.Value()) 59 | } else { 60 | elem := p.BranchPageElement(uint16(i)) 61 | inode.SetPgid(elem.Pgid()) 62 | inode.SetKey(elem.Key()) 63 | } 64 | Assert(len(inode.Key()) > 0, "read: zero-length inode key") 65 | } 66 | 67 | return inodes 68 | } 69 | 70 | func WriteInodeToPage(inodes Inodes, p *Page) uint32 { 71 | // Loop over each item and write it to the page. 72 | // off tracks the offset into p of the start of the next data. 73 | off := unsafe.Sizeof(*p) + p.PageElementSize()*uintptr(len(inodes)) 74 | isLeaf := p.IsLeafPage() 75 | for i, item := range inodes { 76 | Assert(len(item.Key()) > 0, "write: zero-length inode key") 77 | 78 | // Create a slice to write into of needed size and advance 79 | // byte pointer for next iteration. 80 | sz := len(item.Key()) + len(item.Value()) 81 | b := UnsafeByteSlice(unsafe.Pointer(p), off, 0, sz) 82 | off += uintptr(sz) 83 | 84 | // Write the page element. 85 | if isLeaf { 86 | elem := p.LeafPageElement(uint16(i)) 87 | elem.SetPos(uint32(uintptr(unsafe.Pointer(&b[0])) - uintptr(unsafe.Pointer(elem)))) 88 | elem.SetFlags(item.Flags()) 89 | elem.SetKsize(uint32(len(item.Key()))) 90 | elem.SetVsize(uint32(len(item.Value()))) 91 | } else { 92 | elem := p.BranchPageElement(uint16(i)) 93 | elem.SetPos(uint32(uintptr(unsafe.Pointer(&b[0])) - uintptr(unsafe.Pointer(elem)))) 94 | elem.SetKsize(uint32(len(item.Key()))) 95 | elem.SetPgid(item.Pgid()) 96 | Assert(elem.Pgid() != p.Id(), "write: circular dependency occurred") 97 | } 98 | 99 | // Write data for the element to the end of the page. 100 | l := copy(b, item.Key()) 101 | copy(b[l:], item.Value()) 102 | } 103 | 104 | return uint32(off) 105 | } 106 | 107 | func UsedSpaceInPage(inodes Inodes, p *Page) uint32 { 108 | off := unsafe.Sizeof(*p) + p.PageElementSize()*uintptr(len(inodes)) 109 | for _, item := range inodes { 110 | sz := len(item.Key()) + len(item.Value()) 111 | off += uintptr(sz) 112 | } 113 | 114 | return uint32(off) 115 | } 116 | -------------------------------------------------------------------------------- /internal/common/meta.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "hash/fnv" 6 | "io" 7 | "unsafe" 8 | 9 | "go.etcd.io/bbolt/errors" 10 | ) 11 | 12 | type Meta struct { 13 | magic uint32 14 | version uint32 15 | pageSize uint32 16 | flags uint32 17 | root InBucket 18 | freelist Pgid 19 | pgid Pgid 20 | txid Txid 21 | checksum uint64 22 | } 23 | 24 | // Validate checks the marker bytes and version of the meta page to ensure it matches this binary. 25 | func (m *Meta) Validate() error { 26 | if m.magic != Magic { 27 | return errors.ErrInvalid 28 | } else if m.version != Version { 29 | return errors.ErrVersionMismatch 30 | } else if m.checksum != m.Sum64() { 31 | return errors.ErrChecksum 32 | } 33 | return nil 34 | } 35 | 36 | // Copy copies one meta object to another. 37 | func (m *Meta) Copy(dest *Meta) { 38 | *dest = *m 39 | } 40 | 41 | // Write writes the meta onto a page. 42 | func (m *Meta) Write(p *Page) { 43 | if m.root.root >= m.pgid { 44 | panic(fmt.Sprintf("root bucket pgid (%d) above high water mark (%d)", m.root.root, m.pgid)) 45 | } else if m.freelist >= m.pgid && m.freelist != PgidNoFreelist { 46 | // TODO: reject pgidNoFreeList if !NoFreelistSync 47 | panic(fmt.Sprintf("freelist pgid (%d) above high water mark (%d)", m.freelist, m.pgid)) 48 | } 49 | 50 | // Page id is either going to be 0 or 1 which we can determine by the transaction ID. 51 | p.id = Pgid(m.txid % 2) 52 | p.SetFlags(MetaPageFlag) 53 | 54 | // Calculate the checksum. 55 | m.checksum = m.Sum64() 56 | 57 | m.Copy(p.Meta()) 58 | } 59 | 60 | // Sum64 generates the checksum for the meta. 61 | func (m *Meta) Sum64() uint64 { 62 | var h = fnv.New64a() 63 | _, _ = h.Write((*[unsafe.Offsetof(Meta{}.checksum)]byte)(unsafe.Pointer(m))[:]) 64 | return h.Sum64() 65 | } 66 | 67 | func (m *Meta) Magic() uint32 { 68 | return m.magic 69 | } 70 | 71 | func (m *Meta) SetMagic(v uint32) { 72 | m.magic = v 73 | } 74 | 75 | func (m *Meta) Version() uint32 { 76 | return m.version 77 | } 78 | 79 | func (m *Meta) SetVersion(v uint32) { 80 | m.version = v 81 | } 82 | 83 | func (m *Meta) PageSize() uint32 { 84 | return m.pageSize 85 | } 86 | 87 | func (m *Meta) SetPageSize(v uint32) { 88 | m.pageSize = v 89 | } 90 | 91 | func (m *Meta) Flags() uint32 { 92 | return m.flags 93 | } 94 | 95 | func (m *Meta) SetFlags(v uint32) { 96 | m.flags = v 97 | } 98 | 99 | func (m *Meta) SetRootBucket(b InBucket) { 100 | m.root = b 101 | } 102 | 103 | func (m *Meta) RootBucket() *InBucket { 104 | return &m.root 105 | } 106 | 107 | func (m *Meta) Freelist() Pgid { 108 | return m.freelist 109 | } 110 | 111 | func (m *Meta) SetFreelist(v Pgid) { 112 | m.freelist = v 113 | } 114 | 115 | func (m *Meta) IsFreelistPersisted() bool { 116 | return m.freelist != PgidNoFreelist 117 | } 118 | 119 | func (m *Meta) Pgid() Pgid { 120 | return m.pgid 121 | } 122 | 123 | func (m *Meta) SetPgid(id Pgid) { 124 | m.pgid = id 125 | } 126 | 127 | func (m *Meta) Txid() Txid { 128 | return m.txid 129 | } 130 | 131 | func (m *Meta) SetTxid(id Txid) { 132 | m.txid = id 133 | } 134 | 135 | func (m *Meta) IncTxid() { 136 | m.txid += 1 137 | } 138 | 139 | func (m *Meta) DecTxid() { 140 | m.txid -= 1 141 | } 142 | 143 | func (m *Meta) Checksum() uint64 { 144 | return m.checksum 145 | } 146 | 147 | func (m *Meta) SetChecksum(v uint64) { 148 | m.checksum = v 149 | } 150 | 151 | func (m *Meta) Print(w io.Writer) { 152 | fmt.Fprintf(w, "Version: %d\n", m.version) 153 | fmt.Fprintf(w, "Page Size: %d bytes\n", m.pageSize) 154 | fmt.Fprintf(w, "Flags: %08x\n", m.flags) 155 | fmt.Fprintf(w, "Root: \n", m.root.root) 156 | fmt.Fprintf(w, "Freelist: \n", m.freelist) 157 | fmt.Fprintf(w, "HWM: \n", m.pgid) 158 | fmt.Fprintf(w, "Txn ID: %d\n", m.txid) 159 | fmt.Fprintf(w, "Checksum: %016x\n", m.checksum) 160 | fmt.Fprintf(w, "\n") 161 | } 162 | -------------------------------------------------------------------------------- /internal/common/page_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "reflect" 5 | "sort" 6 | "testing" 7 | "testing/quick" 8 | ) 9 | 10 | // Ensure that the page type can be returned in human readable format. 11 | func TestPage_typ(t *testing.T) { 12 | if typ := (&Page{flags: BranchPageFlag}).Typ(); typ != "branch" { 13 | t.Fatalf("exp=branch; got=%v", typ) 14 | } 15 | if typ := (&Page{flags: LeafPageFlag}).Typ(); typ != "leaf" { 16 | t.Fatalf("exp=leaf; got=%v", typ) 17 | } 18 | if typ := (&Page{flags: MetaPageFlag}).Typ(); typ != "meta" { 19 | t.Fatalf("exp=meta; got=%v", typ) 20 | } 21 | if typ := (&Page{flags: FreelistPageFlag}).Typ(); typ != "freelist" { 22 | t.Fatalf("exp=freelist; got=%v", typ) 23 | } 24 | if typ := (&Page{flags: 20000}).Typ(); typ != "unknown<4e20>" { 25 | t.Fatalf("exp=unknown<4e20>; got=%v", typ) 26 | } 27 | } 28 | 29 | // Ensure that the hexdump debugging function doesn't blow up. 30 | func TestPage_dump(t *testing.T) { 31 | (&Page{id: 256}).hexdump(16) 32 | } 33 | 34 | func TestPgids_merge(t *testing.T) { 35 | a := Pgids{4, 5, 6, 10, 11, 12, 13, 27} 36 | b := Pgids{1, 3, 8, 9, 25, 30} 37 | c := a.Merge(b) 38 | if !reflect.DeepEqual(c, Pgids{1, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 25, 27, 30}) { 39 | t.Errorf("mismatch: %v", c) 40 | } 41 | 42 | a = Pgids{4, 5, 6, 10, 11, 12, 13, 27, 35, 36} 43 | b = Pgids{8, 9, 25, 30} 44 | c = a.Merge(b) 45 | if !reflect.DeepEqual(c, Pgids{4, 5, 6, 8, 9, 10, 11, 12, 13, 25, 27, 30, 35, 36}) { 46 | t.Errorf("mismatch: %v", c) 47 | } 48 | } 49 | 50 | func TestPgids_merge_quick(t *testing.T) { 51 | if err := quick.Check(func(a, b Pgids) bool { 52 | // Sort incoming lists. 53 | sort.Sort(a) 54 | sort.Sort(b) 55 | 56 | // Merge the two lists together. 57 | got := a.Merge(b) 58 | 59 | // The expected value should be the two lists combined and sorted. 60 | exp := append(a, b...) 61 | sort.Sort(exp) 62 | 63 | if !reflect.DeepEqual(exp, got) { 64 | t.Errorf("\nexp=%+v\ngot=%+v\n", exp, got) 65 | return false 66 | } 67 | 68 | return true 69 | }, nil); err != nil { 70 | t.Fatal(err) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /internal/common/types.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "os" 5 | "runtime" 6 | "time" 7 | ) 8 | 9 | // MaxMmapStep is the largest step that can be taken when remapping the mmap. 10 | const MaxMmapStep = 1 << 30 // 1GB 11 | 12 | // Version represents the data file format version. 13 | const Version uint32 = 2 14 | 15 | // Magic represents a marker value to indicate that a file is a Bolt DB. 16 | const Magic uint32 = 0xED0CDAED 17 | 18 | const PgidNoFreelist Pgid = 0xffffffffffffffff 19 | 20 | // DO NOT EDIT. Copied from the "bolt" package. 21 | const pageMaxAllocSize = 0xFFFFFFF 22 | 23 | // IgnoreNoSync specifies whether the NoSync field of a DB is ignored when 24 | // syncing changes to a file. This is required as some operating systems, 25 | // such as OpenBSD, do not have a unified buffer cache (UBC) and writes 26 | // must be synchronized using the msync(2) syscall. 27 | const IgnoreNoSync = runtime.GOOS == "openbsd" 28 | 29 | // Default values if not set in a DB instance. 30 | const ( 31 | DefaultMaxBatchSize int = 1000 32 | DefaultMaxBatchDelay = 10 * time.Millisecond 33 | DefaultAllocSize = 16 * 1024 * 1024 34 | ) 35 | 36 | // DefaultPageSize is the default page size for db which is set to the OS page size. 37 | var DefaultPageSize = os.Getpagesize() 38 | 39 | // Txid represents the internal transaction identifier. 40 | type Txid uint64 41 | -------------------------------------------------------------------------------- /internal/common/unsafe.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "unsafe" 5 | ) 6 | 7 | func UnsafeAdd(base unsafe.Pointer, offset uintptr) unsafe.Pointer { 8 | return unsafe.Pointer(uintptr(base) + offset) 9 | } 10 | 11 | func UnsafeIndex(base unsafe.Pointer, offset uintptr, elemsz uintptr, n int) unsafe.Pointer { 12 | return unsafe.Pointer(uintptr(base) + offset + uintptr(n)*elemsz) 13 | } 14 | 15 | func UnsafeByteSlice(base unsafe.Pointer, offset uintptr, i, j int) []byte { 16 | // See: https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices 17 | // 18 | // This memory is not allocated from C, but it is unmanaged by Go's 19 | // garbage collector and should behave similarly, and the compiler 20 | // should produce similar code. Note that this conversion allows a 21 | // subslice to begin after the base address, with an optional offset, 22 | // while the URL above does not cover this case and only slices from 23 | // index 0. However, the wiki never says that the address must be to 24 | // the beginning of a C allocation (or even that malloc was used at 25 | // all), so this is believed to be correct. 26 | return (*[pageMaxAllocSize]byte)(UnsafeAdd(base, offset))[i:j:j] 27 | } 28 | -------------------------------------------------------------------------------- /internal/common/utils.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "unsafe" 8 | ) 9 | 10 | func LoadBucket(buf []byte) *InBucket { 11 | return (*InBucket)(unsafe.Pointer(&buf[0])) 12 | } 13 | 14 | func LoadPage(buf []byte) *Page { 15 | return (*Page)(unsafe.Pointer(&buf[0])) 16 | } 17 | 18 | func LoadPageMeta(buf []byte) *Meta { 19 | return (*Meta)(unsafe.Pointer(&buf[PageHeaderSize])) 20 | } 21 | 22 | func CopyFile(srcPath, dstPath string) error { 23 | // Ensure source file exists. 24 | _, err := os.Stat(srcPath) 25 | if os.IsNotExist(err) { 26 | return fmt.Errorf("source file %q not found", srcPath) 27 | } else if err != nil { 28 | return err 29 | } 30 | 31 | // Ensure output file not exist. 32 | _, err = os.Stat(dstPath) 33 | if err == nil { 34 | return fmt.Errorf("output file %q already exists", dstPath) 35 | } else if !os.IsNotExist(err) { 36 | return err 37 | } 38 | 39 | srcDB, err := os.Open(srcPath) 40 | if err != nil { 41 | return fmt.Errorf("failed to open source file %q: %w", srcPath, err) 42 | } 43 | defer srcDB.Close() 44 | dstDB, err := os.Create(dstPath) 45 | if err != nil { 46 | return fmt.Errorf("failed to create output file %q: %w", dstPath, err) 47 | } 48 | defer dstDB.Close() 49 | written, err := io.Copy(dstDB, srcDB) 50 | if err != nil { 51 | return fmt.Errorf("failed to copy database file from %q to %q: %w", srcPath, dstPath, err) 52 | } 53 | 54 | srcFi, err := srcDB.Stat() 55 | if err != nil { 56 | return fmt.Errorf("failed to get source file info %q: %w", srcPath, err) 57 | } 58 | initialSize := srcFi.Size() 59 | if initialSize != written { 60 | return fmt.Errorf("the byte copied (%q: %d) isn't equal to the initial db size (%q: %d)", dstPath, written, srcPath, initialSize) 61 | } 62 | 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /internal/common/verify.go: -------------------------------------------------------------------------------- 1 | // Copied from https://github.com/etcd-io/etcd/blob/main/client/pkg/verify/verify.go 2 | package common 3 | 4 | import ( 5 | "fmt" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | const ENV_VERIFY = "BBOLT_VERIFY" 11 | 12 | type VerificationType string 13 | 14 | const ( 15 | ENV_VERIFY_VALUE_ALL VerificationType = "all" 16 | ENV_VERIFY_VALUE_ASSERT VerificationType = "assert" 17 | ) 18 | 19 | func getEnvVerify() string { 20 | return strings.ToLower(os.Getenv(ENV_VERIFY)) 21 | } 22 | 23 | func IsVerificationEnabled(verification VerificationType) bool { 24 | env := getEnvVerify() 25 | return env == string(ENV_VERIFY_VALUE_ALL) || env == strings.ToLower(string(verification)) 26 | } 27 | 28 | // EnableVerifications sets `ENV_VERIFY` and returns a function that 29 | // can be used to bring the original settings. 30 | func EnableVerifications(verification VerificationType) func() { 31 | previousEnv := getEnvVerify() 32 | os.Setenv(ENV_VERIFY, string(verification)) 33 | return func() { 34 | os.Setenv(ENV_VERIFY, previousEnv) 35 | } 36 | } 37 | 38 | // EnableAllVerifications enables verification and returns a function 39 | // that can be used to bring the original settings. 40 | func EnableAllVerifications() func() { 41 | return EnableVerifications(ENV_VERIFY_VALUE_ALL) 42 | } 43 | 44 | // DisableVerifications unsets `ENV_VERIFY` and returns a function that 45 | // can be used to bring the original settings. 46 | func DisableVerifications() func() { 47 | previousEnv := getEnvVerify() 48 | os.Unsetenv(ENV_VERIFY) 49 | return func() { 50 | os.Setenv(ENV_VERIFY, previousEnv) 51 | } 52 | } 53 | 54 | // Verify performs verification if the assertions are enabled. 55 | // In the default setup running in tests and skipped in the production code. 56 | func Verify(f func()) { 57 | if IsVerificationEnabled(ENV_VERIFY_VALUE_ASSERT) { 58 | f() 59 | } 60 | } 61 | 62 | // Assert will panic with a given formatted message if the given condition is false. 63 | func Assert(condition bool, msg string, v ...any) { 64 | if !condition { 65 | panic(fmt.Sprintf("assertion failed: "+msg, v...)) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /internal/freelist/array.go: -------------------------------------------------------------------------------- 1 | package freelist 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | 7 | "go.etcd.io/bbolt/internal/common" 8 | ) 9 | 10 | type array struct { 11 | *shared 12 | 13 | ids []common.Pgid // all free and available free page ids. 14 | } 15 | 16 | func (f *array) Init(ids common.Pgids) { 17 | f.ids = ids 18 | f.reindex() 19 | } 20 | 21 | func (f *array) Allocate(txid common.Txid, n int) common.Pgid { 22 | if len(f.ids) == 0 { 23 | return 0 24 | } 25 | 26 | var initial, previd common.Pgid 27 | for i, id := range f.ids { 28 | if id <= 1 { 29 | panic(fmt.Sprintf("invalid page allocation: %d", id)) 30 | } 31 | 32 | // Reset initial page if this is not contiguous. 33 | if previd == 0 || id-previd != 1 { 34 | initial = id 35 | } 36 | 37 | // If we found a contiguous block then remove it and return it. 38 | if (id-initial)+1 == common.Pgid(n) { 39 | // If we're allocating off the beginning then take the fast path 40 | // and just adjust the existing slice. This will use extra memory 41 | // temporarily but the append() in free() will realloc the slice 42 | // as is necessary. 43 | if (i + 1) == n { 44 | f.ids = f.ids[i+1:] 45 | } else { 46 | copy(f.ids[i-n+1:], f.ids[i+1:]) 47 | f.ids = f.ids[:len(f.ids)-n] 48 | } 49 | 50 | // Remove from the free cache. 51 | for i := common.Pgid(0); i < common.Pgid(n); i++ { 52 | delete(f.cache, initial+i) 53 | } 54 | f.allocs[initial] = txid 55 | return initial 56 | } 57 | 58 | previd = id 59 | } 60 | return 0 61 | } 62 | 63 | func (f *array) FreeCount() int { 64 | return len(f.ids) 65 | } 66 | 67 | func (f *array) freePageIds() common.Pgids { 68 | return f.ids 69 | } 70 | 71 | func (f *array) mergeSpans(ids common.Pgids) { 72 | sort.Sort(ids) 73 | common.Verify(func() { 74 | idsIdx := make(map[common.Pgid]struct{}) 75 | for _, id := range f.ids { 76 | // The existing f.ids shouldn't have duplicated free ID. 77 | if _, ok := idsIdx[id]; ok { 78 | panic(fmt.Sprintf("detected duplicated free page ID: %d in existing f.ids: %v", id, f.ids)) 79 | } 80 | idsIdx[id] = struct{}{} 81 | } 82 | 83 | prev := common.Pgid(0) 84 | for _, id := range ids { 85 | // The ids shouldn't have duplicated free ID. Note page 0 and 1 86 | // are reserved for meta pages, so they can never be free page IDs. 87 | if prev == id { 88 | panic(fmt.Sprintf("detected duplicated free ID: %d in ids: %v", id, ids)) 89 | } 90 | prev = id 91 | 92 | // The ids shouldn't have any overlap with the existing f.ids. 93 | if _, ok := idsIdx[id]; ok { 94 | panic(fmt.Sprintf("detected overlapped free page ID: %d between ids: %v and existing f.ids: %v", id, ids, f.ids)) 95 | } 96 | } 97 | }) 98 | f.ids = common.Pgids(f.ids).Merge(ids) 99 | } 100 | 101 | func NewArrayFreelist() Interface { 102 | a := &array{ 103 | shared: newShared(), 104 | ids: []common.Pgid{}, 105 | } 106 | a.Interface = a 107 | return a 108 | } 109 | -------------------------------------------------------------------------------- /internal/freelist/array_test.go: -------------------------------------------------------------------------------- 1 | package freelist 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | 9 | "go.etcd.io/bbolt/internal/common" 10 | ) 11 | 12 | // Ensure that a freelist can find contiguous blocks of pages. 13 | func TestFreelistArray_allocate(t *testing.T) { 14 | f := NewArrayFreelist() 15 | ids := []common.Pgid{3, 4, 5, 6, 7, 9, 12, 13, 18} 16 | f.Init(ids) 17 | if id := int(f.Allocate(1, 3)); id != 3 { 18 | t.Fatalf("exp=3; got=%v", id) 19 | } 20 | if id := int(f.Allocate(1, 1)); id != 6 { 21 | t.Fatalf("exp=6; got=%v", id) 22 | } 23 | if id := int(f.Allocate(1, 3)); id != 0 { 24 | t.Fatalf("exp=0; got=%v", id) 25 | } 26 | if id := int(f.Allocate(1, 2)); id != 12 { 27 | t.Fatalf("exp=12; got=%v", id) 28 | } 29 | if id := int(f.Allocate(1, 1)); id != 7 { 30 | t.Fatalf("exp=7; got=%v", id) 31 | } 32 | if id := int(f.Allocate(1, 0)); id != 0 { 33 | t.Fatalf("exp=0; got=%v", id) 34 | } 35 | if id := int(f.Allocate(1, 0)); id != 0 { 36 | t.Fatalf("exp=0; got=%v", id) 37 | } 38 | if exp := common.Pgids([]common.Pgid{9, 18}); !reflect.DeepEqual(exp, f.freePageIds()) { 39 | t.Fatalf("exp=%v; got=%v", exp, f.freePageIds()) 40 | } 41 | 42 | if id := int(f.Allocate(1, 1)); id != 9 { 43 | t.Fatalf("exp=9; got=%v", id) 44 | } 45 | if id := int(f.Allocate(1, 1)); id != 18 { 46 | t.Fatalf("exp=18; got=%v", id) 47 | } 48 | if id := int(f.Allocate(1, 1)); id != 0 { 49 | t.Fatalf("exp=0; got=%v", id) 50 | } 51 | if exp := common.Pgids([]common.Pgid{}); !reflect.DeepEqual(exp, f.freePageIds()) { 52 | t.Fatalf("exp=%v; got=%v", exp, f.freePageIds()) 53 | } 54 | } 55 | 56 | func TestInvalidArrayAllocation(t *testing.T) { 57 | f := NewArrayFreelist() 58 | // page 0 and 1 are reserved for meta pages, so they should never be free pages. 59 | ids := []common.Pgid{1} 60 | f.Init(ids) 61 | require.Panics(t, func() { 62 | f.Allocate(common.Txid(1), 1) 63 | }) 64 | } 65 | 66 | func Test_Freelist_Array_Rollback(t *testing.T) { 67 | f := newTestArrayFreelist() 68 | 69 | f.Init([]common.Pgid{3, 5, 6, 7, 12, 13}) 70 | 71 | f.Free(100, common.NewPage(20, 0, 0, 1)) 72 | f.Allocate(100, 3) 73 | f.Free(100, common.NewPage(25, 0, 0, 0)) 74 | f.Allocate(100, 2) 75 | 76 | require.Equal(t, map[common.Pgid]common.Txid{5: 100, 12: 100}, f.allocs) 77 | require.Equal(t, map[common.Txid]*txPending{100: { 78 | ids: []common.Pgid{20, 21, 25}, 79 | alloctx: []common.Txid{0, 0, 0}, 80 | }}, f.pending) 81 | 82 | f.Rollback(100) 83 | 84 | require.Equal(t, map[common.Pgid]common.Txid{}, f.allocs) 85 | require.Equal(t, map[common.Txid]*txPending{}, f.pending) 86 | } 87 | 88 | func newTestArrayFreelist() *array { 89 | f := NewArrayFreelist() 90 | return f.(*array) 91 | } 92 | -------------------------------------------------------------------------------- /internal/freelist/freelist.go: -------------------------------------------------------------------------------- 1 | package freelist 2 | 3 | import ( 4 | "go.etcd.io/bbolt/internal/common" 5 | ) 6 | 7 | type ReadWriter interface { 8 | // Read calls Init with the page ids stored in the given page. 9 | Read(page *common.Page) 10 | 11 | // Write writes the freelist into the given page. 12 | Write(page *common.Page) 13 | 14 | // EstimatedWritePageSize returns the size in bytes of the freelist after serialization in Write. 15 | // This should never underestimate the size. 16 | EstimatedWritePageSize() int 17 | } 18 | 19 | type Interface interface { 20 | ReadWriter 21 | 22 | // Init initializes this freelist with the given list of pages. 23 | Init(ids common.Pgids) 24 | 25 | // Allocate tries to allocate the given number of contiguous pages 26 | // from the free list pages. It returns the starting page ID if 27 | // available; otherwise, it returns 0. 28 | Allocate(txid common.Txid, numPages int) common.Pgid 29 | 30 | // Count returns the number of free and pending pages. 31 | Count() int 32 | 33 | // FreeCount returns the number of free pages. 34 | FreeCount() int 35 | 36 | // PendingCount returns the number of pending pages. 37 | PendingCount() int 38 | 39 | // AddReadonlyTXID adds a given read-only transaction id for pending page tracking. 40 | AddReadonlyTXID(txid common.Txid) 41 | 42 | // RemoveReadonlyTXID removes a given read-only transaction id for pending page tracking. 43 | RemoveReadonlyTXID(txid common.Txid) 44 | 45 | // ReleasePendingPages releases any pages associated with closed read-only transactions. 46 | ReleasePendingPages() 47 | 48 | // Free releases a page and its overflow for a given transaction id. 49 | // If the page is already free or is one of the meta pages, then a panic will occur. 50 | Free(txId common.Txid, p *common.Page) 51 | 52 | // Freed returns whether a given page is in the free list. 53 | Freed(pgId common.Pgid) bool 54 | 55 | // Rollback removes the pages from a given pending tx. 56 | Rollback(txId common.Txid) 57 | 58 | // Copyall copies a list of all free ids and all pending ids in one sorted list. 59 | // f.count returns the minimum length required for dst. 60 | Copyall(dst []common.Pgid) 61 | 62 | // Reload reads the freelist from a page and filters out pending items. 63 | Reload(p *common.Page) 64 | 65 | // NoSyncReload reads the freelist from Pgids and filters out pending items. 66 | NoSyncReload(pgIds common.Pgids) 67 | 68 | // freePageIds returns the IDs of all free pages. Returns an empty slice if no free pages are available. 69 | freePageIds() common.Pgids 70 | 71 | // pendingPageIds returns all pending pages by transaction id. 72 | pendingPageIds() map[common.Txid]*txPending 73 | 74 | // release moves all page ids for a transaction id (or older) to the freelist. 75 | release(txId common.Txid) 76 | 77 | // releaseRange moves pending pages allocated within an extent [begin,end] to the free list. 78 | releaseRange(begin, end common.Txid) 79 | 80 | // mergeSpans is merging the given pages into the freelist 81 | mergeSpans(ids common.Pgids) 82 | } 83 | -------------------------------------------------------------------------------- /internal/freelist/hashmap.go: -------------------------------------------------------------------------------- 1 | package freelist 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "sort" 7 | 8 | "go.etcd.io/bbolt/internal/common" 9 | ) 10 | 11 | // pidSet holds the set of starting pgids which have the same span size 12 | type pidSet map[common.Pgid]struct{} 13 | 14 | type hashMap struct { 15 | *shared 16 | 17 | freePagesCount uint64 // count of free pages(hashmap version) 18 | freemaps map[uint64]pidSet // key is the size of continuous pages(span), value is a set which contains the starting pgids of same size 19 | forwardMap map[common.Pgid]uint64 // key is start pgid, value is its span size 20 | backwardMap map[common.Pgid]uint64 // key is end pgid, value is its span size 21 | } 22 | 23 | func (f *hashMap) Init(pgids common.Pgids) { 24 | // reset the counter when freelist init 25 | f.freePagesCount = 0 26 | f.freemaps = make(map[uint64]pidSet) 27 | f.forwardMap = make(map[common.Pgid]uint64) 28 | f.backwardMap = make(map[common.Pgid]uint64) 29 | 30 | if len(pgids) == 0 { 31 | return 32 | } 33 | 34 | if !sort.SliceIsSorted([]common.Pgid(pgids), func(i, j int) bool { return pgids[i] < pgids[j] }) { 35 | panic("pgids not sorted") 36 | } 37 | 38 | size := uint64(1) 39 | start := pgids[0] 40 | 41 | for i := 1; i < len(pgids); i++ { 42 | // continuous page 43 | if pgids[i] == pgids[i-1]+1 { 44 | size++ 45 | } else { 46 | f.addSpan(start, size) 47 | 48 | size = 1 49 | start = pgids[i] 50 | } 51 | } 52 | 53 | // init the tail 54 | if size != 0 && start != 0 { 55 | f.addSpan(start, size) 56 | } 57 | 58 | f.reindex() 59 | } 60 | 61 | func (f *hashMap) Allocate(txid common.Txid, n int) common.Pgid { 62 | if n == 0 { 63 | return 0 64 | } 65 | 66 | // if we have a exact size match just return short path 67 | if bm, ok := f.freemaps[uint64(n)]; ok { 68 | for pid := range bm { 69 | // remove the span 70 | f.delSpan(pid, uint64(n)) 71 | 72 | f.allocs[pid] = txid 73 | 74 | for i := common.Pgid(0); i < common.Pgid(n); i++ { 75 | delete(f.cache, pid+i) 76 | } 77 | return pid 78 | } 79 | } 80 | 81 | // lookup the map to find larger span 82 | for size, bm := range f.freemaps { 83 | if size < uint64(n) { 84 | continue 85 | } 86 | 87 | for pid := range bm { 88 | // remove the initial 89 | f.delSpan(pid, size) 90 | 91 | f.allocs[pid] = txid 92 | 93 | remain := size - uint64(n) 94 | 95 | // add remain span 96 | f.addSpan(pid+common.Pgid(n), remain) 97 | 98 | for i := common.Pgid(0); i < common.Pgid(n); i++ { 99 | delete(f.cache, pid+i) 100 | } 101 | return pid 102 | } 103 | } 104 | 105 | return 0 106 | } 107 | 108 | func (f *hashMap) FreeCount() int { 109 | common.Verify(func() { 110 | expectedFreePageCount := f.hashmapFreeCountSlow() 111 | common.Assert(int(f.freePagesCount) == expectedFreePageCount, 112 | "freePagesCount (%d) is out of sync with free pages map (%d)", f.freePagesCount, expectedFreePageCount) 113 | }) 114 | return int(f.freePagesCount) 115 | } 116 | 117 | func (f *hashMap) freePageIds() common.Pgids { 118 | count := f.FreeCount() 119 | if count == 0 { 120 | return common.Pgids{} 121 | } 122 | 123 | m := make([]common.Pgid, 0, count) 124 | 125 | startPageIds := make([]common.Pgid, 0, len(f.forwardMap)) 126 | for k := range f.forwardMap { 127 | startPageIds = append(startPageIds, k) 128 | } 129 | sort.Sort(common.Pgids(startPageIds)) 130 | 131 | for _, start := range startPageIds { 132 | if size, ok := f.forwardMap[start]; ok { 133 | for i := 0; i < int(size); i++ { 134 | m = append(m, start+common.Pgid(i)) 135 | } 136 | } 137 | } 138 | 139 | return m 140 | } 141 | 142 | func (f *hashMap) hashmapFreeCountSlow() int { 143 | count := 0 144 | for _, size := range f.forwardMap { 145 | count += int(size) 146 | } 147 | return count 148 | } 149 | 150 | func (f *hashMap) addSpan(start common.Pgid, size uint64) { 151 | f.backwardMap[start-1+common.Pgid(size)] = size 152 | f.forwardMap[start] = size 153 | if _, ok := f.freemaps[size]; !ok { 154 | f.freemaps[size] = make(map[common.Pgid]struct{}) 155 | } 156 | 157 | f.freemaps[size][start] = struct{}{} 158 | f.freePagesCount += size 159 | } 160 | 161 | func (f *hashMap) delSpan(start common.Pgid, size uint64) { 162 | delete(f.forwardMap, start) 163 | delete(f.backwardMap, start+common.Pgid(size-1)) 164 | delete(f.freemaps[size], start) 165 | if len(f.freemaps[size]) == 0 { 166 | delete(f.freemaps, size) 167 | } 168 | f.freePagesCount -= size 169 | } 170 | 171 | func (f *hashMap) mergeSpans(ids common.Pgids) { 172 | common.Verify(func() { 173 | ids1Freemap := f.idsFromFreemaps() 174 | ids2Forward := f.idsFromForwardMap() 175 | ids3Backward := f.idsFromBackwardMap() 176 | 177 | if !reflect.DeepEqual(ids1Freemap, ids2Forward) { 178 | panic(fmt.Sprintf("Detected mismatch, f.freemaps: %v, f.forwardMap: %v", f.freemaps, f.forwardMap)) 179 | } 180 | if !reflect.DeepEqual(ids1Freemap, ids3Backward) { 181 | panic(fmt.Sprintf("Detected mismatch, f.freemaps: %v, f.backwardMap: %v", f.freemaps, f.backwardMap)) 182 | } 183 | 184 | sort.Sort(ids) 185 | prev := common.Pgid(0) 186 | for _, id := range ids { 187 | // The ids shouldn't have duplicated free ID. 188 | if prev == id { 189 | panic(fmt.Sprintf("detected duplicated free ID: %d in ids: %v", id, ids)) 190 | } 191 | prev = id 192 | 193 | // The ids shouldn't have any overlap with the existing f.freemaps. 194 | if _, ok := ids1Freemap[id]; ok { 195 | panic(fmt.Sprintf("detected overlapped free page ID: %d between ids: %v and existing f.freemaps: %v", id, ids, f.freemaps)) 196 | } 197 | } 198 | }) 199 | for _, id := range ids { 200 | // try to see if we can merge and update 201 | f.mergeWithExistingSpan(id) 202 | } 203 | } 204 | 205 | // mergeWithExistingSpan merges pid to the existing free spans, try to merge it backward and forward 206 | func (f *hashMap) mergeWithExistingSpan(pid common.Pgid) { 207 | prev := pid - 1 208 | next := pid + 1 209 | 210 | preSize, mergeWithPrev := f.backwardMap[prev] 211 | nextSize, mergeWithNext := f.forwardMap[next] 212 | newStart := pid 213 | newSize := uint64(1) 214 | 215 | if mergeWithPrev { 216 | //merge with previous span 217 | start := prev + 1 - common.Pgid(preSize) 218 | f.delSpan(start, preSize) 219 | 220 | newStart -= common.Pgid(preSize) 221 | newSize += preSize 222 | } 223 | 224 | if mergeWithNext { 225 | // merge with next span 226 | f.delSpan(next, nextSize) 227 | newSize += nextSize 228 | } 229 | 230 | f.addSpan(newStart, newSize) 231 | } 232 | 233 | // idsFromFreemaps get all free page IDs from f.freemaps. 234 | // used by test only. 235 | func (f *hashMap) idsFromFreemaps() map[common.Pgid]struct{} { 236 | ids := make(map[common.Pgid]struct{}) 237 | for size, idSet := range f.freemaps { 238 | for start := range idSet { 239 | for i := 0; i < int(size); i++ { 240 | id := start + common.Pgid(i) 241 | if _, ok := ids[id]; ok { 242 | panic(fmt.Sprintf("detected duplicated free page ID: %d in f.freemaps: %v", id, f.freemaps)) 243 | } 244 | ids[id] = struct{}{} 245 | } 246 | } 247 | } 248 | return ids 249 | } 250 | 251 | // idsFromForwardMap get all free page IDs from f.forwardMap. 252 | // used by test only. 253 | func (f *hashMap) idsFromForwardMap() map[common.Pgid]struct{} { 254 | ids := make(map[common.Pgid]struct{}) 255 | for start, size := range f.forwardMap { 256 | for i := 0; i < int(size); i++ { 257 | id := start + common.Pgid(i) 258 | if _, ok := ids[id]; ok { 259 | panic(fmt.Sprintf("detected duplicated free page ID: %d in f.forwardMap: %v", id, f.forwardMap)) 260 | } 261 | ids[id] = struct{}{} 262 | } 263 | } 264 | return ids 265 | } 266 | 267 | // idsFromBackwardMap get all free page IDs from f.backwardMap. 268 | // used by test only. 269 | func (f *hashMap) idsFromBackwardMap() map[common.Pgid]struct{} { 270 | ids := make(map[common.Pgid]struct{}) 271 | for end, size := range f.backwardMap { 272 | for i := 0; i < int(size); i++ { 273 | id := end - common.Pgid(i) 274 | if _, ok := ids[id]; ok { 275 | panic(fmt.Sprintf("detected duplicated free page ID: %d in f.backwardMap: %v", id, f.backwardMap)) 276 | } 277 | ids[id] = struct{}{} 278 | } 279 | } 280 | return ids 281 | } 282 | 283 | func NewHashMapFreelist() Interface { 284 | hm := &hashMap{ 285 | shared: newShared(), 286 | freemaps: make(map[uint64]pidSet), 287 | forwardMap: make(map[common.Pgid]uint64), 288 | backwardMap: make(map[common.Pgid]uint64), 289 | } 290 | hm.Interface = hm 291 | return hm 292 | } 293 | -------------------------------------------------------------------------------- /internal/freelist/hashmap_test.go: -------------------------------------------------------------------------------- 1 | package freelist 2 | 3 | import ( 4 | "math/rand" 5 | "reflect" 6 | "sort" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "go.etcd.io/bbolt/internal/common" 12 | ) 13 | 14 | func TestFreelistHashmap_init_panics(t *testing.T) { 15 | f := NewHashMapFreelist() 16 | require.Panics(t, func() { 17 | // init expects sorted input 18 | f.Init([]common.Pgid{25, 5}) 19 | }) 20 | } 21 | 22 | func TestFreelistHashmap_allocate(t *testing.T) { 23 | f := NewHashMapFreelist() 24 | 25 | ids := []common.Pgid{3, 4, 5, 6, 7, 9, 12, 13, 18} 26 | f.Init(ids) 27 | 28 | f.Allocate(1, 3) 29 | if x := f.FreeCount(); x != 6 { 30 | t.Fatalf("exp=6; got=%v", x) 31 | } 32 | 33 | f.Allocate(1, 2) 34 | if x := f.FreeCount(); x != 4 { 35 | t.Fatalf("exp=4; got=%v", x) 36 | } 37 | f.Allocate(1, 1) 38 | if x := f.FreeCount(); x != 3 { 39 | t.Fatalf("exp=3; got=%v", x) 40 | } 41 | 42 | f.Allocate(1, 0) 43 | if x := f.FreeCount(); x != 3 { 44 | t.Fatalf("exp=3; got=%v", x) 45 | } 46 | } 47 | 48 | func TestFreelistHashmap_mergeWithExist(t *testing.T) { 49 | bm1 := pidSet{1: struct{}{}} 50 | 51 | bm2 := pidSet{5: struct{}{}} 52 | tests := []struct { 53 | name string 54 | ids common.Pgids 55 | pgid common.Pgid 56 | want common.Pgids 57 | wantForwardmap map[common.Pgid]uint64 58 | wantBackwardmap map[common.Pgid]uint64 59 | wantfreemap map[uint64]pidSet 60 | }{ 61 | { 62 | name: "test1", 63 | ids: []common.Pgid{1, 2, 4, 5, 6}, 64 | pgid: 3, 65 | want: []common.Pgid{1, 2, 3, 4, 5, 6}, 66 | wantForwardmap: map[common.Pgid]uint64{1: 6}, 67 | wantBackwardmap: map[common.Pgid]uint64{6: 6}, 68 | wantfreemap: map[uint64]pidSet{6: bm1}, 69 | }, 70 | { 71 | name: "test2", 72 | ids: []common.Pgid{1, 2, 5, 6}, 73 | pgid: 3, 74 | want: []common.Pgid{1, 2, 3, 5, 6}, 75 | wantForwardmap: map[common.Pgid]uint64{1: 3, 5: 2}, 76 | wantBackwardmap: map[common.Pgid]uint64{6: 2, 3: 3}, 77 | wantfreemap: map[uint64]pidSet{3: bm1, 2: bm2}, 78 | }, 79 | { 80 | name: "test3", 81 | ids: []common.Pgid{1, 2}, 82 | pgid: 3, 83 | want: []common.Pgid{1, 2, 3}, 84 | wantForwardmap: map[common.Pgid]uint64{1: 3}, 85 | wantBackwardmap: map[common.Pgid]uint64{3: 3}, 86 | wantfreemap: map[uint64]pidSet{3: bm1}, 87 | }, 88 | { 89 | name: "test4", 90 | ids: []common.Pgid{2, 3}, 91 | pgid: 1, 92 | want: []common.Pgid{1, 2, 3}, 93 | wantForwardmap: map[common.Pgid]uint64{1: 3}, 94 | wantBackwardmap: map[common.Pgid]uint64{3: 3}, 95 | wantfreemap: map[uint64]pidSet{3: bm1}, 96 | }, 97 | } 98 | for _, tt := range tests { 99 | f := newTestHashMapFreelist() 100 | f.Init(tt.ids) 101 | 102 | f.mergeWithExistingSpan(tt.pgid) 103 | 104 | if got := f.freePageIds(); !reflect.DeepEqual(tt.want, got) { 105 | t.Fatalf("name %s; exp=%v; got=%v", tt.name, tt.want, got) 106 | } 107 | if got := f.forwardMap; !reflect.DeepEqual(tt.wantForwardmap, got) { 108 | t.Fatalf("name %s; exp=%v; got=%v", tt.name, tt.wantForwardmap, got) 109 | } 110 | if got := f.backwardMap; !reflect.DeepEqual(tt.wantBackwardmap, got) { 111 | t.Fatalf("name %s; exp=%v; got=%v", tt.name, tt.wantBackwardmap, got) 112 | } 113 | if got := f.freemaps; !reflect.DeepEqual(tt.wantfreemap, got) { 114 | t.Fatalf("name %s; exp=%v; got=%v", tt.name, tt.wantfreemap, got) 115 | } 116 | } 117 | } 118 | 119 | func TestFreelistHashmap_GetFreePageIDs(t *testing.T) { 120 | f := newTestHashMapFreelist() 121 | 122 | N := int32(100000) 123 | fm := make(map[common.Pgid]uint64) 124 | i := int32(0) 125 | val := int32(0) 126 | for i = 0; i < N; { 127 | val = rand.Int31n(1000) 128 | fm[common.Pgid(i)] = uint64(val) 129 | i += val 130 | f.freePagesCount += uint64(val) 131 | } 132 | 133 | f.forwardMap = fm 134 | res := f.freePageIds() 135 | 136 | if !sort.SliceIsSorted(res, func(i, j int) bool { return res[i] < res[j] }) { 137 | t.Fatalf("pgids not sorted") 138 | } 139 | } 140 | 141 | func Test_Freelist_Hashmap_Rollback(t *testing.T) { 142 | f := newTestHashMapFreelist() 143 | 144 | f.Init([]common.Pgid{3, 5, 6, 7, 12, 13}) 145 | 146 | f.Free(100, common.NewPage(20, 0, 0, 1)) 147 | f.Allocate(100, 3) 148 | f.Free(100, common.NewPage(25, 0, 0, 0)) 149 | f.Allocate(100, 2) 150 | 151 | require.Equal(t, map[common.Pgid]common.Txid{5: 100, 12: 100}, f.allocs) 152 | require.Equal(t, map[common.Txid]*txPending{100: { 153 | ids: []common.Pgid{20, 21, 25}, 154 | alloctx: []common.Txid{0, 0, 0}, 155 | }}, f.pending) 156 | 157 | f.Rollback(100) 158 | 159 | require.Equal(t, map[common.Pgid]common.Txid{}, f.allocs) 160 | require.Equal(t, map[common.Txid]*txPending{}, f.pending) 161 | } 162 | 163 | func Benchmark_freelist_hashmapGetFreePageIDs(b *testing.B) { 164 | f := newTestHashMapFreelist() 165 | N := int32(100000) 166 | fm := make(map[common.Pgid]uint64) 167 | i := int32(0) 168 | val := int32(0) 169 | for i = 0; i < N; { 170 | val = rand.Int31n(1000) 171 | fm[common.Pgid(i)] = uint64(val) 172 | i += val 173 | } 174 | 175 | f.forwardMap = fm 176 | 177 | b.ReportAllocs() 178 | b.ResetTimer() 179 | for n := 0; n < b.N; n++ { 180 | f.freePageIds() 181 | } 182 | } 183 | 184 | func newTestHashMapFreelist() *hashMap { 185 | f := NewHashMapFreelist() 186 | return f.(*hashMap) 187 | } 188 | -------------------------------------------------------------------------------- /internal/guts_cli/guts_cli.go: -------------------------------------------------------------------------------- 1 | package guts_cli 2 | 3 | // Low level access to pages / data-structures of the bbolt file. 4 | 5 | import ( 6 | "errors" 7 | "fmt" 8 | "io" 9 | "os" 10 | 11 | "go.etcd.io/bbolt/internal/common" 12 | ) 13 | 14 | var ( 15 | // ErrCorrupt is returned when a checking a data file finds errors. 16 | ErrCorrupt = errors.New("invalid value") 17 | ) 18 | 19 | // ReadPage reads Page info & full Page data from a path. 20 | // This is not transactionally safe. 21 | func ReadPage(path string, pageID uint64) (*common.Page, []byte, error) { 22 | // Find Page size. 23 | pageSize, hwm, err := ReadPageAndHWMSize(path) 24 | if err != nil { 25 | return nil, nil, fmt.Errorf("read Page size: %s", err) 26 | } 27 | 28 | // Open database file. 29 | f, err := os.Open(path) 30 | if err != nil { 31 | return nil, nil, err 32 | } 33 | defer f.Close() 34 | 35 | // Read one block into buffer. 36 | buf := make([]byte, pageSize) 37 | if n, err := f.ReadAt(buf, int64(pageID*pageSize)); err != nil { 38 | return nil, nil, err 39 | } else if n != len(buf) { 40 | return nil, nil, io.ErrUnexpectedEOF 41 | } 42 | 43 | // Determine total number of blocks. 44 | p := common.LoadPage(buf) 45 | if p.Id() != common.Pgid(pageID) { 46 | return nil, nil, fmt.Errorf("error: %w due to unexpected Page id: %d != %d", ErrCorrupt, p.Id(), pageID) 47 | } 48 | overflowN := p.Overflow() 49 | if overflowN >= uint32(hwm)-3 { // we exclude 2 Meta pages and the current Page. 50 | return nil, nil, fmt.Errorf("error: %w, Page claims to have %d overflow pages (>=hwm=%d). Interrupting to avoid risky OOM", ErrCorrupt, overflowN, hwm) 51 | } 52 | 53 | if overflowN == 0 { 54 | return p, buf, nil 55 | } 56 | 57 | // Re-read entire Page (with overflow) into buffer. 58 | buf = make([]byte, (uint64(overflowN)+1)*pageSize) 59 | if n, err := f.ReadAt(buf, int64(pageID*pageSize)); err != nil { 60 | return nil, nil, err 61 | } else if n != len(buf) { 62 | return nil, nil, io.ErrUnexpectedEOF 63 | } 64 | p = common.LoadPage(buf) 65 | if p.Id() != common.Pgid(pageID) { 66 | return nil, nil, fmt.Errorf("error: %w due to unexpected Page id: %d != %d", ErrCorrupt, p.Id(), pageID) 67 | } 68 | 69 | return p, buf, nil 70 | } 71 | 72 | func WritePage(path string, pageBuf []byte) error { 73 | page := common.LoadPage(pageBuf) 74 | pageSize, _, err := ReadPageAndHWMSize(path) 75 | if err != nil { 76 | return err 77 | } 78 | expectedLen := pageSize * (uint64(page.Overflow()) + 1) 79 | if expectedLen != uint64(len(pageBuf)) { 80 | return fmt.Errorf("WritePage: len(buf):%d != pageSize*(overflow+1):%d", len(pageBuf), expectedLen) 81 | } 82 | f, err := os.OpenFile(path, os.O_WRONLY, 0) 83 | if err != nil { 84 | return err 85 | } 86 | defer f.Close() 87 | _, err = f.WriteAt(pageBuf, int64(page.Id())*int64(pageSize)) 88 | return err 89 | } 90 | 91 | // ReadPageAndHWMSize reads Page size and HWM (id of the last+1 Page). 92 | // This is not transactionally safe. 93 | func ReadPageAndHWMSize(path string) (uint64, common.Pgid, error) { 94 | // Open database file. 95 | f, err := os.Open(path) 96 | if err != nil { 97 | return 0, 0, err 98 | } 99 | defer f.Close() 100 | 101 | // Read 4KB chunk. 102 | buf := make([]byte, 4096) 103 | if _, err := io.ReadFull(f, buf); err != nil { 104 | return 0, 0, err 105 | } 106 | 107 | // Read Page size from metadata. 108 | m := common.LoadPageMeta(buf) 109 | if m.Magic() != common.Magic { 110 | return 0, 0, fmt.Errorf("the Meta Page has wrong (unexpected) magic") 111 | } 112 | return uint64(m.PageSize()), common.Pgid(m.Pgid()), nil 113 | } 114 | 115 | // GetRootPage returns the root-page (according to the most recent transaction). 116 | func GetRootPage(path string) (root common.Pgid, activeMeta common.Pgid, err error) { 117 | m, id, err := GetActiveMetaPage(path) 118 | if err != nil { 119 | return 0, id, err 120 | } 121 | return m.RootBucket().RootPage(), id, nil 122 | } 123 | 124 | // GetActiveMetaPage returns the active meta page and its page ID (0 or 1). 125 | func GetActiveMetaPage(path string) (*common.Meta, common.Pgid, error) { 126 | _, buf0, err0 := ReadPage(path, 0) 127 | if err0 != nil { 128 | return nil, 0, err0 129 | } 130 | m0 := common.LoadPageMeta(buf0) 131 | _, buf1, err1 := ReadPage(path, 1) 132 | if err1 != nil { 133 | return nil, 1, err1 134 | } 135 | m1 := common.LoadPageMeta(buf1) 136 | if m0.Txid() < m1.Txid() { 137 | return m1, 1, nil 138 | } else { 139 | return m0, 0, nil 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /internal/surgeon/surgeon.go: -------------------------------------------------------------------------------- 1 | package surgeon 2 | 3 | import ( 4 | "fmt" 5 | 6 | "go.etcd.io/bbolt/internal/common" 7 | "go.etcd.io/bbolt/internal/guts_cli" 8 | ) 9 | 10 | func CopyPage(path string, srcPage common.Pgid, target common.Pgid) error { 11 | p1, d1, err1 := guts_cli.ReadPage(path, uint64(srcPage)) 12 | if err1 != nil { 13 | return err1 14 | } 15 | p1.SetId(target) 16 | return guts_cli.WritePage(path, d1) 17 | } 18 | 19 | func ClearPage(path string, pgId common.Pgid) (bool, error) { 20 | return ClearPageElements(path, pgId, 0, -1, false) 21 | } 22 | 23 | // ClearPageElements supports clearing elements in both branch and leaf 24 | // pages. Note if the ${abandonFreelist} is true, the freelist may be cleaned 25 | // in the meta pages in the following two cases, and bbolt needs to scan the 26 | // db to reconstruct free list. It may cause some delay on next startup, 27 | // depending on the db size. 28 | // 1. Any branch elements are cleared; 29 | // 2. An object saved in overflow pages is cleared; 30 | // 31 | // Usually ${abandonFreelist} defaults to false, it means it will not clear the 32 | // freelist in meta pages automatically. Users will receive a warning message 33 | // to remind them to explicitly execute `bbolt surgery abandom-freelist` 34 | // afterwards; the first return parameter will be true in such case. But if 35 | // the freelist isn't synced at all, no warning message will be displayed. 36 | func ClearPageElements(path string, pgId common.Pgid, start, end int, abandonFreelist bool) (bool, error) { 37 | // Read the page 38 | p, buf, err := guts_cli.ReadPage(path, uint64(pgId)) 39 | if err != nil { 40 | return false, fmt.Errorf("ReadPage failed: %w", err) 41 | } 42 | 43 | if !p.IsLeafPage() && !p.IsBranchPage() { 44 | return false, fmt.Errorf("can't clear elements in %q page", p.Typ()) 45 | } 46 | 47 | elementCnt := int(p.Count()) 48 | 49 | if elementCnt == 0 { 50 | return false, nil 51 | } 52 | 53 | if start < 0 || start >= elementCnt { 54 | return false, fmt.Errorf("the start index (%d) is out of range [0, %d)", start, elementCnt) 55 | } 56 | 57 | if (end < 0 || end > elementCnt) && end != -1 { 58 | return false, fmt.Errorf("the end index (%d) is out of range [0, %d]", end, elementCnt) 59 | } 60 | 61 | if start > end && end != -1 { 62 | return false, fmt.Errorf("the start index (%d) is bigger than the end index (%d)", start, end) 63 | } 64 | 65 | if start == end { 66 | return false, fmt.Errorf("invalid: the start index (%d) is equal to the end index (%d)", start, end) 67 | } 68 | 69 | preOverflow := p.Overflow() 70 | 71 | var ( 72 | dataWritten uint32 73 | ) 74 | if end == int(p.Count()) || end == -1 { 75 | inodes := common.ReadInodeFromPage(p) 76 | inodes = inodes[:start] 77 | 78 | p.SetCount(uint16(start)) 79 | // no need to write inode & data again, we just need to get 80 | // the data size which will be kept. 81 | dataWritten = common.UsedSpaceInPage(inodes, p) 82 | } else { 83 | inodes := common.ReadInodeFromPage(p) 84 | inodes = append(inodes[:start], inodes[end:]...) 85 | 86 | p.SetCount(uint16(len(inodes))) 87 | dataWritten = common.WriteInodeToPage(inodes, p) 88 | } 89 | 90 | pageSize, _, err := guts_cli.ReadPageAndHWMSize(path) 91 | if err != nil { 92 | return false, fmt.Errorf("ReadPageAndHWMSize failed: %w", err) 93 | } 94 | if dataWritten%uint32(pageSize) == 0 { 95 | p.SetOverflow(dataWritten/uint32(pageSize) - 1) 96 | } else { 97 | p.SetOverflow(dataWritten / uint32(pageSize)) 98 | } 99 | 100 | datasz := pageSize * (uint64(p.Overflow()) + 1) 101 | if err := guts_cli.WritePage(path, buf[0:datasz]); err != nil { 102 | return false, fmt.Errorf("WritePage failed: %w", err) 103 | } 104 | 105 | if preOverflow != p.Overflow() || p.IsBranchPage() { 106 | if abandonFreelist { 107 | return false, ClearFreelist(path) 108 | } 109 | return true, nil 110 | } 111 | 112 | return false, nil 113 | } 114 | 115 | func ClearFreelist(path string) error { 116 | if err := clearFreelistInMetaPage(path, 0); err != nil { 117 | return fmt.Errorf("clearFreelist on meta page 0 failed: %w", err) 118 | } 119 | if err := clearFreelistInMetaPage(path, 1); err != nil { 120 | return fmt.Errorf("clearFreelist on meta page 1 failed: %w", err) 121 | } 122 | return nil 123 | } 124 | 125 | func clearFreelistInMetaPage(path string, pageId uint64) error { 126 | _, buf, err := guts_cli.ReadPage(path, pageId) 127 | if err != nil { 128 | return fmt.Errorf("ReadPage %d failed: %w", pageId, err) 129 | } 130 | 131 | meta := common.LoadPageMeta(buf) 132 | meta.SetFreelist(common.PgidNoFreelist) 133 | meta.SetChecksum(meta.Sum64()) 134 | 135 | if err := guts_cli.WritePage(path, buf); err != nil { 136 | return fmt.Errorf("WritePage %d failed: %w", pageId, err) 137 | } 138 | 139 | return nil 140 | } 141 | 142 | // RevertMetaPage replaces the newer metadata page with the older. 143 | // It usually means that one transaction is being lost. But frequently 144 | // data corruption happens on the last transaction pages and the 145 | // previous state is consistent. 146 | func RevertMetaPage(path string) error { 147 | _, activeMetaPage, err := guts_cli.GetRootPage(path) 148 | if err != nil { 149 | return err 150 | } 151 | if activeMetaPage == 0 { 152 | return CopyPage(path, 1, 0) 153 | } else { 154 | return CopyPage(path, 0, 1) 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /internal/surgeon/surgeon_test.go: -------------------------------------------------------------------------------- 1 | package surgeon_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | 9 | bolt "go.etcd.io/bbolt" 10 | "go.etcd.io/bbolt/internal/btesting" 11 | "go.etcd.io/bbolt/internal/surgeon" 12 | ) 13 | 14 | func TestRevertMetaPage(t *testing.T) { 15 | db := btesting.MustCreateDB(t) 16 | assert.NoError(t, 17 | db.Fill([]byte("data"), 1, 500, 18 | func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) }, 19 | func(tx int, k int) []byte { return make([]byte, 100) }, 20 | )) 21 | assert.NoError(t, 22 | db.Update( 23 | func(tx *bolt.Tx) error { 24 | b := tx.Bucket([]byte("data")) 25 | assert.NoError(t, b.Put([]byte("0123"), []byte("new Value for 123"))) 26 | assert.NoError(t, b.Put([]byte("1234b"), []byte("additional object"))) 27 | assert.NoError(t, b.Delete([]byte("0246"))) 28 | return nil 29 | })) 30 | 31 | assert.NoError(t, 32 | db.View( 33 | func(tx *bolt.Tx) error { 34 | b := tx.Bucket([]byte("data")) 35 | assert.Equal(t, []byte("new Value for 123"), b.Get([]byte("0123"))) 36 | assert.Equal(t, []byte("additional object"), b.Get([]byte("1234b"))) 37 | assert.Nil(t, b.Get([]byte("0246"))) 38 | return nil 39 | })) 40 | 41 | db.Close() 42 | 43 | // This causes the whole tree to be linked to the previous state 44 | assert.NoError(t, surgeon.RevertMetaPage(db.Path())) 45 | 46 | db.MustReopen() 47 | db.MustCheck() 48 | assert.NoError(t, 49 | db.View( 50 | func(tx *bolt.Tx) error { 51 | b := tx.Bucket([]byte("data")) 52 | assert.Equal(t, make([]byte, 100), b.Get([]byte("0123"))) 53 | assert.Nil(t, b.Get([]byte("1234b"))) 54 | assert.Equal(t, make([]byte, 100), b.Get([]byte("0246"))) 55 | return nil 56 | })) 57 | } 58 | -------------------------------------------------------------------------------- /internal/surgeon/xray.go: -------------------------------------------------------------------------------- 1 | package surgeon 2 | 3 | // Library contains raw access to bbolt files for sake of testing or fixing of corrupted files. 4 | // 5 | // The library must not be used bbolt btree - just by CLI or tests. 6 | // It's not optimized for performance. 7 | 8 | import ( 9 | "bytes" 10 | "fmt" 11 | 12 | "go.etcd.io/bbolt/internal/common" 13 | "go.etcd.io/bbolt/internal/guts_cli" 14 | ) 15 | 16 | type XRay struct { 17 | path string 18 | } 19 | 20 | func NewXRay(path string) XRay { 21 | return XRay{path} 22 | } 23 | 24 | func (n XRay) traverse(stack []common.Pgid, callback func(page *common.Page, stack []common.Pgid) error) error { 25 | p, data, err := guts_cli.ReadPage(n.path, uint64(stack[len(stack)-1])) 26 | if err != nil { 27 | return fmt.Errorf("failed reading page (stack %v): %w", stack, err) 28 | } 29 | err = callback(p, stack) 30 | if err != nil { 31 | return fmt.Errorf("failed callback for page (stack %v): %w", stack, err) 32 | } 33 | switch p.Typ() { 34 | case "meta": 35 | { 36 | m := common.LoadPageMeta(data) 37 | r := m.RootBucket().RootPage() 38 | return n.traverse(append(stack, r), callback) 39 | } 40 | case "branch": 41 | { 42 | for i := uint16(0); i < p.Count(); i++ { 43 | bpe := p.BranchPageElement(i) 44 | if err := n.traverse(append(stack, bpe.Pgid()), callback); err != nil { 45 | return err 46 | } 47 | } 48 | } 49 | case "leaf": 50 | for i := uint16(0); i < p.Count(); i++ { 51 | lpe := p.LeafPageElement(i) 52 | if lpe.IsBucketEntry() { 53 | pgid := lpe.Bucket().RootPage() 54 | if pgid > 0 { 55 | if err := n.traverse(append(stack, pgid), callback); err != nil { 56 | return err 57 | } 58 | } else { 59 | inlinePage := lpe.Bucket().InlinePage(lpe.Value()) 60 | if err := callback(inlinePage, stack); err != nil { 61 | return fmt.Errorf("failed callback for inline page (stack %v): %w", stack, err) 62 | } 63 | } 64 | } 65 | } 66 | case "freelist": 67 | return nil 68 | // Free does not have children. 69 | } 70 | return nil 71 | } 72 | 73 | // FindPathsToKey finds all paths from root to the page that contains the given key. 74 | // As it traverses multiple buckets, so in theory there might be multiple keys with the given name. 75 | // Note: For simplicity it's currently implemented as traversing of the whole reachable tree. 76 | // If key is a bucket name, a page-path referencing the key will be returned as well. 77 | func (n XRay) FindPathsToKey(key []byte) ([][]common.Pgid, error) { 78 | var found [][]common.Pgid 79 | 80 | rootPage, _, err := guts_cli.GetRootPage(n.path) 81 | if err != nil { 82 | return nil, err 83 | } 84 | err = n.traverse([]common.Pgid{rootPage}, 85 | func(page *common.Page, stack []common.Pgid) error { 86 | if page.Typ() == "leaf" { 87 | for i := uint16(0); i < page.Count(); i++ { 88 | if bytes.Equal(page.LeafPageElement(i).Key(), key) { 89 | var copyPath []common.Pgid 90 | copyPath = append(copyPath, stack...) 91 | found = append(found, copyPath) 92 | } 93 | } 94 | } 95 | return nil 96 | }) 97 | if err != nil { 98 | return nil, err 99 | } else { 100 | return found, nil 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /internal/surgeon/xray_test.go: -------------------------------------------------------------------------------- 1 | package surgeon_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | 10 | "go.etcd.io/bbolt" 11 | "go.etcd.io/bbolt/internal/btesting" 12 | "go.etcd.io/bbolt/internal/guts_cli" 13 | "go.etcd.io/bbolt/internal/surgeon" 14 | ) 15 | 16 | func TestFindPathsToKey(t *testing.T) { 17 | db := btesting.MustCreateDB(t) 18 | assert.NoError(t, 19 | db.Fill([]byte("data"), 1, 500, 20 | func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) }, 21 | func(tx int, k int) []byte { return make([]byte, 100) }, 22 | )) 23 | assert.NoError(t, db.Close()) 24 | 25 | navigator := surgeon.NewXRay(db.Path()) 26 | path1, err := navigator.FindPathsToKey([]byte("0451")) 27 | assert.NoError(t, err) 28 | assert.NotEmpty(t, path1) 29 | 30 | page := path1[0][len(path1[0])-1] 31 | p, _, err := guts_cli.ReadPage(db.Path(), uint64(page)) 32 | assert.NoError(t, err) 33 | assert.GreaterOrEqual(t, []byte("0451"), p.LeafPageElement(0).Key()) 34 | assert.LessOrEqual(t, []byte("0451"), p.LeafPageElement(p.Count()-1).Key()) 35 | } 36 | 37 | func TestFindPathsToKey_Bucket(t *testing.T) { 38 | rootBucket := []byte("data") 39 | subBucket := []byte("0451A") 40 | 41 | db := btesting.MustCreateDB(t) 42 | assert.NoError(t, 43 | db.Fill(rootBucket, 1, 500, 44 | func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) }, 45 | func(tx int, k int) []byte { return make([]byte, 100) }, 46 | )) 47 | require.NoError(t, db.Update(func(tx *bbolt.Tx) error { 48 | sb, err := tx.Bucket(rootBucket).CreateBucket(subBucket) 49 | require.NoError(t, err) 50 | require.NoError(t, sb.Put([]byte("foo"), []byte("bar"))) 51 | return nil 52 | })) 53 | 54 | assert.NoError(t, db.Close()) 55 | 56 | navigator := surgeon.NewXRay(db.Path()) 57 | path1, err := navigator.FindPathsToKey(subBucket) 58 | assert.NoError(t, err) 59 | assert.NotEmpty(t, path1) 60 | 61 | page := path1[0][len(path1[0])-1] 62 | p, _, err := guts_cli.ReadPage(db.Path(), uint64(page)) 63 | assert.NoError(t, err) 64 | assert.GreaterOrEqual(t, subBucket, p.LeafPageElement(0).Key()) 65 | assert.LessOrEqual(t, subBucket, p.LeafPageElement(p.Count()-1).Key()) 66 | } 67 | -------------------------------------------------------------------------------- /internal/tests/tx_check_test.go: -------------------------------------------------------------------------------- 1 | package tests_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | 9 | bolt "go.etcd.io/bbolt" 10 | "go.etcd.io/bbolt/internal/btesting" 11 | "go.etcd.io/bbolt/internal/guts_cli" 12 | "go.etcd.io/bbolt/internal/surgeon" 13 | ) 14 | 15 | func TestTx_RecursivelyCheckPages_MisplacedPage(t *testing.T) { 16 | db := btesting.MustCreateDB(t) 17 | db.ForceDisableStrictMode() 18 | require.NoError(t, 19 | db.Fill([]byte("data"), 1, 10000, 20 | func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) }, 21 | func(tx int, k int) []byte { return make([]byte, 100) }, 22 | )) 23 | require.NoError(t, db.Close()) 24 | 25 | xRay := surgeon.NewXRay(db.Path()) 26 | 27 | path1, err := xRay.FindPathsToKey([]byte("0451")) 28 | require.NoError(t, err, "cannot find page that contains key:'0451'") 29 | require.Len(t, path1, 1, "Expected only one page that contains key:'0451'") 30 | 31 | path2, err := xRay.FindPathsToKey([]byte("7563")) 32 | require.NoError(t, err, "cannot find page that contains key:'7563'") 33 | require.Len(t, path2, 1, "Expected only one page that contains key:'7563'") 34 | 35 | srcPage := path1[0][len(path1[0])-1] 36 | targetPage := path2[0][len(path2[0])-1] 37 | require.NoError(t, surgeon.CopyPage(db.Path(), srcPage, targetPage)) 38 | 39 | db.MustReopen() 40 | db.ForceDisableStrictMode() 41 | require.NoError(t, db.Update(func(tx *bolt.Tx) error { 42 | // Collect all the errors. 43 | var errors []error 44 | for err := range tx.Check() { 45 | errors = append(errors, err) 46 | } 47 | require.Len(t, errors, 1) 48 | require.ErrorContains(t, errors[0], fmt.Sprintf("leaf page(%v) needs to be >= the key in the ancestor", targetPage)) 49 | return nil 50 | })) 51 | require.NoError(t, db.Close()) 52 | } 53 | 54 | func TestTx_RecursivelyCheckPages_CorruptedLeaf(t *testing.T) { 55 | db := btesting.MustCreateDB(t) 56 | db.ForceDisableStrictMode() 57 | require.NoError(t, 58 | db.Fill([]byte("data"), 1, 10000, 59 | func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) }, 60 | func(tx int, k int) []byte { return make([]byte, 100) }, 61 | )) 62 | require.NoError(t, db.Close()) 63 | 64 | xray := surgeon.NewXRay(db.Path()) 65 | 66 | path1, err := xray.FindPathsToKey([]byte("0451")) 67 | require.NoError(t, err, "cannot find page that contains key:'0451'") 68 | require.Len(t, path1, 1, "Expected only one page that contains key:'0451'") 69 | 70 | srcPage := path1[0][len(path1[0])-1] 71 | p, pbuf, err := guts_cli.ReadPage(db.Path(), uint64(srcPage)) 72 | require.NoError(t, err) 73 | require.Positive(t, p.Count(), "page must be not empty") 74 | p.LeafPageElement(p.Count() / 2).Key()[0] = 'z' 75 | require.NoError(t, guts_cli.WritePage(db.Path(), pbuf)) 76 | 77 | db.MustReopen() 78 | db.ForceDisableStrictMode() 79 | require.NoError(t, db.Update(func(tx *bolt.Tx) error { 80 | // Collect all the errors. 81 | var errors []error 82 | for err := range tx.Check() { 83 | errors = append(errors, err) 84 | } 85 | require.Len(t, errors, 2) 86 | require.ErrorContains(t, errors[0], fmt.Sprintf("leaf page(%v) needs to be < than key of the next element in ancestor", srcPage)) 87 | require.ErrorContains(t, errors[1], fmt.Sprintf("leaf page(%v) needs to be > (found <) than previous element", srcPage)) 88 | return nil 89 | })) 90 | require.NoError(t, db.Close()) 91 | } 92 | -------------------------------------------------------------------------------- /logger.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | // See https://github.com/etcd-io/raft/blob/main/logger.go 4 | import ( 5 | "fmt" 6 | "io" 7 | "log" 8 | "os" 9 | ) 10 | 11 | type Logger interface { 12 | Debug(v ...interface{}) 13 | Debugf(format string, v ...interface{}) 14 | 15 | Error(v ...interface{}) 16 | Errorf(format string, v ...interface{}) 17 | 18 | Info(v ...interface{}) 19 | Infof(format string, v ...interface{}) 20 | 21 | Warning(v ...interface{}) 22 | Warningf(format string, v ...interface{}) 23 | 24 | Fatal(v ...interface{}) 25 | Fatalf(format string, v ...interface{}) 26 | 27 | Panic(v ...interface{}) 28 | Panicf(format string, v ...interface{}) 29 | } 30 | 31 | func getDiscardLogger() Logger { 32 | return discardLogger 33 | } 34 | 35 | var ( 36 | discardLogger = &DefaultLogger{Logger: log.New(io.Discard, "", 0)} 37 | ) 38 | 39 | const ( 40 | calldepth = 2 41 | ) 42 | 43 | // DefaultLogger is a default implementation of the Logger interface. 44 | type DefaultLogger struct { 45 | *log.Logger 46 | debug bool 47 | } 48 | 49 | func (l *DefaultLogger) EnableTimestamps() { 50 | l.SetFlags(l.Flags() | log.Ldate | log.Ltime) 51 | } 52 | 53 | func (l *DefaultLogger) EnableDebug() { 54 | l.debug = true 55 | } 56 | 57 | func (l *DefaultLogger) Debug(v ...interface{}) { 58 | if l.debug { 59 | _ = l.Output(calldepth, header("DEBUG", fmt.Sprint(v...))) 60 | } 61 | } 62 | 63 | func (l *DefaultLogger) Debugf(format string, v ...interface{}) { 64 | if l.debug { 65 | _ = l.Output(calldepth, header("DEBUG", fmt.Sprintf(format, v...))) 66 | } 67 | } 68 | 69 | func (l *DefaultLogger) Info(v ...interface{}) { 70 | _ = l.Output(calldepth, header("INFO", fmt.Sprint(v...))) 71 | } 72 | 73 | func (l *DefaultLogger) Infof(format string, v ...interface{}) { 74 | _ = l.Output(calldepth, header("INFO", fmt.Sprintf(format, v...))) 75 | } 76 | 77 | func (l *DefaultLogger) Error(v ...interface{}) { 78 | _ = l.Output(calldepth, header("ERROR", fmt.Sprint(v...))) 79 | } 80 | 81 | func (l *DefaultLogger) Errorf(format string, v ...interface{}) { 82 | _ = l.Output(calldepth, header("ERROR", fmt.Sprintf(format, v...))) 83 | } 84 | 85 | func (l *DefaultLogger) Warning(v ...interface{}) { 86 | _ = l.Output(calldepth, header("WARN", fmt.Sprint(v...))) 87 | } 88 | 89 | func (l *DefaultLogger) Warningf(format string, v ...interface{}) { 90 | _ = l.Output(calldepth, header("WARN", fmt.Sprintf(format, v...))) 91 | } 92 | 93 | func (l *DefaultLogger) Fatal(v ...interface{}) { 94 | _ = l.Output(calldepth, header("FATAL", fmt.Sprint(v...))) 95 | os.Exit(1) 96 | } 97 | 98 | func (l *DefaultLogger) Fatalf(format string, v ...interface{}) { 99 | _ = l.Output(calldepth, header("FATAL", fmt.Sprintf(format, v...))) 100 | os.Exit(1) 101 | } 102 | 103 | func (l *DefaultLogger) Panic(v ...interface{}) { 104 | l.Logger.Panic(v...) 105 | } 106 | 107 | func (l *DefaultLogger) Panicf(format string, v ...interface{}) { 108 | l.Logger.Panicf(format, v...) 109 | } 110 | 111 | func header(lvl, msg string) string { 112 | return fmt.Sprintf("%s: %s", lvl, msg) 113 | } 114 | -------------------------------------------------------------------------------- /manydbs_test.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | import ( 4 | "crypto/rand" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | ) 10 | 11 | func createDb(t *testing.T) (*DB, func()) { 12 | // First, create a temporary directory to be used for the duration of 13 | // this test. 14 | tempDirName, err := os.MkdirTemp("", "bboltmemtest") 15 | if err != nil { 16 | t.Fatalf("error creating temp dir: %v", err) 17 | } 18 | path := filepath.Join(tempDirName, "testdb.db") 19 | 20 | bdb, err := Open(path, 0600, nil) 21 | if err != nil { 22 | t.Fatalf("error creating bbolt db: %v", err) 23 | } 24 | 25 | cleanup := func() { 26 | bdb.Close() 27 | os.RemoveAll(tempDirName) 28 | } 29 | 30 | return bdb, cleanup 31 | } 32 | 33 | func createAndPutKeys(t *testing.T) { 34 | t.Parallel() 35 | 36 | db, cleanup := createDb(t) 37 | defer cleanup() 38 | 39 | bucketName := []byte("bucket") 40 | 41 | for i := 0; i < 100; i++ { 42 | err := db.Update(func(tx *Tx) error { 43 | nodes, err := tx.CreateBucketIfNotExists(bucketName) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | var key [16]byte 49 | _, rerr := rand.Read(key[:]) 50 | if rerr != nil { 51 | return rerr 52 | } 53 | if err := nodes.Put(key[:], nil); err != nil { 54 | return err 55 | } 56 | 57 | return nil 58 | }) 59 | if err != nil { 60 | t.Fatal(err) 61 | } 62 | } 63 | } 64 | 65 | func TestManyDBs(t *testing.T) { 66 | if testing.Short() { 67 | t.Skip("skipping test in short mode") 68 | } 69 | 70 | for i := 0; i < 100; i++ { 71 | t.Run(fmt.Sprintf("%d", i), createAndPutKeys) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /mlock_unix.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | package bbolt 4 | 5 | import "golang.org/x/sys/unix" 6 | 7 | // mlock locks memory of db file 8 | func mlock(db *DB, fileSize int) error { 9 | sizeToLock := fileSize 10 | if sizeToLock > db.datasz { 11 | // Can't lock more than mmaped slice 12 | sizeToLock = db.datasz 13 | } 14 | if err := unix.Mlock(db.dataref[:sizeToLock]); err != nil { 15 | return err 16 | } 17 | return nil 18 | } 19 | 20 | // munlock unlocks memory of db file 21 | func munlock(db *DB, fileSize int) error { 22 | if db.dataref == nil { 23 | return nil 24 | } 25 | 26 | sizeToUnlock := fileSize 27 | if sizeToUnlock > db.datasz { 28 | // Can't unlock more than mmaped slice 29 | sizeToUnlock = db.datasz 30 | } 31 | 32 | if err := unix.Munlock(db.dataref[:sizeToUnlock]); err != nil { 33 | return err 34 | } 35 | return nil 36 | } 37 | -------------------------------------------------------------------------------- /mlock_windows.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | // mlock locks memory of db file 4 | func mlock(_ *DB, _ int) error { 5 | panic("mlock is supported only on UNIX systems") 6 | } 7 | 8 | // munlock unlocks memory of db file 9 | func munlock(_ *DB, _ int) error { 10 | panic("munlock is supported only on UNIX systems") 11 | } 12 | -------------------------------------------------------------------------------- /node_test.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | import ( 4 | "testing" 5 | "unsafe" 6 | 7 | "go.etcd.io/bbolt/internal/common" 8 | ) 9 | 10 | // Ensure that a node can insert a key/value. 11 | func TestNode_put(t *testing.T) { 12 | m := &common.Meta{} 13 | m.SetPgid(1) 14 | n := &node{inodes: make(common.Inodes, 0), bucket: &Bucket{tx: &Tx{meta: m}}} 15 | n.put([]byte("baz"), []byte("baz"), []byte("2"), 0, 0) 16 | n.put([]byte("foo"), []byte("foo"), []byte("0"), 0, 0) 17 | n.put([]byte("bar"), []byte("bar"), []byte("1"), 0, 0) 18 | n.put([]byte("foo"), []byte("foo"), []byte("3"), 0, common.LeafPageFlag) 19 | 20 | if len(n.inodes) != 3 { 21 | t.Fatalf("exp=3; got=%d", len(n.inodes)) 22 | } 23 | if k, v := n.inodes[0].Key(), n.inodes[0].Value(); string(k) != "bar" || string(v) != "1" { 24 | t.Fatalf("exp=; got=<%s,%s>", k, v) 25 | } 26 | if k, v := n.inodes[1].Key(), n.inodes[1].Value(); string(k) != "baz" || string(v) != "2" { 27 | t.Fatalf("exp=; got=<%s,%s>", k, v) 28 | } 29 | if k, v := n.inodes[2].Key(), n.inodes[2].Value(); string(k) != "foo" || string(v) != "3" { 30 | t.Fatalf("exp=; got=<%s,%s>", k, v) 31 | } 32 | if n.inodes[2].Flags() != uint32(common.LeafPageFlag) { 33 | t.Fatalf("not a leaf: %d", n.inodes[2].Flags()) 34 | } 35 | } 36 | 37 | // Ensure that a node can deserialize from a leaf page. 38 | func TestNode_read_LeafPage(t *testing.T) { 39 | // Create a page. 40 | var buf [4096]byte 41 | page := (*common.Page)(unsafe.Pointer(&buf[0])) 42 | page.SetFlags(common.LeafPageFlag) 43 | page.SetCount(2) 44 | 45 | // Insert 2 elements at the beginning. sizeof(leafPageElement) == 16 46 | nodes := page.LeafPageElements() 47 | //nodes := (*[3]leafPageElement)(unsafe.Pointer(uintptr(unsafe.Pointer(page)) + unsafe.Sizeof(*page))) 48 | nodes[0] = *common.NewLeafPageElement(0, 32, 3, 4) // pos = sizeof(leafPageElement) * 2 49 | nodes[1] = *common.NewLeafPageElement(0, 23, 10, 3) // pos = sizeof(leafPageElement) + 3 + 4 50 | 51 | // Write data for the nodes at the end. 52 | const s = "barfoozhelloworldbye" 53 | data := common.UnsafeByteSlice(unsafe.Pointer(uintptr(unsafe.Pointer(page))+unsafe.Sizeof(*page)+common.LeafPageElementSize*2), 0, 0, len(s)) 54 | copy(data, s) 55 | 56 | // Deserialize page into a leaf. 57 | n := &node{} 58 | n.read(page) 59 | 60 | // Check that there are two inodes with correct data. 61 | if !n.isLeaf { 62 | t.Fatal("expected leaf") 63 | } 64 | if len(n.inodes) != 2 { 65 | t.Fatalf("exp=2; got=%d", len(n.inodes)) 66 | } 67 | if k, v := n.inodes[0].Key(), n.inodes[0].Value(); string(k) != "bar" || string(v) != "fooz" { 68 | t.Fatalf("exp=; got=<%s,%s>", k, v) 69 | } 70 | if k, v := n.inodes[1].Key(), n.inodes[1].Value(); string(k) != "helloworld" || string(v) != "bye" { 71 | t.Fatalf("exp=; got=<%s,%s>", k, v) 72 | } 73 | } 74 | 75 | // Ensure that a node can serialize into a leaf page. 76 | func TestNode_write_LeafPage(t *testing.T) { 77 | // Create a node. 78 | m := &common.Meta{} 79 | m.SetPgid(1) 80 | n := &node{isLeaf: true, inodes: make(common.Inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: m}}} 81 | n.put([]byte("susy"), []byte("susy"), []byte("que"), 0, 0) 82 | n.put([]byte("ricki"), []byte("ricki"), []byte("lake"), 0, 0) 83 | n.put([]byte("john"), []byte("john"), []byte("johnson"), 0, 0) 84 | 85 | // Write it to a page. 86 | var buf [4096]byte 87 | p := (*common.Page)(unsafe.Pointer(&buf[0])) 88 | n.write(p) 89 | 90 | // Read the page back in. 91 | n2 := &node{} 92 | n2.read(p) 93 | 94 | // Check that the two pages are the same. 95 | if len(n2.inodes) != 3 { 96 | t.Fatalf("exp=3; got=%d", len(n2.inodes)) 97 | } 98 | if k, v := n2.inodes[0].Key(), n2.inodes[0].Value(); string(k) != "john" || string(v) != "johnson" { 99 | t.Fatalf("exp=; got=<%s,%s>", k, v) 100 | } 101 | if k, v := n2.inodes[1].Key(), n2.inodes[1].Value(); string(k) != "ricki" || string(v) != "lake" { 102 | t.Fatalf("exp=; got=<%s,%s>", k, v) 103 | } 104 | if k, v := n2.inodes[2].Key(), n2.inodes[2].Value(); string(k) != "susy" || string(v) != "que" { 105 | t.Fatalf("exp=; got=<%s,%s>", k, v) 106 | } 107 | } 108 | 109 | // Ensure that a node can split into appropriate subgroups. 110 | func TestNode_split(t *testing.T) { 111 | // Create a node. 112 | m := &common.Meta{} 113 | m.SetPgid(1) 114 | n := &node{inodes: make(common.Inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: m}}} 115 | n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0) 116 | n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0) 117 | n.put([]byte("00000003"), []byte("00000003"), []byte("0123456701234567"), 0, 0) 118 | n.put([]byte("00000004"), []byte("00000004"), []byte("0123456701234567"), 0, 0) 119 | n.put([]byte("00000005"), []byte("00000005"), []byte("0123456701234567"), 0, 0) 120 | 121 | // Split between 2 & 3. 122 | n.split(100) 123 | 124 | var parent = n.parent 125 | if len(parent.children) != 2 { 126 | t.Fatalf("exp=2; got=%d", len(parent.children)) 127 | } 128 | if len(parent.children[0].inodes) != 2 { 129 | t.Fatalf("exp=2; got=%d", len(parent.children[0].inodes)) 130 | } 131 | if len(parent.children[1].inodes) != 3 { 132 | t.Fatalf("exp=3; got=%d", len(parent.children[1].inodes)) 133 | } 134 | } 135 | 136 | // Ensure that a page with the minimum number of inodes just returns a single node. 137 | func TestNode_split_MinKeys(t *testing.T) { 138 | // Create a node. 139 | m := &common.Meta{} 140 | m.SetPgid(1) 141 | n := &node{inodes: make(common.Inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: m}}} 142 | n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0) 143 | n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0) 144 | 145 | // Split. 146 | n.split(20) 147 | if n.parent != nil { 148 | t.Fatalf("expected nil parent") 149 | } 150 | } 151 | 152 | // Ensure that a node that has keys that all fit on a page just returns one leaf. 153 | func TestNode_split_SinglePage(t *testing.T) { 154 | // Create a node. 155 | m := &common.Meta{} 156 | m.SetPgid(1) 157 | n := &node{inodes: make(common.Inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: m}}} 158 | n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0) 159 | n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0) 160 | n.put([]byte("00000003"), []byte("00000003"), []byte("0123456701234567"), 0, 0) 161 | n.put([]byte("00000004"), []byte("00000004"), []byte("0123456701234567"), 0, 0) 162 | n.put([]byte("00000005"), []byte("00000005"), []byte("0123456701234567"), 0, 0) 163 | 164 | // Split. 165 | n.split(4096) 166 | if n.parent != nil { 167 | t.Fatalf("expected nil parent") 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /quick_test.go: -------------------------------------------------------------------------------- 1 | package bbolt_test 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "fmt" 7 | "math/rand" 8 | "os" 9 | "reflect" 10 | "testing" 11 | "testing/quick" 12 | "time" 13 | ) 14 | 15 | // testing/quick defaults to 5 iterations and a random seed. 16 | // You can override these settings from the command line: 17 | // 18 | // -quick.count The number of iterations to perform. 19 | // -quick.seed The seed to use for randomizing. 20 | // -quick.maxitems The maximum number of items to insert into a DB. 21 | // -quick.maxksize The maximum size of a key. 22 | // -quick.maxvsize The maximum size of a value. 23 | // 24 | 25 | var qcount, qseed, qmaxitems, qmaxksize, qmaxvsize int 26 | 27 | func TestMain(m *testing.M) { 28 | flag.IntVar(&qcount, "quick.count", 5, "") 29 | flag.IntVar(&qseed, "quick.seed", int(time.Now().UnixNano())%100000, "") 30 | flag.IntVar(&qmaxitems, "quick.maxitems", 1000, "") 31 | flag.IntVar(&qmaxksize, "quick.maxksize", 1024, "") 32 | flag.IntVar(&qmaxvsize, "quick.maxvsize", 1024, "") 33 | flag.Parse() 34 | fmt.Fprintln(os.Stderr, "seed:", qseed) 35 | fmt.Fprintf(os.Stderr, "quick settings: count=%v, items=%v, ksize=%v, vsize=%v\n", qcount, qmaxitems, qmaxksize, qmaxvsize) 36 | 37 | os.Exit(m.Run()) 38 | } 39 | 40 | func qconfig() *quick.Config { 41 | return &quick.Config{ 42 | MaxCount: qcount, 43 | Rand: rand.New(rand.NewSource(int64(qseed))), 44 | } 45 | } 46 | 47 | type testdata []testdataitem 48 | 49 | func (t testdata) Len() int { return len(t) } 50 | func (t testdata) Swap(i, j int) { t[i], t[j] = t[j], t[i] } 51 | func (t testdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) == -1 } 52 | 53 | func (t testdata) Generate(rand *rand.Rand, size int) reflect.Value { 54 | n := rand.Intn(qmaxitems-1) + 1 55 | items := make(testdata, n) 56 | used := make(map[string]bool) 57 | for i := 0; i < n; i++ { 58 | item := &items[i] 59 | // Ensure that keys are unique by looping until we find one that we have not already used. 60 | for { 61 | item.Key = randByteSlice(rand, 1, qmaxksize) 62 | if !used[string(item.Key)] { 63 | used[string(item.Key)] = true 64 | break 65 | } 66 | } 67 | item.Value = randByteSlice(rand, 0, qmaxvsize) 68 | } 69 | return reflect.ValueOf(items) 70 | } 71 | 72 | type revtestdata []testdataitem 73 | 74 | func (t revtestdata) Len() int { return len(t) } 75 | func (t revtestdata) Swap(i, j int) { t[i], t[j] = t[j], t[i] } 76 | func (t revtestdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) == 1 } 77 | 78 | type testdataitem struct { 79 | Key []byte 80 | Value []byte 81 | } 82 | 83 | func randByteSlice(rand *rand.Rand, minSize, maxSize int) []byte { 84 | n := rand.Intn(maxSize-minSize) + minSize 85 | b := make([]byte, n) 86 | for i := 0; i < n; i++ { 87 | b[i] = byte(rand.Intn(255)) 88 | } 89 | return b 90 | } 91 | -------------------------------------------------------------------------------- /scripts/compare_benchmarks.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # https://github.com/kubernetes/kube-state-metrics/blob/main/tests/compare_benchmarks.sh (originally written by mxinden) 3 | 4 | # exit immediately when a command fails 5 | set -e 6 | # only exit with zero if all commands of the pipeline exit successfully 7 | set -o pipefail 8 | # error on unset variables 9 | set -u 10 | 11 | [[ "$#" -eq 1 ]] || echo "One argument required, $# provided." 12 | 13 | REF_CURRENT="$(git rev-parse --abbrev-ref HEAD)" 14 | BASE_TO_COMPARE=$1 15 | 16 | RESULT_CURRENT="$(mktemp)-${REF_CURRENT}" 17 | RESULT_TO_COMPARE="$(mktemp)-${BASE_TO_COMPARE}" 18 | 19 | BENCH_COUNT=${BENCH_COUNT:-10} 20 | BENCHSTAT_CONFIDENCE_LEVEL=${BENCHSTAT_CONFIDENCE_LEVEL:-0.9} 21 | BENCHSTAT_FORMAT=${BENCHSTAT_FORMAT:-"text"} 22 | BENCH_PARAMETERS=${BENCH_PARAMETERS:-"-count 2000000 -batch-size 10000"} 23 | 24 | if [[ "${BENCHSTAT_FORMAT}" == "csv" ]] && [[ -z "${BENCHSTAT_OUTPUT_FILE}" ]]; then 25 | echo "BENCHSTAT_FORMAT is set to csv, but BENCHSTAT_OUTPUT_FILE is not set." 26 | exit 1 27 | fi 28 | 29 | function bench() { 30 | local output_file 31 | output_file="$1" 32 | make build 33 | 34 | for _ in $(seq "$BENCH_COUNT"); do 35 | echo ./bin/bbolt bench -gobench-output -profile-mode n ${BENCH_PARAMETERS} 36 | # shellcheck disable=SC2086 37 | ./bin/bbolt bench -gobench-output -profile-mode n ${BENCH_PARAMETERS} >> "${output_file}" 38 | done 39 | } 40 | 41 | function main() { 42 | echo "### Benchmarking PR ${REF_CURRENT}" 43 | bench "${RESULT_CURRENT}" 44 | echo "" 45 | echo "### Done benchmarking ${REF_CURRENT}" 46 | 47 | echo "### Benchmarking base ${BASE_TO_COMPARE}" 48 | git checkout "${BASE_TO_COMPARE}" 49 | bench "${RESULT_TO_COMPARE}" 50 | echo "" 51 | echo "### Done benchmarking ${BASE_TO_COMPARE}" 52 | 53 | git checkout - 54 | 55 | echo "" 56 | echo "### Result" 57 | echo "BASE=${BASE_TO_COMPARE} HEAD=${REF_CURRENT}" 58 | 59 | if [[ "${BENCHSTAT_FORMAT}" == "csv" ]]; then 60 | benchstat -format=csv -confidence="${BENCHSTAT_CONFIDENCE_LEVEL}" BASE="${RESULT_TO_COMPARE}" HEAD="${RESULT_CURRENT}" 2>/dev/null 1>"${BENCHSTAT_OUTPUT_FILE}" 61 | else 62 | if [[ -z "${BENCHSTAT_OUTPUT_FILE}" ]]; then 63 | benchstat -confidence="${BENCHSTAT_CONFIDENCE_LEVEL}" BASE="${RESULT_TO_COMPARE}" HEAD="${RESULT_CURRENT}" 64 | else 65 | benchstat -confidence="${BENCHSTAT_CONFIDENCE_LEVEL}" BASE="${RESULT_TO_COMPARE}" HEAD="${RESULT_CURRENT}" 1>"${BENCHSTAT_OUTPUT_FILE}" 66 | fi 67 | fi 68 | } 69 | 70 | main 71 | -------------------------------------------------------------------------------- /scripts/fix.sh: -------------------------------------------------------------------------------- 1 | GO_CMD="go" 2 | 3 | # TODO(ptabor): Expand to cover different architectures (GOOS GOARCH), or just list go files. 4 | 5 | GOFILES=$(${GO_CMD} list --f "{{with \$d:=.}}{{range .GoFiles}}{{\$d.Dir}}/{{.}}{{\"\n\"}}{{end}}{{end}}" ./...) 6 | TESTGOFILES=$(${GO_CMD} list --f "{{with \$d:=.}}{{range .TestGoFiles}}{{\$d.Dir}}/{{.}}{{\"\n\"}}{{end}}{{end}}" ./...) 7 | XTESTGOFILES=$(${GO_CMD} list --f "{{with \$d:=.}}{{range .XTestGoFiles}}{{\$d.Dir}}/{{.}}{{\"\n\"}}{{end}}{{end}}" ./...) 8 | 9 | 10 | echo "${GOFILES}" "${TESTGOFILES}" "${XTESTGOFILES}"| xargs -n 100 go run golang.org/x/tools/cmd/goimports@latest -w -local go.etcd.io 11 | 12 | go fmt ./... 13 | go mod tidy 14 | -------------------------------------------------------------------------------- /simulation_no_freelist_sync_test.go: -------------------------------------------------------------------------------- 1 | package bbolt_test 2 | 3 | import ( 4 | "testing" 5 | 6 | bolt "go.etcd.io/bbolt" 7 | ) 8 | 9 | func TestSimulateNoFreeListSync_1op_1p(t *testing.T) { 10 | testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 1, 1) 11 | } 12 | func TestSimulateNoFreeListSync_10op_1p(t *testing.T) { 13 | testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 10, 1) 14 | } 15 | func TestSimulateNoFreeListSync_100op_1p(t *testing.T) { 16 | testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 100, 1) 17 | } 18 | func TestSimulateNoFreeListSync_1000op_1p(t *testing.T) { 19 | testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 1000, 1) 20 | } 21 | func TestSimulateNoFreeListSync_10000op_1p(t *testing.T) { 22 | testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 10000, 1) 23 | } 24 | func TestSimulateNoFreeListSync_10op_10p(t *testing.T) { 25 | testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 10, 10) 26 | } 27 | func TestSimulateNoFreeListSync_100op_10p(t *testing.T) { 28 | testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 100, 10) 29 | } 30 | func TestSimulateNoFreeListSync_1000op_10p(t *testing.T) { 31 | testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 1000, 10) 32 | } 33 | func TestSimulateNoFreeListSync_10000op_10p(t *testing.T) { 34 | testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 10000, 10) 35 | } 36 | func TestSimulateNoFreeListSync_100op_100p(t *testing.T) { 37 | testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 100, 100) 38 | } 39 | func TestSimulateNoFreeListSync_1000op_100p(t *testing.T) { 40 | testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 1000, 100) 41 | } 42 | func TestSimulateNoFreeListSync_10000op_100p(t *testing.T) { 43 | testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 10000, 100) 44 | } 45 | func TestSimulateNoFreeListSync_10000op_1000p(t *testing.T) { 46 | testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 10000, 1000) 47 | } 48 | -------------------------------------------------------------------------------- /tests/dmflakey/dmflakey_test.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | 3 | package dmflakey 4 | 5 | import ( 6 | "errors" 7 | "flag" 8 | "fmt" 9 | "os" 10 | "os/exec" 11 | "path/filepath" 12 | "testing" 13 | "time" 14 | 15 | testutils "go.etcd.io/bbolt/tests/utils" 16 | 17 | "github.com/stretchr/testify/assert" 18 | "github.com/stretchr/testify/require" 19 | "golang.org/x/sys/unix" 20 | ) 21 | 22 | func TestMain(m *testing.M) { 23 | flag.Parse() 24 | testutils.RequiresRoot() 25 | os.Exit(m.Run()) 26 | } 27 | 28 | func TestBasic(t *testing.T) { 29 | for _, fsType := range []FSType{FSTypeEXT4, FSTypeXFS} { 30 | t.Run(string(fsType), func(t *testing.T) { 31 | tmpDir := t.TempDir() 32 | 33 | flakey, err := InitFlakey("go-dmflakey", tmpDir, fsType, "") 34 | require.NoError(t, err, "init flakey") 35 | defer func() { 36 | assert.NoError(t, flakey.Teardown()) 37 | }() 38 | 39 | target := filepath.Join(tmpDir, "root") 40 | require.NoError(t, os.MkdirAll(target, 0600)) 41 | 42 | require.NoError(t, mount(target, flakey.DevicePath(), "")) 43 | defer func() { 44 | assert.NoError(t, unmount(target)) 45 | }() 46 | 47 | file := filepath.Join(target, "test") 48 | assert.NoError(t, writeFile(file, []byte("hello, world"), 0600, true)) 49 | 50 | assert.NoError(t, unmount(target)) 51 | 52 | assert.NoError(t, flakey.Teardown()) 53 | }) 54 | } 55 | } 56 | 57 | func TestDropWritesExt4(t *testing.T) { 58 | flakey, root := initFlakey(t, FSTypeEXT4) 59 | 60 | // commit=1000 is to delay commit triggered by writeback thread 61 | require.NoError(t, mount(root, flakey.DevicePath(), "commit=1000")) 62 | 63 | // ensure testdir/f1 is synced. 64 | target := filepath.Join(root, "testdir") 65 | require.NoError(t, os.MkdirAll(target, 0600)) 66 | 67 | f1 := filepath.Join(target, "f1") 68 | assert.NoError(t, writeFile(f1, []byte("hello, world from f1"), 0600, false)) 69 | require.NoError(t, syncfs(f1)) 70 | 71 | // testdir/f2 is created but without fsync 72 | f2 := filepath.Join(target, "f2") 73 | assert.NoError(t, writeFile(f2, []byte("hello, world from f2"), 0600, false)) 74 | 75 | // simulate power failure 76 | assert.NoError(t, flakey.DropWrites()) 77 | assert.NoError(t, unmount(root)) 78 | assert.NoError(t, flakey.AllowWrites()) 79 | require.NoError(t, mount(root, flakey.DevicePath(), "")) 80 | 81 | data, err := os.ReadFile(f1) 82 | assert.NoError(t, err) 83 | assert.Equal(t, "hello, world from f1", string(data)) 84 | 85 | _, err = os.ReadFile(f2) 86 | assert.True(t, errors.Is(err, os.ErrNotExist)) 87 | } 88 | 89 | func TestErrorWritesExt4(t *testing.T) { 90 | flakey, root := initFlakey(t, FSTypeEXT4) 91 | 92 | // commit=1000 is to delay commit triggered by writeback thread 93 | require.NoError(t, mount(root, flakey.DevicePath(), "commit=1000")) 94 | 95 | // inject IO failure on write 96 | assert.NoError(t, flakey.ErrorWrites()) 97 | 98 | f1 := filepath.Join(root, "f1") 99 | err := writeFile(f1, []byte("hello, world during failpoint"), 0600, true) 100 | assert.ErrorContains(t, err, "input/output error") 101 | 102 | // resume 103 | assert.NoError(t, flakey.AllowWrites()) 104 | err = writeFile(f1, []byte("hello, world"), 0600, true) 105 | assert.NoError(t, err) 106 | 107 | assert.NoError(t, unmount(root)) 108 | require.NoError(t, mount(root, flakey.DevicePath(), "")) 109 | 110 | data, err := os.ReadFile(f1) 111 | assert.NoError(t, err) 112 | assert.Equal(t, "hello, world", string(data)) 113 | } 114 | 115 | func initFlakey(t *testing.T, fsType FSType) (_ Flakey, root string) { 116 | tmpDir := t.TempDir() 117 | 118 | target := filepath.Join(tmpDir, "root") 119 | require.NoError(t, os.MkdirAll(target, 0600)) 120 | 121 | flakey, err := InitFlakey("go-dmflakey", tmpDir, fsType, "") 122 | require.NoError(t, err, "init flakey") 123 | 124 | t.Cleanup(func() { 125 | assert.NoError(t, unmount(target)) 126 | assert.NoError(t, flakey.Teardown()) 127 | }) 128 | return flakey, target 129 | } 130 | 131 | func writeFile(name string, data []byte, perm os.FileMode, sync bool) error { 132 | f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) 133 | if err != nil { 134 | return err 135 | } 136 | defer f.Close() 137 | 138 | if _, err = f.Write(data); err != nil { 139 | return err 140 | } 141 | 142 | if sync { 143 | return f.Sync() 144 | } 145 | return nil 146 | } 147 | 148 | func syncfs(file string) error { 149 | f, err := os.Open(file) 150 | if err != nil { 151 | return fmt.Errorf("failed to open %s: %w", file, err) 152 | } 153 | defer f.Close() 154 | 155 | _, _, errno := unix.Syscall(unix.SYS_SYNCFS, uintptr(f.Fd()), 0, 0) 156 | if errno != 0 { 157 | return errno 158 | } 159 | return nil 160 | } 161 | 162 | func mount(target string, devPath string, opt string) error { 163 | args := []string{"-o", opt, devPath, target} 164 | 165 | output, err := exec.Command("mount", args...).CombinedOutput() 166 | if err != nil { 167 | return fmt.Errorf("failed to mount (args: %v) (out: %s): %w", 168 | args, string(output), err) 169 | } 170 | return nil 171 | } 172 | 173 | func unmount(target string) error { 174 | for i := 0; i < 50; i++ { 175 | if err := unix.Unmount(target, 0); err != nil { 176 | switch err { 177 | case unix.EBUSY: 178 | time.Sleep(500 * time.Millisecond) 179 | continue 180 | case unix.EINVAL: 181 | default: 182 | return fmt.Errorf("failed to umount %s: %w", target, err) 183 | } 184 | } 185 | return nil 186 | } 187 | return unix.EBUSY 188 | } 189 | -------------------------------------------------------------------------------- /tests/dmflakey/dmsetup.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | 3 | package dmflakey 4 | 5 | import ( 6 | "fmt" 7 | "os" 8 | "os/exec" 9 | "time" 10 | "unsafe" 11 | 12 | "golang.org/x/sys/unix" 13 | ) 14 | 15 | // newFlakeyDevice creates flakey device. 16 | // 17 | // REF: https://docs.kernel.org/admin-guide/device-mapper/dm-flakey.html 18 | func newFlakeyDevice(flakeyDevice, loopDevice string, interval time.Duration) error { 19 | loopSize, err := getBlkSize(loopDevice) 20 | if err != nil { 21 | return fmt.Errorf("failed to get the size of the loop device %s: %w", loopDevice, err) 22 | } 23 | 24 | // The flakey device will be available in interval.Seconds(). 25 | table := fmt.Sprintf("0 %d flakey %s 0 %d 0", 26 | loopSize, loopDevice, int(interval.Seconds())) 27 | 28 | args := []string{"create", flakeyDevice, "--table", table} 29 | 30 | output, err := exec.Command("dmsetup", args...).CombinedOutput() 31 | if err != nil { 32 | return fmt.Errorf("failed to create flakey device %s with table %s (out: %s): %w", 33 | flakeyDevice, table, string(output), err) 34 | } 35 | return nil 36 | } 37 | 38 | // reloadFlakeyDevice reloads the flakey device with feature table. 39 | func reloadFlakeyDevice(flakeyDevice string, syncFS bool, table string) (retErr error) { 40 | args := []string{"suspend", "--nolockfs", flakeyDevice} 41 | if syncFS { 42 | args[1] = flakeyDevice 43 | args = args[:len(args)-1] 44 | } 45 | 46 | output, err := exec.Command("dmsetup", args...).CombinedOutput() 47 | if err != nil { 48 | return fmt.Errorf("failed to suspend flakey device %s (out: %s): %w", 49 | flakeyDevice, string(output), err) 50 | } 51 | 52 | defer func() { 53 | output, derr := exec.Command("dmsetup", "resume", flakeyDevice).CombinedOutput() 54 | if derr != nil { 55 | derr = fmt.Errorf("failed to resume flakey device %s (out: %s): %w", 56 | flakeyDevice, string(output), derr) 57 | } 58 | 59 | if retErr == nil { 60 | retErr = derr 61 | } 62 | }() 63 | 64 | output, err = exec.Command("dmsetup", "load", flakeyDevice, "--table", table).CombinedOutput() 65 | if err != nil { 66 | return fmt.Errorf("failed to reload flakey device %s with table (%s) (out: %s): %w", 67 | flakeyDevice, table, string(output), err) 68 | } 69 | return nil 70 | } 71 | 72 | // removeFlakeyDevice removes flakey device. 73 | func deleteFlakeyDevice(flakeyDevice string) error { 74 | output, err := exec.Command("dmsetup", "remove", flakeyDevice).CombinedOutput() 75 | if err != nil { 76 | return fmt.Errorf("failed to remove flakey device %s (out: %s): %w", 77 | flakeyDevice, string(output), err) 78 | } 79 | return nil 80 | } 81 | 82 | // getBlkSize64 gets device size in bytes (BLKGETSIZE64). 83 | // 84 | // REF: https://man7.org/linux/man-pages/man8/blockdev.8.html 85 | func getBlkSize64(device string) (int64, error) { 86 | deviceFd, err := os.Open(device) 87 | if err != nil { 88 | return 0, fmt.Errorf("failed to open device %s: %w", device, err) 89 | } 90 | defer deviceFd.Close() 91 | 92 | var size int64 93 | if _, _, err := unix.Syscall(unix.SYS_IOCTL, deviceFd.Fd(), unix.BLKGETSIZE64, uintptr(unsafe.Pointer(&size))); err != 0 { 94 | return 0, fmt.Errorf("failed to get block size: %w", err) 95 | } 96 | return size, nil 97 | } 98 | 99 | // getBlkSize gets size in 512-byte sectors (BLKGETSIZE64 / 512). 100 | // 101 | // REF: https://man7.org/linux/man-pages/man8/blockdev.8.html 102 | func getBlkSize(device string) (int64, error) { 103 | size, err := getBlkSize64(device) 104 | return size / 512, err 105 | } 106 | -------------------------------------------------------------------------------- /tests/dmflakey/loopback.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | 3 | package dmflakey 4 | 5 | import ( 6 | "errors" 7 | "fmt" 8 | "os" 9 | "time" 10 | 11 | "golang.org/x/sys/unix" 12 | ) 13 | 14 | const ( 15 | loopControlDevice = "/dev/loop-control" 16 | loopDevicePattern = "/dev/loop%d" 17 | 18 | maxRetryToAttach = 50 19 | ) 20 | 21 | // attachToLoopDevice associates free loop device with backing file. 22 | // 23 | // There might have race condition. It needs to retry when it runs into EBUSY. 24 | // 25 | // REF: https://man7.org/linux/man-pages/man4/loop.4.html 26 | func attachToLoopDevice(backingFile string) (string, error) { 27 | backingFd, err := os.OpenFile(backingFile, os.O_RDWR, 0) 28 | if err != nil { 29 | return "", fmt.Errorf("failed to open loop device's backing file %s: %w", 30 | backingFile, err) 31 | } 32 | defer backingFd.Close() 33 | 34 | for i := 0; i < maxRetryToAttach; i++ { 35 | loop, err := getFreeLoopDevice() 36 | if err != nil { 37 | return "", fmt.Errorf("failed to get free loop device: %w", err) 38 | } 39 | 40 | err = func() error { 41 | loopFd, err := os.OpenFile(loop, os.O_RDWR, 0) 42 | if err != nil { 43 | return err 44 | } 45 | defer loopFd.Close() 46 | 47 | return unix.IoctlSetInt(int(loopFd.Fd()), 48 | unix.LOOP_SET_FD, int(backingFd.Fd())) 49 | }() 50 | if err != nil { 51 | if errors.Is(err, unix.EBUSY) { 52 | time.Sleep(500 * time.Millisecond) 53 | continue 54 | } 55 | return "", err 56 | } 57 | return loop, nil 58 | } 59 | return "", fmt.Errorf("failed to associate free loop device with backing file %s after retry %v", 60 | backingFile, maxRetryToAttach) 61 | } 62 | 63 | // detachLoopDevice disassociates the loop device from any backing file. 64 | // 65 | // REF: https://man7.org/linux/man-pages/man4/loop.4.html 66 | func detachLoopDevice(loopDevice string) error { 67 | loopFd, err := os.Open(loopDevice) 68 | if err != nil { 69 | return fmt.Errorf("failed to open loop %s: %w", loopDevice, err) 70 | } 71 | defer loopFd.Close() 72 | 73 | return unix.IoctlSetInt(int(loopFd.Fd()), unix.LOOP_CLR_FD, 0) 74 | } 75 | 76 | // getFreeLoopDevice allocates or finds a free loop device for use. 77 | // 78 | // REF: https://man7.org/linux/man-pages/man4/loop.4.html 79 | func getFreeLoopDevice() (string, error) { 80 | control, err := os.OpenFile(loopControlDevice, os.O_RDWR, 0) 81 | if err != nil { 82 | return "", fmt.Errorf("failed to open %s: %w", loopControlDevice, err) 83 | } 84 | 85 | idx, err := unix.IoctlRetInt(int(control.Fd()), unix.LOOP_CTL_GET_FREE) 86 | control.Close() 87 | if err != nil { 88 | return "", fmt.Errorf("failed to get free loop device number: %w", err) 89 | } 90 | return fmt.Sprintf(loopDevicePattern, idx), nil 91 | } 92 | -------------------------------------------------------------------------------- /tests/robustness/main_test.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | 3 | package robustness 4 | 5 | import ( 6 | "flag" 7 | "os" 8 | "testing" 9 | 10 | testutils "go.etcd.io/bbolt/tests/utils" 11 | ) 12 | 13 | func TestMain(m *testing.M) { 14 | flag.Parse() 15 | testutils.RequiresRoot() 16 | os.Exit(m.Run()) 17 | } 18 | -------------------------------------------------------------------------------- /tests/utils/helpers.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | ) 8 | 9 | var enableRoot bool 10 | 11 | func init() { 12 | flag.BoolVar(&enableRoot, "test.root", false, "enable tests that require root") 13 | } 14 | 15 | // RequiresRoot requires root and the test.root flag has been set. 16 | func RequiresRoot() { 17 | if !enableRoot { 18 | fmt.Fprintln(os.Stderr, "Skip tests that require root") 19 | os.Exit(0) 20 | } 21 | 22 | if os.Getuid() != 0 { 23 | fmt.Fprintln(os.Stderr, "This test must be run as root.") 24 | os.Exit(1) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /tx_check_test.go: -------------------------------------------------------------------------------- 1 | package bbolt_test 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | 10 | "go.etcd.io/bbolt" 11 | "go.etcd.io/bbolt/internal/btesting" 12 | "go.etcd.io/bbolt/internal/common" 13 | "go.etcd.io/bbolt/internal/guts_cli" 14 | ) 15 | 16 | func TestTx_Check_CorruptPage(t *testing.T) { 17 | bucketName := []byte("data") 18 | 19 | t.Log("Creating db file.") 20 | db := btesting.MustCreateDBWithOption(t, &bbolt.Options{PageSize: 4096}) 21 | 22 | // Each page can hold roughly 20 key/values pair, so 100 such 23 | // key/value pairs will consume about 5 leaf pages. 24 | err := db.Fill(bucketName, 1, 100, 25 | func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) }, 26 | func(tx int, k int) []byte { return make([]byte, 100) }, 27 | ) 28 | require.NoError(t, err) 29 | 30 | t.Log("Corrupting a random leaf page.") 31 | victimPageId, validPageIds := corruptRandomLeafPageInBucket(t, db.DB, bucketName) 32 | 33 | t.Log("Running consistency check.") 34 | vErr := db.View(func(tx *bbolt.Tx) error { 35 | var cErrs []error 36 | 37 | t.Log("Check corrupted page.") 38 | errChan := tx.Check(bbolt.WithPageId(uint64(victimPageId))) 39 | for cErr := range errChan { 40 | cErrs = append(cErrs, cErr) 41 | } 42 | require.Greater(t, len(cErrs), 0) 43 | 44 | t.Log("Check valid pages.") 45 | cErrs = cErrs[:0] 46 | for _, pgId := range validPageIds { 47 | errChan = tx.Check(bbolt.WithPageId(uint64(pgId))) 48 | for cErr := range errChan { 49 | cErrs = append(cErrs, cErr) 50 | } 51 | require.Equal(t, 0, len(cErrs)) 52 | } 53 | return nil 54 | }) 55 | require.NoError(t, vErr) 56 | t.Log("All check passed") 57 | 58 | // Manually close the db, otherwise the PostTestCleanup will 59 | // check the db again and accordingly fail the test. 60 | db.MustClose() 61 | } 62 | 63 | func TestTx_Check_WithNestBucket(t *testing.T) { 64 | parentBucketName := []byte("parentBucket") 65 | 66 | t.Log("Creating db file.") 67 | db := btesting.MustCreateDBWithOption(t, &bbolt.Options{PageSize: 4096}) 68 | 69 | err := db.Update(func(tx *bbolt.Tx) error { 70 | pb, bErr := tx.CreateBucket(parentBucketName) 71 | if bErr != nil { 72 | return bErr 73 | } 74 | 75 | t.Log("put some key/values under the parent bucket directly") 76 | for i := 0; i < 10; i++ { 77 | k, v := fmt.Sprintf("%04d", i), fmt.Sprintf("value_%4d", i) 78 | if pErr := pb.Put([]byte(k), []byte(v)); pErr != nil { 79 | return pErr 80 | } 81 | } 82 | 83 | t.Log("create a nested bucket and put some key/values under the nested bucket") 84 | cb, bErr := pb.CreateBucket([]byte("nestedBucket")) 85 | if bErr != nil { 86 | return bErr 87 | } 88 | 89 | for i := 0; i < 2000; i++ { 90 | k, v := fmt.Sprintf("%04d", i), fmt.Sprintf("value_%4d", i) 91 | if pErr := cb.Put([]byte(k), []byte(v)); pErr != nil { 92 | return pErr 93 | } 94 | } 95 | 96 | return nil 97 | }) 98 | require.NoError(t, err) 99 | 100 | // Get the bucket's root page. 101 | bucketRootPageId := mustGetBucketRootPage(t, db.DB, parentBucketName) 102 | 103 | t.Logf("Running consistency check starting from pageId: %d", bucketRootPageId) 104 | vErr := db.View(func(tx *bbolt.Tx) error { 105 | var cErrs []error 106 | 107 | errChan := tx.Check(bbolt.WithPageId(uint64(bucketRootPageId))) 108 | for cErr := range errChan { 109 | cErrs = append(cErrs, cErr) 110 | } 111 | require.Equal(t, 0, len(cErrs)) 112 | 113 | return nil 114 | }) 115 | require.NoError(t, vErr) 116 | t.Log("All check passed") 117 | 118 | // Manually close the db, otherwise the PostTestCleanup will 119 | // check the db again and accordingly fail the test. 120 | db.MustClose() 121 | } 122 | 123 | // corruptRandomLeafPage corrupts one random leaf page. 124 | func corruptRandomLeafPageInBucket(t testing.TB, db *bbolt.DB, bucketName []byte) (victimPageId common.Pgid, validPageIds []common.Pgid) { 125 | bucketRootPageId := mustGetBucketRootPage(t, db, bucketName) 126 | bucketRootPage, _, err := guts_cli.ReadPage(db.Path(), uint64(bucketRootPageId)) 127 | require.NoError(t, err) 128 | require.True(t, bucketRootPage.IsBranchPage()) 129 | 130 | // Retrieve all the leaf pages included in the branch page, and pick up random one from them. 131 | var bucketPageIds []common.Pgid 132 | for _, bpe := range bucketRootPage.BranchPageElements() { 133 | bucketPageIds = append(bucketPageIds, bpe.Pgid()) 134 | } 135 | randomIdx := rand.Intn(len(bucketPageIds)) 136 | victimPageId = bucketPageIds[randomIdx] 137 | validPageIds = append(bucketPageIds[:randomIdx], bucketPageIds[randomIdx+1:]...) 138 | 139 | victimPage, victimBuf, err := guts_cli.ReadPage(db.Path(), uint64(victimPageId)) 140 | require.NoError(t, err) 141 | require.True(t, victimPage.IsLeafPage()) 142 | require.True(t, victimPage.Count() > 1) 143 | 144 | // intentionally make the second key < the first key. 145 | element := victimPage.LeafPageElement(1) 146 | key := element.Key() 147 | key[0] = 0 148 | 149 | // Write the corrupt page to db file. 150 | err = guts_cli.WritePage(db.Path(), victimBuf) 151 | require.NoError(t, err) 152 | return victimPageId, validPageIds 153 | } 154 | 155 | // mustGetBucketRootPage returns the root page for the provided bucket. 156 | func mustGetBucketRootPage(t testing.TB, db *bbolt.DB, bucketName []byte) common.Pgid { 157 | var rootPageId common.Pgid 158 | _ = db.View(func(tx *bbolt.Tx) error { 159 | b := tx.Bucket(bucketName) 160 | require.NotNil(t, b) 161 | rootPageId = b.RootPage() 162 | return nil 163 | }) 164 | 165 | return rootPageId 166 | } 167 | -------------------------------------------------------------------------------- /tx_stats_test.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestTxStats_add(t *testing.T) { 11 | statsA := TxStats{ 12 | PageCount: 1, 13 | PageAlloc: 2, 14 | CursorCount: 3, 15 | NodeCount: 100, 16 | NodeDeref: 101, 17 | Rebalance: 1000, 18 | RebalanceTime: 1001 * time.Second, 19 | Split: 10000, 20 | Spill: 10001, 21 | SpillTime: 10001 * time.Second, 22 | Write: 100000, 23 | WriteTime: 100001 * time.Second, 24 | } 25 | 26 | statsB := TxStats{ 27 | PageCount: 2, 28 | PageAlloc: 3, 29 | CursorCount: 4, 30 | NodeCount: 101, 31 | NodeDeref: 102, 32 | Rebalance: 1001, 33 | RebalanceTime: 1002 * time.Second, 34 | Split: 11001, 35 | Spill: 11002, 36 | SpillTime: 11002 * time.Second, 37 | Write: 110001, 38 | WriteTime: 110010 * time.Second, 39 | } 40 | 41 | statsB.add(&statsA) 42 | assert.Equal(t, int64(3), statsB.GetPageCount()) 43 | assert.Equal(t, int64(5), statsB.GetPageAlloc()) 44 | assert.Equal(t, int64(7), statsB.GetCursorCount()) 45 | assert.Equal(t, int64(201), statsB.GetNodeCount()) 46 | assert.Equal(t, int64(203), statsB.GetNodeDeref()) 47 | assert.Equal(t, int64(2001), statsB.GetRebalance()) 48 | assert.Equal(t, 2003*time.Second, statsB.GetRebalanceTime()) 49 | assert.Equal(t, int64(21001), statsB.GetSplit()) 50 | assert.Equal(t, int64(21003), statsB.GetSpill()) 51 | assert.Equal(t, 21003*time.Second, statsB.GetSpillTime()) 52 | assert.Equal(t, int64(210001), statsB.GetWrite()) 53 | assert.Equal(t, 210011*time.Second, statsB.GetWriteTime()) 54 | } 55 | -------------------------------------------------------------------------------- /unix_test.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | package bbolt_test 4 | 5 | import ( 6 | "fmt" 7 | "testing" 8 | 9 | "golang.org/x/sys/unix" 10 | 11 | bolt "go.etcd.io/bbolt" 12 | "go.etcd.io/bbolt/internal/btesting" 13 | ) 14 | 15 | func TestMlock_DbOpen(t *testing.T) { 16 | // 32KB 17 | skipOnMemlockLimitBelow(t, 32*1024) 18 | 19 | btesting.MustCreateDBWithOption(t, &bolt.Options{Mlock: true}) 20 | } 21 | 22 | // Test change between "empty" (16KB) and "non-empty" db 23 | func TestMlock_DbCanGrow_Small(t *testing.T) { 24 | // 32KB 25 | skipOnMemlockLimitBelow(t, 32*1024) 26 | 27 | db := btesting.MustCreateDBWithOption(t, &bolt.Options{Mlock: true}) 28 | 29 | if err := db.Update(func(tx *bolt.Tx) error { 30 | b, err := tx.CreateBucketIfNotExists([]byte("bucket")) 31 | if err != nil { 32 | t.Fatal(err) 33 | } 34 | 35 | key := []byte("key") 36 | value := []byte("value") 37 | if err := b.Put(key, value); err != nil { 38 | t.Fatal(err) 39 | } 40 | 41 | return nil 42 | }); err != nil { 43 | t.Fatal(err) 44 | } 45 | 46 | } 47 | 48 | // Test crossing of 16MB (AllocSize) of db size 49 | func TestMlock_DbCanGrow_Big(t *testing.T) { 50 | if testing.Short() { 51 | t.Skip("skipping test in short mode") 52 | } 53 | 54 | // 32MB 55 | skipOnMemlockLimitBelow(t, 32*1024*1024) 56 | 57 | chunksBefore := 64 58 | chunksAfter := 64 59 | 60 | db := btesting.MustCreateDBWithOption(t, &bolt.Options{Mlock: true}) 61 | 62 | for chunk := 0; chunk < chunksBefore; chunk++ { 63 | insertChunk(t, db, chunk) 64 | } 65 | dbSize := fileSize(db.Path()) 66 | 67 | for chunk := 0; chunk < chunksAfter; chunk++ { 68 | insertChunk(t, db, chunksBefore+chunk) 69 | } 70 | newDbSize := fileSize(db.Path()) 71 | 72 | if newDbSize <= dbSize { 73 | t.Errorf("db didn't grow: %v <= %v", newDbSize, dbSize) 74 | } 75 | } 76 | 77 | func insertChunk(t *testing.T, db *btesting.DB, chunkId int) { 78 | chunkSize := 1024 79 | 80 | if err := db.Update(func(tx *bolt.Tx) error { 81 | b, err := tx.CreateBucketIfNotExists([]byte("bucket")) 82 | if err != nil { 83 | t.Fatal(err) 84 | } 85 | 86 | for i := 0; i < chunkSize; i++ { 87 | key := []byte(fmt.Sprintf("key-%d-%d", chunkId, i)) 88 | value := []byte("value") 89 | if err := b.Put(key, value); err != nil { 90 | t.Fatal(err) 91 | } 92 | } 93 | 94 | return nil 95 | }); err != nil { 96 | t.Fatal(err) 97 | } 98 | } 99 | 100 | // Main reason for this check is travis limiting mlockable memory to 64KB 101 | // https://github.com/travis-ci/travis-ci/issues/2462 102 | func skipOnMemlockLimitBelow(t *testing.T, memlockLimitRequest uint64) { 103 | var info unix.Rlimit 104 | if err := unix.Getrlimit(unix.RLIMIT_MEMLOCK, &info); err != nil { 105 | t.Fatal(err) 106 | } 107 | 108 | if info.Cur < memlockLimitRequest { 109 | t.Skipf( 110 | "skipping as RLIMIT_MEMLOCK is insufficient: %v < %v", 111 | info.Cur, 112 | memlockLimitRequest, 113 | ) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /utils_test.go: -------------------------------------------------------------------------------- 1 | package bbolt_test 2 | 3 | import ( 4 | bolt "go.etcd.io/bbolt" 5 | "go.etcd.io/bbolt/internal/common" 6 | ) 7 | 8 | // `dumpBucket` dumps all the data, including both key/value data 9 | // and child buckets, from the source bucket into the target db file. 10 | func dumpBucket(srcBucketName []byte, srcBucket *bolt.Bucket, dstFilename string) error { 11 | common.Assert(len(srcBucketName) != 0, "source bucket name can't be empty") 12 | common.Assert(srcBucket != nil, "the source bucket can't be nil") 13 | common.Assert(len(dstFilename) != 0, "the target file path can't be empty") 14 | 15 | dstDB, err := bolt.Open(dstFilename, 0600, nil) 16 | if err != nil { 17 | return err 18 | } 19 | defer dstDB.Close() 20 | 21 | return dstDB.Update(func(tx *bolt.Tx) error { 22 | dstBucket, err := tx.CreateBucket(srcBucketName) 23 | if err != nil { 24 | return err 25 | } 26 | return cloneBucket(srcBucket, dstBucket) 27 | }) 28 | } 29 | 30 | func cloneBucket(src *bolt.Bucket, dst *bolt.Bucket) error { 31 | return src.ForEach(func(k, v []byte) error { 32 | if v == nil { 33 | srcChild := src.Bucket(k) 34 | dstChild, err := dst.CreateBucket(k) 35 | if err != nil { 36 | return err 37 | } 38 | if err = dstChild.SetSequence(srcChild.Sequence()); err != nil { 39 | return err 40 | } 41 | 42 | return cloneBucket(srcChild, dstChild) 43 | } 44 | 45 | return dst.Put(k, v) 46 | }) 47 | } 48 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | var ( 4 | // Version shows the last bbolt binary version released. 5 | Version = "1.4.0-alpha.0" 6 | ) 7 | --------------------------------------------------------------------------------