├── .github ├── CODE_OF_CONDUCT.md ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── doc_improve.md │ ├── enhancement.md │ ├── feature_request.md │ ├── github-does-not-support-choosing-PR-template-on-web-yet │ ├── refactor_request.md │ └── testing_request.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml ├── settings.yml └── workflows │ ├── golangci-lint.yml │ ├── local-lint.yml │ ├── test-short.yml │ └── test.yml ├── .gitignore ├── .gitsubrepo ├── .idea ├── modules.xml ├── slim.iml └── vcs.xml ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── _tmpl └── workflow.md.j2 ├── appveyor.yml ├── array ├── array.go ├── array.pb.go ├── array.proto ├── array_test.go ├── base.go ├── base_test.go ├── bitmap.pb.go ├── bitmap.proto ├── errors.go ├── gen.go ├── gen │ └── impl_gen.go ├── int.go ├── int_test.go └── marshal_test.go ├── benchhelper ├── benchhelper.go ├── plot.go └── report.go ├── docs ├── README.md.j2 ├── badges.md ├── change-log.yaml ├── change-log │ ├── v0.4.0.yaml │ ├── v0.4.1.yaml │ ├── v0.4.3.yaml │ ├── v0.5.0.yaml │ ├── v0.5.1.yaml │ ├── v0.5.10.yaml │ ├── v0.5.11.yaml │ ├── v0.5.2.yaml │ ├── v0.5.3.yaml │ ├── v0.5.4.yaml │ ├── v0.5.5.yaml │ ├── v0.5.6.yaml │ ├── v0.5.7.yaml │ ├── v0.5.8.yaml │ └── v0.5.9.yaml ├── performance.md ├── trie-to-slim │ ├── Makefile │ ├── README.md │ ├── bitrie.dot │ ├── bitrie.jpg │ ├── slim-cut-inner.dot │ ├── slim-cut-inner.jpg │ ├── slim-cut-leaf.dot │ ├── slim-cut-leaf.jpg │ ├── slim-final.dot │ ├── slim-final.jpg │ ├── slim-init.dot │ ├── slim-init.jpg │ ├── slim-no-leaf.dot │ └── slim-no-leaf.jpg ├── trie │ ├── benchmark_readme.md │ ├── benchmark_result.md │ ├── cpu.profiling │ └── memusage.md ├── version.md └── workflow.md ├── encode ├── bytes.go ├── bytes_test.go ├── dummy.go ├── dummy_test.go ├── encoder.go ├── encoder_test.go ├── gen.go ├── gen │ └── impl_gen.go ├── int.go ├── int8.go ├── int8_test.go ├── int_test.go ├── nativeint.go ├── nativeint_test.go ├── type_encoder.go └── type_encoder_test.go ├── git-subrepo ├── go.mod ├── go.sum ├── index ├── example_range_test.go ├── example_test.go ├── index.go └── index_test.go ├── int-types.md ├── scripts ├── build_change_log.py ├── build_md.py ├── change-types.yaml └── requirements.txt ├── tools └── reports │ └── memusage │ └── memusage.go └── trie ├── benchmark └── benchmark.go ├── benchmark_btree_test.go ├── bitmap.go ├── bm17with2bit_plot.txt ├── docs ├── README.md ├── imgs │ ├── Makefile │ ├── bitrie.dot │ ├── bitrie.jpg │ ├── slim-cut-inner.dot │ ├── slim-cut-inner.jpg │ ├── slim-cut-leaf.dot │ ├── slim-cut-leaf.jpg │ ├── slim-final.dot │ ├── slim-final.jpg │ ├── slim-init.dot │ ├── slim-init.jpg │ ├── slim-no-leaf.dot │ ├── slim-no-leaf.jpg │ ├── trie_after_squash.png │ └── trie_before_squash.png └── squash.md ├── errors.go ├── example_mem_usage_test.go ├── example_scan_test.go ├── example_slimtrie_range_test.go ├── example_string_value_test.go ├── gen_report.go ├── report ├── bench_get_absent_scan.data ├── bench_get_absent_scan.jpg ├── bench_get_absent_scan.md ├── bench_get_absent_zipf.data ├── bench_get_absent_zipf.jpg ├── bench_get_absent_zipf.md ├── bench_get_present_scan.data ├── bench_get_present_scan.jpg ├── bench_get_present_scan.md ├── bench_get_present_zipf.data ├── bench_get_present_zipf.jpg ├── bench_get_present_zipf.md ├── bench_msab_present_scan.data ├── bench_msab_present_scan.jpg ├── bench_msab_present_scan.md ├── bench_msab_present_zipf.data ├── bench_msab_present_zipf.jpg ├── bench_msab_present_zipf.md ├── fpr_get.data ├── fpr_get.jpg ├── fpr_get.md ├── mem_usage.data ├── mem_usage.jpg ├── mem_usage.md └── report.go ├── slim.pb.go ├── slim.proto ├── slimtrie.go ├── slimtrie_bench_test.go ├── slimtrie_complete_bench_test.go ├── slimtrie_complete_test.go ├── slimtrie_create.go ├── slimtrie_getint.go ├── slimtrie_getint_test.go ├── slimtrie_getnode.go ├── slimtrie_level.go ├── slimtrie_level_test.go ├── slimtrie_marshal.go ├── slimtrie_marshal_test.go ├── slimtrie_new_bench_test.go ├── slimtrie_query.go ├── slimtrie_reduce_dup_value_test.go ├── slimtrie_scan.go ├── slimtrie_scan_test.go ├── slimtrie_stat.go ├── slimtrie_stat_test.go ├── slimtrie_str.go ├── slimtrie_str_test.go ├── slimtrie_string_value_test.go ├── slimtrie_test.go ├── slimtrie_vars.go ├── slimtrie_ver.go ├── slimtrie_vlen_array.go ├── slimtrie_vlen_array_test.go ├── slimtrie_withprefix_test.go └── testdata ├── README.md ├── slimtrie-data-10ll16k-0.5.0 ├── slimtrie-data-10ll16k-0.5.1 ├── slimtrie-data-10ll16k-0.5.2 ├── slimtrie-data-10ll16k-0.5.3 ├── slimtrie-data-10ll16k-0.5.4 ├── slimtrie-data-10ll16k-0.5.5 ├── slimtrie-data-10ll16k-0.5.6 ├── slimtrie-data-10ll16k-0.5.7 ├── slimtrie-data-10ll16k-0.5.8 ├── slimtrie-data-10ll16k-0.5.9 ├── slimtrie-data-10ll16k-allpref-0.5.10 ├── slimtrie-data-10ll16k-innpref-0.5.10 ├── slimtrie-data-10ll16k-nopref-0.5.10 ├── slimtrie-data-10vl5-0.5.0 ├── slimtrie-data-10vl5-0.5.1 ├── slimtrie-data-10vl5-0.5.2 ├── slimtrie-data-10vl5-0.5.3 ├── slimtrie-data-10vl5-0.5.4 ├── slimtrie-data-10vl5-0.5.5 ├── slimtrie-data-10vl5-0.5.6 ├── slimtrie-data-10vl5-0.5.7 ├── slimtrie-data-10vl5-0.5.8 ├── slimtrie-data-10vl5-0.5.9 ├── slimtrie-data-10vl5-allpref-0.5.10 ├── slimtrie-data-10vl5-innpref-0.5.10 ├── slimtrie-data-10vl5-nopref-0.5.10 ├── slimtrie-data-11vl5-0.5.0 ├── slimtrie-data-11vl5-0.5.1 ├── slimtrie-data-11vl5-0.5.2 ├── slimtrie-data-11vl5-0.5.3 ├── slimtrie-data-11vl5-0.5.4 ├── slimtrie-data-11vl5-0.5.5 ├── slimtrie-data-11vl5-0.5.6 ├── slimtrie-data-11vl5-0.5.7 ├── slimtrie-data-11vl5-0.5.8 ├── slimtrie-data-11vl5-0.5.9 ├── slimtrie-data-11vl5-allpref-0.5.10 ├── slimtrie-data-11vl5-innpref-0.5.10 ├── slimtrie-data-11vl5-nopref-0.5.10 ├── slimtrie-data-20kl10-0.5.0 ├── slimtrie-data-20kl10-0.5.1 ├── slimtrie-data-20kl10-0.5.2 ├── slimtrie-data-20kl10-0.5.3 ├── slimtrie-data-20kl10-0.5.4 ├── slimtrie-data-20kl10-0.5.5 ├── slimtrie-data-20kl10-0.5.6 ├── slimtrie-data-20kl10-0.5.7 ├── slimtrie-data-20kl10-0.5.8 ├── slimtrie-data-20kl10-0.5.9 ├── slimtrie-data-20kl10-allpref-0.5.10 ├── slimtrie-data-20kl10-innpref-0.5.10 ├── slimtrie-data-20kl10-nopref-0.5.10 ├── slimtrie-data-20kvl10-0.5.0 ├── slimtrie-data-20kvl10-0.5.1 ├── slimtrie-data-20kvl10-0.5.2 ├── slimtrie-data-20kvl10-0.5.3 ├── slimtrie-data-20kvl10-0.5.4 ├── slimtrie-data-20kvl10-0.5.5 ├── slimtrie-data-20kvl10-0.5.6 ├── slimtrie-data-20kvl10-0.5.7 ├── slimtrie-data-20kvl10-0.5.8 ├── slimtrie-data-20kvl10-0.5.9 ├── slimtrie-data-20kvl10-allpref-0.5.10 ├── slimtrie-data-20kvl10-innpref-0.5.10 ├── slimtrie-data-20kvl10-nopref-0.5.10 ├── slimtrie-data-300vl50-allpref-0.5.10 ├── slimtrie-data-300vl50-innpref-0.5.10 ├── slimtrie-data-300vl50-nopref-0.5.10 ├── slimtrie-data-50kl10-0.5.5 ├── slimtrie-data-50kl10-0.5.6 ├── slimtrie-data-50kl10-0.5.7 ├── slimtrie-data-50kl10-0.5.8 ├── slimtrie-data-50kl10-0.5.9 ├── slimtrie-data-50kl10-allpref-0.5.10 ├── slimtrie-data-50kl10-innpref-0.5.10 ├── slimtrie-data-50kl10-nopref-0.5.10 ├── slimtrie-data-50kvl10-0.5.5 ├── slimtrie-data-50kvl10-0.5.6 ├── slimtrie-data-50kvl10-0.5.7 ├── slimtrie-data-50kvl10-0.5.8 ├── slimtrie-data-50kvl10-0.5.9 ├── slimtrie-data-50kvl10-allpref-0.5.10 ├── slimtrie-data-50kvl10-innpref-0.5.10 ├── slimtrie-data-50kvl10-nopref-0.5.10 ├── slimtrie-data-empty-0.5.0 ├── slimtrie-data-empty-0.5.1 ├── slimtrie-data-empty-0.5.2 ├── slimtrie-data-empty-0.5.3 ├── slimtrie-data-empty-0.5.4 ├── slimtrie-data-empty-0.5.5 ├── slimtrie-data-empty-0.5.6 ├── slimtrie-data-empty-0.5.7 ├── slimtrie-data-empty-0.5.8 ├── slimtrie-data-empty-0.5.9 ├── slimtrie-data-empty-allpref-0.5.10 ├── slimtrie-data-empty-innpref-0.5.10 └── slimtrie-data-empty-nopref-0.5.10 /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: 'bug' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | 12 | **To Reproduce** 13 | 1. Go to '...' 14 | 2. Click on '....' 15 | 3. Scroll down to '....' 16 | 4. See error 17 | 18 | **Expected behavior** 19 | 20 | **Actual behavior** 21 | 22 | **Screenshots** 23 | 24 | **Environment (please complete the following information):** 25 | - OS: [e.g. iOS] 26 | - Version [e.g. 22] 27 | 28 | **Language (please complete the following information):** 29 | - Language: [e.g. Go] 30 | - Version [e.g. 22] 31 | 32 | **Additional context** 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/doc_improve.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Document 3 | about: Add document 4 | title: '' 5 | labels: 'doc' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **What document to add** 11 | 12 | **Additional context** 13 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/enhancement.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Enhancement 3 | about: Enhancement to a feature/API 4 | title: '' 5 | labels: 'enhancement' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is the requested enhancement related to a problem? Please describe.** 11 | 12 | **Describe the solution** 13 | 14 | **Describe alternatives you've considered** 15 | 16 | **Additional context** 17 | 18 | **Affect other component or side effect** 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: 'feature' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | 12 | **Describe the solution you'd like** 13 | 14 | **Describe alternatives you've considered** 15 | 16 | **Additional context** 17 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/github-does-not-support-choosing-PR-template-on-web-yet: -------------------------------------------------------------------------------- 1 | Choosing PR template can be done with URL query. 2 | 3 | For now github will only read PR template from `.github/PULL_REQUEST_TEMPLATE.md` only. 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/refactor_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Refactor 3 | about: Refactoring without impacting on end-user 4 | title: '' 5 | labels: 'refactor' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is the requested refactoring related to a known/potential problem? Please describe.** 11 | 12 | **Describe what to do** 13 | 14 | **Describe alternatives you've considered** 15 | 16 | **Additional context** 17 | 18 | **Affect other component or side effect** 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/testing_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Testing 3 | about: Improve test suite. 4 | title: '' 5 | labels: 'testing' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is the requested test related to a known/potential problem? Please describe.** 11 | 12 | **Describe what test should be added/modified** 13 | 14 | **Describe alternatives you've considered** 15 | 16 | **Additional context** 17 | 18 | **Affect other component or side effect** 19 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Description 2 | 3 | 7 | 8 | Fixes # (issue) 9 | 10 | ## Type of change 11 | 12 | 13 | 14 | - **Bug fix** 15 | - **New feature** 16 | - **Breaking change** 17 | - **Refactoring** 18 | - **Document changes** 19 | - **Test changes** 20 | 21 | 22 | ## How to reproduce it (if it is a bug-fix PR) 23 | 24 | - Env: x86-64, CentOS-7.4, kernel-3.10.0, GO-1.10.1, etc. 25 | 26 | - Step-1: 27 | - Step-2: 28 | 29 | 30 | ## The solution (to fix a bug, implement a new feature etc.) 31 | 32 | 33 | 34 | # Checklist: 35 | 36 | - [ ] **Style**: My code follows the **style guidelines** of this project 37 | - [ ] **Self-review**: I have performed a **self-review** of my own code 38 | - [ ] **Comment**: I have **commented my code**, particularly in hard-to-understand areas 39 | - [ ] **Doc**: I have made corresponding changes to the **documentation** 40 | - [ ] **No-warnings**: My changes generate **no new warnings** 41 | - [ ] **Add-test**: I have added **tests** that prove my fix is effective or that my feature works 42 | - [ ] **Pass**: New and existing **unit tests pass** locally with my changes 43 | - [ ] **Dep**: Any **dependent** changes have been merged and published in downstream modules 44 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | open-pull-requests-limit: 10 8 | ignore: 9 | - dependency-name: github.com/golang/protobuf 10 | versions: 11 | - 1.4.3 12 | - 1.5.1 13 | -------------------------------------------------------------------------------- /.github/settings.yml: -------------------------------------------------------------------------------- 1 | _extends: gh-config 2 | 3 | 4 | repository: 5 | name: slim 6 | description: Surprisingly space efficient trie in Golang(11 bits/key; 100 ns/get). 7 | homepage: https://openacid.github.io/ 8 | topics: go, golang, memory, compacted, compress, datastructure, trie, tree 9 | -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | branches: 7 | - '*' 8 | pull_request: 9 | jobs: 10 | golangci: 11 | name: lint 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: golangci-lint 17 | uses: golangci/golangci-lint-action@v3 18 | with: 19 | # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. 20 | version: v1.29 21 | 22 | # Optional: working directory, useful for monorepos 23 | # working-directory: somedir 24 | 25 | # disable staticcheck: 26 | # SA1019: package github.com/golang/protobuf/proto is deprecated: Use the "google.golang.org/protobuf/proto" package instead 27 | args: --issues-exit-code=0 --exclude SA1019 28 | 29 | # Optional: show only new issues if it's a pull request. The default value is `false`. 30 | # only-new-issues: true 31 | -------------------------------------------------------------------------------- /.github/workflows/local-lint.yml: -------------------------------------------------------------------------------- 1 | name: local-lint 2 | on: 3 | push: 4 | pull_request: 5 | 6 | jobs: 7 | test: 8 | strategy: 9 | matrix: 10 | go-version: 11 | - 1.18.x 12 | os: 13 | - ubuntu-latest 14 | 15 | runs-on: ${{ matrix.os }} 16 | 17 | steps: 18 | - name: Install Go 19 | uses: actions/setup-go@v2 20 | with: 21 | go-version: ${{ matrix.go-version }} 22 | 23 | - name: checkout 24 | uses: actions/checkout@v2 25 | 26 | - name: cache 27 | uses: actions/cache@v2 28 | with: 29 | path: ~/go/pkg/mod 30 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 31 | restore-keys: | 32 | ${{ runner.os }}-go- 33 | 34 | - name: test 35 | run: make lint 36 | -------------------------------------------------------------------------------- /.github/workflows/test-short.yml: -------------------------------------------------------------------------------- 1 | name: test-short 2 | on: 3 | push: 4 | pull_request: 5 | 6 | jobs: 7 | test-short: 8 | strategy: 9 | matrix: 10 | go-version: 11 | - 1.18.x 12 | os: 13 | - ubuntu-latest 14 | 15 | runs-on: ${{ matrix.os }} 16 | 17 | steps: 18 | - name: Install Go 19 | uses: actions/setup-go@v2 20 | with: 21 | go-version: ${{ matrix.go-version }} 22 | 23 | - name: checkout 24 | uses: actions/checkout@v2 25 | 26 | - name: cache 27 | uses: actions/cache@v2 28 | with: 29 | path: ~/go/pkg/mod 30 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 31 | restore-keys: | 32 | ${{ runner.os }}-go- 33 | 34 | - name: test-short 35 | run: make test-short 36 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | on: 3 | push: 4 | pull_request: 5 | 6 | jobs: 7 | test: 8 | strategy: 9 | matrix: 10 | go-version: 11 | - 1.17.x 12 | - 1.18.x 13 | os: 14 | - ubuntu-latest 15 | - macos-latest 16 | - windows-latest 17 | 18 | runs-on: ${{ matrix.os }} 19 | 20 | steps: 21 | - name: Install Go 22 | uses: actions/setup-go@v2 23 | with: 24 | go-version: ${{ matrix.go-version }} 25 | 26 | - name: checkout 27 | uses: actions/checkout@v2 28 | 29 | - name: cache 30 | uses: actions/cache@v2 31 | with: 32 | path: ~/go/pkg/mod 33 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 34 | restore-keys: | 35 | ${{ runner.os }}-go- 36 | 37 | - name: test 38 | run: | 39 | make test 40 | 41 | - name: Send coverage 42 | uses: shogo82148/actions-goveralls@v1 43 | with: 44 | path-to-profile: coverage.out 45 | flag-name: Go-${{ matrix.go }} 46 | parallel: true 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Mac hidden file 15 | .DS_Store 16 | -------------------------------------------------------------------------------- /.gitsubrepo: -------------------------------------------------------------------------------- 1 | [ remote_suffix: .git ] 2 | [ remote: https://github.com/ ] 3 | 4 | [ base: ] 5 | 6 | git-subrepo baishancloud/git-subrepo master git-subrepo 7 | 8 | .github/CODE_OF_CONDUCT.md openacid/gh-config master .github/CODE_OF_CONDUCT.md 9 | .github/ISSUE_TEMPLATE openacid/gh-config master .github/ISSUE_TEMPLATE 10 | .github/PULL_REQUEST_TEMPLATE.md openacid/gh-config master .github/PULL_REQUEST_TEMPLATE.md 11 | 12 | .travis.yml openacid/gh-config master travis/travis-go.yml 13 | 14 | 15 | # git-subrepo 16 | # for maintaining sub git repo 17 | # https://github.com/baishancloud/git-subrepo 18 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/slim.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.14.x 5 | - 1.15.x 6 | - 1.16.x 7 | - 1.17.x 8 | - 1.18.x 9 | 10 | script: 11 | - make travis 12 | 13 | after_success: 14 | - make coveralls 15 | 16 | env: 17 | global: 18 | # coveralls token 19 | secure: fJOPOuwBaZ59iQA1VskxZ3h08Nt5CjGbz5PdrZVT/v5UCG7DOLuVTx3x0Tb+gR9AG9lB8Fqpsnm0jjbBAPvOnyn1KIJDuK9Xj2PvKT78vhJ/SyCnn0BAinmxu9hZqghvyWIzeM8RrA3IrvmnoSUTdE1jnTC7McJ7np6cTRGO9Xe6b4mOO1xQOHJFMyTBFvA84uSKZPbuUHCrh19YH7NKrA4MKunX49R+niEFlFEM4oNM/2FXMca+4+OdlGNJmPkG0kV5exP87ihfqI3Q++9v3Z8SR0KOblL6yRBspaRDHmfKxuGx/YEf71pu0yu7nyT7uVeIABTz5SLrqX2Fhb/cpKb7iqCBQ+ifvgpd86pkfhrPUOsIO9N6pieNxmb+aCNm5WBJ2AaT1zrrfthpfbXvEl66K209rUDL0PV1n/u1pAgY5q7DQD5YuOnyAJNPBNQYYzJnZ+X1GjSNrHKOQPjXmrgwkq7KPVlDoqiaJAh97YwUmjXaULKYOm9JBPwVaToEUeCxzK82ZZRwa4YiYl3MLpJb+SvDMl97hgc58lolfg01wHgLYAT901bbq+qsrQZY4pkW9nDGvBuJg0Mru1bu6hqk/tUA7G4amh2y/5lJxxELednfnyzQ6fBeXKb0FVOTN9xRuFBkpRL1Drmbz3y6J2flAcdpJ4KgAMUP/941J6o= 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 openacid 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # `grep -v` does not work on travis. No time to find out why -- xp 2019 Feb 22 2 | PKGS := $(shell go list ./... | grep -v "^github.com/openacid/slim/\(vendor\|prototype\|iohelper\|serialize\|version\)") 3 | 4 | # PKGS := github.com/openacid/slim/array \ 5 | # github.com/openacid/slim/bit \ 6 | # github.com/openacid/slim/trie \ 7 | 8 | SRCDIRS := $(shell go list -f '{{.Dir}}' $(PKGS)) 9 | 10 | # gofmt check vendor dir. we need to skip vendor manually 11 | GOFILES := $(shell find $(SRCDIRS) -not -path "*/vendor/*" -name "*.go") 12 | GO := go 13 | 14 | check: test vet gofmt misspell unconvert staticcheck ineffassign unparam 15 | 16 | travis: vet gofmt misspell unconvert ineffassign unparam test 17 | 18 | test-short: 19 | # fail fast with severe bugs 20 | $(GO) test -short $(PKGS) 21 | 22 | test: 23 | # $(GO) test -tags debug $(PKGS) 24 | # test release version and generate coverage data for task `coveralls`. 25 | $(GO) test -covermode=count -coverprofile=coverage.out $(PKGS) 26 | 27 | lint: vet gofmt misspell unconvert ineffassign unparam 28 | 29 | vet: 30 | $(GO) vet $(PKGS) 31 | 32 | staticcheck: 33 | $(GO) install honnef.co/go/tools/cmd/staticcheck@latest 34 | # ST1016: methods on the same type should have the same receiver name 35 | # .pb.go have this issue. 36 | staticcheck -checks all,-ST1016 $(PKGS) 37 | 38 | misspell: 39 | $(GO) install github.com/client9/misspell/cmd/misspell@latest 40 | find $(SRCDIRS) -name '*.go' -or -name '*.md' | grep -v "\bvendor/" | xargs misspell \ 41 | -locale US \ 42 | -error 43 | misspell \ 44 | -locale US \ 45 | -error \ 46 | *.md *.go 47 | 48 | unconvert: 49 | $(GO) install github.com/mdempsky/unconvert@latest 50 | unconvert -v $(PKGS) 51 | 52 | ineffassign: 53 | $(GO) install github.com/gordonklaus/ineffassign@latest 54 | # With ineffassign v0.0.0-20210104184537-8eed68eb605f 55 | # The following form now complains error: "named files must all be in one directory" 56 | # find $(SRCDIRS) -name '*.go' | grep -v "\bvendor/" | xargs ineffassign 57 | # Seems this works: 58 | ineffassign $(PKGS) 59 | 60 | pedantic: check errcheck 61 | 62 | unparam: 63 | $(GO) install mvdan.cc/unparam@latest 64 | unparam ./... 65 | 66 | errcheck: 67 | $(GO) install github.com/kisielk/errcheck@latest 68 | errcheck $(PKGS) 69 | 70 | gofmt: 71 | @echo Checking code is gofmted 72 | @test -z "$(shell gofmt -s -l -d -e $(GOFILES) | tee /dev/stderr)" 73 | 74 | ben: test 75 | $(GO) test ./... -run=none -bench=. -benchmem 76 | 77 | gen: 78 | $(GO) generate ./... 79 | 80 | readme: 81 | python ./scripts/build_md.py 82 | # brew install nodejs 83 | # npm install -g doctoc 84 | doctoc --title '' --github README.md 85 | 86 | fix: 87 | gofmt -s -w $(GOFILES) 88 | unconvert -v -apply $(PKGS) 89 | 90 | # local coverage 91 | coverage: 92 | $(GO) test -covermode=count -coverprofile=coverage.out $(PKGS) 93 | go tool cover -func=coverage.out 94 | go tool cover -html=coverage.out 95 | 96 | coveralls: 97 | $(GO) install golang.org/x/tools/cmd/cover@latest 98 | $(GO) install github.com/mattn/goveralls@latest 99 | goveralls -ignore='**/*.pb.go' -coverprofile=coverage.out -service=travis-ci 100 | # -repotoken $$COVERALLS_TOKEN 101 | -------------------------------------------------------------------------------- /_tmpl/workflow.md.j2: -------------------------------------------------------------------------------- 1 | ## Git branching model 2 | 3 | ```txt 4 | release-1.0 release-1.1 release-2.0 5 | ----------- ----------- ----------- 6 | * (feature-a) 7 | | 8 | * (feature-b) 9 | | 10 | | * (feature-c) 11 | pick |/ 12 | .----------------->* (master release-2.0) 13 | / | 14 | * fixbug-z | 15 | | pick | 16 | | .--------------->* (tag: v2.0.2) 17 | pick | / | 18 | .-----------------|-/----------------->* (tag: v2.0.1) 19 | / |/ | 20 | / * fixbug-y * (tag: v2.0.0) 21 | ' pick | | 22 | | .-------------->* / 23 | | / | .----------------' 24 | | / |/ major ver incr 25 | | / * 26 | |/ | 27 | * fixbug-x * (tag: v1.1.0) 28 | | / 29 | | .----------------' 30 | |/ minor ver incr 31 | * 32 | | 33 | * (tag: v1.0.0) 34 | ``` 35 | 36 | ### Version 37 | 38 | We use [semantic-version](http://semver.org/) for versioning. 39 | In a nutshell a version is in form of: `..`, 40 | such as `1.11.0`, `2.3.4`. 41 | 42 | - `major` increment indicates breaking changes. 43 | - `minor` increment indicates compatible changes. 44 | - `patch` increment indicates bug fix etc. 45 | 46 | These rule does not applies to versions `0.x.x`. 47 | 48 | 49 | ### Tag 50 | 51 | Create a tag `v..` for every version and a tag should be kept for ever. 52 | 53 | 54 | ### Branch 55 | 56 | - `release-x.y` is the branch for versions `x.y.*`. 57 | A `relase-x.y` branch should be kept during its maintaining period for applying bug fix to 58 | it or else. 59 | 60 | - **master** is the latest **stable** branch. 61 | It should point to a `release-x.y` branch. 62 | 63 | - Other branches are feature branches. 64 | A feature branch should only contains changes about one feature or one bug fix. 65 | 66 | 67 | ### Commit and message 68 | 69 | **Keep a commit atomic and complete. 70 | Every commit must pass CI**. 71 | 72 | The `subject` of a git commit is in form of: 73 | `: : what changed...` 74 | 75 | `type-of-change` is one of: 76 | 77 | {% include 'scripts/change-types.yaml' %} 78 | 79 | `module` is one of sub module or other aspect of change. 80 | There is no strict restriction on it. 81 | 82 | Example: `new-feature: slimtrie: add String() for print`. 83 | 84 | 85 | ## Working flow with git 86 | 87 | 1. Create a feature branch and hack on it. 88 | 89 | A feature branch may have more than one commits. 90 | 91 | Make sure every commit pass CI. 92 | 93 | A feature branch should base on the latest `release-x.y`. 94 | If a feature is introduced to a former `release-x.y`, make sure to 95 | `cherry-pick`(or merge) it to all newer `release-x.y`. 96 | 97 | 1. A `fix` commit(fix a bug of fix doc) should be committed to the first 98 | `release-x.y` branch that introduced the bug. 99 | 100 | And the fixbug commit should be `cherry-pick`(or `merge`) by all newer `releas-x.y`. 101 | 102 | 1. Review a feature branch by more than one members and push it to a `release-x.y` branch. 103 | 104 | 1. Bump version, create a tag for it when a `release-x.y` is ready. 105 | 106 | 107 | ### Push 108 | 109 | - Only `fast-forward` push to `master`, `release-x.y`. 110 | 111 | - feature branch is free to modify its history such as by `rebase` it. 112 | 113 | - Avoid `merge`, use `rebase` instead if possible. 114 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | version: build-{build}.{branch} 2 | 3 | clone_folder: C:\gopath\src\github.com\openacid\slim 4 | shallow_clone: true # for startup speed 5 | 6 | environment: 7 | GOPATH: C:\gopath 8 | 9 | platform: 10 | - x64 11 | 12 | # http://www.appveyor.com/docs/installed-software 13 | install: 14 | # some helpful output for debugging builds 15 | - go version 16 | - go env 17 | # pre-installed MinGW at C:\MinGW is 32bit only 18 | # but MSYS2 at C:\msys64 has mingw64 19 | - set PATH=C:\msys64\mingw64\bin;%PATH% 20 | - gcc --version 21 | - g++ --version 22 | 23 | build_script: 24 | - go install -v ./... 25 | 26 | test_script: 27 | - set PATH=C:\gopath\bin;%PATH% 28 | - go test -v ./... 29 | 30 | #artifacts: 31 | # - path: '%GOPATH%\bin\*.exe' 32 | deploy: off 33 | -------------------------------------------------------------------------------- /array/array.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option go_package = "array"; 4 | 5 | import "bitmap.proto"; 6 | 7 | message Array32 { 8 | // compatibility guarantee: 9 | // reserved field number: 1, 2, 3, 4 10 | // reserved field name: Cnt, Bitmaps, Offsets, Elts 11 | // 12 | int32 Cnt = 1; // current number of elts 13 | 14 | repeated uint64 Bitmaps = 2; // bitmaps[] about which index has elt 15 | repeated int32 Offsets = 3; // index offset in `elts` for bitmap[i] 16 | bytes Elts = 4; 17 | 18 | 19 | // Flags provides options 20 | // 21 | // Since 0.5.4 22 | uint32 Flags = 10; 23 | 24 | 25 | // EltWidth set width of elt in bits. 26 | // 27 | // Since 0.5.4 28 | int32 EltWidth = 20; 29 | 30 | 31 | // BMElts is optimized for elt itself is a bitmap. 32 | // 33 | // Since 0.5.4 34 | Bits BMElts = 30; 35 | } 36 | -------------------------------------------------------------------------------- /array/array_test.go: -------------------------------------------------------------------------------- 1 | package array_test 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "testing" 7 | 8 | proto "github.com/golang/protobuf/proto" 9 | "github.com/openacid/slim/array" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestArrayNewEmpty(t *testing.T) { 14 | 15 | type D struct { 16 | X int32 17 | Y int16 18 | } 19 | 20 | a, err := array.NewEmpty(D{}) 21 | if err != nil { 22 | t.Fatalf("expect no err but: %v", err) 23 | } 24 | 25 | _ = a 26 | 27 | // TODO test marshal 28 | 29 | } 30 | 31 | func TestArrayOfU32(t *testing.T) { 32 | 33 | a := &array.Array{} 34 | 35 | err := a.Init([]int32{1, 2, 3}, []uint32{4, 5, 6}) 36 | if err != nil { 37 | t.Fatalf("expect no err but: %v", err) 38 | } 39 | 40 | v, found := a.Get(1) 41 | if !found { 42 | t.Fatalf("expect: %v; but: %v", true, false) 43 | } 44 | if v == nil { 45 | t.Fatalf("v should not be nil expect: %v; but: %v", "not nil", v) 46 | } 47 | } 48 | 49 | func TestArrayOfStruct(t *testing.T) { 50 | 51 | a := &array.Array{} 52 | 53 | err := a.Init([]int32{10, 12, 13}, 54 | []struct { 55 | X int32 56 | Y uint16 57 | }{ 58 | {1, 2}, 59 | {3, 4}, 60 | {5, 6}, 61 | }) 62 | if err != nil { 63 | t.Fatalf("expect no err but: %v", err) 64 | } 65 | 66 | v, found := a.Get(12) 67 | if !found { 68 | t.Fatalf("expect: %v; but: %v", true, false) 69 | } 70 | if v == nil { 71 | t.Fatalf("v should not be nil expect: %v; but: %v", "not nil", v) 72 | } 73 | } 74 | 75 | func TestArrayAndU32InterMarshal(t *testing.T) { 76 | 77 | // created with fmt.Printf("%#v\n", data ) 78 | serialized := []byte{ 79 | 0x8, 0x4, 0x12, 0x6, 0xa2, 0x4, 0x0, 0x0, 80 | 0x80, 0x10, 0x1a, 0x4, 0x0, 0x0, 0x0, 0x3, 81 | 0x22, 0x10, 0xc, 0x0, 0x0, 0x0, 0xf, 0x0, 82 | 0x0, 0x0, 0x13, 0x0, 0x0, 0x0, 0x78, 0x0, 83 | 0x0, 0x0, 84 | } 85 | 86 | index := []int32{1, 5, 9, 203} 87 | eltsData := []uint32{12, 15, 19, 120} 88 | 89 | arr, err := array.NewU32(index, eltsData) 90 | if err != nil { 91 | t.Fatalf("create array failure: %s", err) 92 | } 93 | 94 | data, err := proto.Marshal(arr) 95 | if err != nil { 96 | t.Fatalf("proto.Marshal: %s", err) 97 | } 98 | 99 | if !reflect.DeepEqual(serialized, data) { 100 | fmt.Println(serialized) 101 | fmt.Println(data) 102 | t.Fatalf("serialized data incorrect") 103 | } 104 | 105 | loaded, err := array.NewEmpty(uint32(0)) 106 | if err != nil { 107 | t.Fatalf("expected no error but: %+v", err) 108 | } 109 | 110 | err = proto.Unmarshal(data, loaded) 111 | if err != nil { 112 | t.Fatalf("proto.Unmarshal: %+v", err) 113 | } 114 | 115 | second, err := proto.Marshal(loaded) 116 | if err != nil { 117 | t.Fatalf("proto.Marshal: %+v", err) 118 | } 119 | 120 | if !reflect.DeepEqual(serialized, second) { 121 | fmt.Println(serialized) 122 | fmt.Println(second) 123 | t.Fatalf("second serialized data incorrect") 124 | } 125 | } 126 | 127 | func TestArray_Unmarshal_0_5_3(t *testing.T) { 128 | 129 | ta := assert.New(t) 130 | 131 | index := []int32{1, 5, 9, 203} 132 | eltsData := []uint32{12, 15, 19, 120} 133 | 134 | // Made from: 135 | // arr, err := array.NewU32(index, eltsData) 136 | // b, err := proto.Marshal(arr) 137 | // fmt.Printf("%#v\n", b) 138 | marshaled := []byte{0x8, 0x4, 0x12, 0x6, 0xa2, 0x4, 0x0, 0x0, 0x80, 0x10, 0x1a, 0x4, 0x0, 139 | 0x0, 0x0, 0x3, 0x22, 0x10, 0xc, 0x0, 0x0, 0x0, 0xf, 0x0, 0x0, 0x0, 0x13, 140 | 0x0, 0x0, 0x0, 0x78, 0x0, 0x0, 0x0} 141 | 142 | a := &array.U32{} 143 | err := proto.Unmarshal(marshaled, a) 144 | ta.Nil(err) 145 | 146 | for i, idx := range index { 147 | v, found := a.Get(idx) 148 | ta.Equal(eltsData[i], v, "Get(%d)", idx) 149 | ta.True(found, "Get(%d)", idx) 150 | 151 | v, found = a.Get(idx - 1) 152 | ta.Equal(uint32(0), v, "Get(%d-1)", idx) 153 | ta.False(found, "Get(%d-1)", idx) 154 | } 155 | } 156 | 157 | func BenchmarkArrayGet(b *testing.B) { 158 | indexes := []int32{0, 5, 9, 203, 400} 159 | elts := []uint32{12, 15, 19, 120, 300} 160 | a, _ := array.New(indexes, elts) 161 | 162 | for i := 0; i < b.N; i++ { 163 | a.Get(5) 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /array/base.go: -------------------------------------------------------------------------------- 1 | package array 2 | 3 | import ( 4 | "encoding/binary" 5 | "reflect" 6 | 7 | "github.com/openacid/errors" 8 | "github.com/openacid/low/bitmap" 9 | "github.com/openacid/slim/encode" 10 | ) 11 | 12 | // endian is the default endian for array 13 | var endian = binary.LittleEndian 14 | 15 | // Base is the base of: Array and U16 etc. 16 | // 17 | // Since 0.2.0 18 | type Base struct { 19 | Array32 20 | EltEncoder encode.Encoder 21 | } 22 | 23 | const ( 24 | bmShift = uint(6) // log₂64 25 | bmMask = int32(63) 26 | ) 27 | 28 | // bmBit calculates bitamp word index and the bit index in the word. 29 | func bmBit(idx int32) (int32, int32) { 30 | c := idx >> bmShift 31 | r := idx & bmMask 32 | return c, r 33 | } 34 | 35 | // InitIndex initializes index bitmap for an array. 36 | // Index must be an ascending int32 slice, otherwise, it return 37 | // the ErrIndexNotAscending error 38 | // 39 | // Since 0.2.0 40 | func (a *Base) InitIndex(index []int32) error { 41 | 42 | for i := 0; i < len(index)-1; i++ { 43 | if index[i] >= index[i+1] { 44 | return ErrIndexNotAscending 45 | } 46 | } 47 | 48 | a.Bitmaps = bitmap.Of(index) 49 | a.Offsets = bitmap.IndexRank64(a.Bitmaps) 50 | a.Cnt = int32(len(index)) 51 | 52 | // Be compatible to previous issue: 53 | // Since v0.2.0, Offsets is not exactly the same as bitmap ranks. 54 | // It is 0 for empty bitmap word. 55 | // But bitmap ranks set rank[i*64] to rank[(i-1)*64] for empty word. 56 | for i, word := range a.Bitmaps { 57 | if word == 0 { 58 | a.Offsets[i] = 0 59 | } 60 | } 61 | 62 | return nil 63 | } 64 | 65 | // Init initializes an array from the "indexes" and "elts". 66 | // The indexes must be an ascending int32 slice, 67 | // otherwise, return the ErrIndexNotAscending error. 68 | // The "elts" is a slice. 69 | // 70 | // Since 0.2.0 71 | func (a *Base) Init(indexes []int32, elts interface{}) error { 72 | 73 | rElts := reflect.ValueOf(elts) 74 | if rElts.Kind() != reflect.Slice { 75 | panic("elts is not a slice") 76 | } 77 | 78 | n := rElts.Len() 79 | if len(indexes) != n { 80 | return ErrIndexLen 81 | } 82 | 83 | err := a.InitIndex(indexes) 84 | if err != nil { 85 | return err 86 | } 87 | 88 | if len(indexes) == 0 { 89 | return nil 90 | } 91 | 92 | var encoder encode.Encoder 93 | 94 | if a.EltEncoder == nil { 95 | var err error 96 | encoder, err = encode.NewTypeEncoderEndian(rElts.Index(0).Interface(), endian) 97 | if err != nil { 98 | // TODO wrap 99 | return err 100 | } 101 | } else { 102 | encoder = a.EltEncoder 103 | } 104 | 105 | _, err = a.InitElts(elts, encoder) 106 | if err != nil { 107 | return errors.Wrapf(err, "failure Init Array") 108 | } 109 | 110 | return nil 111 | } 112 | 113 | // InitElts initialized a.Elts, by encoding elements in to bytes. 114 | // 115 | // Since 0.2.0 116 | func (a *Base) InitElts(elts interface{}, encoder encode.Encoder) (int, error) { 117 | 118 | rElts := reflect.ValueOf(elts) 119 | n := rElts.Len() 120 | eltsize := encoder.GetEncodedSize(nil) 121 | sz := eltsize * n 122 | 123 | b := make([]byte, 0, sz) 124 | for i := 0; i < n; i++ { 125 | ee := rElts.Index(i).Interface() 126 | bs := encoder.Encode(ee) 127 | b = append(b, bs...) 128 | } 129 | a.Elts = b 130 | 131 | return n, nil 132 | } 133 | 134 | // Get retrieves the value at "idx" and return it. 135 | // If this array has a value at "idx" it returns the value and "true", 136 | // otherwise it returns "nil" and "false". 137 | // 138 | // Since 0.2.0 139 | func (a *Base) Get(idx int32) (interface{}, bool) { 140 | 141 | bs, ok := a.GetBytes(idx, a.EltEncoder.GetEncodedSize(nil)) 142 | if ok { 143 | _, v := a.EltEncoder.Decode(bs) 144 | return v, true 145 | } 146 | 147 | return nil, false 148 | } 149 | 150 | // GetBytes retrieves the raw data of value in []byte at "idx" and return it. 151 | // 152 | // Performance note 153 | // 154 | // Involves 2 memory access: 155 | // a.Bitmaps 156 | // a.Elts 157 | // 158 | // Involves 0 alloc 159 | // 160 | // Since 0.2.0 161 | func (a *Base) GetBytes(idx int32, eltsize int) ([]byte, bool) { 162 | r, b := bitmap.Rank64(a.Bitmaps, a.Offsets, idx) 163 | if b == 0 { 164 | return nil, false 165 | } 166 | 167 | stIdx := int32(eltsize) * r 168 | return a.Elts[stIdx : stIdx+int32(eltsize)], true 169 | } 170 | -------------------------------------------------------------------------------- /array/bitmap.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // source: bitmap.proto 3 | 4 | package array 5 | 6 | import proto "github.com/golang/protobuf/proto" 7 | import fmt "fmt" 8 | import math "math" 9 | 10 | // Reference imports to suppress errors if they are not otherwise used. 11 | var _ = proto.Marshal 12 | var _ = fmt.Errorf 13 | var _ = math.Inf 14 | 15 | // This is a compile-time assertion to ensure that this generated file 16 | // is compatible with the proto package it is being compiled against. 17 | // A compilation error at this line likely means your copy of the 18 | // proto package needs to be updated. 19 | const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package 20 | 21 | // Bits is an array of bits 22 | // with rank(how many 1 upto position x, excluding x) index. 23 | // With option dense, it compresses rank index thus reduces memory usage but a query takes more 24 | // time, about 14 ns. 25 | type Bits struct { 26 | // Flags provides options 27 | Flags uint32 `protobuf:"varint,1,opt,name=Flags,proto3" json:"Flags,omitempty"` 28 | // N is the max index of present elt + 1 29 | N int32 `protobuf:"varint,10,opt,name=N,proto3" json:"N,omitempty"` 30 | // Words contains bitmap 31 | Words []uint64 `protobuf:"varint,20,rep,packed,name=Words,proto3" json:"Words,omitempty"` 32 | // RankIndex speeds up rank() by pre-calcuate it 33 | // Choose by Flags 34 | // 35 | // Since 0.5.4 36 | RankIndex []int32 `protobuf:"varint,30,rep,packed,name=RankIndex,proto3" json:"RankIndex,omitempty"` 37 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 38 | XXX_unrecognized []byte `json:"-"` 39 | XXX_sizecache int32 `json:"-"` 40 | } 41 | 42 | func (m *Bits) Reset() { *m = Bits{} } 43 | func (m *Bits) String() string { return proto.CompactTextString(m) } 44 | func (*Bits) ProtoMessage() {} 45 | func (*Bits) Descriptor() ([]byte, []int) { 46 | return fileDescriptor_bitmap_543ed9b76e11bcdb, []int{0} 47 | } 48 | func (m *Bits) XXX_Unmarshal(b []byte) error { 49 | return xxx_messageInfo_Bits.Unmarshal(m, b) 50 | } 51 | func (m *Bits) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 52 | return xxx_messageInfo_Bits.Marshal(b, m, deterministic) 53 | } 54 | func (dst *Bits) XXX_Merge(src proto.Message) { 55 | xxx_messageInfo_Bits.Merge(dst, src) 56 | } 57 | func (m *Bits) XXX_Size() int { 58 | return xxx_messageInfo_Bits.Size(m) 59 | } 60 | func (m *Bits) XXX_DiscardUnknown() { 61 | xxx_messageInfo_Bits.DiscardUnknown(m) 62 | } 63 | 64 | var xxx_messageInfo_Bits proto.InternalMessageInfo 65 | 66 | func (m *Bits) GetFlags() uint32 { 67 | if m != nil { 68 | return m.Flags 69 | } 70 | return 0 71 | } 72 | 73 | func (m *Bits) GetN() int32 { 74 | if m != nil { 75 | return m.N 76 | } 77 | return 0 78 | } 79 | 80 | func (m *Bits) GetWords() []uint64 { 81 | if m != nil { 82 | return m.Words 83 | } 84 | return nil 85 | } 86 | 87 | func (m *Bits) GetRankIndex() []int32 { 88 | if m != nil { 89 | return m.RankIndex 90 | } 91 | return nil 92 | } 93 | 94 | func init() { 95 | proto.RegisterType((*Bits)(nil), "Bits") 96 | } 97 | 98 | func init() { proto.RegisterFile("bitmap.proto", fileDescriptor_bitmap_543ed9b76e11bcdb) } 99 | 100 | var fileDescriptor_bitmap_543ed9b76e11bcdb = []byte{ 101 | // 130 bytes of a gzipped FileDescriptorProto 102 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x49, 0xca, 0x2c, 0xc9, 103 | 0x4d, 0x2c, 0xd0, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x57, 0x8a, 0xe3, 0x62, 0x71, 0xca, 0x2c, 0x29, 104 | 0x16, 0x12, 0xe1, 0x62, 0x75, 0xcb, 0x49, 0x4c, 0x2f, 0x96, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0d, 105 | 0x82, 0x70, 0x84, 0x78, 0xb8, 0x18, 0xfd, 0x24, 0xb8, 0x14, 0x18, 0x35, 0x58, 0x83, 0x18, 0xfd, 106 | 0x40, 0x6a, 0xc2, 0xf3, 0x8b, 0x52, 0x8a, 0x25, 0x44, 0x14, 0x98, 0x35, 0x58, 0x82, 0x20, 0x1c, 107 | 0x21, 0x19, 0x2e, 0xce, 0xa0, 0xc4, 0xbc, 0x6c, 0xcf, 0xbc, 0x94, 0xd4, 0x0a, 0x09, 0x39, 0x05, 108 | 0x66, 0x0d, 0xd6, 0x20, 0x84, 0x80, 0x13, 0x7b, 0x14, 0x6b, 0x62, 0x51, 0x51, 0x62, 0x65, 0x12, 109 | 0x1b, 0xd8, 0x3e, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0x51, 0x02, 0x99, 0x2d, 0x7f, 0x00, 110 | 0x00, 0x00, 111 | } 112 | -------------------------------------------------------------------------------- /array/bitmap.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option go_package = "array"; 4 | 5 | // Bits is an array of bits 6 | // with rank(how many 1 upto position x, excluding x) index. 7 | // With option dense, it compresses rank index thus reduces memory usage but a query takes more 8 | // time, about 14 ns. 9 | message Bits { 10 | 11 | // Flags provides options 12 | uint32 Flags = 1; 13 | 14 | 15 | // N is the max index of present elt + 1 16 | int32 N = 10; 17 | 18 | 19 | // Words contains bitmap 20 | repeated uint64 Words = 20; 21 | 22 | 23 | // RankIndex speeds up rank() by pre-calcuate it 24 | // Choose by Flags 25 | // 26 | // Since 0.5.4 27 | repeated int32 RankIndex = 30; 28 | } 29 | -------------------------------------------------------------------------------- /array/errors.go: -------------------------------------------------------------------------------- 1 | package array 2 | 3 | import ( 4 | "github.com/openacid/errors" 5 | ) 6 | 7 | var ( 8 | // ErrIndexNotAscending indicates that the indexes to initialize an Array is not 9 | // in ascending order. 10 | // 11 | // Since 0.2.0 12 | ErrIndexNotAscending = errors.New("index must be an ascending ordered slice") 13 | 14 | // ErrIndexLen indicates that the number of indexes does not equal the number of 15 | // elements, when initializing an Array. 16 | // 17 | // Since 0.2.0 18 | ErrIndexLen = errors.New("the length of indexes and elts must be equal") 19 | ) 20 | -------------------------------------------------------------------------------- /array/gen.go: -------------------------------------------------------------------------------- 1 | package array 2 | 3 | // Auto-generated code definition 4 | // Usage: 5 | // go generate ./... 6 | 7 | //go:generate go run gen/impl_gen.go 8 | //go:generate protoc --proto_path=. --go_out=. array.proto 9 | //go:generate protoc --proto_path=. --go_out=. bitmap.proto 10 | -------------------------------------------------------------------------------- /benchhelper/benchhelper.go: -------------------------------------------------------------------------------- 1 | // Package benchhelper provides utilities for large data set memory or cpu 2 | // benchmark. 3 | package benchhelper 4 | 5 | import ( 6 | "math/rand" 7 | "os" 8 | "runtime" 9 | "sort" 10 | "time" 11 | 12 | "github.com/openacid/tablewriter" 13 | ) 14 | 15 | // Allocated returns the in-use heap in bytes. 16 | func Allocated() int64 { 17 | for i := 0; i < 10; i++ { 18 | runtime.GC() 19 | } 20 | 21 | var stats runtime.MemStats 22 | runtime.ReadMemStats(&stats) 23 | 24 | return int64(stats.Alloc) 25 | } 26 | 27 | func NewBytesSlices(eltSize int, n int) [][]byte { 28 | slices := make([][]byte, n) 29 | 30 | for i := 0; i < n; i++ { 31 | slices[i] = make([]byte, eltSize) 32 | } 33 | 34 | return slices 35 | } 36 | 37 | func RandI32SliceBetween(min int32, max int32, factor float64) []int32 { 38 | rnd := rand.New(rand.NewSource(time.Now().Unix())) 39 | 40 | rst := make([]int32, 0) 41 | 42 | for i := min; i < max; i++ { 43 | if rnd.Float64() < factor { 44 | rst = append(rst, i) 45 | } 46 | } 47 | 48 | return rst 49 | } 50 | 51 | func RandSortedStrings(cnt, leng int, from []byte) []string { 52 | 53 | keys := make(map[string]bool, cnt) 54 | 55 | for i := 0; i < cnt; i++ { 56 | k := RandString(leng, from) 57 | if _, ok := keys[k]; ok { 58 | i-- 59 | } else { 60 | keys[k] = true 61 | } 62 | } 63 | 64 | rsts := make([]string, cnt) 65 | j := 0 66 | for i := range keys { 67 | rsts[j] = i 68 | j++ 69 | } 70 | 71 | sort.Strings(rsts) 72 | return rsts 73 | } 74 | 75 | func RandByteSlices(cnt, leng int) [][]byte { 76 | rsts := make([][]byte, cnt) 77 | 78 | for i := int(0); i < cnt; i++ { 79 | rsts[i] = RandBytes(leng, nil) 80 | } 81 | 82 | return rsts 83 | } 84 | 85 | func RandString(leng int, from []byte) string { 86 | return string(RandBytes(leng, from)) 87 | } 88 | 89 | var letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890") 90 | 91 | func RandBytes(leng int, from []byte) []byte { 92 | 93 | if from == nil { 94 | from = letters 95 | } 96 | 97 | b := make([]byte, leng) 98 | for i := range b { 99 | b[i] = from[rand.Intn(len(from))] 100 | } 101 | return b 102 | } 103 | 104 | func newFile(fn string) *os.File { 105 | f, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE, 0644) 106 | if err != nil { 107 | panic(err) 108 | } 109 | 110 | err = f.Truncate(0) 111 | if err != nil { 112 | panic(err) 113 | } 114 | return f 115 | } 116 | 117 | func NewMDFileTable(fn string) (*os.File, *tablewriter.Table) { 118 | 119 | f := newFile(fn) 120 | tb := tablewriter.NewWriter(f) 121 | tb.SetAutoFormatHeaders(false) 122 | tb.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) 123 | tb.SetCenterSeparator("|") 124 | 125 | return f, tb 126 | } 127 | 128 | func NewDataFileTable(fn string) (*os.File, *tablewriter.Table) { 129 | 130 | f := newFile(fn) 131 | tb := tablewriter.NewWriter(f) 132 | tb.SetAutoFormatHeaders(false) 133 | tb.SetBorders(tablewriter.Border{Left: false, Top: false, Right: false, Bottom: false}) 134 | tb.SetCenterSeparator("") 135 | tb.SetColumnSeparator("") 136 | tb.SetHeaderLine(false) 137 | 138 | return f, tb 139 | } 140 | 141 | // WriteTableFiles write a .md file and a .data file 142 | func WriteTableFiles(name string, content interface{}) { 143 | { 144 | f, tb := NewMDFileTable(name + ".md") 145 | defer f.Close() 146 | tb.SetContent(content) 147 | tb.Render() 148 | } 149 | 150 | { 151 | f, tb := NewDataFileTable(name + ".data") 152 | defer f.Close() 153 | tb.SetContent(content) 154 | tb.Render() 155 | } 156 | 157 | } 158 | -------------------------------------------------------------------------------- /benchhelper/report.go: -------------------------------------------------------------------------------- 1 | package benchhelper 2 | 3 | import ( 4 | "flag" 5 | ) 6 | 7 | type ReportCmdFlag struct { 8 | Bench bool 9 | BenchMem bool 10 | FPR bool 11 | Plot bool 12 | } 13 | 14 | func InitCmdFlag() *ReportCmdFlag { 15 | f := &ReportCmdFlag{} 16 | flag.BoolVar(&f.Bench, "bench", true, "whether to re-benchmark") 17 | flag.BoolVar(&f.BenchMem, "benchmem", true, "whether to re-benchmark memory usage") 18 | flag.BoolVar(&f.FPR, "fpr", true, "whether to re-benchmark false positive rate") 19 | flag.BoolVar(&f.Plot, "plot", true, "whether to generate plot picture") 20 | flag.Parse() 21 | return f 22 | } 23 | -------------------------------------------------------------------------------- /docs/badges.md: -------------------------------------------------------------------------------- 1 | [![Travis](https://travis-ci.com/openacid/{{ name }}.svg?branch=main)](https://travis-ci.com/openacid/{{ name }}) 2 | ![test](https://github.com/openacid/{{ name }}/workflows/test/badge.svg) 3 | 4 | [![Report card](https://goreportcard.com/badge/github.com/openacid/{{ name }})](https://goreportcard.com/report/github.com/openacid/{{ name }}) 5 | [![Coverage Status](https://coveralls.io/repos/github/openacid/{{ name }}/badge.svg?branch=main&service=github)](https://coveralls.io/github/openacid/{{ name }}?branch=main&service=github) 6 | 7 | [![GoDoc](https://godoc.org/github.com/openacid/{{ name }}?status.svg)](http://godoc.org/github.com/openacid/{{ name }}) 8 | [![PkgGoDev](https://pkg.go.dev/badge/github.com/openacid/{{ name }})](https://pkg.go.dev/github.com/openacid/{{ name }}) 9 | [![Sourcegraph](https://sourcegraph.com/github.com/openacid/{{ name }}/-/badge.svg)](https://sourcegraph.com/github.com/openacid/{{ name }}?badge) 10 | -------------------------------------------------------------------------------- /docs/change-log/v0.4.0.yaml: -------------------------------------------------------------------------------- 1 | api-changes: 2 | trie: 3 | - trie.Node add squash; by wenbo; 2019-04-11 4 | - remove marshalAt and unmarshalAt; use SectionReader and SectionWriter; by drdr 5 | xp; 2019-04-10 6 | - fix method name encode->marshal; by drdr xp; 2019-04-10 7 | - SlimTrie.Get returns value and found in bool; by drdr xp; 2019-03-27 8 | new-feature: 9 | array: 10 | - add MemSize() to get memory occupied by array; by drdr xp; 2019-04-15 11 | -------------------------------------------------------------------------------- /docs/change-log/v0.4.1.yaml: -------------------------------------------------------------------------------- 1 | new-feature: 2 | encode: 3 | - add encode.Int to convert int to byte and back; by drdr xp; 2019-04-18 4 | slimtrie: 5 | - add proto.Marshaler and proto.Unmarshaler interface; by liubaohai; 2019-04-18 6 | strhelper: 7 | - add func to convert word of bits back to string; by drdr xp; 2019-04-19 8 | -------------------------------------------------------------------------------- /docs/change-log/v0.4.3.yaml: -------------------------------------------------------------------------------- 1 | new-feature: 2 | slimtrie: 3 | - RangeGet() to get value of a key in indexed range; by drdr xp; 2019-04-20 4 | - String(); by drdr xp; 2019-04-23 5 | trie: 6 | - add String() to output human readable trie structure; by drdr xp; 2019-04-19 7 | -------------------------------------------------------------------------------- /docs/change-log/v0.5.0.yaml: -------------------------------------------------------------------------------- 1 | api-change: 2 | trie: 3 | - Append() do not need isStartLeaf; by drdr xp; 2019-04-22 4 | -------------------------------------------------------------------------------- /docs/change-log/v0.5.1.yaml: -------------------------------------------------------------------------------- 1 | new-feature: 2 | slimtrie: 3 | - do not store Step on leaf node; by drdr xp; 2019-04-29 4 | trie: 5 | - Tree convert a tree-like data strcture to string; by drdr xp; 2019-05-02 6 | -------------------------------------------------------------------------------- /docs/change-log/v0.5.10.yaml: -------------------------------------------------------------------------------- 1 | api-change: 2 | array: 3 | - remove NewDenseBits. It saves a few bits for rank but query is slow ~10ns; by 4 | drdr xp; 2020-11-11 5 | - remove polyarray, new repo https//github.com/openacid/polyarray; by drdr xp; 2019-06-04 6 | - remove unused types interface Bitmap and corresponding methods.; by drdr xp; 2020-11-14 7 | - remove Dense-array. It is slow and not used any more. The standalone impl is now 8 | at https//github.com/openacid/polyarray; by drdr xp; 2020-11-21 9 | - remove Base.Indexes(); the same thing can be done with bitmap.ToArray(); see https//github.com/openacid/low/tree/master/bitmap; 10 | by drdr xp; 2020-11-22 11 | - remove Base.MemSize(); the same thing can be done with size.Of(); see https//github.com/openacid/low/tree/master/size; 12 | by drdr xp; 2020-11-22 13 | - remove Has(); It is barely a bitmap.Get1() and no other component uses it.; by 14 | drdr xp; 2020-11-22 15 | - remove array.ExtendIndex(), use bitmap instead, see https//github.com/openacid/low/tree/master/bitmap; 16 | by drdr xp; 2020-11-22 17 | - remove GetEltIndex() use bitmap.Rank64() instead, see https//github.com/openacid/low/tree/master/bitmap; 18 | by drdr xp; 2020-11-22 19 | - remove struct Bitmap16. It is kept only for loading old version data. But it does 20 | not need this type. It only need the underlying type Array32; by drdr xp; 2020-11-22 21 | benhelper: 22 | - remove RandI64Slice and RandI32Slice. moved to https//github.com/openacid/testutil; 23 | by drdr xp; 2020-11-21 24 | bits: 25 | - removed; by drdr xp; 2019-09-24 26 | iohelper: 27 | - removed; by drdr xp; 2019-09-24 28 | polyfit: 29 | - remove polyfit, moved to https//github.com/openacid/slimarray/tree/main/polyfit; 30 | by drdr xp; 2020-11-21 31 | serialize: 32 | - removed; by drdr xp; 2019-09-24 33 | strhelper: 34 | - remove deprecated functions; by drdr xp; 2019-09-24 35 | - remove strhelper moved to https//github.com/openacid/low/tree/master/bitmap; by 36 | drdr xp; 2020-11-21 37 | trie: 38 | - remove Trie; by drdr xp; 2019-09-23 39 | typehelper: 40 | - removed; by drdr xp; 2019-09-24 41 | version: 42 | - removed; by drdr xp; 2019-09-24 43 | fixbug: 44 | scripts/requirements.txt: 45 | - to reduce vulnerabilities; by snyk-bot; 2020-03-20 46 | fixdoc: 47 | slimtrie: 48 | - fix typo GetI32; by drdr xp; 2019-09-25 49 | new-feature: 50 | array: 51 | - add more bitmap flags; by drdr xp; 2019-06-24 52 | encode: 53 | - add Dummy to encode anything to nothing; by drdr xp; 2019-09-24 54 | - add I8 to encode int8; by drdr xp; 2019-09-25 55 | slimtrie: 56 | - add GetIxx to optimize integer value; by drdr xp; 2019-09-25 57 | - add option ReduceSameValue to remove adjasent records with the same value. By 58 | default true; by drdr xp; 2020-11-27 59 | -------------------------------------------------------------------------------- /docs/change-log/v0.5.11.yaml: -------------------------------------------------------------------------------- 1 | fixbug: 2 | slimtrie: 3 | - creating or loading empty slimtrie data should leave the NodeTypeBM a nil so that 4 | querying routine wont access empty bitmap; by drdr xp; 2020-11-29 5 | new-feature: 6 | ScanFrom(): 7 | - and ScanFromTo() scans slimtrie with a callback function; NewIter() returns a 8 | "next" function to iterate items in slimtrie; by drdr xp; 2020-12-03 9 | -------------------------------------------------------------------------------- /docs/change-log/v0.5.2.yaml: -------------------------------------------------------------------------------- 1 | new-feature: 2 | array: 3 | - add Dense array to compress incremental ints; by drdr xp; 2019-05-06 4 | benchhelper: 5 | - add SizeOf() to get size of a value; by drdr xp; 2019-05-06 6 | - add RandI32Slice; by drdr xp; 2019-05-07 7 | -------------------------------------------------------------------------------- /docs/change-log/v0.5.3.yaml: -------------------------------------------------------------------------------- 1 | api-change: 2 | trie: 3 | - values to create trie must be slice or it panic; by drdr xp; 2019-05-01 4 | typehelper: 5 | - ToSlice now just panic if input is not a slice.; by drdr xp; 2019-05-01 6 | -------------------------------------------------------------------------------- /docs/change-log/v0.5.4.yaml: -------------------------------------------------------------------------------- 1 | api-change: 2 | array: 3 | - NewDense do not need eltWidth; only support int32; protobuf structure change; 4 | optimize Dense creating; by drdr xp; 2019-05-13 5 | - rename Bitmap to Bits; by drdr xp; 2019-05-17 6 | new-feature: 7 | array: 8 | - add Bitmap to store series of bits; by drdr xp; 2019-05-16 9 | - Base.Indexes() to retrieve all indexes of present elements; by drdr xp; 2019-05-17 10 | - add NewBitmapJoin() to create a big bitmap by joining sub bitmaps; by drdr xp; 11 | 2019-05-16 12 | - add Bitmap16 which is compatible with U32; by drdr xp; 2019-05-18 13 | - NewBitsJoin() accept a "dense" argument; by drdr xp; 2019-05-19 14 | benchhelper: 15 | - add SizeStat() to describe data struct and size; by drdr xp; 2019-05-18 16 | polyfit: 17 | - add polyfit for fit a polynomial curve over points; by drdr xp; 2019-05-15 18 | slimtrie: 19 | - use Bitmap16 and reduce memory usage; by drdr xp; 2019-05-18 20 | strhelper: 21 | - add ToBin() converts integer or slice of integer to binary format string; by drdr 22 | xp; 2019-05-17 23 | -------------------------------------------------------------------------------- /docs/change-log/v0.5.5.yaml: -------------------------------------------------------------------------------- 1 | api-change: 2 | slimtrie: 3 | - range based SlimTrie must provides all keys; by drdr xp; 2019-05-21 4 | treestr: 5 | - interface Tree adds a new method LabelInfo to format tree branch label; by drdr 6 | xp; 2019-05-21 7 | new-feature: 8 | slimtrie: 9 | - max key limit extends to 2^31; by drdr xp; 2019-05-21 10 | tree: 11 | - add depth-first walker DepthFirst(); by drdr xp; 2019-05-21 12 | trie: 13 | - add removeSameLeaf() to remove leaves with the same value; by drdr xp; 2019-05-21 14 | -------------------------------------------------------------------------------- /docs/change-log/v0.5.6.yaml: -------------------------------------------------------------------------------- 1 | fixbug: 2 | slimtrie: 3 | - 'getStep should use int32 id instead of uint16 id. fix #104; thanks to @aaaton; 4 | by drdr xp; 2019-05-29' 5 | -------------------------------------------------------------------------------- /docs/change-log/v0.5.7.yaml: -------------------------------------------------------------------------------- 1 | api-change: 2 | slimtrie: 3 | - remove LoadTrie(); SlimTrie do not need a trie to create; by drdr xp; 2019-05-28 4 | -------------------------------------------------------------------------------- /docs/change-log/v0.5.8.yaml: -------------------------------------------------------------------------------- 1 | new-feature: 2 | slimtrie: 3 | - add version writing when marshaling and version check when unmarshaling; by drdr 4 | xp; 2019-05-31 5 | -------------------------------------------------------------------------------- /docs/change-log/v0.5.9.yaml: -------------------------------------------------------------------------------- 1 | api-change: 2 | array: 3 | - do not check internal bitmap size, just panic; by drdr xp; 2019-06-02 4 | new-feature: 5 | array: 6 | - add ExtendIndex to allocate additional 0-bits after Bitmaps and Offsets; by drdr 7 | xp; 2019-05-28 8 | -------------------------------------------------------------------------------- /docs/performance.md: -------------------------------------------------------------------------------- 1 | ## Memory overhead 2 | 3 | - Random string, fixed length, default mode, no label is store if possible: 4 | 5 | **Bits/key**: memory or disk-space in bits a key consumed in average. 6 | It does not change when key-length(`k`) becomes larger! 7 | 8 | ![](trie/report/mem_usage.jpg) 9 | 10 | 11 | - 1 million var-length string, 10 to 20 byte in different mode SlimTrie: 12 | 13 | | - | size | gzip-size | 14 | | :-- | --: | --: | 15 | | Original | 15.0M | 14.0M | 16 | | Complete | 14.0M | 10.0M | 17 | | InnerLabel | 1.3M | 0.9M | 18 | | NoLabel | 1.3M | 0.8M | 19 | 20 | Raw string list and serialized slim is stored in: 21 | https://github.com/openacid/testkeys/tree/master/assets 22 | 23 | - Original: raw string lines in a text file. 24 | 25 | - Complete: `NewSlimTrie(..., Opt{Complete:Bool(true)})`: lossless SlimTrie, 26 | stores complete info of every string. This mode provides accurate query. 27 | 28 | - InnerLabel: `NewSlimTrie(..., Opt{InnerPrefix:Bool(true)})` SlimTrie stores 29 | only label strings of inner nodes(but not label to a leaf). There is false positive in this mode. 30 | 31 | - NoLabel: No label info is stored. False positive rate is higher. 32 | 33 | 34 | ## Performance 35 | 36 | Time(in nano second) spent on a `Get()` with golang-map, SlimTrie, array and [btree][] by google. 37 | 38 | - **3.3 times faster** than the [btree][]. 39 | - **2.3 times faster** than binary search. 40 | 41 | ![](trie/report/bench_msab_present_zipf.jpg) 42 | 43 | 44 | Time(in nano second) spent on a `Get()` with different key count(`n`) and key length(`k`): 45 | 46 | ![](trie/report/bench_get_present_zipf.jpg) 47 | 48 | 49 | ## False Positive Rate 50 | 51 | ![](trie/report/fpr_get.jpg) 52 | 53 | > Bloom filter requires about 9 bits/key to archieve less than 1% FPR. 54 | 55 | See: [trie/report/](trie/report/) 56 | -------------------------------------------------------------------------------- /docs/trie-to-slim/Makefile: -------------------------------------------------------------------------------- 1 | # convert dot to jpg 2 | 3 | DOTs = $(wildcard *.dot) 4 | JPGs = $(DOTs:.dot=.jpg) 5 | 6 | all: $(JPGs) 7 | 8 | %.jpg:%.dot 9 | dot -Tjpg -Gsize=3,10\! -Gdpi=100 $< -o $@ 10 | 11 | clean: 12 | rm $(JPGs) 13 | -------------------------------------------------------------------------------- /docs/trie-to-slim/README.md: -------------------------------------------------------------------------------- 1 | # Illustration of Reducing a Trie to SlimTrie 2 | 3 | ## Steps 4 | 5 | ### Bitrie 6 | 7 | ![](bitrie.jpg) 8 | 9 | ### Initialize Trie 10 | 11 | ![](slim-init.jpg) 12 | 13 | 14 | ### Reduce Leaf Nodes 15 | 16 | ![](slim-cut-leaf.jpg) 17 | 18 | 19 | ### Reduce Inner Single Branch 20 | 21 | ![](slim-cut-inner.jpg) 22 | 23 | 24 | ### Remove Skip from Leaf Nodes 25 | 26 | ![](slim-final.jpg) 27 | 28 | 29 | ### Remove Leaf Nodes 30 | 31 | ![](slim-no-leaf.jpg) 32 | 33 | ## Update 34 | 35 | Edit `.dot` files and `make clean; make`. 36 | 37 | 38 | ## Dependency 39 | 40 | - graphviz 41 | -------------------------------------------------------------------------------- /docs/trie-to-slim/bitrie.dot: -------------------------------------------------------------------------------- 1 | digraph bitrie 2 | { 3 | graph [ranksep="0.05"]; 4 | 5 | node [shape=circle, style=filled, fillcolor="white", fixedsize=true] 6 | edge [arrowhead=none] 7 | 8 | root [label="''"] 9 | 10 | root -> abd [label="0.{2}", fontcolor=red] 11 | abd -> abde [label=0] 12 | abde -> abdef [label=0] 13 | abde -> abdeg [label=1] 14 | abd -> abdfg [label="1"] 15 | root -> b1 [label="1.{1}", fontcolor=red] 16 | b1 -> b123 [label="0"] 17 | b1 -> b14 [label=1] 18 | 19 | abd [label="0.."] 20 | abde [label="0..0"] 21 | abdef [label="0..00"] 22 | abdeg [label="0..01"] 23 | abdfg [label="0..1"] 24 | b1 [label="1."] 25 | b123 [label="1.0"] 26 | b14 [label="1.1"] 27 | } 28 | -------------------------------------------------------------------------------- /docs/trie-to-slim/bitrie.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/docs/trie-to-slim/bitrie.jpg -------------------------------------------------------------------------------- /docs/trie-to-slim/slim-cut-inner.dot: -------------------------------------------------------------------------------- 1 | digraph slim_cut_inner 2 | { 3 | graph [ranksep="0.05"]; 4 | 5 | node [shape=circle, style=filled, fillcolor="white", fixedsize=true] 6 | edge [arrowhead=none] 7 | 8 | root [label="''"] 9 | 10 | abd_leaf [fillcolor="grey", style=filled, label="", width=0.2] 11 | abdef_leaf [fillcolor="grey", style=filled, label="", width=0.2] 12 | abdeg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 13 | abdfg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 14 | b123_leaf [fillcolor="grey", style=filled, label="", width=0.2] 15 | b14_leaf [fillcolor="grey", style=filled, label="", width=0.2] 16 | 17 | 18 | root -> abd [label="a.{2}", fontcolor=red] 19 | abd -> abd_leaf [label="$"] 20 | abd -> abde [label=e] 21 | abde -> abdef [label=f] 22 | abdef -> abdef_leaf [label="$"] 23 | abde -> abdeg [label=g] 24 | abdeg -> abdeg_leaf [label="$"] 25 | abd -> abdfg [label="f.{1}", fontcolor=red] 26 | abdfg -> abdfg_leaf [label="$"] 27 | root -> b1 [label="b.{1}", fontcolor=red] 28 | b1 -> b123 [label="2.{1}", fontcolor=red] 29 | b123 -> b123_leaf [label="$"] 30 | b1 -> b14 [label=4] 31 | b14 -> b14_leaf [label="$"] 32 | 33 | abd [label="a..", fillcolor="#bbffbb"] 34 | abde [label="a..e"] 35 | abdef [label="a..ef"] 36 | abdeg [label="a..eg"] 37 | abdfg [label="a..f."] 38 | b1 [label="b.", fillcolor="#bbffbb"] 39 | b123 [label="b.2."] 40 | b14 [label="b.4"] 41 | } 42 | -------------------------------------------------------------------------------- /docs/trie-to-slim/slim-cut-inner.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/docs/trie-to-slim/slim-cut-inner.jpg -------------------------------------------------------------------------------- /docs/trie-to-slim/slim-cut-leaf.dot: -------------------------------------------------------------------------------- 1 | digraph slim_cut_leaf 2 | { 3 | graph [ranksep="0.05"]; 4 | 5 | node [shape=circle, style=filled, fillcolor="white", fixedsize=true] 6 | edge [arrowhead=none] 7 | 8 | root [label="''"] 9 | 10 | abd_leaf [fillcolor="grey", style=filled, label="", width=0.2] 11 | abdef_leaf [fillcolor="grey", style=filled, label="", width=0.2] 12 | abdeg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 13 | abdfg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 14 | b123_leaf [fillcolor="grey", style=filled, label="", width=0.2] 15 | b14_leaf [fillcolor="grey", style=filled, label="", width=0.2] 16 | 17 | 18 | root -> a [label=a] 19 | a -> ab [label=b] 20 | ab -> abd [label=d] 21 | abd -> abd_leaf [label="$"] 22 | abd -> abde [label=e] 23 | abde -> abdef [label=f] 24 | abdef -> abdef_leaf [label="$"] 25 | abde -> abdeg [label=g] 26 | abdeg -> abdeg_leaf [label="$"] 27 | abd -> abdfg [label="f.{1}", fontcolor=red] 28 | abdfg -> abdfg_leaf [label="$"] 29 | root -> b [label=b] 30 | b -> b1 [label=1] 31 | b1 -> b123 [label="2.{1}", fontcolor=red] 32 | b123 -> b123_leaf [label="$"] 33 | b1 -> b14 [label=4] 34 | b14 -> b14_leaf [label="$"] 35 | 36 | abdfg [label="abdf.", fillcolor="#bbffbb"] 37 | b123 [label="b12.", fillcolor="#bbffbb"] 38 | } 39 | -------------------------------------------------------------------------------- /docs/trie-to-slim/slim-cut-leaf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/docs/trie-to-slim/slim-cut-leaf.jpg -------------------------------------------------------------------------------- /docs/trie-to-slim/slim-final.dot: -------------------------------------------------------------------------------- 1 | digraph slim_final 2 | { 3 | graph [ranksep="0.05"]; 4 | 5 | node [shape=circle, style=filled, fillcolor="white", fixedsize=true] 6 | edge [arrowhead=none] 7 | 8 | root [label="''"] 9 | 10 | abd_leaf [fillcolor="grey", style=filled, label="", width=0.2] 11 | abdef_leaf [fillcolor="grey", style=filled, label="", width=0.2] 12 | abdeg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 13 | abdfg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 14 | b123_leaf [fillcolor="grey", style=filled, label="", width=0.2] 15 | b14_leaf [fillcolor="grey", style=filled, label="", width=0.2] 16 | 17 | 18 | root -> abd [label="a.{2}", fontcolor=red] 19 | abd -> abd_leaf [label="$"] 20 | abd -> abde [label=e] 21 | abde -> abdef [label=f] 22 | abdef -> abdef_leaf [label="$"] 23 | abde -> abdeg [label=g] 24 | abdeg -> abdeg_leaf [label="$"] 25 | abd -> abdfg [label="f"] 26 | abdfg -> abdfg_leaf [label="$"] 27 | root -> b1 [label="b.{1}", fontcolor=red] 28 | b1 -> b123 [label="2"] 29 | b123 -> b123_leaf [label="$"] 30 | b1 -> b14 [label=4] 31 | b14 -> b14_leaf [label="$"] 32 | 33 | abd [label="a.."] 34 | abde [label="a..e"] 35 | abdef [label="a..ef"] 36 | abdeg [label="a..eg"] 37 | abdfg [label="a..f"] 38 | b1 [label="b."] 39 | b123 [label="b.2"] 40 | b14 [label="b.4"] 41 | } 42 | -------------------------------------------------------------------------------- /docs/trie-to-slim/slim-final.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/docs/trie-to-slim/slim-final.jpg -------------------------------------------------------------------------------- /docs/trie-to-slim/slim-init.dot: -------------------------------------------------------------------------------- 1 | digraph slim_init 2 | { 3 | graph [ranksep="0.05"]; 4 | 5 | node [shape=circle, style=filled, fillcolor="white", fixedsize=true] 6 | edge [arrowhead=none] 7 | 8 | root [label="''"] 9 | 10 | abd_leaf [fillcolor="grey", style=filled, label="", width=0.2] 11 | abdef_leaf [fillcolor="grey", style=filled, label="", width=0.2] 12 | abdeg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 13 | abdfg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 14 | b123_leaf [fillcolor="grey", style=filled, label="", width=0.2] 15 | b14_leaf [fillcolor="grey", style=filled, label="", width=0.2] 16 | 17 | 18 | root -> a [label=a] 19 | a -> ab [label=b] 20 | ab -> abd [label=d] 21 | abd -> abd_leaf [label="$"] 22 | abd -> abde [label=e] 23 | abde -> abdef [label=f] 24 | abdef -> abdef_leaf [label="$"] 25 | abde -> abdeg [label=g] 26 | abdeg -> abdeg_leaf [label="$"] 27 | abd -> abdf [label=f] 28 | abdf -> abdfg [label=g] 29 | abdfg -> abdfg_leaf [label="$"] 30 | root -> b [label=b] 31 | b -> b1 [label=1] 32 | b1 -> b12 [label=2] 33 | b12 -> b123 [label=3] 34 | b123 -> b123_leaf [label="$"] 35 | b1 -> b14 [label=4] 36 | b14 -> b14_leaf [label="$"] 37 | } 38 | -------------------------------------------------------------------------------- /docs/trie-to-slim/slim-init.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/docs/trie-to-slim/slim-init.jpg -------------------------------------------------------------------------------- /docs/trie-to-slim/slim-no-leaf.dot: -------------------------------------------------------------------------------- 1 | digraph slim_no_leaf 2 | { 3 | graph [ranksep="0.05"]; 4 | 5 | node [shape=circle, style=filled, fillcolor="white", fixedsize=true] 6 | edge [arrowhead=none] 7 | 8 | root [label="''"] 9 | 10 | root -> abd [label="a.{2}", fontcolor=red] 11 | abd -> abde [label=e] 12 | abde -> abdef [label=f] 13 | abde -> abdeg [label=g] 14 | abd -> abdfg [label="f"] 15 | root -> b1 [label="b.{1}", fontcolor=red] 16 | b1 -> b123 [label="2"] 17 | b1 -> b14 [label=4] 18 | 19 | abd [label="a.."] 20 | abde [label="a..e"] 21 | abdef [label="a..ef"] 22 | abdeg [label="a..eg"] 23 | abdfg [label="a..f"] 24 | b1 [label="b."] 25 | b123 [label="b.2"] 26 | b14 [label="b.4"] 27 | } 28 | -------------------------------------------------------------------------------- /docs/trie-to-slim/slim-no-leaf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/docs/trie-to-slim/slim-no-leaf.jpg -------------------------------------------------------------------------------- /docs/trie/benchmark_readme.md: -------------------------------------------------------------------------------- 1 | # SlimTrie Search Performance 2 | 3 | We make benchmarks to test the performance of trie search, and, we also benchmark the search of 4 | B-tree and Map in the same environment to make a comparison. 5 | 6 | Benchmarks search one-key in different key length-count pair and get the one-key-search cost. 7 | It can be considered that the less cost the better performance in slimtrie, map and Btree. 8 | 9 | There is a [benchmark comparison](benchmark_result.md) report. 10 | Benchmark uses the Go language built-in `map` and the [BTree implementation for Go](https://github.com/google/btree). 11 | 12 | Benchmark and optimization steps are as follows: 13 | 14 | 15 | ## Do benchmark 16 | 17 | 1. `go test -benchmem -bench=. -run=none` 18 | 19 | > Run this command in `slim/trie/` will run key search benchmark of trie, map and BTree. 20 | > It return the one-key search cost of trie, map and Btree. 21 | 22 | 2. `go test -cpuprofile trie.cpu.profiling -benchmem -bench=Trie -run=none` 23 | 24 | > Can also use `-bench=Trie` to run the benchmark of trie search only. 25 | > It return the key search cost of trie. 26 | > And use `-cpuprofile` to get the cpu profiling for optimization. 27 | 28 | 29 | ## Profiling with flamegraph 30 | 31 | You must need a tool to view the profile and optimize. 32 | Flamegraph shows a visualization of profile, which is convenient to observe the profile details. 33 | 34 | *steps to use flamegraph* 35 | 36 | 1. Install FlameGraph 37 | 38 | ``` 39 | git clone https://github.com/brendangregg/FlameGraph.git 40 | 41 | # copy 'FlameGraph/flamegraph.pl' to your `$PATH`, like： 42 | cp FlameGraph/flamegraph.pl /usr/local/bin/ 43 | ``` 44 | 45 | 2. Install a visualization tool to use flamegraph 46 | 47 | There are 2 recommanded tools, you can choose *one of those two*. 48 | 49 | [go-torch](https://github.com/uber/go-torch) is a tool to create flamegraph with profiling. 50 | And flamegraph visualization is also available to `go tool pprof` in `Go 1.11`. 51 | 52 | *pprof* 53 | 54 | `pprof` is recommanded because it is an official tool, so it will be more stable in future. 55 | And, if your golang version is `Go 1.11` or higher, your `go tool pprof` is useful to view flamegraph. 56 | Or you can get the latest `pprof` tool: 57 | ``` 58 | go get -u github.com/google/pprof 59 | ``` 60 | 61 | It also needs [Graphviz](https://www.graphviz.org) to use `pprof` to get an interactive web views of profile, 62 | you should ensure that Graphviz is usable before using `pprof`. 63 | 64 | use `pprof` as: 65 | 66 | ``` 67 | # if your Golang version >= 1.11 68 | go tool pprof -http=":8088" 69 | 70 | # if you use pprof tool 71 | pprof -http=":8088" 72 | ``` 73 | you get an interactive web interface that can be used to navigate through various views, 74 | then chose `view > flamegraph` on your browser web view to use flamegraph. 75 | 76 | *go-torch* 77 | 78 | `go-torch` makes a '.svg' file, if you want to save the 'flamegraph', install it: 79 | 80 | ``` 81 | go get -v github.com/uber/go-torch 82 | ``` 83 | 84 | and use it: 85 | 86 | ``` 87 | go-torch -b -f 88 | ``` 89 | you get the flamegraph of profile, then: 90 | ``` 91 | open 92 | ``` 93 | to view flamegraph. 94 | 95 | 96 | ## Trie search cost 97 | 98 | SlimTrie offers a tool in `slim/tools/app/` to run key search benchmark and output a chart result in 99 | string. It show a better view than `go test -bench`. 100 | 101 | This cammand `go run tools/app/trie_search_cost.go` runs the slimtrie key search benchmark, and then 102 | get a result like: 103 | 104 | ``` 105 | cost of trie search with existing & existent key: 106 | 107 | ┌────────┬────────┬────────────────────┬───────────────────────┐ 108 | │ KeyCnt │ KeyLen │ ExsitingKeyNsPerOp │ NonexsitentKeyNsPerOp │ 109 | ├────────┼────────┼────────────────────┼───────────────────────┤ 110 | │ 1 │ 1024 │ 116 │ 85 │ 111 | │ 10 │ 1024 │ 119 │ 96 │ 112 | │ 100 │ 1024 │ 224 │ 197 │ 113 | │ 1000 │ 1024 │ 222 │ 206 │ 114 | │ 1000 │ 512 │ 272 │ 241 │ 115 | │ 1000 │ 256 │ 222 │ 190 │ 116 | └────────┴────────┴────────────────────┴───────────────────────┘ 117 | ``` 118 | -------------------------------------------------------------------------------- /docs/trie/benchmark_result.md: -------------------------------------------------------------------------------- 1 | # Search performance comparison between trie, map and btree 2 | 3 | ## chart of search cost comparison 4 | 5 | There are smooth line charts drew on the search cost data. The less time cost the better 6 | performance. 7 | 8 | 1. comparison of searching existing key: 9 | 10 | ![search-existing result](charts/search_existing.png) 11 | 12 | 2. comparison of searching nonexistent key: 13 | 14 | ![search-nonexistent result](charts/search_nonexistent.png) 15 | 16 | ## search existing key between trie, map and btree: 17 | 18 | | key length | key count | trie cost (ns) | map cost (ns) | btree cost (ns) | 19 | | ---: | ---: | ---: | ---: | ---: | 20 | | 1024 | 1 | 86.367 | 5.037 | 36.933 | 21 | | 1024 | 10 | 90.733 | 59.767 | 99.533 | 22 | | 1024 | 100 | 123.333 | 60.100 | 240.667 | 23 | | 1024 | 1000 | 157.000 | 63.567 | 389.667 | 24 | | 512 | 1000 | 152.667 | 40.033 | 363.000 | 25 | | 256 | 1000 | 152.333 | 28.833 | 332.333 | 26 | 27 | ## search nonexistend key between trie, map and btree: 28 | 29 | | key length | key count | trie cost (ns) | map cost (ns) | btree cost (ns) | 30 | | ---: | ---: | ---: | ---: | ---: | 31 | | 1024 | 1 | 60.267 | 10.900 | 92.567 | 32 | | 1024 | 10 | 63.667 | 68.733 | 178.000 | 33 | | 1024 | 100 | 100.066 | 71.699 | 297.333 | 34 | | 1024 | 1000 | 134.667 | 68.833 | 441.667 | 35 | | 512 | 1000 | 131.333 | 49.300 | 389.667 | 36 | | 256 | 1000 | 136.333 | 38.300 | 362.667 | 37 | 38 | ## ratio of search cost with existing key between trie, map and btree: 39 | 40 | | key length | key count | trie cost / map cost | trie cost / btree cost | 41 | | ---: | ---: | ---: | ---: | 42 | | 1024 | 1 | 1714.7 % | 233.8 % | 43 | | 1024 | 10 | 151.8 % | 91.2 % | 44 | | 1024 | 100 | 205.2 % | 51.2 % | 45 | | 1024 | 1000 | 246.9 % | 42.3 % | 46 | | 512 | 1000 | 381.4 % | 42.1 % | 47 | | 256 | 1000 | 528.3 % | 45.8 % | 48 | 49 | ## ratio of search cost with nonexistent key between trie, map and btree 50 | 51 | | key length | key count | trie cost / map cost | trie cost / btree cost | 52 | | ---: | ---: | ---: | ---: | 53 | | 1024 | 1 | 552.9 % | 65.1 % | 54 | | 1024 | 10 | 92.6 % | 35.8 % | 55 | | 1024 | 100 | 139.6 % | 33.7 % | 56 | | 1024 | 1000 | 195.6 % | 30.5 % | 57 | | 512 | 1000 | 266.4 % | 33.7 % | 58 | | 256 | 1000 | 355.9 % | 37.6 % | 59 | 60 | ## trie search cost of `less than`, `equal`, `greater than` keys at the same time 61 | 62 | | key length | key count | existing key cost (ns) | nonexistent key cost (ns) | 63 | | ---: | ---: | ---: | ---: | 64 | | 1024 | 1 | 119.000 | 106.667 | 65 | | 1024 | 10 | 182.333 | 179.000 | 66 | | 1024 | 100 | 246.000 | 218.000 | 67 | | 1024 | 1000 | 260.667 | 238.000 | 68 | | 512 | 1000 | 262.000 | 238.333 | 69 | | 256 | 1000 | 248.667 | 207.667 | 70 | 71 | -------------------------------------------------------------------------------- /docs/trie/cpu.profiling: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/docs/trie/cpu.profiling -------------------------------------------------------------------------------- /docs/trie/memusage.md: -------------------------------------------------------------------------------- 1 | | Key Count | Key Length | Value Size | Trie Size (Byte/key) | Map Size (Byte/key) | KV Trie Size (Byte/key) | 2 | | --- | --- | --- | --- | --- | --- | 3 | | 1000 | 32 | 2 | 11.2 | 83.0 | 45.7 | 4 | | 1000 | 64 | 2 | 6.7 | 113.2 | 78.4 | 5 | | 1000 | 256 | 2 | 6.7 | 305.2 | 315.9 | 6 | | 1000 | 512 | 2 | 6.8 | 561.3 | 619.1 | 7 | | 2000 | 32 | 2 | 7.4 | 77.1 | 41.8 | 8 | | 2000 | 64 | 2 | 7.4 | 109.1 | 82.8 | 9 | | 2000 | 256 | 2 | 7.4 | 301.1 | 312.1 | 10 | | 2000 | 512 | 2 | 7.4 | 557.1 | 611.2 | 11 | | 5000 | 32 | 2 | 6.9 | 68.1 | 44.1 | 12 | | 5000 | 64 | 2 | 6.9 | 100.1 | 83.5 | 13 | | 5000 | 256 | 2 | 6.9 | 292.1 | 307.9 | 14 | | 5000 | 512 | 2 | 7.2 | 548.1 | 599.9 | 15 | -------------------------------------------------------------------------------- /docs/version.md: -------------------------------------------------------------------------------- 1 | A newer version `y` being compatible with an older version `x` means `y` can 2 | load data serialized by `x`. But `x` should never try to load data serialized by 3 | a newer version `y`. 4 | 5 | - `v0.5.*` is compatible with `0.2.*`, `0.3.*`, `0.4.*`, `0.5.*`. 6 | - `v0.4.*` is compatible with `0.2.*`, `0.3.*`, `0.4.*`. 7 | - `v0.3.*` is compatible with `0.2.*`, `0.3.*`. 8 | - `v0.2.*` is compatible with `0.2.*`. 9 | -------------------------------------------------------------------------------- /docs/workflow.md: -------------------------------------------------------------------------------- 1 | ## Git branching model 2 | 3 | ```txt 4 | release-1.0 release-1.1 release-2.0 5 | ----------- ----------- ----------- 6 | * (feature-a) 7 | | 8 | * (feature-b) 9 | | 10 | | * (feature-c) 11 | pick |/ 12 | .----------------->* (master release-2.0) 13 | / | 14 | * fixbug-z | 15 | | pick | 16 | | .--------------->* (tag: v2.0.2) 17 | pick | / | 18 | .-----------------|-/----------------->* (tag: v2.0.1) 19 | / |/ | 20 | / * fixbug-y * (tag: v2.0.0) 21 | ' pick | | 22 | | .-------------->* / 23 | | / | .----------------' 24 | | / |/ major ver incr 25 | | / * 26 | |/ | 27 | * fixbug-x * (tag: v1.1.0) 28 | | / 29 | | .----------------' 30 | |/ minor ver incr 31 | * 32 | | 33 | * (tag: v1.0.0) 34 | ``` 35 | 36 | ### Version 37 | 38 | We use [semantic-version](http://semver.org/) for versioning. 39 | In a nutshell a version is in form of: `..`, 40 | such as `1.11.0`, `2.3.4`. 41 | 42 | - `major` increment indicates breaking changes. 43 | - `minor` increment indicates compatible changes. 44 | - `patch` increment indicates bug fix etc. 45 | 46 | These rule does not applies to versions `0.x.x`. 47 | 48 | 49 | ### Tag 50 | 51 | Create a tag `v..` for every version and a tag should be kept for ever. 52 | 53 | 54 | ### Branch 55 | 56 | - `release-x.y` is the branch for versions `x.y.*`. 57 | A `relase-x.y` branch should be kept during its maintaining period for applying bug fix to 58 | it or else. 59 | 60 | - **master** is the latest **stable** branch. 61 | It should point to a `release-x.y` branch. 62 | 63 | - Other branches are feature branches. 64 | A feature branch should only contains changes about one feature or one bug fix. 65 | 66 | 67 | ### Commit and message 68 | 69 | **Keep a commit atomic and complete. 70 | Every commit must pass CI**. 71 | 72 | The `subject` of a git commit is in form of: 73 | `: : what changed...` 74 | 75 | `type-of-change` is one of: 76 | 77 | - api-change 78 | - new-feature 79 | - doc 80 | - refactor 81 | - fixbug 82 | - fixdoc 83 | 84 | `module` is one of sub module or other aspect of change. 85 | There is no strict restriction on it. 86 | 87 | Example: `new-feature: slimtrie: add String() for print`. 88 | 89 | 90 | ## Working flow with git 91 | 92 | 1. Create a feature branch and hack on it. 93 | 94 | A feature branch may have more than one commits. 95 | 96 | Make sure every commit pass CI. 97 | 98 | A feature branch should base on the latest `release-x.y`. 99 | If a feature is introduced to a former `release-x.y`, make sure to 100 | `cherry-pick`(or merge) it to all newer `release-x.y`. 101 | 102 | 1. A `fix` commit(fix a bug of fix doc) should be committed to the first 103 | `release-x.y` branch that introduced the bug. 104 | 105 | And the fixbug commit should be `cherry-pick`(or `merge`) by all newer `releas-x.y`. 106 | 107 | 1. Review a feature branch by more than one members and push it to a `release-x.y` branch. 108 | 109 | 1. Bump version, create a tag for it when a `release-x.y` is ready. 110 | 111 | 112 | ### Push 113 | 114 | - Only `fast-forward` push to `master`, `release-x.y`. 115 | 116 | - feature branch is free to modify its history such as by `rebase` it. 117 | 118 | - Avoid `merge`, use `rebase` instead if possible. -------------------------------------------------------------------------------- /encode/bytes.go: -------------------------------------------------------------------------------- 1 | package encode 2 | 3 | // Bytes converts a byte slice into fixed length slice. 4 | // Result slice length is defined by Bytes.Size . 5 | type Bytes struct { 6 | Size int 7 | } 8 | 9 | // Encode converts byte slice to byte slice. 10 | func (c Bytes) Encode(d interface{}) []byte { 11 | return d.([]byte) 12 | } 13 | 14 | // Decode copies fixed length slice out of source byte slice. 15 | // The returned bytes are NOT copied. 16 | func (c Bytes) Decode(b []byte) (int, interface{}) { 17 | s := b[:c.Size] 18 | return c.Size, s 19 | } 20 | 21 | // GetSize returns the length: c.Size. 22 | func (c Bytes) GetSize(d interface{}) int { 23 | return c.Size 24 | } 25 | 26 | // GetEncodedSize returns c.Size 27 | func (c Bytes) GetEncodedSize(b []byte) int { 28 | return c.Size 29 | } 30 | -------------------------------------------------------------------------------- /encode/bytes_test.go: -------------------------------------------------------------------------------- 1 | package encode_test 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/openacid/slim/encode" 8 | ) 9 | 10 | func TestBytes(t *testing.T) { 11 | 12 | var x encode.Encoder = encode.Bytes{} 13 | _ = x 14 | 15 | cases := []struct { 16 | input []byte 17 | want int 18 | }{ 19 | {[]byte(""), 0}, 20 | {[]byte("a"), 1}, 21 | {[]byte("abc"), 3}, 22 | } 23 | 24 | for i, c := range cases { 25 | m := encode.Bytes{Size: c.want} 26 | l := m.GetSize(c.input) 27 | if l != c.want { 28 | t.Fatalf("%d-th: GetSize: input: %v; want: %v; actual: %v", 29 | i+1, c.input, c.want, l) 30 | } 31 | 32 | rst := m.Encode(c.input) 33 | if len(rst) != c.want { 34 | t.Fatalf("%d-th: encoded len: input: %v; want: %v; actual: %v", 35 | i+1, c.input, c.want, len(rst)) 36 | } 37 | 38 | l = m.GetEncodedSize(rst) 39 | if l != c.want { 40 | t.Fatalf("%d-th: encoded size: input: %v; want: %v; actual: %v", 41 | i+1, c.input, c.want, l) 42 | } 43 | 44 | n, s := m.Decode(rst) 45 | if c.want != n { 46 | t.Fatalf("%d-th: decoded size: input: %v; want: %v; actual: %v", 47 | i+1, c.input, c.want, n) 48 | } 49 | if !reflect.DeepEqual(c.input, s) { 50 | t.Fatalf("%d-th: decode: input: %v; want: %v; actual: %v", 51 | i+1, c.input, c.input, s) 52 | } 53 | 54 | if len(rst) > 0 { 55 | rst[0] = 'x' 56 | if s.([]byte)[0] != 'x' { 57 | t.Fatalf("should be not be copied.") 58 | } 59 | } 60 | 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /encode/dummy.go: -------------------------------------------------------------------------------- 1 | package encode 2 | 3 | // Dummy converts anything to nothing. 4 | type Dummy struct { 5 | Size int 6 | } 7 | 8 | // Encode converts something to empty byte slice. 9 | func (c Dummy) Encode(d interface{}) []byte { 10 | return []byte{} 11 | } 12 | 13 | // Decode always returns nil. 14 | func (c Dummy) Decode(b []byte) (int, interface{}) { 15 | return 0, nil 16 | } 17 | 18 | // GetSize returns 0. 19 | func (c Dummy) GetSize(d interface{}) int { 20 | return 0 21 | } 22 | 23 | // GetEncodedSize returns 0. 24 | func (c Dummy) GetEncodedSize(b []byte) int { 25 | return 0 26 | } 27 | -------------------------------------------------------------------------------- /encode/dummy_test.go: -------------------------------------------------------------------------------- 1 | package encode_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/openacid/slim/encode" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestDummy(t *testing.T) { 11 | 12 | ta := require.New(t) 13 | 14 | m := encode.Dummy{} 15 | 16 | cases := []struct { 17 | input interface{} 18 | want int 19 | }{ 20 | {nil, 0}, 21 | {true, 0}, 22 | {1, 0}, 23 | {int32(1), 0}, 24 | {[]byte(""), 0}, 25 | {[]byte("a"), 0}, 26 | {[]byte("abc"), 0}, 27 | } 28 | 29 | for i, c := range cases { 30 | ta.Equal(c.want, m.GetSize(c.input), "%d-th: case: %+v", i+1, c) 31 | ta.Equal(c.want, m.GetEncodedSize([]byte{1, 2, 3}), "%d-th: case: %+v", i+1, c) 32 | ta.Equal([]byte{}, m.Encode(c.input), "%d-th: case: %+v", i+1, c) 33 | n, got := m.Decode([]byte{}) 34 | ta.Equal(0, n, "%d-th: case: %+v", i+1, c) 35 | ta.Nil(got, "%d-th: case: %+v", i+1, c) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /encode/encoder.go: -------------------------------------------------------------------------------- 1 | // Package encode provides encoding API definition and with several commonly 2 | // used Encoder suchas uint32 and uint64 etc. 3 | package encode 4 | 5 | import ( 6 | "errors" 7 | "reflect" 8 | ) 9 | 10 | var ( 11 | // ErrNotSlice indicates it expects a slice type but not 12 | ErrNotSlice = errors.New("it is not a slice") 13 | // ErrUnknownEltType indicates a type this package does not support. 14 | ErrUnknownEltType = errors.New("element type is unknown") 15 | 16 | // ErrNotFixedSize indicates the size of value of a type can not be 17 | // determined by its type. 18 | // Such slice of interface. 19 | ErrNotFixedSize = errors.New("element type is not fixed size") 20 | ) 21 | 22 | // A Encoder converts one element between serialized byte stream 23 | // and in-memory data structure. 24 | type Encoder interface { 25 | // Convert into serialized byte stream. 26 | Encode(interface{}) []byte 27 | 28 | // Read byte stream and convert it back to typed data. 29 | Decode([]byte) (int, interface{}) 30 | 31 | // GetSize returns the size in byte after encoding v. 32 | // If v is of type this encoder can not encode, it panics. 33 | GetSize(v interface{}) int 34 | 35 | // GetEncodedSize returns size of the encoded value. 36 | // Encoded element may be var-length. 37 | // This function is used to determine element size without the need of 38 | // encoding it. 39 | GetEncodedSize([]byte) int 40 | } 41 | 42 | // EncoderOf returns a `Encoder` implementation for type of `e` 43 | func EncoderOf(e interface{}) (Encoder, error) { 44 | k := reflect.ValueOf(e).Kind() 45 | return EncoderByKind(k) 46 | } 47 | 48 | // GetSliceEltEncoder creates a `Encoder` for type of element in slice `s` 49 | func GetSliceEltEncoder(s interface{}) (Encoder, error) { 50 | sl := reflect.ValueOf(s) 51 | if sl.Kind() != reflect.Slice { 52 | return nil, ErrNotSlice 53 | } 54 | 55 | eltKind := reflect.TypeOf(s).Elem().Kind() 56 | 57 | return EncoderByKind(eltKind) 58 | } 59 | 60 | func EncoderByKind(k reflect.Kind) (Encoder, error) { 61 | var m Encoder 62 | switch k { 63 | case reflect.Uint16: 64 | m = U16{} 65 | case reflect.Uint32: 66 | m = U32{} 67 | case reflect.Uint64: 68 | m = U64{} 69 | default: 70 | return nil, ErrUnknownEltType 71 | } 72 | 73 | return m, nil 74 | } 75 | 76 | // String16 converts uint16 to slice of 2 bytes and back. 77 | type String16 struct{} 78 | 79 | // Encode converts uint16 to slice of 2 bytes. 80 | func (s String16) Encode(d interface{}) []byte { 81 | ss := d.(string) 82 | l := len(ss) 83 | rst := make([]byte, 2, 2+l) 84 | rst[0] = byte(l >> 8) 85 | rst[1] = byte(l) 86 | return append(rst, []byte(ss)...) 87 | } 88 | 89 | // Decode converts slice of 2 bytes to uint16. 90 | // It returns number bytes consumed and an uint16. 91 | func (s String16) Decode(b []byte) (int, interface{}) { 92 | l := int(b[0])<<8 + int(b[1]) 93 | ss := string(b[2 : 2+l]) 94 | return 2 + l, ss 95 | } 96 | 97 | // GetSize returns number of byte required to encode a string. 98 | // It is len(str) + 2; 99 | func (s String16) GetSize(d interface{}) int { 100 | ss := d.(string) 101 | l := len(ss) 102 | return 2 + l 103 | } 104 | 105 | // GetEncodedSize returned size of encoded data. 106 | func (s String16) GetEncodedSize(b []byte) int { 107 | l := int(b[0])<<8 + int(b[1]) 108 | return 2 + l 109 | } 110 | -------------------------------------------------------------------------------- /encode/encoder_test.go: -------------------------------------------------------------------------------- 1 | package encode_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/openacid/slim/encode" 7 | ) 8 | 9 | func TestString16(t *testing.T) { 10 | 11 | cases := []struct { 12 | input string 13 | want int 14 | }{ 15 | {"", 2}, 16 | {"a", 3}, 17 | {"abc", 5}, 18 | } 19 | 20 | m := encode.String16{} 21 | 22 | for i, c := range cases { 23 | rst := m.Encode(c.input) 24 | if len(rst) != c.want { 25 | t.Fatalf("%d-th: encoded len: input: %v; want: %v; actual: %v", 26 | i+1, c.input, c.want, len(rst)) 27 | } 28 | 29 | l := m.GetEncodedSize(rst) 30 | if l != c.want { 31 | t.Fatalf("%d-th: encoded size: input: %v; want: %v; actual: %v", 32 | i+1, c.input, c.want, l) 33 | } 34 | 35 | n, s := m.Decode(rst) 36 | if c.want != n { 37 | t.Fatalf("%d-th: decoded size: input: %v; want: %v; actual: %v", 38 | i+1, c.input, c.want, n) 39 | } 40 | if c.input != s { 41 | t.Fatalf("%d-th: decode: input: %v; want: %v; actual: %v", 42 | i+1, c.input, c.input, s) 43 | } 44 | } 45 | } 46 | 47 | func TestGetEncoder(t *testing.T) { 48 | 49 | cases := []struct { 50 | input interface{} 51 | want encode.Encoder 52 | wanterr error 53 | }{ 54 | { 55 | uint16(0), 56 | encode.U16{}, 57 | nil, 58 | }, 59 | { 60 | uint32(0), 61 | encode.U32{}, 62 | nil, 63 | }, 64 | { 65 | uint64(0), 66 | encode.U64{}, 67 | nil, 68 | }, 69 | { 70 | []int{}, 71 | nil, 72 | encode.ErrUnknownEltType, 73 | }, 74 | { 75 | nil, 76 | nil, 77 | encode.ErrUnknownEltType, 78 | }, 79 | } 80 | 81 | for i, c := range cases { 82 | rst, err := encode.EncoderOf(c.input) 83 | if rst != c.want { 84 | t.Fatalf("%d-th: input: %v; want: %v; actual: %v", 85 | i+1, c.input, c.want, rst) 86 | } 87 | if err != c.wanterr { 88 | t.Fatalf("%d-th: input: %v; wanterr: %v; actual: %v", 89 | i+1, c.input, c.wanterr, err) 90 | } 91 | } 92 | } 93 | 94 | func TestGetSliceEltEncoder(t *testing.T) { 95 | 96 | cases := []struct { 97 | input interface{} 98 | want encode.Encoder 99 | wanterr error 100 | }{ 101 | { 102 | []uint16{}, 103 | encode.U16{}, 104 | nil, 105 | }, 106 | { 107 | []uint32{}, 108 | encode.U32{}, 109 | nil, 110 | }, 111 | { 112 | []uint64{}, 113 | encode.U64{}, 114 | nil, 115 | }, 116 | { 117 | []int{}, 118 | nil, 119 | encode.ErrUnknownEltType, 120 | }, 121 | { 122 | int(1), 123 | nil, 124 | encode.ErrNotSlice, 125 | }, 126 | } 127 | 128 | for i, c := range cases { 129 | rst, err := encode.GetSliceEltEncoder(c.input) 130 | if rst != c.want { 131 | t.Fatalf("%d-th: input: %v; want: %v; actual: %v", 132 | i+1, c.input, c.want, rst) 133 | } 134 | if err != c.wanterr { 135 | t.Fatalf("%d-th: input: %v; wanterr: %v; actual: %v", 136 | i+1, c.input, c.wanterr, err) 137 | } 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /encode/gen.go: -------------------------------------------------------------------------------- 1 | package encode 2 | 3 | // Auto-generated code definition 4 | // Usage: 5 | // go generate ./... 6 | 7 | //go:generate go run gen/impl_gen.go 8 | -------------------------------------------------------------------------------- /encode/gen/impl_gen.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/openacid/genr" 5 | ) 6 | 7 | var implHead = `package encode 8 | 9 | import "encoding/binary" 10 | ` 11 | 12 | var implTemplate = ` 13 | // {{.Name}} converts {{.ValType}} to slice of {{.ValLen}} bytes and back. 14 | type {{.Name}} struct{} 15 | 16 | // Encode converts {{.ValType}} to slice of {{.ValLen}} bytes. 17 | func (c {{.Name}}) Encode(d interface{}) []byte { 18 | b := make([]byte, {{.ValLen}}) 19 | v := {{.EncodeCast}}(d.({{.ValType}})) 20 | binary.LittleEndian.Put{{.Codec}}(b, v) 21 | return b 22 | } 23 | 24 | // Decode converts slice of {{.ValLen}} bytes to {{.ValType}}. 25 | // It returns number bytes consumed and an {{.ValType}}. 26 | func (c {{.Name}}) Decode(b []byte) (int, interface{}) { 27 | 28 | size := int({{.ValLen}}) 29 | s := b[:size] 30 | 31 | d := {{.ValType}}(binary.LittleEndian.{{.Codec}}(s)) 32 | return size, d 33 | } 34 | 35 | // GetSize returns the size in byte after encoding v. 36 | func (c {{.Name}}) GetSize(d interface{}) int { 37 | return {{.ValLen}} 38 | } 39 | 40 | // GetEncodedSize returns {{.ValLen}}. 41 | func (c {{.Name}}) GetEncodedSize(b []byte) int { 42 | return {{.ValLen}} 43 | } 44 | ` 45 | 46 | var testHead = `package encode_test 47 | 48 | import ( 49 | "testing" 50 | 51 | "github.com/openacid/slim/encode" 52 | ) 53 | ` 54 | 55 | var testTemplate = ` 56 | func Test{{.Name}}(t *testing.T) { 57 | 58 | v0 := [8]byte{} 59 | v1 := [8]byte{1} 60 | v1234 := [8]byte{0x34, 0x12} 61 | vneg := [8]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} 62 | 63 | cases := []struct { 64 | input {{.ValType}} 65 | want string 66 | wantsize int 67 | }{ 68 | {0, string(v0[:{{.ValLen}}]), {{.ValLen}}}, 69 | {1, string(v1[:{{.ValLen}}]), {{.ValLen}}}, 70 | {0x1234, string(v1234[:{{.ValLen}}]), {{.ValLen}}}, 71 | {^{{.ValType}}(0), string(vneg[:{{.ValLen}}]), {{.ValLen}}}, 72 | } 73 | 74 | m := encode.{{.Name}}{} 75 | 76 | for i, c := range cases { 77 | rst := m.Encode(c.input) 78 | if string(rst) != c.want { 79 | t.Fatalf("%d-th: input: %v; want: %v; actual: %v", 80 | i+1, c.input, []byte(c.want), rst) 81 | } 82 | 83 | n := m.GetSize(c.input) 84 | if c.wantsize != n { 85 | t.Fatalf("%d-th: input: %v; wantsize: %v; actual: %v", 86 | i+1, c.input, c.wantsize, n) 87 | } 88 | 89 | n = m.GetEncodedSize(rst) 90 | if c.wantsize != n { 91 | t.Fatalf("%d-th: input: %v; wantsize: %v; actual: %v", 92 | i+1, c.input, c.wantsize, n) 93 | } 94 | 95 | n, u64 := m.Decode(rst) 96 | if c.input != u64 { 97 | t.Fatalf("%d-th: decode: input: %v; want: %v; actual: %v", 98 | i+1, c.input, c.input, u64) 99 | } 100 | if c.wantsize != n { 101 | t.Fatalf("%d-th: decoded size: input: %v; want: %v; actual: %v", 102 | i+1, c.input, c.wantsize, n) 103 | } 104 | } 105 | } 106 | ` 107 | 108 | func main() { 109 | 110 | pref := "int" 111 | implfn := pref + ".go" 112 | testfn := pref + "_test.go" 113 | 114 | impls := []interface{}{ 115 | genr.NewIntConfig("U16", "uint16"), 116 | genr.NewIntConfig("U32", "uint32"), 117 | genr.NewIntConfig("U64", "uint64"), 118 | genr.NewIntConfig("I16", "int16"), 119 | genr.NewIntConfig("I32", "int32"), 120 | genr.NewIntConfig("I64", "int64"), 121 | } 122 | 123 | genr.Render(implfn, implHead, implTemplate, impls, []string{"gofmt", "unconvert"}) 124 | genr.Render(testfn, testHead, testTemplate, impls, []string{"gofmt", "unconvert"}) 125 | } 126 | -------------------------------------------------------------------------------- /encode/int8.go: -------------------------------------------------------------------------------- 1 | package encode 2 | 3 | // I8 converts int8 to slice of 1 byte and back. 4 | type I8 struct{} 5 | 6 | // Encode converts int8 to slice of 1 byte. 7 | func (c I8) Encode(d interface{}) []byte { 8 | return []byte{byte(d.(int8))} 9 | } 10 | 11 | // Decode converts slice of 1 byte to int8. 12 | // It returns number bytes consumed and an int8. 13 | func (c I8) Decode(b []byte) (int, interface{}) { 14 | return 1, int8(b[0]) 15 | } 16 | 17 | // GetSize returns the size in byte after encoding v. 18 | func (c I8) GetSize(d interface{}) int { 19 | return 1 20 | } 21 | 22 | // GetEncodedSize returns 2. 23 | func (c I8) GetEncodedSize(b []byte) int { 24 | return 1 25 | } 26 | -------------------------------------------------------------------------------- /encode/int8_test.go: -------------------------------------------------------------------------------- 1 | package encode_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/openacid/slim/encode" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestI8(t *testing.T) { 11 | 12 | ta := require.New(t) 13 | 14 | cases := []struct { 15 | input int8 16 | want string 17 | wantsize int 18 | }{ 19 | {0, string([]byte{0}), 1}, 20 | {1, string([]byte{1}), 1}, 21 | {0x12, string([]byte{0x12}), 1}, 22 | {^int8(0), string([]byte{0xff}), 1}, 23 | } 24 | 25 | m := encode.I8{} 26 | 27 | for _, c := range cases { 28 | rst := m.Encode(c.input) 29 | ta.Equal(c.want, string(rst)) 30 | 31 | n := m.GetSize(c.input) 32 | ta.Equal(c.wantsize, n) 33 | 34 | n = m.GetEncodedSize(rst) 35 | ta.Equal(c.wantsize, n) 36 | 37 | n, u64 := m.Decode(rst) 38 | ta.Equal(c.input, u64) 39 | ta.Equal(c.wantsize, n) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /encode/nativeint.go: -------------------------------------------------------------------------------- 1 | package encode 2 | 3 | import ( 4 | "encoding/binary" 5 | "math/bits" 6 | ) 7 | 8 | // Int converts int to slice of bytes and back. 9 | type Int struct{} 10 | 11 | // Encode converts int to slice of bytes. 12 | func (c Int) Encode(d interface{}) []byte { 13 | size := bits.UintSize / 8 14 | b := make([]byte, size) 15 | v := d.(int) 16 | if size == 4 { 17 | binary.LittleEndian.PutUint32(b, uint32(v)) 18 | } else if size == 8 { 19 | binary.LittleEndian.PutUint64(b, uint64(v)) 20 | } else { 21 | panic("unknown int size") 22 | } 23 | 24 | return b 25 | } 26 | 27 | // Decode converts slice of bytes to int. 28 | // It returns number bytes consumed and an int. 29 | func (c Int) Decode(b []byte) (int, interface{}) { 30 | 31 | size := bits.UintSize / 8 32 | s := b[:size] 33 | 34 | var d int 35 | if size == 4 { 36 | d = int(binary.LittleEndian.Uint32(s)) 37 | } else if size == 8 { 38 | d = int(binary.LittleEndian.Uint64(s)) 39 | } else { 40 | panic("unknown int size") 41 | } 42 | return size, d 43 | } 44 | 45 | // GetSize returns native int size in byte after encoding v. 46 | func (c Int) GetSize(d interface{}) int { 47 | return bits.UintSize / 8 48 | } 49 | 50 | // GetEncodedSize returns native int size. 51 | func (c Int) GetEncodedSize(b []byte) int { 52 | return bits.UintSize / 8 53 | } 54 | -------------------------------------------------------------------------------- /encode/nativeint_test.go: -------------------------------------------------------------------------------- 1 | package encode_test 2 | 3 | import ( 4 | "math/bits" 5 | "testing" 6 | 7 | "github.com/openacid/slim/encode" 8 | ) 9 | 10 | func TestInt(t *testing.T) { 11 | 12 | sz := bits.UintSize / 8 13 | 14 | v0 := [8]byte{} 15 | v1 := [8]byte{1} 16 | v1234 := [8]byte{0x34, 0x12} 17 | vneg := [8]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} 18 | 19 | cases := []struct { 20 | input int 21 | want string 22 | wantsize int 23 | }{ 24 | {0, string(v0[:sz]), sz}, 25 | {1, string(v1[:sz]), sz}, 26 | {0x1234, string(v1234[:sz]), sz}, 27 | {^int(0), string(vneg[:sz]), sz}, 28 | } 29 | 30 | m := encode.Int{} 31 | 32 | for i, c := range cases { 33 | rst := m.Encode(c.input) 34 | if string(rst) != c.want { 35 | t.Fatalf("%d-th: input: %v; want: %v; actual: %v", 36 | i+1, c.input, []byte(c.want), rst) 37 | } 38 | 39 | n := m.GetSize(c.input) 40 | if c.wantsize != n { 41 | t.Fatalf("%d-th: input: %v; wantsize: %v; actual: %v", 42 | i+1, c.input, c.wantsize, n) 43 | } 44 | 45 | n = m.GetEncodedSize(rst) 46 | if c.wantsize != n { 47 | t.Fatalf("%d-th: input: %v; wantsize: %v; actual: %v", 48 | i+1, c.input, c.wantsize, n) 49 | } 50 | 51 | n, u64 := m.Decode(rst) 52 | if c.input != u64 { 53 | t.Fatalf("%d-th: decode: input: %v; want: %v; actual: %v", 54 | i+1, c.input, c.input, u64) 55 | } 56 | if c.wantsize != n { 57 | t.Fatalf("%d-th: decoded size: input: %v; want: %v; actual: %v", 58 | i+1, c.input, c.wantsize, n) 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /encode/type_encoder.go: -------------------------------------------------------------------------------- 1 | package encode 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "reflect" 7 | 8 | "github.com/openacid/errors" 9 | ) 10 | 11 | // defaultEndian is default endian 12 | var defaultEndian = binary.LittleEndian 13 | 14 | // TypeEncoder provides encoding for fixed size types. 15 | // Such as int32 or struct { X int32; Y int64; } 16 | // 17 | // "int" is not a fixed size type: int on different platform has different size, 18 | // 4 or 8 bytes. 19 | // 20 | // "[]int32" is not a fixed size type: the data size is also defined by the 21 | // number of elements. 22 | type TypeEncoder struct { 23 | // Endian defines the byte order to encode a value. 24 | // By default it is binary.LittleEndian 25 | Endian binary.ByteOrder 26 | // Type is the data type to encode. 27 | Type reflect.Type 28 | // Size is the encoded size of this type. 29 | Size int 30 | } 31 | 32 | // NewTypeEncoder creates a *TypeEncoder by a value. 33 | // The value "zero" defines what type this Encoder can deal with and must be a 34 | // fixed size type. 35 | func NewTypeEncoder(zero interface{}) (*TypeEncoder, error) { 36 | return NewTypeEncoderEndian(zero, nil) 37 | } 38 | 39 | // NewTypeEncoderEndian creates a *TypeEncoder with a specified byte order. 40 | // 41 | // "endian" could be binary.LittleEndian or binary.BigEndian. 42 | func NewTypeEncoderEndian(zero interface{}, endian binary.ByteOrder) (*TypeEncoder, error) { 43 | if endian == nil { 44 | endian = defaultEndian 45 | } 46 | m := &TypeEncoder{ 47 | Endian: endian, 48 | Type: reflect.Indirect(reflect.ValueOf(zero)).Type(), 49 | Size: binary.Size(zero), 50 | } 51 | 52 | if m.Size == -1 { 53 | return nil, errors.Wrapf(ErrNotFixedSize, "type: %v", reflect.TypeOf(zero)) 54 | } 55 | if m.Type.Kind() == reflect.Slice { 56 | return nil, errors.Wrapf(ErrNotFixedSize, "slice size is not fixed") 57 | } 58 | 59 | return m, nil 60 | } 61 | 62 | // NewTypeEncoderEndianByType creates a *TypeEncoder for specified type and with a specified byte order. 63 | // 64 | // "endian" could be binary.LittleEndian or binary.BigEndian. 65 | func NewTypeEncoderEndianByType(t reflect.Type, endian binary.ByteOrder) (*TypeEncoder, error) { 66 | v := reflect.New(t) 67 | return NewTypeEncoderEndian(v.Interface(), endian) 68 | } 69 | 70 | // Encode converts a m.Type value to byte slice. 71 | // If a different type value from the one used with NewTypeEncoder passed in, 72 | // it panics. 73 | func (m *TypeEncoder) Encode(d interface{}) []byte { 74 | if reflect.Indirect(reflect.ValueOf(d)).Type() != m.Type { 75 | panic("different type from TypeEncoder.Type") 76 | } 77 | 78 | b := bytes.NewBuffer(make([]byte, 0, m.Size)) 79 | err := binary.Write(b, m.Endian, d) 80 | if err != nil { 81 | // there should not be any error if type is fixed size 82 | panic(err) 83 | } 84 | return b.Bytes() 85 | } 86 | 87 | // Decode converts byte slice to a pointer to Type value. 88 | // It returns number bytes consumed and an Type value in interface{}. 89 | func (m *TypeEncoder) Decode(b []byte) (int, interface{}) { 90 | 91 | b = b[0:m.Size] 92 | v := reflect.New(m.Type) 93 | err := binary.Read(bytes.NewBuffer(b), m.Endian, v.Interface()) 94 | if err != nil { 95 | panic(err) 96 | } 97 | return m.Size, reflect.Indirect(v).Interface() 98 | } 99 | 100 | // GetSize returns m.Size. 101 | func (m *TypeEncoder) GetSize(d interface{}) int { 102 | return m.Size 103 | } 104 | 105 | // GetEncodedSize returns m.Size. 106 | func (m *TypeEncoder) GetEncodedSize(b []byte) int { 107 | return m.Size 108 | } 109 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/openacid/slim 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/golang/protobuf v1.3.1 7 | github.com/google/btree v1.1.2 8 | github.com/kr/pretty v0.3.1 9 | github.com/mattn/go-runewidth v0.0.4 // indirect 10 | github.com/openacid/errors v0.8.1 11 | github.com/openacid/genr v0.1.1 12 | github.com/openacid/low v0.1.21 13 | github.com/openacid/must v0.1.3 14 | github.com/openacid/tablewriter v0.0.0-20190429071406-b14f71081b86 15 | github.com/openacid/testkeys v0.1.6 16 | github.com/openacid/testutil v0.1.3 17 | github.com/stretchr/testify v1.8.1 18 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /index/example_range_test.go: -------------------------------------------------------------------------------- 1 | package index_test 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/openacid/slim/index" 8 | ) 9 | 10 | type RangeData string 11 | 12 | func (d RangeData) Read(offset int64, key string) (string, bool) { 13 | for i := 0; i < 4; i++ { 14 | if int(offset) >= len(d) { 15 | break 16 | } 17 | 18 | kv := strings.Split(string(d)[offset:], ",")[0:2] 19 | if kv[0] == key { 20 | return kv[1], true 21 | } 22 | offset += int64(len(kv[0]) + len(kv[1]) + 2) 23 | 24 | } 25 | return "", false 26 | } 27 | 28 | func Example_indexRanges() { 29 | 30 | // Index ranges instead of keys: 31 | // In this example at most 4 keys shares one index item. 32 | 33 | data := RangeData("Aaron,1,Agatha,1,Al,2,Albert,3,Alexander,5,Alison,8") 34 | 35 | // keyOffsets is a prebuilt index that stores range start, range end and its offset. 36 | keyOffsets := []index.OffsetIndexItem{ 37 | // Aaron +--> 0 38 | // Agatha | 39 | // Al | 40 | // Albert | 41 | 42 | // Alexander +--> 31 43 | // Alison | 44 | 45 | {Key: "Aaron", Offset: 0}, 46 | {Key: "Agatha", Offset: 0}, 47 | {Key: "Al", Offset: 0}, 48 | {Key: "Albert", Offset: 0}, 49 | 50 | {Key: "Alexander", Offset: 31}, 51 | {Key: "Alison", Offset: 31}, 52 | } 53 | 54 | st, err := index.NewSlimIndex(keyOffsets, data) 55 | if err != nil { 56 | panic(err) 57 | } 58 | 59 | v, found := st.RangeGet("Aaron") 60 | fmt.Printf("key: %q\n found: %t\n value: %q\n", "Aaron", found, v) 61 | 62 | v, found = st.RangeGet("Al") 63 | fmt.Printf("key: %q\n found: %t\n value: %q\n", "Al", found, v) 64 | 65 | v, found = st.RangeGet("foo") 66 | fmt.Printf("key: %q\n found: %t\n value: %q\n", "foo", found, v) 67 | 68 | // Output: 69 | // key: "Aaron" 70 | // found: true 71 | // value: "1" 72 | // key: "Al" 73 | // found: true 74 | // value: "2" 75 | // key: "foo" 76 | // found: false 77 | // value: "" 78 | } 79 | -------------------------------------------------------------------------------- /index/example_test.go: -------------------------------------------------------------------------------- 1 | package index_test 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/openacid/slim/index" 8 | ) 9 | 10 | type Data string 11 | 12 | func (d Data) Read(offset int64, key string) (string, bool) { 13 | kv := strings.Split(string(d)[offset:], ",")[0:2] 14 | if kv[0] == key { 15 | return kv[1], true 16 | } 17 | return "", false 18 | } 19 | 20 | func Example() { 21 | 22 | // Accelerate external data accessing (in memory or on disk) by indexing 23 | // them with a SlimTrie: 24 | 25 | // `data` is a sample of some unindexed data. In our example it is a comma 26 | // separated key value series. 27 | // 28 | // In order to let SlimTrie be able to read data, `data` should have 29 | // a `Read` method: 30 | // Read(offset int64, key string) (string, bool) 31 | data := Data("Aaron,1,Agatha,1,Al,2,Albert,3,Alexander,5,Alison,8") 32 | 33 | // keyOffsets is a prebuilt index that stores key and its offset in data accordingly. 34 | keyOffsets := []index.OffsetIndexItem{ 35 | {Key: "Aaron", Offset: 0}, 36 | {Key: "Agatha", Offset: 8}, 37 | {Key: "Al", Offset: 17}, 38 | {Key: "Albert", Offset: 22}, 39 | {Key: "Alexander", Offset: 31}, 40 | {Key: "Alison", Offset: 43}, 41 | } 42 | 43 | // `SlimIndex` is simply a container of SlimTrie and its data. 44 | st, err := index.NewSlimIndex(keyOffsets, data) 45 | if err != nil { 46 | fmt.Println(err) 47 | } 48 | 49 | // Lookup 50 | v, found := st.Get("Alison") 51 | fmt.Printf("key: %q\n found: %t\n value: %q\n", "Alison", found, v) 52 | 53 | v, found = st.Get("foo") 54 | fmt.Printf("key: %q\n found: %t\n value: %q\n", "foo", found, v) 55 | 56 | // Output: 57 | // key: "Alison" 58 | // found: true 59 | // value: "8" 60 | // key: "foo" 61 | // found: false 62 | // value: "" 63 | } 64 | -------------------------------------------------------------------------------- /index/index.go: -------------------------------------------------------------------------------- 1 | // Package index provides a data index structure, contains a SlimTrie 2 | // instance as index and a data provider `DataReader`. 3 | // 4 | // The purpose of `index` is to accelerate accessing external data such as a bunch 5 | // of key-values on disk. 6 | // 7 | // SlimIndex is an index only, but not a full functional kv-map. 8 | // The relationship of SlimIndex and its data just like that of B+ tree internal 9 | // nodes and B+ tree leaf nodes: 10 | // In a B+ tree only leaf nodes store record. Internal nodes only help on 11 | // locating a leaf node then a record. 12 | package index 13 | 14 | import ( 15 | "github.com/openacid/slim/encode" 16 | "github.com/openacid/slim/trie" 17 | ) 18 | 19 | // DataReader defines interface to let SlimIndex access the data it indexes. 20 | type DataReader interface { 21 | // Read value at `offset`, of `key`. 22 | // Because the internal SlimTrie does not store complete info of a key(to 23 | // reduce memory consumption). 24 | // Thus the offset SlimTrie returns might not be correct for an abscent 25 | // record. 26 | // It is data providers' responsibility to check if the record at `offset` 27 | // has the exact `key`. 28 | Read(offset int64, key string) (string, bool) 29 | } 30 | 31 | // OffsetIndexItem defines data types for a offset-based index, such as an index 32 | // of on-disk records. 33 | type OffsetIndexItem struct { 34 | // Key is the record identity. 35 | Key string 36 | // Offset is the position of this record in its storage, E.g. the file offset 37 | // where this record is. 38 | Offset int64 39 | } 40 | 41 | // SlimIndex contains a SlimTrie instance as index and a data provider 42 | // `DataReader`. 43 | type SlimIndex struct { 44 | trie.SlimTrie 45 | DataReader 46 | } 47 | 48 | // NewSlimIndex creates SlimIndex instance. 49 | // 50 | // The keys in `index` must be in ascending order. 51 | func NewSlimIndex(index []OffsetIndexItem, dr DataReader) (*SlimIndex, error) { 52 | 53 | l := len(index) 54 | keys := make([]string, 0, l) 55 | offsets := make([]int64, 0, l) 56 | for i := 0; i < l; i++ { 57 | keys = append(keys, index[i].Key) 58 | offsets = append(offsets, index[i].Offset) 59 | } 60 | 61 | st, err := trie.NewSlimTrie(encode.I64{}, keys, offsets) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | return &SlimIndex{*st, dr}, nil 67 | } 68 | 69 | // Get returns the value of `key` which is found by `SlimIndex.DataReader`, and 70 | // a bool value indicating if the `key` is found or not. 71 | func (si *SlimIndex) Get(key string) (string, bool) { 72 | o, found := si.SlimTrie.Get(key) 73 | if !found { 74 | return "", false 75 | } 76 | 77 | offset := o.(int64) 78 | 79 | return si.DataReader.Read(offset, key) 80 | } 81 | 82 | // RangeGet returns the value of `key` that is contained in a range, 83 | // and a bool value indicating if the `key` is found or not. 84 | func (si *SlimIndex) RangeGet(key string) (string, bool) { 85 | o, found := si.SlimTrie.RangeGet(key) 86 | if !found { 87 | return "", false 88 | } 89 | 90 | offset := o.(int64) 91 | 92 | return si.DataReader.Read(offset, key) 93 | } 94 | -------------------------------------------------------------------------------- /index/index_test.go: -------------------------------------------------------------------------------- 1 | package index_test 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/openacid/slim/index" 8 | ) 9 | 10 | type testIndexData string 11 | 12 | func (d testIndexData) Read(offset int64, key string) (string, bool) { 13 | kv := strings.Split(string(d)[offset:], ",")[0:2] 14 | if kv[0] == key { 15 | return kv[1], true 16 | } 17 | return "", false 18 | } 19 | 20 | func TestSlimIndex(t *testing.T) { 21 | 22 | data := testIndexData("Aaron,1,Agatha,1,Al,2,Albert,3,Alexander,5,Alison,8") 23 | 24 | keyOffsets := []index.OffsetIndexItem{ 25 | {Key: "Aaron", Offset: 0}, 26 | {Key: "Agatha", Offset: 8}, 27 | {Key: "Al", Offset: 17}, 28 | {Key: "Albert", Offset: 22}, 29 | {Key: "Alexander", Offset: 31}, 30 | {Key: "Alison", Offset: 43}, 31 | } 32 | 33 | st, err := index.NewSlimIndex(keyOffsets, data) 34 | if err != nil { 35 | t.Fatalf("expect no error but: %+v", err) 36 | } 37 | 38 | cases := []struct { 39 | input string 40 | want string 41 | wantfound bool 42 | }{ 43 | {"Aaron", "1", true}, 44 | {"Agatha", "1", true}, 45 | {"Al", "2", true}, 46 | {"Albert", "3", true}, 47 | {"Alexander", "5", true}, 48 | {"Alison", "8", true}, 49 | {"foo", "", false}, 50 | {"Alexande", "", false}, 51 | {"Alexander0", "", false}, 52 | {"alexander", "", false}, 53 | } 54 | 55 | for i, c := range cases { 56 | rst, found := st.Get(c.input) 57 | if rst != c.want { 58 | t.Fatalf("%d-th: input: %v; want: %v; actual: %v", 59 | i+1, c.input, c.want, rst) 60 | } 61 | if found != c.wantfound { 62 | t.Fatalf("%d-th: input: %v; wantfound: %v; actual: %v", 63 | i+1, c.input, c.wantfound, found) 64 | } 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /int-types.md: -------------------------------------------------------------------------------- 1 | # Choose appropriate integer types 2 | 3 | ## Signed integer 4 | 5 | Signed int is used as arithmetic data. 6 | Such as size, length, array index etc. 7 | 8 | **Do not use unsigned int if possible**. 9 | Minus operation with unsigned int overflows. 10 | E.g.: 11 | 12 | ```go 13 | var a uint32 = 3 14 | var b uint32 = 5 15 | 16 | fmt.Println(a - b) 17 | // Output: 4294967294 18 | ``` 19 | 20 | - `int` for in-memory size, length. 21 | 22 | - `int64` for large size, offset etc. 23 | 24 | ## Unsigned integer 25 | 26 | Unsigned int is used as non-arithmetic data, such as bitmap, bit mask etc. 27 | 28 | - `uint64` for bitmap etc. 29 | -------------------------------------------------------------------------------- /scripts/build_md.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import os 5 | import jinja2 6 | import subprocess 7 | 8 | def render_j2(tmpl_path, tmpl_vars, output_path): 9 | 10 | def include_file(name): 11 | return jinja2.Markup(loader.get_source(env, name)[0]) 12 | 13 | loader = jinja2.FileSystemLoader(searchpath='./') 14 | env = jinja2.Environment(loader=loader, 15 | undefined=jinja2.StrictUndefined) 16 | env.globals['include_file'] = include_file 17 | template = env.get_template(tmpl_path) 18 | 19 | txt = template.render(tmpl_vars) 20 | 21 | with open(output_path, 'w') as f: 22 | f.write(txt) 23 | 24 | 25 | def command(cmd, *arguments, **options): 26 | 27 | close_fds = options.get('close_fds', True) 28 | cwd = options.get('cwd', None) 29 | shell = options.get('shell', False) 30 | env = options.get('env', None) 31 | if env is not None: 32 | env = dict(os.environ, **env) 33 | stdin = options.get('stdin', None) 34 | 35 | subproc = subprocess.Popen([cmd] + list(arguments), 36 | close_fds=close_fds, 37 | shell=shell, 38 | cwd=cwd, 39 | env=env, 40 | encoding='utf-8', 41 | stdin=subprocess.PIPE, 42 | stdout=subprocess.PIPE, 43 | stderr=subprocess.PIPE, ) 44 | 45 | out, err = subproc.communicate(input=stdin) 46 | 47 | subproc.wait() 48 | 49 | if subproc.returncode != 0: 50 | raise Exception(subproc.returncode, out, err) 51 | 52 | return out 53 | 54 | 55 | if __name__ == "__main__": 56 | # pkg = command('go', 'list', '.') 57 | # name = pkg.strip().split('/')[-1] 58 | name = 'slim' 59 | tmpl_vars = { 60 | "name": name 61 | } 62 | 63 | fns = os.listdir('docs') 64 | for fn in fns: 65 | if fn.endswith('.md.j2'): 66 | render_j2('docs/'+fn, tmpl_vars, fn[:-3]) 67 | -------------------------------------------------------------------------------- /scripts/change-types.yaml: -------------------------------------------------------------------------------- 1 | - api-change 2 | - new-feature 3 | - internal 4 | - doc 5 | - test 6 | - refactor 7 | - fixbug 8 | - fixdoc 9 | -------------------------------------------------------------------------------- /scripts/requirements.txt: -------------------------------------------------------------------------------- 1 | semantic_version>=2.6.0 2 | jinja2>=2.11.0 3 | PyYAML>=5.3.0 4 | -------------------------------------------------------------------------------- /tools/reports/memusage/memusage.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/openacid/low/size" 7 | "github.com/openacid/slim/benchhelper" 8 | "github.com/openacid/slim/encode" 9 | "github.com/openacid/slim/trie" 10 | ) 11 | 12 | func main() { 13 | compareTrieMapMemUse() 14 | } 15 | 16 | func compareTrieMapMemUse() { 17 | 18 | writeTableHeader() 19 | 20 | for _, n := range []int{1000, 2000, 5000} { 21 | for _, k := range []int{64, 256, 1024} { 22 | 23 | trieSize := getTrieMem(n, k) 24 | mapSize := getMapMem(n, k) 25 | // make key + value as a value in trie 26 | // kvTrieSize := getKVTrieMem(cnt, l) 27 | kvTrieSize := getKVTrieMem2(n, k) 28 | 29 | mapAvg := float64(mapSize) / float64(n) 30 | trieAvg := float64(trieSize) / float64(n) 31 | kvTrieAvg := float64(kvTrieSize) / float64(n) 32 | 33 | writeTableRow(n, k, 2, trieAvg, mapAvg, kvTrieAvg) 34 | } 35 | } 36 | } 37 | 38 | func writeTableHeader() { 39 | 40 | fmt.Printf("| %s | %s | %s | %s | %s | %s |\n", 41 | "Key Count", "Key Length", "Value Size", "Trie Size (Byte/key)", "Map Size (Byte/key)", 42 | "KV Trie Size (Byte/key)") 43 | 44 | fmt.Printf("| --- | --- | --- | --- | --- | --- |\n") 45 | } 46 | 47 | func writeTableRow(cnt, kLen, vLen int, trieAvg, mapAvg, kvTrieAvg float64) { 48 | 49 | fmt.Printf("| %5d | %5d | %5d | %6.1f | %6.1f | %6.1f |\n", 50 | cnt, kLen, vLen, trieAvg, mapAvg, kvTrieAvg) 51 | } 52 | 53 | func getTrieMem(keyCnt, keyLen int) int64 { 54 | 55 | keys := benchhelper.RandSortedStrings(keyCnt, keyLen, nil) 56 | vals := make([]uint16, keyCnt) 57 | for i := range vals { 58 | vals[i] = uint16(i) 59 | } 60 | 61 | t, err := trie.NewSlimTrie(encode.U16{}, keys, vals) 62 | if err != nil { 63 | panic(err) 64 | } 65 | 66 | return int64(size.Of(t)) 67 | } 68 | 69 | func getKVTrieMem2(keyCnt, keyLen int) int64 { 70 | // make key + value as a value in trie 71 | 72 | keys := benchhelper.RandSortedStrings(keyCnt, keyLen, nil) 73 | indexes := make([]uint32, keyCnt) 74 | for i := 0; i < len(keys); i++ { 75 | indexes[i] = uint32(i) 76 | } 77 | 78 | t, err := trie.NewSlimTrie(encode.U32{}, keys, indexes) 79 | if err != nil { 80 | panic(err) 81 | } 82 | 83 | return int64(size.Of(t)) 84 | } 85 | 86 | func getMapMem(keyCnt, keyLen int) int64 { 87 | 88 | keys := benchhelper.RandSortedStrings(keyCnt, keyLen, nil) 89 | vals := make([]uint16, keyCnt) 90 | 91 | m := make(map[string]uint16, len(keys)) 92 | 93 | for i := 0; i < len(keys); i++ { 94 | m[keys[i]] = vals[i] 95 | } 96 | 97 | return int64(size.Of(m)) 98 | } 99 | -------------------------------------------------------------------------------- /trie/benchmark_btree_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/btree" 7 | "github.com/openacid/low/mathext/zipf" 8 | ) 9 | 10 | type KVElt struct { 11 | Key string 12 | Val int32 13 | } 14 | 15 | func (kv *KVElt) Less(than btree.Item) bool { 16 | o := than.(*KVElt) 17 | return kv.Key < o.Key 18 | } 19 | 20 | func makeKVElts(srcKeys []string, srcVals []int32) []*KVElt { 21 | elts := make([]*KVElt, len(srcKeys)) 22 | for i, k := range srcKeys { 23 | elts[i] = &KVElt{Key: k, Val: srcVals[i]} 24 | } 25 | return elts 26 | } 27 | 28 | var OutputBtree int 29 | 30 | func Benchmark_btree(b *testing.B) { 31 | 32 | benchBigKeySet(b, func(b *testing.B, typ string, keys []string) { 33 | 34 | values := makeI32s(len(keys)) 35 | bt := btree.New(32) 36 | elts := makeKVElts(keys, values) 37 | for _, v := range elts { 38 | bt.ReplaceOrInsert(v) 39 | } 40 | 41 | accesses := zipf.Accesses(2, 1.5, len(keys), b.N, nil) 42 | 43 | b.ResetTimer() 44 | 45 | var id int32 46 | for i := 0; i < b.N; i++ { 47 | idx := accesses[i] 48 | itm := &KVElt{Key: keys[idx], Val: values[idx]} 49 | ee := bt.Get(itm) 50 | id += ee.(*KVElt).Val 51 | 52 | } 53 | OutputBtree = int(id) 54 | }) 55 | 56 | } 57 | -------------------------------------------------------------------------------- /trie/bitmap.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import "github.com/openacid/low/bitmap" 4 | 5 | func newBM(indexes []int32, capa int32, opts ...string) *Bitmap { 6 | bb := &Bitmap{ 7 | Words: bitmap.Of(indexes, capa), 8 | } 9 | bb.indexit(opts...) 10 | return bb 11 | } 12 | 13 | func (b *Bitmap) indexit(opts ...string) { 14 | for _, opt := range opts { 15 | switch opt { 16 | case "r64": 17 | b.RankIndex = bitmap.IndexRank64(b.Words) 18 | case "r128": 19 | b.RankIndex = bitmap.IndexRank128(b.Words) 20 | case "s32": 21 | // select32 also requires rank index to locate a bit 22 | b.SelectIndex, b.RankIndex = bitmap.IndexSelect32R64(b.Words) 23 | default: 24 | panic("unknown " + opt) 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /trie/bm17with2bit_plot.txt: -------------------------------------------------------------------------------- 1 | most used bitmap count 2 | 3 | bitmap17 with 2 bit 4 | 5 | 4500 +-------------------------------------------------------------------+ 6 | | + + + + + + | 7 | 4000 |-+ ++++++....+-| 8 | | ++++++ | 9 | 3500 |-+ ++++++ +-| 10 | | +++++ | 11 | | +++++ | 12 | 3000 |-+ ++++ +-| 13 | | ++++ | 14 | 2500 |-+ ++++ +-| 15 | | +++++ | 16 | 2000 |-+ ++++ +-| 17 | | ++++ | 18 | 1500 |-+ ++++ +-| 19 | | ++++ | 20 | | ++++ | 21 | 1000 |-+ +++ +-| 22 | | +++ | 23 | 500 |-++ +-| 24 | |+ + + + + + + | 25 | 0 +-------------------------------------------------------------------+ 26 | 0 20 40 60 80 100 120 140 27 | 28 | 29 | 30 | 300000 +-----------------------------------------------------------------+ 31 | | + + + + + | 32 | | ++++++++ | 33 | 250000 |-+ ++++++++++ +-| 34 | | ++++++++++ | 35 | | +++++++++ | 36 | | +++++ | 37 | 200000 |-+ +++++ +-| 38 | | ++++ | 39 | | ++++ | 40 | 150000 |-+ ++++ +-| 41 | | +++ | 42 | | +++ | 43 | 100000 |-+ + +-| 44 | | + | 45 | | + | 46 | | + | 47 | 50000 |++ +-| 48 | |+ | 49 | | + + + + + | 50 | 0 +-----------------------------------------------------------------+ 51 | 0 20 40 60 80 100 120 52 | -------------------------------------------------------------------------------- /trie/docs/README.md: -------------------------------------------------------------------------------- 1 | # Illustration of Reducing a Trie to SlimTrie 2 | 3 | ## Steps 4 | 5 | ### Bitrie 6 | 7 | ![](imgs/bitrie.jpg) 8 | 9 | ### Initialize Trie 10 | 11 | ![](imgs/slim-init.jpg) 12 | 13 | 14 | ### Reduce Leaf Nodes 15 | 16 | ![](imgs/slim-cut-leaf.jpg) 17 | 18 | 19 | ### Reduce Inner Single Branch 20 | 21 | ![](imgs/slim-cut-inner.jpg) 22 | 23 | 24 | ### Remove Skip from Leaf Nodes 25 | 26 | ![](imgs/slim-final.jpg) 27 | 28 | 29 | ### Remove Leaf Nodes 30 | 31 | ![](imgs/slim-no-leaf.jpg) 32 | 33 | ## Update 34 | 35 | ``` 36 | cd imgs 37 | ``` 38 | 39 | Edit `.dot` files 40 | 41 | ``` 42 | make clean 43 | make 44 | ``` 45 | 46 | 47 | ## Dependency 48 | 49 | - graphviz 50 | -------------------------------------------------------------------------------- /trie/docs/imgs/Makefile: -------------------------------------------------------------------------------- 1 | # convert dot to jpg 2 | 3 | DOTs = $(wildcard *.dot) 4 | JPGs = $(DOTs:.dot=.jpg) 5 | 6 | all: $(JPGs) 7 | 8 | %.jpg:%.dot 9 | dot -Tjpg -Gsize=3,10\! -Gdpi=100 $< -o $@ 10 | 11 | clean: 12 | rm $(JPGs) 13 | -------------------------------------------------------------------------------- /trie/docs/imgs/bitrie.dot: -------------------------------------------------------------------------------- 1 | digraph bitrie 2 | { 3 | graph [ranksep="0.05"]; 4 | 5 | node [shape=circle, style=filled, fillcolor="white", fixedsize=true] 6 | edge [arrowhead=none] 7 | 8 | root [label="''"] 9 | 10 | root -> abd [label="0.{2}", fontcolor=red] 11 | abd -> abde [label=0] 12 | abde -> abdef [label=0] 13 | abde -> abdeg [label=1] 14 | abd -> abdfg [label="1"] 15 | root -> b1 [label="1.{1}", fontcolor=red] 16 | b1 -> b123 [label="0"] 17 | b1 -> b14 [label=1] 18 | 19 | abd [label="0.."] 20 | abde [label="0..0"] 21 | abdef [label="0..00"] 22 | abdeg [label="0..01"] 23 | abdfg [label="0..1"] 24 | b1 [label="1."] 25 | b123 [label="1.0"] 26 | b14 [label="1.1"] 27 | } 28 | -------------------------------------------------------------------------------- /trie/docs/imgs/bitrie.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/docs/imgs/bitrie.jpg -------------------------------------------------------------------------------- /trie/docs/imgs/slim-cut-inner.dot: -------------------------------------------------------------------------------- 1 | digraph slim_cut_inner 2 | { 3 | graph [ranksep="0.05"]; 4 | 5 | node [shape=circle, style=filled, fillcolor="white", fixedsize=true] 6 | edge [arrowhead=none] 7 | 8 | root [label="''"] 9 | 10 | abd_leaf [fillcolor="grey", style=filled, label="", width=0.2] 11 | abdef_leaf [fillcolor="grey", style=filled, label="", width=0.2] 12 | abdeg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 13 | abdfg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 14 | b123_leaf [fillcolor="grey", style=filled, label="", width=0.2] 15 | b14_leaf [fillcolor="grey", style=filled, label="", width=0.2] 16 | 17 | 18 | root -> abd [label="a.{2}", fontcolor=red] 19 | abd -> abd_leaf [label="$"] 20 | abd -> abde [label=e] 21 | abde -> abdef [label=f] 22 | abdef -> abdef_leaf [label="$"] 23 | abde -> abdeg [label=g] 24 | abdeg -> abdeg_leaf [label="$"] 25 | abd -> abdfg [label="f.{1}", fontcolor=red] 26 | abdfg -> abdfg_leaf [label="$"] 27 | root -> b1 [label="b.{1}", fontcolor=red] 28 | b1 -> b123 [label="2.{1}", fontcolor=red] 29 | b123 -> b123_leaf [label="$"] 30 | b1 -> b14 [label=4] 31 | b14 -> b14_leaf [label="$"] 32 | 33 | abd [label="a..", fillcolor="#bbffbb"] 34 | abde [label="a..e"] 35 | abdef [label="a..ef"] 36 | abdeg [label="a..eg"] 37 | abdfg [label="a..f."] 38 | b1 [label="b.", fillcolor="#bbffbb"] 39 | b123 [label="b.2."] 40 | b14 [label="b.4"] 41 | } 42 | -------------------------------------------------------------------------------- /trie/docs/imgs/slim-cut-inner.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/docs/imgs/slim-cut-inner.jpg -------------------------------------------------------------------------------- /trie/docs/imgs/slim-cut-leaf.dot: -------------------------------------------------------------------------------- 1 | digraph slim_cut_leaf 2 | { 3 | graph [ranksep="0.05"]; 4 | 5 | node [shape=circle, style=filled, fillcolor="white", fixedsize=true] 6 | edge [arrowhead=none] 7 | 8 | root [label="''"] 9 | 10 | abd_leaf [fillcolor="grey", style=filled, label="", width=0.2] 11 | abdef_leaf [fillcolor="grey", style=filled, label="", width=0.2] 12 | abdeg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 13 | abdfg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 14 | b123_leaf [fillcolor="grey", style=filled, label="", width=0.2] 15 | b14_leaf [fillcolor="grey", style=filled, label="", width=0.2] 16 | 17 | 18 | root -> a [label=a] 19 | a -> ab [label=b] 20 | ab -> abd [label=d] 21 | abd -> abd_leaf [label="$"] 22 | abd -> abde [label=e] 23 | abde -> abdef [label=f] 24 | abdef -> abdef_leaf [label="$"] 25 | abde -> abdeg [label=g] 26 | abdeg -> abdeg_leaf [label="$"] 27 | abd -> abdfg [label="f.{1}", fontcolor=red] 28 | abdfg -> abdfg_leaf [label="$"] 29 | root -> b [label=b] 30 | b -> b1 [label=1] 31 | b1 -> b123 [label="2.{1}", fontcolor=red] 32 | b123 -> b123_leaf [label="$"] 33 | b1 -> b14 [label=4] 34 | b14 -> b14_leaf [label="$"] 35 | 36 | abdfg [label="abdf.", fillcolor="#bbffbb"] 37 | b123 [label="b12.", fillcolor="#bbffbb"] 38 | } 39 | -------------------------------------------------------------------------------- /trie/docs/imgs/slim-cut-leaf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/docs/imgs/slim-cut-leaf.jpg -------------------------------------------------------------------------------- /trie/docs/imgs/slim-final.dot: -------------------------------------------------------------------------------- 1 | digraph slim_final 2 | { 3 | graph [ranksep="0.05"]; 4 | 5 | node [shape=circle, style=filled, fillcolor="white", fixedsize=true] 6 | edge [arrowhead=none] 7 | 8 | root [label="''"] 9 | 10 | abd_leaf [fillcolor="grey", style=filled, label="", width=0.2] 11 | abdef_leaf [fillcolor="grey", style=filled, label="", width=0.2] 12 | abdeg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 13 | abdfg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 14 | b123_leaf [fillcolor="grey", style=filled, label="", width=0.2] 15 | b14_leaf [fillcolor="grey", style=filled, label="", width=0.2] 16 | 17 | 18 | root -> abd [label="a.{2}", fontcolor=red] 19 | abd -> abd_leaf [label="$"] 20 | abd -> abde [label=e] 21 | abde -> abdef [label=f] 22 | abdef -> abdef_leaf [label="$"] 23 | abde -> abdeg [label=g] 24 | abdeg -> abdeg_leaf [label="$"] 25 | abd -> abdfg [label="f"] 26 | abdfg -> abdfg_leaf [label="$"] 27 | root -> b1 [label="b.{1}", fontcolor=red] 28 | b1 -> b123 [label="2"] 29 | b123 -> b123_leaf [label="$"] 30 | b1 -> b14 [label=4] 31 | b14 -> b14_leaf [label="$"] 32 | 33 | abd [label="a.."] 34 | abde [label="a..e"] 35 | abdef [label="a..ef"] 36 | abdeg [label="a..eg"] 37 | abdfg [label="a..f"] 38 | b1 [label="b."] 39 | b123 [label="b.2"] 40 | b14 [label="b.4"] 41 | } 42 | -------------------------------------------------------------------------------- /trie/docs/imgs/slim-final.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/docs/imgs/slim-final.jpg -------------------------------------------------------------------------------- /trie/docs/imgs/slim-init.dot: -------------------------------------------------------------------------------- 1 | digraph slim_init 2 | { 3 | graph [ranksep="0.05"]; 4 | 5 | node [shape=circle, style=filled, fillcolor="white", fixedsize=true] 6 | edge [arrowhead=none] 7 | 8 | root [label="''"] 9 | 10 | abd_leaf [fillcolor="grey", style=filled, label="", width=0.2] 11 | abdef_leaf [fillcolor="grey", style=filled, label="", width=0.2] 12 | abdeg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 13 | abdfg_leaf [fillcolor="grey", style=filled, label="", width=0.2] 14 | b123_leaf [fillcolor="grey", style=filled, label="", width=0.2] 15 | b14_leaf [fillcolor="grey", style=filled, label="", width=0.2] 16 | 17 | 18 | root -> a [label=a] 19 | a -> ab [label=b] 20 | ab -> abd [label=d] 21 | abd -> abd_leaf [label="$"] 22 | abd -> abde [label=e] 23 | abde -> abdef [label=f] 24 | abdef -> abdef_leaf [label="$"] 25 | abde -> abdeg [label=g] 26 | abdeg -> abdeg_leaf [label="$"] 27 | abd -> abdf [label=f] 28 | abdf -> abdfg [label=g] 29 | abdfg -> abdfg_leaf [label="$"] 30 | root -> b [label=b] 31 | b -> b1 [label=1] 32 | b1 -> b12 [label=2] 33 | b12 -> b123 [label=3] 34 | b123 -> b123_leaf [label="$"] 35 | b1 -> b14 [label=4] 36 | b14 -> b14_leaf [label="$"] 37 | } 38 | -------------------------------------------------------------------------------- /trie/docs/imgs/slim-init.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/docs/imgs/slim-init.jpg -------------------------------------------------------------------------------- /trie/docs/imgs/slim-no-leaf.dot: -------------------------------------------------------------------------------- 1 | digraph slim_no_leaf 2 | { 3 | graph [ranksep="0.05"]; 4 | 5 | node [shape=circle, style=filled, fillcolor="white", fixedsize=true] 6 | edge [arrowhead=none] 7 | 8 | root [label="''"] 9 | 10 | root -> abd [label="a.{2}", fontcolor=red] 11 | abd -> abde [label=e] 12 | abde -> abdef [label=f] 13 | abde -> abdeg [label=g] 14 | abd -> abdfg [label="f"] 15 | root -> b1 [label="b.{1}", fontcolor=red] 16 | b1 -> b123 [label="2"] 17 | b1 -> b14 [label=4] 18 | 19 | abd [label="a.."] 20 | abde [label="a..e"] 21 | abdef [label="a..ef"] 22 | abdeg [label="a..eg"] 23 | abdfg [label="a..f"] 24 | b1 [label="b."] 25 | b123 [label="b.2"] 26 | b14 [label="b.4"] 27 | } 28 | -------------------------------------------------------------------------------- /trie/docs/imgs/slim-no-leaf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/docs/imgs/slim-no-leaf.jpg -------------------------------------------------------------------------------- /trie/docs/imgs/trie_after_squash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/docs/imgs/trie_after_squash.png -------------------------------------------------------------------------------- /trie/docs/imgs/trie_before_squash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/docs/imgs/trie_before_squash.png -------------------------------------------------------------------------------- /trie/docs/squash.md: -------------------------------------------------------------------------------- 1 | ### How Node.squash works 2 | 3 | #### How trie looks like 4 | 5 | There is an example trie of 6-keys: 6 | 7 | keys: 8 | ``` 9 | abd 10 | abdef 11 | abdeg 12 | abdfg 13 | b123 14 | b14 15 | ``` 16 | 17 | trie: 18 | 19 | **leaf-Node** use a dark color, **mid-Node** use a light color. 20 | 21 | ![trie before squash](imgs/slim-init.jpg) 22 | 23 | There are some nodes that are both leaf-Node and mid-Node. 24 | 25 | To simplify, we add an extra '$' branch used to lead to a value in our trie implement, 26 | thus we can divided nodes into 2 no-intersection kinds: 27 | 28 | **leaf-Node** are those Nodes that have no child. leaf-Node must have value. 29 | **mid-Node** are those Nodes that have at least 1 child. mid-Node must not have value. 30 | 31 | What we can find out: 32 | 33 | 1. 1 leaf-Node represents 1 value, vice versa. 34 | 2. leaf-Node's father node must have one and only one '$' branch, which leads to lead-Node. 35 | 36 | 37 | #### Which can be squashed 38 | 39 | As the trie graph shows, there are some nodes seems not needed when you search an existing key as 40 | the trie path. 41 | 42 | Those 1-child nodes seems can be squashed, but there is a special case. 43 | 44 | If the '$' branch be squashed, there may be 1 mid-Node got 2 leaf-Node, 45 | that is not crroct because 1 key only have 1 value. 46 | So we got that **'$' branch can not be squashed**. 47 | 48 | Then we got which can be squashed: 49 | 50 | **mid-Node that have only 1 branch and this branch is not a '$' branch** 51 | 52 | After squashed, we also need to record how many nodes squashed, then you can know how many nodes 53 | should be skip before you going to the next node. 54 | 55 | there is the trie after squashed: 56 | 57 | ![trie after squash](imgs/slim-final.jpg) 58 | 59 | -------------------------------------------------------------------------------- /trie/errors.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import "errors" 4 | 5 | var ( 6 | 7 | // ErrKeyOutOfOrder means keys to create Trie are not ascendingly ordered. 8 | ErrKeyOutOfOrder = errors.New("keys not ascending sorted") 9 | 10 | // ErrIncompatible means it is trying to unmarshal data from an incompatible 11 | // version. 12 | ErrIncompatible = errors.New("incompatible with marshaled data") 13 | ) 14 | -------------------------------------------------------------------------------- /trie/example_mem_usage_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/openacid/low/size" 7 | "github.com/openacid/slim/encode" 8 | ) 9 | 10 | var ( 11 | m = encode.String16{} 12 | ) 13 | 14 | func Example_memoryUsage() { 15 | 16 | data := make([]byte, 0) 17 | 18 | keys := getKeys("50kl10") 19 | offsets := []uint16{} 20 | 21 | for i, k := range keys { 22 | v := fmt.Sprintf("is the %d-th word", i) 23 | 24 | offsets = append(offsets, uint16(len(data))) 25 | 26 | data = append(data, m.Encode(k)...) 27 | data = append(data, m.Encode(v)...) 28 | 29 | } 30 | 31 | vsize := 2.0 32 | 33 | st, _ := NewSlimTrie(encode.U16{}, keys, offsets) 34 | ksize := size.Of(keys) 35 | 36 | sz := size.Of(st) 37 | 38 | avgIdxLen := float64(sz)/float64(len(keys)) - vsize 39 | avgKeyLen := float64(ksize) / float64(len(keys)) 40 | 41 | ratio := avgIdxLen / avgKeyLen * 100 42 | 43 | fmt.Printf( 44 | "Orignal:: %.1f byte/key --> SlimTrie index: %.1f byte/index\n"+ 45 | "Saved %.1f%%", 46 | avgKeyLen, 47 | avgIdxLen, 48 | 100-ratio, 49 | ) 50 | 51 | } 52 | -------------------------------------------------------------------------------- /trie/example_scan_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/openacid/slim/encode" 7 | ) 8 | 9 | func ExampleSlimTrie_ScanFrom() { 10 | var keys = []string{ 11 | "", 12 | "`", 13 | "a", 14 | "ab", 15 | "abc", 16 | "abca", 17 | "abcd", 18 | "abcd1", 19 | "abce", 20 | "be", 21 | "c", 22 | "cde0", 23 | "d", 24 | } 25 | values := makeI32s(len(keys)) 26 | 27 | codec := encode.I32{} 28 | st, _ := NewSlimTrie(codec, keys, values, Opt{ 29 | Complete: Bool(true), 30 | }) 31 | 32 | // untilD stops when encountering "d". 33 | untilD := func(k, v []byte) bool { 34 | if string(k) == "d" { 35 | return false 36 | } 37 | 38 | _, i32 := codec.Decode(v) 39 | fmt.Println(string(k), i32) 40 | return true 41 | } 42 | 43 | fmt.Println("scan (ab, +∞):") 44 | st.ScanFrom("ab", false, true, untilD) 45 | 46 | fmt.Println() 47 | fmt.Println("scan [be, +∞):") 48 | st.ScanFrom("be", true, true, untilD) 49 | 50 | fmt.Println() 51 | fmt.Println("scan (ab, be):") 52 | st.ScanFromTo( 53 | "ab", false, 54 | "be", false, 55 | true, untilD) 56 | 57 | // Output: 58 | // 59 | // scan (ab, +∞): 60 | // abc 4 61 | // abca 5 62 | // abcd 6 63 | // abcd1 7 64 | // abce 8 65 | // be 9 66 | // c 10 67 | // cde0 11 68 | // 69 | // scan [be, +∞): 70 | // be 9 71 | // c 10 72 | // cde0 11 73 | // 74 | // scan (ab, be): 75 | // abc 4 76 | // abca 5 77 | // abcd 6 78 | // abcd1 7 79 | // abce 8 80 | } 81 | -------------------------------------------------------------------------------- /trie/example_slimtrie_range_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/openacid/slim/encode" 7 | ) 8 | 9 | func ExampleSlimTrie_RangeGet() { 10 | 11 | // To index a map of key range to value with SlimTrie is very simple: 12 | // 13 | // Gives a set of key the same value, and use RangeGet() instead of Get(). 14 | // SlimTrie does not store branches for adjacent leaves with the same value. 15 | 16 | keys := []string{ 17 | "abc", 18 | "abcd", 19 | 20 | "bc", 21 | 22 | "bcd", 23 | "bce", 24 | } 25 | values := []int{ 26 | 1, 1, 27 | 2, 28 | 3, 3, 29 | } 30 | st, err := NewSlimTrie(encode.Int{}, keys, values) 31 | if err != nil { 32 | panic(err) 33 | } 34 | 35 | cases := []struct { 36 | key string 37 | msg string 38 | }{ 39 | {"ab", "out of range"}, 40 | 41 | {"abc", "in range"}, 42 | {"abc1", "FALSE POSITIVE"}, 43 | {"abc2", "FALSE POSITIVE"}, 44 | {"abcd", "in range"}, 45 | 46 | {"abcde", "FALSE POSITIVE: a suffix of abcd"}, 47 | 48 | {"acc", "FALSE POSITIVE"}, 49 | 50 | {"bc", "in single key range [bc]"}, 51 | {"bc1", "FALSE POSITIVE"}, 52 | 53 | {"bcd1", "FALSE POSITIVE"}, 54 | 55 | // {"def", "FALSE POSITIVE"}, 56 | } 57 | 58 | for _, c := range cases { 59 | v, found := st.RangeGet(c.key) 60 | fmt.Printf("%-10s %-5v %-5t: %s\n", c.key, v, found, c.msg) 61 | } 62 | 63 | // Output: 64 | // ab false: out of range 65 | // abc 1 true : in range 66 | // abc1 1 true : FALSE POSITIVE 67 | // abc2 1 true : FALSE POSITIVE 68 | // abcd 1 true : in range 69 | // abcde 1 true : FALSE POSITIVE: a suffix of abcd 70 | // acc 1 true : FALSE POSITIVE 71 | // bc 2 true : in single key range [bc] 72 | // bc1 2 true : FALSE POSITIVE 73 | // bcd1 3 true : FALSE POSITIVE 74 | } 75 | -------------------------------------------------------------------------------- /trie/example_string_value_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | fmt "fmt" 5 | 6 | "github.com/openacid/slim/encode" 7 | ) 8 | 9 | func Example_stringValue() { 10 | keys := []string{"foo", "hello"} 11 | values := []string{"bar", "world"} 12 | 13 | st, err := NewSlimTrie(encode.String16{}, keys, values) 14 | if err != nil { 15 | panic(err) 16 | } 17 | 18 | fmt.Println(st.Get("hello")) 19 | fmt.Println(st.Get("foo")) 20 | fmt.Println(st.Get("unknown")) 21 | 22 | // Output: 23 | // 24 | // world true 25 | // bar true 26 | // false 27 | } 28 | -------------------------------------------------------------------------------- /trie/gen_report.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | // generate report 4 | // Usage: 5 | // go generate ./... 6 | 7 | //go:generate go run report/report.go 8 | -------------------------------------------------------------------------------- /trie/report/bench_get_absent_scan.data: -------------------------------------------------------------------------------- 1 | key-count k=64 k=128 k=256 2 | 1000 61 51 73 3 | 10000 97 98 97 4 | 100000 99 94 112 5 | 1000000 130 130 130 6 | -------------------------------------------------------------------------------- /trie/report/bench_get_absent_scan.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/report/bench_get_absent_scan.jpg -------------------------------------------------------------------------------- /trie/report/bench_get_absent_scan.md: -------------------------------------------------------------------------------- 1 | | key-count | k=64 | k=128 | k=256 | 2 | |-----------|------|-------|-------| 3 | | 1000 | 61 | 51 | 73 | 4 | | 10000 | 97 | 98 | 97 | 5 | | 100000 | 99 | 94 | 112 | 6 | | 1000000 | 130 | 130 | 130 | 7 | -------------------------------------------------------------------------------- /trie/report/bench_get_absent_zipf.data: -------------------------------------------------------------------------------- 1 | key-count k=64 k=128 k=256 2 | 1000 78 67 45 3 | 10000 93 100 92 4 | 100000 83 85 86 5 | 1000000 141 152 127 6 | -------------------------------------------------------------------------------- /trie/report/bench_get_absent_zipf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/report/bench_get_absent_zipf.jpg -------------------------------------------------------------------------------- /trie/report/bench_get_absent_zipf.md: -------------------------------------------------------------------------------- 1 | | key-count | k=64 | k=128 | k=256 | 2 | |-----------|------|-------|-------| 3 | | 1000 | 78 | 67 | 45 | 4 | | 10000 | 93 | 100 | 92 | 5 | | 100000 | 83 | 85 | 86 | 6 | | 1000000 | 141 | 152 | 127 | 7 | -------------------------------------------------------------------------------- /trie/report/bench_get_present_scan.data: -------------------------------------------------------------------------------- 1 | key-count k=64 k=128 k=256 2 | 1000 65 68 72 3 | 10000 105 106 110 4 | 100000 98 98 101 5 | 1000000 137 139 143 6 | -------------------------------------------------------------------------------- /trie/report/bench_get_present_scan.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/report/bench_get_present_scan.jpg -------------------------------------------------------------------------------- /trie/report/bench_get_present_scan.md: -------------------------------------------------------------------------------- 1 | | key-count | k=64 | k=128 | k=256 | 2 | |-----------|------|-------|-------| 3 | | 1000 | 65 | 68 | 72 | 4 | | 10000 | 105 | 106 | 110 | 5 | | 100000 | 98 | 98 | 101 | 6 | | 1000000 | 137 | 139 | 143 | 7 | -------------------------------------------------------------------------------- /trie/report/bench_get_present_zipf.data: -------------------------------------------------------------------------------- 1 | key-count k=64 k=128 k=256 2 | 1000 89 63 92 3 | 10000 94 110 102 4 | 100000 90 93 108 5 | 1000000 161 146 169 6 | -------------------------------------------------------------------------------- /trie/report/bench_get_present_zipf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/report/bench_get_present_zipf.jpg -------------------------------------------------------------------------------- /trie/report/bench_get_present_zipf.md: -------------------------------------------------------------------------------- 1 | | key-count | k=64 | k=128 | k=256 | 2 | |-----------|------|-------|-------| 3 | | 1000 | 89 | 63 | 92 | 4 | | 10000 | 94 | 110 | 102 | 5 | | 100000 | 90 | 93 | 108 | 6 | | 1000000 | 161 | 146 | 169 | 7 | -------------------------------------------------------------------------------- /trie/report/bench_msab_present_scan.data: -------------------------------------------------------------------------------- 1 | key-count map SlimTrie array Btree 2 | 1000 12 88 129 217 3 | 10000 28 104 143 307 4 | 100000 41 100 176 379 5 | 1000000 62 136 210 432 6 | -------------------------------------------------------------------------------- /trie/report/bench_msab_present_scan.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/report/bench_msab_present_scan.jpg -------------------------------------------------------------------------------- /trie/report/bench_msab_present_scan.md: -------------------------------------------------------------------------------- 1 | | key-count | map | SlimTrie | array | Btree | 2 | |-----------|-----|----------|-------|-------| 3 | | 1000 | 12 | 88 | 129 | 217 | 4 | | 10000 | 28 | 104 | 143 | 307 | 5 | | 100000 | 41 | 100 | 176 | 379 | 6 | | 1000000 | 62 | 136 | 210 | 432 | 7 | -------------------------------------------------------------------------------- /trie/report/bench_msab_present_zipf.data: -------------------------------------------------------------------------------- 1 | key-count map SlimTrie array Btree 2 | 1000 11 67 101 219 3 | 10000 23 112 148 326 4 | 100000 27 91 193 448 5 | 1000000 26 162 269 594 6 | -------------------------------------------------------------------------------- /trie/report/bench_msab_present_zipf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/report/bench_msab_present_zipf.jpg -------------------------------------------------------------------------------- /trie/report/bench_msab_present_zipf.md: -------------------------------------------------------------------------------- 1 | | key-count | map | SlimTrie | array | Btree | 2 | |-----------|-----|----------|-------|-------| 3 | | 1000 | 11 | 67 | 101 | 219 | 4 | | 10000 | 23 | 112 | 148 | 326 | 5 | | 100000 | 27 | 91 | 193 | 448 | 6 | | 1000000 | 26 | 162 | 269 | 594 | 7 | -------------------------------------------------------------------------------- /trie/report/fpr_get.data: -------------------------------------------------------------------------------- 1 | key-count fpr 2 | 1000 0.030% 3 | 10000 0.005% 4 | 100000 0.000% 5 | 1000000 0.000% 6 | -------------------------------------------------------------------------------- /trie/report/fpr_get.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/report/fpr_get.jpg -------------------------------------------------------------------------------- /trie/report/fpr_get.md: -------------------------------------------------------------------------------- 1 | | key-count | fpr | 2 | |-----------|--------| 3 | | 1000 | 0.030% | 4 | | 10000 | 0.005% | 5 | | 100000 | 0.000% | 6 | | 1000000 | 0.000% | 7 | -------------------------------------------------------------------------------- /trie/report/mem_usage.data: -------------------------------------------------------------------------------- 1 | key-count k=64 k=128 k=256 2 | 1000 22 21 19 3 | 10000 12 12 12 4 | 100000 10 10 10 5 | 1000000 11 11 11 6 | -------------------------------------------------------------------------------- /trie/report/mem_usage.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/report/mem_usage.jpg -------------------------------------------------------------------------------- /trie/report/mem_usage.md: -------------------------------------------------------------------------------- 1 | | key-count | k=64 | k=128 | k=256 | 2 | |-----------|------|-------|-------| 3 | | 1000 | 22 | 21 | 19 | 4 | | 10000 | 12 | 12 | 12 | 5 | | 100000 | 10 | 10 | 10 | 6 | | 1000000 | 11 | 11 | 11 | 7 | -------------------------------------------------------------------------------- /trie/report/report.go: -------------------------------------------------------------------------------- 1 | // This app runs trie search benchmark. 2 | // Without `go test -bench`, this app show the benchmark result with a chart, which shows a 3 | // better view and is more convenient to compare the cost change with key length and key count. 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "strings" 10 | 11 | "github.com/openacid/slim/benchhelper" 12 | "github.com/openacid/slim/trie/benchmark" 13 | ) 14 | 15 | var ( 16 | // different key counts for benchmark. 17 | keyCounts = []int{ 18 | 1000, 19 | 10 * 1000, 20 | 100 * 1000, 21 | 1000 * 1000, 22 | } 23 | ) 24 | 25 | var flg *benchhelper.ReportCmdFlag 26 | 27 | func main() { 28 | flg = benchhelper.InitCmdFlag() 29 | 30 | for _, workload := range []string{ 31 | "zipf", "scan", 32 | } { 33 | getPresent(workload) 34 | getAbsent(workload) 35 | getMapSlimArrayBtree(workload) 36 | 37 | } 38 | memOverhead() 39 | fprGet() 40 | } 41 | 42 | func getPresent(workload string) { 43 | 44 | fn := fmt.Sprintf("report/bench_get_present_%s", workload) 45 | 46 | if flg.Bench { 47 | results := benchmark.BenchGet(keyCounts, "present", workload) 48 | benchhelper.WriteTableFiles(fn, results) 49 | } 50 | 51 | if flg.Plot { 52 | script := strings.Join([]string{ 53 | fmt.Sprintf(`fn = "%s.data"`, fn), 54 | `set yr [0:300]`, 55 | `set xlabel 'key-count: n'`, 56 | `set ylabel 'ns/Get() present key' offset 1,0`, 57 | ``, 58 | }, "\n") 59 | script += benchhelper.Fformat.JPGHistogramTiny 60 | script += benchhelper.LineStyles.Green 61 | script += benchhelper.Plot.Histogram 62 | 63 | benchhelper.Fplot(fn+".jpg", script) 64 | } 65 | } 66 | 67 | func getAbsent(workload string) { 68 | 69 | fn := fmt.Sprintf("report/bench_get_absent_%s", workload) 70 | 71 | if flg.Bench { 72 | results := benchmark.BenchGet(keyCounts, "absent", workload) 73 | benchhelper.WriteTableFiles(fn, results) 74 | } 75 | 76 | if flg.Plot { 77 | script := strings.Join([]string{ 78 | fmt.Sprintf(`fn = "%s.data"`, fn), 79 | `set yr [0:300]`, 80 | `set xlabel 'key-count: n'`, 81 | `set ylabel 'ns/Get() absent key' offset 1,0`, 82 | ``, 83 | }, "\n") 84 | script += benchhelper.Fformat.JPGHistogramTiny 85 | script += benchhelper.LineStyles.Green 86 | script += benchhelper.Plot.Histogram 87 | 88 | benchhelper.Fplot(fn+".jpg", script) 89 | } 90 | } 91 | 92 | func getMapSlimArrayBtree(workload string) { 93 | 94 | fn := fmt.Sprintf("report/bench_msab_present_%s", workload) 95 | 96 | if flg.Bench { 97 | results := benchmark.GetMapSlimArrayBtree(keyCounts, workload) 98 | benchhelper.WriteTableFiles(fn, results) 99 | } 100 | 101 | if flg.Plot { 102 | script := strings.Join([]string{ 103 | fmt.Sprintf(`fn = "%s.data"`, fn), 104 | `set yr [0:1000]`, 105 | `set xlabel 'key-count: n'`, 106 | `set ylabel 'ns/Get()' offset 1,0`, 107 | ``, 108 | }, "\n") 109 | script += benchhelper.Fformat.JPGHistogramTiny 110 | script += benchhelper.LineStyles.Blue 111 | script += benchhelper.Plot.Histogram 112 | 113 | benchhelper.Fplot(fn+".jpg", script) 114 | } 115 | } 116 | 117 | func memOverhead() { 118 | if flg.BenchMem { 119 | keyCounts := []int{1000, 10 * 1000, 100 * 1000, 1000 * 1000} 120 | results := benchmark.Mem(keyCounts) 121 | benchhelper.WriteTableFiles("report/mem_usage", results) 122 | } 123 | 124 | if flg.Plot { 125 | script := ` 126 | fn = "report/mem_usage.data" 127 | set yr [0:30] 128 | set xlabel 'key-count: n' 129 | set ylabel 'bits/key' offset 1,0 130 | ` 131 | script += benchhelper.Fformat.JPGHistogramTiny 132 | script += benchhelper.LineStyles.Yellow 133 | script += benchhelper.Plot.Histogram 134 | 135 | benchhelper.Fplot("report/mem_usage.jpg", script) 136 | } 137 | } 138 | 139 | func fprGet() { 140 | 141 | if flg.FPR { 142 | rsts := benchmark.GetFPR(keyCounts) 143 | benchhelper.WriteTableFiles("report/fpr_get", rsts) 144 | } 145 | 146 | if flg.Plot { 147 | script := ` 148 | fn = "report/fpr_get.data" 149 | set yr [0:0.05] 150 | set format y "%g%%" 151 | set xlabel 'key-count: n' 152 | set ylabel 'false positive' 153 | ` 154 | script += benchhelper.Fformat.JPGHistogramTiny 155 | script += benchhelper.LineStyles.Purple 156 | script += benchhelper.Plot.Histogram 157 | 158 | benchhelper.Fplot("report/fpr_get.jpg", script) 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /trie/slimtrie_bench_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/openacid/slim/encode" 7 | ) 8 | 9 | var Outputxxx int32 10 | 11 | func BenchmarkSlimTrie_GetID_20k_vlen10(b *testing.B) { 12 | 13 | keys := getKeys("20kvl10") 14 | values := makeI32s(len(keys)) 15 | st, _ := NewSlimTrie(encode.I32{}, keys, values) 16 | 17 | var id int32 18 | 19 | b.ResetTimer() 20 | 21 | i := b.N 22 | for { 23 | for _, k := range keys { 24 | id += st.GetID(k) 25 | 26 | i-- 27 | if i == 0 { 28 | Outputxxx = id 29 | return 30 | } 31 | } 32 | } 33 | } 34 | 35 | func BenchmarkSlimTrie_withPrefixContent_GetID_20k_vlen10(b *testing.B) { 36 | 37 | keys := getKeys("20kvl10") 38 | values := makeI32s(len(keys)) 39 | st, _ := NewSlimTrie(encode.I32{}, keys, values, 40 | Opt{InnerPrefix: Bool(true)}, 41 | ) 42 | 43 | var id int32 44 | 45 | b.ResetTimer() 46 | 47 | i := b.N 48 | for { 49 | for _, k := range keys { 50 | id += st.GetID(k) 51 | 52 | i-- 53 | if i == 0 { 54 | Outputxxx = id 55 | return 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /trie/slimtrie_complete_bench_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/openacid/low/size" 8 | "github.com/openacid/slim/encode" 9 | ) 10 | 11 | var OutputCompleteGetID20kvl10 int32 12 | 13 | func BenchmarkSlimTrie_Complete_GetID_20k_vlen10(b *testing.B) { 14 | 15 | keys := getKeys("20kvl10") 16 | values := makeI32s(len(keys)) 17 | st, _ := NewSlimTrie(encode.I32{}, keys, values, Opt{Complete: Bool(true)}) 18 | 19 | fmt.Println(size.Stat(st, 10, 2)) 20 | 21 | var id int32 22 | 23 | b.ResetTimer() 24 | 25 | i := b.N 26 | for { 27 | for _, k := range keys { 28 | id += st.GetID(k) 29 | 30 | i-- 31 | if i == 0 { 32 | OutputCompleteGetID20kvl10 = id 33 | return 34 | } 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /trie/slimtrie_getint.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | // GetI8 is same as Get() except it is optimized for int8. 4 | // 5 | // Since 0.5.10 6 | func (st *SlimTrie) GetI8(key string) (int8, bool) { 7 | 8 | eqID := st.GetID(key) 9 | 10 | if eqID == -1 { 11 | return 0, false 12 | } 13 | 14 | ith, _ := st.getLeafIndex(eqID) 15 | 16 | v := int8(st.inner.Leaves.Bytes[ith]) 17 | 18 | return v, true 19 | } 20 | 21 | // GetI16 is same as Get() except it is optimized for int16. 22 | // 23 | // Since 0.5.10 24 | func (st *SlimTrie) GetI16(key string) (int16, bool) { 25 | 26 | eqID := st.GetID(key) 27 | 28 | if eqID == -1 { 29 | return 0, false 30 | } 31 | 32 | ith, _ := st.getLeafIndex(eqID) 33 | stIdx := ith << 1 34 | 35 | b := st.inner.Leaves.Bytes[stIdx : stIdx+2] 36 | 37 | v := int16(b[0]) | int16(b[1])<<8 38 | 39 | return v, true 40 | } 41 | 42 | // GetI32 is same as Get() except it is optimized for int32. 43 | // 44 | // Since 0.5.10 45 | func (st *SlimTrie) GetI32(key string) (int32, bool) { 46 | 47 | eqID := st.GetID(key) 48 | 49 | if eqID == -1 { 50 | return 0, false 51 | } 52 | 53 | ith, _ := st.getLeafIndex(eqID) 54 | stIdx := ith << 2 55 | 56 | b := st.inner.Leaves.Bytes[stIdx : stIdx+4] 57 | 58 | v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16 | int32(b[3])<<24 59 | 60 | return v, true 61 | } 62 | 63 | // GetI64 is same as Get() except it is optimized for int64. 64 | // 65 | // Since 0.5.10 66 | func (st *SlimTrie) GetI64(key string) (int64, bool) { 67 | 68 | eqID := st.GetID(key) 69 | 70 | if eqID == -1 { 71 | return 0, false 72 | } 73 | 74 | ith, _ := st.getLeafIndex(eqID) 75 | stIdx := ith << 3 76 | 77 | b := st.inner.Leaves.Bytes[stIdx : stIdx+8] 78 | 79 | v := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 | int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56 80 | 81 | return v, true 82 | } 83 | -------------------------------------------------------------------------------- /trie/slimtrie_getint_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/openacid/slim/encode" 7 | "github.com/openacid/testutil" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func fibhash64(i uint64) uint64 { 12 | return i * 11400714819323198485 13 | } 14 | 15 | func TestSlimTrie_GetI8(t *testing.T) { 16 | 17 | ta := require.New(t) 18 | 19 | keys := getKeys("20kvl10") 20 | values := make([]int8, len(keys)) 21 | for i := 0; i < len(keys); i++ { 22 | values[i] = int8(fibhash64(uint64(i))) 23 | } 24 | 25 | st, err := NewSlimTrie(encode.I8{}, keys, values) 26 | ta.NoError(err) 27 | 28 | testUnknownKeysGRS(t, st, testutil.RandStrSlice(len(keys)*5, 0, 10)) 29 | 30 | for i, key := range keys { 31 | 32 | // dd("test Get: present: %s", key) 33 | 34 | v, found := st.GetI8(key) 35 | ta.True(found, "Get:%v", key) 36 | ta.Equal(values[i], v, "Get:%v", key) 37 | } 38 | } 39 | 40 | func TestSlimTrie_GetI16(t *testing.T) { 41 | 42 | ta := require.New(t) 43 | 44 | keys := getKeys("20kvl10") 45 | values := make([]int16, len(keys)) 46 | for i := 0; i < len(keys); i++ { 47 | values[i] = int16(fibhash64(uint64(i))) 48 | } 49 | 50 | st, err := NewSlimTrie(encode.I16{}, keys, values) 51 | ta.NoError(err) 52 | 53 | testUnknownKeysGRS(t, st, testutil.RandStrSlice(len(keys)*5, 0, 10)) 54 | 55 | for i, key := range keys { 56 | 57 | // dd("test Get: present: %s", key) 58 | 59 | v, found := st.GetI16(key) 60 | ta.True(found, "Get:%v", key) 61 | ta.Equal(values[i], v, "Get:%v", key) 62 | } 63 | } 64 | 65 | func TestSlimTrie_GetI32(t *testing.T) { 66 | 67 | ta := require.New(t) 68 | 69 | keys := getKeys("20kvl10") 70 | values := make([]int32, len(keys)) 71 | for i := 0; i < len(keys); i++ { 72 | values[i] = int32(fibhash64(uint64(i))) 73 | } 74 | 75 | st, err := NewSlimTrie(encode.I32{}, keys, values) 76 | ta.NoError(err) 77 | 78 | testUnknownKeysGRS(t, st, testutil.RandStrSlice(len(keys)*5, 0, 10)) 79 | 80 | for i, key := range keys { 81 | 82 | // dd("test Get: present: %s", key) 83 | 84 | v, found := st.GetI32(key) 85 | ta.True(found, "Get:%v", key) 86 | ta.Equal(values[i], v, "Get:%v", key) 87 | } 88 | } 89 | 90 | func TestSlimTrie_GetI64(t *testing.T) { 91 | 92 | ta := require.New(t) 93 | 94 | keys := getKeys("20kvl10") 95 | values := make([]int64, len(keys)) 96 | for i := 0; i < len(keys); i++ { 97 | values[i] = int64(fibhash64(uint64(i))) 98 | } 99 | 100 | st, err := NewSlimTrie(encode.I64{}, keys, values) 101 | ta.NoError(err) 102 | 103 | testUnknownKeysGRS(t, st, testutil.RandStrSlice(len(keys)*5, 0, 10)) 104 | 105 | for i, key := range keys { 106 | 107 | // dd("test Get: present: %s", key) 108 | 109 | v, found := st.GetI64(key) 110 | ta.True(found, "Get:%v", key) 111 | ta.Equal(values[i], v, "Get:%v", key) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /trie/slimtrie_getnode.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "math/bits" 5 | 6 | "github.com/openacid/low/bitmap" 7 | ) 8 | 9 | func (st *SlimTrie) getIthInner(ithInner int32, qr *querySession) { 10 | ns := st.inner 11 | vars := st.vars 12 | 13 | innWordI := ithInner >> 6 14 | innBitI := ithInner & 63 15 | 16 | if ithInner < ns.BigInnerCnt { 17 | qr.wordSize = bigWordSize 18 | qr.from = ithInner * bigInnerSize 19 | qr.to = qr.from + bigInnerSize 20 | } else { 21 | qr.wordSize = wordSize 22 | 23 | ithShort := ns.ShortBM.RankIndex[innWordI] + int32(bits.OnesCount64(ns.ShortBM.Words[innWordI]&bitmap.Mask[innBitI])) 24 | 25 | qr.from = vars.BigInnerOffset + innerSize*ithInner + vars.ShortMinusInner*ithShort 26 | 27 | // if this is a short node 28 | if ns.ShortBM.Words[innWordI]&bitmap.Bit[innBitI] != 0 { 29 | 30 | qr.to = qr.from + ns.ShortSize 31 | 32 | j := qr.from & 63 33 | w := ns.Inners.Words[qr.from>>6] 34 | 35 | var bm uint64 36 | 37 | if j <= 64-ns.ShortSize { 38 | bm = (w >> uint32(j)) & vars.ShortMask 39 | } else { 40 | w2 := ns.Inners.Words[qr.to>>6] 41 | bm = (w >> uint32(j)) | (w2 << uint(64-j) & vars.ShortMask) 42 | } 43 | 44 | qr.bm = uint64(ns.ShortTable[bm]) 45 | 46 | } else { 47 | qr.to = qr.from + innerSize 48 | } 49 | } 50 | } 51 | 52 | // getIthInnerFrom finds out the start position of the label bitmap of the ith inner node(not by node id). 53 | func (st *SlimTrie) getIthInnerFrom(ithInner int32, qr *querySession) { 54 | ns := st.inner 55 | vars := st.vars 56 | 57 | if ithInner < ns.BigInnerCnt { 58 | qr.from = ithInner * bigInnerSize 59 | } else { 60 | innWordI := ithInner >> 6 61 | 62 | ithShort := ns.ShortBM.RankIndex[innWordI] + int32(bits.OnesCount64(ns.ShortBM.Words[innWordI]&bitmap.Mask[ithInner&63])) 63 | 64 | qr.from = vars.BigInnerOffset + innerSize*ithInner + vars.ShortMinusInner*ithShort 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /trie/slimtrie_level_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/openacid/slim/encode" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | var ( 11 | levelCases = map[string]struct { 12 | keys []string 13 | slimStr string 14 | levels []levelInfo 15 | }{ 16 | "empty": { 17 | keys: []string{}, 18 | slimStr: trim(""), 19 | levels: []levelInfo{{0, 0, 0, nil}}, 20 | }, 21 | "singleKey": { 22 | keys: []string{"foo"}, 23 | slimStr: trim("#000=0"), 24 | levels: []levelInfo{ 25 | {0, 0, 0, nil}, 26 | {1, 0, 1, nil}, 27 | }, 28 | }, 29 | "simple": { 30 | keys: []string{ 31 | "abc", 32 | "abcd", 33 | "abd", 34 | "abde", 35 | "bc", 36 | "bcd", 37 | "bcde", 38 | "cde", 39 | }, 40 | slimStr: trim(` 41 | #000+4*3 42 | -0001->#001+12*2 43 | -0011->#004*2 44 | -->#008=0 45 | -0110->#009=1 46 | -0100->#005*2 47 | -->#010=2 48 | -0110->#011=3 49 | -0010->#002+8*2 50 | -->#006=4 51 | -0110->#007+8*2 52 | -->#012=5 53 | -0110->#013=6 54 | -0011->#003=7 55 | `), 56 | 57 | levels: []levelInfo{ 58 | {0, 0, 0, nil}, 59 | {1, 1, 0, nil}, 60 | {4, 3, 1, nil}, 61 | {8, 6, 2, nil}, 62 | {14, 6, 8, nil}, 63 | }, 64 | }, 65 | "emptyKey": { 66 | keys: []string{ 67 | "", 68 | "a", 69 | "abc", 70 | "abd", 71 | "bc", 72 | "bcd", 73 | "cde", 74 | }, 75 | slimStr: trim(` 76 | #000*2 77 | -->#001=0 78 | -0110->#002*3 79 | -0001->#003*2 80 | -->#006=1 81 | -0110->#007+12*2 82 | -0011->#010=2 83 | -0100->#011=3 84 | -0010->#004+8*2 85 | -->#008=4 86 | -0110->#009=5 87 | -0011->#005=6 88 | `), 89 | levels: []levelInfo{ 90 | {0, 0, 0, nil}, 91 | {1, 1, 0, nil}, 92 | {3, 2, 1, nil}, 93 | {6, 4, 2, nil}, 94 | {10, 5, 5, nil}, 95 | {12, 5, 7, nil}, 96 | }, 97 | }, 98 | } 99 | ) 100 | 101 | func TestSlimTrie_levels(t *testing.T) { 102 | 103 | for name, c := range levelCases { 104 | t.Run(name, func(t *testing.T) { 105 | ta := require.New(t) 106 | 107 | values := makeI32s(len(c.keys)) 108 | st, err := NewSlimTrie(encode.I32{}, c.keys, values, Opt{Complete: Bool(true)}) 109 | ta.NoError(err) 110 | 111 | dd(st) 112 | ta.Equal(c.slimStr, st.String()) 113 | 114 | for i, lvl := range c.levels { 115 | ta.Equal(lvl.total, st.levels[i].total, "total: line %d", i) 116 | ta.Equal(lvl.inner, st.levels[i].inner, "inner: line %d", i) 117 | ta.Equal(lvl.leaf, st.levels[i].leaf, "leaf: line %d", i) 118 | } 119 | }) 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /trie/slimtrie_new_bench_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/openacid/slim/encode" 7 | ) 8 | 9 | var OutputNewSlimTrie int 10 | 11 | func BenchmarkNewSlimTrie(b *testing.B) { 12 | 13 | benchBigKeySet(b, func(b *testing.B, typ string, keys []string) { 14 | 15 | n := len(keys) 16 | values := makeI32s(len(keys)) 17 | 18 | b.ResetTimer() 19 | var s int 20 | for i := 0; i < b.N/n; i++ { 21 | st, err := NewSlimTrie(encode.I32{}, keys, values) 22 | if err != nil { 23 | panic(err) 24 | } 25 | s += int(st.inner.NodeTypeBM.Words[0]) 26 | } 27 | 28 | OutputNewSlimTrie = s 29 | }) 30 | } 31 | -------------------------------------------------------------------------------- /trie/slimtrie_reduce_dup_value_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "github.com/openacid/slim/encode" 5 | "github.com/stretchr/testify/require" 6 | "testing" 7 | ) 8 | 9 | func TestSlimTrie_Opt_DedupValue(t *testing.T) { 10 | 11 | ta := require.New(t) 12 | 13 | // case: a key not to keep and is a leaf: "Al" 14 | keys := []string{ 15 | "Aaron", 16 | "Agatha", 17 | "Al", 18 | "Albert", 19 | 20 | "Alexander", 21 | "Alison", 22 | } 23 | values := []int32{ 24 | 0, 0, 0, 0, 25 | 1, 1, 26 | } 27 | 28 | wantDedupedStr := trim(` 29 | #000+12*2 30 | -0001->#001=0 31 | -1100->#002 32 | -0110->#003 33 | -0101->#004=1 34 | `) 35 | wantNonDedupedStr := trim(` 36 | #000+12*3 37 | -0001->#001=0 38 | -0111->#002=0 39 | -1100->#003*2 40 | -->#004=0 41 | -0110->#005*3 42 | -0010->#006=0 43 | -0101->#007=1 44 | -1001->#008=1 45 | `) 46 | 47 | { // default: dedup 48 | st, err := NewSlimTrie(encode.I32{}, keys, values, Opt{}) 49 | ta.NoError(err) 50 | 51 | dd(keys) 52 | dd(st) 53 | 54 | ta.Equal(wantDedupedStr, st.String()) 55 | 56 | testPresentKeysRangeGet(t, st, keys, values) 57 | } 58 | { // dedup 59 | st, err := NewSlimTrie(encode.I32{}, keys, values, Opt{DedupValue: Bool(true)}) 60 | ta.NoError(err) 61 | 62 | dd(keys) 63 | dd(st) 64 | 65 | ta.Equal(wantDedupedStr, st.String()) 66 | 67 | testPresentKeysRangeGet(t, st, keys, values) 68 | } 69 | { // no-dedup 70 | st, err := NewSlimTrie(encode.I32{}, keys, values, Opt{DedupValue: Bool(false)}) 71 | ta.NoError(err) 72 | 73 | dd(keys) 74 | dd(st) 75 | 76 | ta.Equal(wantNonDedupedStr, st.String()) 77 | 78 | testPresentKeysGRS(t, st, keys, values) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /trie/slimtrie_stat.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | type Stat struct { 4 | LevelCnt int32 5 | Levels []struct { 6 | Total, Inner, Leaf int32 7 | } 8 | KeyCnt int32 9 | NodeCnt int32 10 | } 11 | 12 | // Stat() returns a struct `Stat` describing SlimTrie internal stats. E.g.: 13 | // 14 | // &trie.Stat{ 15 | // LevelCnt: 5, 16 | // Levels: { 17 | // {}, 18 | // {Total:1, Inner:1, Leaf:0}, 19 | // {Total:4, Inner:3, Leaf:1}, 20 | // {Total:8, Inner:6, Leaf:2}, 21 | // {Total:14, Inner:6, Leaf:8}, 22 | // }, 23 | // KeyCnt: 8, 24 | // NodeCnt: 14, 25 | // } 26 | // 27 | // Since 0.5.12 28 | func (st *SlimTrie) Stat() *Stat { 29 | 30 | ns := st.inner 31 | 32 | rst := &Stat{} 33 | 34 | // The 0-th level is a trivial level with {0, 0, 0} 35 | level_cnt := len(st.levels) 36 | rst.LevelCnt = int32(level_cnt) 37 | for i := 0; i < level_cnt; i++ { 38 | l := st.levels[i] 39 | rst.Levels = append(rst.Levels, struct{ Total, Inner, Leaf int32 }{ 40 | l.total, l.inner, l.leaf, 41 | }) 42 | } 43 | 44 | if ns.NodeTypeBM == nil { 45 | rst.KeyCnt = 0 46 | } else { 47 | rst.KeyCnt = st.levels[level_cnt-1].leaf 48 | } 49 | 50 | rst.NodeCnt = st.levels[level_cnt-1].total 51 | 52 | return rst 53 | } 54 | -------------------------------------------------------------------------------- /trie/slimtrie_stat_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/kr/pretty" 7 | "github.com/openacid/slim/encode" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | type statCase struct { 12 | keys []string 13 | slimStr string 14 | stat string 15 | } 16 | 17 | var statCases = map[string]statCase{ 18 | "empty": { 19 | keys: []string{}, 20 | slimStr: trim(""), 21 | stat: trim(` 22 | &trie.Stat{ 23 | LevelCnt: 1, 24 | Levels: { 25 | {}, 26 | }, 27 | KeyCnt: 0, 28 | NodeCnt: 0, 29 | } 30 | `), 31 | }, 32 | "singleKey": { 33 | keys: []string{"foo"}, 34 | slimStr: trim("#000=0"), 35 | stat: trim(` 36 | &trie.Stat{ 37 | LevelCnt: 2, 38 | Levels: { 39 | {}, 40 | {Total:1, Inner:0, Leaf:1}, 41 | }, 42 | KeyCnt: 1, 43 | NodeCnt: 1, 44 | } 45 | `), 46 | }, 47 | "simple": { 48 | keys: []string{ 49 | "abc", 50 | "abcd", 51 | "abd", 52 | "abde", 53 | "bc", 54 | "bcd", 55 | "bcde", 56 | "cde", 57 | }, 58 | slimStr: trim(` 59 | #000+4*3 60 | -0001->#001+12*2 61 | -0011->#004*2 62 | -->#008=0 63 | -0110->#009=1 64 | -0100->#005*2 65 | -->#010=2 66 | -0110->#011=3 67 | -0010->#002+8*2 68 | -->#006=4 69 | -0110->#007+8*2 70 | -->#012=5 71 | -0110->#013=6 72 | -0011->#003=7 73 | `), 74 | stat: trim(` 75 | &trie.Stat{ 76 | LevelCnt: 5, 77 | Levels: { 78 | {}, 79 | {Total:1, Inner:1, Leaf:0}, 80 | {Total:4, Inner:3, Leaf:1}, 81 | {Total:8, Inner:6, Leaf:2}, 82 | {Total:14, Inner:6, Leaf:8}, 83 | }, 84 | KeyCnt: 8, 85 | NodeCnt: 14, 86 | } 87 | `), 88 | }, 89 | } 90 | 91 | func TestSlimTrie_Stat(t *testing.T) { 92 | 93 | for name, c := range statCases { 94 | t.Run(name, func(t *testing.T) { 95 | 96 | values := makeI32s(len(c.keys)) 97 | 98 | t.Run("complete", func(t *testing.T) { 99 | 100 | ta := require.New(t) 101 | 102 | st, err := NewSlimTrie(encode.I32{}, c.keys, values, Opt{Complete: Bool(true)}) 103 | ta.NoError(err) 104 | 105 | dd(st) 106 | ta.Equal(c.slimStr, st.String()) 107 | ta.Equal(c.stat, pretty.Sprint(st.Stat())) 108 | }) 109 | 110 | t.Run("minimal", func(t *testing.T) { 111 | 112 | ta := require.New(t) 113 | 114 | st, err := NewSlimTrie(encode.I32{}, c.keys, values) 115 | ta.NoError(err) 116 | 117 | ta.Equal(c.stat, pretty.Sprint(st.Stat())) 118 | }) 119 | }) 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /trie/slimtrie_str.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | 7 | "github.com/openacid/low/bitmap" 8 | "github.com/openacid/low/bmtree" 9 | "github.com/openacid/low/tree" 10 | ) 11 | 12 | // String implements proto.Message and output human readable multiline 13 | // representation. 14 | // 15 | // A node is in form of 16 | // ->+*= 17 | // E.g.: 18 | // 000->#000+4*3 19 | // 001->#001+4*2 20 | // 003->#004+8=0 21 | // 006->#007+4=1 22 | // 004->#005+1=2 23 | // 006->#008+4=3 24 | // 002->#002+4=4 25 | // 006->#006+4=5 26 | // 006->#009+8=6 27 | // 003->#003+8=7`[1:] 28 | // 29 | // Since 0.4.3 30 | func (st *SlimTrie) String() string { 31 | 32 | // empty SlimTrie 33 | if st.inner.NodeTypeBM == nil { 34 | return "" 35 | } 36 | 37 | s := &slimTrieStringly{ 38 | st: st, 39 | inners: bitmap.ToArray(st.inner.NodeTypeBM.Words), 40 | labels: make(map[int32]map[string]int32), 41 | } 42 | 43 | ch := st.inner 44 | n := &querySession{} 45 | emp := querySession{} 46 | 47 | for _, nid := range s.inners { 48 | *n = emp 49 | 50 | s.labels[nid] = make(map[string]int32) 51 | 52 | st.getNode(nid, n) 53 | 54 | paths := st.getLabels(n) 55 | 56 | leftChildId, _ := bitmap.Rank128(ch.Inners.Words, ch.Inners.RankIndex, n.from) 57 | 58 | for i, l := range paths { 59 | lstr := bmtree.PathStr(l) 60 | s.labels[nid][lstr] = leftChildId + 1 + int32(i) 61 | } 62 | } 63 | 64 | return tree.String(s) 65 | } 66 | 67 | // slimTrieStringly is a wrapper that implements tree.Tree . 68 | // It is a helper to convert SlimTrie to string. 69 | // 70 | // Since 0.5.1 71 | type slimTrieStringly struct { 72 | st *SlimTrie 73 | inners []int32 74 | // node, label(varbits), node 75 | labels map[int32]map[string]int32 76 | } 77 | 78 | // Child implements tree.Tree 79 | // 80 | // Since 0.5.1 81 | func (s *slimTrieStringly) Child(node, branch interface{}) interface{} { 82 | 83 | n := stNodeID(node) 84 | b := branch.(string) 85 | return s.labels[n][b] 86 | } 87 | 88 | // Labels implements tree.Tree 89 | // 90 | // Since 0.5.1 91 | func (s *slimTrieStringly) Labels(node interface{}) []interface{} { 92 | 93 | n := stNodeID(node) 94 | 95 | rst := make([]string, 0, n) 96 | labels := s.labels[n] 97 | for l := range labels { 98 | rst = append(rst, l) 99 | } 100 | sort.Strings(rst) 101 | 102 | r := []interface{}{} 103 | for _, x := range rst { 104 | r = append(r, x) 105 | } 106 | return r 107 | } 108 | 109 | // NodeID implements tree.Tree 110 | // 111 | // Since 0.5.1 112 | func (s *slimTrieStringly) NodeID(node interface{}) string { 113 | return fmt.Sprintf("%03d", stNodeID(node)) 114 | } 115 | 116 | // LabelInfo implements tree.Tree 117 | // 118 | // Since 0.5.1 119 | func (s *slimTrieStringly) LabelInfo(label interface{}) string { 120 | return label.(string) 121 | } 122 | 123 | // NodeInfo implements tree.Tree 124 | // 125 | // Since 0.5.1 126 | func (s *slimTrieStringly) NodeInfo(node interface{}) string { 127 | nid := stNodeID(node) 128 | n := &querySession{} 129 | emp := querySession{} 130 | 131 | if bitmap.Get(s.st.inner.NodeTypeBM.Words, nid) != 0 { 132 | 133 | *n = emp 134 | 135 | s.st.getNode(nid, n) 136 | step := n.innerPrefixLen 137 | if step > 0 { 138 | return fmt.Sprintf("+%d", step) 139 | } 140 | } 141 | return "" 142 | } 143 | 144 | // LeafVal implements tree.Tree 145 | // 146 | // Since 0.5.1 147 | func (s *slimTrieStringly) LeafVal(node interface{}) (interface{}, bool) { 148 | leafI, nodeType := s.st.getLeafIndex(stNodeID(node)) 149 | if nodeType == 1 { 150 | return nil, false 151 | } 152 | 153 | v := s.st.getIthLeaf(leafI) 154 | return v, true 155 | } 156 | 157 | // stNodeID convert a interface to SlimTrie node id. 158 | func stNodeID(node interface{}) int32 { 159 | if node == nil { 160 | node = int32(0) 161 | } 162 | return node.(int32) 163 | } 164 | -------------------------------------------------------------------------------- /trie/slimtrie_str_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/openacid/slim/encode" 7 | "github.com/openacid/testutil" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestSlimTrie_String_empty(t *testing.T) { 12 | 13 | ta := require.New(t) 14 | 15 | keys := []string{} 16 | values := makeI32s(len(keys)) 17 | st, err := NewSlimTrie(encode.I32{}, keys, values) 18 | ta.NoError(err) 19 | 20 | want := trim(` 21 | `) 22 | dd(st) 23 | 24 | ta.Equal(want, st.String()) 25 | 26 | testUnknownKeysGRS(t, st, testutil.RandStrSlice(100, 0, 10)) 27 | testPresentKeysGet(t, st, keys, values) 28 | } 29 | 30 | func TestSlimTrie_String(t *testing.T) { 31 | 32 | ta := require.New(t) 33 | 34 | keys := []string{ 35 | "abc", 36 | "abcd", 37 | "abcdx", 38 | "abcdy", 39 | "abcdz", 40 | "abd", 41 | "abde", 42 | "bc", 43 | "bcd", 44 | "bcde", 45 | "cde", 46 | } 47 | values := makeI32s(len(keys)) 48 | st, err := NewSlimTrie(encode.I32{}, keys, values) 49 | ta.NoError(err) 50 | 51 | dd(st) 52 | 53 | want := trim(` 54 | #000+4*3 55 | -0001->#001+12*2 56 | -0011->#004*2 57 | -->#008=0 58 | -0110->#009+4*2 59 | -->#014=1 60 | -0111->#015*3 61 | -1000->#016=2 62 | -1001->#017=3 63 | -1010->#018=4 64 | -0100->#005*2 65 | -->#010=5 66 | -0110->#011=6 67 | -0010->#002+8*2 68 | -->#006=7 69 | -0110->#007+4*2 70 | -->#012=8 71 | -0110->#013=9 72 | -0011->#003=10 73 | `) 74 | 75 | ta.Equal(want, st.String()) 76 | 77 | testPresentKeysGet(t, st, keys, values) 78 | } 79 | -------------------------------------------------------------------------------- /trie/slimtrie_string_value_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/openacid/slim/encode" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestSlimTrie_stringValue(t *testing.T) { 11 | 12 | ta := require.New(t) 13 | 14 | keys := []string{ 15 | "Aaron", 16 | "Agatha", 17 | "Al", 18 | "Albert", 19 | } 20 | values := []string{ 21 | "abc", 22 | "def", 23 | "ghi", 24 | "jkl", 25 | } 26 | 27 | { 28 | st, err := NewSlimTrie(encode.String16{}, keys, values) 29 | ta.NoError(err) 30 | 31 | dd(keys) 32 | dd(st) 33 | 34 | for i, key := range keys { 35 | 36 | v, found := st.RangeGet(key) 37 | ta.True(found, "RangeGet:%v", key) 38 | ta.Equal(values[i], v, "RangeGet:%v", key) 39 | } 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /trie/slimtrie_vars.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import "github.com/openacid/low/bitmap" 4 | 5 | // slimVars stores several internally used variables by slim, to speed up calculation 6 | // during querying. 7 | // 8 | // Since 0.5.12 9 | type slimVars struct { 10 | // BigInnerOffset is the offset caused by "BigInner" nodes: 11 | // 12 | // Supposing that the i-th inner node is the j-th short inner node(an inner 13 | // node can be a short). 14 | // 15 | // The offset of this node in "Inners" is 16 | // 17 | // 257 * BigInnerCnt + 18 | // 17 * (i-BigInnerCnt-j) + 19 | // ShortSize * j 20 | // 21 | // Thus we could create 2 variables to reduce offset calculation time: 22 | // 23 | // BigInnerOffset = (257 - 17) * BigInnerCnt 24 | // ShortMinusInner = ShortSize - 17 25 | // 26 | // The the offset is: 27 | // 28 | // BigInnerOffset + 17 * i + ShortMinusInner * j 29 | // 30 | // Since 0.5.12 31 | BigInnerOffset int32 32 | 33 | // ShortMinusInner is ShortSize minus 17. 34 | // See BigInnerOffset. 35 | // 36 | // Since 0.5.12 37 | ShortMinusInner int32 38 | 39 | // ShortMask has the lower ShortSize bit set to 1. 40 | // 41 | // Since 0.5.12 42 | ShortMask uint64 43 | } 44 | 45 | // initVars initialize internal st.vars 46 | // 47 | // Since 0.5.12 48 | func (st *SlimTrie) initVars() { 49 | 50 | ns := st.inner 51 | 52 | st.vars = &slimVars{ 53 | BigInnerOffset: (bigInnerSize - innerSize) * ns.BigInnerCnt, 54 | ShortMinusInner: ns.ShortSize - innerSize, 55 | ShortMask: bitmap.Mask[ns.ShortSize], 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /trie/slimtrie_ver.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | const slimtrieVersion = "0.5.12" 4 | -------------------------------------------------------------------------------- /trie/slimtrie_vlen_array.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "math/bits" 5 | 6 | "github.com/openacid/low/bitmap" 7 | ) 8 | 9 | // newVLenArray builds a VLenArray from a slice of []byte. 10 | // 11 | // It returns nil if no need to build at all, i.e., all elements are empty. 12 | func newVLenArray(elts [][]byte) *VLenArray { 13 | allEqual := true 14 | prevSize := -1 15 | totalSize := 0 16 | sizes := make([]int32, 0, len(elts)) 17 | nonEmptyIndexes := make([]int32, 0, len(elts)) 18 | 19 | for i, elt := range elts { 20 | eltSize := len(elt) 21 | sizes = append(sizes, int32(eltSize)) 22 | totalSize += eltSize 23 | 24 | if eltSize > 0 { 25 | nonEmptyIndexes = append(nonEmptyIndexes, int32(i)) 26 | if prevSize != -1 && prevSize != eltSize { 27 | allEqual = false 28 | } 29 | prevSize = eltSize 30 | } 31 | } 32 | 33 | if totalSize == 0 { 34 | return nil 35 | } 36 | 37 | // Pack all []byte into one buffer. 38 | 39 | vlenArray := &VLenArray{} 40 | buf := make([]byte, 0, totalSize) 41 | for i := 0; i < len(elts); i++ { 42 | buf = append(buf, elts[i]...) 43 | } 44 | vlenArray.Bytes = buf 45 | 46 | vlenArray.N = int32(len(sizes)) 47 | vlenArray.EltCnt = int32(len(nonEmptyIndexes)) 48 | vlenArray.PresenceBM = newBM(nonEmptyIndexes, int32(len(elts)), "r64") 49 | 50 | if allEqual { 51 | // All non-empty elements are of the same size. 52 | // Build a fixed size array 53 | vlenArray.FixedSize = int32(prevSize) 54 | } else { 55 | // Build a var-length array 56 | vlenArray.PositionBM = newBM(stepToPos(sizes, 0), 0, "s32") 57 | } 58 | 59 | return vlenArray 60 | } 61 | 62 | // get returns the `index`-th element. 63 | func (va *VLenArray) get(index int32) []byte { 64 | if index >= va.N { 65 | panic("out of bound") 66 | } 67 | wordI := index >> 6 68 | bitI := index & 63 69 | 70 | presence := va.PresenceBM 71 | 72 | if presence.Words[wordI]&bitmap.Bit[bitI] == 0 { 73 | return []byte{} 74 | } 75 | 76 | ithElt := presence.RankIndex[wordI] + int32(bits.OnesCount64(presence.Words[wordI]&bitmap.Mask[bitI])) 77 | 78 | positions := va.PositionBM 79 | 80 | if positions == nil { 81 | // Fixed size elements 82 | from := ithElt * va.FixedSize 83 | return va.Bytes[from : from+va.FixedSize] 84 | } 85 | 86 | // Var-len element 87 | 88 | from, to := bitmap.Select32R64(positions.Words, positions.SelectIndex, positions.RankIndex, ithElt) 89 | return va.Bytes[from:to] 90 | 91 | } 92 | -------------------------------------------------------------------------------- /trie/slimtrie_vlen_array_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestVLenArray_get(t *testing.T) { 10 | 11 | ta := require.New(t) 12 | 13 | // Fixed size 14 | { 15 | elts := [][]byte{ 16 | {'a', 'b'}, 17 | {}, // empty 18 | {}, 19 | {'c', 'd'}, 20 | {'e', 'f'}, 21 | {}, 22 | } 23 | 24 | va := newVLenArray(elts) 25 | 26 | ta.Equal(int32(6), va.N) 27 | ta.Equal(int32(3), va.EltCnt) 28 | ta.Equal(int32(2), va.FixedSize) 29 | 30 | ta.Equal([]byte{'a', 'b'}, va.get(0)) 31 | ta.Equal([]byte{}, va.get(1)) 32 | ta.Equal([]byte{}, va.get(2)) 33 | ta.Equal([]byte{'c', 'd'}, va.get(3)) 34 | ta.Equal([]byte{'e', 'f'}, va.get(4)) 35 | ta.Equal([]byte{}, va.get(5)) 36 | } 37 | 38 | // Var-len size 39 | { 40 | elts := [][]byte{ 41 | {'a', 'b', 'c'}, 42 | {}, // empty 43 | {}, 44 | {'c', 'd'}, 45 | {'e', 'f'}, 46 | {}, 47 | } 48 | 49 | va := newVLenArray(elts) 50 | 51 | ta.Equal(int32(6), va.N) 52 | ta.Equal(int32(3), va.EltCnt) 53 | ta.Equal(int32(0), va.FixedSize) 54 | 55 | ta.Equal([]byte{'a', 'b', 'c'}, va.get(0)) 56 | ta.Equal([]byte{}, va.get(1)) 57 | ta.Equal([]byte{}, va.get(2)) 58 | ta.Equal([]byte{'c', 'd'}, va.get(3)) 59 | ta.Equal([]byte{'e', 'f'}, va.get(4)) 60 | ta.Equal([]byte{}, va.get(5)) 61 | } 62 | 63 | // dd(st) 64 | } 65 | -------------------------------------------------------------------------------- /trie/slimtrie_withprefix_test.go: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/openacid/slim/encode" 8 | "github.com/openacid/testkeys" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestSlimTrie_withPrefixContent_Get(t *testing.T) { 13 | 14 | ta := require.New(t) 15 | 16 | keys := []string{ 17 | "abc", 18 | "abcd", 19 | "abd", 20 | "abde", 21 | "bc", 22 | "bcd", 23 | "bcde", 24 | "cde", 25 | } 26 | values := []int32{0, 1, 2, 3, 4, 5, 6, 7} 27 | searches := []struct { 28 | key string 29 | want interface{} 30 | }{ 31 | {"ab", nil}, 32 | {"abc", int32(0)}, 33 | {"abcde", int32(1)}, // false positive 34 | {"abd", int32(2)}, 35 | {"ac", nil}, 36 | {"acb", nil}, 37 | {"acd", nil}, 38 | {"adc", nil}, 39 | {"bcd", int32(5)}, 40 | {"bce", nil}, 41 | {"c", int32(7)}, // false positive 42 | {"cde", int32(7)}, 43 | {"cfe", int32(7)}, 44 | {"cff", int32(7)}, 45 | } 46 | 47 | st, err := NewSlimTrie(encode.I32{}, keys, values, 48 | Opt{InnerPrefix: Bool(true)}) 49 | ta.NoError(err) 50 | 51 | fmt.Println(st.content()) 52 | fmt.Println(st.String()) 53 | 54 | wantstr := trim(` 55 | #000+4*3 56 | -0001->#001+12*2 57 | -0011->#004*2 58 | -->#008=0 59 | -0110->#009=1 60 | -0100->#005*2 61 | -->#010=2 62 | -0110->#011=3 63 | -0010->#002+8*2 64 | -->#006=4 65 | -0110->#007+8*2 66 | -->#012=5 67 | -0110->#013=6 68 | -0011->#003=7 69 | `) 70 | 71 | _ = wantstr 72 | ta.Equal(wantstr, st.String()) 73 | 74 | for _, c := range searches { 75 | fmt.Println("get:", c.key) 76 | v, found := st.Get(c.key) 77 | ta.Equal(c.want != nil, found, "case: %+v", c) 78 | ta.Equal(c.want, v, "case: %+v", c) 79 | } 80 | } 81 | 82 | func TestSlimTrie_withPrefixContent_Get_small_keyset(t *testing.T) { 83 | 84 | ta := require.New(t) 85 | 86 | for _, typ := range testkeys.AssetNames() { 87 | 88 | keys := getKeys(typ) 89 | 90 | if len(keys) >= 1000 { 91 | continue 92 | } 93 | 94 | dd("small keyset: %s", typ) 95 | 96 | values := makeI32s(len(keys)) 97 | st, err := NewSlimTrie(encode.I32{}, keys, values, 98 | Opt{InnerPrefix: Bool(true)}) 99 | ta.NoError(err) 100 | 101 | dd(st) 102 | 103 | testPresentKeysGRS(t, st, keys, values) 104 | } 105 | } 106 | 107 | func TestSlimTrie_withPrefixContent_GRS_all_keyset(t *testing.T) { 108 | 109 | testBigKeySet(t, func(t *testing.T, typ string, keys []string) { 110 | ta := require.New(t) 111 | values := makeI32s(len(keys)) 112 | st, err := NewSlimTrie(encode.I32{}, keys, values, 113 | Opt{InnerPrefix: Bool(true)}) 114 | ta.NoError(err) 115 | 116 | testPresentKeysGRS(t, st, keys, values) 117 | }) 118 | } 119 | -------------------------------------------------------------------------------- /trie/testdata/README.md: -------------------------------------------------------------------------------- 1 | # test data 2 | 3 | This is a collection of marshaled data by older versions of slim, for backward 4 | compatibility test. 5 | 6 | # Usage 7 | 8 | ``` 9 | go get github.com/openacid/slimcompatible 10 | go run github.com/openacid/slimcompatible 11 | ``` 12 | -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10ll16k-0.5.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10ll16k-0.5.0 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10ll16k-0.5.1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10ll16k-0.5.1 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10ll16k-0.5.2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10ll16k-0.5.2 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10ll16k-0.5.3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10ll16k-0.5.3 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10ll16k-0.5.4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10ll16k-0.5.4 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10ll16k-0.5.5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10ll16k-0.5.5 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10ll16k-0.5.6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10ll16k-0.5.6 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10ll16k-0.5.7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10ll16k-0.5.7 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10ll16k-0.5.8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10ll16k-0.5.8 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10ll16k-0.5.9: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10ll16k-0.5.9 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10ll16k-allpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10ll16k-allpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10ll16k-innpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10ll16k-innpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10ll16k-nopref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10ll16k-nopref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10vl5-0.5.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10vl5-0.5.0 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10vl5-0.5.1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10vl5-0.5.1 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10vl5-0.5.2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10vl5-0.5.2 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10vl5-0.5.3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10vl5-0.5.3 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10vl5-0.5.4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10vl5-0.5.4 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10vl5-0.5.5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10vl5-0.5.5 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10vl5-0.5.6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10vl5-0.5.6 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10vl5-0.5.7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10vl5-0.5.7 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10vl5-0.5.8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10vl5-0.5.8 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10vl5-0.5.9: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10vl5-0.5.9 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10vl5-allpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10vl5-allpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10vl5-innpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10vl5-innpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-10vl5-nopref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-10vl5-nopref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-11vl5-0.5.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-11vl5-0.5.0 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-11vl5-0.5.1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-11vl5-0.5.1 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-11vl5-0.5.2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-11vl5-0.5.2 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-11vl5-0.5.3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-11vl5-0.5.3 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-11vl5-0.5.4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-11vl5-0.5.4 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-11vl5-0.5.5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-11vl5-0.5.5 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-11vl5-0.5.6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-11vl5-0.5.6 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-11vl5-0.5.7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-11vl5-0.5.7 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-11vl5-0.5.8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-11vl5-0.5.8 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-11vl5-0.5.9: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-11vl5-0.5.9 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-11vl5-allpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-11vl5-allpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-11vl5-innpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-11vl5-innpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-11vl5-nopref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-11vl5-nopref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kl10-0.5.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kl10-0.5.0 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kl10-0.5.1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kl10-0.5.1 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kl10-0.5.2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kl10-0.5.2 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kl10-0.5.3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kl10-0.5.3 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kl10-0.5.4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kl10-0.5.4 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kl10-0.5.5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kl10-0.5.5 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kl10-0.5.6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kl10-0.5.6 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kl10-0.5.7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kl10-0.5.7 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kl10-0.5.8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kl10-0.5.8 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kl10-0.5.9: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kl10-0.5.9 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kl10-allpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kl10-allpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kl10-innpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kl10-innpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kl10-nopref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kl10-nopref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kvl10-0.5.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kvl10-0.5.0 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kvl10-0.5.1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kvl10-0.5.1 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kvl10-0.5.2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kvl10-0.5.2 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kvl10-0.5.3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kvl10-0.5.3 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kvl10-0.5.4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kvl10-0.5.4 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kvl10-0.5.5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kvl10-0.5.5 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kvl10-0.5.6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kvl10-0.5.6 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kvl10-0.5.7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kvl10-0.5.7 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kvl10-0.5.8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kvl10-0.5.8 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kvl10-0.5.9: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kvl10-0.5.9 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kvl10-allpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kvl10-allpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kvl10-innpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kvl10-innpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-20kvl10-nopref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-20kvl10-nopref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-300vl50-allpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-300vl50-allpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-300vl50-innpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-300vl50-innpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-300vl50-nopref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-300vl50-nopref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kl10-0.5.5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kl10-0.5.5 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kl10-0.5.6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kl10-0.5.6 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kl10-0.5.7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kl10-0.5.7 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kl10-0.5.8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kl10-0.5.8 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kl10-0.5.9: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kl10-0.5.9 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kl10-allpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kl10-allpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kl10-innpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kl10-innpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kl10-nopref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kl10-nopref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kvl10-0.5.5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kvl10-0.5.5 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kvl10-0.5.6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kvl10-0.5.6 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kvl10-0.5.7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kvl10-0.5.7 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kvl10-0.5.8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kvl10-0.5.8 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kvl10-0.5.9: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kvl10-0.5.9 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kvl10-allpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kvl10-allpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kvl10-innpref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kvl10-innpref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-50kvl10-nopref-0.5.10: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-50kvl10-nopref-0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-empty-0.5.0: -------------------------------------------------------------------------------- 1 | 1.0.0 1.0.0 1.0.0 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-empty-0.5.1: -------------------------------------------------------------------------------- 1 | 1.0.0 1.0.0 1.0.0 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-empty-0.5.2: -------------------------------------------------------------------------------- 1 | 1.0.0 1.0.0 1.0.0 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-empty-0.5.3: -------------------------------------------------------------------------------- 1 | 1.0.0 1.0.0 1.0.0 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-empty-0.5.4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-empty-0.5.4 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-empty-0.5.5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-empty-0.5.5 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-empty-0.5.6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openacid/slim/e045937bf4e53e06893314bb71a2c710e3132be4/trie/testdata/slimtrie-data-empty-0.5.6 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-empty-0.5.7: -------------------------------------------------------------------------------- 1 | 1.0.0 1.0.0 1.0.0 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-empty-0.5.8: -------------------------------------------------------------------------------- 1 | 0.5.8 0.5.8 0.5.8 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-empty-0.5.9: -------------------------------------------------------------------------------- 1 | 0.5.9 0.5.9 0.5.9 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-empty-allpref-0.5.10: -------------------------------------------------------------------------------- 1 | 0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-empty-innpref-0.5.10: -------------------------------------------------------------------------------- 1 | 0.5.10 -------------------------------------------------------------------------------- /trie/testdata/slimtrie-data-empty-nopref-0.5.10: -------------------------------------------------------------------------------- 1 | 0.5.10 --------------------------------------------------------------------------------