├── .gitattributes ├── .github └── workflows │ └── test.yml ├── .golangci.yml ├── LICENSE ├── README.md ├── benchmark_test.go ├── benchmarks ├── README.md ├── bbloom.go ├── benchmark_test.go ├── blobloom.go ├── blobloom_xxh3.go ├── blobloom_xxhash.go ├── boom.go ├── dcso.go ├── devopsfaith.go ├── go.mod ├── go.sum ├── ring.go ├── sync.go └── willf.go ├── bloomfilter.go ├── bloomfilter_test.go ├── example_test.go ├── examples ├── bloomstat │ └── main.go └── spellcheck │ └── main.go ├── go.mod ├── go.sum ├── io.go ├── io_fuzz_test.go ├── io_test.go ├── optimize.go ├── optimize_test.go ├── setop_64bit.go ├── setop_other.go ├── sync.go ├── sync_go124.go ├── sync_nogo124.go ├── sync_test.go └── test.sh /.gitattributes: -------------------------------------------------------------------------------- 1 | # Work around https://github.com/golang/go/issues/52268. 2 | **/testdata/fuzz/*/* eol=lf 3 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | 8 | jobs: 9 | test: 10 | strategy: 11 | matrix: 12 | goversion: [1.16, 1.19, 1.23] 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - name: Install Go 17 | uses: actions/setup-go@v4 18 | with: 19 | go-version: ${{ matrix.goversion }} 20 | - name: Checkout 21 | uses: actions/checkout@v2 22 | - name: Test 23 | # Run benchmarks just to see that they don't crash. 24 | run: go test -bench=. -benchtime=.1s ./... 25 | - name: Test nounsafe 26 | run: go test -tags nounsafe ./... 27 | - name: Test 386 28 | run: go test -tags nounsafe ./... 29 | env: 30 | GOARCH: 386 31 | - name: Build benchmarks 32 | run: cd benchmarks && go test -c 33 | 34 | test-qemu: 35 | strategy: 36 | matrix: 37 | arch: [arm, arm64] 38 | runs-on: ubuntu-latest 39 | 40 | steps: 41 | - name: Install Go 42 | uses: actions/setup-go@v4 43 | with: 44 | go-version: 1.19 45 | - name: Install QEMU 46 | uses: docker/setup-qemu-action@v1 47 | - name: Checkout 48 | uses: actions/checkout@v2 49 | - name: Test 50 | run: go test -bench=. -benchtime=.1s ./... 51 | env: 52 | GOARCH: ${{ matrix.arch }} 53 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | # Configuration for golangci-lint. 2 | 3 | linters: 4 | disable: 5 | - asciicheck 6 | enable: 7 | - gocognit 8 | - gocyclo 9 | - godot 10 | - gofumpt 11 | - lll 12 | - misspell 13 | - nakedret 14 | - thelper 15 | 16 | issues: 17 | exclude-rules: 18 | - path: _test\.go 19 | linters: 20 | errcheck 21 | 22 | linters-settings: 23 | govet: 24 | enable: 25 | - atomicalign 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Blobloom 2 | ======== 3 | 4 | A Bloom filter package for Go (golang) with no compile-time dependencies. 5 | 6 | This package implements a version of Bloom filters called [blocked Bloom filters]( 7 | https://www.cs.amherst.edu/~ccmcgeoch/cs34/papers/cacheefficientbloomfilters-jea.pdf), 8 | which get a speed boost from using the CPU cache more efficiently 9 | than regular Bloom filters. 10 | 11 | Unlike most Bloom filter packages for Go, 12 | this one doesn't run a hash function for you. 13 | That's a benefit if you need a custom hash 14 | or you want pick the fastest one for an application. 15 | 16 | Usage 17 | ----- 18 | 19 | To construct a Bloom filter, you need to know how many keys you want to store 20 | and what rate of false positives you find acceptable. 21 | 22 | f := blobloom.NewOptimized(blobloom.Config{ 23 | Capacity: nkeys, // Expected number of keys. 24 | FPRate: 1e-4, // Accept one false positive per 10,000 lookups. 25 | }) 26 | 27 | To add a key: 28 | 29 | // import "github.com/cespare/xxhash/v2" 30 | f.Add(xxhash.Sum64(key)) 31 | 32 | To test for the presence of a key in the filter: 33 | 34 | if f.Has(xxhash.Sum64(key)) { 35 | // Key is probably in f. 36 | } else { 37 | // Key is certainly not in f. 38 | } 39 | 40 | The false positive rate is defined as usual: 41 | if you look up 10,000 random keys in a Bloom filter filled to capacity, 42 | an expected one of those is a false positive for FPRate 1e-4. 43 | 44 | See the examples/ directory and the 45 | [package documentation](https://pkg.go.dev/github.com/greatroar/blobloom) 46 | for further usage information and examples. 47 | 48 | Hash functions 49 | -------------- 50 | 51 | Blobloom does not provide hash functions. Instead, it requires client code to 52 | represent each key as a single 64-bit hash value, leaving it to the user to 53 | pick the right hash function for a particular problem. Here are some general 54 | suggestions: 55 | 56 | * If you use Bloom filters to speed up access to a key-value store, you might 57 | want to look at [xxh3](https://github.com/zeebo/xxh3) or [xxhash]( 58 | https://github.com/cespare/xxhash). 59 | * If your keys are cryptographic hashes, consider using the first 8 bytes of those hashes. 60 | * If you use Bloom filters to make probabilistic decisions, a randomized hash 61 | function such as [maphash](https://golang.org/pkg/hash/maphash) should prevent 62 | the same false positives occurring every time. 63 | 64 | When evaluating a hash function, or designing a custom one, 65 | make sure it is a 64-bit hash that properly mixes its input bits. 66 | Casting a 32-bit hash to uint64 gives suboptimal results. 67 | So does passing integer keys in without running them through a mixing function. 68 | 69 | 70 | 71 | License 72 | ------- 73 | 74 | Copyright © 2020-2024 the Blobloom authors 75 | 76 | Licensed under the Apache License, Version 2.0 (the "License"); 77 | you may not use this file except in compliance with the License. 78 | You may obtain a copy of the License at 79 | 80 | http://www.apache.org/licenses/LICENSE-2.0 81 | 82 | Unless required by applicable law or agreed to in writing, software 83 | distributed under the License is distributed on an "AS IS" BASIS, 84 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 85 | See the License for the specific language governing permissions and 86 | limitations under the License. 87 | -------------------------------------------------------------------------------- /benchmark_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Benchmarks for the basic operations live in the benchmarks/ subpackage. 16 | 17 | package blobloom 18 | 19 | import ( 20 | "math/rand" 21 | "sync" 22 | "sync/atomic" 23 | "testing" 24 | ) 25 | 26 | // Baseline for BenchmarkAddSync. 27 | func benchmarkAddLocked(b *testing.B, nbits uint64) { 28 | b.Helper() 29 | 30 | const nhashes = 22 // Large number of hashes to create collisions. 31 | 32 | var ( 33 | f = New(nbits, nhashes) 34 | mu sync.Mutex 35 | seed uint32 36 | ) 37 | 38 | b.ResetTimer() 39 | 40 | b.RunParallel(func(pb *testing.PB) { 41 | r := rand.New(rand.NewSource(int64(atomic.AddUint32(&seed, 1)))) 42 | for pb.Next() { 43 | mu.Lock() 44 | f.Add(r.Uint64()) 45 | mu.Unlock() 46 | } 47 | }) 48 | } 49 | 50 | func BenchmarkAddLocked128kB(b *testing.B) { benchmarkAddLocked(b, 1<<20) } 51 | func BenchmarkAddLocked1MB(b *testing.B) { benchmarkAddLocked(b, 1<<23) } 52 | func BenchmarkAddLocked16MB(b *testing.B) { benchmarkAddLocked(b, 1<<27) } 53 | 54 | func benchmarkAddSync(b *testing.B, nbits uint64) { 55 | b.Helper() 56 | 57 | const nhashes = 22 // Large number of hashes to create collisions. 58 | 59 | f := NewSync(nbits, nhashes) 60 | var seed uint32 61 | 62 | b.ResetTimer() 63 | 64 | b.RunParallel(func(pb *testing.PB) { 65 | r := rand.New(rand.NewSource(int64(atomic.AddUint32(&seed, 1)))) 66 | for pb.Next() { 67 | f.Add(r.Uint64()) 68 | } 69 | }) 70 | } 71 | 72 | func BenchmarkAddSync128kB(b *testing.B) { benchmarkAddSync(b, 1<<20) } 73 | func BenchmarkAddSync1MB(b *testing.B) { benchmarkAddSync(b, 1<<23) } 74 | func BenchmarkAddSync16MB(b *testing.B) { benchmarkAddSync(b, 1<<27) } 75 | 76 | func BenchmarkCardinalityDense(b *testing.B) { 77 | f := New(1<<20, 2) 78 | for i := range f.b { 79 | for j := range f.b[i] { 80 | f.b[i][j] = rand.Uint32() 81 | } 82 | } 83 | 84 | b.ResetTimer() 85 | 86 | for i := 0; i < b.N; i++ { 87 | f.Cardinality() 88 | } 89 | } 90 | 91 | func BenchmarkCardinalitySparse(b *testing.B) { 92 | f := New(1<<20, 2) 93 | for i := 0; i < len(f.b); i += 2 { 94 | for _, j := range []int{4, 8, 13} { 95 | f.b[i][j] = rand.Uint32() 96 | } 97 | } 98 | 99 | b.ResetTimer() 100 | 101 | for i := 0; i < b.N; i++ { 102 | f.Cardinality() 103 | } 104 | } 105 | 106 | func BenchmarkOnescount(b *testing.B) { 107 | var blk block 108 | for i := range blk { 109 | blk[i] = rand.Uint32() 110 | } 111 | 112 | b.ResetTimer() 113 | 114 | for i := 0; i < b.N; i++ { 115 | onescount(&blk) 116 | } 117 | } 118 | 119 | func BenchmarkUnion(b *testing.B) { 120 | const n = 1e6 121 | 122 | var ( 123 | cfg = Config{Capacity: n, FPRate: 1e-5} 124 | f = NewOptimized(cfg) 125 | g = NewOptimized(cfg) 126 | fRef = NewOptimized(cfg) 127 | gRef = NewOptimized(cfg) 128 | hashes = randomU64(n, 0xcb6231119) 129 | ) 130 | 131 | b.Logf("NumBits = %d", f.NumBits()) 132 | 133 | for _, h := range hashes[:n/2] { 134 | fRef.Add(h) 135 | } 136 | for _, h := range hashes[n/2:] { 137 | gRef.Add(h) 138 | } 139 | 140 | b.ResetTimer() 141 | 142 | for i := 0; i < b.N; i++ { 143 | b.StopTimer() 144 | f.Clear() 145 | f.Union(fRef) 146 | g.Clear() 147 | g.Union(gRef) 148 | b.StartTimer() 149 | 150 | f.Union(g) 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /benchmarks/README.md: -------------------------------------------------------------------------------- 1 | This module contains benchmarks for comparison against other Bloom filter 2 | packages. To run these benchmarks, pick a build tag from the following table: 3 | 4 | | Tag | Package | 5 | | -------- | ----------------------------------------------------------- | 6 | | (no tag) | This package with pre-hashed inputs | 7 | | bbloom | github.com/ipfs/bbloom | 8 | | boom | github.com/tylertreat/BoomFilters ("classic" Bloom filters) | 9 | | dcso | github.com/DCSO/bloom | 10 | | ring | github.com/tannerryan/ring | 11 | | sync | This package's SyncFilter with pre-hashed inputs | 12 | | willf | github.com/bits-and-blooms/bloom (formerly willf/bloom) | 13 | | xxhash | This package + github.com/cespare/xxhash | 14 | | xxh3 | This package + github.com/zeebo/xxh3 | 15 | 16 | Then invoke go test as follows: 17 | 18 | go test -tags="$tag" -bench=. 19 | 20 | Omit -tags and its argument to run the benchmarks for Blobloom. These assume 21 | that the input keys (which are random strings) can be used as hashes without 22 | any processing. This reflects the original use case (in [Syncthing]( 23 | https://syncthing.net)) where SHA-256 hashes were stored in a Bloom filter. 24 | If this does not describe your use case, benchmark with the tag xxhash 25 | to run the keys through the [xxhash](https://github.com/cespare/xxhash) 26 | function. 27 | 28 | The benchmarks are set up to work with the benchstat tool. 29 | To compare Blobloom+xxh3 to bbloom, do 30 | 31 | go get golang.org/x/perf/cmd/benchstat 32 | go test -bench=. -count=5 -timeout=30m -tags "bbloom" | tee bbloom.bench 33 | go test -bench=. -count=5 -timeout=30m -tags "xx3" | tee xxh3.bench 34 | benchstat bbloom.bench xxh3.bench 35 | 36 | The sync benchmark only measures sequential performance. 37 | -------------------------------------------------------------------------------- /benchmarks/bbloom.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build bbloom 16 | // +build bbloom 17 | 18 | package benchmarks 19 | 20 | import "github.com/ipfs/bbloom" 21 | 22 | type bloomFilter = bbloom.Bloom 23 | 24 | func newBF(capacity int, fpr float64) *bloomFilter { 25 | f, err := bbloom.New(float64(capacity), fpr) 26 | if err != nil { 27 | panic(err) 28 | } 29 | return (*bloomFilter)(f) 30 | } 31 | -------------------------------------------------------------------------------- /benchmarks/benchmark_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package benchmarks contains benchmarks for various Bloom filter 16 | // implementations, selected by build tags. See README.md for details. 17 | package benchmarks 18 | 19 | import ( 20 | "math/rand" 21 | "testing" 22 | ) 23 | 24 | const hashSize = 32 25 | 26 | func makehashes(n int, seed int64) []byte { 27 | h := make([]byte, n*hashSize) 28 | r := rand.New(rand.NewSource(seed)) 29 | r.Read(h) 30 | 31 | return h 32 | } 33 | 34 | // In each iteration, add a SHA-256 into a Bloom filter with the given capacity 35 | // and desired FPR. 36 | func benchmarkAdd(b *testing.B, capacity int, fpr float64) { 37 | b.Helper() 38 | 39 | hashes := makehashes(b.N, 51251991517) 40 | f := newBF(capacity, fpr) 41 | 42 | b.ResetTimer() 43 | 44 | for i := 0; i < b.N; i++ { 45 | h := hashes[i*hashSize : (i+1)*hashSize] 46 | f.Add(h) 47 | } 48 | } 49 | 50 | func BenchmarkAdd1e5_1e2(b *testing.B) { benchmarkAdd(b, 1e5, 1e-2) } 51 | func BenchmarkAdd1e6_1e2(b *testing.B) { benchmarkAdd(b, 1e6, 1e-2) } 52 | func BenchmarkAdd1e7_1e2(b *testing.B) { benchmarkAdd(b, 1e7, 1e-2) } 53 | func BenchmarkAdd1e8_1e2(b *testing.B) { benchmarkAdd(b, 1e8, 1e-2) } 54 | func BenchmarkAdd1e5_1e3(b *testing.B) { benchmarkAdd(b, 1e5, 1e-3) } 55 | func BenchmarkAdd1e6_1e3(b *testing.B) { benchmarkAdd(b, 1e6, 1e-3) } 56 | func BenchmarkAdd1e7_1e3(b *testing.B) { benchmarkAdd(b, 1e7, 1e-3) } 57 | func BenchmarkAdd1e8_1e3(b *testing.B) { benchmarkAdd(b, 1e8, 1e-3) } 58 | 59 | // In each iteration, test for a SHA-256 in a Bloom filter with the given capacity 60 | // and desired FPR that has that SHA-256 added to it. 61 | func benchmarkTestPos(b *testing.B, capacity int, fpr float64) { 62 | b.Helper() 63 | 64 | const ntest = 8192 65 | hashes := makehashes(ntest, 0x5128351a) 66 | 67 | f := newBF(capacity, fpr) 68 | 69 | for i := 0; i < capacity && i < ntest; i++ { 70 | h := hashes[i*hashSize : (i+1)*hashSize] 71 | f.Add(h) 72 | } 73 | for i := ntest; i < capacity; i++ { 74 | h := make([]byte, hashSize) 75 | f.Add(h) 76 | } 77 | 78 | b.ResetTimer() 79 | 80 | for i := 0; i < b.N; i++ { 81 | j := i % ntest 82 | h := hashes[j*hashSize : (j+1)*hashSize] 83 | if !f.Has(h) { 84 | b.Fatalf("%x added to Bloom filter but not retrieved", h) 85 | } 86 | } 87 | } 88 | 89 | func BenchmarkTestPos1e5_1e2(b *testing.B) { benchmarkTestPos(b, 1e5, 1e-2) } 90 | func BenchmarkTestPos1e6_1e2(b *testing.B) { benchmarkTestPos(b, 1e6, 1e-2) } 91 | func BenchmarkTestPos1e7_1e2(b *testing.B) { benchmarkTestPos(b, 1e7, 1e-2) } 92 | func BenchmarkTestPos1e8_1e2(b *testing.B) { benchmarkTestPos(b, 1e8, 1e-2) } 93 | func BenchmarkTestPos1e5_1e3(b *testing.B) { benchmarkTestPos(b, 1e5, 1e-3) } 94 | func BenchmarkTestPos1e6_1e3(b *testing.B) { benchmarkTestPos(b, 1e6, 1e-3) } 95 | func BenchmarkTestPos1e7_1e3(b *testing.B) { benchmarkTestPos(b, 1e7, 1e-3) } 96 | func BenchmarkTestPos1e8_1e3(b *testing.B) { benchmarkTestPos(b, 1e8, 1e-3) } 97 | 98 | // In each iteration, test for the presence of a SHA-256 in a filled Bloom filter 99 | // with the given capacity and desired FPR. 100 | func benchmarkTestNeg(b *testing.B, capacity int, fpr float64) { 101 | b.Helper() 102 | 103 | r := rand.New(rand.NewSource(0xae694)) 104 | f := newBF(capacity, fpr) 105 | 106 | h := make([]byte, hashSize) 107 | for i := 0; i < capacity; i++ { 108 | r.Read(h) 109 | f.Add(h) 110 | } 111 | 112 | // Make new hashes. Assume these are all distinct from the inserted ones. 113 | const ntest = 8192 114 | hashes := makehashes(ntest, 562175) 115 | 116 | b.ResetTimer() 117 | 118 | fp := 0 119 | for i := 0; i < b.N; i++ { 120 | j := i % ntest 121 | h := hashes[j*hashSize : (j+1)*hashSize] 122 | if f.Has(h) { 123 | fp++ 124 | } 125 | } 126 | 127 | b.Logf("false positive rate = %.3f%%", 100*float64(fp)/float64(b.N)) 128 | } 129 | 130 | func BenchmarkTestNeg1e5_1e2(b *testing.B) { benchmarkTestNeg(b, 1e5, 1e-2) } 131 | func BenchmarkTestNeg1e6_1e2(b *testing.B) { benchmarkTestNeg(b, 1e6, 1e-2) } 132 | func BenchmarkTestNeg1e7_1e2(b *testing.B) { benchmarkTestNeg(b, 1e7, 1e-2) } 133 | func BenchmarkTestNeg1e8_1e2(b *testing.B) { benchmarkTestNeg(b, 1e8, 1e-2) } 134 | func BenchmarkTestNeg1e5_1e3(b *testing.B) { benchmarkTestNeg(b, 1e5, 1e-3) } 135 | func BenchmarkTestNeg1e6_1e3(b *testing.B) { benchmarkTestNeg(b, 1e6, 1e-3) } 136 | func BenchmarkTestNeg1e7_1e3(b *testing.B) { benchmarkTestNeg(b, 1e7, 1e-3) } 137 | func BenchmarkTestNeg1e8_1e3(b *testing.B) { benchmarkTestNeg(b, 1e8, 1e-3) } 138 | 139 | // In each iteration, test for the presence of a SHA-256 in an empty Bloom filter 140 | // with the given capacity and desired FPR. 141 | func benchmarkTestEmpty(b *testing.B, capacity int, fpr float64) { 142 | b.Helper() 143 | 144 | const ntest = 65536 145 | hashes := makehashes(ntest, 054271) 146 | f := newBF(capacity, fpr) 147 | 148 | b.ResetTimer() 149 | 150 | for i := 0; i < b.N; i++ { 151 | j := i % ntest 152 | f.Has(hashes[j*hashSize : (j+1)*hashSize]) 153 | } 154 | } 155 | 156 | func BenchmarkTestEmpty1e5_1e2(b *testing.B) { benchmarkTestEmpty(b, 1e5, 1e-2) } 157 | func BenchmarkTestEmpty1e6_1e2(b *testing.B) { benchmarkTestEmpty(b, 1e6, 1e-2) } 158 | func BenchmarkTestEmpty1e7_1e2(b *testing.B) { benchmarkTestEmpty(b, 1e7, 1e-2) } 159 | func BenchmarkTestEmpty1e8_1e2(b *testing.B) { benchmarkTestEmpty(b, 1e8, 1e-2) } 160 | func BenchmarkTestEmpty1e5_1e3(b *testing.B) { benchmarkTestEmpty(b, 1e5, 1e-3) } 161 | func BenchmarkTestEmpty1e6_1e3(b *testing.B) { benchmarkTestEmpty(b, 1e6, 1e-3) } 162 | func BenchmarkTestEmpty1e7_1e3(b *testing.B) { benchmarkTestEmpty(b, 1e7, 1e-3) } 163 | func BenchmarkTestEmpty1e8_1e3(b *testing.B) { benchmarkTestEmpty(b, 1e8, 1e-3) } 164 | -------------------------------------------------------------------------------- /benchmarks/blobloom.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !bbloom && !boom && !dcso && !devopsfaith && !ring && !sync && !willf && !xxh3 && !xxhash 16 | // +build !bbloom,!boom,!dcso,!devopsfaith,!ring,!sync,!willf,!xxh3,!xxhash 17 | 18 | package benchmarks 19 | 20 | import ( 21 | "encoding/binary" 22 | 23 | "github.com/greatroar/blobloom" 24 | ) 25 | 26 | type bloomFilter blobloom.Filter 27 | 28 | func (f *bloomFilter) Add(hash []byte) { 29 | h := binary.BigEndian.Uint64(hash[:8]) 30 | ((*blobloom.Filter)(f)).Add(h) 31 | } 32 | 33 | func (f *bloomFilter) Has(hash []byte) bool { 34 | h := binary.BigEndian.Uint64(hash[:8]) 35 | return ((*blobloom.Filter)(f)).Has(h) 36 | } 37 | 38 | func newBF(capacity int, fpr float64) *bloomFilter { 39 | f := blobloom.NewOptimized(blobloom.Config{ 40 | Capacity: uint64(capacity), 41 | FPRate: fpr, 42 | }) 43 | return (*bloomFilter)(f) 44 | } 45 | -------------------------------------------------------------------------------- /benchmarks/blobloom_xxh3.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build xxh3 16 | // +build xxh3 17 | 18 | package benchmarks 19 | 20 | import ( 21 | "github.com/greatroar/blobloom" 22 | "github.com/zeebo/xxh3" 23 | ) 24 | 25 | type bloomFilter blobloom.Filter 26 | 27 | func (f *bloomFilter) Add(hash []byte) { 28 | h := xxh3.Hash(hash) 29 | ((*blobloom.Filter)(f)).Add(h) 30 | } 31 | 32 | func (f *bloomFilter) Has(hash []byte) bool { 33 | h := xxh3.Hash(hash) 34 | return ((*blobloom.Filter)(f)).Has(h) 35 | } 36 | 37 | func newBF(capacity int, fpr float64) *bloomFilter { 38 | f := blobloom.NewOptimized(blobloom.Config{ 39 | Capacity: uint64(capacity), 40 | FPRate: fpr, 41 | }) 42 | return (*bloomFilter)(f) 43 | } 44 | -------------------------------------------------------------------------------- /benchmarks/blobloom_xxhash.go: -------------------------------------------------------------------------------- 1 | // Licensed under the Apache License, Version 2.0 (the "License"); 2 | // you may not use this file except in compliance with the License. 3 | // You may obtain a copy of the License at 4 | // 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software 8 | // distributed under the License is distributed on an "AS IS" BASIS, 9 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | // See the License for the specific language governing permissions and 11 | // limitations under the License. 12 | 13 | //go:build xxhash 14 | // +build xxhash 15 | 16 | package benchmarks 17 | 18 | import ( 19 | "github.com/cespare/xxhash/v2" 20 | "github.com/greatroar/blobloom" 21 | ) 22 | 23 | type bloomFilter blobloom.Filter 24 | 25 | func (f *bloomFilter) Add(hash []byte) { 26 | h := xxhash.Sum64(hash) 27 | ((*blobloom.Filter)(f)).Add(h) 28 | } 29 | 30 | func (f *bloomFilter) Has(hash []byte) bool { 31 | h := xxhash.Sum64(hash) 32 | return ((*blobloom.Filter)(f)).Has(h) 33 | } 34 | 35 | func newBF(capacity int, fpr float64) *bloomFilter { 36 | f := blobloom.NewOptimized(blobloom.Config{ 37 | Capacity: uint64(capacity), 38 | FPRate: fpr, 39 | }) 40 | return (*bloomFilter)(f) 41 | } 42 | -------------------------------------------------------------------------------- /benchmarks/boom.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build boom 16 | // +build boom 17 | 18 | package benchmarks 19 | 20 | import "github.com/tylertreat/BoomFilters" 21 | 22 | type bloomFilter boom.BloomFilter 23 | 24 | func (f *bloomFilter) Add(hash []byte) { 25 | ((*boom.BloomFilter)(f)).Add(hash) 26 | } 27 | 28 | func (f *bloomFilter) Has(hash []byte) bool { 29 | return ((*boom.BloomFilter)(f)).Test(hash) 30 | } 31 | 32 | func newBF(capacity int, fpr float64) *bloomFilter { 33 | f := boom.NewBloomFilter(uint(capacity), fpr) 34 | f.SetHash(&nopHash{}) 35 | return (*bloomFilter)(f) 36 | } 37 | 38 | // No-op hash function. Assumes all data is written to it in one Write call. 39 | type nopHash struct{ data []byte } 40 | 41 | func (h *nopHash) BlockSize() int { return 1 } 42 | func (h *nopHash) Reset() {} 43 | func (h *nopHash) Size() int { return 8 } 44 | func (h *nopHash) Sum(d []byte) []byte { return append(d, h.data...) } 45 | func (h *nopHash) Sum64() uint64 { panic("not used by BoomFilters") } 46 | 47 | func (h *nopHash) Write(p []byte) (n int, err error) { 48 | if len(p) > 8 { 49 | p = p[:8] 50 | } 51 | h.data = p 52 | return len(p), nil 53 | } 54 | -------------------------------------------------------------------------------- /benchmarks/dcso.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build dcso 16 | // +build dcso 17 | 18 | package benchmarks 19 | 20 | import "github.com/DCSO/bloom" 21 | 22 | type bloomFilter struct{ bloom.BloomFilter } 23 | 24 | func newBF(capacity int, fpr float64) *bloomFilter { 25 | f := bloom.Initialize(uint64(capacity), fpr) 26 | return &bloomFilter{f} 27 | } 28 | 29 | func (f *bloomFilter) Has(hash []byte) bool { 30 | return f.Check(hash) 31 | } 32 | -------------------------------------------------------------------------------- /benchmarks/devopsfaith.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build devopsfaith 16 | // +build devopsfaith 17 | 18 | package benchmarks 19 | 20 | import ( 21 | "github.com/devopsfaith/bloomfilter" 22 | "github.com/devopsfaith/bloomfilter/bloomfilter" 23 | ) 24 | 25 | type bloomFilter baseBloomfilter.Bloomfilter 26 | 27 | func newBF(capacity int, fpr float64) *bloomFilter { 28 | f := baseBloomfilter.New(bloomfilter.Config{ 29 | N: uint(capacity), 30 | P: fpr, 31 | HashName: "default", 32 | }) 33 | return (*bloomFilter)(f) 34 | } 35 | 36 | func (f *bloomFilter) Add(hash []byte) { 37 | ((*baseBloomfilter.Bloomfilter)(f)).Add(hash) 38 | } 39 | 40 | func (f *bloomFilter) Has(hash []byte) bool { 41 | return ((*baseBloomfilter.Bloomfilter)(f)).Check(hash) 42 | } 43 | -------------------------------------------------------------------------------- /benchmarks/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/greatroar/blobloom/benchmarks 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/DCSO/bloom v0.2.4 7 | github.com/bits-and-blooms/bitset v1.7.0 // indirect 8 | github.com/bits-and-blooms/bloom/v3 v3.3.1 9 | github.com/cespare/xxhash/v2 v2.2.0 10 | github.com/d4l3k/messagediff v1.2.1 // indirect 11 | github.com/devopsfaith/bloomfilter v1.4.0 12 | github.com/greatroar/blobloom v0.7.2 13 | github.com/ipfs/bbloom v0.0.4 14 | github.com/klauspost/cpuid/v2 v2.2.4 // indirect 15 | github.com/tannerryan/ring v1.1.2 16 | github.com/tmthrgd/atomics v0.0.0-20190904060638-dc7a5fcc7e0d // indirect 17 | github.com/tmthrgd/go-bitset v0.0.0-20190904054048-394d9a556c05 // indirect 18 | github.com/tmthrgd/go-bitwise v0.0.0-20190904053232-1430ee983fca // indirect 19 | github.com/tmthrgd/go-byte-test v0.0.0-20190904060354-2794345b9929 // indirect 20 | github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect 21 | github.com/tmthrgd/go-memset v0.0.0-20190904060434-6fb7a21f88f1 // indirect 22 | github.com/tmthrgd/go-popcount v0.0.0-20190904054823-afb1ace8b04f // indirect 23 | github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 24 | github.com/zeebo/xxh3 v1.0.2 25 | golang.org/x/sys v0.7.0 // indirect 26 | ) 27 | 28 | replace github.com/greatroar/blobloom => ../ 29 | -------------------------------------------------------------------------------- /benchmarks/go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/Azure/azure-sdk-for-go v16.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= 3 | github.com/Azure/go-autorest v10.7.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= 4 | github.com/Azure/go-autorest v10.15.3+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= 5 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 6 | github.com/DCSO/bloom v0.2.4 h1:GYQuxfH7Q4mS+LM1iEVSSjkXbS5FyFSyh/n4y45X14c= 7 | github.com/DCSO/bloom v0.2.4/go.mod h1:G/tm7BJ6j725OJIGl5MCNpbbJbl09+8QWMNLr9mDLJ0= 8 | github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= 9 | github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= 10 | github.com/Microsoft/go-winio v0.4.3/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= 11 | github.com/NYTimes/gziphandler v1.0.1/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= 12 | github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= 13 | github.com/abdullin/seq v0.0.0-20160510034733-d5467c17e7af/go.mod h1:5Jv4cbFiHJMsVxt52+i0Ha45fjshj6wxYr1r19tB9bw= 14 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 15 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 16 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 17 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 18 | github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= 19 | github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= 20 | github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= 21 | github.com/armon/go-metrics v0.3.4/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= 22 | github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= 23 | github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= 24 | github.com/aws/aws-sdk-go v1.15.24/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= 25 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 26 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 27 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 28 | github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= 29 | github.com/bits-and-blooms/bitset v1.3.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= 30 | github.com/bits-and-blooms/bitset v1.7.0 h1:YjAGVd3XmtK9ktAbX8Zg2g2PwLIMjGREZJHlV4j7NEo= 31 | github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= 32 | github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= 33 | github.com/bits-and-blooms/bloom/v3 v3.3.1/go.mod h1:bhUUknWd5khVbTe4UgMCSiOOVJzr3tMoijSK3WwvW90= 34 | github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= 35 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 36 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 37 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 38 | github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= 39 | github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= 40 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 41 | github.com/coredns/coredns v1.1.2/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0= 42 | github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= 43 | github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= 44 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 45 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 46 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 47 | github.com/denverdino/aliyungo v0.0.0-20170926055100-d3308649c661/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= 48 | github.com/devopsfaith/bloomfilter v1.4.0 h1:h8UVgr0sWxwSciqAZKd4LFugljZk8a/+DW2omzVgB08= 49 | github.com/devopsfaith/bloomfilter v1.4.0/go.mod h1:LQuMjwtBekCk8qdTHOJ7wZpEYB3J6zQEwwCPABbWMLY= 50 | github.com/devopsfaith/flatmap v0.0.0-20200601181759-8521186182fc/go.mod h1:J9Y/58s7wx7HbHT3i4UKNwLGuBB9qCf0/JUdEFGDPmA= 51 | github.com/devopsfaith/krakend-consul v1.4.0/go.mod h1:76v8AByTEzlBbiGWEzHFvT4g9BOtr8fpotYIMoac64Q= 52 | github.com/devopsfaith/krakend-gologging v1.4.0/go.mod h1:0IBy8rXN5ck5nHp5DRxOki3nPVm3Akta4X78qeNATwA= 53 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 54 | github.com/digitalocean/godo v1.1.1/go.mod h1:h6faOIcZ8lWIwNQ+DN7b3CgX4Kwby5T+nbpNqkUIozU= 55 | github.com/digitalocean/godo v1.10.0/go.mod h1:h6faOIcZ8lWIwNQ+DN7b3CgX4Kwby5T+nbpNqkUIozU= 56 | github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= 57 | github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= 58 | github.com/elazarl/go-bindata-assetfs v0.0.0-20160803192304-e1a2a7ec64b0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= 59 | github.com/envoyproxy/go-control-plane v0.8.0/go.mod h1:GSSbY9P1neVhdY7G4wu+IK1rk/dqhiCC/4ExuWJZVuk= 60 | github.com/envoyproxy/protoc-gen-validate v0.0.14/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 61 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 62 | github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= 63 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 64 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 65 | github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= 66 | github.com/gin-gonic/gin v1.1.5-0.20170702092826-d459835d2b07/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= 67 | github.com/go-chi/chi v4.0.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= 68 | github.com/go-contrib/uuid v1.2.0/go.mod h1:R9zf5oXjEfersQve5ceWY37X8JR3qtDTU2WSVxbWXGE= 69 | github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= 70 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 71 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 72 | github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= 73 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 74 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 75 | github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= 76 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 77 | github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= 78 | github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= 79 | github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= 80 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 81 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 82 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 83 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 84 | github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 85 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 86 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 87 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 88 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 89 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 90 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 91 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 92 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 93 | github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= 94 | github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= 95 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 96 | github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= 97 | github.com/gophercloud/gophercloud v0.0.0-20180828235145-f29afc2cceca/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= 98 | github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 99 | github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 100 | github.com/gorilla/mux v1.6.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 101 | github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= 102 | github.com/hashicorp/consul v1.6.10/go.mod h1:rQ8araXgdbaX5K/ZoZHDpVSIX+jF0o0Zm7+A9egVur4= 103 | github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= 104 | github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= 105 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 106 | github.com/hashicorp/go-bexpr v0.1.2/go.mod h1:ANbpTX1oAql27TZkKVeW8p1w8NTdnyzPe/0qqPCKohU= 107 | github.com/hashicorp/go-checkpoint v0.0.0-20171009173528-1545e56e46de/go.mod h1:xIwEieBHERyEvaeKF/TcHh1Hu+lxPM+n2vT1+g9I4m4= 108 | github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 109 | github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 110 | github.com/hashicorp/go-connlimit v0.1.0/go.mod h1:OUj9FGL1tPIhl/2RCfzYHrIiWj+VVPGNyVPnUX8AqS0= 111 | github.com/hashicorp/go-discover v0.0.0-20190403160810-22221edb15cd/go.mod h1:ueUgD9BeIocT7QNuvxSyJyPAM9dfifBcaWmeybb67OY= 112 | github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= 113 | github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= 114 | github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= 115 | github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= 116 | github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 117 | github.com/hashicorp/go-memdb v0.0.0-20180223233045-1289e7fffe71/go.mod h1:kbfItVoBJwCfKXDXN4YoAXjxcFVZ7MRrJzyTX6H4giE= 118 | github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= 119 | github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= 120 | github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= 121 | github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= 122 | github.com/hashicorp/go-raftchunking v0.6.1/go.mod h1:cGlg3JtDy7qy6c/3Bu660Mic1JF+7lWqIwCFSb08fX0= 123 | github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= 124 | github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= 125 | github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= 126 | github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= 127 | github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= 128 | github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= 129 | github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= 130 | github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 131 | github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 132 | github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 133 | github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= 134 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 135 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 136 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 137 | github.com/hashicorp/hil v0.0.0-20160711231837-1e86c6b523c5/go.mod h1:KHvg/R2/dPtaePb16oW4qIyzkMxXOL38xjRN64adsts= 138 | github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= 139 | github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= 140 | github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= 141 | github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= 142 | github.com/hashicorp/memberlist v0.1.5/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= 143 | github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69/go.mod h1:/z+jUGRBlwVpUZfjute9jWaF6/HuhjuFQuL1YXzVD1Q= 144 | github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= 145 | github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= 146 | github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= 147 | github.com/hashicorp/serf v0.8.6/go.mod h1:P/AVgr4UHsUYqVHG1y9eFhz8S35pqhGhLZaDpfGKIMo= 148 | github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= 149 | github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= 150 | github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443/go.mod h1:bEpDU35nTu0ey1EXjwNwPjI9xErAsoOCmcMb9GKvyxo= 151 | github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= 152 | github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= 153 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 154 | github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 155 | github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= 156 | github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= 157 | github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= 158 | github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 159 | github.com/joyent/triton-go v0.0.0-20180628001255-830d2b111e62/go.mod h1:U+RSyWxWd04xTqnuOQxnai7XGS2PrPY2cfGoDKtMHjA= 160 | github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 161 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 162 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 163 | github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 164 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 165 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 166 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 167 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 168 | github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= 169 | github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= 170 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 171 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 172 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 173 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 174 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 175 | github.com/letgoapp/krakend-consul v0.0.0-20180406153423-b4d135ce6994/go.mod h1:cFcys9MH7oD42Dw7NVy/971Sn4zuCmtE/XjwVgk3Srw= 176 | github.com/luraproject/lura v1.4.0/go.mod h1:KIo1/+nsRZVxIO04Hkbth0GXSSzypvkFpF5KaIoLvlo= 177 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 178 | github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 179 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 180 | github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= 181 | github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= 182 | github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= 183 | github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= 184 | github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 185 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 186 | github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= 187 | github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= 188 | github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= 189 | github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= 190 | github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= 191 | github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= 192 | github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 193 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 194 | github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= 195 | github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= 196 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 197 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 198 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 199 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 200 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 201 | github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2/go.mod h1:TLb2Sg7HQcgGdloNxkrmtgDNR9uVYF3lfdFIN4Ro6Sk= 202 | github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= 203 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 204 | github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= 205 | github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= 206 | github.com/packethost/packngo v0.1.1-0.20180711074735-b9cb5096f54c/go.mod h1:otzZQXgoO96RTzDB/Hycg0qZcXZsWJGJRSXbmEIJ+4M= 207 | github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 208 | github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 209 | github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= 210 | github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 211 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 212 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 213 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 214 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 215 | github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= 216 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 217 | github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= 218 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 219 | github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= 220 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 221 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 222 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 223 | github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 224 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 225 | github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= 226 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 227 | github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 228 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 229 | github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= 230 | github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03/go.mod h1:gRAiPF5C5Nd0eyyRdqIu9qTiFSoZzpTq727b5B8fkkU= 231 | github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 232 | github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 233 | github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= 234 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= 235 | github.com/shirou/gopsutil v0.0.0-20181107111621-48177ef5f880/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= 236 | github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= 237 | github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= 238 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 239 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 240 | github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 241 | github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= 242 | github.com/softlayer/softlayer-go v0.0.0-20180806151055-260589d94c7d/go.mod h1:Cw4GTlQccdRGSEf6KiMju767x0NEHE0YIVPJSaXjlsw= 243 | github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 244 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 245 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 246 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 247 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 248 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 249 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 250 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 251 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 252 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 253 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 254 | github.com/tannerryan/ring v1.1.2 h1:iXayOjqHQOLzuy9GwSKuG3nhWfzQkldMlQivcgIr7gQ= 255 | github.com/tannerryan/ring v1.1.2/go.mod h1:DkELJEjbZhJBtFKR9Xziwj3HKZnb/knRgljNqp65vH4= 256 | github.com/tent/http-link-go v0.0.0-20130702225549-ac974c61c2f9/go.mod h1:RHkNRtSLfOK7qBTHaeSX1D6BNpI3qw7NTxsmNr4RvN8= 257 | github.com/tmthrgd/asm v0.0.0-20180106020940-4be3ab5ca701/go.mod h1:eH36fdzMoEurHdB3rmh4o3AGc67+EDT2TGaYzwdqTe4= 258 | github.com/tmthrgd/atomics v0.0.0-20180217065130-6910de195248/go.mod h1:J2+dTgaX/1g3PkyL6sLBglBWfaLmAp5bQbRhSfKw9XI= 259 | github.com/tmthrgd/atomics v0.0.0-20190904060638-dc7a5fcc7e0d h1:2QXSQjy/gDm0QeP9G9NaO9Hm2Cl1LAle4ZV0JeYK7XY= 260 | github.com/tmthrgd/atomics v0.0.0-20190904060638-dc7a5fcc7e0d/go.mod h1:J2+dTgaX/1g3PkyL6sLBglBWfaLmAp5bQbRhSfKw9XI= 261 | github.com/tmthrgd/go-bitset v0.0.0-20180828125936-62ad9ed7ff29/go.mod h1:SooM96OIpihI7iMZhVGbpiiO9Qevqv8vXxHlwNtefd4= 262 | github.com/tmthrgd/go-bitset v0.0.0-20190904054048-394d9a556c05 h1:5jOF3BEex8XyBKMbaDUN1SiPQJRAKVuP24/sbwC2aWA= 263 | github.com/tmthrgd/go-bitset v0.0.0-20190904054048-394d9a556c05/go.mod h1:SooM96OIpihI7iMZhVGbpiiO9Qevqv8vXxHlwNtefd4= 264 | github.com/tmthrgd/go-bitwise v0.0.0-20170218093117-01bef038b6bd/go.mod h1:Ba4ek/h+sJUzTQ03ZGD1r0lazhxd7CBoEQzFk/icxxU= 265 | github.com/tmthrgd/go-bitwise v0.0.0-20190904053232-1430ee983fca h1:Ns4/7EvYZ7FxKiKnEMkMMAPtoR/ifUgRsvk7lzlOtPY= 266 | github.com/tmthrgd/go-bitwise v0.0.0-20190904053232-1430ee983fca/go.mod h1:Ba4ek/h+sJUzTQ03ZGD1r0lazhxd7CBoEQzFk/icxxU= 267 | github.com/tmthrgd/go-byte-test v0.0.0-20170223110042-2eb5216b83f7/go.mod h1:MEz1Lt0fxSL/ZgE7VN3yUJV0sP5I5aYecYLd9y/viEs= 268 | github.com/tmthrgd/go-byte-test v0.0.0-20190904060354-2794345b9929 h1:EV5x10oS2/QrR1RwniFQW1i22d/iyX/emvvMjHT32CA= 269 | github.com/tmthrgd/go-byte-test v0.0.0-20190904060354-2794345b9929/go.mod h1:MEz1Lt0fxSL/ZgE7VN3yUJV0sP5I5aYecYLd9y/viEs= 270 | github.com/tmthrgd/go-hex v0.0.0-20180828131331-d1fb3dbb16a1/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= 271 | github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= 272 | github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= 273 | github.com/tmthrgd/go-memset v0.0.0-20180828131805-6f4e59bf1e1d/go.mod h1:xUkvcKF3VBDKFmmqCtW333lognWBHzSScj4fgjVB0Ek= 274 | github.com/tmthrgd/go-memset v0.0.0-20190904060434-6fb7a21f88f1 h1:KfSLDmc6rrLHr//urKbNApu51nt2AzdrwBCxqq0gRlk= 275 | github.com/tmthrgd/go-memset v0.0.0-20190904060434-6fb7a21f88f1/go.mod h1:xUkvcKF3VBDKFmmqCtW333lognWBHzSScj4fgjVB0Ek= 276 | github.com/tmthrgd/go-popcount v0.0.0-20180111143836-3918361d3e97/go.mod h1:FcUQfrsAsSSqM3n9xf4EtPzB8tWzt58/y0AV+wNNM8Q= 277 | github.com/tmthrgd/go-popcount v0.0.0-20190904054823-afb1ace8b04f h1:Phf2p9+twoHct5ZjSTrI8K7iWeSxO4x1p5pShTl0J00= 278 | github.com/tmthrgd/go-popcount v0.0.0-20190904054823-afb1ace8b04f/go.mod h1:FcUQfrsAsSSqM3n9xf4EtPzB8tWzt58/y0AV+wNNM8Q= 279 | github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= 280 | github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= 281 | github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= 282 | github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 h1:QEePdg0ty2r0t1+qwfZmQ4OOl/MB2UXIeJSpIZv56lg= 283 | github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43/go.mod h1:OYRfF6eb5wY9VRFkXJH8FFBi3plw2v+giaIu7P054pM= 284 | github.com/ugorji/go v0.0.0-20180112141927-9831f2c3ac10/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= 285 | github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= 286 | github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= 287 | github.com/vmware/govmomi v0.18.0/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= 288 | github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= 289 | github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= 290 | github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= 291 | github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= 292 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 293 | golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 294 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 295 | golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= 296 | golang.org/x/crypto v0.0.0-20191106202628-ed6320f186d4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 297 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 298 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 299 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 300 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 301 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 302 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 303 | golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 304 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 305 | golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 306 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 307 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 308 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 309 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 310 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 311 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 312 | golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 313 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 314 | golang.org/x/oauth2 v0.0.0-20170807180024-9a379c6b3e95/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 315 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 316 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 317 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 318 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 319 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 320 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 321 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 322 | golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 323 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 324 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 325 | golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 326 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 327 | golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 328 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 329 | golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 330 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 331 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 332 | golang.org/x/sys v0.0.0-20190508220229-2d0786266e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 333 | golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 334 | golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 335 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 336 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 337 | golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 338 | golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= 339 | golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 340 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 341 | golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 342 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 343 | golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 344 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 345 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 346 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 347 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 348 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 349 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 350 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 351 | golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 352 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 353 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 354 | google.golang.org/api v0.0.0-20180829000535-087779f1d2c9/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= 355 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 356 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 357 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 358 | google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 359 | google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= 360 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 361 | google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 362 | google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 363 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 364 | gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= 365 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 366 | gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= 367 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 368 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 369 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 370 | gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= 371 | gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= 372 | gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= 373 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 374 | gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= 375 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 376 | gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= 377 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 378 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 379 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 380 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 381 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 382 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 383 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 384 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 385 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 386 | istio.io/gogo-genproto v0.0.0-20190124151557-6d926a6e6feb/go.mod h1:eIDJ6jNk/IeJz6ODSksHl5Aiczy5JUq6vFhJWI5OtiI= 387 | k8s.io/api v0.0.0-20180806132203-61b11ee65332/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= 388 | k8s.io/api v0.0.0-20190325185214-7544f9db76f6/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= 389 | k8s.io/apimachinery v0.0.0-20180821005732-488889b0007f/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= 390 | k8s.io/apimachinery v0.0.0-20190223001710-c182ff3b9841/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= 391 | k8s.io/client-go v8.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= 392 | -------------------------------------------------------------------------------- /benchmarks/ring.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build ring 16 | // +build ring 17 | 18 | package benchmarks 19 | 20 | import "github.com/tannerryan/ring" 21 | 22 | type bloomFilter ring.Ring 23 | 24 | func (f *bloomFilter) Add(hash []byte) { 25 | ((*ring.Ring)(f)).Add(hash) 26 | } 27 | 28 | func (f *bloomFilter) Has(hash []byte) bool { 29 | return ((*ring.Ring)(f)).Test(hash) 30 | } 31 | 32 | func newBF(capacity int, fpr float64) *bloomFilter { 33 | f, err := ring.Init(capacity, fpr) 34 | if err != nil { 35 | panic(err) 36 | } 37 | return (*bloomFilter)(f) 38 | } 39 | -------------------------------------------------------------------------------- /benchmarks/sync.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build sync 16 | // +build sync 17 | 18 | package benchmarks 19 | 20 | import ( 21 | "encoding/binary" 22 | 23 | "github.com/greatroar/blobloom" 24 | ) 25 | 26 | type bloomFilter blobloom.SyncFilter 27 | 28 | func (f *bloomFilter) Add(hash []byte) { 29 | h := binary.BigEndian.Uint64(hash[:8]) 30 | ((*blobloom.SyncFilter)(f)).Add(h) 31 | } 32 | 33 | func (f *bloomFilter) Has(hash []byte) bool { 34 | h := binary.BigEndian.Uint64(hash[:8]) 35 | return ((*blobloom.SyncFilter)(f)).Has(h) 36 | } 37 | 38 | func newBF(capacity int, fpr float64) *bloomFilter { 39 | f := blobloom.NewSync(blobloom.Optimize(blobloom.Config{ 40 | Capacity: uint64(capacity), 41 | FPRate: fpr, 42 | })) 43 | return (*bloomFilter)(f) 44 | } 45 | -------------------------------------------------------------------------------- /benchmarks/willf.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build willf 16 | // +build willf 17 | 18 | package benchmarks 19 | 20 | import "github.com/bits-and-blooms/bloom/v3" 21 | 22 | type bloomFilter bloom.BloomFilter 23 | 24 | func (f *bloomFilter) Add(hash []byte) { 25 | ((*bloom.BloomFilter)(f)).Add(hash) 26 | } 27 | 28 | func (f *bloomFilter) Has(hash []byte) bool { 29 | return ((*bloom.BloomFilter)(f)).Test(hash) 30 | } 31 | 32 | func newBF(capacity int, fpr float64) *bloomFilter { 33 | f := bloom.NewWithEstimates(uint(capacity), fpr) 34 | return (*bloomFilter)(f) 35 | } 36 | -------------------------------------------------------------------------------- /bloomfilter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2024 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package blobloom implements blocked Bloom filters. 16 | // 17 | // Blocked Bloom filters are an approximate set data structure: if a key has 18 | // been added to a filter, a lookup of that key returns true, but if the key 19 | // has not been added, there is a non-zero probability that the lookup still 20 | // returns true (a false positive). False negatives are impossible: if the 21 | // lookup for a key returns false, that key has not been added. 22 | // 23 | // In this package, keys are represented exclusively as hashes. Client code 24 | // is responsible for supplying a 64-bit hash value. 25 | // 26 | // Compared to standard Bloom filters, blocked Bloom filters use the CPU 27 | // cache more efficiently. A blocked Bloom filter is an array of ordinary 28 | // Bloom filters of fixed size BlockBits (the blocks). The lower half of the 29 | // hash selects the block to use. 30 | // 31 | // To achieve the same false positive rate (FPR) as a standard Bloom filter, 32 | // a blocked Bloom filter requires more memory. For an FPR of at most 2e-6 33 | // (two in a million), it uses ~20% more memory. At 1e-10, the space required 34 | // is double that of standard Bloom filter. 35 | // 36 | // For more details, see the 2010 paper by Putze, Sanders and Singler, 37 | // https://www.cs.amherst.edu/~ccmcgeoch/cs34/papers/cacheefficientbloomfilters-jea.pdf. 38 | package blobloom 39 | 40 | import "math" 41 | 42 | // BlockBits is the number of bits per block and the minimum number of bits 43 | // in a Filter. 44 | // 45 | // The value of this constant is chosen to match the L1 cache line size 46 | // of popular architectures (386, amd64, arm64). 47 | const BlockBits = 512 48 | 49 | // MaxBits is the maximum number of bits supported by a Filter. 50 | const MaxBits = BlockBits << 32 // 256GiB. 51 | 52 | // A Filter is a blocked Bloom filter. 53 | type Filter struct { 54 | b []block // Shards. 55 | k int // Number of hash functions required. 56 | } 57 | 58 | // New constructs a Bloom filter with given numbers of bits and hash functions. 59 | // 60 | // The number of bits should be at least BlockBits; smaller values are silently 61 | // increased. 62 | // 63 | // The number of hashes reflects the number of hashes synthesized from the 64 | // single hash passed in by the client. It is silently increased to two if 65 | // a lower value is given. 66 | func New(nbits uint64, nhashes int) *Filter { 67 | nbits, nhashes = fixBitsAndHashes(nbits, nhashes) 68 | 69 | return &Filter{ 70 | b: make([]block, nbits/BlockBits), 71 | k: nhashes, 72 | } 73 | } 74 | 75 | func fixBitsAndHashes(nbits uint64, nhashes int) (uint64, int) { 76 | if nbits < 1 { 77 | nbits = BlockBits 78 | } 79 | if nhashes < 2 { 80 | nhashes = 2 81 | } 82 | if nbits > MaxBits { 83 | panic("nbits exceeds MaxBits") 84 | } 85 | 86 | // Round nbits up to a multiple of BlockBits. 87 | if nbits%BlockBits != 0 { 88 | nbits += BlockBits - nbits%BlockBits 89 | } 90 | 91 | return nbits, nhashes 92 | } 93 | 94 | // Add insert a key with hash value h into f. 95 | func (f *Filter) Add(h uint64) { 96 | h1, h2 := uint32(h>>32), uint32(h) 97 | b := getblock(f.b, h2) 98 | 99 | for i := 1; i < f.k; i++ { 100 | h1, h2 = doublehash(h1, h2, i) 101 | b.setbit(h1) 102 | } 103 | } 104 | 105 | // log(1 - 1/BlockBits) computed with 128 bits precision. 106 | // Note that this is extremely close to -1/BlockBits, 107 | // which is what Wikipedia would have us use: 108 | // https://en.wikipedia.org/wiki/Bloom_filter#Approximating_the_number_of_items_in_a_Bloom_filter. 109 | const log1minus1divBlockbits = -0.0019550348358033505576274922418668121377 110 | 111 | // Cardinality estimates the number of distinct keys added to f. 112 | // 113 | // The estimate is most reliable when f is filled to roughly its capacity. 114 | // It gets worse as f gets more densely filled. When one of the blocks is 115 | // entirely filled, the estimate becomes +Inf. 116 | // 117 | // The return value is the maximum likelihood estimate of Papapetrou, Siberski 118 | // and Nejdl, summed over the blocks 119 | // (https://www.win.tue.nl/~opapapetrou/papers/Bloomfilters-DAPD.pdf). 120 | func (f *Filter) Cardinality() float64 { 121 | return cardinality(f.k, f.b, onescount) 122 | } 123 | 124 | func cardinality(nhashes int, b []block, onescount func(*block) int) float64 { 125 | k := float64(nhashes - 1) 126 | 127 | var n float64 128 | for i := range b { 129 | ones := onescount(&b[i]) 130 | if ones == 0 { 131 | continue 132 | } 133 | n += math.Log1p(-float64(ones) / BlockBits) 134 | } 135 | 136 | // The probability of some bit not being set in a single insertion is 137 | // p0 = (1-1/BlockBits)^k. Divide by the log of that. 138 | logP0 := k * log1minus1divBlockbits 139 | return n / logP0 140 | } 141 | 142 | // Clear resets f to its empty state. 143 | func (f *Filter) Clear() { 144 | for i := 0; i < len(f.b); i++ { 145 | f.b[i] = block{} 146 | } 147 | } 148 | 149 | // Empty reports whether f contains no keys. 150 | func (f *Filter) Empty() bool { 151 | for i := 0; i < len(f.b); i++ { 152 | if f.b[i] != (block{}) { 153 | return false 154 | } 155 | } 156 | return true 157 | } 158 | 159 | // Equals returns true if f and g contain the same keys (in terms of Has) 160 | // when used with the same hash function. 161 | func (f *Filter) Equals(g *Filter) bool { 162 | if g.k != f.k || len(g.b) != len(f.b) { 163 | return false 164 | } 165 | for i := range g.b { 166 | if f.b[i] != g.b[i] { 167 | return false 168 | } 169 | } 170 | return true 171 | } 172 | 173 | // Fill set f to a completely full filter. 174 | // After Fill, Has returns true for any key. 175 | func (f *Filter) Fill() { 176 | for i := 0; i < len(f.b); i++ { 177 | for j := 0; j < blockWords; j++ { 178 | f.b[i][j] = ^uint32(0) 179 | } 180 | } 181 | } 182 | 183 | // Has reports whether a key with hash value h has been added. 184 | // It may return a false positive. 185 | func (f *Filter) Has(h uint64) bool { 186 | h1, h2 := uint32(h>>32), uint32(h) 187 | b := getblock(f.b, h2) 188 | 189 | for i := 1; i < f.k; i++ { 190 | h1, h2 = doublehash(h1, h2, i) 191 | if !b.getbit(h1) { 192 | return false 193 | } 194 | } 195 | return true 196 | } 197 | 198 | // doublehash generates the hash values to use in iteration i of 199 | // enhanced double hashing from the values h1, h2 of the previous iteration. 200 | // See https://www.ccs.neu.edu/home/pete/pub/bloom-filters-verification.pdf. 201 | func doublehash(h1, h2 uint32, i int) (uint32, uint32) { 202 | h1 = h1 + h2 203 | h2 = h2 + uint32(i) 204 | return h1, h2 205 | } 206 | 207 | // NumBits returns the number of bits of f. 208 | func (f *Filter) NumBits() uint64 { 209 | return BlockBits * uint64(len(f.b)) 210 | } 211 | 212 | func checkBinop(f, g *Filter) { 213 | if len(f.b) != len(g.b) { 214 | panic("Bloom filters do not have the same number of bits") 215 | } 216 | if f.k != g.k { 217 | panic("Bloom filters do not have the same number of hash functions") 218 | } 219 | } 220 | 221 | // Intersect sets f to the intersection of f and g. 222 | // 223 | // Intersect panics when f and g do not have the same number of bits and 224 | // hash functions. Both Filters must be using the same hash function(s), 225 | // but Intersect cannot check this. 226 | // 227 | // Since Bloom filters may return false positives, Has may return true for 228 | // a key that was not in both f and g. 229 | // 230 | // After Intersect, the estimates from f.Cardinality and f.FPRate should be 231 | // considered unreliable. 232 | func (f *Filter) Intersect(g *Filter) { 233 | checkBinop(f, g) 234 | f.intersect(g) 235 | } 236 | 237 | // Union sets f to the union of f and g. 238 | // 239 | // Union panics when f and g do not have the same number of bits and 240 | // hash functions. Both Filters must be using the same hash function(s), 241 | // but Union cannot check this. 242 | func (f *Filter) Union(g *Filter) { 243 | checkBinop(f, g) 244 | f.union(g) 245 | } 246 | 247 | const ( 248 | wordSize = 32 249 | blockWords = BlockBits / wordSize 250 | ) 251 | 252 | // A block is a fixed-size Bloom filter, used as a shard of a Filter. 253 | type block [blockWords]uint32 254 | 255 | func getblock(b []block, h2 uint32) *block { 256 | i := reducerange(h2, uint32(len(b))) 257 | return &b[i] 258 | } 259 | 260 | // reducerange maps i to an integer in the range [0,n). 261 | // https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ 262 | func reducerange(i, n uint32) uint32 { 263 | return uint32((uint64(i) * uint64(n)) >> 32) 264 | } 265 | 266 | // getbit reports whether bit (i modulo BlockBits) is set. 267 | func (b *block) getbit(i uint32) bool { 268 | bit := uint32(1) << (i % wordSize) 269 | x := (*b)[(i/wordSize)%blockWords] & bit 270 | return x != 0 271 | } 272 | 273 | // setbit sets bit (i modulo BlockBits) of b. 274 | func (b *block) setbit(i uint32) { 275 | bit := uint32(1) << (i % wordSize) 276 | (*b)[(i/wordSize)%blockWords] |= bit 277 | } 278 | -------------------------------------------------------------------------------- /bloomfilter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobloom 16 | 17 | import ( 18 | "crypto/sha256" 19 | "encoding/binary" 20 | "encoding/hex" 21 | "math" 22 | "math/rand" 23 | "testing" 24 | 25 | "github.com/stretchr/testify/assert" 26 | ) 27 | 28 | func TestSimple(t *testing.T) { 29 | t.Parallel() 30 | 31 | keys := randomU64(10000, 0x758e326) 32 | 33 | for _, config := range []struct { 34 | nbits uint64 35 | nhashes int 36 | }{ 37 | {1, 2}, 38 | {1024, 4}, 39 | {100, 3}, 40 | {10000, 7}, 41 | {1000000, 14}, 42 | } { 43 | f := New(config.nbits, config.nhashes) 44 | assert.GreaterOrEqual(t, f.NumBits(), config.nbits) 45 | assert.LessOrEqual(t, f.NumBits(), config.nbits+BlockBits) 46 | assert.True(t, f.Empty()) 47 | 48 | for _, k := range keys { 49 | assert.False(t, f.Has(k)) 50 | } 51 | for _, k := range keys { 52 | f.Add(k) 53 | } 54 | assert.False(t, f.Empty()) 55 | for _, k := range keys { 56 | assert.True(t, f.Has(k)) 57 | } 58 | 59 | f.Clear() 60 | assert.True(t, f.Empty()) 61 | for _, k := range keys { 62 | assert.False(t, f.Has(k)) 63 | } 64 | 65 | f.Fill() 66 | assert.False(t, f.Empty()) 67 | for _, k := range keys { 68 | assert.True(t, f.Has(k)) 69 | } 70 | } 71 | } 72 | 73 | func TestUse(t *testing.T) { 74 | t.Parallel() 75 | 76 | const n = 100000 77 | 78 | // For FPR = .01, n = 100000, the optimal number of bits is 958505.84 79 | // for a standard Bloom filter. 80 | f := NewOptimized(Config{ 81 | Capacity: n, 82 | FPRate: .01, 83 | }) 84 | if f.NumBits() < 958506 { 85 | t.Fatalf("bloom filter with %d bits too small", f.NumBits()) 86 | } 87 | 88 | t.Logf("k = %d; m/n = %d/%d = %.3f", 89 | f.k, f.NumBits(), n, float64(f.NumBits())/n) 90 | 91 | // Generate random hash values for n keys. Pretend the keys are all distinct, 92 | // even if the hashes are not. 93 | r := rand.New(rand.NewSource(0xb1007)) 94 | hashes := make([]uint64, n) 95 | for i := range hashes { 96 | hashes[i] = r.Uint64() 97 | } 98 | 99 | for _, h := range hashes { 100 | f.Add(h) 101 | } 102 | 103 | for _, h := range hashes { 104 | if !f.Has(h) { 105 | t.Errorf("%032x added to Bloom filter but not found", h) 106 | } 107 | } 108 | 109 | // Generate some more random hashes to get a sense of the FPR. 110 | // Pretend these represent unique keys, distinct from the ones we added. 111 | const nTest = 10000 112 | fp := 0 113 | for i := 0; i < nTest; i++ { 114 | if f.Has(r.Uint64()) { 115 | fp++ 116 | } 117 | } 118 | 119 | fpr := float64(fp) / nTest 120 | assert.Less(t, fpr, .02) 121 | t.Logf("FPR = %.5f\n", fpr) 122 | } 123 | 124 | // Test robustness against 32-bit hash functions. 125 | func TestHash32(t *testing.T) { 126 | t.Parallel() 127 | 128 | const n = 400 129 | 130 | f := NewOptimized(Config{ 131 | Capacity: n, 132 | FPRate: .01, 133 | }) 134 | 135 | r := rand.New(rand.NewSource(32)) 136 | 137 | for i := 0; i < n; i++ { 138 | f.Add(uint64(r.Uint32())) 139 | } 140 | 141 | const nrounds = 8 142 | fp := 0 143 | for i := n; i < nrounds*n; i++ { 144 | if f.Has(uint64(r.Uint32())) { 145 | fp++ 146 | } 147 | } 148 | 149 | fprate := float64(fp) / (nrounds * n) 150 | t.Logf("FP rate = %.2f%%", 100*fprate) 151 | assert.LessOrEqual(t, fprate, .1) 152 | } 153 | 154 | func TestDoubleHashing(t *testing.T) { 155 | t.Parallel() 156 | 157 | var h1, h2 uint32 = 0, 0 158 | 159 | for i := 0; i < 20; i++ { 160 | h1, h2 = doublehash(h1, h2, i) 161 | assert.NotEqual(t, h2, 0) 162 | } 163 | } 164 | 165 | func TestReducerange(t *testing.T) { 166 | t.Parallel() 167 | 168 | for i := 0; i < 40000; i++ { 169 | m := rand.Uint32() 170 | j := reducerange(rand.Uint32(), m) 171 | if m == 0 { 172 | assert.Equal(t, j, 0) 173 | } 174 | assert.Less(t, j, m) 175 | } 176 | } 177 | 178 | func TestCardinality(t *testing.T) { 179 | t.Parallel() 180 | 181 | const cap = 1e4 182 | f := NewOptimized(Config{ 183 | Capacity: cap, 184 | FPRate: .0015, 185 | }) 186 | 187 | assert.EqualValues(t, 0, f.Cardinality()) 188 | 189 | r := rand.New(rand.NewSource(0x81feae2b)) 190 | 191 | var sumN, sumNhat float64 192 | for n := 1.0; n <= 5*cap; n++ { 193 | f.Add(r.Uint64()) 194 | 195 | nhat := f.Cardinality() 196 | assert.InDelta(t, 1, nhat/float64(n), 0.09) 197 | 198 | sumN += n 199 | sumNhat += nhat 200 | if int(n)%cap == 0 { 201 | // On average, we want to be less than a percent off. 202 | assert.InDelta(t, 1, sumNhat/sumN, 0.008) 203 | } 204 | } 205 | } 206 | 207 | func TestCardinalityFull(t *testing.T) { 208 | t.Parallel() 209 | 210 | f := New(BlockBits, 2) 211 | for i := range f.b { 212 | for j := range f.b[i] { 213 | f.b[i][j] = ^uint32(0) 214 | } 215 | } 216 | 217 | assert.Equal(t, math.Inf(+1), f.Cardinality()) 218 | } 219 | 220 | func TestIntersect(t *testing.T) { 221 | t.Parallel() 222 | 223 | const n uint64 = 1e4 224 | const seed = 0x5544332211 225 | hashes := randomU64(int(n), seed) 226 | 227 | f := NewOptimized(Config{Capacity: n, FPRate: 1e-3}) 228 | g := NewOptimized(Config{Capacity: n, FPRate: 1e-3}) 229 | i := NewOptimized(Config{Capacity: n, FPRate: 1e-3}) 230 | 231 | for _, h := range hashes[:n/3] { 232 | f.Add(h) 233 | } 234 | for _, h := range hashes[n/3 : 2*n/3] { 235 | f.Add(h) 236 | g.Add(h) 237 | i.Add(h) 238 | } 239 | for _, h := range hashes[n/3:] { 240 | g.Add(h) 241 | } 242 | 243 | expectFPR := math.Min(f.FPRate(n), g.FPRate(n)) 244 | 245 | f.Intersect(g) 246 | assert.NotEqual(t, i, g) 247 | 248 | for _, h := range hashes[n/3 : 2*n/3] { 249 | assert.True(t, f.Has(h)) 250 | } 251 | 252 | var fp uint64 253 | for _, h := range hashes { 254 | if f.Has(h) && !i.Has(h) { 255 | fp++ 256 | } 257 | } 258 | actualFPR := float64(fp) / float64(n) 259 | assert.Less(t, actualFPR, 2*expectFPR) 260 | t.Logf("FPR = %f", actualFPR) 261 | 262 | assert.Panics(t, func() { f.Intersect(New(f.NumBits(), 9)) }) 263 | assert.Panics(t, func() { f.Union(New(n+BlockBits, f.k)) }) 264 | } 265 | 266 | func TestUnion(t *testing.T) { 267 | t.Parallel() 268 | 269 | const n = 1e5 270 | hashes := randomU64(n, 0xa6e98fb) 271 | 272 | f := New(n, 5) 273 | g := New(n, 5) 274 | u := New(n, 5) 275 | 276 | for _, h := range hashes[:n/2] { 277 | f.Add(h) 278 | u.Add(h) 279 | } 280 | for _, h := range hashes[n/2:] { 281 | g.Add(h) 282 | u.Add(h) 283 | } 284 | 285 | assert.NotEqual(t, f, g) 286 | 287 | f.Union(g) 288 | assert.Equal(t, u, f) 289 | assert.NotEqual(t, u, g) 290 | 291 | g.Union(f) 292 | assert.Equal(t, u, g) 293 | 294 | assert.Panics(t, func() { f.Union(New(n, 4)) }) 295 | assert.Panics(t, func() { f.Union(New(n+BlockBits, 5)) }) 296 | } 297 | 298 | func randomU64(n int, seed int64) []uint64 { 299 | r := rand.New(rand.NewSource(seed)) 300 | p := make([]uint64, n) 301 | for i := range p { 302 | p[i] = r.Uint64() 303 | } 304 | return p 305 | } 306 | 307 | func TestUnionSmall(t *testing.T) { 308 | t.Parallel() 309 | 310 | f := New(BlockBits, 2) 311 | g := New(BlockBits, 2) 312 | 313 | g.Add(42) 314 | 315 | f.Union(g) 316 | assert.True(t, f.Has(42)) 317 | } 318 | 319 | // This test ensures that the switch from 64-bit to 32-bit words did not 320 | // alter the little-endian serialization of blocks. 321 | func TestBlockLayout(t *testing.T) { 322 | t.Parallel() 323 | 324 | var b block 325 | b.setbit(0) 326 | b.setbit(1) 327 | b.setbit(111) 328 | b.setbit(499) 329 | 330 | assert.Equal(t, BlockBits, 8*binary.Size(b)) 331 | 332 | h := sha256.New() 333 | binary.Write(h, binary.LittleEndian, b) 334 | expect := "aa7f8c411600fa387f0c10641eab428a7ed2f27a86171ac69f0e2087b2aa9140" 335 | assert.Equal(t, expect, hex.EncodeToString(h.Sum(nil))) 336 | } 337 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobloom_test 16 | 17 | import ( 18 | "crypto/sha256" 19 | "encoding/binary" 20 | "fmt" 21 | "hash/fnv" 22 | "io" 23 | "math" 24 | "sync" 25 | 26 | "github.com/greatroar/blobloom" 27 | ) 28 | 29 | func Example_fnv() { 30 | // This example uses the hash/fnv package from the standard Go library. 31 | 32 | f := blobloom.New(10000, 5) 33 | h := fnv.New64() 34 | 35 | messages := []string{ 36 | "Hello!", 37 | "Welcome!", 38 | "Mind your step!", 39 | "Have fun!", 40 | "Goodbye!", 41 | } 42 | 43 | for _, msg := range messages { 44 | h.Reset() 45 | io.WriteString(h, msg) 46 | f.Add(h.Sum64()) 47 | } 48 | 49 | for _, msg := range messages { 50 | h.Reset() 51 | io.WriteString(h, msg) 52 | if f.Has(h.Sum64()) { 53 | fmt.Println(msg) 54 | } else { 55 | panic("Bloom filter didn't get the message") 56 | } 57 | } 58 | 59 | // Output: 60 | // Hello! 61 | // Welcome! 62 | // Mind your step! 63 | // Have fun! 64 | // Goodbye! 65 | } 66 | 67 | func Example_sha224() { 68 | // If you have items addressed by a cryptographic hash, 69 | // you can use a prefix of it as the hash value for a Bloom filter. 70 | // 71 | // If the cryptohashes denote objects from an untrusted source, 72 | // the Bloom filter can be tricked into giving false positives for 73 | // chosen objects, because it only uses a small part of the hash 74 | // that can easily be broken (by a birthday attack). If that can 75 | // cause problems in your application, first run SipHash on the 76 | // full cryptohash to get the hash value for the Bloom filter: 77 | // 78 | // import "github.com/dchest/siphash" 79 | // h := siphash.Hash(secret1, secret2, key[:]) 80 | 81 | // A list of files, identified by their SHA-224. 82 | files := []string{ 83 | "\x85\x52\xd8\xb7\xa7\xdc\x54\x76\xcb\x9e\x25\xde\xe6\x9a\x80\x91\x29\x07\x64\xb7\xf2\xa6\x4f\xe6\xe7\x8e\x95\x68", 84 | "\xa0\xad\x8f\x63\x90\x72\x74\x7b\xc3\x43\x09\x45\x94\x0e\x7c\x73\xb8\x34\x93\xf1\x77\x90\x0f\xd2\x7d\x09\x65\x94", 85 | "\x7b\xd3\xdb\x48\x1e\x7b\x05\x2c\x88\x18\x68\xcc\x13\xc3\x04\x34\x43\x2d\x7b\x49\x24\x74\x70\x33\xd2\xe8\x6e\x73", 86 | } 87 | 88 | // first64 extracts the first 64 bits of a key as a uint64. 89 | // The choice of big vs. little-endian is arbitrary. 90 | first64 := func(key []byte) uint64 { 91 | return binary.BigEndian.Uint64(key[:8]) 92 | } 93 | 94 | f := blobloom.NewOptimized(blobloom.Config{Capacity: 600, FPRate: .002}) 95 | 96 | for _, filehash := range files { 97 | f.Add(first64([]byte(filehash))) 98 | } 99 | 100 | for _, s := range []string{"Hello, world!", "Goodbye"} { 101 | h := sha256.Sum224([]byte(s)) 102 | found := f.Has(first64(h[:])) 103 | if found { 104 | fmt.Printf("Found: %v\n", s) 105 | } 106 | } 107 | 108 | // Output: 109 | // Found: Hello, world! 110 | } 111 | 112 | func ExampleOptimize() { 113 | cfg := blobloom.Config{ 114 | // We want to insert a billion keys and get a false positive rate of 115 | // one in a million, but we only have 2GiB (= 2^31 bytes) to spare. 116 | Capacity: 1e9, 117 | FPRate: 1e-6, 118 | MaxBits: 8 * 1 << 31, 119 | } 120 | nbits, nhashes := blobloom.Optimize(cfg) 121 | fpr := blobloom.FPRate(cfg.Capacity, nbits, nhashes) 122 | 123 | // How big will the filter be and what FP rate will we achieve? 124 | fmt.Printf("size = %dMiB\nfpr = %.3f\n", nbits/(8<<20), fpr) 125 | 126 | // Output: 127 | // size = 2048MiB 128 | // fpr = 0.001 129 | } 130 | 131 | var hashes [200]uint64 132 | 133 | func init() { 134 | for i := range hashes { 135 | hashes[i] = uint64(i) 136 | } 137 | } 138 | 139 | func ExampleFilter_Cardinality_infinity() { 140 | // To handle the case of Cardinality returning +Inf, track the number of 141 | // calls to Add and compute the minimum. 142 | 143 | // This Bloom filter is constructed with too many hash functions 144 | // to force +Inf. 145 | f := blobloom.New(512, 100) 146 | var numAdded int 147 | 148 | add := func(h uint64) { 149 | f.Add(h) 150 | numAdded++ 151 | } 152 | 153 | for _, h := range hashes { 154 | add(h) 155 | } 156 | 157 | estimate := f.Cardinality() 158 | fmt.Printf("blobloom's estimate: %.2f\n", estimate) 159 | fmt.Printf("number of calls to Add: %d\n", numAdded) 160 | estimate = math.Min(estimate, float64(numAdded)) 161 | fmt.Printf("combined estimate: %.2f\n", estimate) 162 | 163 | // Output: 164 | // blobloom's estimate: +Inf 165 | // number of calls to Add: 200 166 | // combined estimate: 200.00 167 | } 168 | 169 | const nworkers = 4 170 | 171 | func getKeys(keys chan<- string) { 172 | keys <- "hello" 173 | keys <- "goodbye" 174 | close(keys) 175 | } 176 | 177 | func hash(key string) uint64 { 178 | h := fnv.New64() 179 | io.WriteString(h, key) 180 | return h.Sum64() 181 | } 182 | 183 | func ExampleFilter_Union() { 184 | // Union can be used to fill a Bloom filter using multiple goroutines. 185 | // 186 | // Each goroutine allocates a filter, so the memory use increases 187 | // by a factor nworkers-1 compared to a sequential version 188 | // or a SyncFilter. 189 | 190 | keys := make(chan string, nworkers) 191 | filters := make(chan *blobloom.Filter, nworkers) 192 | 193 | go getKeys(keys) 194 | 195 | for i := 0; i < nworkers; i++ { 196 | go func() { 197 | f := blobloom.New(1<<20, 6) 198 | for key := range keys { 199 | f.Add(hash(key)) 200 | } 201 | 202 | filters <- f 203 | }() 204 | } 205 | 206 | f := <-filters 207 | for i := 1; i < nworkers; i++ { 208 | f.Union(<-filters) 209 | } 210 | 211 | // Output: 212 | } 213 | 214 | func ExampleSyncFilter() { 215 | // Multiple goroutines can Add to a SyncFilter concurrently, 216 | // without requiring separate synchronization. 217 | 218 | f := blobloom.NewSync(1<<20, 6) 219 | var wg sync.WaitGroup 220 | 221 | add := func(hs []uint64) { 222 | for _, h := range hs { 223 | f.Add(h) 224 | } 225 | wg.Done() 226 | } 227 | 228 | wg.Add(2) 229 | half := len(hashes) / 2 230 | go add(hashes[:half]) 231 | go add(hashes[half:]) 232 | 233 | wg.Wait() // Wait for updating goroutines to complete. 234 | 235 | for _, h := range hashes { 236 | if !f.Has(h) { 237 | fmt.Printf("hash %d added but not retrieved\n", h) 238 | } 239 | } 240 | 241 | // Output: 242 | } 243 | -------------------------------------------------------------------------------- /examples/bloomstat/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Bloomstat is a utility for estimating Bloom filter sizes. 16 | package main 17 | 18 | import ( 19 | "fmt" 20 | "io" 21 | "log" 22 | "os" 23 | "strconv" 24 | "strings" 25 | 26 | "github.com/greatroar/blobloom" 27 | ) 28 | 29 | const usage = `usage: bloomstat capacity false-positive-rate [max-memory] 30 | The maximum memory may be specified as "10MB", "1.5GiB", etc.` 31 | 32 | func main() { 33 | if len(os.Args) < 2 { 34 | fmt.Fprintln(os.Stderr, usage) 35 | os.Exit(1) 36 | } 37 | 38 | var ( 39 | capacity = parse("capacity", os.Args[1]) 40 | fpr = parse("false positive rate", os.Args[2]) 41 | maxsize float64 42 | ) 43 | if len(os.Args) > 3 { 44 | maxsize = parseMem(os.Args[3]) 45 | } 46 | 47 | bits, hashes := blobloom.Optimize(blobloom.Config{ 48 | Capacity: uint64(capacity), 49 | FPRate: fpr, 50 | MaxBits: uint64(8 * maxsize), 51 | }) 52 | 53 | size, unit := memsize(float64(bits)) 54 | bitsPerKey := float64(bits) / capacity 55 | 56 | expectedFpr := blobloom.FPRate(uint64(capacity), bits, hashes) 57 | 58 | fmt.Printf("%d bits, %.02f %s\n"+ 59 | "%.02f bits/%.02f B per key\n"+ 60 | "%d hashes\n"+ 61 | "%.04f expected false positive rate\n", 62 | bits, size, unit, bitsPerKey, bitsPerKey/8, hashes, expectedFpr) 63 | } 64 | 65 | const ( 66 | kiB = 1 << 10 67 | MiB = 1 << 20 68 | GiB = 1 << 30 69 | ) 70 | 71 | func memsize(bits float64) (size float64, unit string) { 72 | size = float64(bits) / 8 73 | 74 | switch { 75 | case size >= GiB: 76 | size /= GiB 77 | unit = "GiB" 78 | case size >= MiB: 79 | size /= MiB 80 | unit = "MiB" 81 | case size >= kiB: 82 | size /= kiB 83 | unit = "kiB" 84 | default: 85 | unit = "B" 86 | } 87 | return 88 | } 89 | 90 | func parse(name, num string) float64 { 91 | v, err := strconv.ParseFloat(num, 64) 92 | 93 | switch e := err.(type) { 94 | case nil: 95 | case *strconv.NumError: 96 | log.Fatalf("%s %q: %v", name, e.Num, e.Err) 97 | default: 98 | log.Fatalf("%s: %v", name, err) 99 | } 100 | if v < 0 { 101 | log.Fatalf("%s must be >= 0", name) 102 | } 103 | 104 | return v 105 | } 106 | 107 | func parseMem(s string) float64 { 108 | var ( 109 | size float64 110 | unit string 111 | ) 112 | n, err := fmt.Sscanf(s, "%f%s", &size, &unit) 113 | switch err { 114 | case nil: 115 | case io.EOF: 116 | if n == 1 { 117 | // Default to bytes. 118 | unit = "b" 119 | } else { 120 | log.Fatal("max memory: invalid input") 121 | } 122 | default: 123 | log.Fatal("max memory:", err) 124 | } 125 | 126 | switch strings.ToLower(unit) { 127 | case "kb", "kib": 128 | size *= kiB 129 | case "mb", "mib": 130 | size *= MiB 131 | case "gb", "gib": 132 | size *= GiB 133 | } 134 | 135 | return size 136 | } 137 | -------------------------------------------------------------------------------- /examples/spellcheck/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2021 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This package implements a toy interactive spell checker. 16 | // 17 | // It reads a dictionary from /usr/share/dict/words and a text from standard 18 | // input. It then reports any misspelled words on standard output. 19 | package main 20 | 21 | import ( 22 | "bufio" 23 | "bytes" 24 | "fmt" 25 | "hash/maphash" 26 | "log" 27 | "os" 28 | "unicode" 29 | 30 | "github.com/greatroar/blobloom" 31 | ) 32 | 33 | func main() { 34 | dict := loadDictionary() 35 | 36 | sc := bufio.NewScanner(os.Stdin) 37 | sc.Split(bufio.ScanWords) 38 | 39 | for sc.Scan() { 40 | word := normalize(sc.Bytes()) 41 | if !dict.has(word) { 42 | fmt.Printf(">>> %s\n", word) 43 | } 44 | } 45 | if err := sc.Err(); err != nil { 46 | log.Fatal(err) 47 | } 48 | } 49 | 50 | // A Bloom filter with a randomized hash function. 51 | type bloomfilter struct { 52 | *blobloom.Filter 53 | maphash.Seed 54 | } 55 | 56 | func newBloomfilter(capacity uint64, fprate float64) *bloomfilter { 57 | cfg := blobloom.Config{Capacity: capacity, FPRate: .001} 58 | return &bloomfilter{ 59 | Filter: blobloom.NewOptimized(cfg), 60 | Seed: maphash.MakeSeed(), 61 | } 62 | } 63 | 64 | func (f *bloomfilter) add(key []byte) { f.Filter.Add(f.hash(key)) } 65 | func (f *bloomfilter) has(key []byte) bool { return f.Filter.Has(f.hash(key)) } 66 | 67 | func (f *bloomfilter) hash(key []byte) uint64 { 68 | var h maphash.Hash 69 | h.SetSeed(f.Seed) 70 | _, _ = h.Write(key) 71 | return h.Sum64() 72 | } 73 | 74 | func normalize(word []byte) []byte { 75 | word = bytes.TrimFunc(word, unicode.IsPunct) 76 | word = bytes.ToLower(word) 77 | return word 78 | } 79 | 80 | // To estimate the number of keys without scanning the file twice, we need 81 | // an estimate of the average length of a word. This comes close for English. 82 | const avgWordLength = 10 83 | 84 | func loadDictionary() *bloomfilter { 85 | f, err := os.Open("/usr/share/dict/words") 86 | if err != nil { 87 | log.Fatal(err) 88 | } 89 | defer f.Close() 90 | 91 | info, err := f.Stat() 92 | if err != nil { 93 | log.Fatal(err) 94 | } 95 | filesize := uint64(info.Size()) 96 | 97 | dict := newBloomfilter(filesize/avgWordLength, .001) 98 | 99 | sc := bufio.NewScanner(f) 100 | for sc.Scan() { 101 | dict.add(normalize(sc.Bytes())) 102 | filesize-- // Subtract newline, for fairness. 103 | } 104 | if err := sc.Err(); err != nil { 105 | log.Fatal(err) 106 | } 107 | 108 | log.Printf("dictionary loaded: %dkiB on disk, %dkiB in memory", 109 | filesize/1024, dict.NumBits()/(8*1024)) 110 | 111 | return dict 112 | } 113 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/greatroar/blobloom 2 | 3 | go 1.14 4 | 5 | require github.com/stretchr/testify v1.8.0 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 5 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 6 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 7 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 8 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 9 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 10 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 11 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 12 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 13 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 14 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 15 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 16 | -------------------------------------------------------------------------------- /io.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobloom 16 | 17 | import ( 18 | "bytes" 19 | "encoding/binary" 20 | "errors" 21 | "fmt" 22 | "io" 23 | "strings" 24 | "sync/atomic" 25 | ) 26 | 27 | const maxCommentLen = 44 28 | 29 | // Dump writes f to w, with an optional comment string, in the binary format 30 | // that a Loader accepts. It returns the number of bytes written to w. 31 | // 32 | // The comment may contain arbitrary data, within the limits layed out by the 33 | // format description. It can be used to record the hash function to be used 34 | // with a Filter. 35 | func Dump(w io.Writer, f *Filter, comment string) (int64, error) { 36 | return dump(w, f.b, f.k, comment) 37 | } 38 | 39 | // DumpSync is like Dump, but for SyncFilters. 40 | // 41 | // If other goroutines are simultaneously modifying f, 42 | // their modifications may not be reflected in the dump. 43 | // Separate synchronization is required to prevent this. 44 | // 45 | // The format produced is the same as Dump's. The fact that 46 | // the argument is a SyncFilter is not encoded in the dump. 47 | func DumpSync(w io.Writer, f *SyncFilter, comment string) (n int64, err error) { 48 | return dump(w, f.b, f.k, comment) 49 | } 50 | 51 | func dump(w io.Writer, b []block, nhashes int, comment string) (n int64, err error) { 52 | switch { 53 | case len(b) == 0 || nhashes == 0: 54 | err = errors.New("blobloom: won't dump uninitialized Filter") 55 | case len(comment) > maxCommentLen: 56 | err = fmt.Errorf("blobloom: comment of length %d too long", len(comment)) 57 | case strings.IndexByte(comment, 0) != -1: 58 | err = fmt.Errorf("blobloom: comment %q contains zero byte", len(comment)) 59 | } 60 | if err != nil { 61 | return 0, err 62 | } 63 | 64 | var buf [64]byte 65 | copy(buf[:8], "blobloom") 66 | // As documented in the comment for Loader, we store one less than the 67 | // number of blocks. This way, we can use the otherwise invalid value 0 68 | // and store 2³² blocks instead of at most 2³²-1. 69 | binary.LittleEndian.PutUint32(buf[12:], uint32(len(b)-1)) 70 | binary.LittleEndian.PutUint32(buf[16:], uint32(nhashes)) 71 | copy(buf[20:], comment) 72 | 73 | k, err := w.Write(buf[:]) 74 | n = int64(k) 75 | if err != nil { 76 | return n, err 77 | } 78 | 79 | for i := range b { 80 | for j := range b[i] { 81 | x := atomic.LoadUint32(&b[i][j]) 82 | binary.LittleEndian.PutUint32(buf[4*j:], x) 83 | } 84 | k, err = w.Write(buf[:]) 85 | n += int64(k) 86 | if err != nil { 87 | break 88 | } 89 | } 90 | 91 | return n, err 92 | } 93 | 94 | // A Loader reads a Filter or SyncFilter from an io.Reader. 95 | // 96 | // A Loader accepts the binary format produced by Dump. The format starts 97 | // with a 64-byte header: 98 | // - the string "blobloom", in ASCII; 99 | // - a four-byte version number, which must be zero; 100 | // - the number of Bloom filter blocks, minus one, as a 32-bit integer; 101 | // - the number of hashes, as a 32-bit integer; 102 | // - a comment of at most 44 non-zero bytes, padded to 44 bytes with zeros. 103 | // 104 | // After the header come the 512-bit blocks, divided into sixteen 32-bit limbs. 105 | // All integers are little-endian. 106 | type Loader struct { 107 | buf [64]byte 108 | r io.Reader 109 | err error 110 | 111 | Comment string // Comment field. Filled in by NewLoader. 112 | nblocks uint64 113 | nhashes int 114 | } 115 | 116 | // NewLoader parses the format header from r and returns a Loader 117 | // that can be used to load a Filter from it. 118 | func NewLoader(r io.Reader) (*Loader, error) { 119 | l := &Loader{r: r} 120 | 121 | err := l.fillbuf() 122 | if err != nil { 123 | return nil, err 124 | } 125 | 126 | version := binary.LittleEndian.Uint32(l.buf[8:]) 127 | // See comment in dump for the +1. 128 | l.nblocks = 1 + uint64(binary.LittleEndian.Uint32(l.buf[12:])) 129 | l.nhashes = int(binary.LittleEndian.Uint32(l.buf[16:])) 130 | comment := l.buf[20:] 131 | 132 | switch { 133 | case string(l.buf[:8]) != "blobloom": 134 | err = errors.New("blobloom: not a Bloom filter dump") 135 | case version != 0: 136 | err = errors.New("blobloom: unsupported dump version") 137 | case l.nhashes == 0: 138 | err = errors.New("blobloom: zero hashes in Bloom filter dump") 139 | } 140 | if err == nil { 141 | comment, err = checkComment(comment) 142 | l.Comment = string(comment) 143 | } 144 | 145 | if err != nil { 146 | l = nil 147 | } 148 | return l, err 149 | } 150 | 151 | // Load sets f to the union of f and the Loader's filter, then returns f. 152 | // If f is nil, a new Filter of the appropriate size is constructed. 153 | // 154 | // If f is not nil and an error occurs while reading from the Loader, 155 | // f may end up in an inconsistent state. 156 | func (l *Loader) Load(f *Filter) (*Filter, error) { 157 | if f == nil { 158 | nbits := BlockBits * l.nblocks 159 | if nbits > MaxBits { 160 | return nil, fmt.Errorf("blobloom: %d blocks is too large", l.nblocks) 161 | } 162 | f = New(nbits, int(l.nhashes)) 163 | } else if err := l.checkBitsAndHashes(len(f.b), f.k); err != nil { 164 | return nil, err 165 | } 166 | 167 | for i := range f.b { 168 | if err := l.fillbuf(); err != nil { 169 | return nil, err 170 | } 171 | 172 | for j := range f.b[i] { 173 | f.b[i][j] |= binary.LittleEndian.Uint32(l.buf[4*j:]) 174 | } 175 | } 176 | 177 | return f, nil 178 | } 179 | 180 | // Load sets f to the union of f and the Loader's filter, then returns f. 181 | // If f is nil, a new SyncFilter of the appropriate size is constructed. 182 | // Else, LoadSync may run concurrently with other modifications to f. 183 | // 184 | // If f is not nil and an error occurs while reading from the Loader, 185 | // f may end up in an inconsistent state. 186 | func (l *Loader) LoadSync(f *SyncFilter) (*SyncFilter, error) { 187 | if f == nil { 188 | nbits := BlockBits * l.nblocks 189 | if nbits > MaxBits { 190 | return nil, fmt.Errorf("blobloom: %d blocks is too large", l.nblocks) 191 | } 192 | f = NewSync(nbits, int(l.nhashes)) 193 | } else if err := l.checkBitsAndHashes(len(f.b), f.k); err != nil { 194 | return nil, err 195 | } 196 | 197 | for i := range f.b { 198 | if err := l.fillbuf(); err != nil { 199 | return nil, err 200 | } 201 | 202 | for j := range f.b[i] { 203 | p := &f.b[i][j] 204 | x := binary.LittleEndian.Uint32(l.buf[4*j:]) 205 | 206 | for { 207 | old := atomic.LoadUint32(p) 208 | if atomic.CompareAndSwapUint32(p, old, old|x) { 209 | break 210 | } 211 | } 212 | } 213 | } 214 | 215 | return f, nil 216 | } 217 | 218 | func (l *Loader) checkBitsAndHashes(nblocks, nhashes int) error { 219 | switch { 220 | case nblocks != int(l.nblocks): 221 | return fmt.Errorf("blobloom: Filter has %d blocks, but dump has %d", nblocks, l.nblocks) 222 | case nhashes != l.nhashes: 223 | return fmt.Errorf("blobloom: Filter has %d hashes, but dump has %d", nhashes, l.nhashes) 224 | } 225 | return nil 226 | } 227 | 228 | func (l *Loader) fillbuf() error { 229 | _, err := io.ReadFull(l.r, l.buf[:]) 230 | if err == io.EOF { 231 | err = io.ErrUnexpectedEOF 232 | } 233 | return err 234 | } 235 | 236 | func checkComment(p []byte) ([]byte, error) { 237 | eos := bytes.IndexByte(p, 0) 238 | if eos != -1 { 239 | tail := p[eos+1:] 240 | if !bytes.Equal(tail, make([]byte, len(tail))) { 241 | return nil, fmt.Errorf("blobloom: comment block %q contains zero byte", p) 242 | } 243 | p = p[:eos] 244 | } 245 | return p, nil 246 | } 247 | -------------------------------------------------------------------------------- /io_fuzz_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build go1.18 16 | // +build go1.18 17 | 18 | package blobloom 19 | 20 | import ( 21 | "bytes" 22 | "io" 23 | "strings" 24 | "testing" 25 | ) 26 | 27 | func FuzzLoader(f *testing.F) { 28 | const validHeader = "blobloom\x00\x00\x00\x00" + 29 | "\x00\x00\x00\x00\x00\x00\x00\x02" + // one block, two hashes 30 | "this is a valid zero-padded UTF-8 comment\x00\x00\x00" 31 | var zeroblock [64]byte 32 | 33 | f.Add(zeroblock[:]) 34 | f.Add([]byte(validHeader + string(zeroblock[:]))) 35 | 36 | f.Fuzz(func(t *testing.T, p []byte) { 37 | r := bytes.NewReader(p) 38 | l, err := NewLoader(r) 39 | 40 | switch { 41 | case err != nil: 42 | if l != nil { 43 | t.Error("l should be nil when err != nil") 44 | } 45 | return 46 | case l.nblocks == 0: 47 | t.Fatal("l.nblocks == 0") 48 | case l.nhashes == 0: 49 | t.Fatal("l.nhashes == 0") 50 | case strings.IndexByte(l.Comment, 0) != -1: 51 | t.Fatal("zero in comment") 52 | } 53 | 54 | // Prevent large allocations. 55 | const maxMem = 1 << 20 56 | if l.nblocks > maxMem/(BlockBits/8) { 57 | t.Skip() 58 | } 59 | 60 | f, err := l.Load(nil) 61 | if err == nil { 62 | if f == nil { 63 | t.Error("err == nil and f == nil") 64 | } 65 | } else { 66 | if f != nil { 67 | t.Error("f != nil and err != nil") 68 | } 69 | if err != io.ErrUnexpectedEOF && !strings.HasPrefix(err.Error(), "blobloom: ") { 70 | t.Fatal("unexpected error", err) 71 | } 72 | } 73 | }) 74 | } 75 | -------------------------------------------------------------------------------- /io_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobloom 16 | 17 | import ( 18 | "bytes" 19 | "io" 20 | "math/rand" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | func TestDumpLoad(t *testing.T) { 28 | f := New(12345, 6) 29 | r := rand.New(rand.NewSource(55)) 30 | for i := 0; i < 100; i++ { 31 | f.Add(r.Uint64()) 32 | } 33 | 34 | buf := new(bytes.Buffer) 35 | n, err := Dump(buf, f, "random bytes") 36 | require.NoError(t, err) 37 | assert.EqualValues(t, 26*64, n) 38 | 39 | l, err := NewLoader(buf) 40 | require.NoError(t, err) 41 | assert.Equal(t, "random bytes", l.Comment) 42 | 43 | g := New(12345, 6) 44 | g2, err := l.Load(g) 45 | require.NoError(t, err) 46 | assert.True(t, g == g2) 47 | assert.True(t, f.Equals(g)) 48 | 49 | g2, err = l.Load(nil) 50 | assert.Nil(t, g2) 51 | assert.Equal(t, io.ErrUnexpectedEOF, err) 52 | } 53 | -------------------------------------------------------------------------------- /optimize.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobloom 16 | 17 | import "math" 18 | 19 | // A Config holds parameters for Optimize or NewOptimized. 20 | type Config struct { 21 | // Trigger the "contains filtered or unexported fields" message for 22 | // forward compatibility and force the caller to use named fields. 23 | _ struct{} 24 | 25 | // Capacity is the expected number of distinct keys to be added. 26 | // More keys can always be added, but the false positive rate can be 27 | // expected to drop below FPRate if their number exceeds the Capacity. 28 | Capacity uint64 29 | 30 | // Desired lower bound on the false positive rate when the Bloom filter 31 | // has been filled to its capacity. FPRate must be between zero 32 | // (exclusive) and one (inclusive). 33 | FPRate float64 34 | 35 | // Maximum size of the Bloom filter in bits. Zero means the global 36 | // MaxBits constant. A value less than BlockBits means BlockBits. 37 | MaxBits uint64 38 | } 39 | 40 | // NewOptimized is shorthand for New(Optimize(config)). 41 | func NewOptimized(config Config) *Filter { 42 | return New(Optimize(config)) 43 | } 44 | 45 | // NewSyncOptimized is shorthand for New(Optimize(config)). 46 | func NewSyncOptimized(config Config) *SyncFilter { 47 | return NewSync(Optimize(config)) 48 | } 49 | 50 | // Optimize returns numbers of keys and hash functions that achieve the 51 | // desired false positive described by config. 52 | // 53 | // Optimize panics when config.FPRate is invalid. 54 | // 55 | // The estimated number of bits is imprecise for false positives rates below 56 | // ca. 1e-15. 57 | func Optimize(config Config) (nbits uint64, nhashes int) { 58 | n := float64(config.Capacity) 59 | p := config.FPRate 60 | 61 | if p <= 0 || p > 1 { 62 | panic("false positive rate for a Bloom filter must be > 0, <= 1") 63 | } 64 | if n == 0 { 65 | // Assume the client wants to add at least one key; log2(0) = -inf. 66 | n = 1 67 | } 68 | 69 | // The optimal nbits/n is c = -log2(p) / ln(2) for a vanilla Bloom filter. 70 | c := math.Ceil(-math.Log2(p) / math.Ln2) 71 | if c < float64(len(correctC)) { 72 | c = float64(correctC[int(c)]) 73 | } else { 74 | // We can't achieve the desired FPR. Just triple the number of bits. 75 | c *= 3 76 | } 77 | nbits = uint64(c * n) 78 | 79 | // Round up to a multiple of BlockBits. 80 | if nbits%BlockBits != 0 { 81 | nbits += BlockBits - nbits%BlockBits 82 | } 83 | 84 | var maxbits uint64 = MaxBits 85 | if config.MaxBits != 0 && config.MaxBits < maxbits { 86 | maxbits = config.MaxBits 87 | if maxbits < BlockBits { 88 | maxbits = BlockBits 89 | } 90 | } 91 | if nbits > maxbits { 92 | nbits = maxbits 93 | // Round down to a multiple of BlockBits. 94 | nbits -= nbits % BlockBits 95 | } 96 | 97 | // The corresponding optimal number of hash functions is k = c * log(2). 98 | // Try rounding up and down to see which rounding is better. 99 | c = float64(nbits) / n 100 | k := c * math.Ln2 101 | if k < 1 { 102 | nhashes = 1 103 | return nbits, nhashes 104 | } 105 | 106 | ceilK, floorK := math.Floor(k), math.Ceil(k) 107 | if ceilK == floorK { 108 | return nbits, int(ceilK) 109 | } 110 | 111 | fprCeil, _ := fpRate(c, math.Ceil(k)) 112 | fprFloor, _ := fpRate(c, math.Floor(k)) 113 | if fprFloor < fprCeil { 114 | k = floorK 115 | } else { 116 | k = ceilK 117 | } 118 | 119 | return nbits, int(k) 120 | } 121 | 122 | // correctC maps c = m/n for a vanilla Bloom filter to the c' for a 123 | // blocked Bloom filter. 124 | // 125 | // This is Putze et al.'s Table I, extended down to zero. 126 | // For c > 34, the values become huge and are hard to compute. 127 | var correctC = []byte{ 128 | 1, 1, 2, 4, 5, 129 | 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 20, 21, 23, 130 | 25, 26, 28, 30, 32, 35, 38, 40, 44, 48, 51, 58, 64, 74, 90, 131 | } 132 | 133 | // FPRate computes an estimate of the false positive rate of a Bloom filter 134 | // after nkeys distinct keys have been added. 135 | func FPRate(nkeys, nbits uint64, nhashes int) float64 { 136 | if nkeys == 0 { 137 | return 0 138 | } 139 | p, _ := fpRate(float64(nbits)/float64(nkeys), float64(nhashes)) 140 | return p 141 | } 142 | 143 | func fpRate(c, k float64) (p float64, iter int) { 144 | switch { 145 | case c == 0: 146 | panic("0 bits per key is too few") 147 | case k == 0: 148 | panic("0 hashes is too few") 149 | } 150 | 151 | // Putze et al.'s Equation (3). 152 | // 153 | // The Poisson distribution has a single spike around its mean 154 | // BlockBits/c that gets slimmer and further away from zero as c tends 155 | // to zero (the Bloom filter gets more filled). We start at the mean, 156 | // then add terms left and right of it until their relative contribution 157 | // drops below ε. 158 | const ε = 1e-9 159 | mean := BlockBits / c 160 | 161 | // Ceil to make sure we start at one, not zero. 162 | i := math.Ceil(mean) 163 | p = math.Exp(logPoisson(mean, i) + logFprBlock(BlockBits/i, k)) 164 | 165 | for j := i - 1; j > 0; j-- { 166 | add := math.Exp(logPoisson(mean, j) + logFprBlock(BlockBits/j, k)) 167 | p += add 168 | iter++ 169 | if add/p < ε { 170 | break 171 | } 172 | } 173 | 174 | for j := i + 1; ; j++ { 175 | add := math.Exp(logPoisson(mean, j) + logFprBlock(BlockBits/j, k)) 176 | p += add 177 | iter++ 178 | if add/p < ε { 179 | break 180 | } 181 | } 182 | 183 | return p, iter 184 | } 185 | 186 | // FPRate computes an estimate of f's false positive rate after nkeys distinct 187 | // keys have been added. 188 | func (f *Filter) FPRate(nkeys uint64) float64 { 189 | return FPRate(nkeys, f.NumBits(), f.k) 190 | } 191 | 192 | // Log of the FPR of a single block, FPR = (1 - exp(-k/c))^k. 193 | func logFprBlock(c, k float64) float64 { 194 | return k * math.Log1p(-math.Exp(-k/c)) 195 | } 196 | 197 | // Log of the Poisson distribution's pmf. 198 | func logPoisson(λ, k float64) float64 { 199 | lg, _ := math.Lgamma(k + 1) 200 | return k*math.Log(λ) - λ - lg 201 | } 202 | -------------------------------------------------------------------------------- /optimize_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobloom 16 | 17 | import ( 18 | "fmt" 19 | "math" 20 | "testing" 21 | 22 | "github.com/stretchr/testify/assert" 23 | ) 24 | 25 | func TestFPRate(t *testing.T) { 26 | t.Parallel() 27 | 28 | // FP rate is zero when no keys have been inserted. 29 | assert.EqualValues(t, 0, FPRate(0, 100, 3)) 30 | 31 | // FP rate is close to one when the capacity is greatly exceeded. 32 | nhashes := 100.0 * math.Ln2 33 | assert.InDelta(t, 1.0, FPRate(1e9, 1e8, int(nhashes)), 1e-7) 34 | 35 | // Examples from Putze et al., page 4. 36 | 37 | // XXX We compute 0.023041, which is confirmed by PARI/GP and SciPy. 38 | // Is the rounding in the paper off? 39 | assert.InDelta(t, 0.0231, FPRate(1, 8, 5), 6e-5) 40 | 41 | // XXX This one is only accurate to one digit. 42 | // The required number does not occur in the series expansion either, 43 | // the closest partial sum being 1.9536e-4. 44 | assert.InDelta(t, 1.94e-4, FPRate(1, 20, 14), 3e-5) 45 | } 46 | 47 | func TestFPRateConvergence(t *testing.T) { 48 | for _, c := range []struct { 49 | c, k float64 50 | iter int 51 | }{ 52 | {.01, 1, 2500}, 53 | {.1, 1, 2000}, 54 | {3, 2, 200}, 55 | {4, 2, 200}, 56 | {6, 3, 200}, 57 | {8, 5, 200}, 58 | {20, 14, 100}, 59 | {30, 20, 100}, 60 | } { 61 | c := c 62 | t.Run(fmt.Sprintf("c=%f,k=%d", c.c, int(c.k)), func(t *testing.T) { 63 | t.Parallel() 64 | 65 | fpr, iterations := fpRate(c.c, c.k) 66 | t.Logf("fpr = %f", fpr) 67 | assert.Less(t, iterations, c.iter) 68 | }) 69 | } 70 | } 71 | 72 | func TestFPRateCorrectC(t *testing.T) { 73 | t.Parallel() 74 | 75 | // Try to reconstruct the correction table. We may be one bit off. 76 | for i, expect := range correctC[1:] { 77 | c := float64(i + 1) 78 | k := float64(c) * math.Ln2 79 | fprBlock := math.Exp(logFprBlock(c, k)) 80 | 81 | cprime := c 82 | for { 83 | if p, _ := fpRate(cprime, k); p <= fprBlock { 84 | break 85 | } 86 | cprime++ 87 | k = cprime * math.Ln2 88 | } 89 | 90 | assert.InDelta(t, float64(expect), cprime, 1) 91 | } 92 | } 93 | 94 | func TestFPRateInvalidInput(t *testing.T) { 95 | assert.Panics(t, func() { FPRate(10, 0, 2) }) 96 | assert.Panics(t, func() { FPRate(10, 2, 0) }) 97 | } 98 | 99 | func TestNewOptimizedMaxFPR(t *testing.T) { 100 | t.Parallel() 101 | 102 | f := NewOptimized(Config{ 103 | Capacity: 0, 104 | FPRate: 1, 105 | }) 106 | assert.EqualValues(t, BlockBits, f.NumBits()) 107 | } 108 | 109 | func TestMaxBits(t *testing.T) { 110 | t.Parallel() 111 | 112 | for _, c := range []struct { 113 | want, expect uint64 114 | }{ 115 | {1, BlockBits}, 116 | {BlockBits - 1, BlockBits}, 117 | {BlockBits + 1, BlockBits}, 118 | {2*BlockBits - 1, BlockBits}, 119 | {4<<20 - 1, 4<<20 - BlockBits}, 120 | {4<<20 + 1, 4 << 20}, 121 | {4<<20 + BlockBits, 4<<20 + BlockBits}, 122 | } { 123 | nbits, nhashes := Optimize(Config{ 124 | // Ask for tiny FPR with a huge number of keys. 125 | Capacity: 2 * c.want, 126 | FPRate: 1e-10, 127 | MaxBits: c.want, 128 | }) 129 | // Optimize should round down to multiple of BlockBits. 130 | assert.LessOrEqual(t, nbits, c.expect) 131 | assert.EqualValues(t, 0, nbits%BlockBits) 132 | 133 | f := New(nbits, nhashes) 134 | assert.Equal(t, c.expect, f.NumBits()) 135 | } 136 | } 137 | 138 | func TestOptimizeFewBits(t *testing.T) { 139 | t.Parallel() 140 | 141 | for _, config := range []Config{ 142 | { 143 | Capacity: 1, 144 | FPRate: .99, 145 | MaxBits: 1, 146 | }, 147 | { 148 | Capacity: 100000, 149 | FPRate: 0.01, 150 | MaxBits: 408, 151 | }, 152 | } { 153 | // Optimize should give nbits >= BlockBits. 154 | nbits, nhashes := Optimize(config) 155 | assert.EqualValues(t, BlockBits, nbits) 156 | assert.Greater(t, nhashes, 0) 157 | } 158 | } 159 | 160 | func TestOptimizeInvalidFPRate(t *testing.T) { 161 | t.Parallel() 162 | 163 | assert.Panics(t, func() { Optimize(Config{FPRate: 0}) }) 164 | assert.Panics(t, func() { Optimize(Config{FPRate: 1.0000001}) }) 165 | } 166 | -------------------------------------------------------------------------------- /setop_64bit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2022 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build (amd64 || arm64) && !nounsafe 16 | // +build amd64 arm64 17 | // +build !nounsafe 18 | 19 | package blobloom 20 | 21 | import ( 22 | "math/bits" 23 | "sync/atomic" 24 | "unsafe" 25 | ) 26 | 27 | // Block reinterpreted as array of uint64. 28 | type block64 [BlockBits / 64]uint64 29 | 30 | func (f *Filter) intersect(g *Filter) { 31 | a, b := f.b, g.b 32 | for len(a) >= 2 && len(b) >= 2 { 33 | p := (*block64)(unsafe.Pointer(&a[0])) 34 | q := (*block64)(unsafe.Pointer(&b[0])) 35 | 36 | p[0] &= q[0] 37 | p[1] &= q[1] 38 | p[2] &= q[2] 39 | p[3] &= q[3] 40 | p[4] &= q[4] 41 | p[5] &= q[5] 42 | p[6] &= q[6] 43 | p[7] &= q[7] 44 | 45 | p = (*block64)(unsafe.Pointer(&a[1])) 46 | q = (*block64)(unsafe.Pointer(&b[1])) 47 | 48 | p[0] &= q[0] 49 | p[1] &= q[1] 50 | p[2] &= q[2] 51 | p[3] &= q[3] 52 | p[4] &= q[4] 53 | p[5] &= q[5] 54 | p[6] &= q[6] 55 | p[7] &= q[7] 56 | 57 | a, b = a[2:], b[2:] 58 | } 59 | 60 | if len(a) > 0 && len(b) > 0 { 61 | p := (*block64)(unsafe.Pointer(&a[0])) 62 | q := (*block64)(unsafe.Pointer(&b[0])) 63 | 64 | p[0] &= q[0] 65 | p[1] &= q[1] 66 | p[2] &= q[2] 67 | p[3] &= q[3] 68 | p[4] &= q[4] 69 | p[5] &= q[5] 70 | p[6] &= q[6] 71 | p[7] &= q[7] 72 | } 73 | } 74 | 75 | func (f *Filter) union(g *Filter) { 76 | a, b := f.b, g.b 77 | for len(a) >= 2 && len(b) >= 2 { 78 | p := (*block64)(unsafe.Pointer(&a[0])) 79 | q := (*block64)(unsafe.Pointer(&b[0])) 80 | 81 | p[0] |= q[0] 82 | p[1] |= q[1] 83 | p[2] |= q[2] 84 | p[3] |= q[3] 85 | p[4] |= q[4] 86 | p[5] |= q[5] 87 | p[6] |= q[6] 88 | p[7] |= q[7] 89 | 90 | p = (*block64)(unsafe.Pointer(&a[1])) 91 | q = (*block64)(unsafe.Pointer(&b[1])) 92 | 93 | p[0] |= q[0] 94 | p[1] |= q[1] 95 | p[2] |= q[2] 96 | p[3] |= q[3] 97 | p[4] |= q[4] 98 | p[5] |= q[5] 99 | p[6] |= q[6] 100 | p[7] |= q[7] 101 | 102 | a, b = a[2:], b[2:] 103 | } 104 | 105 | if len(a) > 0 && len(b) > 0 { 106 | p := (*block64)(unsafe.Pointer(&a[0])) 107 | q := (*block64)(unsafe.Pointer(&b[0])) 108 | 109 | p[0] |= q[0] 110 | p[1] |= q[1] 111 | p[2] |= q[2] 112 | p[3] |= q[3] 113 | p[4] |= q[4] 114 | p[5] |= q[5] 115 | p[6] |= q[6] 116 | p[7] |= q[7] 117 | } 118 | } 119 | 120 | func onescount(b *block) (n int) { 121 | p := (*block64)(unsafe.Pointer(&b[0])) 122 | 123 | n += bits.OnesCount64(p[0]) 124 | n += bits.OnesCount64(p[1]) 125 | n += bits.OnesCount64(p[2]) 126 | n += bits.OnesCount64(p[3]) 127 | n += bits.OnesCount64(p[4]) 128 | n += bits.OnesCount64(p[5]) 129 | n += bits.OnesCount64(p[6]) 130 | n += bits.OnesCount64(p[7]) 131 | 132 | return n 133 | } 134 | 135 | func onescountAtomic(b *block) (n int) { 136 | p := (*block64)(unsafe.Pointer(&b[0])) 137 | 138 | n += bits.OnesCount64(atomic.LoadUint64(&p[0])) 139 | n += bits.OnesCount64(atomic.LoadUint64(&p[1])) 140 | n += bits.OnesCount64(atomic.LoadUint64(&p[2])) 141 | n += bits.OnesCount64(atomic.LoadUint64(&p[3])) 142 | n += bits.OnesCount64(atomic.LoadUint64(&p[4])) 143 | n += bits.OnesCount64(atomic.LoadUint64(&p[5])) 144 | n += bits.OnesCount64(atomic.LoadUint64(&p[6])) 145 | n += bits.OnesCount64(atomic.LoadUint64(&p[7])) 146 | 147 | return n 148 | } 149 | -------------------------------------------------------------------------------- /setop_other.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2022 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build (!amd64 && !arm64) || nounsafe 16 | // +build !amd64,!arm64 nounsafe 17 | 18 | package blobloom 19 | 20 | import ( 21 | "math/bits" 22 | "sync/atomic" 23 | ) 24 | 25 | func (f *Filter) intersect(g *Filter) { 26 | for i := range f.b { 27 | f.b[i].intersect(&g.b[i]) 28 | } 29 | } 30 | 31 | func (f *Filter) union(g *Filter) { 32 | for i := range f.b { 33 | f.b[i].union(&g.b[i]) 34 | } 35 | } 36 | 37 | func (b *block) intersect(c *block) { 38 | b[0] &= c[0] 39 | b[1] &= c[1] 40 | b[2] &= c[2] 41 | b[3] &= c[3] 42 | b[4] &= c[4] 43 | b[5] &= c[5] 44 | b[6] &= c[6] 45 | b[7] &= c[7] 46 | b[8] &= c[8] 47 | b[9] &= c[9] 48 | b[10] &= c[10] 49 | b[11] &= c[11] 50 | b[12] &= c[12] 51 | b[13] &= c[13] 52 | b[14] &= c[14] 53 | b[15] &= c[15] 54 | } 55 | 56 | func (b *block) union(c *block) { 57 | b[0] |= c[0] 58 | b[1] |= c[1] 59 | b[2] |= c[2] 60 | b[3] |= c[3] 61 | b[4] |= c[4] 62 | b[5] |= c[5] 63 | b[6] |= c[6] 64 | b[7] |= c[7] 65 | b[8] |= c[8] 66 | b[9] |= c[9] 67 | b[10] |= c[10] 68 | b[11] |= c[11] 69 | b[12] |= c[12] 70 | b[13] |= c[13] 71 | b[14] |= c[14] 72 | b[15] |= c[15] 73 | } 74 | 75 | func onescount(b *block) (n int) { 76 | n += bits.OnesCount32(b[0]) 77 | n += bits.OnesCount32(b[1]) 78 | n += bits.OnesCount32(b[2]) 79 | n += bits.OnesCount32(b[3]) 80 | n += bits.OnesCount32(b[4]) 81 | n += bits.OnesCount32(b[5]) 82 | n += bits.OnesCount32(b[6]) 83 | n += bits.OnesCount32(b[7]) 84 | n += bits.OnesCount32(b[8]) 85 | n += bits.OnesCount32(b[9]) 86 | n += bits.OnesCount32(b[10]) 87 | n += bits.OnesCount32(b[11]) 88 | n += bits.OnesCount32(b[12]) 89 | n += bits.OnesCount32(b[13]) 90 | n += bits.OnesCount32(b[14]) 91 | n += bits.OnesCount32(b[15]) 92 | 93 | return n 94 | } 95 | 96 | func onescountAtomic(b *block) (n int) { 97 | n += bits.OnesCount32(atomic.LoadUint32(&b[0])) 98 | n += bits.OnesCount32(atomic.LoadUint32(&b[1])) 99 | n += bits.OnesCount32(atomic.LoadUint32(&b[2])) 100 | n += bits.OnesCount32(atomic.LoadUint32(&b[3])) 101 | n += bits.OnesCount32(atomic.LoadUint32(&b[4])) 102 | n += bits.OnesCount32(atomic.LoadUint32(&b[5])) 103 | n += bits.OnesCount32(atomic.LoadUint32(&b[6])) 104 | n += bits.OnesCount32(atomic.LoadUint32(&b[7])) 105 | n += bits.OnesCount32(atomic.LoadUint32(&b[8])) 106 | n += bits.OnesCount32(atomic.LoadUint32(&b[9])) 107 | n += bits.OnesCount32(atomic.LoadUint32(&b[10])) 108 | n += bits.OnesCount32(atomic.LoadUint32(&b[11])) 109 | n += bits.OnesCount32(atomic.LoadUint32(&b[12])) 110 | n += bits.OnesCount32(atomic.LoadUint32(&b[13])) 111 | n += bits.OnesCount32(atomic.LoadUint32(&b[14])) 112 | n += bits.OnesCount32(atomic.LoadUint32(&b[15])) 113 | 114 | return n 115 | } 116 | -------------------------------------------------------------------------------- /sync.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobloom 16 | 17 | import "sync/atomic" 18 | 19 | // A SyncFilter is a Bloom filter that can be accessed and updated 20 | // by multiple goroutines concurrently. 21 | // 22 | // A SyncFilter mostly behaves as a regular filter protected by a lock, 23 | // 24 | // type SyncFilter struct { 25 | // Filter 26 | // lock sync.Mutex 27 | // } 28 | // 29 | // with each method taking and releasing the lock, 30 | // but is implemented much more efficiently. 31 | // See the method descriptions for exceptions to the previous rule. 32 | type SyncFilter struct { 33 | b []block // Shards. 34 | k int // Number of hash functions required. 35 | } 36 | 37 | // NewSync constructs a Bloom filter with given numbers of bits and hash functions. 38 | // 39 | // The number of bits should be at least BlockBits; smaller values are silently 40 | // increased. 41 | // 42 | // The number of hashes reflects the number of hashes synthesized from the 43 | // single hash passed in by the client. It is silently increased to two if 44 | // a lower value is given. 45 | func NewSync(nbits uint64, nhashes int) *SyncFilter { 46 | nbits, nhashes = fixBitsAndHashes(nbits, nhashes) 47 | 48 | return &SyncFilter{ 49 | b: make([]block, nbits/BlockBits), 50 | k: nhashes, 51 | } 52 | 53 | } 54 | 55 | // Add insert a key with hash value h into f. 56 | func (f *SyncFilter) Add(h uint64) { 57 | h1, h2 := uint32(h>>32), uint32(h) 58 | b := getblock(f.b, h2) 59 | 60 | for i := 1; i < f.k; i++ { 61 | h1, h2 = doublehash(h1, h2, i) 62 | setbitAtomic(b, h1) 63 | } 64 | } 65 | 66 | // Cardinality estimates the number of distinct keys added to f. 67 | // 68 | // The estimate is most reliable when f is filled to roughly its capacity. 69 | // It gets worse as f gets more densely filled. When one of the blocks is 70 | // entirely filled, the estimate becomes +Inf. 71 | // 72 | // The return value is the maximum likelihood estimate of Papapetrou, Siberski 73 | // and Nejdl, summed over the blocks 74 | // (https://www.win.tue.nl/~opapapetrou/papers/Bloomfilters-DAPD.pdf). 75 | // 76 | // If other goroutines are concurrently adding keys, 77 | // the estimate may lie in between what would have been returned 78 | // before the concurrent updates started and what is returned 79 | // after the updates complete. 80 | func (f *SyncFilter) Cardinality() float64 { 81 | return cardinality(f.k, f.b, onescountAtomic) 82 | } 83 | 84 | // Empty reports whether f contains no keys. 85 | // 86 | // If other goroutines are concurrently adding keys, 87 | // Empty may return a false positive. 88 | func (f *SyncFilter) Empty() bool { 89 | for i := 0; i < len(f.b); i++ { 90 | for j := 0; j < blockWords; j++ { 91 | if atomic.LoadUint32(&f.b[i][j]) != 0 { 92 | return false 93 | } 94 | } 95 | } 96 | return true 97 | } 98 | 99 | // Fill sets f to a completely full filter. 100 | // After Fill, Has returns true for any key. 101 | func (f *SyncFilter) Fill() { 102 | for i := 0; i < len(f.b); i++ { 103 | for j := 0; j < blockWords; j++ { 104 | atomic.StoreUint32(&f.b[i][j], ^uint32(0)) 105 | } 106 | } 107 | } 108 | 109 | // Has reports whether a key with hash value h has been added. 110 | // It may return a false positive. 111 | func (f *SyncFilter) Has(h uint64) bool { 112 | h1, h2 := uint32(h>>32), uint32(h) 113 | b := getblock(f.b, h2) 114 | 115 | for i := 1; i < f.k; i++ { 116 | h1, h2 = doublehash(h1, h2, i) 117 | if !getbitAtomic(b, h1) { 118 | return false 119 | } 120 | } 121 | return true 122 | } 123 | 124 | // getbitAtomic reports whether bit (i modulo BlockBits) is set. 125 | func getbitAtomic(b *block, i uint32) bool { 126 | bit := uint32(1) << (i % wordSize) 127 | x := atomic.LoadUint32(&(*b)[(i/wordSize)%blockWords]) 128 | return x&bit != 0 129 | } 130 | -------------------------------------------------------------------------------- /sync_go124.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build go1.24 16 | // +build go1.24 17 | 18 | package blobloom 19 | 20 | import "sync/atomic" 21 | 22 | // setbit sets bit (i modulo BlockBits) of b, atomically. 23 | func setbitAtomic(b *block, i uint32) { 24 | bit := uint32(1) << (i % wordSize) 25 | p := &(*b)[(i/wordSize)%blockWords] 26 | atomic.OrUint32(p, bit) 27 | } 28 | -------------------------------------------------------------------------------- /sync_nogo124.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !go1.24 16 | // +build !go1.24 17 | 18 | package blobloom 19 | 20 | import "sync/atomic" 21 | 22 | // setbit sets bit (i modulo BlockBits) of b, atomically. 23 | func setbitAtomic(b *block, i uint32) { 24 | bit := uint32(1) << (i % wordSize) 25 | p := &(*b)[(i/wordSize)%blockWords] 26 | 27 | for { 28 | old := atomic.LoadUint32(p) 29 | if old&bit != 0 { 30 | // Checking here instead of checking the return value from 31 | // the CAS is between 50% and 80% faster on the benchmark. 32 | return 33 | } 34 | atomic.CompareAndSwapUint32(p, old, old|bit) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /sync_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 the Blobloom authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobloom 16 | 17 | import ( 18 | "math" 19 | "math/rand" 20 | "sync" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | func TestSync(t *testing.T) { 28 | const ( 29 | nkeys = 1e4 30 | nworkers = 4 31 | ) 32 | 33 | var ( 34 | config = Config{Capacity: nkeys, FPRate: 1e-5} 35 | hashes = make([]uint64, nkeys) 36 | r = rand.New(rand.NewSource(0xaeb15)) 37 | ref = NewOptimized(config) 38 | ) 39 | 40 | for i := range hashes { 41 | h := r.Uint64() 42 | hashes[i] = h 43 | ref.Add(h) 44 | } 45 | 46 | card := ref.Cardinality() 47 | require.False(t, ref.Empty()) 48 | require.False(t, math.IsInf(card, 0)) 49 | 50 | check := func(f *SyncFilter) { 51 | t.Helper() 52 | 53 | assert.Equal(t, ref.b, f.b) 54 | assert.False(t, f.Empty()) 55 | 56 | for i := 0; i < 2e4; i++ { 57 | h := r.Uint64() 58 | assert.Equal(t, ref.Has(h), f.Has(h)) 59 | } 60 | assert.Equal(t, card, f.Cardinality()) 61 | } 62 | 63 | t.Run("all hashes", func(t *testing.T) { 64 | // Each worker adds all hashes to f. 65 | t.Parallel() 66 | 67 | f := NewSyncOptimized(config) 68 | assert.True(t, f.Empty()) 69 | 70 | var wg sync.WaitGroup 71 | wg.Add(nworkers) 72 | 73 | for i := 0; i < nworkers; i++ { 74 | go func() { 75 | for _, h := range hashes { 76 | f.Add(h) 77 | } 78 | wg.Done() 79 | }() 80 | } 81 | 82 | wg.Wait() 83 | check(f) 84 | }) 85 | 86 | t.Run("split hashes", func(t *testing.T) { 87 | // Hashes divided across workers. 88 | t.Parallel() 89 | 90 | var ( 91 | ch = make(chan uint64, nworkers) 92 | f = NewSyncOptimized(config) 93 | wg sync.WaitGroup 94 | ) 95 | wg.Add(nworkers) 96 | 97 | go func() { 98 | for _, h := range hashes { 99 | ch <- h 100 | } 101 | close(ch) 102 | }() 103 | 104 | for i := 0; i < nworkers; i++ { 105 | go func() { 106 | for h := range ch { 107 | f.Add(h) 108 | } 109 | wg.Done() 110 | }() 111 | } 112 | 113 | wg.Wait() 114 | check(f) 115 | }) 116 | } 117 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e -x 4 | 5 | golangci-lint run . examples/* 6 | 7 | go test 8 | 9 | if [ "$(go env GOARCH)" = amd64 ]; then 10 | go test -tags nounsafe 11 | GOARCH=386 go test 12 | fi 13 | 14 | for e in examples/*; do 15 | (cd $e && go build && rm $(basename $e)) 16 | done 17 | --------------------------------------------------------------------------------