├── LICENSE
├── README.md
├── cuckoo.go
├── cuckoo_test.go
├── demo
    └── demo.html
├── example
    └── example.go
├── go.mod
├── go.sum
├── hash-builtin.go
├── internal
    ├── dstest
    │   └── dstest.go
    ├── jenkins264
    │   └── jenkins264.go
    ├── jenkins3
    │   ├── jenkins3.go
    │   └── jenkins3_test.go
    └── siginfo
    │   ├── example
    │       └── example.go
    │   └── siginfo.go
├── kv_default.go
├── kv_string.go
├── kvt_array.go
├── kvt_slice.go
├── murmur3
    ├── License-murmur3_test
    ├── murmur3.go
    └── murmur3_test.go
└── primes
    └── primes.go


/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2014 Lawrence E. Bakst
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 
23 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | <img src="http://clipart.coolclips.com/100/wjm/tf05110/CoolClips_hous1343.jpg"></img>
  2 | 
  3 | Cuckoo Hash Tables
  4 | ==================
  5 | 
  6 | This package is an implementation of a Cuckoo Hash Table (CHT). [^1] A Cuckoo Hash Table is similar to Go's builtin hash map but uses multiple hash tables with a cascading random walk slot eviction strategy when hashing conflicts occur. Additional hash tables can optionally be added on the fly. A Cuckoo Hash Table is a 3D data structure. Multiple hash tables are comprised of buckets. Each bucket contains slots. Each slot contains a key/value pair. The hash tables all use the same hash function but with different seeds.
  7 | 
  8 | Go's builtin map is well designed and implemented. The author uses it all the time. This CHT is a boutique and bespoke data structure better suited for special cases where the datasets are large, memory efficiency is key, or both.
  9 | 
 10 | *Why use a CHT instead of Go's builtin map?*
 11 | 
 12 | 1. Memory Efficiency. In a benchmark below `map[uint64]uint64` the CHT used 4.0X (15 MiB vs 59 MiB) less memory than Go's built-in map at competitive speeds for insert and lookup. This is because you can tune CHT's key and value to your specific needs, while Go's map current uses a single hash table with 8 slots and an overflow pointer. For small lookup tables this means CHT stores more data in L1/L2/L3 cache. For larger tables it means greater overall memory efficiency.
 13 | 
 14 | 2. Memory Efficiency. In addition to being the above, a CHT can handle load factors as high as .999 with some tradeoff in insert efficiency. If you have a mostly read only data structure a CHT is perfect. Even if you don't, the knobs and dials in this implementation can be set to give you the insert efficiency you desire.
 15 | 
 16 | 3. Large Scale Memory Efficiency. As of 1.4 I believe Go's maps uses power of two sizes for its hash tables. This allows a mask to be used to calculate the bucket index instead of a MOD instruction. Unfortunately, large data sets often require a much larger allocation than required. For example a 2 GB + 1 byte data structure will require 4 GB. (*needs to be confirmed*)
 17 | 
 18 | 4. No GC pauses. As of Go 1.4 Go's maps can be subject to significant GC pauses as the overflow pointers are scanned. The CHT has no overflow pointers and no pointers at all, unless you add them to the value you store. This may change in Go 1.5.
 19 | 
 20 | 5. Knobs and Dials. This CHT has a number of knobs you can tweak to get the results you want
 21 | 
 22 | Load factors as high as .999 are achievable with the caveats that the amount of work per insertion increases as the hash table fills up (load factor increases) and the amount of work per delete increases with the number of hash tables and slots. The amount of work on Insert can be ameliorated by decreasing the load factor, increasing the number of hash tables, the number of slots per bucket, or both.
 23 | 
 24 | Additionally, Cuckoo Hash Tables are subject to pathological cases (cycles) that can prevent an insert from completing. If a cycle does occur, it can be automatically detected, another hash table is added on the fly, and the insert is guaranteed to complete. The amount of work done before a cycle is assumed can also be configured by the user via an API call.
 25 | 
 26 | In this implementation there are three ways to reduce the probability of running into a pathological case:
 27 | 
 28 | 1. Increase the number of slots per bucket (again, 4, 8, or 16 are good numbers)
 29 | 2. Increase the number of hash tables to a number greater than 2 helps (4 is a good number)
 30 | 3. Reduce the load factor  
 31 | 
 32 | Slots are a very effective way of achieving high load factors efficiently. 8, 16, and 32 slots per bucket allow for very high load factors. Adding hash tables is not nearly as efficient as more hashes per insert need to be calculated. Therefore more slots are preferred over more hash tables, but some balance is required between the two. Hash tables can be added on the fly, slots can not.
 33 | 
 34 | The implementation keeps a number of counters that can be used to derive statistics about how well the implementation is performing. I could not have easily developed this package without the counters.
 35 | 
 36 | An example program is included which easily allows one to quickly try out new combinations of parameters and explore the results. Unit tests verify that the implementation works as advertised. Benchmarks are also included.
 37 | 
 38 | Status
 39 | ------
 40 | *The code should be considered beta quality.*
 41 | 
 42 | Performance optimizations caused me to switch the hash function used to be the Intel X86-64 accelerated [AES hash function](https://github.com/tildeleb/aeshash) from Go's runtime.  
 43 | 
 44 | Optimizations were put into place to raise the performance level to be competitive with Go's builtin map. The optimizations made use of the inline functionality of the Go's gc compiler. Now that Go 1.7 has a new SSA backend, all the optimizations need to be verified. Internally the standard hash function interface is supported. It should be possible to support a pluggable hash function interface in the future, with some performance hit as the standard hash function interface does not support optimizations for hashing 32 and 64 bit numeric keys among other issues.
 45 | 
 46 | Support currently exists for cross platform Jenkins264 and Jeknins364.
 47 | 
 48 | The package is untested on any architecture other than Intel X86-64
 49 | 
 50 | How to Build
 51 | ------------
 52 | 
 53 | **NB: Before you build you need to define the types of your key and value. Edit the file "kv_default.go" and define the types for "Key" and "Value".**
 54 | 
 55 | ###Simple (builds a portable version with runtime selection of slots)
 56 | 
 57 | 	% go build -tags="noaes slice"
 58 | 
 59 | Note the above build uses a slice per bucket to implement slots. This allows for experimentation with the number of slots without recompiling but is inefficient from a memory usage perspective. When the number of slots needed for your application is finalized do the following:
 60 | 
 61 | ###Optimized
 62 | Edit "kvt_array.go" and define Slots to be the number of slots needed for your application.  
 63 | 
 64 | 	% go build -tags=noaes
 65 | 
 66 | This will build a version that uses arrays. Calls to cuckoo.New where the number of slots passed in does not match the number specified in "kvt_array.go" will fail.
 67 | 
 68 | ###X86-64 optimized
 69 | 
 70 | 	% go build
 71 | 
 72 | This version uses an accelerated hash function that works on most Intel X86-64 architecture machines. The specific feature is AESNI and the instruction used is AESENC. This is default since that's a very popular architecture these days for desktops, laptops, an cloud machines.
 73 | 
 74 | Included Sub-Packages
 75 | ---------------------
 76 | * jenkins 264 hash package
 77 | * jenkins 364 hash package
 78 | * dtest test framework
 79 | * primes provides prime numbers for table sizes
 80 | 
 81 | Dependent Packages
 82 | -------------------
 83 | * [aeshash](https://github.com/tildeleb/aeshash)
 84 | 
 85 | Goals For This Version
 86 | ----------------------
 87 | * Allow for many configuration options to explore the design space
 88 | * 64 bit hash functions
 89 | * Clear code that facilitates understanding the algorithm
 90 | * Top notch memory and CPU efficiency within the bounds of pure Go
 91 | * Comprehensive counters to allow for dynamic debugging and statistical analysis
 92 | * Good example program with lots of options to play with various parameters
 93 | * User selectable hash functions
 94 | * Support for non power of two table size and therefore the use of mod to calculate a bucket index.
 95 | * Production quality code with testing
 96 | * 100% written in Go with ~~no~~ few external dependencies (for the main package)
 97 | 
 98 | Future Development
 99 | ------------------
100 | * Concurrent lock free access
101 | * Stable iteration even with concurrent access 
102 | * More hash functions like CityHash, SIPHash, and others
103 | * More test cases
104 | 
105 | Bugs and Issues
106 | ---------------
107 | * Simple iteration support in this version, works well when table is full-ish, slow when table close to empty.
108 | 
109 | Example of a Pathological Case
110 | ------------------------------
111 | In the following example a 4 table x 11 buckets x 8 slot cuckoo hash is constructed and 1 million trials are run doing inserts/verify/delete.. In all but a single case the CHT was able to achieve a perfect load factor of 1.0, which means that the table was completely filled. In the single case that failed, only a single insert, the final insert, could not be completed. This defines life with a cuckoo table.
112 | 
113 | If the single failure makes you unhappy I suggest you change the number of slots from 8 to 16 and investigate how many trials it takes to find a failure.
114 | 
115 | 	leb@hula:~/gotest/src/leb/cuckoo/example % time ./example -t 4 -b 11 -s 8 -nt=1000000 -flf=1.0 -lf=10 -dg -rb=true
116 | 	trials: size=8 kbytes
117 | 	trials: tables=4, buckets=11, slots=8, size=352, max=352, trials=1000000, fails=1, avg=1.0000
118 | 	trials: MaxRemaining=1, LowestLevel=-168, Aborts=168, bpi=2.15, api=21.68, ipi=0.1618
119 | 	./example -t 4 -b 11 -s 8 -nt=1000000 -flf=1.0 -lf=10 -dg -rb=true  524.49s user 3.76s system 101% cpu 8:42.49 total
120 | 	leb@hula:~/gotest/src/leb/cuckoo/example % 
121 | 
122 | 
123 | Benchmarks
124 | ----------
125 | The following benchmark data is from a run on my MacBook Pro 2.5 GHz Core i7. The Cuckoo Hashtable configuration is 2 hash tables with 8 slots per bucket with the array optimization. Another optimization is turned on that marshals numeric quantities (currently 32 and 64 bit only) more efficiently than using the binary package.
126 | 
127 | 	leb@hula:~/gotest/src/github.com/tildeleb/cuckoo % go test -bench=. -v
128 | 	=== RUN   TestBasic
129 | 	--- PASS: TestBasic (0.34s)
130 | 	=== RUN   TestMemoryEfficiency
131 | 	--- PASS: TestMemoryEfficiency (0.46s)
132 | 		cuckoo_test.go:151: Cuckoo Hash LoadFactor:       0.99
133 | 		cuckoo_test.go:152: Cuckoo Hash memory allocated: 15 MiB
134 | 		cuckoo_test.go:153: Go map memory allocated:      59 MiB
135 | 	=== RUN   Example
136 | 	--- PASS: Example (0.00s)
137 | 	PASS
138 | 	BenchmarkCuckoo2T2SInsert-11	10000000	       191 ns/op	       0 B/op	       0 allocs/op
139 | 	BenchmarkCuckoo2T2SSearch-11	10000000	       146 ns/op	       0 B/op	       0 allocs/op
140 | 	BenchmarkCuckoo2T2SDelete-11	10000000	       305 ns/op	       0 B/op	       0 allocs/op
141 | 	BenchmarkGoMapInsert-11     	 5000000	       245 ns/op	      17 B/op	       0 allocs/op
142 | 	BenchmarkGoMapSearch-11     	20000000	       144 ns/op	       0 B/op	       0 allocs/op
143 | 	BenchmarkGoMapDelete-11     	100000000	        20.0 ns/op	       0 B/op	       0 allocs/op
144 | 	ok  	github.com/tildeleb/cuckoo	26.872s
145 | 	leb@hula:~/gotest/src/github.com/tildeleb/cuckoo % 
146 | 
147 | Benchmarks Discussion
148 | ---------------------
149 | For the case "var map[uint64]uint64 Cuckoo Hash uses 6.5X less memory than Go's builtin map and does so while achieving a load factor of 99% with similar efficiency. Again, the Cuckoo Hash for this example uses 2 hash tables and each bucket has 8 slots. From a performance standpoint the Cuckoo hash achieves 242 ns/op on Inserts vs 264 ns/op for the build-in map.
150 | 
151 | 
152 | Selectable Hash Functions
153 | -------------------------
154 | The hash function used by this package can be selected. Currently only two hash functions are supported.
155 | 
156 | 1. "aes" This hash function is the same hash function used by Go's map. AESNI instructions are used to generates a very fast high quality hash function. Special versions for 32 and 64 bit data are supported. This hash function is about 5x faster than "j264".
157 | 
158 | 2. "j264" This is a version of Jenkin's 2nd generation hash functions. There is some optimization for speed but no special versions of 32 and 64 bit data. No assembler optimization. No fast path. No inlining.
159 | 
160 | Defining Your Own Key/Value Types
161 | ---------------
162 | The package supports almost any kind of key and value type by simply creating a new "kvt" file. The file "kvt_default.go" can be edited to change the definitions for Key and Value.
163 | 
164 | 
165 | Support for Arrays or Slices via Build Tags
166 | ------------------------------------------
167 | The package has an optimization to implement slots as either slices or arrays. Slices allow the number of slots to be selected at runtime but the slice overhead per bucket is high. Therefore, once the number of slots is known, it's best to switch to a static array size. 
168 | 
169 | [fix]
170 | 
171 | Slices are not very efficient but you can try out new sizes without having to edit a file.
172 | Arrays are more efficient wither cpu or memory wise because they are not a reference type so there is the overhead of an 8 byte pointer on a 64 bit system and the cache miss(es) that go along with that pointer dereference.
173 | 
174 | 
175 | As an example here is a file "kvt_uint32_uint32_slice.go" that defines a "uint32" key, a "uint32" value, and a uses slices.
176 | 	
177 | 	// +build kuint32,vuint32,slice
178 | 	
179 | 	package cuckoo
180 | 	
181 | 	type Key uint32
182 | 	type Value uint32
183 | 	
184 | 	type Buckets	[]Bucket		// slots
185 | 	func makeSlots(b Buckets, slots int) Buckets {
186 | 		return make(Buckets, slots, slots)
187 | 	}
188 | 
189 | To build this version of the Cuckoo Hash you would issue the following command
190 | 
191 | 	go build -tags="kuint32 vuint32 slice"
192 | 
193 | and here is a similar file "kvt_uint32_uint32_array.go" that uses an array type instead of a slice:
194 | 
195 | 	// +build kuint32,vuint32,array
196 | 	
197 | 	package cuckoo
198 | 	
199 | 	type Key uint32
200 | 	type Value uint32
201 | 	
202 | 	const Slots = 4
203 | 	
204 | 	type Buckets	[Slots]Bucket		// slots
205 | 	func makeSlots(b Buckets, slots int) Buckets {
206 | 		return b
207 | 	}
208 | 	
209 | To build this version of the Cuckoo Hash you would issue the following command
210 | 
211 | 	go build -tags="kuint32 vuint32 array"
212 | 
213 | Example Program
214 | ---------------
215 | There is an example program which is useful or exploring the tuning of cuckoo hash tables and verifying the implementation.
216 | 
217 | 	Usage of ./example:
218 | 	  -a=false: automatic
219 | 	  -b=31: buckets
220 | 	  -base=1: base of fill series, -1 for random
221 | 	  -cp="": write cpu profile to file
222 | 	  -dg=false: dont't add hash tables automatically
223 | 	  -flf=1: fill load factor
224 | 	  -fo=false: fill only
225 | 	  -h="aes": name of hash function (aes or j264)
226 | 	  -lf=0.96: maximum load factor
227 | 	  -ll=-8000: lowest level
228 | 	  -mp="": write memory profile to this file
229 | 	  -nt=5: number of trials
230 | 	  -pl=false: print level of each insert
231 | 	  -pr=false: print progress
232 | 	  -ps=false: print stats at the end of all trails
233 | 	  -pt=false: print summary for each trail
234 | 	  -rb=true: random base
235 | 	  -rr=true: random run
236 | 	  -s=8: slots
237 | 	  -sl=2000: starting level
238 | 	  -t=4: tables
239 | 	  -v=false: verbose
240 | 
241 | Let's take a simple example of a classic (two table) cuckoo table. This example creates a cuckoo hash table with 2 hash tables, 11 buckets, and 1 slot per bucket. The occupancy of the hash table won't exceed a load factor of greater than 40%.
242 | 
243 | 	% ./example -t 2 -b 11 -s 1 -nt=5 -lf=0.4 -ps 
244 | 	trials: size=352 bytes
245 | 	trials: trial=0, Remaining=14, Aborts=0, LowestLevel=2000, MaxAttemps=2, MaxIterations=0, bpi=0.25, api=1.25, ipi=0.0000
246 | 	trials: trial=1, Remaining=14, Aborts=0, LowestLevel=2000, MaxAttemps=2, MaxIterations=0, bpi=0.12, api=1.12, ipi=0.0000
247 | 	trials: trial=2, Remaining=14, Aborts=0, LowestLevel=2000, MaxAttemps=1, MaxIterations=0, bpi=0.00, api=1.00, ipi=0.0000
248 | 	trials: trial=3, Remaining=14, Aborts=0, LowestLevel=2000, MaxAttemps=2, MaxIterations=0, bpi=0.25, api=1.25, ipi=0.0000
249 | 	trials: trial=4, Remaining=14, Aborts=0, LowestLevel=2000, MaxAttemps=2, MaxIterations=0, bpi=0.38, api=1.38, ipi=0.0000
250 | 	trials: tables=2, buckets=11, slots=1, size=22, max=22, trials=5, fails=5, avg=0.3636
251 | 	trials: Aborts=0, bpi=0.20, api=1.20, ipi=0.0000
252 | 	trials: MaxRemaining=14
253 | 	trials: LowestLevel=2000
254 | 	trials: c=&cuckoo.CuckooStat{BucketSize:16, Elements:40, Inserts:40, Attempts:48, Iterations:0, Deletes:40, Lookups:40, Fails:0, Bumps:8, Aborts:0, MaxAttempts:0, MaxIterations:0, Limited:false}
255 | 	 % 
256 | 
257 | So this creates a cuckoo table with 2 hash tables, 11 buckets, and 1 slot per bucket. It runs 5 trials with a load factor of 40%. 2 x 11 x 1 = 22 x .4 = 8.8 = 8. 22 slots - 8 = 14 slots remaining. The average load achieved for all 3 trials is 0.36.
258 | 
259 | The stats "bpi", "api", and "ili" stand for "bumper per insert", "attempts per insert", and "iterations per insert". 
260 | 
261 | Now let's look at cuckoo table can support a load factor of 99.9%, albeit with some time consuming insertions as the table fills up.
262 | 
263 | 	leb% ./example -t 4 -b 14009 -s 4 -nt=5 -lf=0.999 -ps     
264 | 	trials: size=3 Mbytes
265 | 	trials: trial=0, Remaining=225, Aborts=0, LowestLevel=1390, MaxAttemps=9775, MaxIterations=610, bpi=3.20, api=15.30, ipi=0.4255
266 | 	trials: trial=1, Remaining=225, Aborts=0, LowestLevel=1301, MaxAttemps=11200, MaxIterations=699, bpi=3.18, api=15.22, ipi=0.4205
267 | 	trials: trial=2, Remaining=225, Aborts=0, LowestLevel=1472, MaxAttemps=8464, MaxIterations=528, bpi=3.13, api=15.02, ipi=0.4081
268 | 	trials: trial=3, Remaining=225, Aborts=0, LowestLevel=1631, MaxAttemps=5920, MaxIterations=369, bpi=3.17, api=15.16, ipi=0.4169
269 | 	trials: trial=4, Remaining=225, Aborts=0, LowestLevel=1375, MaxAttemps=10016, MaxIterations=625, bpi=3.19, api=15.27, ipi=0.4237
270 | 	trials: tables=4, buckets=14009, slots=4, size=224144, max=224144, trials=5, fails=5, avg=0.9990
271 | 	trials: Aborts=0, bpi=3.17, api=15.20, ipi=0.4189
272 | 	trials: MaxRemaining=225
273 | 	trials: LowestLevel=1301
274 | 	trials: c=&cuckoo.CuckooStat{BucketSize:16, Elements:1119595, Inserts:1119595, Attempts:17013688, Iterations:469042, Deletes:1119595, Lookups:1119595, Fails:0, Bumps:3554019, Aborts:0, MaxAttempts:0, MaxIterations:0, Limited:false}
275 | 	leb% 
276 | 
277 | The key number to look at here is the api which has moved from 1.20 on the classic hash table to a 15.20 here. So the CHT has to try 15 locations on average to insert a key.
278 | 
279 | 
280 | Implementation [Must Proofread]
281 | --------------
282 | This version of a cuckoo hash table implements a three dimensional hash table. In concrete terms we have "t" hash tables, each has tables has "b" buckets, and each bucket has "s" slots. Total entries is simply t * b * s. In practical terms t can range from 2 to 4 and maybe as high as 8 and slots can range from 1 to 8 and maybe as high as 16 or 32. Access to slots is fast because the pre-fetcher gets them into the L1 cache. The number of buckets should be a prime number. Within reason slots are more efficient execution time wise then hash tables, so prefer slots to tables. For expositional purpose consider hash tables laid out in left to right order.
283 | 
284 | The insert algorithm is as follows. For the given key, a hash value is calculated for each hash table. The bucket in the leftmost table is indexed by its key and if a free slot is found it is used. If none of the slots are free a random slot is evicted and the new key/value pair is stored where there and the evicted slots becomes the new ke/value pair to be inserted in the next rightmost hash table.
285 | 
286 | The evicted key and it's value are then attempted to be stored in the next hash table to the right and the same procedure is followed until hopefully a home is found for all key/value pairs.
287 | 
288 | The entire procedure is repeated until the end of the left to right hash tables is reached. again for a her specified number of iterations. When the number of iterations has expired (== 0) the algorithm goes into recovery mode, where instead of trying to insert a value it tries to get the value to be inserted back as the value to be inserted. This isn't alway possible in which case data loss happens.
289 | 
290 | Cuckoo tables are known for their efficiency. Go to the example folder and run: 
291 | 
292 | 	leb@hula:~/gotest/src/leb/cuckoo/example % time ./example -t 4 -b 11 -s 8 -nt=1000000 -flf=1.0 -lf=10 -dg -rb=true
293 | 	trials: size=8 kbytes
294 | 	trials: tables=4, buckets=11, slots=8, size=352, max=352, trials=1000000, fails=1, avg=1.0000
295 | 	trials: MaxRemaining=1, LowestLevel=-168, Aborts=168, bpi=2.15, api=21.68, ipi=0.1618
296 | 	./example -t 4 -b 11 -s 8 -nt=1000000 -flf=1.0 -lf=10 -dg -rb=true  524.49s user 3.76s system 101% cpu 8:42.49 total
297 | 	leb@hula:~/gotest/src/leb/cuckoo/example % 
298 | 
299 | A cuckoo has table with 352 locations in it was constructed and 352 random numbers were inserted into this hash table. 1 millions trials were run and with the exception of a single trial all trials achieved a perfect load factor of 1.0, e.g. all the numbers could be inserted. In the single failure case only the last number could not be inserted. So you feel lucky today? If not, I suggest you up the number of tables or slots until you feel lucky.
300 | 
301 | 	leb@hula: % time ./example -t 4 -b 31 -s 16 -flf=1.0 -lf=1.0 -dg -rb=true -nt=1000000
302 | 	trials: cucko hash table size=248 Kibytes
303 | 	trials: tables=4, buckets=31, slots=16, size=1984, max=1984, trials=1000000, fails=0, avg=1.0000
304 | 	trials: MaxRemaining=0, LowestLevel=1438, Aborts=0, bpi=2.09, api=41.98, ipi=0.1481
305 | 	./example -t 4 -b 31 -s 16 -flf=1.0 -lf=1.0 -dg -rb=true -nt=1000000  5420.39s user 28.36s system 100% cpu 1:30:42.80 total
306 | 	leb@hula: % # 	load: 3.28  cmd: example 75597 running 5339.27u 27.95s
307 | 
308 | Since the example above had a failure rate of 0.0001% let's see if we can improve that. The easiest way is to increase the associativity per bucket and go from 8 slots/bucket to 16 slocks/bucket.
309 | 
310 | Note the size of the hash tables is 248 Kibytes which just fits in the L2 cache of the processor in my laptop. When I test there are 3 sizes that makes sense to test with. 32KB or less means everything fits in the L1 data cache and this runs very quickly. 256KB is the size of my L2 cache. Memory latency here is 10X L1. My L3 cache is 8 MB. Latencies are about 4, 12, and 28 cycles respectively.
311 | 
312 | A cuckoo has table with 1984 locations in it was constructed and 1984 random numbers were inserted into this hash table, verified, and deleted. 1 millions trials were run and all trials achieved a perfect load factor of 1.0, e.g. all the numbers could be inserted. 
313 | 
314 | I am lucky today.
315 | 
316 | 
317 | Implementation FAQ
318 | ------------------
319 | **Q** Why is delete so slow?
320 | **A** Two reasons. First, Go's map uses a trick where some of the hash bits are used to index into the slots, saving a scan of the slots. The CHT can't use that trick. Second, Go's map just sets a bit to delete a slot but the CHT currently copies the "empty key" into the key of the slot being deleted.
321 | 
322 | **Q**: Why do you use mod instead of power of two tables with a bit mask for bucket indexing?  
323 | **A**: This was a difficult decision. Calculating MOD is much slower than performing a power of two masking AND operation. Also to (always) be considered are the memory caching effects. ‘MOD’ is slower than AND by an amount larger than an L1-miss-L2-hit time. So assuming that miss-hit pattern (unclear, depends on table size and other factors) it might be better to re-probe once than calculate the MOD.
324 | 
325 | I am interested in working with large datasets. In the end the main reason I choose MOD over power of two table sizes with AND masking is because the latter doesn't scale efficiently to large datasets. e.g. if I have a 16 GB  dataset and it grows by one more entry, it will need a 32 GiB allocation and waste 16 GiB.
326 | 
327 | **Q**: Don't you know MOD is slow?  
328 | **A**: Sure, but see above.
329 | 
330 | **Q**: What hash functions are used?
331 | **A**: Currently only the accelerated 32 bit hash calculated using AESNI on X86-64 platforms.
332 | 
333 | Support for other hash functions has to be put back in. Siphash also accelerated on X86-64 but has fallback to pure Go for platforms other than X86-64. Jenkins and Murmur are also supported.
334 | 
335 | **Q**: Why is Delete so slow?  
336 | **A**: Essentially because Delete has to look is t * s places to find the key whereas Go's build in map only has to look in a single place. In the example benchmarks t == 2 and s == 8 so s * t == 16. Therefor on average Delete has to do 8 lookups to find the key. The speed of Delete can be increased by decreasing the number of slots and tables.
337 | 
338 | **Q**: Why isn't there a stash?  
339 | **A**: I read the Microsoft paper and wasn't impresses. Adding a stash is like adding a bag to the design. In order to guarantee insert doesn't fail we just add another hash table if needed. Adding a stash buffer would add a parallel data structure and code to manipulate it in Insert, Delete, and Lookup. 
340 | 
341 | 
342 | References
343 | ----------
344 | [^1]: [21] R. Pagh and F. Rodler. Cuckoo hashing. Journal of Algorithms,
345 | 51(2):122–144, May 2004.
346 | 
347 | 


--------------------------------------------------------------------------------
/cuckoo.go:
--------------------------------------------------------------------------------
  1 | // Copyright © 2014-2017 Lawrence E. Bakst. All rights reserved.
  2 | 
  3 | // Package cuckoo implements a cuckoo hash table.
  4 | // With the correct options this data structure can achieve 5X more storage efficiency
  5 | // over Go's builtin map with similar performance. See the "README.md" file for all the details.
  6 | // Edit the file "kv_default.go" to define the types for you Key and Value.
  7 | package cuckoo
  8 | 
  9 | import (
 10 | 	_ "bytes"
 11 | 	_ "encoding/binary"
 12 | 	"fmt"
 13 | 	"hash"
 14 | 	"math"
 15 | 	"math/rand"
 16 | 	"unsafe"
 17 | 
 18 | 	"github.com/alecthomas/binary"
 19 | 	"leb.io/cuckoo/primes"
 20 | )
 21 | 
 22 | const (
 23 | 	aes  = 1
 24 | 	j264 = iota
 25 | 	j364 = iota
 26 | )
 27 | 
 28 | type Container interface {
 29 | 	Lookup(key Key) (Value, bool)
 30 | 	Delete(key Key) (Value, bool)
 31 | 	Insert(key Key, val Value) (ok bool)
 32 | 	Map(iter func(key Key, val Value) (stop bool))
 33 | }
 34 | 
 35 | var zeroKey Key
 36 | var zeroVal Value
 37 | 
 38 | // For historical reasons this is called a Bucket but should really be called an element
 39 | type Bucket struct {
 40 | 	key Key
 41 | 	val Value
 42 | }
 43 | 
 44 | // Counters. All public but we now have an API to access them.
 45 | type Counters struct {
 46 | 	BucketSize    int  // size of a single bucket (1 slot) in bytes
 47 | 	SlotsSize     int  // size of a single bucket * slots
 48 | 	Elements      int  // number of elements currently residing in the data structure
 49 | 	Inserts       int  // number of time insert has been called
 50 | 	Probes        int  // number of probes to find a free element
 51 | 	Iterations    int  // number of iterations through all the hash tables in an attemp an insert
 52 | 	Deletes       int  // number of times delete has been called
 53 | 	Lookups       int  // number of lookups
 54 | 	Aborts        int  // number of times an insert had to aborted
 55 | 	Fails         int  // number of times that insert failed
 56 | 	Bumps         int  // number of evicted buckets
 57 | 	TableGrows    int  // number of hash tables added
 58 | 	TraceCnt      int  // number of trance records out
 59 | 	MaxPathLen    int  // longest chain of bumps
 60 | 	MaxProbes     int  // highest number of probes
 61 | 	MaxIterations int  // highest number of interations
 62 | 	MinLevel      int  // lowest level achieved
 63 | 	MinTraceCnt   int  // lowest trace count
 64 | 	Limited       bool // were inserts limited by a load factor
 65 | }
 66 | 
 67 | // Per table stats, again all public.
 68 | type TableCounters struct {
 69 | 	Size     int // c.Nbuckets * c.Nslots
 70 | 	Elements int // number of elements currently residing in this hash table
 71 | 	Bumps    int // number of evicted buckets
 72 | }
 73 | 
 74 | // These two constants seem to work well for many cases, but not all
 75 | const (
 76 | 	InitialStartLevel  = 2000
 77 | 	InitialLowestLevel = -8000
 78 | )
 79 | 
 80 | // Configuration info for the cucko hash is collected in this structure.
 81 | // All fields are exported/public.
 82 | type Config struct {
 83 | 	MaxLoadFactor float64 // don't allow more than MaxElements = Tables * Buckets * Slots elements
 84 | 	StartLevel    int     // starting value for level which is decremented for each insertion attempt
 85 | 	LowestLevel   int     // This is usually a negative number and defines how far level can be decremented
 86 | 	Ntables       int     // number of hash tables
 87 | 	Nbuckets      int     // number of buckets
 88 | 	Nslots        int     // number of slots
 89 | 	Size          int     // Size = Tables * Buckets * Slots
 90 | 	MaxElements   int     // maximum number of elements the data structure can hold
 91 | 	HashName      string  // name of hashing function used
 92 | }
 93 | 
 94 | // A Table is a 2 dimensional matrix of buckets, the first index is the bucket number
 95 | // and the second index is the slot number
 96 | type Table struct {
 97 | 	buckets       []Slots     // each indexed bucket contains a slice of Bucket, called Slots, defined in kv_array.go or kv_slice.go
 98 | 	c             *Cuckoo     // point back to main data structure
 99 | 	seed          uint64      // seed used per table to make a unique hash function
100 | 	hfs           hash.Hash64 // hash function to use, the design allows for different hash functions per table but that is not used
101 | 	Nbuckets      int         // number of buckets
102 | 	Nslots        int         // number of slots
103 | 	Size          int         // Size = Tables * Buckets * Slots
104 | 	MaxElements   int         // maximum number of elements the data structure can hold
105 | 	TableCounters             // per Table stats
106 | }
107 | 
108 | // The main data structure for cuckoo hash.
109 | // Most fields are private but the counters and config are public.
110 | //indexed Slots defined in kv_array.go or kv_slice.go
111 | type Cuckoo struct {
112 | 	tables []*Table // a slice  of Tables, each table having a slice of Slots, each slot holding a Bucket
113 | 	//TableCounters []TableCounters // per table stats
114 | 	//seeds         []uint64        // seeds used per table
115 | 	//hfs           []hash.Hash64   // one for each table + the last one reserved for fingerprints
116 | 	//hs            []uint64        // hash sums for each table and fingerprint, no longer used
117 | 	//r             uint64          // reciprocal of Buckets
118 | 	//n             uint64          // Size
119 | 	rot      int  // table rotator
120 | 	fp       bool // first pass of table insert
121 | 	Config        // config data
122 | 	Counters      // stats
123 | 
124 | 	hashno int         // hash function
125 | 	hf     hash.Hash64 // generic hash function
126 | 	hf32   func(data uint32, seed uint64) uint64
127 | 	hf64   func(data, seed uint64) uint64
128 | 	hfb    func(data []byte, seed uint64) uint64
129 | 
130 | 	//b				[]byte			// used for result of marshalled data
131 | 	buf     *buf            // for marshalling data
132 | 	encoder *binary.Encoder // encoder for serializing Key
133 | 	//rnd				func() float64	// random numbers for eviction
134 | 	rnd            *rand.Rand // random numbers used for eviction
135 | 	eseed          int64      // seed for evictions
136 | 	emptyKey       Key        // empty key
137 | 	emptyValue     Value      // if empty key store value lives here and not in a hash table
138 | 	emptyKeyValid  bool       // something store here
139 | 	ekiz           bool       // empty key is zero
140 | 	grow           bool       // are we allowed to add a hash table as needed?
141 | 	Trace          bool       // produce a trace on stdout
142 | 	NumericKeySize int        // if key is numeric what is size in bytes
143 | }
144 | 
145 | // Simple struct and a couple of methods that satisfy the io.Writer interface.
146 | // buf saves the data in a slice that can be accessed without a copy.
147 | // Used to serialize the key.
148 | type buf struct {
149 | 	b    []byte
150 | 	base [4096]byte
151 | 	i    int
152 | }
153 | 
154 | func (b *buf) Reset() {
155 | 	b.i = 0
156 | }
157 | 
158 | func newBuf(size int) (b *buf) {
159 | 	buf := buf{}
160 | 	//buf.base = make([]byte, size, size)
161 | 	//buf.base = buf.base[0:0] // makes printing buf cleaner
162 | 	return &buf
163 | }
164 | 
165 | // capture io.Writer data in a slice
166 | func (b *buf) Write(p []byte) (n int, err error) {
167 | 	b.b = b.base[b.i : b.i+len(p)]
168 | 	copy(b.b, p)
169 | 	b.i += len(p)
170 | 	//fmt.Printf("Write: len(b.b)=%d, len(p)=%d, % #X\n", len(b.b), len(p), p)
171 | 	//fmt.Printf("b=%#v\n", b)
172 | 	return len(p), nil
173 | }
174 | 
175 | /*
176 | 	BucketSize    int  // size of a single bucket (1 slot) in bytes
177 | 	BucketsSize   int  // size of a single bucket * slots
178 | 	Elements      int  // number of elements currently residing in the data structure
179 | 	Inserts       int  // number of time insert has been called
180 | 	Attempts      int  // number of attempts to insert all elements
181 | 	Iterations    int  // number of iterations through all the hash tables to attemps an insert
182 | 	Deletes       int  // number of times delete has been called
183 | 	Lookups       int  // number of lookups
184 | 	Aborts        int  // number of times an insert had to aborted
185 | 	Fails         int  // number of times that insert failed
186 | 	Bumps         int  // number of evicted buckets
187 | 	TableGrows    int  // number of hash tables added
188 | 	MaxPathLen    int  // longest chain of bumps
189 | 	MaxAttempts   int  // highest number of attempts
190 | 	MaxIterations int  // highest number of interations
191 | 	MinLevel      int  // lowest level achieved
192 | 	Limited       bool // were inserts limited by a load factor
193 | */
194 | 
195 | func (c *Counters) InitCounters() {
196 | 	c.MinLevel = InitialStartLevel
197 | 	c.MinTraceCnt = math.MaxInt64
198 | }
199 | 
200 | func (c *Counters) CountersAdd(add *Counters) {
201 | 	var max = func(a, b int) int {
202 | 		if a < b {
203 | 			return b
204 | 		}
205 | 		return a
206 | 	}
207 | 	var min = func(a, b int) int {
208 | 		if a < b {
209 | 			return a
210 | 		}
211 | 		return b
212 | 	}
213 | 	c.Elements += add.Elements
214 | 	c.Inserts += add.Inserts
215 | 	c.Probes += add.Probes
216 | 	c.Iterations += add.Iterations
217 | 	c.Deletes += add.Deletes
218 | 	c.Lookups += add.Lookups
219 | 	c.Aborts += add.Aborts
220 | 	c.Fails += add.Fails
221 | 	c.Bumps += add.Bumps
222 | 	c.TableGrows += add.TableGrows
223 | 	//tot.BucketSize = add.BucketSize
224 | 	//tot.BucketsSize = add.BucketsSize
225 | 	c.MaxPathLen = max(c.MaxPathLen, add.MaxPathLen)
226 | 	c.MaxProbes = max(c.MaxProbes, add.MaxProbes)
227 | 	c.MaxIterations = max(c.MaxIterations, add.MaxIterations)
228 | 	c.MinLevel = min(c.MinLevel, add.MinLevel)
229 | 	c.MinTraceCnt = min(c.MinTraceCnt, add.TraceCnt)
230 | 	if add.Limited {
231 | 		c.Limited = true
232 | 	}
233 | }
234 | 
235 | // Get the value of some of the counters, need to finish them all XXX
236 | func (c *Cuckoo) GetCounter(s string) int {
237 | 	switch s {
238 | 	case "bumps":
239 | 		return c.Bumps
240 | 	case "inserts":
241 | 		return c.Inserts
242 | 	case "elements":
243 | 		return c.Elements
244 | 	case "size":
245 | 		return c.Size
246 | 	case "MaxPathLen":
247 | 		return c.MaxPathLen
248 | 	default:
249 | 		panic("GetCounter")
250 | 	}
251 | }
252 | 
253 | // Get the value of some of the table counters
254 | func (c *Cuckoo) GetTableCounter(t int, s string) int {
255 | 	if t > c.Ntables {
256 | 		panic("GetTableCounter")
257 | 	}
258 | 	switch s {
259 | 	case "size":
260 | 		return c.tables[t].Size
261 | 	case "elements":
262 | 		return c.tables[t].Elements
263 | 	case "bumps":
264 | 		return c.tables[t].Bumps
265 | 	default:
266 | 		panic("GetTableCounter")
267 | 	}
268 | }
269 | 
270 | // This function used to select a victim bucket to be evicted.
271 | func (c *Cuckoo) rbetween(a int, b int) int {
272 | 	//rf := c.rnd()
273 | 	rf := c.rnd.Float64()
274 | 	diff := float64(b - a + 1)
275 | 	r2 := rf * diff
276 | 	r3 := r2 + float64(a)
277 | 	//	fmt.Printf("rbetween: a=%d, b=%d, rf=%f, diff=%f, r2=%f, r3=%f\n", a, b, rf, diff, r2, r3)
278 | 	ret := int(r3)
279 | 	return ret
280 | }
281 | 
282 | // Dynamicall exapnd the data structure by adding a hash table. Called from Insert and friends.
283 | func (c *Cuckoo) addTable(growFactor float64) {
284 | 	//fmt.Printf("table: %d\n", c.Ntables)
285 | 	c.Ntables++
286 | 	buckets := int(float64(c.Nbuckets) * growFactor)
287 | 	slots := c.Nslots
288 | 	c.Size += buckets * slots
289 | 	c.MaxElements = int(float64(c.Size) * c.MaxLoadFactor)
290 | 	t := new(Table)
291 | 	t.buckets = make([]Slots, buckets, buckets)
292 | 	// we should do this lazily
293 | 	for b, _ := range t.buckets {
294 | 		if len(t.buckets[b]) == 0 {
295 | 			t.buckets[b] = makeSlots(t.buckets[b], slots)
296 | 			for s, _ := range t.buckets[b] {
297 | 				t.buckets[b][s].val = c.emptyValue // ???
298 | 			}
299 | 		}
300 | 	}
301 | 	t.seed = uint64(len(c.tables) + 1)
302 | 	t.hfs = c.getHash(c.HashName, t.seed)
303 | 	t.Nbuckets = c.Nbuckets
304 | 	t.Nslots = c.Nslots
305 | 	t.Size = t.Nbuckets * t.Nslots
306 | 	t.MaxElements = int(float64(t.Size) * c.MaxLoadFactor)
307 | 	t.c = c
308 | 	c.tables = append(c.tables, t)
309 | 
310 | 	// perhaps reset the stats ???
311 | }
312 | 
313 | // Create a new cuckoo hash table of size  = tables * buckets * slots.
314 | // If buckets is negative, the next prime number greater than abs(buckets) is automatically generated,
315 | // You can pass an eseed to seed the random number generator used to select a bucket for eviction.
316 | // Don't allow more than size * loadFactor elements to be stored.
317 | // Therefore, a loadFactor of 1.0 means the hash table can be completely full.
318 | // Use a lower loadFactor to reduce the amount of CPU time used for Inserts when the table gets full.
319 | // Use hashName to specify which hash function to use.
320 | // Currently the only valid hashName strings are "j364" and "aes".
321 | // Only use "aes" on Intel 64 bit machines with the AES instructions.
322 | // If specified, use emptyKey as the key that signifies that an element is unused.
323 | // However, often the default, the Go zero initialization suffices as the emptyKey.
324 | func New(tables, buckets, slots int, eseed int64, loadFactor float64, hashName string, emptyKey ...Key) *Cuckoo {
325 | 	var s Slots
326 | 	var b Bucket
327 | 
328 | 	//fmt.Printf("New: tables=%d, buckets=%d, slots=%d, loadFactor=%f, hashName=%q\n", tables, buckets, slots, loadFactor, hashName)
329 | 	if len(s) > 0 && len(s) != slots {
330 | 		fmt.Printf("New: slot mismatch compiled slots=%d, requested slots=%d\n", len(s), slots)
331 | 		return nil
332 | 	}
333 | 
334 | 	if buckets < 0 {
335 | 		pbuckets := primes.NextPrime(-buckets)
336 | 		//fmt.Printf("buckets=%d, pbuckets=%d\n", buckets, pbuckets)
337 | 		buckets = pbuckets
338 | 	}
339 | 
340 | 	if tables < 1 || buckets < 1 || slots < 1 || loadFactor < 0.0 || loadFactor > 1.0 {
341 | 		fmt.Printf("New: tables=%d, buckets=%d, slots=%d, loadFactor=%f, hashName=%q\n", tables, buckets, slots, loadFactor, hashName)
342 | 		return nil
343 | 	}
344 | 
345 | 	//fmt.Printf("New: tables=%d, buckets=%d, slots=%d, loadFactor=%f, hashName=%q\n", tables, buckets, slots, loadFactor, hashName)
346 | 	c := &Cuckoo{}
347 | 
348 | 	h, err := c.setHash(hashName)
349 | 	if err != nil {
350 | 		return nil
351 | 	}
352 | 	c.hashno = h
353 | 	c.HashName = hashName
354 | 
355 | 	//fmt.Printf("unsafe.Sizeof(akey)=%d\n", unsafe.Sizeof(akey))
356 | 	/*
357 | 		c.b = make([]byte, unsafe.Sizeof(akey), unsafe.Sizeof(akey))
358 | 		c.b = c.b[:]
359 | 	*/
360 | 
361 | 	c.Nbuckets, c.Nslots = buckets, slots
362 | 	c.buf = newBuf(2048)
363 | 	c.encoder = binary.NewEncoder(c.buf)
364 | 	c.grow = true
365 | 	c.StartLevel, c.LowestLevel = InitialStartLevel, InitialLowestLevel
366 | 	c.MaxLoadFactor = loadFactor
367 | 	if len(emptyKey) > 0 {
368 | 		c.emptyKey = emptyKey[0]
369 | 	}
370 | 	c.ekiz = c.emptyKey == zeroKey
371 | 	//c.rnd = rand.Float64
372 | 
373 | 	c.eseed = int64(eseed)
374 | 	src := rand.NewSource(int64(c.eseed))
375 | 	r := rand.New(src)
376 | 	c.rnd = r
377 | 
378 | 	c.BucketSize = int(unsafe.Sizeof(b))
379 | 	c.SlotsSize = int(unsafe.Sizeof(s))
380 | 
381 | 	for i := 0; i < tables; i++ {
382 | 		c.addTable(1.0)
383 | 	}
384 | 	//fmt.Printf("c=%#v\n", c)
385 | 	return c
386 | }
387 | 
388 | // If the Key is a numeric data type set the length here.
389 | func (c *Cuckoo) SetNumericKeySize(size int) {
390 | 	switch size {
391 | 	case 4:
392 | 		c.buf.b = c.buf.base[0:4]
393 | 	case 8:
394 | 		c.buf.b = c.buf.base[0:8]
395 | 	default:
396 | 		panic("SetNumericKeySize")
397 | 	}
398 | 	c.NumericKeySize = size
399 | }
400 | 
401 | // Get the current load factor.
402 | func (c *Cuckoo) GetLoadFactor() float64 {
403 | 	return float64(c.Elements) / float64(c.Size)
404 | }
405 | 
406 | // Set the starting value for level, used by Insert and friends.
407 | func (c *Cuckoo) SetStartLevel(sl int) {
408 | 	c.StartLevel = sl
409 | }
410 | 
411 | // Set the lowest value level call decemnet to.
412 | func (c *Cuckoo) SetLowestLevel(ll int) {
413 | 	c.LowestLevel = ll
414 | }
415 | 
416 | // Set if hash tables can be added dynamically if an insert fails.
417 | func (c *Cuckoo) SetGrow(b bool) {
418 | 	c.grow = b
419 | }
420 | 
421 | // Set if hash tables can be added dynamically if an insert fails.
422 | func (c *Cuckoo) SetEvictionSeed(seed int64) {
423 | 	c.eseed = seed
424 | 	rand.Seed(seed)
425 | }
426 | 
427 | /*
428 |    seed := int64(0)
429 |    // fixed pattern or different values each time
430 |    if *ranf {
431 | 	   seed = time.Now().UTC().UnixNano()
432 |    } else {
433 | 	   seed = int64(0)
434 |    }
435 |    rand.Seed(seed)
436 | */
437 | 
438 | // Given key calculate the hash for the specified table
439 | func (t *Table) calcHashForTable(key Key) uint64 {
440 | 	return t.c.calcHash(t.hfs, t.seed, key)
441 | }
442 | 
443 | // end inlined functions
444 | 
445 | /*
446 | func  (c *Cuckoo) calcHashForTable(t int, key Key) {
447 | 	c.hs[t] = c.calcHash(c.hf[t], c.seeds[t], key)
448 | }
449 | */
450 | 
451 | /*
452 | func (c *Cuckoo) lowHash(hash int64) {
453 | 	switch c.sectors {
454 | 	case 1:
455 | 		return 0
456 | 	case 2:
457 | 		return hash & 1
458 | 	case 4:
459 | 		return hash & 3
460 | 	case 8:
461 | 		return hash & 7
462 | 	case 16:
463 | 		return hash & 15
464 | 	}
465 | }
466 | */
467 | 
468 | // Given key return the value and a "ok" bool indicating success or failure.
469 | func (c *Cuckoo) Lookup(key Key) (Value, bool) {
470 | 	c.Lookups++
471 | 
472 | 	if key == c.emptyKey {
473 | 		if c.emptyKeyValid {
474 | 			return c.emptyValue, true
475 | 		} else {
476 | 			return zeroVal, false
477 | 		}
478 | 	}
479 | 
480 | 	for _, t := range c.tables {
481 | 		h := uint64(t.calcHashForTable(key))
482 | 		b := h % uint64(t.Nbuckets)
483 | 
484 | 		for s, _ := range t.buckets[b] {
485 | 			//fmt.Printf("Lookup: key=%d, table=%d, bucket=%d, slot=%d, found key=%d\n", key, t, b, s, c.tbs[t][b][s].key)
486 | 			if t.buckets[b][s].key == key {
487 | 				//fmt.Printf("Lookup: table=%d, bucket=%d, slot=%d, key=%d, value=%d\n", t, b, s, key, c.tbs[t][b][s].val)
488 | 				return t.buckets[b][s].val, true
489 | 			}
490 | 		}
491 | 	}
492 | 	return zeroVal, false
493 | }
494 | 
495 | // Given key delete the bucket. Return the value found and a bool "ok" indicating success
496 | func (c *Cuckoo) Delete(key Key) (Value, bool) {
497 | 	c.Deletes++
498 | 
499 | 	//fmt.Printf("key=%v, c.emptyKey=%v\n", key, c.emptyKey)
500 | 	if key == c.emptyKey {
501 | 		if c.emptyKeyValid {
502 | 			c.Elements--
503 | 			c.emptyKeyValid = false
504 | 			return c.emptyValue, true
505 | 		} else {
506 | 			//fmt.Printf("Delete: can't find emptyKey %v\n", key)
507 | 			return zeroVal, false
508 | 		}
509 | 	}
510 | 
511 | 	for _, t := range c.tables {
512 | 		b := t.calcHashForTable(key) % uint64(t.Nbuckets)
513 | 		for s, _ := range t.buckets[b] {
514 | 			//fmt.Printf("Delete: check key=%d, table=%d, bucket=%d, slot=%d, found key=%d\n", key, t, b, s, c.tbs[t][b][s].key)
515 | 			if t.buckets[b][s].key == key {
516 | 				//fmt.Printf("Delete: found key=%d, value=%d, table=%d, bucket=%d, slot=%d\n", key, c.tbs[t][b][s].val, t, b, s)
517 | 				t.buckets[b][s].key = c.emptyKey
518 | 				t.Elements--
519 | 				c.Elements--
520 | 				if c.Elements < 0 {
521 | 					panic("Delete")
522 | 				}
523 | 				return t.buckets[b][s].val, true
524 | 			}
525 | 		}
526 | 	}
527 | 	//fmt.Printf("Delete: can't find %v\n", key)
528 | 	return zeroVal, false
529 | }
530 | 
531 | var calls int
532 | 
533 | // Internal version of insert routine.
534 | // Given key, value, and a starting level insert the KV pair. Return ok and level needed to insert.
535 | // If level 0 is returned it means the insert failed
536 | func (c *Cuckoo) insert(key Key, val Value, ilevel int) (ok bool, level int) {
537 | 	var k Key
538 | 	var v Value
539 | 	var bumps int
540 | 	var depth int
541 | 
542 | 	var ins func(kx Key, vx Value) bool // forward declare the closure so we can call it recursively
543 | 	ins = func(kx Key, vx Value) bool {
544 | 		var sk Key
545 | 		var sv Value
546 | 		var pk Key
547 | 		//fmt.Printf("Insert: level=%d, key=%d, ", level, kx)
548 | 		depth++
549 | 		k = kx // was :=
550 | 		v = vx // was :=
551 | 		// we used to move left to right, with the chance of an insert increasing as
552 | 		// we move because the tables filled up left to right.
553 | 		// Now we rotate the starting point. Why has no one done this before.
554 | 		ti := c.rot
555 | 		for _, _ = range c.tables {
556 | 			t := c.tables[ti]
557 | 			h := uint64(t.calcHashForTable(k))
558 | 			//fmt.Printf("h=%#x\n", h)
559 | 			b := h % uint64(t.Nbuckets)
560 | 
561 | 			//fmt.Printf("Insert: next table, h=%#x, level=%d, table=%d, bucket=%d, key=%d, value=%d\n", h, level, t, b, k, v)
562 | 			// check all the slots in the current table and see if we can insert
563 | 			//s := lowHash(h, )
564 | 			//for {
565 | 			for s, _ := range t.buckets[b] {
566 | 				c.Probes++
567 | 				pk = t.buckets[b][s].key // avoid previous allocation
568 | 				c.TraceCnt++
569 | 				if c.Trace {
570 | 					fmt.Printf("{%q: %d, %q: %d, %q: %q, %q: %d, %q: %d, %q: %d, %q: %v, %q: %v},\n",
571 | 						"i", c.TraceCnt, "l", level, "op", "P", "t", t, "b", b, "s", s, "k", k, "v", v)
572 | 				}
573 | 				if pk == c.emptyKey || pk == k { // added replacement semantics
574 | 					t.buckets[b][s].key, t.buckets[b][s].val = k, v
575 | 					c.TraceCnt++
576 | 					if c.Trace {
577 | 						fmt.Printf("{%q: %d, %q: %d, %q: %q, %q: %d, %q: %d, %q: %d, %q: %v, %q: %v},\n",
578 | 							"i", c.TraceCnt, "l", level, "op", "I", "t", t, "b", b, "s", s, "k", k, "v", v)
579 | 					}
580 | 					if pk == c.emptyKey || pk == k {
581 | 						//fmt.Printf("Insert: h=%#x, level=%d, table=%d, bucket=%d, slot=%d, pk=%d, key=%d, value=%d\n", h, level, t, b, s, pk, k, v)
582 | 					}
583 | 					c.Elements++
584 | 					t.Elements++
585 | 					return true
586 | 				}
587 | 			}
588 | 			// unproven and untested optimization below XXX
589 | 			// if first insert attempt and no slots in this table and more than 2 tables, try the next table
590 | 			if depth == 0 && len(c.tables) > 2 {
591 | 				continue
592 | 			}
593 | 			// No slots available in this table available, evict a random KV pair and store the current KV where it was.
594 | 			// move to the next table and different bucket and hope it works out better.
595 | 			bumps++
596 | 			c.Bumps++
597 | 			t.Bumps++
598 | 			victim := c.rbetween(0, t.Nslots-1)
599 | 			//fmt.Printf("insert: level=%d, bump value=%d for value=%d, table=%d, bucket=%d, slot=%d\n", level, c.tbs[t][b][victim].val, val, t, b, victim)
600 | 			sk, sv = t.buckets[b][victim].key, t.buckets[b][victim].val // avoid previous stack allocation
601 | 			c.TraceCnt++
602 | 			if c.Trace {
603 | 				fmt.Printf("{%q: %d, %q: %d, %q: %q, %q: %d, %q: %d, %q: %d, %q: %v, %q: %v},\n",
604 | 					"i", c.TraceCnt, "l", level, "op", "E", "t", t, "b", b, "s", victim, "k", sk, "v", v)
605 | 			}
606 | 			t.buckets[b][victim].key = k
607 | 			t.buckets[b][victim].val = v
608 | 			c.TraceCnt++
609 | 			if c.Trace {
610 | 				fmt.Printf("{%q: %d, %q: %d, %q: %q, %q: %d, %q: %d, %q: %d, %q: %v, %q: %v},\n",
611 | 					"i", c.TraceCnt, "l", level, "op", "I", "t", t, "b", b, "s", victim, "k", k, "v", v)
612 | 			}
613 | 			k = sk
614 | 			v = sv
615 | 			//c.calcHashes(k) ??? XXX ???
616 | 			//fmt.Printf("insert: level=%d, new key=%d, val=%d\n", level, k, v)
617 | 			ti++
618 | 			if ti > len(c.tables)-1 {
619 | 				ti = 0
620 | 			}
621 | 		}
622 | 		// Could not find any space for key in any table. Since the key has now changed,
623 | 		// we try again with what will probably be different buckets hoping for a place.
624 | 		c.Iterations++
625 | 		level--
626 | 
627 | 		// If we reach level 0 we have failed to insert after InitialStartLevel interations, each examining
628 | 		// t hash tables with s slots each. We don't stop because they current key
629 | 		// is probaly not be the key we started with, so we keep going, hoping to finally get the original
630 | 		// key back, to avoid data loss.
631 | 		// It can also happen that in the process of doing this the key ends up being inserted because
632 | 		// the loop and logic is identical except we stop trying if the key is inserted OR we get the original
633 | 		// key back.
634 | 		// We skip 0 because it's used as a return value that Insert failed because of load factor constraint.
635 | 		// We call this an abort. An abort does not imply the KV failed to insert.
636 | 		if level == 0 {
637 | 			//fmt.Printf("insert: begin abort key=%d, val=%d, calls=%d, depth=%d, c.Iterations=%d\n", key, val, calls, depth, c.Iterations)
638 | 			c.Aborts++ // stop trying to insert and recover displaced data
639 | 			level = -1
640 | 		}
641 | 		// At this point we have failed to recover the original KV after abs(InitialLowestLevel) more interations.
642 | 		// This means that the insert failed AND a random KV was also deleted from the Cuckoo table.
643 | 		// Give up, we call this a "fail"
644 | 		if level <= c.LowestLevel {
645 | 			c.Fails++
646 | 			fmt.Printf("cukcoo: Insert FAILED, val=%v, key=%v\n", k, v)
647 | 			return false
648 | 		}
649 | 		if level <= 0 {
650 | 			// NB: fine point, on insert failure, the key NOT inserted may NOT be the original key.
651 | 			// Keep interating until the original key is not found to prevent random data loss
652 | 			// So level less than 0 means had to work to get a displaced key back into the hash table
653 | 			_, found := c.Lookup(key)
654 | 			//fmt.Printf("key %d found=%v\n", key, found)
655 | 			// if we can't find the key that was passed in then it is safe to stop because there will
656 | 			// be no data loss. If we can find they key that was passed in, then some other key
657 | 			// has been displaced.
658 | 			// This is an interesting case that I had never seen before. Insert fails and a random
659 | 			// piece of data that was previusly inserted has been lost. Luckily the fix is pretty easy.
660 | 			if !found {
661 | 				fmt.Printf("insert: aborted at key=%d, value=%d, calls=%d, depth=%d, level=%d, aborts=%d\n", key, val, calls, depth, level, c.Aborts)
662 | 				return false
663 | 			}
664 | 		}
665 | 		// ??? consider bumping c.rot here as opposed to below
666 | 		return ins(k, v) // try to insert again, tail recursively
667 | 	}
668 | 
669 | 	// insert starts here
670 | 	//fmt.Printf("Insert: level=%d, key=%d, value=%d\n", level, key, val)
671 | 	calls++
672 | 	k = key
673 | 	v = val
674 | 	sva, svi := c.Probes, c.Iterations
675 | 	level = ilevel
676 | again:
677 | 	if c.Elements >= c.MaxElements {
678 | 		//fmt.Printf("insert: limited at %v\n", key)
679 | 		c.Limited = true
680 | 		return false, 0
681 | 	}
682 | 	if k == c.emptyKey {
683 | 		if c.emptyKeyValid {
684 | 			panic("emptyKeyValid")
685 | 		} else {
686 | 			c.Inserts++
687 | 			c.Elements++
688 | 			c.emptyKeyValid = true
689 | 			c.emptyValue = v
690 | 		}
691 | 		return true, level
692 | 	}
693 | 	ok = ins(k, v)
694 | 	if ok {
695 | 		c.Inserts++
696 | 	} else {
697 | 		if c.grow {
698 | 			fmt.Printf("insert: add a table, level=%d, key=%v, val=%v\n", level, k, v)
699 | 			c.TableGrows++
700 | 			//c.Ntables++
701 | 			c.addTable(0)
702 | 			goto again
703 | 		}
704 | 	}
705 | 	if c.Probes-sva > c.MaxProbes {
706 | 		c.MaxProbes = c.Probes - sva
707 | 	}
708 | 	if c.Iterations-svi > c.MaxIterations {
709 | 		c.MaxIterations = c.Iterations - svi
710 | 	}
711 | 	if level < c.MinLevel {
712 | 		c.MinLevel = level
713 | 	}
714 | 	if bumps > c.MaxPathLen {
715 | 		c.MaxPathLen = bumps
716 | 	}
717 | 	c.rot++
718 | 	c.rot %= c.Ntables
719 | 	//fmt.Printf("c.rot=%d, c.Ntables=%d\n", c.rot, c.Ntables)
720 | 	//fmt.Printf("%d/%d ", c.Attempts - sva, c.Iterations - svi)
721 | 	return
722 | }
723 | 
724 | // Given key, value insert a KV pair and return ok.
725 | func (c *Cuckoo) Insert(key Key, val Value) (ok bool) {
726 | 	ok, _ = c.insert(key, val, c.StartLevel)
727 | 	return
728 | }
729 | 
730 | // Given key, value insert a KV pair and return ok and level needed to insert
731 | func (c *Cuckoo) InsertL(key Key, val Value) (ok bool, rlevel int) {
732 | 	ok, rlevel = c.insert(key, val, c.StartLevel)
733 | 	return
734 | }
735 | 
736 | // should this be redone??
737 | func (c *Cuckoo) Map(iter func(c *Cuckoo, key Key, val Value) (stop bool)) {
738 | 	if c.emptyKeyValid {
739 | 		iter(c, c.emptyKey, c.emptyValue)
740 | 	}
741 | 
742 | 	for _, t := range c.tables {
743 | 		for _, s := range t.buckets {
744 | 			for _, b := range s {
745 | 				if b.key != c.emptyKey {
746 | 					if iter(c, b.key, b.val) {
747 | 						return
748 | 					}
749 | 				}
750 | 			}
751 | 		}
752 | 	}
753 | }
754 | 
755 | // doesn't print the value if c.emptyKeyValid is true
756 | func (c *Cuckoo) Print() {
757 | 	for ti, t := range c.tables {
758 | 		for si, s := range t.buckets {
759 | 			fmt.Printf("[%d][%d]: ", ti, si)
760 | 			cnt := 0
761 | 			for _, b := range s {
762 | 				if b.key != c.emptyKey {
763 | 					cnt++
764 | 				}
765 | 			}
766 | 			fmt.Printf("%d\n", cnt)
767 | 		}
768 | 	}
769 | }
770 | 
771 | /*
772 | func init() {
773 | 	var k Key
774 | 	var v Value = 1// "foobar"
775 | 
776 | 	fmt.Printf("Key=%T\n", k)
777 | 	fmt.Printf("Value=%T\n", v)
778 | }
779 | */
780 | 


--------------------------------------------------------------------------------
/cuckoo_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright © 2014-2017 Lawrence E. Bakst. All rights reserved.
  2 | package cuckoo_test
  3 | 
  4 | import (
  5 | 	"fmt"
  6 | 	"math/rand"
  7 | 	"runtime"
  8 | 	"testing"
  9 | 
 10 | 	. "leb.io/cuckoo"
 11 | 	. "leb.io/cuckoo/internal/dstest"
 12 | 	"leb.io/hrff"
 13 | )
 14 | 
 15 | //import "flag"
 16 | 
 17 | //import "math"
 18 | 
 19 | var r = rand.Float64
 20 | var b = int(0)
 21 | var n = int(2e6)
 22 | 
 23 | const hashName = "aes" // aes" "j264"
 24 | 
 25 | type KeySet struct {
 26 | 	Keys       []Key
 27 | 	Vals       []Value
 28 | 	M          map[Key]Value
 29 | 	AllocBytes uint64
 30 | }
 31 | 
 32 | var ks *KeySet
 33 | 
 34 | func hu(v uint64, u string) hrff.Int64 {
 35 | 	return hrff.Int64{V: int64(v), U: u}
 36 | }
 37 | 
 38 | func hi(v int64, u string) hrff.Int64 {
 39 | 	return hrff.Int64{V: int64(v), U: u}
 40 | }
 41 | 
 42 | type config struct {
 43 | 	ef     float64
 44 | 	add    float64
 45 | 	lf     float64
 46 | 	flf    float64
 47 | 	tables int
 48 | 	slots  int
 49 | 	n      int
 50 | }
 51 | 
 52 | const ef = 1.01
 53 | const add = 32.0
 54 | const lf = 1.0
 55 | const flf = 1.0
 56 | const tables = 2
 57 | const slots = 8
 58 | 
 59 | var cf = config{ef: 1.01, add: 32.0, lf: 1.0, flf: 0.9, tables: 4, slots: 8, n: 1000000}
 60 | 
 61 | func dump_mstats(m *runtime.MemStats, mstats, cstats, gc bool) {
 62 | 	if mstats {
 63 | 		fmt.Printf("Alloc=%h, TotalAlloc=%h, Sys=%h, Lookups=%h, Mallocs=%h, Frees=%h\n",
 64 | 			hu(m.Alloc, "B"), hu(m.TotalAlloc, "B"), hu(m.Sys, "B"), hu(m.Lookups, ""), hu(m.Mallocs, ""), hu(m.Frees, ""))
 65 | 		fmt.Printf("HeapAlloc=%h, HeapSys=%h, HeapIdle=%h, HeapInuse=%h, HeapReleased=%h, HeapObjects=%h\n",
 66 | 			hu(m.HeapAlloc, "B"), hu(m.HeapSys, "B"), hu(m.HeapIdle, "B"), hu(m.HeapInuse, "B"), hu(m.HeapReleased, "B"), hu(m.HeapObjects, ""))
 67 | 		fmt.Printf("StackInuse=%d, StackSys=%d, MSpanInuse=%d, MSpanSys=%d, MCacheSys=%d, BuckHashSys=%d\n", m.StackInuse, m.StackSys, m.MSpanInuse, m.MSpanSys, m.MCacheSys, m.BuckHashSys)
 68 | 		fmt.Printf("NextGC=%d, LastGC=%d, PauseTotalNs=%d, NumGC=%d, EnableGC=%v, DebugGC=%v\n", m.NextGC, m.LastGC, m.PauseTotalNs, m.NumGC, m.EnableGC, m.DebugGC)
 69 | 	}
 70 | 	if cstats {
 71 | 		for i, b := range m.BySize {
 72 | 			if b.Mallocs == 0 {
 73 | 				continue
 74 | 			}
 75 | 			fmt.Printf("BySize[%d]: Size=%d, Malloc=%d, Frees=%d\n", i, b.Size, b.Mallocs, b.Frees)
 76 | 		}
 77 | 	}
 78 | 	if gc {
 79 | 		for i := range m.PauseNs {
 80 | 			fmt.Printf("PauseNs: ")
 81 | 			fmt.Printf("%d, ", m.PauseNs[(int(m.NumGC)+255+i)%256])
 82 | 			fmt.Printf("\n")
 83 | 		}
 84 | 	}
 85 | }
 86 | 
 87 | func CreateKeysValuesMap(b, n int) *KeySet {
 88 | 	var v Value
 89 | 	var msb, msa runtime.MemStats
 90 | 	var ks KeySet
 91 | 
 92 | 	ks.Keys = make([]Key, n, n)
 93 | 	ks.Vals = make([]Value, n, n)
 94 | 
 95 | 	runtime.ReadMemStats(&msb)
 96 | 	ks.M = make(map[Key]Value)
 97 | 	for i := b; i < b+n; i++ {
 98 | 		k := Key(rand.Uint32())
 99 | 		ks.M[k] = v
100 | 		ks.Keys[i] = k
101 | 		ks.Vals[i] = v
102 | 	}
103 | 	runtime.ReadMemStats(&msa)
104 | 	ks.AllocBytes = msa.Alloc - msb.Alloc
105 | 	return &ks
106 | }
107 | 
108 | func init() {
109 | 	ks = CreateKeysValuesMap(b, n)
110 | }
111 | 
112 | type IB interface {
113 | 	Logf(string, ...interface{})
114 | 	FailNow()
115 | }
116 | 
117 | func setup(t IB, cf config) (d *DSTest) { // testing.T
118 | 	//New(tables, -int(float64(n)*ef+add)/(tables*slots), slots, 0, lf, hashName)
119 | 	//start := time.Now()
120 | 	c := New(cf.tables, -int(float64(cf.n)*cf.ef+cf.add)/(cf.tables*cf.slots), cf.slots, 0, cf.lf, hashName)
121 | 	if c == nil {
122 | 		t.Logf("TestBasic: failed probably because slots don't match")
123 | 		t.FailNow()
124 | 	}
125 | 	d = NewTester(c, 2000, 0) // ???
126 | 	d.I = c
127 | 	//t.Logf("Config=%#v\n", c.Config)
128 | 
129 | 	c.SetNumericKeySize(8)
130 | 	return
131 | }
132 | 
133 | func TestBasic(t *testing.T) {
134 | 	var cf = config{ef: 1.01, add: 32.0, lf: 1.0, flf: 0.9, tables: 4, slots: 8, n: 1000000}
135 | 	d := setup(t, cf)
136 | 	_ = d.Fill(cf.tables, cf.n/(cf.tables*cf.slots), cf.slots, 1, cf.flf, false, false, false, false)
137 | 	ok := d.Verify(1, n/(tables*slots), false)
138 | 	if !ok {
139 | 		t.FailNow()
140 | 	}
141 | 	//t.Logf("Stats=%#v\n", c.CuckooStat)
142 | }
143 | 
144 | func TestMemoryEfficiency(t *testing.T) {
145 | 	var cf = config{ef: 1.01, add: 32.0, lf: 1.0, flf: 1.0, tables: 2, slots: 8, n: 1000000}
146 | 	var msb, msa runtime.MemStats
147 | 
148 | 	runtime.ReadMemStats(&msb)
149 | 	d := setup(t, cf)
150 | 	//d := setup(t, cf)
151 | 	//for k, v := range ks.M {
152 | 	//	c.Insert(k, v)
153 | 	//}
154 | 	fs := d.Fill(cf.tables, cf.n/(cf.tables*cf.slots), cf.slots, 1.0, cf.flf, false, false, false, true)
155 | 	runtime.ReadMemStats(&msa)
156 | 
157 | 	//dump_mstats(&msb, true, false, false)
158 | 	//fmt.Printf("\n")
159 | 	//dump_mstats(&msa, true, false, false)
160 | 	//fmt.Printf("msb=%#v\n", msb)
161 | 	//fmt.Printf("msa=%#v\n", msa)
162 | 
163 | 	c := (d.I).(*Cuckoo)
164 | 	t.Logf("Cuckoo Hash LoadFactor:       %0.2f", c.GetLoadFactor())
165 | 	t.Logf("Cuckoo Hash memory allocated: %0.0f MiB", float64(msa.Alloc-msb.Alloc)/float64(1<<20))
166 | 	t.Logf("Go map memory allocated:      %0.0f MiB", float64(ks.AllocBytes)/float64(1<<20))
167 | 	//t.Logf("stats=%#v\n", fs)
168 | 	fs.Fails = fs.Fails
169 | 	//fmt.Printf("Config=%#v\n", c.Config)
170 | 	//fmt.Printf("Counters=%#v\n\n", c.Counters)
171 | }
172 | 
173 | func benchmarkCuckooInsert(ef, add, lf float64, tables, slots int, hash string, b *testing.B) {
174 | 	//t.Logf("BenchmarkCuckooInsert: N=%d, ef=%f, add=%f, lf=%f, tables=%d, slots=%d\n", b.N, ef, add, lf, tables, slots)
175 | 	d := setup(b, cf)
176 | 	//fmt.Printf("Config=%#v\n", c.Config)
177 | 	//fmt.Printf("N=%d\n", b.N)
178 | 	b.ResetTimer()
179 | 
180 | 	if true {
181 | 		for i := 0; i < b.N; i++ {
182 | 			d.I.Insert(ks.Keys[i%n], ks.Vals[i%n])
183 | 		}
184 | 	} else {
185 | 		// func (d *DSTest) Fill(tables, buckets, slots, ibase int, flf float64, verbose, pl, progress bool, r bool) *FillStats {
186 | 
187 | 		fs := d.Fill(tables, b.N, slots, 1, flf, false, false, false, true)
188 | 		fs.Fails = fs.Fails
189 | 		//b.Logf("stats=%#v\n", fs)
190 | 	}
191 | 	//fmt.Printf("Config=%#v\n", c.Config)
192 | 	//fmt.Printf("Counters=%#v\n\n", c.Counters)
193 | 	b.ReportAllocs()
194 | }
195 | 
196 | func benchmarkCuckooSearch(ef, add, lf float64, tables, slots int, hash string, b *testing.B) {
197 | 	var cf = config{ef: 1.01, add: 32.0, lf: 1.0, flf: 0.8, tables: 4, slots: 8, n: 1000000}
198 | 	d := setup(b, cf)
199 | 	for i := 0; i < b.N; i++ {
200 | 		d.I.Insert(ks.Keys[i%n], ks.Vals[i%n])
201 | 	}
202 | 	b.ResetTimer()
203 | 	b.ReportAllocs()
204 | 
205 | 	for i := 0; i < b.N; i++ {
206 | 		d.I.Lookup(ks.Keys[i%n])
207 | 	}
208 | }
209 | 
210 | func benchmarkCuckooDelete(ef, add, lf float64, tables, slots int, hash string, b *testing.B) {
211 | 	d := setup(b, cf)
212 | 	for i := 0; i < b.N; i++ {
213 | 		d.I.Insert(ks.Keys[i%n], ks.Vals[i%n])
214 | 	}
215 | 	b.ResetTimer()
216 | 	b.ReportAllocs()
217 | 
218 | 	for i := 0; i < b.N; i++ {
219 | 		d.I.Delete(ks.Keys[i%n])
220 | 	}
221 | }
222 | 
223 | func BenchmarkCuckoo2T2SInsert(b *testing.B) {
224 | 	benchmarkCuckooInsert(ef, add, lf, tables, slots, hashName, b)
225 | }
226 | 
227 | func BenchmarkCuckoo2T2SSearch(b *testing.B) {
228 | 	benchmarkCuckooSearch(ef, add, lf, tables, slots, hashName, b)
229 | }
230 | 
231 | func BenchmarkCuckoo2T2SDelete(b *testing.B) {
232 | 	benchmarkCuckooDelete(ef, add, lf, tables, slots, hashName, b)
233 | }
234 | 
235 | /*
236 | func BenchmarkCuckoo4T4SInsert(b *testing.B) {
237 | 	benchmarkCuckooInsert(1.0, 32.0, 0.99, 4, 4, "m332", b)
238 | }
239 | 
240 | func BenchmarkCuckoo4T4SSearch(b *testing.B) {
241 | 	benchmarkCuckooSearch(1.0, 32.0, 0.99, 4, 4, "m332", b)
242 | }
243 | 
244 | func BenchmarkCuckoo4T4SDelete(b *testing.B) {
245 | 	benchmarkCuckooDelete(1.0, 32.0, 0.99, 4, 4, "m332", b)
246 | }
247 | */
248 | 
249 | func GoMapInsert(m map[Key]Value, nn int) {
250 | 	for i := 0; i < nn; i++ {
251 | 		m[ks.Keys[i%n]] = ks.Vals[i%n]
252 | 	}
253 | }
254 | 
255 | func BenchmarkGoMapInsert(b *testing.B) {
256 | 	m := make(map[Key]Value)
257 | 	b.ResetTimer()
258 | 	b.ReportAllocs()
259 | 	GoMapInsert(m, b.N)
260 | }
261 | 
262 | func BenchmarkGoMapSearch(b *testing.B) {
263 | 	m := make(map[Key]Value)
264 | 
265 | 	for i := 0; i < len(ks.Keys); i++ {
266 | 		m[ks.Keys[i%n]] = ks.Vals[i%n]
267 | 	}
268 | 	b.ResetTimer()
269 | 	b.ReportAllocs()
270 | 
271 | 	for i := 0; i < b.N; i++ {
272 | 		_, _ = m[ks.Keys[i%n]]
273 | 	}
274 | }
275 | 
276 | func BenchmarkGoMapDelete(b *testing.B) {
277 | 	m := make(map[Key]Value)
278 | 
279 | 	for i := 0; i < len(ks.Keys); i++ {
280 | 		m[ks.Keys[i%n]] = ks.Vals[i%n]
281 | 	}
282 | 	b.ResetTimer()
283 | 	b.ReportAllocs()
284 | 
285 | 	for i := 0; i < b.N; i++ {
286 | 		delete(m, ks.Keys[i%n])
287 | 	}
288 | }
289 | 
290 | // Demonstrate how to create a cuckoo table and insert, lookup, and delete elemebts
291 | func Example() {
292 | 	const tables = 4
293 | 	const buckets = 11
294 | 	const slots = 8
295 | 	//const hashName = "m332"
296 | 	var lf = 0.95 // has to be a var or we get an err
297 | 	var cnt int
298 | 
299 | 	var countf = func(c *Cuckoo, key Key, val Value) (stop bool) {
300 | 		cnt++
301 | 		return
302 | 	}
303 | 
304 | 	c := New(tables, buckets, slots, 0, lf, hashName)
305 | 	if c == nil {
306 | 		fmt.Printf("Example: New failed probably because slots don't match")
307 | 	}
308 | 	c.SetNumericKeySize(8)
309 | 
310 | 	n := int(float64(tables*buckets*slots) * lf)
311 | 
312 | 	// insert
313 | 	for i := 0; i < n; i++ {
314 | 		k, v := Key(i), Value(i)
315 | 		ok := c.Insert(k, v)
316 | 		if !ok {
317 | 			fmt.Printf("Example: Insert failed")
318 | 			return
319 | 		}
320 | 	}
321 | 
322 | 	// lookup
323 | 	for i := 0; i < n; i++ {
324 | 		k := Key(i)
325 | 		v, ok := c.Lookup(k)
326 | 		if !ok {
327 | 			fmt.Printf("Example: Lookup failed")
328 | 			return
329 | 		}
330 | 		if v != Value(i) {
331 | 			fmt.Printf("Example: Values don't match %v vs %v\n", v, Value(i))
332 | 		}
333 | 	}
334 | 
335 | 	// iterate
336 | 	c.Map(countf)
337 | 	s := fmt.Sprintf("cnt=%d vs %d\n", cnt, c.Counters.Elements)
338 | 	if cnt != c.Counters.Elements {
339 | 		panic(s)
340 | 	}
341 | 
342 | 	// delete
343 | 	for i := 0; i < n; i++ {
344 | 		k := Key(i)
345 | 		v, ok := c.Delete(k)
346 | 		if !ok {
347 | 			fmt.Printf("Example: Delete failed")
348 | 			return
349 | 		}
350 | 		if v != Value(i) {
351 | 			fmt.Printf("Example: Values don't match %v vs %v\n", v, Value(i))
352 | 		}
353 | 	}
354 | 
355 | 	// iterate
356 | 	cnt = 0
357 | 	c.Map(countf)
358 | 	if cnt != 0 {
359 | 		panic("cnt 2")
360 | 	}
361 | 
362 | 	fmt.Printf("Example: Passed\n")
363 | 	// Output:
364 | 	// Example: Passed
365 | }
366 | 
367 | var _ DSTester = New(4, 11, 8, 0, 1.0, "aes")
368 | 


--------------------------------------------------------------------------------
/example/example.go:
--------------------------------------------------------------------------------
  1 | // Copyright © 2014-2017 Lawrence E. Bakst. All rights reserved.
  2 | 
  3 | // This program provides a test interface to the cuckoo hash tables.
  4 | // The only test it currently knows how to do is crete the table,
  5 | // fill it with values, verify the values are in the table, and
  6 | // then verify they are not in the table
  7 | package main
  8 | 
  9 | import (
 10 | 	cr "crypto/rand"
 11 | 	"flag"
 12 | 	"fmt"
 13 | 	"log"
 14 | 	_ "math"
 15 | 	_ "math/rand"
 16 | 	"os"
 17 | 	"runtime"
 18 | 	"runtime/pprof"
 19 | 	"time"
 20 | 	"unsafe"
 21 | 
 22 | 	"leb.io/cuckoo"
 23 | 	"leb.io/cuckoo/internal/dstest"
 24 | 	"leb.io/cuckoo/internal/siginfo"
 25 | 	"leb.io/cuckoo/primes"
 26 | 	"leb.io/hrff"
 27 | )
 28 | 
 29 | func tdiff(begin, end time.Time) time.Duration {
 30 | 	d := end.Sub(begin)
 31 | 	return d
 32 | }
 33 | 
 34 | var ranb = flag.Bool("rb", false, "ignore base, use random base value")
 35 | var ranr = flag.Bool("rr", false, "random run, seed for base values is random")
 36 | var rane = flag.Bool("re", false, "use a random seed for evictions source")
 37 | var ranh = flag.Bool("rh", false, "use a random seed for table hash functions")
 38 | var seedb = flag.Int("sb", 0, "seed for base values")
 39 | var seede = flag.Int("se", 0, "seed for eviction values")
 40 | 
 41 | var auto = flag.Bool("a", false, "automatic")
 42 | var fo = flag.Bool("fo", false, "fill only")
 43 | var dg = flag.Bool("dg", false, "dont't add hash tables automatically")
 44 | var hash = flag.String("h", "aes", "name of hash function {aes, j264, j364}")
 45 | var ntables = flag.Int("t", 4, "tables")
 46 | var nbuckets = flag.Int("b", 31, "buckets")
 47 | var nslots = flag.Int("s", 8, "slots")
 48 | var _ntrials = hrff.Int{V: 5, U: "trials"}
 49 | 
 50 | //var ntrials = flag.Int("nt", 5, "number of trials")
 51 | var ntrials *int = &_ntrials.V
 52 | var ibase = flag.Int("base", 1, "base of fill series, -1 for random")
 53 | var startLevel = flag.Int("sl", 2000, "starting level")
 54 | var lowLevel = flag.Int("ll", -8000, "lowest level")
 55 | var lf = flag.Float64("lf", 0.96, "maximum load factor")
 56 | var flf = flag.Float64("flf", 1.0, "fill load factor")
 57 | 
 58 | var pl = flag.Bool("pl", false, "print level of each insert")
 59 | var pt = flag.Bool("pt", false, "print summary for each trail")
 60 | var ps = flag.Bool("ps", false, "print stats at the end of all trails")
 61 | var pr = flag.Bool("pr", false, "print progress")
 62 | var pf = flag.Bool("pf", false, "print info on failure")
 63 | var trace = flag.Bool("trace", false, "produce trace on stdout")
 64 | var verbose = flag.Bool("v", false, "verbose")
 65 | 
 66 | var cp = flag.String("cp", "", "write cpu profile to file")
 67 | var mp = flag.String("mp", "", "write memory profile to this file")
 68 | 
 69 | var printStatusOneShot bool
 70 | 
 71 | var bseed = int64(0) // seed for base values
 72 | var cseed = int64(0) // seed for evictions
 73 | 
 74 | func hu(v uint64, u string) hrff.Int64 {
 75 | 	return hrff.Int64{V: int64(v), U: u}
 76 | }
 77 | 
 78 | func hi(v int64, u string) hrff.Int64 {
 79 | 	return hrff.Int64{V: int64(v), U: u}
 80 | }
 81 | 
 82 | /*
 83 | func rbetween(r int, a int, b int) int {
 84 | 	rf := r.Float64()
 85 | 	diff := float64(b - a + 1)
 86 | 	r2 := rf * diff
 87 | 	r3 := r2 + float64(a)
 88 | 	ret := int(r3)
 89 | 	fmt.Printf("rbetween: a=%d, b=%d, rf=%f, diff=%f, r2=%f, r3=%f, ret=%d\n", a, b, rf, diff, r2, r3, ret)
 90 | 	return ret
 91 | }
 92 | */
 93 | 
 94 | func dump_mstats(m *runtime.MemStats, mstats, cstats, gc bool) {
 95 | 	if mstats {
 96 | 		fmt.Printf("Alloc=%h, TotalAlloc=%h, Sys=%h, Lookups=%h, Mallocs=%h, Frees=%h\n",
 97 | 			hu(m.Alloc, "B"), hu(m.TotalAlloc, "B"), hu(m.Sys, "B"), hu(m.Lookups, ""), hu(m.Mallocs, ""), hu(m.Frees, ""))
 98 | 		fmt.Printf("HeapAlloc=%h, HeapSys=%h, HeapIdle=%h, HeapInuse=%h, HeapReleased=%h, HeapObjects=%h\n",
 99 | 			hu(m.HeapAlloc, "B"), hu(m.HeapSys, "B"), hu(m.HeapIdle, "B"), hu(m.HeapInuse, "B"), hu(m.HeapReleased, "B"), hu(m.HeapObjects, ""))
100 | 		fmt.Printf("StackInuse=%d, StackSys=%d, MSpanInuse=%d, MSpanSys=%d, MCacheSys=%d, BuckHashSys=%d\n", m.StackInuse, m.StackSys, m.MSpanInuse, m.MSpanSys, m.MCacheSys, m.BuckHashSys)
101 | 		fmt.Printf("NextGC=%d, LastGC=%d, PauseTotalNs=%d, NumGC=%d, EnableGC=%v, DebugGC=%v\n", m.NextGC, m.LastGC, m.PauseTotalNs, m.NumGC, m.EnableGC, m.DebugGC)
102 | 	}
103 | 	if cstats {
104 | 		for i, b := range m.BySize {
105 | 			if b.Mallocs == 0 {
106 | 				continue
107 | 			}
108 | 			fmt.Printf("BySize[%d]: Size=%d, Malloc=%d, Frees=%d\n", i, b.Size, b.Mallocs, b.Frees)
109 | 		}
110 | 	}
111 | 	if gc {
112 | 		for i := range m.PauseNs {
113 | 			fmt.Printf("PauseNs: ")
114 | 			fmt.Printf("%d, ", m.PauseNs[(int(m.NumGC)+255+i)%256])
115 | 			fmt.Printf("\n")
116 | 		}
117 | 	}
118 | }
119 | 
120 | //func trials(tables, buckets, slots, trials int, lf float64, ibase int, verbose, r bool) (cs *cuckoo.Counters, avg float64, rmax int, fails int) {
121 | func trials(tables, buckets, slots, trials int, eseed int64, lf float64, ibase int, verbose, ranr, ranb bool) (d *dstest.DSTest, cs *cuckoo.Counters, avg float64, used int, rmax int, fails int) {
122 | 	var key cuckoo.Key
123 | 	var acs cuckoo.Counters
124 | 	var labels = []string{"init", "fill", "verify", "delete", "verify"}
125 | 	var durations = make([]time.Duration, 5)
126 | 	var msb, msa runtime.MemStats
127 | 
128 | 	var print = func(i, used int) {
129 | 		if verbose {
130 | 			tmp := labels[i]
131 | 			f2 := hrff.Float64{float64(used) * (float64(time.Second) / float64(durations[i])), "ops/sec"}
132 | 			fmt.Printf("    %s: %v %h\n", tmp, durations[i], f2)
133 | 		}
134 | 	}
135 | 
136 | 	cs = &acs
137 | 	cs.InitCounters()
138 | 	tot := float64(0)
139 | 	fails = 0
140 | 
141 | 	if ranr {
142 | 		bseed = time.Now().UTC().UnixNano()
143 | 		// fixed pattern or different values each time
144 | 		b := make([]byte, 8)
145 | 		_, err := cr.Read(b)
146 | 		if err != nil {
147 | 			fmt.Println("error:", err)
148 | 			return
149 | 		}
150 | 		bseed = int64(uint64(b[0])<<56 | uint64(b[1])<<48 | uint64(b[2])<<40 | uint64(b[3])<<32 |
151 | 			uint64(b[4])<<24 | uint64(b[5])<<16 | uint64(b[6])<<8 | uint64(b[7])<<0)
152 | 	} else {
153 | 		bseed = int64(*seedb)
154 | 	}
155 | 	//r := rand.New(rand.NewSource(int64(bseed)))
156 | 	//rand.Seed(seed)
157 | 
158 | 	td := dstest.NewTester(nil, *startLevel, 0) // ???
159 | 	for t := 0; t < trials; t++ {
160 | 		//fmt.Printf("t=%d, fails=%d\n", t, fails)
161 | 		// init
162 | 		//fmt.Printf("trials: init\n")
163 | 		start := time.Now()
164 | 		c := cuckoo.New(tables, buckets, slots, int64(*seede), lf, *hash)
165 | 		if c == nil {
166 | 			panic("New failed")
167 | 		}
168 | 		if *trace {
169 | 			c.Trace = true
170 | 		}
171 | 		d = dstest.NewTester(c, *startLevel, bseed) // ???
172 | 		//d.I = c
173 | 
174 | 		siz := int(unsafe.Sizeof(key))
175 | 		switch siz {
176 | 		case 4, 8:
177 | 			//fmt.Printf("Set SetNumericKeySize(%d)\n", siz)
178 | 			c.SetNumericKeySize(siz)
179 | 		}
180 | 		c.SetGrow(!*dg)
181 | 		c.StartLevel = *startLevel
182 | 		c.LowestLevel = *lowLevel
183 | 		stop := time.Now()
184 | 		if t == 0 {
185 | 			sz := hrff.Int64{int64(c.Size * c.BucketSize), "bytes"}
186 | 			fmt.Printf("trials: bseed=%#x, seede=%#x, cucko hash table size=%H, trials=%d\n", uint64(bseed), *seede, sz, trials)
187 | 		}
188 | 		durations[0] = tdiff(start, stop)
189 | 		print(0, tables*buckets*slots)
190 | 
191 | 		// fill
192 | 		//fmt.Printf("trials: fill\n")
193 | 
194 | 		runtime.ReadMemStats(&msb)
195 | 		//dump_mstats(&msa, true, false, false)
196 | 		//fmt.Printf("\n")
197 | 		start = time.Now()
198 | 		fs := d.Fill(tables, buckets, slots, ibase, *flf, verbose, *pl, *pr, ranb)
199 | 		used = fs.Used
200 | 		stop = time.Now()
201 | 		runtime.ReadMemStats(&msa)
202 | 		//dump_mstats(&msa, true, false, false)
203 | 		bpi := float64(c.Bumps) / float64(c.Inserts)
204 | 		ppi := float64(c.Probes) / float64(c.Inserts)
205 | 		ipi := float64(c.Iterations) / float64(c.Inserts)
206 | 
207 | 		rmax = fs.Thresh
208 | 		durations[1] = tdiff(start, stop)
209 | 		print(1, fs.Used)
210 | 		//c.Print() // xxx
211 | 
212 | 		tot += fs.Load
213 | 		//fmt.Printf("fs=%#v\n", fs)
214 | 		if fs.Failed {
215 | 			fails++
216 | 			//fmt.Printf("fails=%d\n", fails)
217 | 		}
218 | 		if d.Limited {
219 | 			td.Limited = true
220 | 		}
221 | 		if *fo {
222 | 			//c.Print()
223 | 			continue
224 | 		}
225 | 
226 | 		// verify
227 | 		//fmt.Printf("trials: verify base=%d, n=%d\n", fs.Base, c.Elements)
228 | 		start = time.Now()
229 | 		d.Verify(fs.Base, c.Elements, *pr)
230 | 		stop = time.Now()
231 | 		durations[2] = tdiff(start, stop)
232 | 		print(2, fs.Used)
233 | 		savElements := c.Elements
234 | 
235 | 		// delete
236 | 		//fmt.Printf("trials: delete\n")
237 | 		start = time.Now()
238 | 		ok := d.Delete(fs.Base, c.Elements, verbose, *pr)
239 | 		if !ok || c.Elements != 0 {
240 | 			s := fmt.Sprintf("Delete failed ok=%v, c.Elements=%d\n", ok, c.Elements)
241 | 			fmt.Printf(s)
242 | 			//panic(s)
243 | 		}
244 | 		stop = time.Now()
245 | 		durations[3] = tdiff(start, stop)
246 | 		print(3, fs.Used)
247 | 
248 | 		c.Elements = savElements
249 | 		cs.CountersAdd(&c.Counters)
250 | 		//statAdd(cs, &c.Counters)
251 | 		if d.Ll < td.Ll {
252 | 			td.Ll = d.Ll
253 | 			//fmt.Printf("setting td.Ll=%d\n", td.Ll)
254 | 		}
255 | 		//fmt.Printf("change d.Mr=%d, td.Mr=%d\n", d.Mr, td.Mr)
256 | 		if d.Mr > td.Mr {
257 | 			td.Mr = d.Mr
258 | 			//fmt.Printf("setting td.Mr=%d\n", td.Mr)
259 | 		}
260 | 
261 | 		// print information about operational rates
262 | 		if false {
263 | 			for k, v := range labels {
264 | 				f2 := hrff.Float64{float64(fs.Used) * (float64(time.Second) / float64(durations[k])), "ops/sec"}
265 | 				fmt.Printf("    %s: %v %h\n", v, durations[k], f2)
266 | 			}
267 | 			fmt.Printf("\n")
268 | 		}
269 | 		if verbose {
270 | 			fmt.Printf("trials: cf=%#v\n", c.Config)
271 | 			fmt.Printf("trials: cs=%#v\n", c.Counters)
272 | 			fmt.Printf("trials: fs=%#v\n", fs)
273 | 		}
274 | 		if *pt || printStatusOneShot || (*pf && fs.Failed) {
275 | 			fmt.Printf("trials: trial=%d, TraceCnt=%d, fails=%d, L=%v, F=%v, Remaining=%d, Aborts=%d, LowestLevel=%d, MaxProbes=%d, MaxIterations=%d, bpi=%0.2f, ppi=%0.2f, ipi=%0.4f, lf=%0.2f (%d/%d)\n",
276 | 				t, c.TraceCnt, fails, fs.Limited, fs.Failed, fs.Remaining, c.Aborts, fs.LowestLevel, c.MaxProbes, c.MaxIterations, bpi, ppi, ipi, float64(c.Elements)/float64(c.Size), c.Elements, c.Size)
277 | 			fmt.Printf("trials: fs=%#v\n", fs)
278 | 			printStatusOneShot = false
279 | 		}
280 | 
281 | 		bseed++
282 | 		if verbose {
283 | 			fmt.Printf("\n")
284 | 		}
285 | 	}
286 | 	avg = tot / float64(trials)
287 | 	/*
288 | 	   <<<<<<< HEAD
289 | 	   	fmt.Printf("trials: tables=%d, buckets=%d, slots=%d, trials=%d, avg=%0.2f max=%d, fails=%d\n", tables, buckets, slots, trials,
290 | 	   		avg, rmax, fails)
291 | 	   	fmt.Printf("trials: cs=%#v\n", cs)
292 | 	   =======
293 | 	   	d = td
294 | 	   	//fmt.Printf("trials: tables=%d, buckets=%d, slots=%d, trials=%d, fails=%d, avg=%0.2f\n", tables, buckets, slots, trials, fails, avg)
295 | 	*/
296 | 	return // avg, max, fails
297 | }
298 | 
299 | func f() {
300 | 	printStatusOneShot = true
301 | 	//*pt = !*pt
302 | }
303 | 
304 | func runTrials() {
305 | 	//tables := []int{2, 3, 4, 5, 6, 7, 8}
306 | 	//slots := []int{1, 2, 3, 4, 5, 6, 7, 8}
307 | 
308 | 	//st := 0
309 | 	//ss := 0
310 | 	siginfo.SetHandler(f)
311 | 	fails := 0
312 | 	//verbose := false
313 | 	if *ntrials == 1 {
314 | 		*verbose = true
315 | 	}
316 | 	/*
317 | 	       if *auto {
318 | 	       	maxAvg := float64(0)
319 | 	   		for _, b := range primes.Primes {
320 | 	       		for t := 1; t <= *ntables; t++ {
321 | 	       			for s := 1; s <= *nslots; s++ {
322 | 	   					//fmt.Printf("tables=%d, buckets=%d, slots=%d, trials=%d\n", t, b, s, *ntrials)
323 | 	       				_, avg, _, f := trials(t, b, s, *ntrials, *lf, 0, *verbose, *ranb)
324 | 	       				fails += f
325 | 	    					//fmt.Printf("tables=%d, buckets=%d, slots=%d, trials=%d\n", t, b, s, *ntrials)
326 | 	       				if avg > maxAvg {
327 | 	       					maxAvg = avg
328 | 	       					st = t
329 | 	       					ss = s
330 | 	       				}
331 | 	       			}
332 | 	       		}
333 | 	   	    	fmt.Printf("tables=%d, buckets=%d, slots=%d, trials=%d, fails=%d, avg=%0.4f\n", st, b, ss, *ntrials, fails, maxAvg)
334 | 	       		maxAvg = 0.0
335 | 	       	}
336 | 	       } else {
337 | 	*/
338 | 	nb := *nbuckets
339 | 	if nb < 0 {
340 | 		nb = primes.NextPrime(-nb)
341 | 	}
342 | 	tot := *ntables * nb * *nslots
343 | 	d, c, avg, used, max, fails := trials(*ntables, nb, *nslots, *ntrials, 0, *lf, *ibase, *verbose, *ranr, *ranb)
344 | 
345 | 	//fmt.Printf("dbg: avg=%0.4f, max=%d, fails=%d\n", avg, max, fails)
346 | 	//fmt.Printf("dbg: d=%#v\n", d)
347 | 	//fmt.Printf("dbg: c=%#v\n", c)
348 | 
349 | 	bpi := float64(c.Bumps) / float64(c.Inserts)
350 | 	ppi := float64(c.Probes) / float64(c.Inserts)
351 | 	ipi := float64(c.Iterations) / float64(c.Inserts)
352 | 
353 | 	c2 := (d.I).(*cuckoo.Cuckoo)
354 | 
355 | 	fmt.Printf("trials: tables=%d, buckets=%d, slots=%d, size=%d, used=%d, max=%d, lf=%0.2f, trials=%h, fails=%d, limited=%v, avg=%0.4f\n",
356 | 		c2.Ntables, nb, *nslots, tot, used, max, *lf, _ntrials, fails, d.Limited, avg)
357 | 	fmt.Printf("trials: MinTraceCnt=%d, MaxRemaining=%d, LowestLevel=%d, Aborts=%d, bpi=%0.2f, ppi=%0.2f, ipi=%0.4f\n",
358 | 		c.MinTraceCnt, d.Mr, d.Ll, c.Aborts, bpi, ppi, ipi)
359 | 	//fmt.Printf("trials: MaxRemaining=%d\n", dstest.Mr)
360 | 	//fmt.Printf("trials: LowestLevel=%d\n", dstest.Ll)
361 | 	if *ps {
362 | 		fmt.Printf("trials: c=%#v\n", c)
363 | 	}
364 | 	//	}
365 | }
366 | 
367 | func main() {
368 | 	flag.Var(&_ntrials, "nt", "number of trials.")
369 | 	flag.Parse()
370 | 	if *mp != "" {
371 | 		f, err := os.Create(*mp)
372 | 		if err != nil {
373 | 			log.Fatal(err)
374 | 		}
375 | 		runTrials()
376 | 		pprof.WriteHeapProfile(f)
377 | 		f.Close()
378 | 		return
379 | 	}
380 | 
381 | 	if *cp != "" {
382 | 		f, err := os.Create(*cp)
383 | 		if err != nil {
384 | 			log.Fatal(err)
385 | 		}
386 | 		runTrials()
387 | 		pprof.StartCPUProfile(f)
388 | 		defer pprof.StopCPUProfile()
389 | 		return
390 | 	}
391 | 	runTrials()
392 | }
393 | 


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
 1 | module leb.io/cuckoo
 2 | 
 3 | go 1.16
 4 | 
 5 | require (
 6 | 	github.com/alecthomas/binary v0.0.0-20190922233330-fb1b1d9c299c
 7 | 	github.com/dataence/bloom v0.0.0-20151026233158-e24b032dccb1
 8 | 	github.com/dataence/cityhash v0.0.0-20131128155616-cdd6a94144ab
 9 | 	github.com/spaolacci/murmur3 v1.1.0
10 | 	github.com/stretchr/testify v1.7.0 // indirect
11 | 	github.com/willf/bitset v1.1.11
12 | 	github.com/zhenjl/bloom v0.0.0-20151026233158-e24b032dccb1 // indirect
13 | 	github.com/zhenjl/cityhash v0.0.0-20131128155616-cdd6a94144ab // indirect
14 | 	leb.io/aeshash v0.0.0-20190627052759-9e6b40329b3b
15 | 	leb.io/hashland v0.0.0-20171003003232-07375b562dea // indirect
16 | 	leb.io/hrff v0.1.0
17 | )
18 | 


--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
 1 | github.com/alecthomas/binary v0.0.0-20190922233330-fb1b1d9c299c h1:SnUAzBu0FguUHChHbuy2HInhc2YBBTmbDcZOOByAVt8=
 2 | github.com/alecthomas/binary v0.0.0-20190922233330-fb1b1d9c299c/go.mod h1:v4e05/vzE8ubOim1No9Xx5eIQ/WRq6AtcnQIy/Z/JPs=
 3 | github.com/dataence/bloom v0.0.0-20151026233158-e24b032dccb1 h1:ZZZOG1+mzcgW/z/HOHj0RDWTxHQAn+9L5UejB/ZK+XI=
 4 | github.com/dataence/bloom v0.0.0-20151026233158-e24b032dccb1/go.mod h1:hQ0yBWNA+0SlYMlAkHsylNyF2c85ZPgnJ6pvMOkzpRY=
 5 | github.com/dataence/cityhash v0.0.0-20131128155616-cdd6a94144ab h1:MjAkdXxk4H78lm7W17Qd85nEPHlEdx23Xk0KaESrmnc=
 6 | github.com/dataence/cityhash v0.0.0-20131128155616-cdd6a94144ab/go.mod h1:8k8m9EQDOnG3a2Z27FvW8LPScFO9P2JglEeq2kjKP9Y=
 7 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 9 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
10 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
11 | github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
12 | github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
13 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
14 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
15 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
16 | github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=
17 | github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
18 | github.com/zhenjl/bloom v0.0.0-20151026233158-e24b032dccb1 h1:s7Fcgy4ILv5n3dz+dQvpRntBtPPw7Ffbyh3Q4JY+kk4=
19 | github.com/zhenjl/bloom v0.0.0-20151026233158-e24b032dccb1/go.mod h1:m5109HBotwhAcpovJ+nory3OI1OrcUmaluV7vxjX+ig=
20 | github.com/zhenjl/cityhash v0.0.0-20131128155616-cdd6a94144ab h1:BWHvAOZz0pBILkGl/ebPQKZDrqbaWj/iN9RE8AvaTvg=
21 | github.com/zhenjl/cityhash v0.0.0-20131128155616-cdd6a94144ab/go.mod h1:P6L88wrqK99Njntah9SB7AyzFpUXsXYq06LkjixxQmY=
22 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
23 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
24 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
25 | leb.io/aeshash v0.0.0-20190627052759-9e6b40329b3b h1:MG17Tc0pA3XmFTsPwklMMEfcos3pTFnVYM4A0YfVSbU=
26 | leb.io/aeshash v0.0.0-20190627052759-9e6b40329b3b/go.mod h1:BKjvdzZnV5WaMoO/kz/Fygl60dA79GTFQTSuKJmSdNs=
27 | leb.io/hashland v0.0.0-20171003003232-07375b562dea h1:s9IkzZTqYqw77voO6taUZHc0C1B096h4T/kQtujGApE=
28 | leb.io/hashland v0.0.0-20171003003232-07375b562dea/go.mod h1:OSwp5BNKniWS6joctn4Grabrxd2EERBB7pNzK9Ni3BQ=
29 | leb.io/hrff v0.1.0 h1:HAhaPEnNEJp7pCbH/qgCT0VMNMQC7I10unCRuu08cKc=
30 | leb.io/hrff v0.1.0/go.mod h1:EZ02Yctbwtxev2S/3jw6aKNrw2e/gkaaRk4fgnYFTec=
31 | 


--------------------------------------------------------------------------------
/hash-builtin.go:
--------------------------------------------------------------------------------
 1 | // Copyright © 2014, 2015, 2016 Lawrence E. Bakst. All rights reserved.
 2 | // +build !noaes
 3 | 
 4 | package cuckoo
 5 | 
 6 | import (
 7 | 	"errors"
 8 | 	"hash"
 9 | 
10 | 	"leb.io/aeshash"
11 | 	"leb.io/cuckoo/internal/jenkins264"
12 | 	"leb.io/cuckoo/internal/jenkins3"
13 | )
14 | 
15 | // Set hash function used
16 | func (c *Cuckoo) setHash(hashName string) (int, error) {
17 | 	switch hashName {
18 | 	case "":
19 | 		fallthrough
20 | 	case "aes":
21 | 		c.hf64 = aeshash.Hash64
22 | 		c.hf32 = aeshash.Hash32
23 | 		c.hfb = aeshash.Hash
24 | 		c.hf = aeshash.NewAES(0)
25 | 		return aes, nil
26 | 	case "j364":
27 | 		c.hf64 = nil
28 | 		c.hf32 = nil
29 | 		c.hfb = jenkins3.HashBytes
30 | 		c.hf = jenkins3.New(uint32(0))
31 | 		return j364, nil
32 | 	case "j264":
33 | 		c.hf64 = nil
34 | 		c.hf32 = nil
35 | 		c.hfb = jenkins264.Hash
36 | 		return j264, nil
37 | 	default:
38 | 		// fallthrough generated bad error messaage Go bug
39 | 	}
40 | 	return 0, errors.New("cuckoo: invalid hash name")
41 | }
42 | 
43 | // Get a hash function with a specific seed
44 | func (c *Cuckoo) getHash(hashName string, seed uint64) hash.Hash64 {
45 | 	switch hashName {
46 | 	case "":
47 | 		fallthrough
48 | 	case "aes":
49 | 		return aeshash.NewAES(seed)
50 | 	case "j264":
51 | 		return jenkins264.New(seed)
52 | 	case "j364":
53 | 		return jenkins3.New(uint32(seed))
54 | 	}
55 | 	panic("getHash")
56 | }
57 | 


--------------------------------------------------------------------------------
/internal/dstest/dstest.go:
--------------------------------------------------------------------------------
  1 | // Copyright © 2014-2017 Lawrence E. Bakst. All rights reserved.
  2 | 
  3 | // small step towards creating a package that can test data structures
  4 | package dstest
  5 | 
  6 | import "fmt"
  7 | import "math/rand"
  8 | import c "leb.io/cuckoo"
  9 | 
 10 | //var Mr int
 11 | //var Ll int
 12 | 
 13 | // basic data structures methods and a method to get stats, what do we do about level?
 14 | type DSTester interface {
 15 | 	Insert(key c.Key, value c.Value) (ok bool)
 16 | 	InsertL(key c.Key, value c.Value) (ok bool, rlevel int)
 17 | 	Lookup(key c.Key) (v c.Value, ok bool)
 18 | 	Delete(key c.Key) (c.Value, bool)
 19 | 	GetCounter(stat string) int
 20 | 	GetTableCounter(t int, stat string) int
 21 | }
 22 | 
 23 | //var r = rand.Float64
 24 | 
 25 | // return information about what happened during a fill
 26 | type FillStats struct {
 27 | 	Load        float64
 28 | 	Base        int
 29 | 	Total       int
 30 | 	Thresh      int
 31 | 	Used        int
 32 | 	Remaining   int
 33 | 	LowestLevel int
 34 | 	Fails       int
 35 | 	Failed      bool
 36 | 	Limited     bool
 37 | }
 38 | 
 39 | type DSTest struct {
 40 | 	Seed      int64      // seed used to control fill stream
 41 | 	Mr        int        // Max remaining
 42 | 	Ll        int        // Lowest level
 43 | 	FillStats            // stats
 44 | 	I         DSTester   // functions
 45 | 	R         *rand.Rand // random number generator with no lock
 46 | }
 47 | 
 48 | func NewTester(i DSTester, ll int, seed int64) *DSTest {
 49 | 	d := DSTest{Seed: seed, Ll: ll, I: i}
 50 | 	d.I = i
 51 | 	//d.Remaining = 1<<32
 52 | 	d.R = rand.New(rand.NewSource(int64(seed))) // no lock
 53 | 	return &d
 54 | }
 55 | 
 56 | func (d *DSTest) rbetween(a int, b int) int {
 57 | 	//rf := r()
 58 | 	rf := d.R.Float64()
 59 | 	diff := float64(b - a + 1)
 60 | 	r2 := rf * diff
 61 | 	r3 := r2 + float64(a)
 62 | 	ret := int(r3)
 63 | 	//fmt.Printf("rbetween: a=%d, b=%d, rf=%f, diff=%f, r2=%f, r3=%f, ret=%d\n", a, b, rf, diff, r2, r3, ret)
 64 | 	return ret
 65 | }
 66 | 
 67 | func (d *DSTest) _fill(tables, buckets, slots, ibase int, flf float64, verbose, printLevels, progress, r bool) *FillStats {
 68 | 	var fs FillStats
 69 | 	base := 0
 70 | 	if r {
 71 | 		base = d.rbetween(1, 1<<29)
 72 | 		//fmt.Printf("random generated base=%d\n", base)
 73 | 	} else {
 74 | 		base = ibase
 75 | 	}
 76 | 	fs.Base = base
 77 | 	fs.Total = tables * buckets * slots
 78 | 	amt := float64(tables * buckets * slots)
 79 | 	amt *= flf
 80 | 	max := int(amt)
 81 | 	fs.Used = max
 82 | 	fs.Thresh = max
 83 | 	amax := base + max
 84 | 	//fmt.Printf("_fill: base=%d, amax=%d, max=%d\n", base, amax, max)
 85 | 	fs.Load = float64(1.0)
 86 | 	cnt := 1
 87 | 	svi := amax
 88 | 	lowestLevel := 1 << 31
 89 | 	onep := (amax - base) / 100
 90 | 	thresh := 0
 91 | 
 92 | 	if verbose {
 93 | 		fmt.Printf("    fill: base=%d, amax=%d, n=%d\n", base, amax, amax-base)
 94 | 	}
 95 | 	if progress {
 96 | 		fmt.Printf("F: ")
 97 | 	}
 98 | 
 99 | 	for i := base; i < amax; i++ {
100 | 		//fmt.Printf("%d\n", i)
101 | 		ok, l := d.I.InsertL(c.Key(i), c.Value(uint64(cnt)))
102 | 		if l < lowestLevel && l != 0 {
103 | 			lowestLevel = l
104 | 		}
105 | 		if !ok {
106 | 			// Two reasons we fail, load constraint (fs.Limited) or other (fs.Failed)
107 | 			if l == 0 {
108 | 				fs.Limited = true
109 | 			} else {
110 | 				fs.Failed = true
111 | 			}
112 | 			if verbose {
113 | 				if printLevels {
114 | 					fmt.Printf("%d/%d\n", l, lowestLevel)
115 | 				}
116 | 				fmt.Printf("    fill: %d/%d, remain=%d, MaxPathLen=%d, bumps=%d, %d/%d=%0.4f, level=%d, bpi=%0.2f\n",
117 | 					i, amax, amax-i, d.I.GetCounter("MaxPathLen"), d.I.GetCounter("bumps"), d.I.GetCounter("elements"), d.I.GetCounter("size"),
118 | 					fs.Load, l, float64(d.I.GetCounter("bumps"))/float64(d.I.GetCounter("inserts")))
119 | 			}
120 | 			fs.Used = i - base
121 | 			fs.LowestLevel = lowestLevel
122 | 			svi = i
123 | 			break
124 | 		} else {
125 | 			//fmt.Printf("%d\n", i)
126 | 			if printLevels {
127 | 				fmt.Printf("%d/%d ", l, lowestLevel)
128 | 			}
129 | 		}
130 | 		if progress && cnt >= thresh {
131 | 			pcnt := cnt / onep
132 | 			if pcnt%10 == 0 {
133 | 				fmt.Printf("%d", pcnt/10)
134 | 			} else {
135 | 				fmt.Printf("%%")
136 | 			}
137 | 			fmt.Printf("%d: MaxPathLen=%d\n", cnt/onep, d.I.GetCounter("MaxPathLen"))
138 | 			thresh += onep
139 | 		}
140 | 		cnt++
141 | 	}
142 | 	if progress {
143 | 		fmt.Printf("\n")
144 | 	}
145 | 	fs.LowestLevel = lowestLevel
146 | 	fs.Load = float64(d.I.GetCounter("elements")) / float64(d.I.GetCounter("size"))
147 | 	fs.Remaining = amax - svi
148 | 	if verbose {
149 | 		fmt.Printf("    fill: fail=%v @ %d/%d, remain=%d, MaxPathLen=%d, bumps=%d, %d/%d=%0.4f, bpi=%0.2f\n",
150 | 			fs.Failed, svi, amax, amax-svi,
151 | 			d.I.GetCounter("MaxPathLen"), d.I.GetCounter("bumps"), d.I.GetCounter("inserts"), d.I.GetCounter("elements"),
152 | 			fs.Load, float64(d.I.GetCounter("bumps"))/float64(d.I.GetCounter("inserts")))
153 | 	}
154 | 	if fs.Remaining > d.Mr {
155 | 		d.Mr = fs.Remaining
156 | 	}
157 | 	if fs.LowestLevel < d.Ll {
158 | 		d.Ll = fs.LowestLevel
159 | 	}
160 | 	if printLevels && !verbose {
161 | 		fmt.Printf("\n")
162 | 	}
163 | 	//fmt.Printf("\n")
164 | 	/*
165 | 		if level == -8000 {
166 | 			panic("_fill")
167 | 		}
168 | 	*/
169 | 	//avg := tot / float64(trials)
170 | 	//fmt.Printf("tables=%d, buckets=%d, slots=%d, trials=%d, avg=%0.2f\n", tables, buckets, slots, trials, avg)
171 | 	return &fs
172 | }
173 | 
174 | func (d *DSTest) Fill(tables, buckets, slots, ibase int, flf float64, verbose, pl, progress bool, r bool) *FillStats {
175 | 	fs := d._fill(tables, buckets, slots, ibase, flf, verbose, pl, progress, r)
176 | 	if verbose {
177 | 		for i := 0; i < tables; i++ {
178 | 			fmt.Printf("    fill: table[%d]: %d/%d=%0.4f\n", i,
179 | 				d.I.GetTableCounter(i, "elements"), d.I.GetTableCounter(i, "size"), float64(d.I.GetTableCounter(i, "elements"))/float64(d.I.GetTableCounter(i, "size")))
180 | 		}
181 | 	}
182 | 	return fs
183 | }
184 | 
185 | // test lookup by looking for a sequence of keys and making sure the values match the keys
186 | func (d *DSTest) Verify(base, n int, progress bool) bool {
187 | 	//fmt.Printf("Verify: base=%d, n=%d \n", base, n)
188 | 	if progress {
189 | 		fmt.Printf("V: ")
190 | 	}
191 | 	onep := n / 100
192 | 	thresh := onep
193 | 	cnt := 0
194 | 	if false {
195 | 		fmt.Printf("    verify: base=%d, base+n=%d, n=%d\n", base, base+n, n)
196 | 	}
197 | 	//fmt.Printf("    verify: base=%d, base+n=%d, n=%d\n", base, base+n, n)
198 | 	for i := base; i < base+n; i++ {
199 | 		cnt++
200 | 		v, ok := d.I.Lookup(c.Key(i))
201 | 		if !ok {
202 | 			fmt.Printf("Verify: lookup FAILED i=%d, cnt=%d\n", i, cnt)
203 | 			return false
204 | 		}
205 | 		//fmt.Printf("Verify: check i=%d, cnt=%d == v=%d\n", i, cnt, uint32(v))
206 | 		if uint32(cnt) != uint32(v) {
207 | 			fmt.Printf("Verify: FAIL i=%d, cnt=%d != v=%d\n", i, cnt, uint64(v))
208 | 			return false
209 | 		}
210 | 		if progress && cnt > thresh {
211 | 			fmt.Printf("%%")
212 | 			thresh += onep
213 | 		}
214 | 	}
215 | 	if progress {
216 | 		fmt.Printf("\n")
217 | 	}
218 | 	//fmt.Printf("Verify: OK\n")
219 | 	return true
220 | }
221 | 
222 | func (d *DSTest) Delete(base, n int, verbose, progress bool) bool {
223 | 	//fmt.Printf("delete from=%d, n=%d\n", base, n)
224 | 
225 | 	if progress {
226 | 		fmt.Printf("D: ")
227 | 	}
228 | 	onep := n / 100
229 | 	thresh := onep
230 | 	cnt := 0
231 | 	for i := base; i < base+n; i++ {
232 | 		if _, b := d.I.Delete(c.Key(i)); !b {
233 | 			return false
234 | 		}
235 | 		cnt++
236 | 		if progress && cnt > thresh {
237 | 			fmt.Printf("%%")
238 | 			thresh += onep
239 | 		}
240 | 	}
241 | 	if progress {
242 | 		fmt.Printf("\n")
243 | 	}
244 | 	return true
245 | }
246 | 


--------------------------------------------------------------------------------
/internal/jenkins264/jenkins264.go:
--------------------------------------------------------------------------------
  1 | // Copyright © 2014 Lawrence E. Bakst. All rights reserved.
  2 | // See http://burtleburtle.net/bob/c/lookup3.c and http://burtleburtle.net/bob/hash/evahash.html
  3 | 
  4 | package jenkins264
  5 | 
  6 | import (
  7 | 	_ "fmt"
  8 | 	"hash"
  9 | 	"unsafe"
 10 | )
 11 | 
 12 | // original mix function
 13 | func mix64(a, b, c uint64) (uint64, uint64, uint64) {
 14 | 	a = a - b
 15 | 	a = a - c
 16 | 	a = a ^ (c >> 43)
 17 | 	b = b - c
 18 | 	b = b - a
 19 | 	b = b ^ (a << 9)
 20 | 	c = c - a
 21 | 	c = c - b
 22 | 	c = c ^ (b >> 8)
 23 | 	a = a - b
 24 | 	a = a - c
 25 | 	a = a ^ (c >> 38)
 26 | 	b = b - c
 27 | 	b = b - a
 28 | 	b = b ^ (a << 23)
 29 | 	c = c - a
 30 | 	c = c - b
 31 | 	c = c ^ (b >> 5)
 32 | 	a = a - b
 33 | 	a = a - c
 34 | 	a = a ^ (c >> 35)
 35 | 	b = b - c
 36 | 	b = b - a
 37 | 	b = b ^ (a << 49)
 38 | 	c = c - a
 39 | 	c = c - b
 40 | 	c = c ^ (b >> 11)
 41 | 	a = a - b
 42 | 	a = a - c
 43 | 	a = a ^ (c >> 12)
 44 | 	b = b - c
 45 | 	b = b - a
 46 | 	b = b ^ (a << 18)
 47 | 	c = c - a
 48 | 	c = c - b
 49 | 	c = c ^ (b >> 22)
 50 | 	return a, b, c
 51 | }
 52 | 
 53 | // restated mix function better for gofmt
 54 | func mix64alt(a, b, c uint64) (uint64, uint64, uint64) {
 55 | 	a -= b - c ^ (c >> 43)
 56 | 	b -= c - a ^ (a << 9)
 57 | 	c -= a - b ^ (b >> 8)
 58 | 	a -= b - c ^ (c >> 38)
 59 | 	b -= c - a ^ (a << 23)
 60 | 	c -= a - b ^ (b >> 5)
 61 | 	a -= b - c ^ (c >> 35)
 62 | 	b -= c - a ^ (a << 49)
 63 | 	c -= a - b ^ (b >> 11)
 64 | 	a -= b - c ^ (c >> 12)
 65 | 	b -= c - a ^ (a << 18)
 66 | 	c -= a - b ^ (b >> 22)
 67 | 	return a, b, c
 68 | }
 69 | 
 70 | // the following functions can be inlined
 71 | func mix64a(a, b, c uint64) (uint64, uint64, uint64) {
 72 | 	a -= b - c ^ (c >> 43)
 73 | 	b -= c - a ^ (a << 9)
 74 | 	return a, b, c
 75 | }
 76 | 
 77 | func mix64b(a, b, c uint64) (uint64, uint64, uint64) {
 78 | 	c -= a - b ^ (b >> 8)
 79 | 	a -= b - c ^ (c >> 38)
 80 | 	return a, b, c
 81 | }
 82 | 
 83 | func mix64c(a, b, c uint64) (uint64, uint64, uint64) {
 84 | 	b -= c - a ^ (a << 23)
 85 | 	c -= a - b ^ (b >> 5)
 86 | 	return a, b, c
 87 | }
 88 | 
 89 | func mix64d(a, b, c uint64) (uint64, uint64, uint64) {
 90 | 	a -= b - c ^ (c >> 35)
 91 | 	b -= c - a ^ (a << 49)
 92 | 	return a, b, c
 93 | }
 94 | 
 95 | func mix64e(a, b, c uint64) (uint64, uint64, uint64) {
 96 | 	c -= a - b ^ (b >> 11)
 97 | 	a -= b - c ^ (c >> 12)
 98 | 	return a, b, c
 99 | }
100 | 
101 | func mix64f(a, b, c uint64) (uint64, uint64, uint64) {
102 | 	b -= c - a ^ (a << 18)
103 | 	c -= a - b ^ (b >> 22)
104 | 	return a, b, c
105 | }
106 | 
107 | // This makes a new slice of uint64 that points to the same slice passed in as []byte.
108 | // We should check alignment for architectures that don't handle unaligned reads and
109 | // fallback to a copy.
110 | // Unclear is we guarentee the same hash for different endianess.
111 | func sliceUI64(in []byte) []uint64 {
112 | 	return (*(*[]uint64)(unsafe.Pointer(&in)))[:len(in)/8]
113 | }
114 | 
115 | // Jenkin's second generation 64 bit hash.
116 | // Benchmarked with 24 byte key, inlining, store of hash in memory (cache miss every 4 hashes) and fast=true at:
117 | // benchmark64: 26 Mhashes/sec
118 | // benchmark64: 623 MB/sec
119 | func Hash(k []byte, seed uint64) uint64 {
120 | 	var fast = true // fast is really much faster
121 | 	//fmt.Printf("k=%v\n", k)
122 | 	//fmt.Printf("length=%d, len(k)=%d\n", length, len(k))
123 | 
124 | 	//The 64-bit golden ratio is 0x9e3779b97f4a7c13LL
125 | 	length := uint64(len(k))
126 | 	a := uint64(0x9e3779b97f4a7c13)
127 | 	b := a
128 | 	c := seed
129 | 	if fast {
130 | 		k64 := sliceUI64(k)
131 | 		cnt := 0
132 | 		for i := length; i >= 24; i -= 24 {
133 | 			a += k64[0+cnt]
134 | 			b += k64[1+cnt]
135 | 			c += k64[2+cnt]
136 | 			// inlining is slightly faster
137 | 			a, b, c = mix64a(a, b, c)
138 | 			a, b, c = mix64b(a, b, c)
139 | 			a, b, c = mix64c(a, b, c)
140 | 			a, b, c = mix64d(a, b, c)
141 | 			a, b, c = mix64e(a, b, c)
142 | 			a, b, c = mix64f(a, b, c)
143 | 			k = k[24:]
144 | 			cnt += 3
145 | 			length -= 24
146 | 		}
147 | 	} else {
148 | 		for i := length; i >= 24; i -= 24 {
149 | 			a += uint64(k[0]) | uint64(k[1])<<8 | uint64(k[2])<<16 | uint64(k[3])<<24 | uint64(k[4])<<32 | uint64(k[5])<<40 | uint64(k[6])<<48 | uint64(k[7])<<56
150 | 			b += uint64(k[8]) | uint64(k[9])<<8 | uint64(k[10])<<16 | uint64(k[11])<<24 | uint64(k[12])<<32 | uint64(k[13])<<40 | uint64(k[14])<<48 | uint64(k[15])<<56
151 | 			c += uint64(k[16]) | uint64(k[17])<<8 | uint64(k[18])<<16 | uint64(k[19])<<24 | uint64(k[20])<<32 | uint64(k[21])<<40 | uint64(k[22])<<48 | uint64(k[23])<<56
152 | 			a, b, c = mix64alt(a, b, c)
153 | 			k = k[24:]
154 | 			length -= 24
155 | 		}
156 | 	}
157 | 	c += length
158 | 	if len(k) > 23 {
159 | 		panic("Hash264")
160 | 	}
161 | 	switch length {
162 | 	case 23:
163 | 		c += uint64(k[22]) << 56
164 | 		fallthrough
165 | 	case 22:
166 | 		c += uint64(k[21]) << 48
167 | 		fallthrough
168 | 	case 21:
169 | 		c += uint64(k[20]) << 40
170 | 		fallthrough
171 | 	case 20:
172 | 		c += uint64(k[19]) << 32
173 | 		fallthrough
174 | 	case 19:
175 | 		c += uint64(k[18]) << 24
176 | 		fallthrough
177 | 	case 18:
178 | 		c += uint64(k[17]) << 16
179 | 		fallthrough
180 | 	case 17:
181 | 		c += uint64(k[16]) << 8
182 | 		fallthrough
183 | 	case 16:
184 | 		b += uint64(k[15]) << 56 // the first byte of c is reserved for the length
185 | 		fallthrough
186 | 	case 15:
187 | 		b += uint64(k[14]) << 48
188 | 		fallthrough
189 | 	case 14:
190 | 		b += uint64(k[13]) << 40
191 | 		fallthrough
192 | 	case 13:
193 | 		b += uint64(k[12]) << 32
194 | 		fallthrough
195 | 	case 12:
196 | 		b += uint64(k[11]) << 24
197 | 		fallthrough
198 | 	case 11:
199 | 		b += uint64(k[10]) << 16
200 | 		fallthrough
201 | 	case 10:
202 | 		b += uint64(k[9]) << 8
203 | 		fallthrough
204 | 	case 9:
205 | 		b += uint64(k[8])
206 | 		fallthrough
207 | 	case 8:
208 | 		a += uint64(k[7]) << 56
209 | 		fallthrough
210 | 	case 7:
211 | 		a += uint64(k[6]) << 48
212 | 		fallthrough
213 | 	case 6:
214 | 		a += uint64(k[5]) << 40
215 | 		fallthrough
216 | 	case 5:
217 | 		a += uint64(k[4]) << 32
218 | 		fallthrough
219 | 	case 4:
220 | 		a += uint64(k[3]) << 24
221 | 		fallthrough
222 | 	case 3:
223 | 		a += uint64(k[2]) << 16
224 | 		fallthrough
225 | 	case 2:
226 | 		a += uint64(k[1]) << 8
227 | 		fallthrough
228 | 	case 1:
229 | 		a += uint64(k[0])
230 | 	case 0:
231 | 		break
232 | 	default:
233 | 		panic("HashWords64")
234 | 	}
235 | 	a, b, c = mix64alt(a, b, c)
236 | 	return c
237 | }
238 | 
239 | type Digest struct {
240 | 	hash uint64
241 | 	seed uint64
242 | 	clen int
243 | 	tail []byte
244 | }
245 | 
246 | // The size of an jenkins3 32 bit hash in bytes.
247 | const Size = 4
248 | 
249 | var (
250 | 	_ hash.Hash64 = new(Digest)
251 | )
252 | 
253 | // New returns a new hash.Hash64 interface that computes the a 32 bit murmur3 hash.
254 | func New(seed uint64) hash.Hash64 {
255 | 	d := new(Digest)
256 | 	d.seed = seed
257 | 	d.Reset()
258 | 	return d
259 | }
260 | 
261 | // Reset the hash state.
262 | func (d *Digest) Reset() {
263 | 	d.hash = uint64(d.seed)
264 | 	d.clen = 0
265 | 	d.tail = nil
266 | }
267 | 
268 | // Return the size of the resulting hash.
269 | func (d *Digest) Size() int { return Size }
270 | 
271 | // Return the blocksize of the hash which in this case is 1 byte.
272 | func (d *Digest) BlockSize() int { return 1 }
273 | 
274 | // Accept a byte stream p used for calculating the hash. For now this call is lazy and the actual hash calculations take place in Sum() and Sum32().
275 | func (d *Digest) Write(p []byte) (nn int, err error) {
276 | 	l := len(p)
277 | 	d.clen += l
278 | 	d.tail = append(d.tail, p...)
279 | 	return l, nil
280 | }
281 | 
282 | // Return the current hash as a byte slice.
283 | func (d *Digest) Sum(b []byte) []byte {
284 | 	h := Hash(d.tail, d.seed)
285 | 	d.hash = h
286 | 	return append(b, byte(h>>56), byte(h>>48), byte(h>>42), byte(h>>32), byte(h>>24), byte(h>>16), byte(h>>8), byte(h))
287 | }
288 | 
289 | // Return the current hash as a 32 bit unsigned type.
290 | func (d *Digest) Sum64() uint64 {
291 | 	d.hash = Hash(d.tail, d.seed)
292 | 	return d.hash
293 | }
294 | 


--------------------------------------------------------------------------------
/internal/jenkins3/jenkins3.go:
--------------------------------------------------------------------------------
  1 | // Copyright © 2014 Lawrence E. Bakst. All rights reserved.
  2 | 
  3 | // This package is a transliteration of Jenkins lookup3.c
  4 | // having lots of fun trying to get the inliner to work
  5 | 
  6 | package jenkins3
  7 | 
  8 | import (
  9 | 	"fmt"
 10 | 	"hash"
 11 | 	"unsafe"
 12 | )
 13 | 
 14 | type Digest struct {
 15 | 	hash uint64
 16 | 	seed uint32
 17 | 	pc   uint32
 18 | 	pb   uint32
 19 | 	clen int
 20 | 	tail []byte
 21 | }
 22 | 
 23 | // The size of an jenkins3 32 bit hash in bytes.
 24 | const Size = 4
 25 | 
 26 | // Make sure interfaces are correctly implemented. Stolen from another implementation.
 27 | // I did something similar in another package to verify the interface but didn't know you could elide the variable in a var.
 28 | // What a cute wart it is.
 29 | var (
 30 | 	//_ hash.Hash   = new(Digest)
 31 | 	//_ hash.Hash32 = new(Digest)
 32 | 	_ hash.Hash64 = new(Digest)
 33 | )
 34 | 
 35 | /*
 36 | uint32_t jenkins_one_at_a_time_hash(char *key, size_t len)
 37 | {
 38 |     uint32_t hash, i;
 39 |     for(hash = i = 0; i < len; ++i)
 40 |     {
 41 |         hash += key[i];
 42 |         hash += (hash << 10);
 43 |         hash ^= (hash >> 6);
 44 |     }
 45 |     hash += (hash << 3);
 46 |     hash ^= (hash >> 11);
 47 |     hash += (hash << 15);
 48 |     return hash;
 49 | }
 50 | */
 51 | 
 52 | func HashWords64(k []uint64, length int, seed uint64) uint64 {
 53 | 	var a, b, c uint64
 54 | 
 55 | 	//fmt.Printf("k=%v\n", k)
 56 | 	//fmt.Printf("length=%d, len(k)=%d\n", length, len(k))
 57 | 	if length > len(k) {
 58 | 		fmt.Printf("length=%d, len(k)=%d\n", length, len(k))
 59 | 		panic("HashWords")
 60 | 	}
 61 | 
 62 | 	//The 64-bit golden ratio is 0x9e3779b97f4a7c13LL
 63 | 	ul := uint64(len(k))
 64 | 	a = 0x9e3779b97f4a7c13 + ul<<2 + seed
 65 | 	b, c = a, a
 66 | 	cnt := 0
 67 | 	for i := 0; i < length; i++ {
 68 | 		switch cnt {
 69 | 		case 0:
 70 | 			a = k[i]
 71 | 		case 1:
 72 | 			b = k[i]
 73 | 		case 2:
 74 | 			c = k[i]
 75 | 			cnt = 0
 76 | 		}
 77 | 		a = a - b
 78 | 		a = a - c
 79 | 		a = a ^ (c >> 43)
 80 | 		b = b - c
 81 | 		b = b - a
 82 | 		b = b ^ (a << 9)
 83 | 		c = c - a
 84 | 		c = c - b
 85 | 		c = c ^ (b >> 8)
 86 | 		a = a - b
 87 | 		a = a - c
 88 | 		a = a ^ (c >> 38)
 89 | 		b = b - c
 90 | 		b = b - a
 91 | 		b = b ^ (a << 23)
 92 | 		c = c - a
 93 | 		c = c - b
 94 | 		c = c ^ (b >> 5)
 95 | 		a = a - b
 96 | 		a = a - c
 97 | 		a = a ^ (c >> 35)
 98 | 		b = b - c
 99 | 		b = b - a
100 | 		b = b ^ (a << 49)
101 | 		c = c - a
102 | 		c = c - b
103 | 		c = c ^ (b >> 11)
104 | 		a = a - b
105 | 		a = a - c
106 | 		a = a ^ (c >> 12)
107 | 		b = b - c
108 | 		b = b - a
109 | 		b = b ^ (a << 18)
110 | 		c = c - a
111 | 		c = c - b
112 | 		c = c ^ (b >> 22)
113 | 	}
114 | 	return c
115 | }
116 | 
117 | /*
118 | func omix(a, b, c uint32) (uint32, uint32, uint32) {
119 | 	a -= c;  a ^= rot(c, 4);  c += b;
120 | 	b -= a;  b ^= rot(a, 6);  a += c;
121 | 	c -= b;  c ^= rot(b, 8);  b += a;
122 | 	a -= c;  a ^= rot(c,16);  c += b;
123 | 	b -= a;  b ^= rot(a,19);  a += c;
124 | 	c -= b;  c ^= rot(b, 4);  b += a;
125 | 	return a, b, c
126 | }
127 | 
128 | func ofinal(a, b, c uint32) (uint32, uint32, uint32) {
129 | 	c ^= b; c -= rot(b,14);
130 | 	a ^= c; a -= rot(c,11);
131 | 	b ^= a; b -= rot(a,25);
132 | 	c ^= b; c -= rot(b,16);
133 | 	a ^= c; a -= rot(c,4);
134 | 	b ^= a; b -= rot(a,14);
135 | 	c ^= b; c -= rot(b,24);
136 | 	return a, b, c
137 | }
138 | 
139 | 
140 | func mix(a, b, c uint32) (uint32, uint32, uint32) {
141 | 	a -= c;  a ^= c << 4 | c >> (32 - 4);  c += b;
142 | 	b -= a;  b ^= a << 6 | a >> (32 - 6);  a += c;
143 | 	c -= b;  c ^= b << 8 | b >> (32 - 8);  b += a;
144 | 	a -= c;  a ^= c << 16 | c >> (32 - 16);  c += b;
145 | 	b -= a;  b ^= a << 19 | a >> (32 - 19);  a += c;
146 | 	c -= b;  c ^= b << 4 | b >> (32 - 4);  b += a;
147 | 	return a, b, c
148 | }
149 | 
150 | 
151 | func final(a, b, c uint32) (uint32, uint32, uint32) {
152 | 	c ^= b; c -= b << 14 | b >> (32 - 14);
153 | 	a ^= c; a -= c << 11 | c >> (32 - 11);
154 | 	b ^= a; b -= a << 25 | a >> (32 - 25);
155 | 	c ^= b; c -= b << 16 | b >> (32 - 16);
156 | 	a ^= c; a -= c << 4 | c >> (32 - 4);
157 | 	b ^= a; b -= a << 14 | a >> (32 - 14);
158 | 	c ^= b; c -= b << 24 | b >> (32 - 24);
159 | 	return a, b, c
160 | }
161 | */
162 | 
163 | //var a, b, c uint32
164 | 
165 | func rot(x, k uint32) uint32 {
166 | 	return x<<k | x>>(32-k)
167 | }
168 | 
169 | // current gc compilers can't inline long functions so we have to split mix into 2
170 | func mix1(a, b, c uint32) (uint32, uint32, uint32) {
171 | 	a -= c
172 | 	a ^= rot(c, 4)
173 | 	c += b
174 | 	b -= a
175 | 	b ^= rot(a, 6)
176 | 	a += c
177 | 	c -= b
178 | 	c ^= rot(b, 8)
179 | 	b += a
180 | 	//a -= c;  a ^= c << 4 | c >> (32 - 4);  c += b;
181 | 	//b -= a;  b ^= a << 6 | a >> (32 - 6);  a += c;
182 | 	return a, b, c
183 | }
184 | 
185 | func mix2(a, b, c uint32) (uint32, uint32, uint32) {
186 | 	a -= c
187 | 	a ^= rot(c, 16)
188 | 	c += b
189 | 	b -= a
190 | 	b ^= rot(a, 19)
191 | 	a += c
192 | 	c -= b
193 | 	c ^= rot(b, 4)
194 | 	b += a
195 | 	//	c -= b;  c ^= b << 8 | b >> (32 - 8);  b += a;
196 | 	//	a -= c;  a ^= c << 16 | c >> (32 - 16);  c += b;
197 | 	return a, b, c
198 | }
199 | 
200 | /*
201 | func mix3(a, b, c uint32) (uint32, uint32, uint32) {
202 | 	b -= a;  b ^= a << 19 | a >> (32 - 19);  a += c;
203 | 	c -= b;  c ^= b << 4 | b >> (32 - 4);  b += a;
204 | 	return a, b, c
205 | }
206 | */
207 | 
208 | func final1(a, b, c uint32) (uint32, uint32, uint32) {
209 | 	c ^= b
210 | 	c -= rot(b, 14)
211 | 	a ^= c
212 | 	a -= rot(c, 11)
213 | 	b ^= a
214 | 	b -= rot(a, 25)
215 | 	c ^= b
216 | 	c -= rot(b, 16)
217 | 	//c ^= b; c -= b << 14 | b >> (32 - 14);
218 | 	//a ^= c; a -= c << 11 | c >> (32 - 11);
219 | 	//b ^= a; b -= a << 25 | a >> (32 - 25);
220 | 	//c ^= b; c -= b << 16 | b >> (32 - 16);
221 | 	return a, b, c
222 | }
223 | 
224 | func final2(a, b, c uint32) (uint32, uint32, uint32) {
225 | 	a ^= c
226 | 	a -= rot(c, 4)
227 | 	b ^= a
228 | 	b -= rot(a, 14)
229 | 	c ^= b
230 | 	c -= rot(b, 24)
231 | 	//a ^= c; a -= c << 4 | c >> (32 - 4);
232 | 	//b ^= a; b -= a << 14 | a >> (32 - 14);
233 | 	//c ^= b; c -= b << 24 | b >> (32 - 24);
234 | 	return a, b, c
235 | }
236 | 
237 | func HashWords32(k []uint32, seed uint32) uint32 {
238 | 	var a, b, c uint32
239 | 
240 | 	length := uint32(len(k))
241 | 	a = 0xdeadbeef + length<<2 + seed
242 | 	b, c = a, a
243 | 
244 | 	i := 0
245 | 	for ; length > 3; length -= 3 {
246 | 		a += k[i+0]
247 | 		b += k[i+1]
248 | 		c += k[i+2]
249 | 		a, b, c = mix1(a, b, c)
250 | 		a, b, c = mix2(a, b, c)
251 | 		i += 3
252 | 	}
253 | 
254 | 	switch length {
255 | 	case 3:
256 | 		c += k[i+2]
257 | 		fallthrough
258 | 	case 2:
259 | 		b += k[i+1]
260 | 		fallthrough
261 | 	case 1:
262 | 		a += k[i+0]
263 | 		a, b, c = final1(a, b, c)
264 | 		a, b, c = final2(a, b, c)
265 | 	case 0:
266 | 		break
267 | 	}
268 | 	return c
269 | }
270 | 
271 | func HashWordsLen(k []uint32, length int, seed uint32) uint32 {
272 | 	var a, b, c uint32
273 | 
274 | 	//fmt.Printf("k=%v\n", k)
275 | 	//fmt.Printf("length=%d, len(k)=%d\n", length, len(k))
276 | 	if length > len(k) {
277 | 		fmt.Printf("length=%d, len(k)=%d\n", length, len(k))
278 | 		panic("HashWords")
279 | 	}
280 | 
281 | 	ul := uint32(len(k))
282 | 	a = 0xdeadbeef + ul<<2 + seed
283 | 	b, c = a, a
284 | 
285 | 	i := 0
286 | 	//length := 0
287 | 	for ; length > 3; length -= 3 {
288 | 		a += k[i+0]
289 | 		b += k[i+1]
290 | 		c += k[i+2]
291 | 		a, b, c = mix1(a, b, c)
292 | 		a, b, c = mix2(a, b, c)
293 | 		//a, b, c = mix3(a, b, c)
294 | 		i += 3
295 | 	}
296 | 
297 | 	//fmt.Printf("remaining length=%d, len(k)=%d, i=%d, k[i + 2]=%d, k[i + 1]=%d, k[i + 0]=%d\n", length, len(k), i, k[i + 2], k[i + 1], k[i + 0])
298 | 	switch length {
299 | 	case 3:
300 | 		c += k[i+2]
301 | 		fallthrough
302 | 	case 2:
303 | 		b += k[i+1]
304 | 		fallthrough
305 | 	case 1:
306 | 		a += k[i+0]
307 | 		a, b, c = final1(a, b, c)
308 | 		a, b, c = final2(a, b, c)
309 | 	case 0:
310 | 		break
311 | 	}
312 | 	//fmt.Printf("end\n")
313 | 	return c
314 | }
315 | 
316 | func XHashWords(k []uint32, length int, seed uint32) uint32 {
317 | 	var a, b, c uint32
318 | 	var rot = func(x, k uint32) uint32 {
319 | 		return x<<k | x>>(32-k)
320 | 	}
321 | 	var mix = func() {
322 | 		a -= c
323 | 		a ^= rot(c, 4)
324 | 		c += b
325 | 		b -= a
326 | 		b ^= rot(a, 6)
327 | 		a += c
328 | 		c -= b
329 | 		c ^= rot(b, 8)
330 | 		b += a
331 | 		a -= c
332 | 		a ^= rot(c, 16)
333 | 		c += b
334 | 		b -= a
335 | 		b ^= rot(a, 19)
336 | 		a += c
337 | 		c -= b
338 | 		c ^= rot(b, 4)
339 | 		b += a
340 | 	}
341 | 	var final = func() {
342 | 		c ^= b
343 | 		c -= rot(b, 14)
344 | 		a ^= c
345 | 		a -= rot(c, 11)
346 | 		b ^= a
347 | 		b -= rot(a, 25)
348 | 		c ^= b
349 | 		c -= rot(b, 16)
350 | 		a ^= c
351 | 		a -= rot(c, 4)
352 | 		b ^= a
353 | 		b -= rot(a, 14)
354 | 		c ^= b
355 | 		c -= rot(b, 24)
356 | 	}
357 | 	ul := uint32(len(k))
358 | 	a = 0xdeadbeef + ul<<2 + seed
359 | 	b, c = a, a
360 | 
361 | 	i := 0
362 | 	//length := 0
363 | 	for length = len(k); length > 3; length -= 3 {
364 | 		a += k[i+0]
365 | 		b += k[i+1]
366 | 		c += k[i+2]
367 | 		mix()
368 | 		i += 3
369 | 	}
370 | 
371 | 	switch length {
372 | 	case 3:
373 | 		c += k[i+2]
374 | 		fallthrough
375 | 	case 2:
376 | 		b += k[i+1]
377 | 		fallthrough
378 | 	case 1:
379 | 		a += k[i+0]
380 | 		final()
381 | 	case 0:
382 | 		break
383 | 	}
384 | 	return c
385 | }
386 | 
387 | // jenkins364: return 2 32-bit hash values.
388 | // Returns two 32-bit hash values instead of just one.
389 | // This is good enough for hash table lookup with 2^^64 buckets,
390 | // or if you want a second hash if you're not happy with the first,
391 | // or if you want a probably-unique 64-bit ID for the key.
392 | // *pc is better mixed than *pb, so use *pc first.
393 | // If you want a 64-bit value do something like "*pc + (((uint64_t)*pb)<<32)"
394 | func Jenkins364(k []byte, length int, pc, pb uint32) (rpc, rpb uint32) {
395 | 	var a, b, c uint32
396 | 	/*
397 | 		var rot = func(x, k uint32) uint32 {
398 | 			return x << k | x >> (32 - k)
399 | 		}
400 | 
401 | 		var mix = func() {
402 | 			a -= c;  a ^= rot(c, 4); c += b;
403 | 			b -= a;  b ^= rot(a, 6);  a += c;
404 | 			c -= b;  c ^= rot(b, 8);  b += a;
405 | 			a -= c;  a ^= rot(c,16);  c += b;
406 | 			b -= a;  b ^= rot(a,19);  a += c;
407 | 			c -= b;  c ^= rot(b, 4);  b += a;
408 | 		}
409 | 		var final = func() {
410 | 			c ^= b; c -= rot(b,14);
411 | 			a ^= c; a -= rot(c,11);
412 | 			b ^= a; b -= rot(a,25);
413 | 			c ^= b; c -= rot(b,16);
414 | 			a ^= c; a -= rot(c,4);
415 | 			b ^= a; b -= rot(a,14);
416 | 			c ^= b; c -= rot(b,24);
417 | 		}
418 | 	*/
419 | 	ul := uint32(len(k))
420 | 	//fmt.Printf("s=%q, k=%v, len(s)=%d, len(k)=%d\n", s, k, len(s), len(k))
421 | 
422 | 	/* Set up the internal state */
423 | 	a = 0xdeadbeef + ul + pc
424 | 	b, c = a, a
425 | 	c += pb
426 | 
427 | 	for ; length > 12; length -= 12 {
428 | 		//fmt.Printf("k=%q, length=%d\n", k, length)
429 | 		a += *(*uint32)(unsafe.Pointer(&k[0]))
430 | 		b += *(*uint32)(unsafe.Pointer(&k[4]))
431 | 		c += *(*uint32)(unsafe.Pointer(&k[8]))
432 | 		a, b, c = mix1(a, b, c)
433 | 		a, b, c = mix2(a, b, c)
434 | 		k = k[12:]
435 | 	}
436 | 	//fmt.Printf("k=%q, length=%d\n", k, length)
437 | 
438 | 	/* handle the last (probably partial) block */
439 | 	/*
440 | 	 * "k[2]&0xffffff" actually reads beyond the end of the string, but
441 | 	 * then masks off the part it's not allowed to read.  Because the
442 | 	 * string is aligned, the masked-off tail is in the same word as the
443 | 	 * rest of the string.  Every machine with memory protection I've seen
444 | 	 * does it on word boundaries, so is OK with this.  But VALGRIND will
445 | 	 * still catch it and complain.  The masking trick does make the hash
446 | 	 * noticably faster for short strings (like English words).
447 | 	 */
448 | 
449 | 	//fmt.Printf("length now=%d\n", length)
450 | 	switch length {
451 | 	case 12:
452 | 		a += *(*uint32)(unsafe.Pointer(&k[0]))
453 | 		b += *(*uint32)(unsafe.Pointer(&k[4]))
454 | 		c += *(*uint32)(unsafe.Pointer(&k[8]))
455 | 	case 11:
456 | 		c += uint32(k[10]) << 16
457 | 		fallthrough
458 | 	case 10:
459 | 		c += uint32(k[9]) << 8
460 | 		fallthrough
461 | 	case 9:
462 | 		c += uint32(k[8])
463 | 		fallthrough
464 | 	case 8:
465 | 		a += *(*uint32)(unsafe.Pointer(&k[0]))
466 | 		b += *(*uint32)(unsafe.Pointer(&k[4]))
467 | 		break
468 | 	case 7:
469 | 		b += uint32(k[6]) << 16
470 | 		fallthrough
471 | 	case 6:
472 | 		b += uint32(k[5]) << 8
473 | 		fallthrough
474 | 	case 5:
475 | 		b += uint32(k[4])
476 | 		fallthrough
477 | 	case 4:
478 | 		a += *(*uint32)(unsafe.Pointer(&k[0]))
479 | 		break
480 | 	case 3:
481 | 		a += uint32(k[2]) << 16
482 | 		fallthrough
483 | 	case 2:
484 | 		a += uint32(k[1]) << 8
485 | 		fallthrough
486 | 	case 1:
487 | 		a += uint32(k[0])
488 | 		break
489 | 	case 0:
490 | 		//fmt.Printf("case 0\n")
491 | 		return c, b /* zero length strings require no mixing */
492 | 	}
493 | 	a, b, c = final1(a, b, c)
494 | 	a, b, c = final2(a, b, c)
495 | 	return c, b
496 | }
497 | 
498 | func HashString(s string, pc, pb uint32) (rpc, rpb uint32) {
499 | 	k := ([]byte)(s)
500 | 	rpc, rpb = Jenkins364(k, len(k), pc, pb)
501 | 	return
502 | }
503 | 
504 | func HashBytesLength(k []byte, length int, seed uint32) uint32 {
505 | 	if length > len(k) {
506 | 		fmt.Printf("len(k)=%d, length=%d\n", len(k), length)
507 | 		panic("HashBytesLength")
508 | 	}
509 | 	ret, _ := Jenkins364(k, length, seed, seed)
510 | 	return ret
511 | }
512 | 
513 | // Sum32 returns the 32 bit hash of data given the seed.
514 | // This is code is what I started with before I added the hash.Hash and hash.Hash32 interfaces.
515 | func Sum32(data []byte, seed uint32) uint32 {
516 | 	rpc, _ := Jenkins364(data, len(data), seed, seed)
517 | 	return rpc
518 | }
519 | 
520 | // Sum64 returns the 64 bit hash of data given the seed.
521 | // This is code is what I started with before I added the hash.Hash and hash.Hash32 interfaces.
522 | func Sum64(data []byte, seed uint32) uint64 {
523 | 	rpc, rpb := Jenkins364(data, len(data), seed, seed)
524 | 	return uint64(rpc)<<32 | uint64(rpb)
525 | }
526 | 
527 | func HashBytes(data []byte, seed uint64) uint64 {
528 | 	rpc, rpb := Jenkins364(data, len(data), uint32(seed>>32), uint32(seed&0xFFFFFFFF))
529 | 	return uint64(rpc)<<32 | uint64(rpb)
530 | }
531 | 
532 | // New returns a new hash.Hash64 interface that computes the a 32 bit murmur3 hash.
533 | func New(seed uint32) hash.Hash64 {
534 | 	d := new(Digest)
535 | 	d.seed = seed
536 | 	d.Reset()
537 | 	return d
538 | }
539 | 
540 | // Reset the hash state.
541 | func (d *Digest) Reset() {
542 | 	d.hash = uint64(d.seed)
543 | 	d.clen = 0
544 | 	d.tail = nil
545 | }
546 | 
547 | // Return the size of the resulting hash.
548 | func (d *Digest) Size() int { return Size }
549 | 
550 | // Return the blocksize of the hash which in this case is 1 byte.
551 | func (d *Digest) BlockSize() int { return 1 }
552 | 
553 | // Accept a byte stream p used for calculating the hash. For now this call is lazy and the actual hash calculations take place in Sum() and Sum32().
554 | func (d *Digest) Write(p []byte) (nn int, err error) {
555 | 	l := len(p)
556 | 	d.clen += l
557 | 	d.tail = append(d.tail, p...)
558 | 	return l, nil
559 | }
560 | 
561 | // Return the current hash as a byte slice.
562 | func (d *Digest) Sum(b []byte) []byte {
563 | 	d.pc, d.pb = Jenkins364(d.tail, len(d.tail), d.pc, d.pb)
564 | 	d.hash = uint64(d.pc)
565 | 	h := d.pc
566 | 	return append(b, byte(h>>24), byte(h>>16), byte(h>>8), byte(h))
567 | }
568 | 
569 | // Return the current hash as a 32 bit unsigned type.
570 | /*
571 | func (d *Digest) Sum32() uint32 {
572 | 	d.pc, d.pb = Jenkins364(d.tail, len(d.tail), d.pc, d.pb)
573 | 	d.hash = uint64(d.pc)
574 | 	return uint32(d.hash)
575 | }
576 | */
577 | 
578 | // Return the current hash as a 32 bit unsigned type.
579 | func (d *Digest) Sum64() uint64 {
580 | 	d.pc, d.pb = Jenkins364(d.tail, len(d.tail), d.pc, d.pb)
581 | 	d.hash = uint64(d.pc)<<32 | uint64(d.pb)
582 | 	return d.hash
583 | }
584 | 
585 | /*
586 | 	var mix = func() {
587 | 		a -= c;  a ^= c << 4 | c >> (32 - 4);  c += b;
588 | 		b -= a;  b ^= a << 6 | a >> (32 - 6);  a += c;
589 | 		c -= b;  c ^= b << 8 | b >> (32 - 8);  b += a;
590 | 		a -= c;  a ^= c << 16 | c >> (32 - 16);  c += b;
591 | 		b -= a;  b ^= a << 19 | a >> (32 - 19);  a += c;
592 | 		c -= b;  c ^= b << 4 | b >> (32 - 4);  b += a;
593 | 	}
594 | 
595 | 	var final = func() {
596 | 		c ^= b; c -= b << 14 | b >> (32 - 14);
597 | 		a ^= c; a -= c << 11 | c >> (32 - 11);
598 | 		b ^= a; b -= a << 25 | a >> (32 - 25);
599 | 		c ^= b; c -= b << 16 | b >> (32 - 16);
600 | 		a ^= c; a -= c << 4 | c >> (32 - 4);
601 | 		b ^= a; b -= a << 14 | a >> (32 - 14);
602 | 		c ^= b; c -= b << 24 | b >> (32 - 24);
603 | 	}
604 | */
605 | 


--------------------------------------------------------------------------------
/internal/jenkins3/jenkins3_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright © 2014 Lawrence E. Bakst. All rights reserved.
  2 | package jenkins3_test
  3 | 
  4 | //import "flag"
  5 | import (
  6 | 	"fmt"
  7 | 	"testing"
  8 | 
  9 | 	"leb.io/cuckoo/internal/jenkins3"
 10 | )
 11 | 
 12 | //import "math"
 13 | //import "math/rand"
 14 | //import "runtime"
 15 | 
 16 | func TestBasic(t *testing.T) {
 17 | 
 18 | 	q := "This is the time for all good men to come to the aid of their country..."
 19 | 	//qq := []byte{"xThis is the time for all good men to come to the aid of their country..."}
 20 | 	//qqq := []byte{"xxThis is the time for all good men to come to the aid of their country..."}
 21 | 	//qqqq[] := []byte{"xxxThis is the time for all good men to come to the aid of their country..."}
 22 | 
 23 | 	u := stu(q)
 24 | 	h1 := jenkins3.HashWordsLen(u, 13)
 25 | 	fmt.Printf("%08x, %0x8, %08x\n", h1)
 26 | 
 27 | 	b, c := uint32(0), uint32(0)
 28 | 	c, b = jenkins3.HashString("", c, b)
 29 | 	//fmt.Printf("%08x, %08x\n", c, b)
 30 | 	if c != 0xdeadbeef || b != 0xdeadbeef {
 31 | 		t.Logf("c=0x%x != 0xdeadbeef || b=0x%x != 0xdeadbeef\n", c, b)
 32 | 		t.FailNow()
 33 | 	}
 34 | 
 35 | 	b, c = 0xdeadbeef, 0
 36 | 	c, b = jenkins3.HashString("", c, b)
 37 | 	//fmt.Printf("%08x, %08x\n", c, b)	// bd5b7dde deadbeef
 38 | 	if c != 0xbd5b7dde || b != 0xdeadbeef {
 39 | 		t.Logf("c=0x%x != 0xbd5b7dde || b=0x%x != 0xdeadbeef\n", c, b)
 40 | 		t.FailNow()
 41 | 	}
 42 | 
 43 | 	b, c = 0xdeadbeef, 0xdeadbeef
 44 | 	c, b = jenkins3.HashString("", c, b)
 45 | 	//fmt.Printf("%08x, %08x\n", c, b)	// 9c093ccd bd5b7dde
 46 | 	if c != 0x9c093ccd || b != 0xbd5b7dde {
 47 | 		t.Logf("c=0x%x != 0x9c093ccd || b=0x%x != 0xbd5b7dde\n", c, b)
 48 | 		t.FailNow()
 49 | 	}
 50 | 
 51 | 	b, c = 0, 0
 52 | 	c, b = jenkins3.HashString("Four score and seven years ago", c, b)
 53 | 	//fmt.Printf("%08x, %08x\n", c, b)	// 17770551 ce7226e6
 54 | 	if c != 0x17770551 || b != 0xce7226e6 {
 55 | 		t.Logf("c=0x%x != 0x17770551 || b=0x%x != 0xce7226e6\n", c, b)
 56 | 		t.FailNow()
 57 | 	}
 58 | 
 59 | 	b, c = 1, 0
 60 | 	c, b = jenkins3.HashString("Four score and seven years ago", c, b)
 61 | 	//fmt.Printf("%08x, %08x\n", c, b)	// e3607cae bd371de4
 62 | 	if c != 0xe3607cae || b != 0xbd371de4 {
 63 | 		t.Logf("c=0x%x != 0xe3607cae || b=0x%x != 0xbd371de4\n", c, b)
 64 | 		t.FailNow()
 65 | 	}
 66 | 
 67 | 	b, c = 0, 1
 68 | 	c, b = jenkins3.HashString("Four score and seven years ago", c, b)
 69 | 	//fmt.Printf("%08x, %08x\n", c, b)	// cd628161 6cbea4b3
 70 | 	if c != 0xcd628161 || b != 0x6cbea4b3 {
 71 | 		t.Logf("c=0x%x != 0xcd628161 || b=0x%x != 0x6cbea4b3\n", c, b)
 72 | 		t.FailNow()
 73 | 	}
 74 | 
 75 | }
 76 | 
 77 | func BenchmarkJenkins(b *testing.B) {
 78 | 	//tmp := make([]byte, 4, 4)
 79 | 	us := make([]uint32, 1)
 80 | 	b.SetBytes(int64(b.N * 4))
 81 | 	for i := 1; i <= b.N; i++ {
 82 | 		us[0] = uint32(i)
 83 | 		//tmp[0], tmp[1], tmp[2], tmp[3] = byte(key&0xFF), byte((key>>8)&0xFF), byte((key>>16)&0xFF), byte((key>>24)&0xFF)
 84 | 		jenkins3.HashWords(us, 0)
 85 | 	}
 86 | }
 87 | 
 88 | /*
 89 | func main() {
 90 | 	q := "This is the time for all good men to come to the aid of their country..."
 91 | 	//qq := []byte{"xThis is the time for all good men to come to the aid of their country..."}
 92 | 	//qqq := []byte{"xxThis is the time for all good men to come to the aid of their country..."}
 93 | 	//qqqq[] := []byte{"xxxThis is the time for all good men to come to the aid of their country..."}
 94 | 
 95 | 	u := stu(q)
 96 | 	h1 := hashword(u, (len(q)-1)/4, 13)
 97 | 	h2 := hashword(u, (len(q)-5)/4, 13)
 98 | 	h3 := hashword(u, (len(q)-9)/4, 13)
 99 | 	fmt.Printf("%08x, %0x8, %08x\n", h1, h2, h3)
100 | 
101 | 
102 | }
103 | */
104 | 


--------------------------------------------------------------------------------
/internal/siginfo/example/example.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"log"
 5 | 
 6 | 	"leb.io/cuckoo/internal/siginfo"
 7 | )
 8 | 
 9 | func main() {
10 | 	var f = func() {
11 | 		log.Printf("You pressed ^T")
12 | 	}
13 | 	siginfo.SetHandler(f)
14 | 	select {}
15 | }
16 | 
17 | /*
18 | func main() {
19 | 	ch := make(chan os.Signal, 1)
20 | 	signal.Notify(ch, SIGINFO)
21 | 
22 | 	go func() {
23 | 		for _ = range ch {
24 | 			// f()
25 | 			log.Printf("You pressed ^T")
26 | 		}
27 | 	}()
28 | 
29 | 	select {}
30 | }*/
31 | 


--------------------------------------------------------------------------------
/internal/siginfo/siginfo.go:
--------------------------------------------------------------------------------
 1 | package siginfo
 2 | 
 3 | import (
 4 | 	_ "log"
 5 | 	"os"
 6 | 	"os/signal"
 7 | 	"syscall"
 8 | )
 9 | 
10 | // SIGINFO isn't part of the stdlib, but it's 29 on most systems
11 | const SIGINFO = syscall.Signal(29)
12 | 
13 | func SetHandler(f func()) {
14 | 	ch := make(chan os.Signal, 1)
15 | 	signal.Notify(ch, SIGINFO)
16 | 
17 | 	go func() {
18 | 		for _ = range ch {
19 | 			f()
20 | 		}
21 | 	}()
22 | }
23 | 


--------------------------------------------------------------------------------
/kv_default.go:
--------------------------------------------------------------------------------
 1 | // Copyright © 2014-2017 Lawrence E. Bakst. All rights reserved.
 2 | // +build !string
 3 | 
 4 | // !kuint32,!vuint32
 5 | 
 6 | package cuckoo
 7 | 
 8 | import "hash"
 9 | 
10 | type Key uint64
11 | type Value uint64
12 | 
13 | func (c *Cuckoo) _calcHash(hf hash.Hash64, seed uint64, key Key) (h uint64) {
14 | 	// ok we have to copy the key now as all the other hash functions want a slice of bytes.
15 | 	switch c.NumericKeySize {
16 | 	case 4:
17 | 		c.buf.b = c.buf.base[0:4]
18 | 		c.buf.b[0], c.buf.b[1], c.buf.b[2], c.buf.b[3] = byte(key), byte(key>>8), byte(key>>16), byte(key>>24)
19 | 	case 8:
20 | 		c.buf.b = c.buf.base[0:8]
21 | 		c.buf.b[0], c.buf.b[1], c.buf.b[2], c.buf.b[3], c.buf.b[4], c.buf.b[5], c.buf.b[6], c.buf.b[7] =
22 | 			byte(key), byte(key>>8), byte(key>>16), byte(key>>24), byte(key>>32), byte(key>>40), byte(key>>48), byte(key>>56)
23 | 	default:
24 | 		c.buf.Reset()
25 | 		if err := c.encoder.Encode(&key); err != nil {
26 | 			//fmt.Printf("Write: err=%q\n", err)
27 | 			panic("Insert: binary.Write")
28 | 		}
29 | 		c.buf.b = c.buf.base[0:c.buf.i]
30 | 	}
31 | 	if c.hfb != nil {
32 | 		h = c.hfb(c.buf.b, seed) % uint64(c.Nbuckets)
33 | 	} else {
34 | 		hf.Reset()
35 | 		hf.Write(c.buf.b)
36 | 		h1 := uint64(hf.Sum64())
37 | 		h = h1 % uint64(c.Nbuckets)
38 | 	}
39 | 	return
40 | }
41 | 
42 | // inlined functions
43 | 
44 | // the following 5 functions/methods can be inlined.
45 | func ui32tob(b []byte, key Key) {
46 | 	b[0], b[1], b[2], b[3] = byte(key), byte(key>>8), byte(key>>16), byte(key>>24)
47 | }
48 | 
49 | func ui64tob(b []byte, key Key) {
50 | 	b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7] = byte(key), byte(key>>8), byte(key>>16), byte(key>>24), byte(key>>32), byte(key>>40), byte(key>>48), byte(key>>56)
51 | }
52 | 
53 | func ui64tob1(b []byte, key Key) {
54 | 	b[0], b[1], b[2], b[3] = byte(key), byte(key>>8), byte(key>>16), byte(key>>24)
55 | }
56 | 
57 | func ui64tob2(b []byte, key Key) {
58 | 	b[4], b[5], b[6], b[7] = byte(key>>32), byte(key>>40), byte(key>>48), byte(key>>56)
59 | }
60 | 
61 | // Given a key and a hash function to use, calculate the hash for the specified table.
62 | // To do this we have to serialize the key
63 | // To get this to inline the optimization for NumericKeySize == 4 was moved to _calcHash ???
64 | // check to see this this inlines with SSA
65 | func (c *Cuckoo) calcHash(hf hash.Hash64, seed uint64, key Key) uint64 {
66 | 	// speed up a common key case
67 | 	//fmt.Printf("%d ", c.NumericKeySize)
68 | 	//fmt.Printf("calcHash: seed=%d, key=%v\n", seed, key)
69 | 	if c.hashno == aes {
70 | 		if c.NumericKeySize == 8 && c.hf64 != nil {
71 | 			//fmt.Printf("8 key=%v, h=%v\n", uint64(key), c.hf64(uint64(key), seed))
72 | 			//ui64tob1(c.buf.b, key)
73 | 			//ui64tob2(c.buf.b, key)
74 | 			return c.hf64(uint64(key), seed)
75 | 		} else {
76 | 			//fmt.Printf("4")
77 | 			if c.NumericKeySize == 4 && c.hf32 != nil {
78 | 				//ui32tob(c.buf.b, key)
79 | 				return c.hf32(uint32(key), seed)
80 | 			}
81 | 		}
82 | 	}
83 | 	return c._calcHash(hf, seed, key)
84 | }
85 | 
86 | //type Value interface{}
87 | 


--------------------------------------------------------------------------------
/kv_string.go:
--------------------------------------------------------------------------------
 1 | // Copyright © 2014-2017 Lawrence E. Bakst. All rights reserved.
 2 | // +build string
 3 | 
 4 | package cuckoo
 5 | 
 6 | import "hash"
 7 | 
 8 | type Key string
 9 | type Value string
10 | 
11 | func (c *Cuckoo) _calcHash(hf hash.Hash64, seed uint64, key Key) (h uint64) {
12 | 	// ok we have to copy the key now as all the other hash functions want a slice of bytes.
13 | 	switch c.NumericKeySize {
14 | 	case 4:
15 | 		panic("_calcHash: 4")
16 | 	case 8:
17 | 		panic("_calcHash: 8")
18 | 	default:
19 | 		c.buf.Reset()
20 | 		if err := c.encoder.Encode(&key); err != nil {
21 | 			//fmt.Printf("Write: err=%q\n", err)
22 | 			panic("Insert: binary.Write")
23 | 		}
24 | 		c.buf.b = c.buf.base[0:c.buf.i]
25 | 	}
26 | 	if c.hfb != nil {
27 | 		h = c.hfb(c.buf.b, seed) % uint64(c.Buckets)
28 | 	} else {
29 | 		hf.Reset()
30 | 		hf.Write(c.buf.b)
31 | 		h1 := uint64(hf.Sum64())
32 | 		h = h1 % uint64(c.Buckets)
33 | 	}
34 | 	return
35 | }
36 | 
37 | // Given a key and a hash function to use, calculate the hash for the specified table.
38 | // To do this we have to serialize the key
39 | // To get this to inline the optimization for NumericKeySize == 4 was moved to _calcHash ???
40 | // check to see this this inlines with SSA
41 | func (c *Cuckoo) calcHash(hf hash.Hash64, seed uint64, key Key) uint64 {
42 | 	// speed up a common key case
43 | 	//fmt.Printf("%d ", c.NumericKeySize)
44 | 	return c._calcHash(hf, seed, key)
45 | }
46 | 


--------------------------------------------------------------------------------
/kvt_array.go:
--------------------------------------------------------------------------------
 1 | // Copyright © 2014-2017 Lawrence E. Bakst. All rights reserved.
 2 | // +build !slice
 3 | 
 4 | package cuckoo
 5 | 
 6 | const Nslots = 8
 7 | 
 8 | type Slots [Nslots]Bucket // slots
 9 | func makeSlots(s Slots, slots int) Slots {
10 | 	return s
11 | }
12 | 


--------------------------------------------------------------------------------
/kvt_slice.go:
--------------------------------------------------------------------------------
 1 | // Copyright © 2014-2017 Lawrence E. Bakst. All rights reserved.
 2 | // +build slice
 3 | 
 4 | package cuckoo
 5 | 
 6 | type Slots []Bucket
 7 | 
 8 | func makeSlots(s Slots, slots int) Slots {
 9 | 	return make(Slots, slots, slots)
10 | }
11 | 


--------------------------------------------------------------------------------
/murmur3/License-murmur3_test:
--------------------------------------------------------------------------------
 1 | Copyright 2013, Sébastien Paolacci.
 2 | All rights reserved.
 3 | 
 4 | Redistribution and use in source and binary forms, with or without
 5 | modification, are permitted provided that the following conditions are met:
 6 |     * Redistributions of source code must retain the above copyright
 7 |       notice, this list of conditions and the following disclaimer.
 8 |     * Redistributions in binary form must reproduce the above copyright
 9 |       notice, this list of conditions and the following disclaimer in the
10 |       documentation and/or other materials provided with the distribution.
11 |     * Neither the name of the library nor the
12 |       names of its contributors may be used to endorse or promote products
13 |       derived from this software without specific prior written permission.
14 | 
15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 | DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


--------------------------------------------------------------------------------
/murmur3/murmur3.go:
--------------------------------------------------------------------------------
  1 | // © Copyright 2014 Lawrence E. Bakst All Rights Reserved
  2 | // THIS SOURCE CODE IS THE PROPRIETARY INTELLECTUAL PROPERTY AND CONFIDENTIAL
  3 | // INFORMATION OF LAWRENCE E. BAKST AND IS PROTECTED UNDER U.S. AND
  4 | // INTERNATIONAL LAW. ANY USE OF THIS SOURCE CODE WITHOUT THE
  5 | // AUTHORIZATION OF LAWRENCE E. BAKST IS STRICTLY PROHIBITED.
  6 | 
  7 | // This package implements the 32 bit version of the MurmurHash3 hash code.
  8 | // With the exception of the interface check, this version was developed independtly.
  9 | // However, the "spaolacci" implementation with it's bmixer interface is da bomb, although
 10 | // this version is slightly faster.
 11 | //
 12 | // https://en.wikipedia.org/wiki/MurmurHash
 13 | // https://github.com/spaolacci/murmur3
 14 | package murmur3
 15 | 
 16 | import _ "fmt"
 17 | import "hash"
 18 | import "unsafe"
 19 | 
 20 | const (
 21 | 	c1	uint32 = 0xcc9e2d51
 22 | 	c2	uint32 = 0x1b873593
 23 | 	r1	uint32 = 15
 24 | 	r2	uint32 = 13
 25 | 	m	uint32 = 5
 26 | 	n	uint32 = 0xe6546b64
 27 | )
 28 | 
 29 | type Digest struct {
 30 | 	hash	uint32
 31 | 	seed	uint32
 32 | 	clen	int
 33 | 	tail	[]byte
 34 | }
 35 | 
 36 | // The size of an murmur3 32 bit hash in bytes.
 37 | const Size = 4
 38 | 
 39 | // Make sure interfaces are correctly implemented. Stolen from another implementation.
 40 | // I did something similar in another package to verify the interface but didn't know you could elide the variable in a var.
 41 | // What a cute wart it is.
 42 | var (
 43 | 	_ hash.Hash   = new(Digest)
 44 | 	_ hash.Hash32 = new(Digest)
 45 | )
 46 | 
 47 | // New returns a new hash.Hash32 interface that computes the a 32 bit murmur3 hash.
 48 | func New(seed uint32) hash.Hash32 {
 49 | 	d := new(Digest)
 50 | 	d.seed = seed
 51 | 	d.Reset()
 52 | 	return d
 53 | }
 54 | 
 55 | // The following methods implment the hash.Hash and hash.Hash32 interfaces
 56 | 
 57 | // Reset the hash state.
 58 | func (d *Digest) Reset() {
 59 | 	d.hash = d.seed
 60 | 	d.clen = 0
 61 | 	d.tail = nil
 62 | }
 63 | 
 64 | // Return the size of the resulting hash.
 65 | func (d *Digest) Size() int { return Size }
 66 | 
 67 | // Return the blocksize of the hash which in this case is 1 byte.
 68 | func (d *Digest) BlockSize() int { return 1 }
 69 | 
 70 | func (d *Digest) murmur332Blocks() {
 71 | 	nblocks := len(d.tail) / 4
 72 | 	if nblocks <= 0 {
 73 | 		return
 74 | 	}
 75 | 	for i := 0; i < nblocks; i++ {
 76 | 		k := *(*uint32)(unsafe.Pointer(&d.tail[i*4]))
 77 | 		//k = uint32(d.tail[i*4+0])<<0 | uint32(d.tail[i*4+1])<<8 | uint32(d.tail[i*4+2])<<16 | uint32(d.tail[i*4+3])<<24
 78 | 		k *= c1
 79 | 		k = (k << r1) | (k >> (32 - r1))
 80 | 		k *= c2
 81 | 		d.hash ^= k
 82 | 		d.hash = ((d.hash << r2) | (d.hash >> (32 - r2))) * m + n
 83 | 	}
 84 | 	d.tail = d.tail[nblocks*4:]
 85 | }
 86 | 
 87 | func (d *Digest) murmur332Tail() {
 88 | 	hash := d.hash
 89 | 	k1 := uint32(0)
 90 | 	l := len(d.tail) & 3; switch (l) {
 91 | 	case 3:
 92 | 		k1 ^= uint32(d.tail[2]) << 16
 93 | 		fallthrough
 94 | 	case 2:
 95 | 		k1 ^= uint32(d.tail[1]) << 8
 96 | 		fallthrough
 97 | 	case 1:
 98 | 		k1 ^= uint32(d.tail[0])
 99 | 		k1 *= c1
100 | 		k1 = (k1 << r1) | (k1 >> (32 - r1))
101 | 		k1 *= c2
102 | 		hash ^= k1
103 | 	case 0:
104 | 		break
105 | 	default:
106 | 		panic("murmur332Tail")
107 | 	}
108 | 	hash ^= uint32(d.clen)
109 | 	hash ^= hash >> 16
110 | 	hash *= 0x85ebca6b
111 | 	hash ^= hash >> 13
112 | 	hash *= 0xc2b2ae35
113 | 	hash ^= hash >> 16
114 | 	d.hash = hash
115 | }
116 | 
117 | 
118 | // Accept a byte stream p used for calculating the hash. For now this call is lazy and the actual hash calculations take place in Sum() and Sum32().
119 | func (d *Digest) Write(p []byte) (nn int, err error) {
120 | 	l := len(p)
121 | 	d.clen += l
122 | 	d.tail = append(d.tail, p...)
123 | 	return l, nil
124 | }
125 | 
126 | // Return the current hash as a byte slice.
127 | func (d *Digest) Sum(b []byte) []byte {
128 | 	d.murmur332Blocks()
129 | 	d.murmur332Tail()
130 | 	h := d.hash
131 | 	return append(b, byte(h>>24), byte(h>>16), byte(h>>8), byte(h))
132 | }
133 | 
134 | // Return the current hash as a 32 bit unsigned type.
135 | func (d *Digest) Sum32() uint32 {
136 | 	d.murmur332Blocks()
137 | 	d.murmur332Tail()
138 | 	return d.hash
139 | }
140 | 
141 | // Sum32 returns the 32 bit hash of data given the seed.
142 | // This is code is what I started with before I added the hash.Hash and hash.Hash32 interfaces.
143 | func Sum32(data []byte, seed uint32) uint32 {
144 | 	hash := seed
145 | 	nblocks := len(data) / 4
146 | 	for i := 0; i < nblocks; i++ {
147 | 		k := *(*uint32)(unsafe.Pointer(&data[i*4]))
148 | 		//k := uint32(data[i*4+0])<<0 | uint32(data[i*4+1])<<8 | uint32(data[i*4+2])<<16 | uint32(data[i*4+3])<<24
149 | 		k *= c1
150 | 		k = (k << r1) | (k >> (32 - r1))
151 | 		k *= c2
152 | 		hash ^= k
153 | 		hash = ((hash << r2) | (hash >> (32 - r2))) * m + n
154 | 	}
155 | 
156 | 	l := nblocks * 4; k1 := uint32(0); switch (len(data) & 3) {
157 | 	case 3:
158 | 		k1 ^= uint32(data[l+2]) << 16
159 | 		fallthrough
160 | 	case 2:
161 | 		k1 ^= uint32(data[l+1]) << 8
162 | 		fallthrough
163 | 	case 1:
164 | 		k1 ^= uint32(data[l+0])
165 | 		k1 *= c1
166 | 		k1 = (k1 << r1) | (k1 >> (32 - r1))
167 | 		k1 *= c2
168 | 		hash ^= k1
169 | 	}
170 |  
171 | 	hash ^= uint32(len(data))
172 | 	hash ^= hash >> 16
173 | 	hash *= 0x85ebca6b
174 | 	hash ^= hash >> 13
175 | 	hash *= 0xc2b2ae35
176 | 	hash ^= hash >> 16
177 | 	return hash
178 | }
179 | 


--------------------------------------------------------------------------------
/murmur3/murmur3_test.go:
--------------------------------------------------------------------------------
  1 | //Copyright 2013, Sébastien Paolacci. All rights reserved.
  2 | //© Copyright 2014 Lawrence E. Bakst All Rights Reserved
  3 | package murmur3
  4 | 
  5 | import (
  6 | 	"hash"
  7 | 	"testing"
  8 | 	_ "fmt"
  9 | )
 10 | 
 11 | var tests = []struct{
 12 | 	hash	uint32
 13 | 	hex		bool
 14 | 	s		string
 15 | }{
 16 | 	{0x00000000, false, ""},
 17 | 	{0x3c2569b2, false, "a"},
 18 | 	{0x4f31114c, false, "bc"},
 19 | 	{0xf5797de2, false, "def"},
 20 | 	{0x13704969, false, "ghij"},
 21 | 	{0x248bfa47, false, "hello"},
 22 | 	{0x149bbb7f, false, "hello, world"},
 23 | 	{0xe31e8a70, false, "19 Jan 2038 at 3:14:07 AM"},
 24 | 	{0xd5c48bfc, false, "The quick brown fox jumps over the lazy dog."},
 25 | 	{0xb61e6dcc, true,	"daf1596449909da0dde3987638909728"},				// same hash with all seeds!
 26 | }
 27 | 
 28 | func unhex(c byte) uint8 {
 29 |         switch {
 30 |         case '0' <= c && c <= '9':
 31 |                 return c - '0'
 32 |         case 'a' <= c && c <= 'f':
 33 |                 return c - 'a' + 10
 34 |         case 'A' <= c && c <= 'F':
 35 |                 return c - 'A' + 10
 36 |         }
 37 |         panic("unhex: bad input")
 38 | }
 39 | 
 40 | func hexToBytes(s string) []byte {
 41 |         var data = make([]byte, 1000, 1000)
 42 |         data = data[0:len(s)/2]
 43 | 
 44 |         n := len(s)
 45 |         if (n&1) == 1 {
 46 |                 panic("gethex: string must be even")
 47 |         }
 48 |         j = len()
 49 |         for i := range data {
 50 |                 data[i] = unhex(s[2*i])<<4 | unhex(s[2*i+1])
 51 |         }
 52 |         //fmt.Printf("hexToBytes: len(data)=%d, len(s)=%d\n", len(data), len(s))
 53 |         return data[0:len(s)/2]
 54 | }
 55 | 
 56 | func TestRef(t *testing.T) {
 57 | 	var h32 hash.Hash32 = New(0)
 58 | 	for _, elem := range tests {
 59 | 		h32.Reset()
 60 | 		b := []byte(elem.s)
 61 | 		if elem.hex {
 62 | 			b = hexToBytes(elem.s)
 63 | 		}
 64 | 		h32.Write(b)
 65 | 		//fmt.Printf("TestRef: %q, len=%d\n", elem.s, len(string(elem.s)))
 66 | 		if v := h32.Sum32(); v != elem.hash {
 67 | 			t.Errorf("h32.Sum32: %q 0x%x (want 0x%x)", elem.s, v, elem.hash)
 68 | 		}
 69 | 
 70 | 		if v := Sum32(b, 0); v != elem.hash {
 71 | 			t.Errorf("Sum32: %q 0x%x (want 0x%x)", elem.s, v, elem.hash)
 72 | 		}
 73 | 
 74 | 	}
 75 | }
 76 | 
 77 | //---
 78 | 
 79 | func bench32(b *testing.B, length int) {
 80 | 	buf := make([]byte, length)
 81 | 	b.SetBytes(int64(length))
 82 | 	b.ResetTimer()
 83 | 	for i := 0; i < b.N; i++ {
 84 | 		Sum32(buf, 0)
 85 | 	}
 86 | }
 87 | 
 88 | func Benchmark32_1(b *testing.B) {
 89 | 	bench32(b, 1)
 90 | }
 91 | func Benchmark32_2(b *testing.B) {
 92 | 	bench32(b, 2)
 93 | }
 94 | func Benchmark32_4(b *testing.B) {
 95 | 	bench32(b, 4)
 96 | }
 97 | func Benchmark32_8(b *testing.B) {
 98 | 	bench32(b, 8)
 99 | }
100 | func Benchmark32_16(b *testing.B) {
101 | 	bench32(b, 16)
102 | }
103 | func Benchmark32_32(b *testing.B) {
104 | 	bench32(b, 32)
105 | }
106 | func Benchmark32_64(b *testing.B) {
107 | 	bench32(b, 64)
108 | }
109 | func Benchmark32_128(b *testing.B) {
110 | 	bench32(b, 128)
111 | }
112 | func Benchmark32_256(b *testing.B) {
113 | 	bench32(b, 256)
114 | }
115 | func Benchmark32_512(b *testing.B) {
116 | 	bench32(b, 512)
117 | }
118 | func Benchmark32_1024(b *testing.B) {
119 | 	bench32(b, 1024)
120 | }
121 | func Benchmark32_2048(b *testing.B) {
122 | 	bench32(b, 2048)
123 | }
124 | func Benchmark32_4096(b *testing.B) {
125 | 	bench32(b, 4096)
126 | }
127 | func Benchmark32_8192(b *testing.B) {
128 | 	bench32(b, 8192)
129 | }
130 | 
131 | //---
132 | 
133 | func benchPartial32(b *testing.B, length int) {
134 | 	buf := make([]byte, length)
135 | 	b.SetBytes(int64(length))
136 | 
137 | 	start := (32 / 8) / 2
138 | 	chunks := 7
139 | 	k := length / chunks
140 | 	tail := (length - start) % k
141 | 
142 | 	b.ResetTimer()
143 | 	for i := 0; i < b.N; i++ {
144 | 		hasher := New(0)
145 | 		hasher.Write(buf[0:start])
146 | 
147 | 		for j := start; j+k <= length; j += k {
148 | 			hasher.Write(buf[j : j+k])
149 | 		}
150 | 
151 | 		hasher.Write(buf[length-tail:])
152 | 		hasher.Sum32()
153 | 	}
154 | }
155 | 
156 | func BenchmarkPartial32_8(b *testing.B) {
157 | 	benchPartial32(b, 8)
158 | }
159 | func BenchmarkPartial32_16(b *testing.B) {
160 | 	benchPartial32(b, 16)
161 | }
162 | func BenchmarkPartial32_32(b *testing.B) {
163 | 	benchPartial32(b, 32)
164 | }
165 | func BenchmarkPartial32_64(b *testing.B) {
166 | 	benchPartial32(b, 64)
167 | }
168 | func BenchmarkPartial32_128(b *testing.B) {
169 | 	benchPartial32(b, 128)
170 | }
171 | 
172 | 
173 | /*
174 | //     hasher := New32()
175 | //     hasher.Write(data)
176 | //     return hasher.Sum32()
177 | // 	{0x248bfa47, 0xcbd8a7b341bd9b02, 0x5b1e906a48ae1d19, "hello"},
178 | func main() {
179 | 	var h32 hash.Hash32 = murmur3.New32()
180 | 
181 | 	//value := "now is the time for all good men to come to the aid of their country"
182 | 	value := "hello"
183 | 	fmt.Printf("hash=0x%x\n", 0x248bfa47)
184 | 	for i := 0; i < 10; i++ {
185 | 		h := murmur332([]byte(value), uint32(i))
186 | 		fmt.Printf("%d: hash=0x%x\n", i, h)
187 | 	}
188 | 
189 | 	h32.Write([]byte(value))
190 | 	h := h32.Sum32()
191 | 	fmt.Printf("hash=0x%x\n", h)
192 | }
193 | */


--------------------------------------------------------------------------------
/primes/primes.go:
--------------------------------------------------------------------------------
  1 | // quick 10 minute transliteration of plan9/p9p primes.c
  2 | package primes
  3 | 
  4 | import "math"
  5 | 
  6 | var big = 9.007199254740992e15
  7 | 
  8 | var pt = []int{
  9 | 	2, 3, 5, 7, 11, 13, 17, 19, 23, 29,
 10 | 	31, 37, 41, 43, 47, 53, 59, 61, 67, 71,
 11 | 	73, 79, 83, 89, 97, 101, 103, 107, 109, 113,
 12 | 	127, 131, 137, 139, 149, 151, 157, 163, 167, 173,
 13 | 	179, 181, 191, 193, 197, 199, 211, 223, 227, 229,
 14 | }
 15 | 
 16 | var wheel = []int{
 17 | 	10, 2, 4, 2, 4, 6, 2, 6, 4, 2,
 18 | 	4, 6, 6, 2, 6, 4, 2, 6, 4, 6,
 19 | 	8, 4, 2, 4, 2, 4, 8, 6, 4, 6,
 20 | 	2, 4, 6, 2, 6, 6, 4, 2, 4, 6,
 21 | 	2, 6, 4, 2, 4, 2, 10, 2,
 22 | }
 23 | var table [10000]byte
 24 | var tsiz8 = len(table) * 8
 25 | var bittab = []byte{1, 2, 4, 8, 16, 32, 64, 128}
 26 | 
 27 | func mark(nn float64, k int) {
 28 | 	t1, _ := math.Modf(nn / float64(k))
 29 | 	j := int(float64(k)*t1 - nn)
 30 | 	if j < 0 {
 31 | 		j += k
 32 | 	}
 33 | 	for ; j < 8; j += k {
 34 | 		table[j>>3] |= bittab[j&07]
 35 | 	}
 36 | }
 37 | 
 38 | // collect primes between a and b and call f with each prime
 39 | func Primes(nn, limit float64, f func(v int)) {
 40 | 	if limit == 0.0 {
 41 | 		limit = big
 42 | 	}
 43 | 
 44 | 	if limit > big {
 45 | 		panic("limit > big")
 46 | 	}
 47 | 
 48 | 	if nn < 0 || nn > big {
 49 | 		panic("nn < 0 || nn > big")
 50 | 	}
 51 | 
 52 | 	if nn == 0 {
 53 | 		nn = 1
 54 | 	}
 55 | 
 56 | 	if nn < 230 {
 57 | 		for k, vi := range pt {
 58 | 			v := float64(vi)
 59 | 			if v < nn {
 60 | 				continue
 61 | 			}
 62 | 			if v > limit {
 63 | 				return
 64 | 			}
 65 | 			f(int(pt[k]))
 66 | 			if limit >= big {
 67 | 				return
 68 | 			}
 69 | 		}
 70 | 		nn = 230
 71 | 	}
 72 | 
 73 | 	temp, _ := math.Modf(nn / 2)
 74 | 	nn = 2.0*temp + 1
 75 | 
 76 | 	// clear the sieve table.
 77 | 	for {
 78 | 		for k := range table {
 79 | 			table[k] = 0
 80 | 		}
 81 | 
 82 | 		// run the sieve.
 83 | 		max := int(math.Sqrt(nn + float64(tsiz8)))
 84 | 		mark(nn, 3)
 85 | 		mark(nn, 5)
 86 | 		mark(nn, 7)
 87 | 		for i, k := 0, 11; k <= max; k += wheel[i] {
 88 | 			mark(nn, k)
 89 | 			i++
 90 | 			if i >= len(wheel) {
 91 | 				i = 0
 92 | 			}
 93 | 		}
 94 | 
 95 | 		// now get the primes from the table and print them.
 96 | 		for i := 0; i < tsiz8; i += 2 {
 97 | 			if (table[i>>3] & bittab[i&07]) != 0 {
 98 | 				continue
 99 | 			}
100 | 			temp = nn + float64(i)
101 | 			if temp > limit {
102 | 				return
103 | 			}
104 | 			f(int(temp))
105 | 			if limit >= big {
106 | 				return
107 | 			}
108 | 		}
109 | 		nn += float64(tsiz8)
110 | 	}
111 | }
112 | 
113 | func NextPrime(n int) (p int) {
114 | 	Primes(float64(n), 0.0, func(v int) { p = v })
115 | 	return
116 | }
117 | 


--------------------------------------------------------------------------------