├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── map.go └── map_test.go /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019, Joshua J Baker 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 10 | SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 12 | OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 13 | CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **This project has been archived. Please check out [tidwall/hashmap](https://github.com/tidwall/hashmap) for a fitter, happier, more productive hashmap library.** 2 | 3 | # rhh 4 | 5 | [![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://godoc.org/github.com/tidwall/rhh) 6 | 7 | A simple and efficient hashmap package for Go using the 8 | [`xxhash`](http://www.xxhash.com) algorithm, 9 | [open addressing](https://en.wikipedia.org/wiki/Hash_table#Open_addressing), and 10 | [robin hood hashing](https://en.wikipedia.org/wiki/Hash_table#Robin_Hood_hashing). 11 | 12 | This is an alternative to the standard [Go map](https://golang.org/ref/spec#Map_types). 13 | 14 | # Getting Started 15 | 16 | ## Installing 17 | 18 | To start using `rhh`, install Go and run `go get`: 19 | 20 | ```sh 21 | $ go get github.com/tidwall/rhh 22 | ``` 23 | 24 | This will retrieve the library. 25 | 26 | ## Usage 27 | 28 | The `Map` type works similar to a standard Go map, and includes four methods: 29 | `Set`, `Get`, `Delete`, `Len`. 30 | 31 | ```go 32 | var m rhh.Map 33 | m.Set("Hello", "Dolly!") 34 | val, _ := m.Get("Hello") 35 | fmt.Printf("%v\n", val) 36 | val, _ = m.Delete("Hello") 37 | fmt.Printf("%v\n", val) 38 | val, _ = m.Get("Hello") 39 | fmt.Printf("%v\n", val) 40 | 41 | // Output: 42 | // Dolly! 43 | // Dolly! 44 | // 45 | ``` 46 | 47 | ## Contact 48 | 49 | Josh Baker [@tidwall](http://twitter.com/tidwall) 50 | 51 | ## License 52 | 53 | Source code is available under the MIT [License](/LICENSE). 54 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tidwall/rhh 2 | 3 | go 1.15 4 | 5 | require github.com/cespare/xxhash/v2 v2.1.2 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= 2 | github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 3 | -------------------------------------------------------------------------------- /map.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Joshua J Baker. All rights reserved. 2 | // Use of this source code is governed by an ISC-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package rhh 6 | 7 | import "github.com/cespare/xxhash/v2" 8 | 9 | const ( 10 | loadFactor = 0.85 // must be above 50% 11 | dibBitSize = 16 // 0xFFFF 12 | hashBitSize = 64 - dibBitSize // 0xFFFFFFFFFFFF 13 | maxHash = ^uint64(0) >> dibBitSize // max 28,147,497,671,0655 14 | maxDIB = ^uint64(0) >> hashBitSize // max 65,535 15 | ) 16 | 17 | type entry struct { 18 | hdib uint64 // bitfield { hash:48 dib:16 } 19 | key string // user key 20 | value interface{} // user value 21 | } 22 | 23 | func (e *entry) dib() int { 24 | return int(e.hdib & maxDIB) 25 | } 26 | func (e *entry) hash() int { 27 | return int(e.hdib >> dibBitSize) 28 | } 29 | func (e *entry) setDIB(dib int) { 30 | e.hdib = e.hdib>>dibBitSize<> dibBitSize) 43 | } 44 | 45 | // Map is a hashmap. Like map[string]interface{} 46 | type Map struct { 47 | cap int 48 | length int 49 | mask int 50 | growAt int 51 | shrinkAt int 52 | buckets []entry 53 | } 54 | 55 | // New returns a new Map. Like map[string]interface{} 56 | func New(cap int) *Map { 57 | m := new(Map) 58 | m.cap = cap 59 | sz := 8 60 | for sz < m.cap { 61 | sz *= 2 62 | } 63 | m.buckets = make([]entry, sz) 64 | m.mask = len(m.buckets) - 1 65 | m.growAt = int(float64(len(m.buckets)) * loadFactor) 66 | m.shrinkAt = int(float64(len(m.buckets)) * (1 - loadFactor)) 67 | return m 68 | } 69 | 70 | func (m *Map) resize(newCap int) { 71 | nmap := New(newCap) 72 | for i := 0; i < len(m.buckets); i++ { 73 | if m.buckets[i].dib() > 0 { 74 | nmap.set(m.buckets[i].hash(), m.buckets[i].key, m.buckets[i].value) 75 | } 76 | } 77 | cap := m.cap 78 | *m = *nmap 79 | m.cap = cap 80 | } 81 | 82 | // Set assigns a value to a key. 83 | // Returns the previous value, or false when no value was assigned. 84 | func (m *Map) Set(key string, value interface{}) (interface{}, bool) { 85 | if len(m.buckets) == 0 { 86 | *m = *New(0) 87 | } 88 | if m.length >= m.growAt { 89 | m.resize(len(m.buckets) * 2) 90 | } 91 | return m.set(m.hash(key), key, value) 92 | } 93 | 94 | func (m *Map) set(hash int, key string, value interface{}) (interface{}, bool) { 95 | e := entry{makeHDIB(hash, 1), key, value} 96 | i := e.hash() & m.mask 97 | for { 98 | if m.buckets[i].dib() == 0 { 99 | m.buckets[i] = e 100 | m.length++ 101 | return nil, false 102 | } 103 | if e.hash() == m.buckets[i].hash() && e.key == m.buckets[i].key { 104 | old := m.buckets[i].value 105 | m.buckets[i].value = e.value 106 | return old, true 107 | } 108 | if m.buckets[i].dib() < e.dib() { 109 | e, m.buckets[i] = m.buckets[i], e 110 | } 111 | i = (i + 1) & m.mask 112 | e.setDIB(e.dib() + 1) 113 | } 114 | } 115 | 116 | // Get returns a value for a key. 117 | // Returns false when no value has been assign for key. 118 | func (m *Map) Get(key string) (interface{}, bool) { 119 | if len(m.buckets) == 0 { 120 | return nil, false 121 | } 122 | hash := m.hash(key) 123 | i := hash & m.mask 124 | for { 125 | if m.buckets[i].dib() == 0 { 126 | return nil, false 127 | } 128 | if m.buckets[i].hash() == hash && m.buckets[i].key == key { 129 | return m.buckets[i].value, true 130 | } 131 | i = (i + 1) & m.mask 132 | } 133 | } 134 | 135 | // Len returns the number of values in map. 136 | func (m *Map) Len() int { 137 | return m.length 138 | } 139 | 140 | // Delete deletes a value for a key. 141 | // Returns the deleted value, or false when no value was assigned. 142 | func (m *Map) Delete(key string) (interface{}, bool) { 143 | if len(m.buckets) == 0 { 144 | return nil, false 145 | } 146 | hash := m.hash(key) 147 | i := hash & m.mask 148 | for { 149 | if m.buckets[i].dib() == 0 { 150 | return nil, false 151 | } 152 | if m.buckets[i].hash() == hash && m.buckets[i].key == key { 153 | old := m.buckets[i].value 154 | m.remove(i) 155 | return old, true 156 | } 157 | i = (i + 1) & m.mask 158 | } 159 | } 160 | 161 | func (m *Map) remove(i int) { 162 | m.buckets[i].setDIB(0) 163 | for { 164 | pi := i 165 | i = (i + 1) & m.mask 166 | if m.buckets[i].dib() <= 1 { 167 | m.buckets[pi] = entry{} 168 | break 169 | } 170 | m.buckets[pi] = m.buckets[i] 171 | m.buckets[pi].setDIB(m.buckets[pi].dib() - 1) 172 | } 173 | m.length-- 174 | if len(m.buckets) > m.cap && m.length <= m.shrinkAt { 175 | m.resize(m.length) 176 | } 177 | } 178 | 179 | // Range iterates over all key/values. 180 | // It's not safe to call or Set or Delete while ranging. 181 | func (m *Map) Range(iter func(key string, value interface{}) bool) { 182 | for i := 0; i < len(m.buckets); i++ { 183 | if m.buckets[i].dib() > 0 { 184 | if !iter(m.buckets[i].key, m.buckets[i].value) { 185 | return 186 | } 187 | } 188 | } 189 | } 190 | 191 | // GetPos gets a single keys/value nearby a position 192 | // The pos param can be any valid uint64. Useful for grabbing a random item 193 | // from the map. 194 | // It's not safe to call or Set or Delete while ranging. 195 | func (m *Map) GetPos(pos uint64) (key string, value interface{}, ok bool) { 196 | for i := 0; i < len(m.buckets); i++ { 197 | index := (pos + uint64(i)) & uint64(m.mask) 198 | if m.buckets[index].dib() > 0 { 199 | return m.buckets[index].key, m.buckets[index].value, true 200 | } 201 | } 202 | return "", nil, false 203 | } 204 | -------------------------------------------------------------------------------- /map_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Joshua J Baker. All rights reserved. 2 | // Use of this source code is governed by an ISC-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package rhh 6 | 7 | import ( 8 | "fmt" 9 | "math/rand" 10 | "os" 11 | "runtime" 12 | "strconv" 13 | "testing" 14 | "time" 15 | ) 16 | 17 | type keyT = string 18 | type valueT = interface{} 19 | 20 | func k(key int) keyT { 21 | return strconv.FormatInt(int64(key), 10) 22 | } 23 | 24 | func add(x keyT, delta int) int { 25 | i, err := strconv.ParseInt(x, 10, 64) 26 | if err != nil { 27 | panic(err) 28 | } 29 | return int(i + int64(delta)) 30 | } 31 | 32 | /////////////////////////// 33 | func random(N int, perm bool) []keyT { 34 | nums := make([]keyT, N) 35 | if perm { 36 | for i, x := range rand.Perm(N) { 37 | nums[i] = k(x) 38 | } 39 | } else { 40 | m := make(map[keyT]bool) 41 | for len(m) < N { 42 | m[k(int(rand.Uint64()))] = true 43 | } 44 | var i int 45 | for k := range m { 46 | nums[i] = k 47 | i++ 48 | } 49 | } 50 | return nums 51 | } 52 | 53 | func shuffle(nums []keyT) { 54 | for i := range nums { 55 | j := rand.Intn(i + 1) 56 | nums[i], nums[j] = nums[j], nums[i] 57 | } 58 | } 59 | 60 | func init() { 61 | //var seed int64 = 1519776033517775607 62 | seed := (time.Now().UnixNano()) 63 | println("seed:", seed) 64 | rand.Seed(seed) 65 | } 66 | 67 | func TestRandomData(t *testing.T) { 68 | N := 10000 69 | start := time.Now() 70 | for time.Since(start) < time.Second*2 { 71 | nums := random(N, true) 72 | var m *Map 73 | switch rand.Int() % 5 { 74 | default: 75 | m = New(N / ((rand.Int() % 3) + 1)) 76 | case 1: 77 | m = new(Map) 78 | case 2: 79 | m = New(0) 80 | } 81 | v, ok := m.Get(k(999)) 82 | if ok || v != nil { 83 | t.Fatalf("expected %v, got %v", nil, v) 84 | } 85 | v, ok = m.Delete(k(999)) 86 | if ok || v != nil { 87 | t.Fatalf("expected %v, got %v", nil, v) 88 | } 89 | if m.Len() != 0 { 90 | t.Fatalf("expected %v, got %v", 0, m.Len()) 91 | } 92 | // set a bunch of items 93 | for i := 0; i < len(nums); i++ { 94 | v, ok := m.Set(nums[i], nums[i]) 95 | if ok || v != nil { 96 | t.Fatalf("expected %v, got %v", nil, v) 97 | } 98 | } 99 | if m.Len() != N { 100 | t.Fatalf("expected %v, got %v", N, m.Len()) 101 | } 102 | // retrieve all the items 103 | shuffle(nums) 104 | for i := 0; i < len(nums); i++ { 105 | v, ok := m.Get(nums[i]) 106 | if !ok || v == nil || v != nums[i] { 107 | t.Fatalf("expected %v, got %v", nums[i], v) 108 | } 109 | } 110 | // replace all the items 111 | shuffle(nums) 112 | for i := 0; i < len(nums); i++ { 113 | v, ok := m.Set(nums[i], add(nums[i], 1)) 114 | if !ok || v != nums[i] { 115 | t.Fatalf("expected %v, got %v", nums[i], v) 116 | } 117 | } 118 | if m.Len() != N { 119 | t.Fatalf("expected %v, got %v", N, m.Len()) 120 | } 121 | // retrieve all the items 122 | shuffle(nums) 123 | for i := 0; i < len(nums); i++ { 124 | v, ok := m.Get(nums[i]) 125 | if !ok || v != add(nums[i], 1) { 126 | t.Fatalf("expected %v, got %v", add(nums[i], 1), v) 127 | } 128 | } 129 | // remove half the items 130 | shuffle(nums) 131 | for i := 0; i < len(nums)/2; i++ { 132 | v, ok := m.Delete(nums[i]) 133 | if !ok || v != add(nums[i], 1) { 134 | t.Fatalf("expected %v, got %v", add(nums[i], 1), v) 135 | } 136 | } 137 | if m.Len() != N/2 { 138 | t.Fatalf("expected %v, got %v", N/2, m.Len()) 139 | } 140 | // check to make sure that the items have been removed 141 | for i := 0; i < len(nums)/2; i++ { 142 | v, ok := m.Get(nums[i]) 143 | if ok || v != nil { 144 | t.Fatalf("expected %v, got %v", nil, v) 145 | } 146 | } 147 | // check the second half of the items 148 | for i := len(nums) / 2; i < len(nums); i++ { 149 | v, ok := m.Get(nums[i]) 150 | if !ok || v != add(nums[i], 1) { 151 | t.Fatalf("expected %v, got %v", add(nums[i], 1), v) 152 | } 153 | } 154 | // try to delete again, make sure they don't exist 155 | for i := 0; i < len(nums)/2; i++ { 156 | v, ok := m.Delete(nums[i]) 157 | if ok || v != nil { 158 | t.Fatalf("expected %v, got %v", nil, v) 159 | } 160 | } 161 | if m.Len() != N/2 { 162 | t.Fatalf("expected %v, got %v", N/2, m.Len()) 163 | } 164 | m.Range(func(key keyT, value valueT) bool { 165 | if value != add(key, 1) { 166 | t.Fatalf("expected %v, got %v", add(key, 1), value) 167 | } 168 | return true 169 | }) 170 | var n int 171 | m.Range(func(key keyT, value valueT) bool { 172 | n++ 173 | return false 174 | }) 175 | if n != 1 { 176 | t.Fatalf("expected %v, got %v", 1, n) 177 | } 178 | for i := len(nums) / 2; i < len(nums); i++ { 179 | v, ok := m.Delete(nums[i]) 180 | if !ok || v != add(nums[i], 1) { 181 | t.Fatalf("expected %v, got %v", add(nums[i], 1), v) 182 | } 183 | } 184 | } 185 | } 186 | 187 | func TestBench(t *testing.T) { 188 | N, _ := strconv.ParseUint(os.Getenv("MAPBENCH"), 10, 64) 189 | if N == 0 { 190 | fmt.Printf("Enable benchmarks with MAPBENCH=1000000\n") 191 | return 192 | } 193 | nums := random(int(N), false) 194 | var pnums []valueT 195 | for i := range nums { 196 | pnums = append(pnums, valueT(&nums[i])) 197 | } 198 | fmt.Printf("\n## STRING KEYS\n\n") 199 | t.Run("Tidwall", func(t *testing.T) { 200 | testPerf(nums, pnums, "tidwall") 201 | }) 202 | t.Run("Stdlib", func(t *testing.T) { 203 | testPerf(nums, pnums, "stdlib") 204 | }) 205 | } 206 | 207 | func printItem(s string, size int, dir int) { 208 | for len(s) < size { 209 | if dir == -1 { 210 | s += " " 211 | } else { 212 | s = " " + s 213 | } 214 | } 215 | fmt.Printf("%s ", s) 216 | } 217 | 218 | func testPerf(nums []keyT, pnums []valueT, which keyT) { 219 | var ms1, ms2 runtime.MemStats 220 | initSize := 0 //len(nums) * 2 221 | defer func() { 222 | heapBytes := int(ms2.HeapAlloc - ms1.HeapAlloc) 223 | fmt.Printf("memory %13s bytes %19s/entry \n", 224 | commaize(heapBytes), commaize(heapBytes/len(nums))) 225 | fmt.Printf("\n") 226 | }() 227 | runtime.GC() 228 | time.Sleep(time.Millisecond * 100) 229 | runtime.ReadMemStats(&ms1) 230 | 231 | var setop, getop, delop func(int, int) 232 | var scnop func() 233 | switch which { 234 | case "stdlib": 235 | m := make(map[keyT]valueT, initSize) 236 | setop = func(i, _ int) { m[nums[i]] = pnums[i] } 237 | getop = func(i, _ int) { _ = m[nums[i]] } 238 | delop = func(i, _ int) { delete(m, nums[i]) } 239 | scnop = func() { 240 | for range m { 241 | } 242 | } 243 | case "tidwall": 244 | m := New(initSize) 245 | setop = func(i, _ int) { m.Set(nums[i], pnums[i]) } 246 | getop = func(i, _ int) { m.Get(nums[i]) } 247 | delop = func(i, _ int) { m.Delete(nums[i]) } 248 | scnop = func() { 249 | m.Range(func(key keyT, value valueT) bool { 250 | return true 251 | }) 252 | } 253 | } 254 | fmt.Printf("-- %s --", which) 255 | fmt.Printf("\n") 256 | 257 | ops := []func(int, int){setop, getop, setop, nil, delop} 258 | tags := []keyT{"set", "get", "reset", "scan", "delete"} 259 | for i := range ops { 260 | shuffle(nums) 261 | var na bool 262 | var n int 263 | start := time.Now() 264 | if tags[i] == "scan" { 265 | op := scnop 266 | if op == nil { 267 | na = true 268 | } else { 269 | n = 20 270 | for i := 0; i < n; i++ { 271 | op() 272 | } 273 | } 274 | } else { 275 | n = len(nums) 276 | for j := 0; j < n; j++ { 277 | ops[i](j, 1) 278 | } 279 | } 280 | dur := time.Since(start) 281 | if i == 0 { 282 | runtime.GC() 283 | time.Sleep(time.Millisecond * 100) 284 | runtime.ReadMemStats(&ms2) 285 | } 286 | printItem(tags[i], 9, -1) 287 | if na { 288 | printItem("-- unavailable --", 14, 1) 289 | } else { 290 | if n == -1 { 291 | printItem("unknown ops", 14, 1) 292 | } else { 293 | printItem(fmt.Sprintf("%s ops", commaize(n)), 14, 1) 294 | } 295 | printItem(fmt.Sprintf("%.0fms", dur.Seconds()*1000), 8, 1) 296 | if n != -1 { 297 | printItem(fmt.Sprintf("%s/sec", commaize(int(float64(n)/dur.Seconds()))), 18, 1) 298 | } 299 | } 300 | fmt.Printf("\n") 301 | } 302 | } 303 | 304 | func commaize(n int) string { 305 | s1, s2 := fmt.Sprintf("%d", n), "" 306 | for i, j := len(s1)-1, 0; i >= 0; i, j = i-1, j+1 { 307 | if j%3 == 0 && j != 0 { 308 | s2 = "," + s2 309 | } 310 | s2 = string(s1[i]) + s2 311 | } 312 | return s2 313 | } 314 | 315 | func TestHashDIB(t *testing.T) { 316 | var e entry 317 | e.setDIB(100) 318 | e.setHash(90000) 319 | if e.dib() != 100 { 320 | t.Fatalf("expected %v, got %v", 100, e.dib()) 321 | } 322 | if e.hash() != 90000 { 323 | t.Fatalf("expected %v, got %v", 90000, e.hash()) 324 | } 325 | } 326 | --------------------------------------------------------------------------------