├── README.md ├── LICENSE ├── example_test.go ├── consistent.go └── consistent_test.go /README.md: -------------------------------------------------------------------------------- 1 | consistent 2 | ========== 3 | 4 | Consistent hash package for Go. 5 | 6 | Installation 7 | ------------ 8 | 9 | go get stathat.com/c/consistent 10 | 11 | Examples 12 | -------- 13 | 14 | Look at the [godoc](http://godoc.org/stathat.com/c/consistent). 15 | 16 | Status 17 | ------ 18 | 19 | This package was extracted from production code powering [StatHat](http://www.stathat.com), 20 | so clearly we feel that it is production-ready, but it should still be considered 21 | experimental as other uses of it could reveal issues we aren't experiencing. 22 | 23 | Contact us 24 | ---------- 25 | 26 | We'd love to hear from you if you are using `consistent`. 27 | Get in touch: [@stathat](http://twitter.com/stathat) or [contact us here](http://www.stathat.com/docs/contact). 28 | 29 | About 30 | ----- 31 | 32 | Written by Patrick Crosby at [StatHat](http://www.stathat.com). Twitter: [@stathat](http://twitter.com/stathat) 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2012 Numerotron Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2012 Numerotron Inc. 2 | // Use of this source code is governed by an MIT-style license 3 | // that can be found in the LICENSE file. 4 | 5 | package consistent_test 6 | 7 | import ( 8 | "fmt" 9 | "log" 10 | "stathat.com/c/consistent" 11 | ) 12 | 13 | func ExampleNew() { 14 | c := consistent.New() 15 | c.Add("cacheA") 16 | c.Add("cacheB") 17 | c.Add("cacheC") 18 | users := []string{"user_mcnulty", "user_bunk", "user_omar", "user_bunny", "user_stringer"} 19 | for _, u := range users { 20 | server, err := c.Get(u) 21 | if err != nil { 22 | log.Fatal(err) 23 | } 24 | fmt.Printf("%s => %s\n", u, server) 25 | } 26 | // Output: 27 | // user_mcnulty => cacheA 28 | // user_bunk => cacheA 29 | // user_omar => cacheA 30 | // user_bunny => cacheC 31 | // user_stringer => cacheC 32 | } 33 | 34 | func ExampleAdd() { 35 | c := consistent.New() 36 | c.Add("cacheA") 37 | c.Add("cacheB") 38 | c.Add("cacheC") 39 | users := []string{"user_mcnulty", "user_bunk", "user_omar", "user_bunny", "user_stringer"} 40 | fmt.Println("initial state [A, B, C]") 41 | for _, u := range users { 42 | server, err := c.Get(u) 43 | if err != nil { 44 | log.Fatal(err) 45 | } 46 | fmt.Printf("%s => %s\n", u, server) 47 | } 48 | c.Add("cacheD") 49 | c.Add("cacheE") 50 | fmt.Println("\nwith cacheD, cacheE [A, B, C, D, E]") 51 | for _, u := range users { 52 | server, err := c.Get(u) 53 | if err != nil { 54 | log.Fatal(err) 55 | } 56 | fmt.Printf("%s => %s\n", u, server) 57 | } 58 | // Output: 59 | // initial state [A, B, C] 60 | // user_mcnulty => cacheA 61 | // user_bunk => cacheA 62 | // user_omar => cacheA 63 | // user_bunny => cacheC 64 | // user_stringer => cacheC 65 | // 66 | // with cacheD, cacheE [A, B, C, D, E] 67 | // user_mcnulty => cacheE 68 | // user_bunk => cacheA 69 | // user_omar => cacheA 70 | // user_bunny => cacheE 71 | // user_stringer => cacheE 72 | } 73 | 74 | func ExampleRemove() { 75 | c := consistent.New() 76 | c.Add("cacheA") 77 | c.Add("cacheB") 78 | c.Add("cacheC") 79 | users := []string{"user_mcnulty", "user_bunk", "user_omar", "user_bunny", "user_stringer"} 80 | fmt.Println("initial state [A, B, C]") 81 | for _, u := range users { 82 | server, err := c.Get(u) 83 | if err != nil { 84 | log.Fatal(err) 85 | } 86 | fmt.Printf("%s => %s\n", u, server) 87 | } 88 | c.Remove("cacheC") 89 | fmt.Println("\ncacheC removed [A, B]") 90 | for _, u := range users { 91 | server, err := c.Get(u) 92 | if err != nil { 93 | log.Fatal(err) 94 | } 95 | fmt.Printf("%s => %s\n", u, server) 96 | } 97 | // Output: 98 | // initial state [A, B, C] 99 | // user_mcnulty => cacheA 100 | // user_bunk => cacheA 101 | // user_omar => cacheA 102 | // user_bunny => cacheC 103 | // user_stringer => cacheC 104 | // 105 | // cacheC removed [A, B] 106 | // user_mcnulty => cacheA 107 | // user_bunk => cacheA 108 | // user_omar => cacheA 109 | // user_bunny => cacheB 110 | // user_stringer => cacheB 111 | } 112 | -------------------------------------------------------------------------------- /consistent.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2012 Numerotron Inc. 2 | // Use of this source code is governed by an MIT-style license 3 | // that can be found in the LICENSE file. 4 | 5 | // Package consistent provides a consistent hashing function. 6 | // 7 | // Consistent hashing is often used to distribute requests to a changing set of servers. For example, 8 | // say you have some cache servers cacheA, cacheB, and cacheC. You want to decide which cache server 9 | // to use to look up information on a user. 10 | // 11 | // You could use a typical hash table and hash the user id 12 | // to one of cacheA, cacheB, or cacheC. But with a typical hash table, if you add or remove a server, 13 | // almost all keys will get remapped to different results, which basically could bring your service 14 | // to a grinding halt while the caches get rebuilt. 15 | // 16 | // With a consistent hash, adding or removing a server drastically reduces the number of keys that 17 | // get remapped. 18 | // 19 | // Read more about consistent hashing on wikipedia: http://en.wikipedia.org/wiki/Consistent_hashing 20 | // 21 | package consistent // import "stathat.com/c/consistent" 22 | 23 | import ( 24 | "errors" 25 | "hash/crc32" 26 | "hash/fnv" 27 | "sort" 28 | "strconv" 29 | "sync" 30 | ) 31 | 32 | type uints []uint32 33 | 34 | // Len returns the length of the uints array. 35 | func (x uints) Len() int { return len(x) } 36 | 37 | // Less returns true if element i is less than element j. 38 | func (x uints) Less(i, j int) bool { return x[i] < x[j] } 39 | 40 | // Swap exchanges elements i and j. 41 | func (x uints) Swap(i, j int) { x[i], x[j] = x[j], x[i] } 42 | 43 | // ErrEmptyCircle is the error returned when trying to get an element when nothing has been added to hash. 44 | var ErrEmptyCircle = errors.New("empty circle") 45 | 46 | // Consistent holds the information about the members of the consistent hash circle. 47 | type Consistent struct { 48 | circle map[uint32]string 49 | members map[string]bool 50 | sortedHashes uints 51 | NumberOfReplicas int 52 | count int64 53 | scratch [64]byte 54 | UseFnv bool 55 | sync.RWMutex 56 | } 57 | 58 | // New creates a new Consistent object with a default setting of 20 replicas for each entry. 59 | // 60 | // To change the number of replicas, set NumberOfReplicas before adding entries. 61 | func New() *Consistent { 62 | c := new(Consistent) 63 | c.NumberOfReplicas = 20 64 | c.circle = make(map[uint32]string) 65 | c.members = make(map[string]bool) 66 | return c 67 | } 68 | 69 | // eltKey generates a string key for an element with an index. 70 | func (c *Consistent) eltKey(elt string, idx int) string { 71 | // return elt + "|" + strconv.Itoa(idx) 72 | return strconv.Itoa(idx) + elt 73 | } 74 | 75 | // Add inserts a string element in the consistent hash. 76 | func (c *Consistent) Add(elt string) { 77 | c.Lock() 78 | defer c.Unlock() 79 | c.add(elt) 80 | } 81 | 82 | // need c.Lock() before calling 83 | func (c *Consistent) add(elt string) { 84 | for i := 0; i < c.NumberOfReplicas; i++ { 85 | c.circle[c.hashKey(c.eltKey(elt, i))] = elt 86 | } 87 | c.members[elt] = true 88 | c.updateSortedHashes() 89 | c.count++ 90 | } 91 | 92 | // Remove removes an element from the hash. 93 | func (c *Consistent) Remove(elt string) { 94 | c.Lock() 95 | defer c.Unlock() 96 | c.remove(elt) 97 | } 98 | 99 | // need c.Lock() before calling 100 | func (c *Consistent) remove(elt string) { 101 | for i := 0; i < c.NumberOfReplicas; i++ { 102 | delete(c.circle, c.hashKey(c.eltKey(elt, i))) 103 | } 104 | delete(c.members, elt) 105 | c.updateSortedHashes() 106 | c.count-- 107 | } 108 | 109 | // Set sets all the elements in the hash. If there are existing elements not 110 | // present in elts, they will be removed. 111 | func (c *Consistent) Set(elts []string) { 112 | c.Lock() 113 | defer c.Unlock() 114 | for k := range c.members { 115 | found := false 116 | for _, v := range elts { 117 | if k == v { 118 | found = true 119 | break 120 | } 121 | } 122 | if !found { 123 | c.remove(k) 124 | } 125 | } 126 | for _, v := range elts { 127 | _, exists := c.members[v] 128 | if exists { 129 | continue 130 | } 131 | c.add(v) 132 | } 133 | } 134 | 135 | func (c *Consistent) Members() []string { 136 | c.RLock() 137 | defer c.RUnlock() 138 | var m []string 139 | for k := range c.members { 140 | m = append(m, k) 141 | } 142 | return m 143 | } 144 | 145 | // Get returns an element close to where name hashes to in the circle. 146 | func (c *Consistent) Get(name string) (string, error) { 147 | c.RLock() 148 | defer c.RUnlock() 149 | if len(c.circle) == 0 { 150 | return "", ErrEmptyCircle 151 | } 152 | key := c.hashKey(name) 153 | i := c.search(key) 154 | return c.circle[c.sortedHashes[i]], nil 155 | } 156 | 157 | func (c *Consistent) search(key uint32) (i int) { 158 | f := func(x int) bool { 159 | return c.sortedHashes[x] > key 160 | } 161 | i = sort.Search(len(c.sortedHashes), f) 162 | if i >= len(c.sortedHashes) { 163 | i = 0 164 | } 165 | return 166 | } 167 | 168 | // GetTwo returns the two closest distinct elements to the name input in the circle. 169 | func (c *Consistent) GetTwo(name string) (string, string, error) { 170 | c.RLock() 171 | defer c.RUnlock() 172 | if len(c.circle) == 0 { 173 | return "", "", ErrEmptyCircle 174 | } 175 | key := c.hashKey(name) 176 | i := c.search(key) 177 | a := c.circle[c.sortedHashes[i]] 178 | 179 | if c.count == 1 { 180 | return a, "", nil 181 | } 182 | 183 | start := i 184 | var b string 185 | for i = start + 1; i != start; i++ { 186 | if i >= len(c.sortedHashes) { 187 | i = 0 188 | } 189 | b = c.circle[c.sortedHashes[i]] 190 | if b != a { 191 | break 192 | } 193 | } 194 | return a, b, nil 195 | } 196 | 197 | // GetN returns the N closest distinct elements to the name input in the circle. 198 | func (c *Consistent) GetN(name string, n int) ([]string, error) { 199 | c.RLock() 200 | defer c.RUnlock() 201 | 202 | if len(c.circle) == 0 { 203 | return nil, ErrEmptyCircle 204 | } 205 | 206 | if c.count < int64(n) { 207 | n = int(c.count) 208 | } 209 | 210 | var ( 211 | key = c.hashKey(name) 212 | i = c.search(key) 213 | start = i 214 | res = make([]string, 0, n) 215 | elem = c.circle[c.sortedHashes[i]] 216 | ) 217 | 218 | res = append(res, elem) 219 | 220 | if len(res) == n { 221 | return res, nil 222 | } 223 | 224 | for i = start + 1; i != start; i++ { 225 | if i >= len(c.sortedHashes) { 226 | i = 0 227 | } 228 | elem = c.circle[c.sortedHashes[i]] 229 | if !sliceContainsMember(res, elem) { 230 | res = append(res, elem) 231 | } 232 | if len(res) == n { 233 | break 234 | } 235 | } 236 | 237 | return res, nil 238 | } 239 | 240 | func (c *Consistent) hashKey(key string) uint32 { 241 | if c.UseFnv { 242 | return c.hashKeyFnv(key) 243 | } 244 | return c.hashKeyCRC32(key) 245 | } 246 | 247 | func (c *Consistent) hashKeyCRC32(key string) uint32 { 248 | if len(key) < 64 { 249 | var scratch [64]byte 250 | copy(scratch[:], key) 251 | return crc32.ChecksumIEEE(scratch[:len(key)]) 252 | } 253 | return crc32.ChecksumIEEE([]byte(key)) 254 | } 255 | 256 | func (c *Consistent) hashKeyFnv(key string) uint32 { 257 | h := fnv.New32a() 258 | h.Write([]byte(key)) 259 | return h.Sum32() 260 | } 261 | 262 | func (c *Consistent) updateSortedHashes() { 263 | hashes := c.sortedHashes[:0] 264 | //reallocate if we're holding on to too much (1/4th) 265 | if cap(c.sortedHashes)/(c.NumberOfReplicas*4) > len(c.circle) { 266 | hashes = nil 267 | } 268 | for k := range c.circle { 269 | hashes = append(hashes, k) 270 | } 271 | sort.Sort(hashes) 272 | c.sortedHashes = hashes 273 | } 274 | 275 | func sliceContainsMember(set []string, member string) bool { 276 | for _, m := range set { 277 | if m == member { 278 | return true 279 | } 280 | } 281 | return false 282 | } 283 | -------------------------------------------------------------------------------- /consistent_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2012-2014 Numerotron Inc. 2 | // Use of this source code is governed by an MIT-style license 3 | // that can be found in the LICENSE file. 4 | 5 | package consistent 6 | 7 | import ( 8 | "bufio" 9 | "encoding/base64" 10 | "math/rand" 11 | "os" 12 | "runtime" 13 | "sort" 14 | "strconv" 15 | "sync" 16 | "testing" 17 | "testing/quick" 18 | "time" 19 | ) 20 | 21 | func checkNum(num, expected int, t *testing.T) { 22 | if num != expected { 23 | t.Errorf("got %d, expected %d", num, expected) 24 | } 25 | } 26 | 27 | func TestNew(t *testing.T) { 28 | x := New() 29 | if x == nil { 30 | t.Errorf("expected obj") 31 | } 32 | checkNum(x.NumberOfReplicas, 20, t) 33 | } 34 | 35 | func TestAdd(t *testing.T) { 36 | x := New() 37 | x.Add("abcdefg") 38 | checkNum(len(x.circle), 20, t) 39 | checkNum(len(x.sortedHashes), 20, t) 40 | if sort.IsSorted(x.sortedHashes) == false { 41 | t.Errorf("expected sorted hashes to be sorted") 42 | } 43 | x.Add("qwer") 44 | checkNum(len(x.circle), 40, t) 45 | checkNum(len(x.sortedHashes), 40, t) 46 | if sort.IsSorted(x.sortedHashes) == false { 47 | t.Errorf("expected sorted hashes to be sorted") 48 | } 49 | } 50 | 51 | func TestRemove(t *testing.T) { 52 | x := New() 53 | x.Add("abcdefg") 54 | x.Remove("abcdefg") 55 | checkNum(len(x.circle), 0, t) 56 | checkNum(len(x.sortedHashes), 0, t) 57 | } 58 | 59 | func TestRemoveNonExisting(t *testing.T) { 60 | x := New() 61 | x.Add("abcdefg") 62 | x.Remove("abcdefghijk") 63 | checkNum(len(x.circle), 20, t) 64 | } 65 | 66 | func TestGetEmpty(t *testing.T) { 67 | x := New() 68 | _, err := x.Get("asdfsadfsadf") 69 | if err == nil { 70 | t.Errorf("expected error") 71 | } 72 | if err != ErrEmptyCircle { 73 | t.Errorf("expected empty circle error") 74 | } 75 | } 76 | 77 | func TestGetSingle(t *testing.T) { 78 | x := New() 79 | x.Add("abcdefg") 80 | f := func(s string) bool { 81 | y, err := x.Get(s) 82 | if err != nil { 83 | t.Logf("error: %q", err) 84 | return false 85 | } 86 | t.Logf("s = %q, y = %q", s, y) 87 | return y == "abcdefg" 88 | } 89 | if err := quick.Check(f, nil); err != nil { 90 | t.Fatal(err) 91 | } 92 | } 93 | 94 | type gtest struct { 95 | in string 96 | out string 97 | } 98 | 99 | var gmtests = []gtest{ 100 | {"ggg", "abcdefg"}, 101 | {"hhh", "opqrstu"}, 102 | {"iiiii", "hijklmn"}, 103 | } 104 | 105 | func TestGetMultiple(t *testing.T) { 106 | x := New() 107 | x.Add("abcdefg") 108 | x.Add("hijklmn") 109 | x.Add("opqrstu") 110 | for i, v := range gmtests { 111 | result, err := x.Get(v.in) 112 | if err != nil { 113 | t.Fatal(err) 114 | } 115 | if result != v.out { 116 | t.Errorf("%d. got %q, expected %q", i, result, v.out) 117 | } 118 | } 119 | } 120 | 121 | func TestGetMultipleQuick(t *testing.T) { 122 | x := New() 123 | x.Add("abcdefg") 124 | x.Add("hijklmn") 125 | x.Add("opqrstu") 126 | f := func(s string) bool { 127 | y, err := x.Get(s) 128 | if err != nil { 129 | t.Logf("error: %q", err) 130 | return false 131 | } 132 | t.Logf("s = %q, y = %q", s, y) 133 | return y == "abcdefg" || y == "hijklmn" || y == "opqrstu" 134 | } 135 | if err := quick.Check(f, nil); err != nil { 136 | t.Fatal(err) 137 | } 138 | } 139 | 140 | var rtestsBefore = []gtest{ 141 | {"ggg", "abcdefg"}, 142 | {"hhh", "opqrstu"}, 143 | {"iiiii", "hijklmn"}, 144 | } 145 | 146 | var rtestsAfter = []gtest{ 147 | {"ggg", "abcdefg"}, 148 | {"hhh", "opqrstu"}, 149 | {"iiiii", "opqrstu"}, 150 | } 151 | 152 | func TestGetMultipleRemove(t *testing.T) { 153 | x := New() 154 | x.Add("abcdefg") 155 | x.Add("hijklmn") 156 | x.Add("opqrstu") 157 | for i, v := range rtestsBefore { 158 | result, err := x.Get(v.in) 159 | if err != nil { 160 | t.Fatal(err) 161 | } 162 | if result != v.out { 163 | t.Errorf("%d. got %q, expected %q before rm", i, result, v.out) 164 | } 165 | } 166 | x.Remove("hijklmn") 167 | for i, v := range rtestsAfter { 168 | result, err := x.Get(v.in) 169 | if err != nil { 170 | t.Fatal(err) 171 | } 172 | if result != v.out { 173 | t.Errorf("%d. got %q, expected %q after rm", i, result, v.out) 174 | } 175 | } 176 | } 177 | 178 | func TestGetMultipleRemoveQuick(t *testing.T) { 179 | x := New() 180 | x.Add("abcdefg") 181 | x.Add("hijklmn") 182 | x.Add("opqrstu") 183 | x.Remove("opqrstu") 184 | f := func(s string) bool { 185 | y, err := x.Get(s) 186 | if err != nil { 187 | t.Logf("error: %q", err) 188 | return false 189 | } 190 | t.Logf("s = %q, y = %q", s, y) 191 | return y == "abcdefg" || y == "hijklmn" 192 | } 193 | if err := quick.Check(f, nil); err != nil { 194 | t.Fatal(err) 195 | } 196 | } 197 | 198 | func TestGetTwo(t *testing.T) { 199 | x := New() 200 | x.Add("abcdefg") 201 | x.Add("hijklmn") 202 | x.Add("opqrstu") 203 | a, b, err := x.GetTwo("99999999") 204 | if err != nil { 205 | t.Fatal(err) 206 | } 207 | if a == b { 208 | t.Errorf("a shouldn't equal b") 209 | } 210 | if a != "abcdefg" { 211 | t.Errorf("wrong a: %q", a) 212 | } 213 | if b != "hijklmn" { 214 | t.Errorf("wrong b: %q", b) 215 | } 216 | } 217 | 218 | func TestGetTwoQuick(t *testing.T) { 219 | x := New() 220 | x.Add("abcdefg") 221 | x.Add("hijklmn") 222 | x.Add("opqrstu") 223 | f := func(s string) bool { 224 | a, b, err := x.GetTwo(s) 225 | if err != nil { 226 | t.Logf("error: %q", err) 227 | return false 228 | } 229 | if a == b { 230 | t.Logf("a == b") 231 | return false 232 | } 233 | if a != "abcdefg" && a != "hijklmn" && a != "opqrstu" { 234 | t.Logf("invalid a: %q", a) 235 | return false 236 | } 237 | 238 | if b != "abcdefg" && b != "hijklmn" && b != "opqrstu" { 239 | t.Logf("invalid b: %q", b) 240 | return false 241 | } 242 | return true 243 | } 244 | if err := quick.Check(f, nil); err != nil { 245 | t.Fatal(err) 246 | } 247 | } 248 | 249 | func TestGetTwoOnlyTwoQuick(t *testing.T) { 250 | x := New() 251 | x.Add("abcdefg") 252 | x.Add("hijklmn") 253 | f := func(s string) bool { 254 | a, b, err := x.GetTwo(s) 255 | if err != nil { 256 | t.Logf("error: %q", err) 257 | return false 258 | } 259 | if a == b { 260 | t.Logf("a == b") 261 | return false 262 | } 263 | if a != "abcdefg" && a != "hijklmn" { 264 | t.Logf("invalid a: %q", a) 265 | return false 266 | } 267 | 268 | if b != "abcdefg" && b != "hijklmn" { 269 | t.Logf("invalid b: %q", b) 270 | return false 271 | } 272 | return true 273 | } 274 | if err := quick.Check(f, nil); err != nil { 275 | t.Fatal(err) 276 | } 277 | } 278 | 279 | func TestGetTwoOnlyOneInCircle(t *testing.T) { 280 | x := New() 281 | x.Add("abcdefg") 282 | a, b, err := x.GetTwo("99999999") 283 | if err != nil { 284 | t.Fatal(err) 285 | } 286 | if a == b { 287 | t.Errorf("a shouldn't equal b") 288 | } 289 | if a != "abcdefg" { 290 | t.Errorf("wrong a: %q", a) 291 | } 292 | if b != "" { 293 | t.Errorf("wrong b: %q", b) 294 | } 295 | } 296 | 297 | func TestGetN(t *testing.T) { 298 | x := New() 299 | x.Add("abcdefg") 300 | x.Add("hijklmn") 301 | x.Add("opqrstu") 302 | members, err := x.GetN("9999999", 3) 303 | if err != nil { 304 | t.Fatal(err) 305 | } 306 | if len(members) != 3 { 307 | t.Errorf("expected 3 members instead of %d", len(members)) 308 | } 309 | if members[0] != "opqrstu" { 310 | t.Errorf("wrong members[0]: %q", members[0]) 311 | } 312 | if members[1] != "abcdefg" { 313 | t.Errorf("wrong members[1]: %q", members[1]) 314 | } 315 | if members[2] != "hijklmn" { 316 | t.Errorf("wrong members[2]: %q", members[2]) 317 | } 318 | } 319 | 320 | func TestGetNLess(t *testing.T) { 321 | x := New() 322 | x.Add("abcdefg") 323 | x.Add("hijklmn") 324 | x.Add("opqrstu") 325 | members, err := x.GetN("99999999", 2) 326 | if err != nil { 327 | t.Fatal(err) 328 | } 329 | if len(members) != 2 { 330 | t.Errorf("expected 2 members instead of %d", len(members)) 331 | } 332 | if members[0] != "abcdefg" { 333 | t.Errorf("wrong members[0]: %q", members[0]) 334 | } 335 | if members[1] != "hijklmn" { 336 | t.Errorf("wrong members[1]: %q", members[1]) 337 | } 338 | } 339 | 340 | func TestGetNMore(t *testing.T) { 341 | x := New() 342 | x.Add("abcdefg") 343 | x.Add("hijklmn") 344 | x.Add("opqrstu") 345 | members, err := x.GetN("9999999", 5) 346 | if err != nil { 347 | t.Fatal(err) 348 | } 349 | if len(members) != 3 { 350 | t.Errorf("expected 3 members instead of %d", len(members)) 351 | } 352 | if members[0] != "opqrstu" { 353 | t.Errorf("wrong members[0]: %q", members[0]) 354 | } 355 | if members[1] != "abcdefg" { 356 | t.Errorf("wrong members[1]: %q", members[1]) 357 | } 358 | if members[2] != "hijklmn" { 359 | t.Errorf("wrong members[2]: %q", members[2]) 360 | } 361 | } 362 | 363 | func TestGetNQuick(t *testing.T) { 364 | x := New() 365 | x.Add("abcdefg") 366 | x.Add("hijklmn") 367 | x.Add("opqrstu") 368 | f := func(s string) bool { 369 | members, err := x.GetN(s, 3) 370 | if err != nil { 371 | t.Logf("error: %q", err) 372 | return false 373 | } 374 | if len(members) != 3 { 375 | t.Logf("expected 3 members instead of %d", len(members)) 376 | return false 377 | } 378 | set := make(map[string]bool, 4) 379 | for _, member := range members { 380 | if set[member] { 381 | t.Logf("duplicate error") 382 | return false 383 | } 384 | set[member] = true 385 | if member != "abcdefg" && member != "hijklmn" && member != "opqrstu" { 386 | t.Logf("invalid member: %q", member) 387 | return false 388 | } 389 | } 390 | return true 391 | } 392 | if err := quick.Check(f, nil); err != nil { 393 | t.Fatal(err) 394 | } 395 | } 396 | 397 | func TestGetNLessQuick(t *testing.T) { 398 | x := New() 399 | x.Add("abcdefg") 400 | x.Add("hijklmn") 401 | x.Add("opqrstu") 402 | f := func(s string) bool { 403 | members, err := x.GetN(s, 2) 404 | if err != nil { 405 | t.Logf("error: %q", err) 406 | return false 407 | } 408 | if len(members) != 2 { 409 | t.Logf("expected 2 members instead of %d", len(members)) 410 | return false 411 | } 412 | set := make(map[string]bool, 4) 413 | for _, member := range members { 414 | if set[member] { 415 | t.Logf("duplicate error") 416 | return false 417 | } 418 | set[member] = true 419 | if member != "abcdefg" && member != "hijklmn" && member != "opqrstu" { 420 | t.Logf("invalid member: %q", member) 421 | return false 422 | } 423 | } 424 | return true 425 | } 426 | if err := quick.Check(f, nil); err != nil { 427 | t.Fatal(err) 428 | } 429 | } 430 | 431 | func TestGetNMoreQuick(t *testing.T) { 432 | x := New() 433 | x.Add("abcdefg") 434 | x.Add("hijklmn") 435 | x.Add("opqrstu") 436 | f := func(s string) bool { 437 | members, err := x.GetN(s, 5) 438 | if err != nil { 439 | t.Logf("error: %q", err) 440 | return false 441 | } 442 | if len(members) != 3 { 443 | t.Logf("expected 3 members instead of %d", len(members)) 444 | return false 445 | } 446 | set := make(map[string]bool, 4) 447 | for _, member := range members { 448 | if set[member] { 449 | t.Logf("duplicate error") 450 | return false 451 | } 452 | set[member] = true 453 | if member != "abcdefg" && member != "hijklmn" && member != "opqrstu" { 454 | t.Logf("invalid member: %q", member) 455 | return false 456 | } 457 | } 458 | return true 459 | } 460 | if err := quick.Check(f, nil); err != nil { 461 | t.Fatal(err) 462 | } 463 | } 464 | 465 | func TestSet(t *testing.T) { 466 | x := New() 467 | x.Add("abc") 468 | x.Add("def") 469 | x.Add("ghi") 470 | x.Set([]string{"jkl", "mno"}) 471 | if x.count != 2 { 472 | t.Errorf("expected 2 elts, got %d", x.count) 473 | } 474 | a, b, err := x.GetTwo("qwerqwerwqer") 475 | if err != nil { 476 | t.Fatal(err) 477 | } 478 | if a != "jkl" && a != "mno" { 479 | t.Errorf("expected jkl or mno, got %s", a) 480 | } 481 | if b != "jkl" && b != "mno" { 482 | t.Errorf("expected jkl or mno, got %s", b) 483 | } 484 | if a == b { 485 | t.Errorf("expected a != b, they were both %s", a) 486 | } 487 | x.Set([]string{"pqr", "mno"}) 488 | if x.count != 2 { 489 | t.Errorf("expected 2 elts, got %d", x.count) 490 | } 491 | a, b, err = x.GetTwo("qwerqwerwqer") 492 | if err != nil { 493 | t.Fatal(err) 494 | } 495 | if a != "pqr" && a != "mno" { 496 | t.Errorf("expected jkl or mno, got %s", a) 497 | } 498 | if b != "pqr" && b != "mno" { 499 | t.Errorf("expected jkl or mno, got %s", b) 500 | } 501 | if a == b { 502 | t.Errorf("expected a != b, they were both %s", a) 503 | } 504 | x.Set([]string{"pqr", "mno"}) 505 | if x.count != 2 { 506 | t.Errorf("expected 2 elts, got %d", x.count) 507 | } 508 | a, b, err = x.GetTwo("qwerqwerwqer") 509 | if err != nil { 510 | t.Fatal(err) 511 | } 512 | if a != "pqr" && a != "mno" { 513 | t.Errorf("expected jkl or mno, got %s", a) 514 | } 515 | if b != "pqr" && b != "mno" { 516 | t.Errorf("expected jkl or mno, got %s", b) 517 | } 518 | if a == b { 519 | t.Errorf("expected a != b, they were both %s", a) 520 | } 521 | } 522 | 523 | // allocBytes returns the number of bytes allocated by invoking f. 524 | func allocBytes(f func()) uint64 { 525 | var stats runtime.MemStats 526 | runtime.ReadMemStats(&stats) 527 | t := stats.TotalAlloc 528 | f() 529 | runtime.ReadMemStats(&stats) 530 | return stats.TotalAlloc - t 531 | } 532 | 533 | func mallocNum(f func()) uint64 { 534 | var stats runtime.MemStats 535 | runtime.ReadMemStats(&stats) 536 | t := stats.Mallocs 537 | f() 538 | runtime.ReadMemStats(&stats) 539 | return stats.Mallocs - t 540 | } 541 | 542 | func BenchmarkAllocations(b *testing.B) { 543 | x := New() 544 | x.Add("stays") 545 | b.ResetTimer() 546 | allocSize := allocBytes(func() { 547 | for i := 0; i < b.N; i++ { 548 | x.Add("Foo") 549 | x.Remove("Foo") 550 | } 551 | }) 552 | b.Logf("%d: Allocated %d bytes (%.2fx)", b.N, allocSize, float64(allocSize)/float64(b.N)) 553 | } 554 | 555 | func BenchmarkMalloc(b *testing.B) { 556 | x := New() 557 | x.Add("stays") 558 | b.ResetTimer() 559 | mallocs := mallocNum(func() { 560 | for i := 0; i < b.N; i++ { 561 | x.Add("Foo") 562 | x.Remove("Foo") 563 | } 564 | }) 565 | b.Logf("%d: Mallocd %d times (%.2fx)", b.N, mallocs, float64(mallocs)/float64(b.N)) 566 | } 567 | 568 | func BenchmarkCycle(b *testing.B) { 569 | x := New() 570 | x.Add("nothing") 571 | b.ResetTimer() 572 | for i := 0; i < b.N; i++ { 573 | x.Add("foo" + strconv.Itoa(i)) 574 | x.Remove("foo" + strconv.Itoa(i)) 575 | } 576 | } 577 | 578 | func BenchmarkCycleLarge(b *testing.B) { 579 | x := New() 580 | for i := 0; i < 10; i++ { 581 | x.Add("start" + strconv.Itoa(i)) 582 | } 583 | b.ResetTimer() 584 | for i := 0; i < b.N; i++ { 585 | x.Add("foo" + strconv.Itoa(i)) 586 | x.Remove("foo" + strconv.Itoa(i)) 587 | } 588 | } 589 | 590 | func BenchmarkGet(b *testing.B) { 591 | x := New() 592 | x.Add("nothing") 593 | b.ResetTimer() 594 | for i := 0; i < b.N; i++ { 595 | x.Get("nothing") 596 | } 597 | } 598 | 599 | func BenchmarkGetLarge(b *testing.B) { 600 | x := New() 601 | for i := 0; i < 10; i++ { 602 | x.Add("start" + strconv.Itoa(i)) 603 | } 604 | b.ResetTimer() 605 | for i := 0; i < b.N; i++ { 606 | x.Get("nothing") 607 | } 608 | } 609 | 610 | func BenchmarkGetN(b *testing.B) { 611 | x := New() 612 | x.Add("nothing") 613 | b.ResetTimer() 614 | for i := 0; i < b.N; i++ { 615 | x.GetN("nothing", 3) 616 | } 617 | } 618 | 619 | func BenchmarkGetNLarge(b *testing.B) { 620 | x := New() 621 | for i := 0; i < 10; i++ { 622 | x.Add("start" + strconv.Itoa(i)) 623 | } 624 | b.ResetTimer() 625 | for i := 0; i < b.N; i++ { 626 | x.GetN("nothing", 3) 627 | } 628 | } 629 | 630 | func BenchmarkGetTwo(b *testing.B) { 631 | x := New() 632 | x.Add("nothing") 633 | b.ResetTimer() 634 | for i := 0; i < b.N; i++ { 635 | x.GetTwo("nothing") 636 | } 637 | } 638 | 639 | func BenchmarkGetTwoLarge(b *testing.B) { 640 | x := New() 641 | for i := 0; i < 10; i++ { 642 | x.Add("start" + strconv.Itoa(i)) 643 | } 644 | b.ResetTimer() 645 | for i := 0; i < b.N; i++ { 646 | x.GetTwo("nothing") 647 | } 648 | } 649 | 650 | // from @edsrzf on github: 651 | func TestAddCollision(t *testing.T) { 652 | // These two strings produce several crc32 collisions after "|i" is 653 | // appended added by Consistent.eltKey. 654 | const s1 = "abear" 655 | const s2 = "solidiform" 656 | x := New() 657 | x.Add(s1) 658 | x.Add(s2) 659 | elt1, err := x.Get("abear") 660 | if err != nil { 661 | t.Fatal("unexpected error:", err) 662 | } 663 | 664 | y := New() 665 | // add elements in opposite order 666 | y.Add(s2) 667 | y.Add(s1) 668 | elt2, err := y.Get(s1) 669 | if err != nil { 670 | t.Fatal("unexpected error:", err) 671 | } 672 | 673 | if elt1 != elt2 { 674 | t.Error(elt1, "and", elt2, "should be equal") 675 | } 676 | } 677 | 678 | // inspired by @or-else on github 679 | func TestCollisionsCRC(t *testing.T) { 680 | t.SkipNow() 681 | c := New() 682 | f, err := os.Open("/usr/share/dict/words") 683 | if err != nil { 684 | t.Fatal(err) 685 | } 686 | defer f.Close() 687 | found := make(map[uint32]string) 688 | scanner := bufio.NewScanner(f) 689 | count := 0 690 | for scanner.Scan() { 691 | word := scanner.Text() 692 | for i := 0; i < c.NumberOfReplicas; i++ { 693 | ekey := c.eltKey(word, i) 694 | // ekey := word + "|" + strconv.Itoa(i) 695 | k := c.hashKey(ekey) 696 | exist, ok := found[k] 697 | if ok { 698 | t.Logf("found collision: %s, %s", ekey, exist) 699 | count++ 700 | } else { 701 | found[k] = ekey 702 | } 703 | } 704 | } 705 | t.Logf("number of collisions: %d", count) 706 | } 707 | 708 | func TestConcurrentGetSet(t *testing.T) { 709 | x := New() 710 | x.Set([]string{"abc", "def", "ghi", "jkl", "mno"}) 711 | 712 | var wg sync.WaitGroup 713 | for i := 0; i < 10; i++ { 714 | wg.Add(1) 715 | go func() { 716 | for i := 0; i < 1000; i++ { 717 | x.Set([]string{"abc", "def", "ghi", "jkl", "mno"}) 718 | time.Sleep(time.Duration(rand.Intn(10)) * time.Millisecond) 719 | x.Set([]string{"pqr", "stu", "vwx"}) 720 | time.Sleep(time.Duration(rand.Intn(10)) * time.Millisecond) 721 | } 722 | wg.Done() 723 | }() 724 | } 725 | 726 | for i := 0; i < 100; i++ { 727 | wg.Add(1) 728 | go func() { 729 | for i := 0; i < 1000; i++ { 730 | a, err := x.Get("xxxxxxx") 731 | if err != nil { 732 | t.Error(err) 733 | } 734 | if a != "def" && a != "vwx" { 735 | t.Errorf("got %s, expected abc", a) 736 | } 737 | time.Sleep(time.Duration(rand.Intn(10)) * time.Millisecond) 738 | } 739 | wg.Done() 740 | }() 741 | } 742 | wg.Wait() 743 | } 744 | 745 | func TestDistributionFnv(t *testing.T) { 746 | x := New() 747 | x.UseFnv = true 748 | x.Add("abcdefg") 749 | x.Add("hijklmn") 750 | x.Add("opqrstu") 751 | dist := make(map[string]int) 752 | g := make([]byte, 12) 753 | for i := 0; i < 10000; i++ { 754 | _, err := rand.Read(g) 755 | if err != nil { 756 | t.Fatal(err) 757 | } 758 | s := base64.StdEncoding.EncodeToString(g) 759 | r, err := x.Get(s) 760 | if err != nil { 761 | t.Fatal(err) 762 | } 763 | dist[r] = dist[r] + 1 764 | } 765 | for k, v := range dist { 766 | t.Logf("%s: %d", k, v) 767 | } 768 | } 769 | 770 | func TestDistributionCRC(t *testing.T) { 771 | x := New() 772 | x.Add("abcdefg") 773 | x.Add("hijklmn") 774 | x.Add("opqrstu") 775 | dist := make(map[string]int) 776 | g := make([]byte, 12) 777 | for i := 0; i < 10000; i++ { 778 | _, err := rand.Read(g) 779 | if err != nil { 780 | t.Fatal(err) 781 | } 782 | s := base64.StdEncoding.EncodeToString(g) 783 | r, err := x.Get(s) 784 | if err != nil { 785 | t.Fatal(err) 786 | } 787 | dist[r] = dist[r] + 1 788 | } 789 | for k, v := range dist { 790 | t.Logf("%s: %d", k, v) 791 | } 792 | } 793 | --------------------------------------------------------------------------------