├── .travis.yml ├── perm ├── feistel_test.go └── feistel.go ├── LICENSE ├── safe ├── randmap.go └── randmap_test.go ├── README.md ├── randmap.go ├── runtime_go1.7.go └── randmap_test.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | os: 4 | - linux 5 | - osx 6 | 7 | go: 8 | - 1.7 9 | - master 10 | 11 | script: go test -v -bench=. ./... 12 | 13 | sudo: false 14 | 15 | notifications: 16 | email: false 17 | -------------------------------------------------------------------------------- /perm/feistel_test.go: -------------------------------------------------------------------------------- 1 | package perm 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | ) 7 | 8 | func TestGenerator(t *testing.T) { 9 | const numElems = 50 10 | const iters = 10000 11 | counts := make([][]int, numElems) 12 | for i := range counts { 13 | counts[i] = make([]int, numElems) 14 | } 15 | for i := 0; i < iters; i++ { 16 | g := NewGenerator(numElems, rand.Uint32()) 17 | for j := 0; ; j++ { 18 | u, ok := g.Next() 19 | if !ok { 20 | break 21 | } 22 | // u appeared at index j 23 | counts[u][j]++ 24 | } 25 | } 26 | 27 | // each key should have appeared at each index about iters/numElems times 28 | for k, cs := range counts { 29 | for i, c := range cs { 30 | if (iters/numElems)/2 > c || c > (iters/numElems)*2 { 31 | t.Errorf("suspicious count for key %v index %v: expected %v-%v, got %v", k, i, (iters/numElems)/2, (iters/numElems)*2, c) 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Luke Champine 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /perm/feistel.go: -------------------------------------------------------------------------------- 1 | // Package perm provides a generator object which can be used to iterate 2 | // through a permutation of a given range in constant space. 3 | // 4 | // The current implementation uses a Feistel network, with blake2b as the 5 | // round function. Note that since the entropy seeding the round function is 6 | // typically far smaller than the full permutation space, the generator cannot 7 | // enumerate the full space in such cases. For example, even a 128-bit key can 8 | // only enumerate the permutation space of 34 elements, since 2^128 < 35!. 9 | // This is a known limitation of Feistel networks. 10 | // 11 | // For small n, it may be more efficient to enumerate [0,n) and permute the 12 | // values in-place using the well-known Fisher-Yates algorithm (this is what 13 | // math/rand.Perm uses). For convenience, such a function is provided in this 14 | // package as well. 15 | package perm 16 | 17 | import ( 18 | "hash" 19 | "math/rand" 20 | 21 | "github.com/minio/blake2b-simd" 22 | ) 23 | 24 | // FisherYates returns a pseudorandom permutation of the integers [0,n). 25 | func FisherYates(n int) []int { return rand.Perm(n) } 26 | 27 | type feistelGenerator struct { 28 | nextPow4 uint32 29 | halfNumBits uint32 30 | leftMask uint32 31 | rightMask uint32 32 | seed uint32 33 | numElems uint32 34 | i uint32 35 | 36 | hash hash.Hash 37 | arena [32]byte 38 | } 39 | 40 | // NewGenerator returns a new Feistel network-based permutation generator. 41 | func NewGenerator(numElems, seed uint32) *feistelGenerator { 42 | nextPow4 := uint32(4) 43 | log4 := uint32(1) 44 | for nextPow4 < numElems { 45 | nextPow4 *= 4 46 | log4++ 47 | } 48 | 49 | return &feistelGenerator{ 50 | nextPow4: nextPow4, 51 | halfNumBits: log4, 52 | leftMask: ((uint32(1) << log4) - 1) << log4, // e.g. 0xFFFF0000 53 | rightMask: (uint32(1) << log4) - 1, // e.g. 0x0000FFFF 54 | seed: seed, 55 | numElems: numElems, 56 | 57 | hash: blake2b.New256(), 58 | } 59 | } 60 | 61 | func (f *feistelGenerator) Next() (uint32, bool) { 62 | for f.i < f.nextPow4 { 63 | n := f.encryptIndex(f.i) 64 | f.i++ 65 | if n < f.numElems { 66 | return n, true 67 | } 68 | } 69 | return 0, false 70 | } 71 | 72 | func (f *feistelGenerator) encryptIndex(index uint32) uint32 { 73 | // split index into left and right bits 74 | left := (index & f.leftMask) >> f.halfNumBits 75 | right := (index & f.rightMask) 76 | 77 | // do 4 Feistel rounds 78 | for i := uint32(0); i < 4; i++ { 79 | ki := f.seed + i 80 | left, right = right, left^f.round(right, ki) 81 | } 82 | 83 | // join left and right bits to form permuted index 84 | return (left << f.halfNumBits) | right 85 | } 86 | 87 | func (f *feistelGenerator) round(right uint32, subkey uint32) uint32 { 88 | data := f.arena[:8] 89 | data[0] = byte(right >> 24) 90 | data[1] = byte(right >> 16) 91 | data[2] = byte(right >> 8) 92 | data[3] = byte(right) 93 | data[4] = byte(subkey >> 24) 94 | data[5] = byte(subkey >> 16) 95 | data[6] = byte(subkey >> 8) 96 | data[7] = byte(subkey) 97 | f.hash.Reset() 98 | f.hash.Write(data[:8]) 99 | data = f.hash.Sum(f.arena[:0]) 100 | 101 | r := uint32(data[0])<<24 | uint32(data[1])<<16 | uint32(data[2])<<8 | uint32(data[3]) 102 | return r & f.rightMask 103 | } 104 | -------------------------------------------------------------------------------- /safe/randmap.go: -------------------------------------------------------------------------------- 1 | package randmap 2 | 3 | import ( 4 | "math/big" 5 | "reflect" 6 | 7 | crand "crypto/rand" 8 | mrand "math/rand" 9 | ) 10 | 11 | // A randIntn function returns a random value in [0, n). 12 | type randIntn func(n int) int 13 | 14 | func cRandInt(n int) int { 15 | i, _ := crand.Int(crand.Reader, big.NewInt(int64(n))) 16 | return int(i.Int64()) 17 | } 18 | 19 | func mRandInt(n int) int { 20 | return mrand.Intn(n) 21 | } 22 | 23 | func randKey(m interface{}, Intn randIntn) interface{} { 24 | mv := reflect.ValueOf(m) 25 | keys := mv.MapKeys() 26 | return keys[Intn(len(keys))].Interface() 27 | } 28 | 29 | func randVal(m interface{}, Intn randIntn) interface{} { 30 | mv := reflect.ValueOf(m) 31 | keys := mv.MapKeys() 32 | val := mv.MapIndex(keys[Intn(len(keys))]) 33 | return val.Interface() 34 | } 35 | 36 | // An Iterator iterates over a map in random or pseudorandom order. It is 37 | // intended to be used in a for loop like so: 38 | // 39 | // m := make(map[int]int) 40 | // var k, v int 41 | // i := Iterator(m, &k, &v) 42 | // for i.Next() { 43 | // // use k and v 44 | // } 45 | // 46 | type Iterator struct { 47 | // map, stored so that we can lookup values via keys 48 | m reflect.Value 49 | // random permutation of map keys, resliced each time we iterate 50 | perm []reflect.Value 51 | // settable Values for k and v 52 | k, v reflect.Value 53 | } 54 | 55 | // Next advances the Iterator to the next element in the map, storing its key 56 | // and value in the pointers passed during initialization. It returns false 57 | // when all of the elements have been enumerated. 58 | func (i *Iterator) Next() bool { 59 | if i == nil || len(i.perm) == 0 { 60 | return false 61 | } 62 | var k reflect.Value 63 | k, i.perm = i.perm[0], i.perm[1:] 64 | i.k.Set(k) 65 | i.v.Set(i.m.MapIndex(k)) 66 | return true 67 | } 68 | 69 | func randIter(m, k, v interface{}, Intn randIntn) *Iterator { 70 | mt, kt, vt := reflect.TypeOf(m), reflect.TypeOf(k), reflect.TypeOf(v) 71 | if exp := reflect.PtrTo(mt.Key()); kt != exp { 72 | panic("wrong type for k: expected " + exp.String() + ", got " + kt.String()) 73 | } else if exp = reflect.PtrTo(mt.Elem()); vt != exp { 74 | panic("wrong type for v: expected " + exp.String() + ", got " + vt.String()) 75 | } 76 | 77 | // grab pointers to k and v's memory 78 | kptr := reflect.ValueOf(k).Elem() 79 | vptr := reflect.ValueOf(v).Elem() 80 | 81 | // create a random permutation of m's keys 82 | mv := reflect.ValueOf(m) 83 | keys := mv.MapKeys() 84 | for i := len(keys) - 1; i >= 1; i-- { 85 | j := Intn(i + 1) 86 | keys[i], keys[j] = keys[j], keys[i] 87 | } 88 | 89 | return &Iterator{ 90 | m: mv, 91 | perm: keys, 92 | k: kptr, 93 | v: vptr, 94 | } 95 | } 96 | 97 | // Key returns a uniform random key of m, which must be a non-empty map. 98 | func Key(m interface{}) interface{} { return randKey(m, cRandInt) } 99 | 100 | // Val returns a uniform random value of m, which must be a non-empty map. 101 | func Val(m interface{}) interface{} { return randVal(m, cRandInt) } 102 | 103 | // Iter returns a random iterator for m. Each call to Next will store the next 104 | // key/value pair in k and v, which must be pointers. Modifying the map during 105 | // iteration will result in undefined behavior. 106 | func Iter(m, k, v interface{}) *Iterator { return randIter(m, k, v, cRandInt) } 107 | 108 | // FastKey returns a pseudorandom key of m, which must be a non-empty map. 109 | func FastKey(m interface{}) interface{} { return randKey(m, mRandInt) } 110 | 111 | // FastVal returns a pseudorandom value of m, which must be a non-empty map. 112 | func FastVal(m interface{}) interface{} { return randVal(m, mRandInt) } 113 | 114 | // FastIter returns a pseudorandom iterator for m. Each call to Next will 115 | // store the next key/value pair in k and v, which must be pointers. Modifying 116 | // the map during iteration will result in undefined behavior. 117 | func FastIter(m, k, v interface{}) *Iterator { return randIter(m, k, v, mRandInt) } 118 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | randmap 2 | ------- 3 | 4 | [![GoDoc](https://godoc.org/github.com/lukechampine/randmap?status.svg)](https://godoc.org/github.com/lukechampine/randmap) 5 | [![Go Report Card](http://goreportcard.com/badge/github.com/lukechampine/randmap)](https://goreportcard.com/report/github.com/lukechampine/randmap) 6 | 7 | ``` 8 | go get github.com/lukechampine/randmap 9 | ``` 10 | 11 | randmap provides methods for (efficiently) accessing random elements of maps, and iterating 12 | through maps in random order. [Here is a blog post](http://lukechampine.com/hackmap.html) that dives deeper into how I 13 | accomplished this, if you're interested in that sort of thing. 14 | 15 | **WARNING:** randmap uses the `unsafe` package to access the internal Go map 16 | type. In general, you should think twice before running any code that imports 17 | `unsafe`. Please read the full README (and the code!) if you are considering 18 | using randmap in any serious capacity. If you want the `randmap` functionality 19 | without the risks, you can import the `randmap/safe` package instead, which 20 | is far less efficient but does not use `unsafe`. 21 | 22 | First, it is important to clear up a misconception about Go's map type: that 23 | `range` iterates through maps in random order. Well, what does the Language 24 | Specification say? 25 | 26 | >The iteration order over maps is not specified and is not guaranteed to be 27 | >the same from one iteration to the next. [[1]](https://golang.org/ref/spec#For_statements) 28 | 29 | This says nothing at all about whether the iteration is random! In fact, you 30 | can confirm for yourself that iteration order is not very random at all. The 31 | (current) map iterator implementation simply chooses a random index in the map 32 | data and iterates forward from there, skipping empty "buckets" and wrapping 33 | around at the end. This can lead to surprising behavior: 34 | 35 | ```go 36 | m := map[int]int{ 37 | 0: 0, 38 | 1: 1, 39 | } 40 | for i := range m { 41 | fmt.Printf("selected %v!\n", i) 42 | break 43 | } 44 | ``` 45 | 46 | How frequently do you think 0 will be selected, and how frequently will 1 be 47 | selected? The answer is that 0 will be selected **7x** more frequently than 1! 48 | So we can conclude that map iteration does not produce uniformly random 49 | elements. (Actually, it's even worse than that; for certain map states, there 50 | are elements that `range` will _never_ start on!) 51 | 52 | This leaves us with only two options: we can flatten the map and select a 53 | random index, or we can iterate a random number of times. But these incur 54 | costs of O(_n_) memory and O(_n_) time, respectively. That might be ok for 55 | small maps, but maps aren't always small. 56 | 57 | Neither of these options were acceptable to me, so I now present a third 58 | option: random map access in constant space and time. Specifically, this 59 | approach uses O(1) space and O(_L_ * (1 + _k_/_n_)) time, where _n_ is the 60 | number of elements in the map, _k_ is the "capacity" of the map (how many 61 | elements it can hold before being resized), and _L_ is the length of the 62 | longest "bucket chain." Since Go maps double in size when they grow, the total 63 | time will generally not exceed 2x the normal map lookup time. 64 | 65 | The algorithm is as follows: we begin the same way as the builtin algorithm, 66 | by selecting a random index. If the index contains an element, we return it. 67 | If it is empty, we try another random index. (The builtin algorithm seeks 68 | forward until it hits a non-empty index.) That's it! In theory, this approach 69 | actually has an unbounded run time (because you may select the same index 70 | twice), but in practice this is rare. I may improve the implementation later 71 | to iterate through a random permutation of indices, which would guarantee 72 | eventual termination. 73 | 74 | ## Random Iteration ## 75 | 76 | randmap provides `Iter` and `FastIter` functions for iterating through maps in 77 | random or pseudeorandom order. The standard way to do this is to flatten the 78 | map and use `math/rand.Perm` to iterate through the slice. Although this is 79 | sufficiently random, it requires O(_n_) space and time, which is infeasible 80 | for large maps. randmap instead uses a Feistel network to simultaneously 81 | generate and iterate through permutations in constant space. This approach is 82 | further detailed in the docstring of the `perm` subpackage. The main tradeoff 83 | is that the generator approach will be much slower when _n_ is small. 84 | 85 | ## Examples ## 86 | 87 | ```go 88 | m := map[int]int{ 89 | 0: 0, 90 | 1: 1, 91 | } 92 | 93 | // select a random key 94 | k := randmap.Key(m).(int) 95 | 96 | // select a random value 97 | v := randmap.Val(m).(int) 98 | 99 | // select a pseudorandom key 100 | k := randmap.FastKey(m).(int) 101 | 102 | // select a pseudorandom value 103 | v := randmap.FastVal(m).(int) 104 | 105 | // iterate in random order 106 | var k, v int 107 | i := randmap.Iter(m, &k, &v) 108 | for i.Next() { 109 | // use k and v 110 | } 111 | 112 | // iterate in pseudorandom order 113 | i := randmap.FastIter(m, &k, &v) 114 | for i.Next() { 115 | // use k and v 116 | } 117 | ``` 118 | 119 | In case it wasn't obvious, `Key`/`Val`/`Iter` use `crypto/rand`, while their 120 | `Fast` equivalents use `math/rand`. You should use the former in any code that 121 | requires cryptographically strong randomness. 122 | 123 | ## Caveats ## 124 | 125 | This package obviously depends heavily on the internal representation of the 126 | `map` type. If it changes, this package may break. By the way, the `map` type 127 | is changing in Go 1.8. As stated above, use the `randmap/safe` package if you 128 | want the functionality of `randmap` without the risks. 129 | 130 | The runtime code governing maps is a bit esoteric, and uses constructs that 131 | aren't available outside of the runtime. Concurrent map operations are 132 | especially tricky. For now, no guarantees are made about concurrent use of the 133 | functions in this package. Guarding map accesses with a mutex should be 134 | sufficient to prevent any problems. 135 | 136 | The provided Iterators are not guaranteed to uniformly cover the full 137 | permutation space of a given map. This is because the number of permutations 138 | may be much larger than the entropy of the iterator's seed. Nevertheless, the 139 | seed is sufficient to prevent an attacker from guessing which permutation was 140 | selected. 141 | -------------------------------------------------------------------------------- /randmap.go: -------------------------------------------------------------------------------- 1 | // Package randmap provides methods for accessing random elements of maps, and 2 | // iterating through maps in random order. 3 | package randmap 4 | 5 | import ( 6 | "reflect" 7 | "unsafe" 8 | 9 | crand "crypto/rand" 10 | mrand "math/rand" 11 | 12 | "github.com/lukechampine/randmap/perm" 13 | ) 14 | 15 | const ptrSize = unsafe.Sizeof(uintptr(0)) 16 | 17 | type emptyInterface struct { 18 | typ unsafe.Pointer 19 | val unsafe.Pointer 20 | } 21 | 22 | // mrand doesn't give us access to its globalRand, so we'll just use a 23 | // function instead of an io.Reader 24 | type randReader func(p []byte) (int, error) 25 | 26 | func randInts(read randReader, numBuckets uintptr, numOver uint8) (uintptr, uint8, uint8) { 27 | space := numBuckets * uintptr(numOver) * bucketCnt 28 | var arena [ptrSize]byte 29 | read(arena[:]) 30 | r := *(*uintptr)(unsafe.Pointer(&arena[0])) % space 31 | 32 | bucket := r / (uintptr(numOver) * bucketCnt) 33 | over := (r / bucketCnt) % uintptr(numOver) 34 | offi := r % bucketCnt 35 | 36 | return bucket, uint8(over), uint8(offi) 37 | } 38 | 39 | func randKey(m interface{}, src randReader) interface{} { 40 | ei := (*emptyInterface)(unsafe.Pointer(&m)) 41 | t := (*maptype)(ei.typ) 42 | h := (*hmap)(ei.val) 43 | if h == nil || h.count == 0 { 44 | panic("empty map") 45 | } 46 | it := new(hiter) 47 | numBuckets := uintptr(1) << h.B 48 | numOver := maxOverflow(t, h) + 1 49 | bucket, over, offi := randInts(src, numBuckets, numOver) 50 | for !mapaccessi(t, h, it, bucket, over, offi) { 51 | bucket, over, offi = randInts(src, numBuckets, numOver) 52 | } 53 | return *(*interface{})(unsafe.Pointer(&emptyInterface{ 54 | typ: unsafe.Pointer(t.key), 55 | val: it.key, 56 | })) 57 | } 58 | 59 | func randVal(m interface{}, src randReader) interface{} { 60 | ei := (*emptyInterface)(unsafe.Pointer(&m)) 61 | t := (*maptype)(ei.typ) 62 | h := (*hmap)(ei.val) 63 | if h == nil || h.count == 0 { 64 | panic("empty map") 65 | } 66 | it := new(hiter) 67 | numBuckets := uintptr(1) << h.B 68 | numOver := maxOverflow(t, h) + 1 69 | bucket, over, offi := randInts(src, numBuckets, numOver) 70 | for !mapaccessi(t, h, it, bucket, over, offi) { 71 | bucket, over, offi = randInts(src, numBuckets, numOver) 72 | } 73 | return *(*interface{})(unsafe.Pointer(&emptyInterface{ 74 | typ: unsafe.Pointer(t.elem), 75 | val: it.value, 76 | })) 77 | } 78 | 79 | // An Iterator iterates over a map in random or pseudorandom order. It is 80 | // intended to be used in a for loop like so: 81 | // 82 | // m := make(map[int]int) 83 | // var k, v int 84 | // i := Iterator(m, &k, &v) 85 | // for i.Next() { 86 | // // use k and v 87 | // } 88 | // 89 | type Iterator struct { 90 | // permutation generator 91 | gen interface { 92 | Next() (uint32, bool) 93 | } 94 | 95 | it *hiter 96 | k, v reflect.Value 97 | 98 | // constants 99 | t *maptype 100 | h *hmap 101 | over uint32 102 | } 103 | 104 | // Next advances the Iterator to the next element in the map, storing its key 105 | // and value in the pointers passed during initialization. It returns false 106 | // when all of the elements have been enumerated. 107 | func (i *Iterator) Next() bool { 108 | if i == nil { 109 | return false 110 | } 111 | t, h, it := i.t, i.h, i.it 112 | 113 | for { 114 | r, ok := i.gen.Next() 115 | if !ok { 116 | return false 117 | } 118 | 119 | bucket := uintptr(r / (i.over * bucketCnt)) 120 | over := (r / bucketCnt) % i.over 121 | offi := r % bucketCnt 122 | if mapaccessi(t, h, it, bucket, uint8(over), uint8(offi)) { 123 | // unfortunately, there doesn't seem to be a faster way than this 124 | k := *(*interface{})(unsafe.Pointer(&emptyInterface{ 125 | typ: unsafe.Pointer(t.key), 126 | val: it.key, 127 | })) 128 | v := *(*interface{})(unsafe.Pointer(&emptyInterface{ 129 | typ: unsafe.Pointer(t.elem), 130 | val: it.value, 131 | })) 132 | i.k.Set(reflect.ValueOf(k)) 133 | i.v.Set(reflect.ValueOf(v)) 134 | return true 135 | } 136 | } 137 | } 138 | 139 | func randIter(m, k, v interface{}, read randReader) *Iterator { 140 | mt, kt, vt := reflect.TypeOf(m), reflect.TypeOf(k), reflect.TypeOf(v) 141 | if exp := reflect.PtrTo(mt.Key()); kt != exp { 142 | panic("wrong type for k: expected " + exp.String() + ", got " + kt.String()) 143 | } else if exp = reflect.PtrTo(mt.Elem()); vt != exp { 144 | panic("wrong type for v: expected " + exp.String() + ", got " + vt.String()) 145 | } 146 | 147 | // determine total rand space for m 148 | ei := (*emptyInterface)(unsafe.Pointer(&m)) 149 | t := (*maptype)(ei.typ) 150 | h := (*hmap)(ei.val) 151 | if h == nil || h.count == 0 { 152 | return nil 153 | } 154 | numOver := uint32(maxOverflow(t, h) + 1) 155 | numBuckets := uint32(1 << h.B) 156 | space := numBuckets * numOver * bucketCnt 157 | 158 | // create a permutation generator for the space 159 | var seed [4]byte 160 | read(seed[:]) 161 | g := perm.NewGenerator(space, *(*uint32)(unsafe.Pointer(&seed[0]))) 162 | 163 | // grab pointers to k and v's memory 164 | kptr := reflect.ValueOf(k).Elem() 165 | vptr := reflect.ValueOf(v).Elem() 166 | 167 | return &Iterator{ 168 | gen: g, 169 | it: new(hiter), 170 | k: kptr, 171 | v: vptr, 172 | t: t, 173 | h: h, 174 | over: numOver, 175 | } 176 | } 177 | 178 | // Key returns a uniform random key of m, which must be a non-empty map. 179 | func Key(m interface{}) interface{} { return randKey(m, crand.Read) } 180 | 181 | // Val returns a uniform random value of m, which must be a non-empty map. 182 | func Val(m interface{}) interface{} { return randVal(m, crand.Read) } 183 | 184 | // Iter returns a random iterator for m. Each call to Next will store the next 185 | // key/value pair in k and v, which must be pointers. Modifying the map during 186 | // iteration will result in undefined behavior. 187 | func Iter(m, k, v interface{}) *Iterator { return randIter(m, k, v, crand.Read) } 188 | 189 | // FastKey returns a pseudorandom key of m, which must be a non-empty map. 190 | func FastKey(m interface{}) interface{} { return randKey(m, mrand.Read) } 191 | 192 | // FastVal returns a pseudorandom value of m, which must be a non-empty map. 193 | func FastVal(m interface{}) interface{} { return randVal(m, mrand.Read) } 194 | 195 | // FastIter returns a pseudorandom iterator for m. Each call to Next will 196 | // store the next key/value pair in k and v, which must be pointers. Modifying 197 | // the map during iteration will result in undefined behavior. 198 | func FastIter(m, k, v interface{}) *Iterator { return randIter(m, k, v, mrand.Read) } 199 | -------------------------------------------------------------------------------- /safe/randmap_test.go: -------------------------------------------------------------------------------- 1 | package randmap 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "math/rand" 7 | "testing" 8 | ) 9 | 10 | // builtinSeekKey selects a key by advancing the map iterator a random number 11 | // of times. It is unbiased. It runs in O(1) space and O(n) time. 12 | func builtinSeekKey(m map[int]int) (n int) { 13 | r := rand.Intn(len(m)) + 1 14 | for n = range m { 15 | if r--; r <= 0 { 16 | return 17 | } 18 | } 19 | panic("empty map") 20 | } 21 | 22 | // builtinFlattenKey selects a key by flattening the map into a slice of its keys 23 | // and selecting a random index. It is unbiased. It runs in O(n) space and 24 | // O(n) time. 25 | func builtinFlattenKey(m map[int]int) int { 26 | flat := make([]int, 0, len(m)) 27 | for n := range m { 28 | flat = append(flat, n) 29 | } 30 | return flat[rand.Intn(len(flat))] 31 | } 32 | 33 | func TestKey(t *testing.T) { 34 | const iters = 100000 35 | m := map[int]int{ 36 | 0: 0, 37 | 1: 1, 38 | 2: 2, 39 | 3: 3, 40 | 4: 4, 41 | 5: 5, 42 | 6: 6, 43 | 7: 7, 44 | 8: 8, 45 | 9: 9, 46 | } 47 | counts := make([]int, len(m)) 48 | for i := 0; i < iters; i++ { 49 | counts[Key(m).(int)]++ 50 | } 51 | 52 | for n, c := range counts { 53 | if (iters/len(m))/2 > c || c > (iters/len(m))*2 { 54 | t.Errorf("suspicious count: expected %v-%v, got %v (%v)", (iters/len(m))/2, (iters/len(m))*2, c, n) 55 | } 56 | } 57 | } 58 | 59 | func TestVal(t *testing.T) { 60 | const iters = 100000 61 | m := map[int]int{ 62 | 0: 0, 63 | 1: 1, 64 | 2: 2, 65 | 3: 3, 66 | 4: 4, 67 | 5: 5, 68 | 6: 6, 69 | 7: 7, 70 | 8: 8, 71 | 9: 9, 72 | } 73 | counts := make([]int, len(m)) 74 | for i := 0; i < iters; i++ { 75 | counts[Val(m).(int)]++ 76 | } 77 | 78 | for n, c := range counts { 79 | if (iters/len(m))/2 > c || c > (iters/len(m))*2 { 80 | t.Errorf("suspicious count: expected %v-%v, got %v (%v)", (iters/len(m))/2, (iters/len(m))*2, c, n) 81 | } 82 | } 83 | } 84 | 85 | func TestFastKey(t *testing.T) { 86 | const iters = 100000 87 | m := map[int]int{ 88 | 0: 0, 89 | 1: 1, 90 | 2: 2, 91 | 3: 3, 92 | 4: 4, 93 | 5: 5, 94 | 6: 6, 95 | 7: 7, 96 | 8: 8, 97 | 9: 9, 98 | } 99 | counts := make([]int, len(m)) 100 | for i := 0; i < iters; i++ { 101 | counts[FastKey(m).(int)]++ 102 | } 103 | 104 | for n, c := range counts { 105 | if (iters/len(m))/2 > c || c > (iters/len(m))*2 { 106 | t.Errorf("suspicious count: expected %v-%v, got %v (%v)", (iters/len(m))/2, (iters/len(m))*2, c, n) 107 | } 108 | } 109 | } 110 | 111 | func TestFastVal(t *testing.T) { 112 | const iters = 100000 113 | m := map[int]int{ 114 | 0: 0, 115 | 1: 1, 116 | 2: 2, 117 | 3: 3, 118 | 4: 4, 119 | } 120 | counts := make([]int, len(m)) 121 | for i := 0; i < iters; i++ { 122 | counts[FastVal(m).(int)]++ 123 | } 124 | 125 | for n, c := range counts { 126 | if (iters/len(m))/2 > c || c > (iters/len(m))*2 { 127 | t.Errorf("suspicious count: expected %v-%v, got %v (%v)", (iters/len(m))/2, (iters/len(m))*2, c, n) 128 | } 129 | } 130 | } 131 | 132 | func TestEmpty(t *testing.T) { 133 | defer func() { 134 | if recover() == nil { 135 | t.Fatal("expected panic when accessing empty map") 136 | } 137 | }() 138 | _ = Key(make(map[int]int)) 139 | } 140 | 141 | func TestEntropy(t *testing.T) { 142 | m := make(map[int]byte) 143 | for j := 0; j < 255; j++ { 144 | m[j] = byte(j) 145 | } 146 | b := make([]byte, 10000) 147 | for j := range b { 148 | b[j] = FastVal(m).(byte) 149 | } 150 | var buf bytes.Buffer 151 | w, _ := gzip.NewWriterLevel(&buf, gzip.BestCompression) 152 | w.Write(b) 153 | w.Close() 154 | if buf.Len() < len(b) { 155 | t.Fatalf("gzip was able to compress random keys by %.2f%%! (%v total bytes)", float64(100*buf.Len())/float64(len(b)), buf.Len()) 156 | } 157 | } 158 | 159 | func BenchmarkKey(b *testing.B) { 160 | m := make(map[int]int, 10000) 161 | for i := 0; i < 10000; i++ { 162 | m[i] = i 163 | } 164 | 165 | b.Run("key", func(b *testing.B) { 166 | b.ReportAllocs() 167 | for i := 0; i < b.N; i++ { 168 | _ = Key(m).(int) 169 | } 170 | }) 171 | 172 | b.Run("fastkey", func(b *testing.B) { 173 | b.ReportAllocs() 174 | for i := 0; i < b.N; i++ { 175 | _ = FastKey(m).(int) 176 | } 177 | }) 178 | 179 | b.Run("seek", func(b *testing.B) { 180 | b.ReportAllocs() 181 | for i := 0; i < b.N; i++ { 182 | _ = builtinSeekKey(m) 183 | } 184 | }) 185 | 186 | b.Run("flatten", func(b *testing.B) { 187 | b.ReportAllocs() 188 | for i := 0; i < b.N; i++ { 189 | _ = builtinFlattenKey(m) 190 | } 191 | }) 192 | } 193 | 194 | func TestIter(t *testing.T) { 195 | const iters = 1000 196 | m := map[int]int{ 197 | 0: 0, 198 | 1: 1, 199 | 2: 2, 200 | 3: 3, 201 | 4: 4, 202 | 5: 5, 203 | 6: 6, 204 | 7: 7, 205 | 8: 8, 206 | 9: 9, 207 | } 208 | counts := make([][]int, len(m)) 209 | for i := range counts { 210 | counts[i] = make([]int, len(m)) 211 | } 212 | var k, v int 213 | for i := 0; i < iters; i++ { 214 | it := Iter(m, &k, &v) 215 | for j := 0; it.Next(); j++ { 216 | // key k appeared at index j 217 | counts[k][j]++ 218 | } 219 | } 220 | 221 | // each key should have appeared at each index about iters/len(m) times 222 | for k, cs := range counts { 223 | for i, c := range cs { 224 | if (iters/len(m))/2 > c || c > (iters/len(m))*2 { 225 | t.Errorf("suspicious count for key %v index %v: expected %v-%v, got %v", k, i, (iters/len(m))/2, (iters/len(m))*2, c) 226 | } 227 | } 228 | } 229 | } 230 | 231 | func TestFastIter(t *testing.T) { 232 | const iters = 1000 233 | m := map[int]int{ 234 | 0: 0, 235 | 1: 1, 236 | 2: 2, 237 | 3: 3, 238 | 4: 4, 239 | 5: 5, 240 | 6: 6, 241 | 7: 7, 242 | 8: 8, 243 | 9: 9, 244 | } 245 | counts := make([][]int, len(m)) 246 | for i := range counts { 247 | counts[i] = make([]int, len(m)) 248 | } 249 | var k, v int 250 | for i := 0; i < iters; i++ { 251 | it := FastIter(m, &k, &v) 252 | for j := 0; it.Next(); j++ { 253 | // key k appeared at index j 254 | counts[k][j]++ 255 | } 256 | } 257 | 258 | // each key should have appeared at each index about iters/len(m) times 259 | for k, cs := range counts { 260 | for i, c := range cs { 261 | if (iters/len(m))/2 > c || c > (iters/len(m))*2 { 262 | t.Errorf("suspicious count for key %v index %v: expected %v-%v, got %v", k, i, (iters/len(m))/2, (iters/len(m))*2, c) 263 | } 264 | } 265 | } 266 | } 267 | 268 | func TestIterBadType(t *testing.T) { 269 | defer func() { 270 | if recover() == nil { 271 | t.Fatal("expected panic when passing wrong type to Iter") 272 | } 273 | }() 274 | _ = Iter(make(map[int]int), new(uint8), new(uint8)) 275 | } 276 | 277 | func BenchmarkIter(b *testing.B) { 278 | m := make(map[int]int, 1000) 279 | for i := 0; i < 1000; i++ { 280 | m[i] = i 281 | } 282 | 283 | b.Run("iter", func(b *testing.B) { 284 | b.ReportAllocs() 285 | for i := 0; i < b.N; i++ { 286 | var k, v int 287 | it := FastIter(m, &k, &v) 288 | for it.Next() { 289 | } 290 | } 291 | }) 292 | 293 | b.Run("fastiter", func(b *testing.B) { 294 | b.ReportAllocs() 295 | for i := 0; i < b.N; i++ { 296 | var k, v int 297 | it := FastIter(m, &k, &v) 298 | for it.Next() { 299 | } 300 | } 301 | }) 302 | 303 | b.Run("flatten", func(b *testing.B) { 304 | b.ReportAllocs() 305 | for i := 0; i < b.N; i++ { 306 | flat := make([]int, 0, len(m)) 307 | for n := range m { 308 | flat = append(flat, n) 309 | } 310 | for _, k := range rand.Perm(len(flat)) { 311 | _ = k 312 | } 313 | } 314 | }) 315 | } 316 | -------------------------------------------------------------------------------- /runtime_go1.7.go: -------------------------------------------------------------------------------- 1 | // +build go1.7 2 | 3 | package randmap 4 | 5 | import "unsafe" 6 | 7 | const ( 8 | // Maximum number of key/value pairs a bucket can hold. 9 | bucketCntBits = 3 10 | bucketCnt = 1 << bucketCntBits 11 | 12 | // data offset should be the size of the bmap struct, but needs to be 13 | // aligned correctly. For amd64p32 this means 64-bit alignment 14 | // even though pointers are 32 bit. 15 | dataOffset = unsafe.Offsetof(struct { 16 | b bmap 17 | v int64 18 | }{}.v) 19 | 20 | // Possible tophash values. We reserve a few possibilities for special marks. 21 | // Each bucket (including its overflow buckets, if any) will have either all or none of its 22 | // entries in the evacuated* states (except during the evacuate() method, which only happens 23 | // during map writes and thus no one else can observe the map during that time). 24 | empty = 0 // cell is empty 25 | evacuatedEmpty = 1 // cell is empty, bucket is evacuated. 26 | evacuatedX = 2 // key/value is valid. Entry has been evacuated to first half of larger table. 27 | evacuatedY = 3 // same as above, but evacuated to second half of larger table. 28 | minTopHash = 4 // minimum tophash for a normal filled cell. 29 | 30 | noCheck = 1<<(8*unsafe.Sizeof(uintptr(0))) - 1 31 | 32 | kindNoPointers = 1 << 7 33 | ) 34 | 35 | type ( 36 | hmap struct { 37 | count int // # live cells == size of map. Must be first (used by len() builtin) 38 | flags uint8 39 | B uint8 // log_2 of # of buckets (can hold up to loadFactor * 2^B items) 40 | hash0 uint32 // hash seed 41 | 42 | buckets unsafe.Pointer // array of 2^B Buckets. may be nil if count==0. 43 | oldbuckets unsafe.Pointer // previous bucket array of half the size, non-nil only when growing 44 | nevacuate uintptr // progress counter for evacuation (buckets less than this have been evacuated) 45 | 46 | // If both key and value do not contain pointers and are inline, then we mark bucket 47 | // type as containing no pointers. This avoids scanning such maps. 48 | // However, bmap.overflow is a pointer. In order to keep overflow buckets 49 | // alive, we store pointers to all overflow buckets in hmap.overflow. 50 | // Overflow is used only if key and value do not contain pointers. 51 | // overflow[0] contains overflow buckets for hmap.buckets. 52 | // overflow[1] contains overflow buckets for hmap.oldbuckets. 53 | // The first indirection allows us to reduce static size of hmap. 54 | // The second indirection allows to store a pointer to the slice in hiter. 55 | overflow *[2]*[]*bmap 56 | } 57 | 58 | bmap struct { 59 | tophash [bucketCnt]uint8 60 | // Followed by bucketCnt keys and then bucketCnt values. 61 | // NOTE: packing all the keys together and then all the values together makes the 62 | // code a bit more complicated than alternating key/value/key/value/... but it allows 63 | // us to eliminate padding which would be needed for, e.g., map[int64]int8. 64 | // Followed by an overflow pointer. 65 | } 66 | 67 | hiter struct { 68 | key unsafe.Pointer // Must be in first position. Write nil to indicate iteration end (see cmd/internal/gc/range.go). 69 | value unsafe.Pointer // Must be in second position (see cmd/internal/gc/range.go). 70 | overflow [2]*[]*bmap // keeps overflow buckets alive 71 | } 72 | 73 | _type struct { 74 | size uintptr 75 | ptrdata uintptr // size of memory prefix holding all pointers 76 | hash uint32 77 | tflag uint8 78 | align uint8 79 | fieldalign uint8 80 | kind uint8 81 | alg *typeAlg 82 | // gcdata stores the GC type data for the garbage collector. 83 | // If the KindGCProg bit is set in kind, gcdata is a GC program. 84 | // Otherwise it is a ptrmask bitmap. See mbitmap.go for details. 85 | gcdata uintptr 86 | str int32 87 | ptrToThis int32 88 | } 89 | 90 | maptype struct { 91 | typ _type 92 | key *_type 93 | elem *_type 94 | bucket *_type // internal type representing a hash bucket 95 | hmap *_type // internal type representing a hmap 96 | keysize uint8 // size of key slot 97 | indirectkey bool // store ptr to key instead of key itself 98 | valuesize uint8 // size of value slot 99 | indirectvalue bool // store ptr to value instead of value itself 100 | bucketsize uint16 // size of bucket 101 | reflexivekey bool // true if k==k for all keys 102 | needkeyupdate bool // true if we need to update key on an overwrite 103 | } 104 | 105 | // typeAlg is also copied/used in reflect/type.go. 106 | // keep them in sync. 107 | typeAlg struct { 108 | // function for hashing objects of this type 109 | // (ptr to object, seed) -> hash 110 | hash func(unsafe.Pointer, uintptr) uintptr 111 | // function for comparing objects of this type 112 | // (ptr to object A, ptr to object B) -> ==? 113 | equal func(unsafe.Pointer, unsafe.Pointer) bool 114 | } 115 | ) 116 | 117 | func evacuated(b *bmap) bool { 118 | h := b.tophash[0] 119 | return h > empty && h < minTopHash 120 | } 121 | 122 | func (b *bmap) overflow(t *maptype) *bmap { 123 | return *(**bmap)(add(unsafe.Pointer(b), uintptr(t.bucketsize)-unsafe.Sizeof(uintptr(0)))) 124 | } 125 | 126 | func (h *hmap) createOverflow() { 127 | if h.overflow == nil { 128 | h.overflow = new([2]*[]*bmap) 129 | } 130 | if h.overflow[0] == nil { 131 | h.overflow[0] = new([]*bmap) 132 | } 133 | } 134 | 135 | func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { 136 | return unsafe.Pointer(uintptr(p) + x) 137 | } 138 | 139 | // maxOverflow returns the length of the longest bucket chain in the map. 140 | func maxOverflow(t *maptype, h *hmap) uint8 { 141 | var max uint8 142 | if h.oldbuckets != nil { 143 | for i := uintptr(0); i < (1 << (h.B - 1)); i++ { 144 | var over uint8 145 | b := (*bmap)(add(h.oldbuckets, i*uintptr(t.bucketsize))) 146 | if evacuated(b) { 147 | continue 148 | } 149 | for b = b.overflow(t); b != nil; over++ { 150 | b = b.overflow(t) 151 | } 152 | if over > max { 153 | max = over 154 | } 155 | } 156 | } 157 | for i := uintptr(0); i < (1 << h.B); i++ { 158 | var over uint8 159 | for b := (*bmap)(add(h.buckets, i*uintptr(t.bucketsize))).overflow(t); b != nil; over++ { 160 | b = b.overflow(t) 161 | } 162 | if over > max { 163 | max = over 164 | } 165 | } 166 | return max 167 | } 168 | 169 | // mapaccessi moves 'it' to offset 'offi' in overflow bucket 'over' of bucket 170 | // 'bucket' in hmap, which may or may not contain valid data. It returns true 171 | // if the data is valid, and false otherwise. 172 | func mapaccessi(t *maptype, h *hmap, it *hiter, bucket uintptr, over, offi uint8) bool { 173 | // grab snapshot of bucket state 174 | if t.bucket.kind&kindNoPointers != 0 { 175 | // Allocate the current slice and remember pointers to both current and old. 176 | // This preserves all relevant overflow buckets alive even if 177 | // the table grows and/or overflow buckets are added to the table 178 | // while we are iterating. 179 | h.createOverflow() 180 | it.overflow = *h.overflow 181 | } 182 | 183 | b := (*bmap)(add(h.buckets, bucket*uintptr(t.bucketsize))) 184 | 185 | checkBucket := false 186 | if h.oldbuckets != nil { 187 | // Iterator was started in the middle of a grow, and the grow isn't done yet. 188 | // If the bucket we're looking at hasn't been filled in yet (i.e. the old 189 | // bucket hasn't been evacuated) then we need to use that pointer instead. 190 | oldbucket := bucket & (uintptr(1)<<(h.B-1) - 1) 191 | oldB := (*bmap)(add(h.oldbuckets, oldbucket*uintptr(t.bucketsize))) 192 | if !evacuated(oldB) { 193 | b = oldB 194 | checkBucket = true 195 | } 196 | } 197 | 198 | // seek to overflow bucket 199 | for i := uint8(0); i < over; i++ { 200 | b = b.overflow(t) 201 | if b == nil { 202 | return false 203 | } 204 | } 205 | 206 | // check that bucket is not empty 207 | if b.tophash[offi] == empty || b.tophash[offi] == evacuatedEmpty { 208 | return false 209 | } 210 | 211 | // grab the key and value 212 | k := add(unsafe.Pointer(b), dataOffset+uintptr(offi)*uintptr(t.keysize)) 213 | v := add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+uintptr(offi)*uintptr(t.valuesize)) 214 | if t.indirectkey { 215 | k = *((*unsafe.Pointer)(k)) 216 | } 217 | if t.indirectvalue { 218 | v = *((*unsafe.Pointer)(v)) 219 | } 220 | 221 | // if this is an old bucket, we need to check whether this key is destined 222 | // for the new bucket. Otherwise, we will have a 2x bias towards oldbucket 223 | // values, since two different bucket selections can result in the same 224 | // oldbucket. 225 | if checkBucket { 226 | if t.reflexivekey || t.key.alg.equal(k, k) { 227 | // If the item in the oldbucket is not destined for 228 | // the current new bucket in the iteration, skip it. 229 | hash := t.key.alg.hash(k, uintptr(h.hash0)) 230 | if hash&(uintptr(1)<>(h.B-1) != uintptr(b.tophash[offi]&1) { 239 | return false 240 | } 241 | } 242 | } 243 | 244 | it.key = k 245 | it.value = v 246 | return true 247 | } 248 | -------------------------------------------------------------------------------- /randmap_test.go: -------------------------------------------------------------------------------- 1 | package randmap 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "math/rand" 7 | "runtime" 8 | "testing" 9 | ) 10 | 11 | // builtinInitKey selects a key by ranging over m and returning the key at the 12 | // initial position of the iterator. It is biased when the map is sparse or 13 | // has overflow buckets. It runs in O(1) space and time. 14 | func builtinInitKey(m map[int]int) (n int) { 15 | for n = range m { 16 | return 17 | } 18 | panic("empty map") 19 | } 20 | 21 | // builtinSeekKey selects a key by advancing the map iterator a random number 22 | // of times. It is unbiased. It runs in O(1) space and O(n) time. 23 | func builtinSeekKey(m map[int]int) (n int) { 24 | r := rand.Intn(len(m)) + 1 25 | for n = range m { 26 | if r--; r <= 0 { 27 | return 28 | } 29 | } 30 | panic("empty map") 31 | } 32 | 33 | // builtinFlattenKey selects a key by flattening the map into a slice of its keys 34 | // and selecting a random index. It is unbiased. It runs in O(n) space and 35 | // O(n) time. 36 | func builtinFlattenKey(m map[int]int) int { 37 | flat := make([]int, 0, len(m)) 38 | for n := range m { 39 | flat = append(flat, n) 40 | } 41 | return flat[rand.Intn(len(flat))] 42 | } 43 | 44 | func TestBuiltinMapInit(t *testing.T) { 45 | const iters = 100000 46 | m := map[int]int{ 47 | 0: 0, 48 | 1: 1, 49 | 2: 2, 50 | 3: 3, 51 | 4: 4, 52 | } 53 | counts := make([]int, len(m)) 54 | for i := 0; i < iters; i++ { 55 | counts[builtinInitKey(m)]++ 56 | } 57 | // 0 should be selected 45-55% of the time 58 | if (iters/2-iters/20) > counts[0] || counts[0] > (iters/2+iters/20) { 59 | t.Errorf("expected builtin map to be less random: expected ~%v for elem 0, got %v", iters/2, counts[0]) 60 | } 61 | } 62 | 63 | func TestBuiltinMapSeek(t *testing.T) { 64 | const iters = 100000 65 | m := map[int]int{ 66 | 0: 0, 67 | 1: 1, 68 | 2: 2, 69 | 3: 3, 70 | 4: 4, 71 | } 72 | counts := make([]int, len(m)) 73 | for i := 0; i < iters; i++ { 74 | counts[builtinSeekKey(m)]++ 75 | } 76 | 77 | // should be unbiased 78 | for n, c := range counts { 79 | if (iters/len(m))/2 > c || c > (iters/len(m))*2 { 80 | t.Errorf("suspicious count: expected %v-%v, got %v (%v)", (iters/len(m))/2, (iters/len(m))*2, c, n) 81 | } 82 | } 83 | } 84 | 85 | func TestBuiltinMapFlatten(t *testing.T) { 86 | const iters = 100000 87 | m := map[int]int{ 88 | 0: 0, 89 | 1: 1, 90 | 2: 2, 91 | 3: 3, 92 | 4: 4, 93 | } 94 | counts := make([]int, len(m)) 95 | for i := 0; i < iters; i++ { 96 | counts[builtinFlattenKey(m)]++ 97 | } 98 | 99 | // should be unbiased 100 | for n, c := range counts { 101 | if (iters/len(m))/2 > c || c > (iters/len(m))*2 { 102 | t.Errorf("suspicious count: expected %v-%v, got %v (%v)", (iters/len(m))/2, (iters/len(m))*2, c, n) 103 | } 104 | } 105 | } 106 | 107 | func TestKey(t *testing.T) { 108 | const iters = 100000 109 | m := map[int]int{ 110 | 0: 0, 111 | 1: 1, 112 | 2: 2, 113 | 3: 3, 114 | 4: 4, 115 | 5: 5, 116 | 6: 6, 117 | 7: 7, 118 | 8: 8, 119 | 9: 9, 120 | } 121 | counts := make([]int, len(m)) 122 | for i := 0; i < iters; i++ { 123 | counts[Key(m).(int)]++ 124 | } 125 | 126 | for n, c := range counts { 127 | if (iters/len(m))/2 > c || c > (iters/len(m))*2 { 128 | t.Errorf("suspicious count: expected %v-%v, got %v (%v)", (iters/len(m))/2, (iters/len(m))*2, c, n) 129 | } 130 | } 131 | } 132 | 133 | func TestVal(t *testing.T) { 134 | const iters = 100000 135 | m := map[int]int{ 136 | 0: 0, 137 | 1: 1, 138 | 2: 2, 139 | 3: 3, 140 | 4: 4, 141 | 5: 5, 142 | 6: 6, 143 | 7: 7, 144 | 8: 8, 145 | 9: 9, 146 | } 147 | counts := make([]int, len(m)) 148 | for i := 0; i < iters; i++ { 149 | counts[Val(m).(int)]++ 150 | } 151 | 152 | for n, c := range counts { 153 | if (iters/len(m))/2 > c || c > (iters/len(m))*2 { 154 | t.Errorf("suspicious count: expected %v-%v, got %v (%v)", (iters/len(m))/2, (iters/len(m))*2, c, n) 155 | } 156 | } 157 | } 158 | 159 | func TestFastKey(t *testing.T) { 160 | const iters = 100000 161 | m := map[int]int{ 162 | 0: 0, 163 | 1: 1, 164 | 2: 2, 165 | 3: 3, 166 | 4: 4, 167 | 5: 5, 168 | 6: 6, 169 | 7: 7, 170 | 8: 8, 171 | 9: 9, 172 | } 173 | counts := make([]int, len(m)) 174 | for i := 0; i < iters; i++ { 175 | counts[FastKey(m).(int)]++ 176 | } 177 | 178 | for n, c := range counts { 179 | if (iters/len(m))/2 > c || c > (iters/len(m))*2 { 180 | t.Errorf("suspicious count: expected %v-%v, got %v (%v)", (iters/len(m))/2, (iters/len(m))*2, c, n) 181 | } 182 | } 183 | } 184 | 185 | func TestGhostIndex(t *testing.T) { 186 | // sometimes, an element is never selected despite thousands of 187 | // iterations. This affects the builtin map range as well. 188 | const outer = 1000 189 | const inner = 1000 190 | for i := 0; i < outer; i++ { 191 | m := map[int]int{ 192 | 0: 0, 193 | 1: 1, 194 | 2: 2, 195 | 3: 3, 196 | 4: 4, 197 | 5: 5, 198 | 6: 6, 199 | 7: 7, 200 | 8: 8, 201 | } 202 | counts := make([]int, len(m)) 203 | for j := 0; j < inner; j++ { 204 | counts[FastKey(m).(int)]++ 205 | } 206 | 207 | for n, c := range counts { 208 | if c == 0 { 209 | t.Fatalf("%v: key %v was never selected!", i, n) 210 | } 211 | } 212 | } 213 | } 214 | 215 | func TestInsert(t *testing.T) { 216 | // Go maps incrementally copy values after each resize. The randmap 217 | // functions should continue to work in the middle of an incremental copy. 218 | const outer = 100 219 | const inner = 100 220 | m := make(map[int]int) 221 | for i := 0; i < outer; i++ { 222 | m[i] = i 223 | 224 | counts := make([]int, len(m)) 225 | for j := 0; j < inner*len(m); j++ { 226 | counts[FastKey(m).(int)]++ 227 | } 228 | 229 | for n, c := range counts { 230 | if inner/2 > c || c > inner*2 { 231 | t.Errorf("suspicious count: expected %v-%v, got %v (%v)", inner/2, inner*2, c, n) 232 | } 233 | } 234 | } 235 | } 236 | 237 | func TestFastVal(t *testing.T) { 238 | const iters = 100000 239 | m := map[int]int{ 240 | 0: 0, 241 | 1: 1, 242 | 2: 2, 243 | 3: 3, 244 | 4: 4, 245 | } 246 | counts := make([]int, len(m)) 247 | for i := 0; i < iters; i++ { 248 | counts[FastVal(m).(int)]++ 249 | } 250 | 251 | for n, c := range counts { 252 | if (iters/len(m))/2 > c || c > (iters/len(m))*2 { 253 | t.Errorf("suspicious count: expected %v-%v, got %v (%v)", (iters/len(m))/2, (iters/len(m))*2, c, n) 254 | } 255 | } 256 | } 257 | 258 | func TestEmpty(t *testing.T) { 259 | defer func() { 260 | if recover() == nil { 261 | t.Fatal("expected panic when accessing empty map") 262 | } 263 | }() 264 | _ = Key(make(map[int]int)) 265 | } 266 | 267 | func TestEntropy(t *testing.T) { 268 | m := make(map[int]int) 269 | for j := 0; j < 417; j++ { // 417 chosen to ensure unevacuated map buckets 270 | m[j] = j 271 | } 272 | b := make([]byte, 10000) 273 | for j := range b { 274 | b[j] = byte(FastKey(m).(int)) 275 | } 276 | var buf bytes.Buffer 277 | w, _ := gzip.NewWriterLevel(&buf, gzip.BestCompression) 278 | w.Write(b) 279 | w.Close() 280 | if buf.Len() < len(b) { 281 | t.Fatalf("gzip was able to compress random keys by %.2f%%! (%v total bytes)", float64(100*buf.Len())/float64(len(b)), buf.Len()) 282 | } 283 | } 284 | 285 | func TestGC(t *testing.T) { 286 | // if pointers are not preserved, overflow buckets can get garbage 287 | // collected. The randmap functions should continue to work in the 288 | // presence of GC. 289 | m := make(map[int]int) 290 | for i := 0; i < 1000; i++ { 291 | m[i] = i 292 | FastKey(m) 293 | runtime.GC() 294 | } 295 | } 296 | 297 | func TestConcurrent(t *testing.T) { 298 | m := map[int]int{ 299 | 0: 0, 300 | 1: 1, 301 | 2: 2, 302 | 3: 3, 303 | 4: 4, 304 | } 305 | const iters = 10000 306 | go func() { 307 | for i := 0; i < iters/len(m); i++ { 308 | for range m { 309 | } 310 | } 311 | }() 312 | for i := 0; i < 10; i++ { 313 | go func() { 314 | for i := 0; i < iters; i++ { 315 | Key(m) 316 | } 317 | }() 318 | } 319 | for i := 0; i < iters; i++ { 320 | Key(m) 321 | } 322 | } 323 | 324 | func BenchmarkKey(b *testing.B) { 325 | m := make(map[int]int, 10000) 326 | for i := 0; i < 10000; i++ { 327 | m[i] = i 328 | } 329 | 330 | b.Run("key", func(b *testing.B) { 331 | b.ReportAllocs() 332 | for i := 0; i < b.N; i++ { 333 | _ = Key(m).(int) 334 | } 335 | }) 336 | 337 | b.Run("fastkey", func(b *testing.B) { 338 | b.ReportAllocs() 339 | for i := 0; i < b.N; i++ { 340 | _ = FastKey(m).(int) 341 | } 342 | }) 343 | 344 | b.Run("seek", func(b *testing.B) { 345 | b.ReportAllocs() 346 | for i := 0; i < b.N; i++ { 347 | _ = builtinSeekKey(m) 348 | } 349 | }) 350 | 351 | b.Run("flatten", func(b *testing.B) { 352 | b.ReportAllocs() 353 | for i := 0; i < b.N; i++ { 354 | _ = builtinFlattenKey(m) 355 | } 356 | }) 357 | } 358 | 359 | func TestIter(t *testing.T) { 360 | const iters = 1000 361 | m := map[int]int{ 362 | 0: 0, 363 | 1: 1, 364 | 2: 2, 365 | 3: 3, 366 | 4: 4, 367 | 5: 5, 368 | 6: 6, 369 | 7: 7, 370 | 8: 8, 371 | 9: 9, 372 | } 373 | counts := make([][]int, len(m)) 374 | for i := range counts { 375 | counts[i] = make([]int, len(m)) 376 | } 377 | var k, v int 378 | for i := 0; i < iters; i++ { 379 | it := Iter(m, &k, &v) 380 | for j := 0; it.Next(); j++ { 381 | // key k appeared at index j 382 | counts[k][j]++ 383 | } 384 | } 385 | 386 | // each key should have appeared at each index about iters/len(m) times 387 | for k, cs := range counts { 388 | for i, c := range cs { 389 | if (iters/len(m))/2 > c || c > (iters/len(m))*2 { 390 | t.Errorf("suspicious count for key %v index %v: expected %v-%v, got %v", k, i, (iters/len(m))/2, (iters/len(m))*2, c) 391 | } 392 | } 393 | } 394 | } 395 | 396 | func TestFastIter(t *testing.T) { 397 | const iters = 1000 398 | m := map[int]int{ 399 | 0: 0, 400 | 1: 1, 401 | 2: 2, 402 | 3: 3, 403 | 4: 4, 404 | 5: 5, 405 | 6: 6, 406 | 7: 7, 407 | 8: 8, 408 | 9: 9, 409 | } 410 | counts := make([][]int, len(m)) 411 | for i := range counts { 412 | counts[i] = make([]int, len(m)) 413 | } 414 | var k, v int 415 | for i := 0; i < iters; i++ { 416 | it := FastIter(m, &k, &v) 417 | for j := 0; it.Next(); j++ { 418 | // key k appeared at index j 419 | counts[k][j]++ 420 | } 421 | } 422 | 423 | // each key should have appeared at each index about iters/len(m) times 424 | for k, cs := range counts { 425 | for i, c := range cs { 426 | if (iters/len(m))/2 > c || c > (iters/len(m))*2 { 427 | t.Errorf("suspicious count for key %v index %v: expected %v-%v, got %v", k, i, (iters/len(m))/2, (iters/len(m))*2, c) 428 | } 429 | } 430 | } 431 | } 432 | 433 | func TestIterBadType(t *testing.T) { 434 | defer func() { 435 | if recover() == nil { 436 | t.Fatal("expected panic when passing wrong type to Iter") 437 | } 438 | }() 439 | _ = Iter(make(map[int]int), new(uint8), new(uint8)) 440 | } 441 | 442 | func BenchmarkIter(b *testing.B) { 443 | m := make(map[int]int, 1000) 444 | for i := 0; i < 1000; i++ { 445 | m[i] = i 446 | } 447 | 448 | b.Run("iter", func(b *testing.B) { 449 | b.ReportAllocs() 450 | for i := 0; i < b.N; i++ { 451 | var k, v int 452 | it := FastIter(m, &k, &v) 453 | for it.Next() { 454 | } 455 | } 456 | }) 457 | 458 | b.Run("fastiter", func(b *testing.B) { 459 | b.ReportAllocs() 460 | for i := 0; i < b.N; i++ { 461 | var k, v int 462 | it := FastIter(m, &k, &v) 463 | for it.Next() { 464 | } 465 | } 466 | }) 467 | 468 | b.Run("flatten", func(b *testing.B) { 469 | b.ReportAllocs() 470 | for i := 0; i < b.N; i++ { 471 | flat := make([]int, 0, len(m)) 472 | for n := range m { 473 | flat = append(flat, n) 474 | } 475 | for _, k := range rand.Perm(len(flat)) { 476 | _ = k 477 | } 478 | } 479 | }) 480 | } 481 | --------------------------------------------------------------------------------