├── .gitignore ├── .golangci.yml ├── LICENSE ├── Makefile ├── README.md ├── binheap ├── README.md ├── generic.go ├── generic_test.go ├── ordered.go ├── ordered_test.go ├── topn.go ├── topn_bench_test.go ├── topn_test.go ├── topstream.go └── topstream_test.go ├── examples ├── heap │ └── main.go └── topn │ └── main.go ├── go.mod ├── go.sum └── smap ├── README.md ├── comparable.go ├── generic.go ├── integer.go ├── integer_bench_test.go └── integer_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | testdata/ 3 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | concurrency: 4 3 | timeout: 1m 4 | issue-exit-code: 1 5 | tests: false 6 | 7 | output: 8 | format: colored-line-number # use 'golangci-lint run --out-format junit-xml' to have Jenkins-compatible results 9 | print-issued-lines: true 10 | print-linter-name: true 11 | uniq-by-line: true 12 | sort-results: true 13 | 14 | linters: 15 | enable-all: false 16 | presets: 17 | - bugs 18 | - comment 19 | - error 20 | - format 21 | - import 22 | - performance 23 | - style 24 | - metalinter 25 | disable: 26 | - scopelint # deprecated 27 | - interfacer # deprecated 28 | - golint # deprecated, 'style' preset uses Revive 29 | - maligned # deprecated, replaced with govet fieldalignment 30 | - varnamelen 31 | - wsl 32 | - gomnd 33 | - ireturn 34 | - nlreturn 35 | 36 | linters-settings: 37 | govet: 38 | check-shadowing: true 39 | enable: 40 | - asmdecl 41 | - assign 42 | - atomic 43 | - atomicalign 44 | - bools 45 | - buildtag 46 | - cgocall 47 | - composites 48 | - copylocks 49 | - deepequalerrors 50 | - errorsas 51 | - fieldalignment 52 | - findcall 53 | - framepointer 54 | - httpresponse 55 | - ifaceassert 56 | - loopclosure 57 | - lostcancel 58 | - nilfunc 59 | - nilness 60 | - printf 61 | - reflectvaluecompare 62 | - shadow 63 | - shift 64 | - sigchanyzer 65 | - sortslice 66 | - stdmethods 67 | - stringintconv 68 | - structtag 69 | - testinggoroutine 70 | - tests 71 | - unmarshal 72 | - unreachable 73 | - unsafeptr 74 | - unusedresult 75 | - unusedwrite 76 | tagliatelle: 77 | case: 78 | rules: 79 | json: snake 80 | yaml: snake 81 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 lispad.me 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | test: fuzz 2 | go test -cover github.com/lispad/go-generics-tools/binheap 3 | 4 | fuzz: 5 | go test -fuzz=FuzzEmptyMinHeap -fuzztime=10s github.com/lispad/go-generics-tools/binheap 6 | go test -fuzz=FuzzEmptyMaxHeap -fuzztime=10s github.com/lispad/go-generics-tools/binheap 7 | go test -fuzz=FuzzTopN$$ -fuzztime=10s github.com/lispad/go-generics-tools/binheap 8 | go test -fuzz=FuzzTopNImmutable -fuzztime=10s github.com/lispad/go-generics-tools/binheap 9 | 10 | lint: 11 | golangci-lint run binheap -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GoLang Generics tools: Heap structure, sharded rw-locked map. 2 | [![Go Report Card](https://goreportcard.com/badge/github.com/lispad/go-generics-tools)](https://goreportcard.com/report/github.com/lispad/go-generics-tools) 3 | [![MIT License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) 4 | 5 | Introduction 6 | ------------ 7 | 8 | The [Heap](binheap/README.md) package contains simple [binary heap](https://en.wikipedia.org/wiki/Binary_heap) implementation, using Golang 9 | generics. There are several heap implementations 10 | [Details](binheap/README.md). 11 | 12 | 13 | The [ShardedLockMap](smap/README.md) package contains implementation of sharded lock map. 14 | Interface is similar to sync.map, but sharded lock map is faster on scenarios with huge read load with rare updates, 15 | and uses less memory, doing less allocations. 16 | [Details](smap/README.md) 17 | 18 | Compatibility 19 | ------------- 20 | Minimal Golang version is 1.18. Generics and fuzz testing are used. 21 | 22 | Installation 23 | ---------------------- 24 | 25 | To install package, run: 26 | 27 | go get github.com/lispad/go-generics-tools/binheap 28 | or 29 | 30 | go get github.com/lispad/go-generics-tools/smap 31 | 32 | 33 | License 34 | ------- 35 | 36 | The binheap package is licensed under the MIT license. Please see the LICENSE file for details. -------------------------------------------------------------------------------- /binheap/README.md: -------------------------------------------------------------------------------- 1 | # Heap structure, using go generics 2 | [![MIT License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) 3 | 4 | Introduction 5 | ------------ 6 | 7 | The Heap package contains simple [binary heap](https://en.wikipedia.org/wiki/Binary_heap) implementation, using Golang 8 | generics. There are several heap implementations 9 | 10 | - generic Heap implementation, that could be used for `any` type, 11 | - `ComparableHeap` for [comparable](https://go.dev/ref/spec#Comparison_operators) types. Additional `Search` 12 | and `Delete` are implemented, 13 | - for [`constraints.Ordered`](https://pkg.go.dev/golang.org/x/exp/constraints#Ordered) there are 14 | constructors for min, max heaps; 15 | 16 | Also use-cases provided: 17 | 18 | - `TopN` that allows getting N top elements from slice. 19 | `TopN` swaps top N elements to first N elements of slice, no additional allocations are done. All slice elements are 20 | kept, only order is changed. 21 | - `TopNHeap` allows to get N top, pushing elements from stream without allocation slice for all elements. Only O(N) 22 | memory is used. 23 | - `TopNImmutable` allocated new slice for heap, input slice is not mutated. 24 | 25 | Both TopN and TopNImmutable has methods for creating min and max tops for `constraints.Ordered`. 26 | 27 | Usage Example 28 | ----------------- 29 | 30 | package main 31 | 32 | import ( 33 | "fmt" 34 | 35 | "github.com/lispad/go-generics-tools/binheap" 36 | ) 37 | 38 | func main() { 39 | someData := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0} 40 | mins := binheap.MinN[float64](someData, 3) 41 | fmt.Printf("--- top 3 min elements: %v\n", mins) 42 | maxs := binheap.MaxN[float64](someData, 3) 43 | fmt.Printf("--- top 3 max elements: %v\n\n", maxs) 44 | 45 | heap := binheap.EmptyMaxHeap[string]() 46 | heap.Push("foo") 47 | heap.Push("zzz") 48 | heap.Push("bar") 49 | heap.Push("baz") 50 | heap.Push("foobar") 51 | heap.Push("foobaz") 52 | fmt.Printf("--- heap has %d elements, max element:\n%s\n\n", heap.Len(), heap.Peak()) 53 | } 54 | 55 | A bit more examples could be found in `examples` directory 56 | 57 | Benchmark 58 | ----------------- 59 | Theoretical complexity for getting TopN from slice with size M, N <= M: O(N*ln(M)). When N << M, the heap-based TopN 60 | could be much faster than sorting slice and getting top. E.g. For top-3 from 10k elements approach is ln(10^5)/ln(3) ~= 61 | 8.38 times faster. 62 | 63 | #### Benchmark 64 | 65 | BenchmarkSortedMaxN-8 10303648 136.0 ns/op 0 B/op 0 allocs/op 66 | BenchmarkMaxNImmutable-8 398996316 3.029 ns/op 0 B/op 0 allocs/op 67 | BenchmarkMaxN-8 804041455 1.819 ns/op 0 B/op 0 allocs/op 68 | 69 | Compatibility 70 | ------------- 71 | Minimal Golang version is 1.18. Generics and fuzz testing are used. 72 | 73 | Installation 74 | ---------------------- 75 | 76 | To install package, run: 77 | 78 | go get github.com/lispad/go-generics-tools/binheap 79 | 80 | License 81 | ------- 82 | 83 | The binheap package is licensed under the MIT license. Please see the LICENSE file for details. 84 | -------------------------------------------------------------------------------- /binheap/generic.go: -------------------------------------------------------------------------------- 1 | // Package binheap provides implementation of binary heap for any type 2 | // with use of golang generics, and top-N heap usecases. 3 | package binheap 4 | 5 | // Heap provides basic methods: Push, Peak, Pop, Replace top element and PushPop. 6 | type Heap[T any] struct { 7 | comparator func(x, y T) bool 8 | data []T 9 | } 10 | 11 | // EmptyHeap creates empty heap with provided comparator. 12 | func EmptyHeap[T any](comparator func(x, y T) bool) Heap[T] { 13 | return Heap[T]{ 14 | comparator: comparator, 15 | } 16 | } 17 | 18 | // FromSlice creates heap, based on provided slice. 19 | // Slice could be reordered. 20 | func FromSlice[T any](data []T, comparator func(x, y T) bool) Heap[T] { 21 | h := Heap[T]{ 22 | data: data, 23 | comparator: comparator, 24 | } 25 | n := h.Len() 26 | for i := n/2 - 1; i >= 0; i-- { 27 | h.down(i, n) 28 | } 29 | 30 | return h 31 | } 32 | 33 | // Push inserts element to heap. 34 | func (h *Heap[T]) Push(x T) { 35 | h.data = append(h.data, x) 36 | h.up(h.Len() - 1) 37 | } 38 | 39 | // Len returns count of elements in heap. 40 | func (h *Heap[T]) Len() int { 41 | return len(h.data) 42 | } 43 | 44 | // Peak returns top element without deleting. 45 | func (h *Heap[T]) Peak() T { 46 | return h.data[0] 47 | } 48 | 49 | // Pop returns top element with removing it. 50 | func (h *Heap[T]) Pop() T { 51 | n := h.Len() - 1 52 | h.swap(0, n) 53 | h.down(0, n) 54 | result := h.data[n] 55 | h.data = h.data[0:n] 56 | 57 | return result 58 | } 59 | 60 | // PushPop pushes x to the heap and then pops top element. 61 | func (h *Heap[T]) PushPop(x T) T { 62 | if h.Len() > 0 && h.comparator(h.data[0], x) { 63 | x, h.data[0] = h.data[0], x 64 | h.down(0, h.Len()) 65 | } 66 | 67 | return x 68 | } 69 | 70 | // Replace extracts the root of the heap, and push a new item. 71 | func (h *Heap[T]) Replace(x T) (result T) { 72 | result, h.data[0] = h.data[0], x 73 | h.fix(0) 74 | return 75 | } 76 | 77 | func (h *Heap[T]) fix(i int) (result T) { 78 | if !h.down(i, h.Len()) { 79 | h.up(i) 80 | } 81 | 82 | return result 83 | } 84 | 85 | func (h *Heap[T]) swap(i, j int) { 86 | h.data[i], h.data[j] = h.data[j], h.data[i] 87 | } 88 | 89 | func (h *Heap[T]) up(j int) { 90 | for { 91 | i := (j - 1) / 2 // parent 92 | if i == j || !h.comparator(h.data[j], h.data[i]) { 93 | break 94 | } 95 | h.swap(i, j) 96 | j = i 97 | } 98 | } 99 | 100 | func (h *Heap[T]) down(i0, n int) bool { 101 | i := i0 102 | for { 103 | j1 := 2*i + 1 104 | if j1 >= n || j1 < 0 { // j1 < 0 after int overflow 105 | break 106 | } 107 | j := j1 // left child 108 | if j2 := j1 + 1; j2 < n && h.comparator(h.data[j2], h.data[j1]) { 109 | j = j2 // = 2*i + 2 // right child 110 | } 111 | if !h.comparator(h.data[j], h.data[i]) { 112 | break 113 | } 114 | h.swap(i, j) 115 | i = j 116 | } 117 | 118 | return i > i0 119 | } 120 | -------------------------------------------------------------------------------- /binheap/generic_test.go: -------------------------------------------------------------------------------- 1 | package binheap_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/lispad/go-generics-tools/binheap" 9 | ) 10 | 11 | func TestEmptyHeap(t *testing.T) { 12 | h := binheap.EmptyHeap[string](func(a, b string) bool { return len(a) > len(b) }) 13 | assert.Equal(t, 0, h.Len()) 14 | h.Push("1") 15 | assert.Equal(t, 1, h.Len()) 16 | h.Push("22") 17 | assert.Equal(t, 2, h.Len()) 18 | h.Push("4444") 19 | assert.Equal(t, 3, h.Len()) 20 | h.Push("88888888") 21 | assert.Equal(t, 4, h.Len()) 22 | h.Push("") 23 | assert.Equal(t, 5, h.Len()) 24 | h.Push("55555") 25 | assert.Equal(t, 6, h.Len()) 26 | h.Push("1") 27 | assert.Equal(t, 7, h.Len()) 28 | h.Push("7777777") 29 | assert.Equal(t, 8, h.Len()) 30 | h.Push("999999999") 31 | assert.Equal(t, 9, h.Len()) 32 | h.Push("333") 33 | assert.Equal(t, 10, h.Len()) 34 | 35 | assert.Equal(t, "999999999", h.Peak()) 36 | assert.Equal(t, 10, h.Len()) 37 | assert.Equal(t, "999999999", h.PushPop("4444")) // push less than max value, max will be returned 38 | assert.Equal(t, 10, h.Len()) 39 | assert.Equal(t, "88888888", h.Peak()) 40 | assert.Equal(t, 10, h.Len()) 41 | assert.Equal(t, "0000000000", h.PushPop("0000000000")) // value will be returned, heap is unchanged 42 | assert.Equal(t, 10, h.Len()) 43 | assert.Equal(t, "88888888", h.Peak()) 44 | assert.Equal(t, 10, h.Len()) 45 | 46 | assert.Equal(t, "88888888", h.Replace("22")) 47 | assert.Equal(t, 10, h.Len()) 48 | assert.Equal(t, "7777777", h.Pop()) 49 | assert.Equal(t, 9, h.Len()) 50 | assert.Equal(t, "55555", h.Pop()) 51 | assert.Equal(t, 8, h.Len()) 52 | assert.Equal(t, "4444", h.Pop()) 53 | assert.Equal(t, 7, h.Len()) 54 | assert.Equal(t, "4444", h.Pop()) 55 | assert.Equal(t, 6, h.Len()) 56 | assert.Equal(t, "333", h.Pop()) 57 | assert.Equal(t, 5, h.Len()) 58 | h.Push("999999999") 59 | assert.Equal(t, 6, h.Len()) 60 | assert.Equal(t, "999999999", h.Pop()) 61 | assert.Equal(t, 5, h.Len()) 62 | assert.Equal(t, "22", h.Pop()) 63 | assert.Equal(t, 4, h.Len()) 64 | assert.Equal(t, "22", h.Pop()) 65 | assert.Equal(t, 3, h.Len()) 66 | assert.Equal(t, "1", h.Pop()) 67 | assert.Equal(t, 2, h.Len()) 68 | assert.Equal(t, "1", h.Pop()) 69 | assert.Equal(t, 1, h.Len()) 70 | assert.Equal(t, "", h.Pop()) 71 | assert.Equal(t, 0, h.Len()) 72 | } 73 | -------------------------------------------------------------------------------- /binheap/ordered.go: -------------------------------------------------------------------------------- 1 | package binheap 2 | 3 | import ( 4 | "golang.org/x/exp/constraints" 5 | ) 6 | 7 | // ComparableHeap provides additional Search and Delete methods. 8 | // Could be used for comparable types. 9 | type ComparableHeap[T comparable] struct { 10 | Heap[T] 11 | } 12 | 13 | // EmptyComparableHeap creates heap for comparable types. 14 | func EmptyComparableHeap[T comparable](comparator func(x, y T) bool) ComparableHeap[T] { 15 | return ComparableHeap[T]{ 16 | Heap: EmptyHeap(comparator), 17 | } 18 | } 19 | 20 | // ComparableFromSlice creates heap, based on provided slice. 21 | // Slice could be reordered. 22 | func ComparableFromSlice[T comparable](data []T, comparator func(x, y T) bool) ComparableHeap[T] { 23 | return ComparableHeap[T]{ 24 | Heap: FromSlice(data, comparator), 25 | } 26 | } 27 | 28 | // Search returns if element presents in heap. 29 | func (h *ComparableHeap[T]) Search(x T) bool { 30 | return h.search(x) != -1 31 | } 32 | 33 | // Delete removes element from heap, and returns true if x presents in heap. 34 | func (h *ComparableHeap[T]) Delete(x T) bool { 35 | pos := h.search(x) 36 | if pos == -1 { 37 | return false 38 | } 39 | newLen := h.Len() - 1 40 | h.swap(pos, newLen) 41 | h.data = h.data[:newLen] 42 | 43 | return true 44 | } 45 | 46 | func (h *ComparableHeap[T]) search(x T) int { 47 | for i := range h.data { 48 | if h.data[i] == x { 49 | return i 50 | } 51 | } 52 | return -1 53 | } 54 | 55 | // EmptyMinHeap creates empty Min-Heap for ordered types. 56 | func EmptyMinHeap[T constraints.Ordered]() ComparableHeap[T] { 57 | return EmptyComparableHeap(func(x, y T) bool { 58 | return x < y 59 | }) 60 | } 61 | 62 | // EmptyMaxHeap creates empty Max-Heap for ordered types. 63 | func EmptyMaxHeap[T constraints.Ordered]() ComparableHeap[T] { 64 | return EmptyComparableHeap(func(x, y T) bool { 65 | return x > y 66 | }) 67 | } 68 | 69 | // EmptyMinHeap creates Min-Heap, based on provided slice. 70 | // Slice could be reordered. 71 | func MinHeapFromSlice[T constraints.Ordered](data []T) ComparableHeap[T] { 72 | return ComparableFromSlice(data, func(x, y T) bool { 73 | return x < y 74 | }) 75 | } 76 | 77 | // EmptyMaxHeap creates Max-Heap, based on provided slice. 78 | // Slice could be reordered. 79 | func MaxHeapFromSlice[T constraints.Ordered](data []T) ComparableHeap[T] { 80 | return ComparableFromSlice(data, func(x, y T) bool { 81 | return x > y 82 | }) 83 | } 84 | -------------------------------------------------------------------------------- /binheap/ordered_test.go: -------------------------------------------------------------------------------- 1 | package binheap_test 2 | 3 | import ( 4 | "sort" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "golang.org/x/exp/slices" 9 | 10 | "github.com/lispad/go-generics-tools/binheap" 11 | ) 12 | 13 | func TestMinHeapFromSlice(t *testing.T) { 14 | t.Run("uint8", func(t *testing.T) { 15 | heapWithData := binheap.MinHeapFromSlice([]uint8{5, 10, 1, 5, 2, 7}) 16 | top := make([]uint8, 0) 17 | for heapWithData.Len() > 0 { 18 | top = append(top, heapWithData.Pop()) 19 | } 20 | assert.Equal(t, []uint8{1, 2, 5, 5, 7, 10}, top) 21 | }) 22 | 23 | t.Run("strings", func(t *testing.T) { 24 | heapWithData := binheap.MinHeapFromSlice([]string{"foo", "bar", "foobar", "zzz", "aaa"}) 25 | top := make([]string, 0) 26 | for heapWithData.Len() > 0 { 27 | top = append(top, heapWithData.Pop()) 28 | } 29 | assert.Equal(t, []string{"aaa", "bar", "foo", "foobar", "zzz"}, top) 30 | }) 31 | } 32 | 33 | func TestMaxHeapFromSlice(t *testing.T) { 34 | t.Run("uint8", func(t *testing.T) { 35 | heapWithData := binheap.MaxHeapFromSlice([]uint8{5, 10, 1, 5, 2, 7}) 36 | top := make([]uint8, 0) 37 | for heapWithData.Len() > 0 { 38 | top = append(top, heapWithData.Pop()) 39 | } 40 | assert.Equal(t, []uint8{10, 7, 5, 5, 2, 1}, top) 41 | }) 42 | 43 | t.Run("strings", func(t *testing.T) { 44 | heapWithData := binheap.MaxHeapFromSlice([]string{"foo", "bar", "foobar", "zzz", "aaa"}) 45 | top := make([]string, 0) 46 | for heapWithData.Len() > 0 { 47 | top = append(top, heapWithData.Pop()) 48 | } 49 | assert.Equal(t, []string{"zzz", "foobar", "foo", "bar", "aaa"}, top) 50 | }) 51 | } 52 | 53 | func FuzzEmptyMinHeap(f *testing.F) { 54 | min := binheap.EmptyMinHeap[byte]() 55 | f.Fuzz(func(t *testing.T, data []byte) { 56 | for _, b := range data { 57 | min.Push(b) 58 | } 59 | 60 | sorted := make([]byte, len(data)) 61 | copy(sorted, data) 62 | slices.Sort(sorted) 63 | for _, b := range sorted { 64 | v := min.Pop() 65 | assert.Equal(t, b, v) 66 | } 67 | assert.Equal(t, 0, min.Len()) 68 | }) 69 | } 70 | 71 | func FuzzEmptyMaxHeap(f *testing.F) { 72 | max := binheap.EmptyMaxHeap[byte]() 73 | f.Fuzz(func(t *testing.T, data []byte) { 74 | for _, b := range data { 75 | max.Push(b) 76 | } 77 | reverseSorted := make([]byte, len(data)) 78 | copy(reverseSorted, data) 79 | sort.Slice(reverseSorted, func(i, j int) bool { 80 | return reverseSorted[i] > reverseSorted[j] 81 | }) 82 | for _, b := range reverseSorted { 83 | v := max.Pop() 84 | assert.Equal(t, b, v) 85 | } 86 | assert.Equal(t, 0, max.Len()) 87 | }) 88 | } 89 | 90 | func TestEmptyComparableHeapHeap(t *testing.T) { 91 | h := binheap.EmptyComparableHeap[string](func(a, b string) bool { return len(a) > len(b) }) 92 | assert.Equal(t, 0, h.Len()) 93 | assert.False(t, h.Search("333")) 94 | assert.False(t, h.Delete("333")) 95 | 96 | h.Push("1") 97 | assert.Equal(t, 1, h.Len()) 98 | assert.False(t, h.Search("333")) 99 | assert.False(t, h.Delete("333")) 100 | assert.True(t, h.Search("1")) 101 | 102 | h.Push("22") 103 | assert.Equal(t, 2, h.Len()) 104 | assert.True(t, h.Search("1")) 105 | assert.True(t, h.Delete("1")) 106 | assert.False(t, h.Search("1")) 107 | assert.True(t, h.Search("22")) 108 | assert.Equal(t, 1, h.Len()) 109 | 110 | h.Push("4444") 111 | h.Push("88888888") 112 | h.Push("55555") 113 | h.Push("1") 114 | h.Push("4444") 115 | h.Push("4444") 116 | h.Push("7777777") 117 | assert.Equal(t, 8, h.Len()) 118 | 119 | assert.True(t, h.Search("88888888")) 120 | assert.True(t, h.Delete("88888888")) // test root deletion 121 | assert.False(t, h.Search("88888888")) 122 | assert.False(t, h.Delete("88888888")) 123 | 124 | assert.True(t, h.Search("4444")) // first entry 125 | assert.True(t, h.Delete("4444")) 126 | assert.True(t, h.Search("4444")) // second entry 127 | assert.True(t, h.Delete("4444")) 128 | assert.True(t, h.Search("4444")) // third entry 129 | assert.True(t, h.Delete("4444")) 130 | assert.False(t, h.Search("4444")) // no more "4444" 131 | assert.False(t, h.Delete("4444")) 132 | } 133 | -------------------------------------------------------------------------------- /binheap/topn.go: -------------------------------------------------------------------------------- 1 | package binheap 2 | 3 | import ( 4 | "golang.org/x/exp/constraints" 5 | "golang.org/x/exp/slices" 6 | ) 7 | 8 | // TopN mutates data slice, moving TopN elements of slice to the beginning and returns subslice of first n elements. 9 | // O(M *ln(N)), where M is slice size 10 | // Mutates source data slice. No additional allocations are done. 11 | func TopN[T any](data []T, n int, comparator func(x, y T) bool) []T { 12 | if n > len(data) { 13 | n = len(data) 14 | } 15 | heap := FromSlice(data[0:n], reverse(comparator)) 16 | for i := n; i < len(data); i++ { 17 | if !comparator(data[0], data[i]) { 18 | data[0], data[i] = data[i], data[0] 19 | heap.fix(0) 20 | } 21 | } 22 | slices.SortFunc(data[0:n], comparator) 23 | return data[0:n] 24 | } 25 | 26 | // MinN moves N Min elements of slice to the beginning and returns subslice of first n elements. 27 | // Mutates source data slice. 28 | // No additional allocations are done. 29 | func MinN[T constraints.Ordered](data []T, n int) []T { 30 | return TopN(data, n, func(x, y T) bool { 31 | return x < y 32 | }) 33 | } 34 | 35 | // MaxN moves N Max elements of slice to the beginning and returns subslice of first n elements. 36 | // Mutates source data slice. 37 | // No additional allocations are done. 38 | func MaxN[T constraints.Ordered](data []T, n int) []T { 39 | return TopN(data, n, func(x, y T) bool { 40 | return x > y 41 | }) 42 | } 43 | 44 | func reverse[T any](comparator func(x, y T) bool) func(x, y T) bool { 45 | return func(x, y T) bool { 46 | return !comparator(x, y) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /binheap/topn_bench_test.go: -------------------------------------------------------------------------------- 1 | package binheap_test 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | 7 | "golang.org/x/exp/slices" 8 | 9 | "github.com/lispad/go-generics-tools/binheap" 10 | ) 11 | 12 | func BenchmarkSortedMaxN(b *testing.B) { 13 | sl := testSlice(b.N) 14 | k := 5 15 | b.ReportAllocs() 16 | b.ResetTimer() 17 | 18 | if k > len(sl) { 19 | k = len(sl) 20 | } 21 | slices.Sort(sl) 22 | result := sl[:k] 23 | _ = result 24 | } 25 | 26 | func BenchmarkMaxNImmutable(b *testing.B) { 27 | sl := testSlice(b.N) 28 | k := 5 29 | b.ReportAllocs() 30 | b.ResetTimer() 31 | 32 | if k > len(sl) { 33 | k = len(sl) 34 | } 35 | result := binheap.MaxNImmutable(sl, k) 36 | _ = result 37 | } 38 | 39 | func BenchmarkMaxN(b *testing.B) { 40 | sl := testSlice(b.N) 41 | k := 5 42 | b.ReportAllocs() 43 | b.ResetTimer() 44 | 45 | if k > len(sl) { 46 | k = len(sl) 47 | } 48 | result := binheap.MinN(sl, k) 49 | _ = result 50 | } 51 | 52 | func testSlice(size int) []int32 { 53 | sl := make([]int32, size) 54 | for i := 0; i < size; i++ { 55 | sl[i] = rand.Int31() 56 | } 57 | return sl 58 | } 59 | -------------------------------------------------------------------------------- /binheap/topn_test.go: -------------------------------------------------------------------------------- 1 | package binheap_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "golang.org/x/exp/slices" 8 | 9 | "github.com/lispad/go-generics-tools/binheap" 10 | ) 11 | 12 | func TestMinN(t *testing.T) { 13 | t.Run("float32", func(t *testing.T) { 14 | source := []float32{11.3, 5, 10.12, 1, 5, 2, 7} 15 | input := make([]float32, len(source)) 16 | copy(input, source) 17 | 18 | min3 := binheap.MinN(input, 3) 19 | assert.Equal(t, []float32{1, 2, 5}, min3) 20 | assert.Equal(t, []float32{1, 2, 5, 11.3, 10.12, 5, 7}, input) // check that rest of elements are kept 21 | 22 | min5 := binheap.MinN(input, 5) 23 | assert.Equal(t, []float32{1, 2, 5, 5, 7}, min5) 24 | assert.ElementsMatch(t, input, source) 25 | 26 | min10 := binheap.MinN(input, 10) 27 | assert.Equal(t, []float32{1, 2, 5, 5, 7, 10.12, 11.3}, min10) 28 | assert.ElementsMatch(t, input, source) 29 | }) 30 | 31 | t.Run("strings", func(t *testing.T) { 32 | source := []string{"foo", "bar", "foobar", "zzz", "aaa", "some more text"} 33 | input := make([]string, len(source)) 34 | copy(input, source) 35 | 36 | min1 := binheap.MinN(input, 1) 37 | assert.Equal(t, []string{"aaa"}, min1) 38 | assert.ElementsMatch(t, input, source) 39 | 40 | min3 := binheap.MinN(input, 3) 41 | assert.Equal(t, []string{"aaa", "bar", "foo"}, min3) 42 | assert.ElementsMatch(t, input, source) 43 | }) 44 | } 45 | 46 | func TestMaxN(t *testing.T) { 47 | t.Run("int16", func(t *testing.T) { 48 | source := []int16{5, 1, 5, 2, 7, -111} 49 | input := make([]int16, len(source)) 50 | copy(input, source) 51 | 52 | max3 := binheap.MaxN(input, 3) 53 | assert.Equal(t, []int16{7, 5, 5}, max3) 54 | assert.ElementsMatch(t, input, source) 55 | 56 | max5 := binheap.MaxN(input, 5) 57 | assert.Equal(t, []int16{7, 5, 5, 2, 1}, max5) 58 | assert.ElementsMatch(t, input, source) 59 | 60 | max10 := binheap.MaxN(input, 10) 61 | assert.Equal(t, []int16{7, 5, 5, 2, 1, -111}, max10) 62 | assert.ElementsMatch(t, input, source) 63 | }) 64 | 65 | t.Run("strings", func(t *testing.T) { 66 | source := []string{"foo", "bar", "foobar", "zzz", "aaa", "some more text"} 67 | input := make([]string, len(source)) 68 | copy(input, source) 69 | 70 | max1 := binheap.MaxN(input, 1) 71 | assert.Equal(t, []string{"zzz"}, max1) 72 | assert.ElementsMatch(t, input, source) 73 | 74 | max3 := binheap.MaxN(input, 3) 75 | assert.Equal(t, []string{"zzz", "some more text", "foobar"}, max3) 76 | assert.ElementsMatch(t, input, source) 77 | }) 78 | } 79 | 80 | func FuzzTopN(f *testing.F) { 81 | f.Add([]byte{1, 10, 21, 211, 12, 2, 7, 13, 10}, uint(5)) 82 | f.Fuzz(func(t *testing.T, data []byte, i uint) { 83 | n := int(i) 84 | if n > len(data) { 85 | n = len(data) 86 | } 87 | 88 | sorted := make([]byte, len(data)) 89 | copy(sorted, data) 90 | slices.Sort(sorted) 91 | 92 | topN := binheap.MinN(data, n) 93 | assert.Equal(t, sorted[0:n], topN) // top should match with order 94 | assert.ElementsMatch(t, sorted[n:], data[n:]) // rest of elements should match but could be unsorted 95 | 96 | slices.Sort(data) 97 | topN = binheap.MinN(data, n) 98 | assert.Equal(t, sorted[0:n], topN) // top should match with order 99 | assert.Equal(t, sorted, data) // data should not be changed if it was already sorted 100 | }) 101 | } 102 | -------------------------------------------------------------------------------- /binheap/topstream.go: -------------------------------------------------------------------------------- 1 | package binheap 2 | 3 | import ( 4 | "golang.org/x/exp/constraints" 5 | "golang.org/x/exp/slices" 6 | ) 7 | 8 | // TopNHeap keeps N top elements in reverse order. 9 | type TopNHeap[T any] struct { 10 | comparator func(x, y T) bool 11 | heap Heap[T] 12 | maxSize int 13 | } 14 | 15 | // EmptyTopNHeap creates new heap for storing n top elements. 16 | func EmptyTopNHeap[T any](n int, comparator func(x, y T) bool) TopNHeap[T] { 17 | return TopNHeap[T]{ 18 | comparator: comparator, 19 | heap: EmptyHeap(reverse(comparator)), 20 | maxSize: n, 21 | } 22 | } 23 | 24 | // Push stores x, if x is better than top n, or ignores otherwise. 25 | func (h *TopNHeap[T]) Push(x T) { 26 | if h.heap.Len() < h.maxSize { 27 | h.heap.Push(x) 28 | return 29 | } 30 | 31 | if !h.comparator(h.heap.data[0], x) { 32 | h.heap.Replace(x) 33 | } 34 | } 35 | 36 | // PopTopN returns current top N elements, and empties slice. 37 | func (h *TopNHeap[T]) PopTopN() []T { 38 | result := h.PeekTopN() 39 | h.heap.data = h.heap.data[:0] 40 | 41 | return result 42 | } 43 | 44 | // PeekTopN returns current top N elements. 45 | func (h *TopNHeap[T]) PeekTopN() []T { 46 | result := make([]T, h.heap.Len()) 47 | copy(result, h.heap.data) 48 | slices.SortFunc(result, h.comparator) 49 | 50 | return result 51 | } 52 | 53 | // TopNImmutable returns top n elements from slice. 54 | // O(M *ln(N)), where M is slice size 55 | // Allocates new slice, source data isn't mutated. O(N) additional allocations. 56 | func TopNImmutable[T any](data []T, n int, comparator func(x, y T) bool) []T { 57 | h := EmptyTopNHeap(n, comparator) 58 | for _, x := range data { 59 | h.Push(x) 60 | } 61 | return h.PopTopN() 62 | } 63 | 64 | // MinNImmutable return n minimal elements from slice. 65 | // O(M *ln(N)), where M is slice size 66 | // Allocates new slice, source data isn't mutated. O(N) additional allocations. 67 | func MinNImmutable[T constraints.Ordered](data []T, n int) []T { 68 | return TopNImmutable(data, n, func(x, y T) bool { 69 | return x < y 70 | }) 71 | } 72 | 73 | // MaxNImmutable return n maximal elements from slice. 74 | // O(M *ln(N)), where M is slice size 75 | // Allocates new slice, source data isn't mutated. O(N) additional allocations. 76 | func MaxNImmutable[T constraints.Ordered](data []T, n int) []T { 77 | return TopNImmutable(data, n, func(x, y T) bool { 78 | return x > y 79 | }) 80 | } 81 | -------------------------------------------------------------------------------- /binheap/topstream_test.go: -------------------------------------------------------------------------------- 1 | package binheap_test 2 | 3 | import ( 4 | "sort" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "golang.org/x/exp/slices" 9 | 10 | "github.com/lispad/go-generics-tools/binheap" 11 | ) 12 | 13 | func TestMaxNImmutable(t *testing.T) { 14 | source := []string{"foo", "bar", "baz", "zzz", "aaa"} 15 | input := make([]string, len(source)) 16 | copy(input, source) 17 | 18 | max := binheap.MaxNImmutable(input, 3) 19 | assert.Equal(t, source, input) // input should not be mutated 20 | assert.Equal(t, []string{"zzz", "foo", "baz"}, max) 21 | } 22 | 23 | func TestMinNImmutable(t *testing.T) { 24 | source := []string{"foo", "bar", "baz", "zzz", "aaa"} 25 | input := make([]string, len(source)) 26 | copy(input, source) 27 | 28 | max := binheap.MinNImmutable(input, 3) 29 | assert.Equal(t, source, input) // input should not be mutated 30 | assert.Equal(t, []string{"aaa", "bar", "baz"}, max) 31 | } 32 | 33 | func FuzzTopNImmutable(f *testing.F) { 34 | f.Fuzz(func(t *testing.T, data []byte, i uint) { 35 | n := int(i) 36 | if n == 0 { 37 | n = 1 38 | } 39 | if n > len(data) { 40 | n = len(data) 41 | } 42 | 43 | input := make([]byte, len(data)) 44 | copy(input, data) 45 | 46 | sorted := make([]byte, len(data)) 47 | copy(sorted, data) 48 | slices.Sort(sorted) 49 | 50 | topN := binheap.MinNImmutable(input, n) 51 | assert.Equal(t, sorted[0:n], topN) // top should match with order 52 | assert.Equal(t, data, input) // input should not be mutated 53 | 54 | topN = binheap.MaxNImmutable(input, n) 55 | // reverse sort 56 | sort.Slice(sorted, func(i, j int) bool { 57 | return sorted[i] > sorted[j] 58 | }) 59 | assert.Equal(t, sorted[0:n], topN) // top should match with order 60 | assert.Equal(t, data, input) // input should not be mutated 61 | }) 62 | } 63 | -------------------------------------------------------------------------------- /examples/heap/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/lispad/go-generics-tools/binheap" 7 | ) 8 | 9 | type someStruct struct { 10 | someData []int // make struct non comparable 11 | } 12 | 13 | func main() { 14 | heap := binheap.EmptyHeap[someStruct](func(x, y someStruct) bool { return len(x.someData) > len(y.someData) }) 15 | heap.Push(someStruct{someData: []int{1, 2, 3, 4, 5, 6}}) 16 | heap.Push(someStruct{someData: nil}) 17 | heap.Push(someStruct{someData: []int{1, 2, 3}}) 18 | heap.Push(someStruct{someData: []int{11}}) 19 | heap.Push(someStruct{someData: []int{11, 2222, 3333, 44}}) 20 | heap.Push(someStruct{someData: []int{1}}) 21 | heap.Push(someStruct{someData: nil}) 22 | heap.Push(someStruct{someData: []int{1111}}) 23 | heap.Push(someStruct{someData: []int{1, 2}}) 24 | 25 | fmt.Printf("Heap has len: %d\nElements sorted by slice length:\n\n", heap.Len()) 26 | for heap.Len() > 0 { 27 | fmt.Printf(" %v\n", heap.Pop()) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /examples/topn/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/lispad/go-generics-tools/binheap" 7 | ) 8 | 9 | func main() { 10 | someData := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0} 11 | fmt.Printf("--- top 3 min ---\n source slice: %v\n", someData) 12 | mins := binheap.MinN[float64](someData, 3) 13 | fmt.Printf(" top 3 min elements: %v\n", mins) 14 | fmt.Printf(" slice was mutated: %v\n\n", someData) 15 | 16 | someIntData := []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0} 17 | fmt.Printf("--- top 4 max ---\n source slice: %v\n", someIntData) 18 | maxs := binheap.MaxN[int64](someIntData, 4) 19 | fmt.Printf(" top 4 max elements: %v\n\n", maxs) 20 | fmt.Printf(" slice was mutated: %v\n\n", someIntData) 21 | 22 | someFloat32Data := []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0} 23 | fmt.Printf("--- top 5 min without slice mutation ---\n source slice: %v\n", someFloat32Data) 24 | minsFloat32 := binheap.MinNImmutable[float32](someFloat32Data, 5) 25 | fmt.Printf(" top 5 min elements: %v\n", minsFloat32) 26 | fmt.Printf(" slice was not mutated: %v\n\n", someFloat32Data) 27 | 28 | someInt32Data := []int32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0} 29 | fmt.Printf("--- top 4 by remainder of devicion to 3 ---\n source slice: %v\n", someInt32Data) 30 | topByDivisionTo3Remainder := binheap.TopN[int64](someIntData, 4, func(x, y int64) bool { return x%3 > y%3 }) 31 | fmt.Printf(" top 4 elements: %v\n", topByDivisionTo3Remainder) 32 | fmt.Printf(" slice was mutated: %v\n\n", someIntData) 33 | } 34 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/lispad/go-generics-tools 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/stretchr/testify v1.7.1 7 | golang.org/x/exp v0.0.0-20220609121020-a51bd0440498 8 | ) 9 | 10 | require ( 11 | github.com/davecgh/go-spew v1.1.0 // indirect 12 | github.com/pmezard/go-difflib v1.0.0 // indirect 13 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect 14 | ) 15 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 4 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 5 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 6 | github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= 7 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 8 | golang.org/x/exp v0.0.0-20220328175248-053ad81199eb h1:pC9Okm6BVmxEw76PUu0XUbOTQ92JX11hfvqTjAV3qxM= 9 | golang.org/x/exp v0.0.0-20220328175248-053ad81199eb/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= 10 | golang.org/x/exp v0.0.0-20220609121020-a51bd0440498 h1:TF0FvLUGEq/8wOt/9AV1nj6D4ViZGUIGCMQfCv7VRXY= 11 | golang.org/x/exp v0.0.0-20220609121020-a51bd0440498/go.mod h1:yh0Ynu2b5ZUe3MQfp2nM0ecK7wsgouWTDN0FNeJuIys= 12 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 13 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 14 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 15 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 16 | -------------------------------------------------------------------------------- /smap/README.md: -------------------------------------------------------------------------------- 1 | # Sharded RWLocked Map, using go generics 2 | [![MIT License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) 3 | 4 | Introduction 5 | ------------ 6 | 7 | In some scenarios sync.Map could allocate large space for dirty-read copies, or even heavily use locks (when getting 8 | missing values, that should be rechecked in map with lock). In such scenarios go internal map with rw-mutex, divided to 9 | several shards could perform much better, with cost of memory for additional locks storage. But this amount could be 10 | much less, than sync.map uses. 11 | 12 | Interface incompatibility 13 | ------------ 14 | 15 | - `LoadOrStore` method changed to `LoadOrCreate`, with callback that generates value. Could be used to avoid 16 | unnecessary creating huge values, in case if key already exists. 17 | 18 | Usage Example 19 | ----------------- 20 | 21 | package main 22 | 23 | import ( 24 | "fmt" 25 | 26 | "github.com/lispad/go-generics-tools/smap" 27 | ) 28 | 29 | func main() { 30 | m := smap.NewIntegerComparable[int, int](8, 128) 31 | m.Store(123, 456) 32 | 33 | value, ok := m.Load(123) 34 | fmt.Printf("%d, %t", value, ok) 35 | } 36 | 37 | A bit more examples could be found in tests. 38 | 39 | Benchmark 40 | ----------------- 41 | 42 | 43 | #### Benchmark 44 | 45 | Benchmark performed on Lenovo Ideapad laptop with AMD Ryzen 7 4700U, Linux Mint 20.3 with 5.13.0 kernel 46 | 47 | BenchmarkIntegerSMap_ConcurrentGet-8 82540485 12.95 ns/op 0 B/op 0 allocs/op 48 | BenchmarkSyncMap_ConcurrentGet-8 73431339 18.35 ns/op 0 B/op 0 allocs/op 49 | BenchmarkLockMap_ConcurrentGet-8 19327282 54.03 ns/op 0 B/op 0 allocs/op 50 | BenchmarkIntegerShardedMap_ConcurrentSet-8 25605380 42.33 ns/op 0 B/op 0 allocs/op 51 | BenchmarkSyncMap_ConcurrentSet-8 2138496 536.1 ns/op 36 B/op 3 allocs/op 52 | BenchmarkLockMap_ConcurrentSet-8 3827476 302.9 ns/op 0 B/op 0 allocs/op 53 | BenchmarkIntegerShardedMap_ConcurrentGetSet5-8 3377473 357.1 ns/op 0 B/op 0 allocs/op 54 | BenchmarkSyncMap_ConcurrentGetSet5-8 323318 3540 ns/op 266 B/op 3 allocs/op 55 | BenchmarkLockMap_ConcurrentGetSet5-8 204633 6176 ns/op 0 B/op 0 allocs/op 56 | BenchmarkIntegerShardedMap_ConcurrentGetSet50-8 18236305 65.03 ns/op 0 B/op 0 allocs/op 57 | BenchmarkSyncMap_ConcurrentGetSet50-8 18337423 56.55 ns/op 32 B/op 2 allocs/op 58 | BenchmarkLockMap_ConcurrentGetSet50-8 2697315 431.2 ns/op 0 B/op 0 allocs/op 59 | BenchmarkIntegerShardedMap_ConcurrentGetSet1-8 1003506 1076 ns/op 0 B/op 0 allocs/op 60 | BenchmarkSyncMap_ConcurrentGetSet1-8 32668 44423 ns/op 3508 B/op 5 allocs/op 61 | BenchmarkLockMap_ConcurrentGetSet1-8 78486 16019 ns/op 0 B/op 0 allocs/op 62 | 63 | Sharded Lock map is approximately equal to sync.Map on 50% read + 50% concurrent writes, and is much faster on 64 | 5% writes+95% reads, and 1% writes+99% reads. 65 | Also sharded map allocated about 40x less memory in 5% writes+95% reads scenario, than sync.Map does. 66 | 67 | Compatibility 68 | ------------- 69 | Minimal Golang version is 1.18. Generics are used. 70 | 71 | Installation 72 | ---------------------- 73 | 74 | To install package, run: 75 | 76 | go get github.com/lispad/go-generics-tools/smap 77 | 78 | License 79 | ------- 80 | 81 | The smap package is licensed under the MIT license. Please see the LICENSE file for details. 82 | -------------------------------------------------------------------------------- /smap/comparable.go: -------------------------------------------------------------------------------- 1 | package smap 2 | 3 | // GenericComparable stores data in N shards, with rw mutex for each. 4 | // Additional CompareAndSwap method added for comparable values. 5 | type GenericComparable[K comparable, V comparable] struct { 6 | Generic[K, V] 7 | } 8 | 9 | // NewGenericComparable creates generic RWLocked Sharded map for comparable values. 10 | // shardDetector should be idempotent function. 11 | func NewGenericComparable[K comparable, V comparable](shardsCount, defaultSize int, shardDetector func(key K) int) GenericComparable[K, V] { 12 | return GenericComparable[K, V]{ 13 | Generic: NewGeneric[K, V](shardsCount, defaultSize, shardDetector), 14 | } 15 | } 16 | 17 | // CompareAndSwap executes the compare-and-swap operation for the Key & Value pair. 18 | // If and only if key exists, and value for key equals old, value will be changed to new. 19 | // Otherwise, returns current value. 20 | // The ok result indicates whether value was changed to new in the map. 21 | func (sm GenericComparable[K, V]) CompareAndSwap(key K, old, new V) (V, bool) { 22 | shardID := sm.shardDetector(key) 23 | sm.locks[shardID].Lock() 24 | if current, ok := sm.shards[shardID][key]; ok && current == old { 25 | sm.shards[shardID][key] = new 26 | sm.locks[shardID].Unlock() 27 | return new, true 28 | } else { 29 | sm.locks[shardID].Unlock() 30 | return current, false 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /smap/generic.go: -------------------------------------------------------------------------------- 1 | package smap 2 | 3 | import ( 4 | "runtime" 5 | "sync" 6 | ) 7 | 8 | // Generic stores data in N shards, with rw mutex for each. 9 | type Generic[K comparable, V any] struct { 10 | shards []map[K]V 11 | locks []sync.RWMutex 12 | shardDetector func(key K) int 13 | } 14 | 15 | // NewGeneric creates generic RWLocked Sharded map. 16 | // shardDetector should be idempotent function. 17 | func NewGeneric[K comparable, V any](shardsCount, defaultSize int, shardDetector func(key K) int) Generic[K, V] { 18 | sm := Generic[K, V]{ 19 | shards: make([]map[K]V, shardsCount), 20 | locks: make([]sync.RWMutex, shardsCount), 21 | shardDetector: shardDetector, 22 | } 23 | for i := 0; i < shardsCount; i++ { 24 | sm.shards[i] = make(map[K]V, defaultSize) 25 | sm.locks[i] = sync.RWMutex{} 26 | } 27 | return sm 28 | } 29 | 30 | // Load returns the value stored in the map for a key, or nil if no value is present. 31 | // The ok result indicates whether value was found in the map. 32 | func (sm Generic[K, V]) Load(key K) (V, bool) { 33 | shardID := sm.shardDetector(key) 34 | sm.locks[shardID].RLock() 35 | value, ok := sm.shards[shardID][key] 36 | sm.locks[shardID].RUnlock() 37 | return value, ok 38 | } 39 | 40 | // Store sets the value for a key. 41 | func (sm Generic[K, V]) Store(key K, value V) { 42 | shardID := sm.shardDetector(key) 43 | sm.locks[shardID].Lock() 44 | sm.shards[shardID][key] = value 45 | sm.locks[shardID].Unlock() 46 | } 47 | 48 | // LoadAndDelete deletes the value for a key, returning the previous value if any. 49 | // The loaded result reports whether the key was present. 50 | func (sm Generic[K, V]) LoadAndDelete(key K) (V, bool) { 51 | shardID := sm.shardDetector(key) 52 | sm.locks[shardID].Lock() 53 | value, ok := sm.shards[shardID][key] 54 | if ok { 55 | delete(sm.shards[shardID], key) 56 | } 57 | sm.locks[shardID].Unlock() 58 | return value, ok 59 | } 60 | 61 | // LoadOrCreate returns the existing value for the key if present. 62 | // Otherwise, it calls generator func, stores and returns the generator's result. 63 | // Generator will not be called if key present. 64 | // The loaded result is true if the value was loaded, false if stored. 65 | func (sm Generic[K, V]) LoadOrCreate(key K, generator func() V) (V, bool) { 66 | shardID := sm.shardDetector(key) 67 | sm.locks[shardID].RLock() 68 | value, ok := sm.shards[shardID][key] 69 | sm.locks[shardID].RUnlock() 70 | if ok { 71 | return value, ok 72 | } 73 | 74 | sm.locks[shardID].Lock() 75 | value, ok = sm.shards[shardID][key] 76 | if !ok { 77 | value = generator() 78 | sm.shards[shardID][key] = value 79 | } 80 | sm.locks[shardID].Unlock() 81 | return value, ok 82 | } 83 | 84 | // Delete deletes the value for a key. 85 | func (sm Generic[K, V]) Delete(key K) { 86 | shardID := sm.shardDetector(key) 87 | sm.locks[shardID].Lock() 88 | delete(sm.shards[shardID], key) 89 | sm.locks[shardID].Unlock() 90 | } 91 | 92 | // Range calls cb sequentially for each key and value present in the map. 93 | // If cb returns false, range stops the iteration. 94 | // 95 | // Range performs like sync.Range: does not correspond to any consistent snapshot of the Map's contents: 96 | // no key will be visited more than once, but if the value for any key 97 | // is stored or deleted concurrently (including by cb), Range may reflect any 98 | // mapping for that key from any point during the Range call. Range does not 99 | // block other methods on the receiver; even cb itself may call any method on sm. 100 | // 101 | // Range may be O(N) with the number of elements in the map even if f returns 102 | // false after a constant number of calls. 103 | func (sm Generic[K, V]) Range(cb func(K, V) bool) { 104 | keys := make([]K, 0) 105 | for i := range sm.locks { 106 | keys = keys[:0] 107 | sm.locks[i].RLock() 108 | for k := range sm.shards[i] { 109 | keys = append(keys, k) 110 | } 111 | sm.locks[i].RUnlock() 112 | 113 | for _, key := range keys { 114 | sm.locks[i].RLock() 115 | value, ok := sm.shards[i][key] 116 | sm.locks[i].RUnlock() 117 | if ok { 118 | if !cb(key, value) { 119 | return 120 | } 121 | } 122 | } 123 | } 124 | } 125 | 126 | // ShardID returns shard number for given key. 127 | func (sm Generic[K, V]) ShardID(key K) int { 128 | return sm.shardDetector(key) 129 | } 130 | 131 | // ShardsCount returns shards count, given on initialisation. 132 | func (sm Generic[K, V]) ShardsCount() int { 133 | return len(sm.locks) 134 | } 135 | 136 | // LockShard locks shard with given id. 137 | // Could be useful with Unblocked* functions. Other calls to sm could be locked. 138 | // Use with caution, only when benchmark shows significant performance changes. 139 | func (sm Generic[K, V]) LockShard(id int) { 140 | sm.locks[id].Lock() 141 | } 142 | 143 | // RLockShard locks for read shard with given id. 144 | // Could be useful with Unblocked* functions. Other calls to sm could be rlocked. 145 | // Use with caution, only when benchmark shows significant performance changes. 146 | func (sm Generic[K, V]) RLockShard(id int) { 147 | sm.locks[id].RLock() 148 | } 149 | 150 | // UnlockShard unlocks shard with given id. 151 | func (sm Generic[K, V]) UnlockShard(id int) { 152 | sm.locks[id].Unlock() 153 | } 154 | 155 | // RUnlockShard unlocks for read shard with given id. 156 | func (sm Generic[K, V]) RUnlockShard(id int) { 157 | sm.locks[id].RUnlock() 158 | } 159 | 160 | // UnblockedGet returns value, without locks. 161 | // Use with caution, only when lock or rlock were taken for shard. 162 | func (sm Generic[K, V]) UnblockedGet(key K) (V, bool) { 163 | value, ok := sm.shards[sm.shardDetector(key)][key] 164 | return value, ok 165 | } 166 | 167 | // UnblockedSet sets value, without locks. 168 | // Use with caution, only when lock were taken for shard. 169 | func (sm Generic[K, V]) UnblockedSet(key K, value V) { 170 | sm.shards[sm.shardDetector(key)][key] = value 171 | } 172 | 173 | // UnblockedShardRange calls cb sequentially for each key and value present in the maps shard. 174 | // Use with caution, only when lock or rlock were taken for shard. 175 | func (sm Generic[K, V]) UnblockedShardRange(shardID int, cb func(key K, value V) bool) { 176 | for key, value := range sm.shards[shardID] { 177 | if !cb(key, value) { 178 | break 179 | } 180 | } 181 | } 182 | 183 | // HeuristicOptimalShardsCount returns shards count. 184 | // Use with caution, It's point for change, and could be changed in the future. 185 | func HeuristicOptimalShardsCount() int { 186 | procs := runtime.GOMAXPROCS(-1) 187 | return procs * procs * 24 188 | } 189 | 190 | // HeuristicOptimalDistribution returns shards count, and shard size for given map size. 191 | // Use with caution, It's point for change, and could be changed in the future. 192 | func HeuristicOptimalDistribution(expectedItemsCount int) (int, int) { 193 | shards := HeuristicOptimalShardsCount() 194 | shardSize := expectedItemsCount/shards + 1 195 | return shards, shardSize 196 | } 197 | -------------------------------------------------------------------------------- /smap/integer.go: -------------------------------------------------------------------------------- 1 | package smap 2 | 3 | import ( 4 | "golang.org/x/exp/constraints" 5 | ) 6 | 7 | // NewInteger creates sharded rwlock maps with shard detection based on key division to shards count modulo. 8 | func NewInteger[K constraints.Integer, V any](shardsCount, defaultSize int) Generic[K, V] { 9 | return NewGeneric[K, V](shardsCount, defaultSize, func(key K) int { 10 | return int(key) % shardsCount 11 | }) 12 | } 13 | 14 | // NewIntegerComparable creates sharded rwlock maps with comparable values. 15 | func NewIntegerComparable[K constraints.Integer, V comparable](shardsCount, defaultSize int) GenericComparable[K, V] { 16 | return NewGenericComparable[K, V](shardsCount, defaultSize, func(key K) int { 17 | return int(key) % shardsCount 18 | }) 19 | } 20 | -------------------------------------------------------------------------------- /smap/integer_bench_test.go: -------------------------------------------------------------------------------- 1 | package smap_test 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "math/rand" 7 | "runtime" 8 | "sync" 9 | "testing" 10 | 11 | "github.com/lispad/go-generics-tools/smap" 12 | ) 13 | 14 | func BenchmarkIntegerSMap_ConcurrentGet(b *testing.B) { 15 | sm := smap.NewIntegerComparable[uint16, uint64](smap.HeuristicOptimalDistribution(math.MaxUint16)) 16 | for i := uint16(0); i < math.MaxUint16; i++ { 17 | sm.Store(i, uint64(i)) 18 | } 19 | 20 | b.ReportAllocs() 21 | b.ResetTimer() 22 | b.RunParallel(func(pb *testing.PB) { 23 | i := uint16(rand.Uint32()) 24 | for pb.Next() { 25 | sm.Load(i) 26 | i++ 27 | } 28 | }) 29 | } 30 | 31 | func BenchmarkSyncMap_ConcurrentGet(b *testing.B) { 32 | sm := sync.Map{} 33 | for i := uint16(0); i < math.MaxUint16; i++ { 34 | sm.Store(i, uint64(i)) 35 | } 36 | 37 | b.ReportAllocs() 38 | b.ResetTimer() 39 | b.RunParallel(func(pb *testing.PB) { 40 | i := uint16(rand.Uint32()) 41 | for pb.Next() { 42 | sm.Load(i) 43 | i++ 44 | } 45 | }) 46 | } 47 | 48 | func BenchmarkLockMap_ConcurrentGet(b *testing.B) { 49 | sm := make(map[uint16]uint64) 50 | var mutex sync.RWMutex 51 | for i := uint16(0); i < math.MaxUint16; i++ { 52 | sm[i] = uint64(i) 53 | } 54 | 55 | b.ReportAllocs() 56 | b.ResetTimer() 57 | b.RunParallel(func(pb *testing.PB) { 58 | i := uint16(rand.Uint32()) 59 | var ( 60 | val uint64 61 | ok bool 62 | ) 63 | for pb.Next() { 64 | mutex.RLock() 65 | val, ok = sm[i] 66 | mutex.RUnlock() 67 | i++ 68 | } 69 | _ = val 70 | _ = ok 71 | }) 72 | } 73 | 74 | func BenchmarkIntegerShardedMap_ConcurrentSet(b *testing.B) { 75 | sm := smap.NewIntegerComparable[uint16, uint64](smap.HeuristicOptimalDistribution(math.MaxUint16)) 76 | b.ReportAllocs() 77 | b.ResetTimer() 78 | b.RunParallel(func(pb *testing.PB) { 79 | i := uint16(rand.Uint32()) 80 | for pb.Next() { 81 | sm.Store(i, uint64(i)) 82 | i++ 83 | } 84 | }) 85 | } 86 | 87 | func BenchmarkSyncMap_ConcurrentSet(b *testing.B) { 88 | sm := sync.Map{} 89 | b.ReportAllocs() 90 | b.ResetTimer() 91 | b.RunParallel(func(pb *testing.PB) { 92 | i := uint16(rand.Uint32()) 93 | for pb.Next() { 94 | sm.Store(i, uint64(i)) 95 | i++ 96 | } 97 | }) 98 | } 99 | 100 | func BenchmarkLockMap_ConcurrentSet(b *testing.B) { 101 | sm := make(map[uint16]uint64) 102 | var mutex sync.RWMutex 103 | 104 | b.ReportAllocs() 105 | b.ResetTimer() 106 | b.RunParallel(func(pb *testing.PB) { 107 | i := uint16(rand.Uint32()) 108 | for pb.Next() { 109 | mutex.Lock() 110 | sm[i] = uint64(i) 111 | mutex.Unlock() 112 | i++ 113 | } 114 | }) 115 | } 116 | 117 | func BenchmarkIntegerShardedMap_ConcurrentGetSet5(b *testing.B) { 118 | startMemory, endMemory := prepareMemStats() 119 | benchmarkIntegerShardedMapConcurrentGetSet(b, 19) // 95% reads, 5% writes 120 | reportMemStats("sharded rwlocked map", &startMemory, &endMemory) 121 | } 122 | 123 | func BenchmarkSyncMap_ConcurrentGetSet5(b *testing.B) { 124 | startMemory, endMemory := prepareMemStats() 125 | benchmarkSyncMapConcurrentGetSet(b, 19) // 95% reads, 5% writes 126 | reportMemStats("sync map", &startMemory, &endMemory) 127 | } 128 | 129 | func BenchmarkLockMap_ConcurrentGetSet5(b *testing.B) { 130 | startMemory, endMemory := prepareMemStats() 131 | benchmarkLockMapConcurrentGetSet(b, 19) // 95% reads, 5% writes 132 | reportMemStats("lock map", &startMemory, &endMemory) 133 | } 134 | 135 | func BenchmarkIntegerShardedMap_ConcurrentGetSet50(b *testing.B) { 136 | benchmarkIntegerShardedMapConcurrentGetSet(b, 1) // 50% reads, 50% writes 137 | } 138 | 139 | func BenchmarkSyncMap_ConcurrentGetSet50(b *testing.B) { 140 | benchmarkSyncMapConcurrentGetSet(b, 1) // 50% reads, 50% writes 141 | } 142 | 143 | func BenchmarkLockMap_ConcurrentGetSet50(b *testing.B) { 144 | benchmarkLockMapConcurrentGetSet(b, 1) // 50% reads, 50% writes 145 | } 146 | 147 | func BenchmarkIntegerShardedMap_ConcurrentGetSet1(b *testing.B) { 148 | benchmarkIntegerShardedMapConcurrentGetSet(b, 99) // 99% reads, 1% writes 149 | } 150 | 151 | func BenchmarkSyncMap_ConcurrentGetSet1(b *testing.B) { 152 | benchmarkSyncMapConcurrentGetSet(b, 99) // 99% reads, 1% writes 153 | } 154 | 155 | func BenchmarkLockMap_ConcurrentGetSet1(b *testing.B) { 156 | benchmarkLockMapConcurrentGetSet(b, 99) // 99% reads, 1% writes 157 | } 158 | 159 | func benchmarkIntegerShardedMapConcurrentGetSet(b *testing.B, ratio int) { 160 | sm := smap.NewIntegerComparable[uint16, uint64](smap.HeuristicOptimalDistribution(math.MaxUint16)) 161 | b.ReportAllocs() 162 | b.ResetTimer() 163 | 164 | benchmarkConcurrentGetSetRatio(b, ratio, func(k uint16) (uint64, bool) { 165 | return sm.Load(k) 166 | }, func(k uint16, v uint64) { 167 | sm.Store(k, v) 168 | }) 169 | } 170 | 171 | func benchmarkSyncMapConcurrentGetSet(b *testing.B, ratio int) { 172 | sm := sync.Map{} 173 | b.ReportAllocs() 174 | b.ResetTimer() 175 | 176 | benchmarkConcurrentGetSetRatio(b, ratio, func(k uint16) (uint64, bool) { 177 | val, ok := sm.Load(k) 178 | if ok { 179 | return val.(uint64), ok 180 | } 181 | return 0, false 182 | }, func(k uint16, v uint64) { 183 | sm.Store(k, v) 184 | }) 185 | } 186 | 187 | func benchmarkLockMapConcurrentGetSet(b *testing.B, ratio int) { 188 | sm := make(map[uint16]uint64, math.MaxUint16) 189 | var mutex sync.RWMutex 190 | b.ReportAllocs() 191 | b.ResetTimer() 192 | 193 | benchmarkConcurrentGetSetRatio(b, ratio, func(k uint16) (uint64, bool) { 194 | mutex.RLock() 195 | val, ok := sm[k] 196 | mutex.RUnlock() 197 | return val, ok 198 | }, func(k uint16, v uint64) { 199 | mutex.Lock() 200 | sm[k] = v 201 | mutex.Unlock() 202 | }) 203 | } 204 | 205 | func benchmarkConcurrentGetSetRatio(b *testing.B, ratio int, get func(k uint16) (uint64, bool), set func(uint16, uint64)) { 206 | b.ReportAllocs() 207 | b.ResetTimer() 208 | b.RunParallel(func(pb *testing.PB) { 209 | i := uint16(rand.Uint32()) 210 | j := uint16(rand.Uint32()) 211 | for pb.Next() { 212 | set(i, uint64(i)) 213 | for k := 0; k < ratio; k++ { 214 | get(j) 215 | j++ 216 | } 217 | i++ 218 | } 219 | }) 220 | } 221 | 222 | func prepareMemStats() (startMemory, endMemory runtime.MemStats) { 223 | runtime.ReadMemStats(&startMemory) 224 | return 225 | } 226 | 227 | func reportMemStats(test string, startMemory, endMemory *runtime.MemStats) { 228 | runtime.ReadMemStats(endMemory) 229 | fmt.Printf("Test %s. Memory used %.1fKb\n", test, float32(endMemory.TotalAlloc-startMemory.TotalAlloc)/1024) 230 | } 231 | -------------------------------------------------------------------------------- /smap/integer_test.go: -------------------------------------------------------------------------------- 1 | package smap 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestGeneric_Delete(t *testing.T) { 10 | m := NewInteger[uint32, string](8, 128) 11 | val, ok := m.Load(123) 12 | assert.False(t, ok) 13 | assert.Equal(t, "", val) 14 | 15 | m.Delete(123) // no error on deleting 16 | val, ok = m.Load(123) // nothing changed 17 | assert.False(t, ok) 18 | assert.Equal(t, "", val) 19 | 20 | m.Store(123, "value set") 21 | val, ok = m.Load(123) 22 | assert.True(t, ok) 23 | assert.Equal(t, "value set", val) 24 | 25 | m.Delete(123) 26 | val, ok = m.Load(123) 27 | assert.False(t, ok) 28 | assert.Equal(t, "", val) 29 | } 30 | 31 | func TestGeneric_GetAndDelete(t *testing.T) { 32 | m := NewInteger[uint32, string](8, 128) 33 | m.Store(123, "value set") 34 | val, ok := m.Load(123) 35 | assert.True(t, ok) 36 | assert.Equal(t, "value set", val) 37 | 38 | val, ok = m.Load(123) // nothing changed 39 | assert.True(t, ok) 40 | assert.Equal(t, "value set", val) 41 | 42 | val, ok = m.LoadAndDelete(123) 43 | assert.True(t, ok) 44 | assert.Equal(t, "value set", val) 45 | 46 | val, ok = m.LoadAndDelete(123) 47 | assert.False(t, ok) 48 | assert.Equal(t, "", val) 49 | } 50 | 51 | func TestGeneric_GetOrCreate(t *testing.T) { 52 | m := NewInteger[uint32, string](8, 128) 53 | val, ok := m.Load(123) 54 | assert.False(t, ok) 55 | assert.Equal(t, "", val) 56 | 57 | val, ok = m.LoadOrCreate(123, func() string { return "new value created" }) 58 | assert.False(t, ok) 59 | assert.Equal(t, "new value created", val) 60 | 61 | val, ok = m.LoadOrCreate(123, func() string { panic("no generator should be called") }) 62 | assert.True(t, ok) 63 | assert.Equal(t, "new value created", val) 64 | 65 | val, ok = m.LoadAndDelete(123) 66 | assert.True(t, ok) 67 | assert.Equal(t, "new value created", val) 68 | } 69 | 70 | func TestGeneric_Range(t *testing.T) { 71 | m := NewInteger[int, int](8, 128) 72 | expected := make(map[int]int, 256) 73 | for i := 0; i < 256; i++ { 74 | m.Store(i, i*i) 75 | expected[i] = i * i 76 | } 77 | 78 | result := make(map[int]int, 256) 79 | m.Range(func(k int, v int) bool { 80 | result[k] = v 81 | return true 82 | }) 83 | assert.Equal(t, expected, result) 84 | } 85 | 86 | func TestGenericComparable_CompareAndSwap(t *testing.T) { 87 | m := NewIntegerComparable[int, int](8, 128) 88 | m.CompareAndSwap(123, 0, 23) // no value with key 123 => no change 89 | val, ok := m.Load(123) 90 | assert.False(t, ok) 91 | assert.Equal(t, 0, val) 92 | 93 | m.Store(123, 10) 94 | m.CompareAndSwap(123, 11, 23) // value differs from 11 => no change 95 | val, ok = m.Load(123) 96 | assert.True(t, ok) 97 | assert.Equal(t, 10, val) 98 | 99 | m.CompareAndSwap(123, 10, 23) // value equal 11 => change to 23 100 | val, ok = m.Load(123) 101 | assert.True(t, ok) 102 | assert.Equal(t, 23, val) 103 | } 104 | --------------------------------------------------------------------------------