├── .gitignore ├── gopie.png ├── scripts ├── help.sh ├── doc.sh └── test.sh ├── Makefile ├── pkg ├── singleton │ ├── singleton_test.go │ └── singleton.go ├── nocopy │ ├── nocopy_test.go │ └── nocopy.go ├── sequence │ ├── iceflake.go │ ├── memsequence.go │ ├── sequencer.go │ ├── memflake_test.go │ ├── memsequence_test.go │ └── memflake.go ├── spsc │ ├── util.go │ └── spsc_test.go ├── log │ └── log_test.go ├── randomsequence │ ├── randomseq_test.go │ └── randomseq.go ├── heapsort │ ├── heapsort_test.go │ └── heapsort.go ├── mergesort │ ├── mergesort_test.go │ └── mergesort.go ├── quicksort │ ├── quicksort_test.go │ └── quicksort.go ├── tlv │ ├── tlv_test.go │ └── tlv.go ├── deadline │ ├── deadline_test.go │ └── deadline.go ├── semaphore │ ├── semaphore_test.go │ └── semaphore.go ├── subset │ ├── subset.go │ └── subset_test.go ├── quickselect │ ├── quickselect_test.go │ └── quickselect.go ├── latest │ ├── latest_test.go │ └── latest.go ├── ratelimit │ ├── sliding_window_test.go │ └── ratelimit_test.go ├── skiplist │ ├── types.go │ ├── typestr.go │ └── skiplist_test.go ├── spinlock │ ├── spinlock_test.go │ └── spinlock.go ├── drf │ ├── drf_test.go │ └── types.go ├── base58 │ ├── base58.go │ └── base58_test.go ├── pushsum │ ├── store.go │ └── pushsum_test.go ├── jumphash │ ├── jumphash.go │ └── jumphash_test.go ├── bitflag │ ├── bitflag.go │ └── bitflag_test.go ├── batch │ ├── batch_test.go │ └── batch.go ├── dll │ └── amr.go ├── backoff │ ├── backoff_test.go │ └── backoff.go ├── lru │ ├── lru_test.go │ └── lru.go ├── pubsub │ └── pubsub_test.go ├── barrier │ └── barrier.go ├── fileutil │ ├── fileutil.go │ └── fileutil_test.go ├── bloom │ ├── bloombit.go │ ├── bloombit_test.go │ ├── bloomscale_test.go │ ├── bloom_test.go │ └── bloomscale.go ├── rwspinlock │ ├── rwspinlock_test.go │ └── rwspinlock.go ├── countminsketch │ ├── countmin_test.go │ └── countmin.go ├── multilane │ └── multilane.go ├── ringhash │ └── ringhash_test.go └── hyperloglog │ └── hyperloglog.go ├── go.mod ├── docs ├── heapsort.md ├── mergesort.md ├── quicksort.md ├── quickselect.md ├── subset.md ├── singleton.md ├── latest.md ├── nocopy.md ├── randomsequence.md ├── base58.md ├── batch.md ├── deadline.md ├── semaphore.md ├── backoff.md ├── jumphash.md ├── spinlock.md ├── barrier.md ├── log.md ├── lru.md ├── spsc.md ├── fileutil.md ├── hyperloglog.md ├── bitflag.md ├── tlv.md ├── rwspinlock.md ├── cached.md ├── multilane.md ├── bloom.md └── dll.md └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | .DS_Store 3 | -------------------------------------------------------------------------------- /gopie.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andy2046/gopie/HEAD/gopie.png -------------------------------------------------------------------------------- /scripts/help.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | echo 'usage: make [target] ...' 6 | echo 7 | echo 'targets:' 8 | fgrep -h "##" ./Makefile | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//' 9 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all help test doc 2 | 3 | all: help 4 | 5 | help: ## Show this help 6 | @scripts/help.sh 7 | 8 | test: ## Test potential bugs and race conditions 9 | @scripts/test.sh 10 | 11 | doc: ## Generate docs 12 | @scripts/doc.sh 13 | -------------------------------------------------------------------------------- /scripts/doc.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | for f in pkg/*; do 6 | if [[ -d $f ]]; then 7 | i=$(basename $f) 8 | godoc2md github.com/andy2046/gopie/pkg/$i \ 9 | > $GOPATH/src/github.com/andy2046/gopie/docs/$i.md 10 | fi 11 | done; 12 | -------------------------------------------------------------------------------- /pkg/singleton/singleton_test.go: -------------------------------------------------------------------------------- 1 | package singleton 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestSingleton(t *testing.T) { 8 | s := New() 9 | s.Values["this"] = "that" 10 | 11 | s2 := New() 12 | if s2.Values["this"] != "that" { 13 | t.Fatal("wrong singleton implementation") 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/andy2046/gopie 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/andy2046/bitmap v0.2.0 7 | github.com/andy2046/tik v0.1.0 8 | github.com/go-redis/redis v6.14.1+incompatible 9 | github.com/onsi/ginkgo v1.12.2 // indirect 10 | go.uber.org/goleak v1.0.0 11 | golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 12 | ) 13 | -------------------------------------------------------------------------------- /pkg/nocopy/nocopy_test.go: -------------------------------------------------------------------------------- 1 | package nocopy_test 2 | 3 | import ( 4 | . "github.com/andy2046/gopie/pkg/nocopy" 5 | "testing" 6 | ) 7 | 8 | type MyStruct struct { 9 | noCopy NoCopy 10 | } 11 | 12 | func TestNoCopy(t *testing.T) { 13 | // go vet fails 14 | var m1 MyStruct 15 | m2 := m1 16 | var m3 = m1 17 | m2 = m1 18 | _, _ = m2, m3 19 | t.Log("go vet fails here") 20 | } 21 | -------------------------------------------------------------------------------- /pkg/nocopy/nocopy.go: -------------------------------------------------------------------------------- 1 | // Package nocopy implements the interface for -copylocks checker from `go vet`. 2 | package nocopy 3 | 4 | // NoCopy can be embedded into structs which must not be copied 5 | // after the first use. 6 | type NoCopy struct{} 7 | 8 | // Lock is a no-op used by -copylocks checker from `go vet`. 9 | func (*NoCopy) Lock() {} 10 | 11 | // Unlock is not required by -copylocks checker from `go vet`. 12 | func (*NoCopy) Unlock() {} 13 | -------------------------------------------------------------------------------- /pkg/sequence/iceflake.go: -------------------------------------------------------------------------------- 1 | package sequence 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // Iceflake is the interface for snowflake similar sequence generator. 8 | type Iceflake interface { 9 | Sequencer 10 | // StartTime defines the time since which 11 | // the Iceflake time is defined as the elapsed time. 12 | StartTime() time.Time 13 | // BitLenSequence defines the bit length of sequence number, 14 | // and the bit length of time is 63 - BitLenSequence(). 15 | BitLenSequence() uint8 16 | } 17 | -------------------------------------------------------------------------------- /pkg/singleton/singleton.go: -------------------------------------------------------------------------------- 1 | // Package singleton provides a singleton implementation. 2 | package singleton 3 | 4 | import "sync" 5 | 6 | // Instance is the singleton instance. 7 | type Instance struct { 8 | Values map[interface{}]interface{} 9 | } 10 | 11 | var ( 12 | once sync.Once 13 | instance *Instance 14 | ) 15 | 16 | // New returns the singleton instance. 17 | func New() *Instance { 18 | once.Do(func() { 19 | instance = &Instance{ 20 | Values: make(map[interface{}]interface{}), 21 | } 22 | }) 23 | return instance 24 | } 25 | -------------------------------------------------------------------------------- /pkg/spsc/util.go: -------------------------------------------------------------------------------- 1 | package spsc 2 | 3 | import ( 4 | "unsafe" 5 | ) 6 | 7 | type iface struct { 8 | t, d unsafe.Pointer 9 | } 10 | 11 | func extractptr(i interface{}) unsafe.Pointer { 12 | return (*iface)(unsafe.Pointer(&i)).d 13 | } 14 | 15 | func inject(i interface{}, ptr unsafe.Pointer) { 16 | var v = (*unsafe.Pointer)((*iface)(unsafe.Pointer(&i)).d) 17 | *v = ptr 18 | } 19 | 20 | func nextPowerOf2(v uint32) uint32 { 21 | v-- 22 | v |= v >> 1 23 | v |= v >> 2 24 | v |= v >> 4 25 | v |= v >> 8 26 | v |= v >> 16 27 | return v + 1 28 | } 29 | -------------------------------------------------------------------------------- /pkg/log/log_test.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import "testing" 4 | 5 | func TestNewLogger(t *testing.T) { 6 | logger := NewLogger(func(c *Config) error { 7 | c.Level = DEBUG 8 | c.Prefix = "test:" 9 | return nil 10 | }) 11 | 12 | logger.Debug("Debug") 13 | logger.Debugf("format:%s\n", "Debug") 14 | 15 | logger.SetLevel(INFO) 16 | 17 | logger.Info("Info") 18 | logger.Infof("format:%s\n", "Info") 19 | 20 | logger.Warn("Warn") 21 | logger.Warnf("format:%s\n", "Warn") 22 | 23 | logger.Error("Error") 24 | logger.Errorf("format:%s\n", "Error") 25 | } 26 | -------------------------------------------------------------------------------- /scripts/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | for f in pkg/*; do 6 | if [[ -d $f ]]; then 7 | i=$(basename $f) 8 | echo 9 | echo === Testing pkg $i 10 | dir=$GOPATH/src/github.com/andy2046/gopie/pkg/$i 11 | cd $dir 12 | go test -count=1 -v -race 13 | GOGC=off go test -bench=. -run=none -benchtime=3s 14 | go fmt 15 | if [[ $i =~ "nocopy" || $i =~ "spinlock" ]]; then 16 | echo "ignore go vet" 17 | else 18 | go vet 19 | fi 20 | golint 21 | cd - 22 | echo === Tested pkg $i 23 | echo 24 | fi 25 | done; 26 | -------------------------------------------------------------------------------- /pkg/randomsequence/randomseq_test.go: -------------------------------------------------------------------------------- 1 | package randomsequence_test 2 | 3 | import ( 4 | bt "github.com/andy2046/bitmap" 5 | . "github.com/andy2046/gopie/pkg/randomsequence" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestUnique(t *testing.T) { 11 | size := uint64(^uint32(0) >> 10) // by right the size is 4294967295 but it is too slow 12 | t.Log("sample size is", size) 13 | btmap := bt.New(size + 1) 14 | seed := uint32(time.Now().UTC().UnixNano() >> 32) 15 | rnd := New(seed, seed+1) 16 | for i := uint64(0); i <= size; i++ { 17 | n := rnd.Next() 18 | if btmap.GetBit(uint64(n)) { 19 | t.Fatal("dup") 20 | } 21 | btmap.SetBit(uint64(n), true) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /pkg/heapsort/heapsort_test.go: -------------------------------------------------------------------------------- 1 | package heapsort_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "math/rand" 8 | 9 | "github.com/andy2046/gopie/pkg/heapsort" 10 | ) 11 | 12 | func TestHeapsort(t *testing.T) { 13 | size := 20 14 | max := 999 15 | s := make([]int, 0, size) 16 | rand.Seed(time.Now().UTC().UnixNano()) 17 | for range make([]struct{}, size) { 18 | s = append(s, rand.Intn(max)) 19 | } 20 | t.Logf("before -> %v", s) 21 | heapsort.Sort(s) 22 | t.Logf("after -> %v", s) 23 | } 24 | 25 | func BenchmarkHeapsort(b *testing.B) { 26 | size := 1000000 27 | s := make([]int, 0, size) 28 | rand.Seed(time.Now().UTC().UnixNano()) 29 | for range make([]struct{}, size) { 30 | s = append(s, rand.Int()) 31 | } 32 | b.ResetTimer() 33 | heapsort.Sort(s) 34 | } 35 | -------------------------------------------------------------------------------- /docs/heapsort.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # heapsort 4 | `import "github.com/andy2046/gopie/pkg/heapsort"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package heapsort implements Heapsort. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [func Sort(data []int)](#Sort) 17 | 18 | 19 | #### Package files 20 | [heapsort.go](/src/github.com/andy2046/gopie/pkg/heapsort/heapsort.go) 21 | 22 | 23 | 24 | 25 | 26 | ## func [Sort](/src/target/heapsort.go?s=84:105#L5) 27 | ``` go 28 | func Sort(data []int) 29 | ``` 30 | Sort sorts the slice. 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | - - - 40 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 41 | -------------------------------------------------------------------------------- /pkg/mergesort/mergesort_test.go: -------------------------------------------------------------------------------- 1 | package mergesort_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "math/rand" 8 | 9 | "github.com/andy2046/gopie/pkg/mergesort" 10 | ) 11 | 12 | func TestMergesort(t *testing.T) { 13 | size := 20 14 | max := 999 15 | s := make([]int, 0, size) 16 | rand.Seed(time.Now().UTC().UnixNano()) 17 | for range make([]struct{}, size) { 18 | s = append(s, rand.Intn(max)) 19 | } 20 | t.Logf("before -> %v", s) 21 | mergesort.Sort(s) 22 | t.Logf("after -> %v", s) 23 | } 24 | 25 | func BenchmarkMergesort(b *testing.B) { 26 | size := 1000000 27 | s := make([]int, 0, size) 28 | rand.Seed(time.Now().UTC().UnixNano()) 29 | for range make([]struct{}, size) { 30 | s = append(s, rand.Int()) 31 | } 32 | b.ResetTimer() 33 | mergesort.Sort(s) 34 | } 35 | -------------------------------------------------------------------------------- /pkg/quicksort/quicksort_test.go: -------------------------------------------------------------------------------- 1 | package quicksort_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "math/rand" 8 | 9 | "github.com/andy2046/gopie/pkg/quicksort" 10 | ) 11 | 12 | func TestQuicksort(t *testing.T) { 13 | size := 20 14 | max := 999 15 | s := make([]int, 0, size) 16 | rand.Seed(time.Now().UTC().UnixNano()) 17 | for range make([]struct{}, size) { 18 | s = append(s, rand.Intn(max)) 19 | } 20 | t.Logf("before -> %v", s) 21 | quicksort.Sort(s) 22 | t.Logf("after -> %v", s) 23 | } 24 | 25 | func BenchmarkQuicksort(b *testing.B) { 26 | size := 1000000 27 | s := make([]int, 0, size) 28 | rand.Seed(time.Now().UTC().UnixNano()) 29 | for range make([]struct{}, size) { 30 | s = append(s, rand.Int()) 31 | } 32 | b.ResetTimer() 33 | quicksort.Sort(s) 34 | } 35 | -------------------------------------------------------------------------------- /docs/mergesort.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # mergesort 4 | `import "github.com/andy2046/gopie/pkg/mergesort"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package mergesort implements Mergesort. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [func Sort(data []int)](#Sort) 17 | 18 | 19 | #### Package files 20 | [mergesort.go](/src/github.com/andy2046/gopie/pkg/mergesort/mergesort.go) 21 | 22 | 23 | 24 | 25 | 26 | ## func [Sort](/src/target/mergesort.go?s=87:108#L5) 27 | ``` go 28 | func Sort(data []int) 29 | ``` 30 | Sort sorts the slice. 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | - - - 40 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 41 | -------------------------------------------------------------------------------- /docs/quicksort.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # quicksort 4 | `import "github.com/andy2046/gopie/pkg/quicksort"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package quicksort implements Quicksort. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [func Sort(data []int)](#Sort) 17 | 18 | 19 | #### Package files 20 | [quicksort.go](/src/github.com/andy2046/gopie/pkg/quicksort/quicksort.go) 21 | 22 | 23 | 24 | 25 | 26 | ## func [Sort](/src/target/quicksort.go?s=87:108#L5) 27 | ``` go 28 | func Sort(data []int) 29 | ``` 30 | Sort sorts the slice. 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | - - - 40 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 41 | -------------------------------------------------------------------------------- /pkg/tlv/tlv_test.go: -------------------------------------------------------------------------------- 1 | package tlv 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | ) 7 | 8 | func TestTLV(t *testing.T) { 9 | v := "hola, tlv!" 10 | typ := uint(8) 11 | buf := new(bytes.Buffer) 12 | codec := &Codec{TypeBytes: Bytes1, LenBytes: Bytes2} 13 | writer := NewWriter(buf, codec) 14 | 15 | record := &Record{ 16 | Payload: []byte(v), 17 | Type: typ, 18 | } 19 | writer.Write(record) 20 | 21 | reader := bytes.NewReader(buf.Bytes()) 22 | tlvReader := NewReader(reader, codec) 23 | next, _ := tlvReader.Next() 24 | 25 | if next.Type != typ { 26 | t.Errorf("expected %d got %d", typ, next.Type) 27 | } 28 | 29 | if r := string(next.Payload); r != v { 30 | t.Errorf("expected %s got %s", v, r) 31 | } 32 | 33 | t.Logf("type: %d\n", next.Type) 34 | t.Logf("payload: %s\n", string(next.Payload)) 35 | } 36 | -------------------------------------------------------------------------------- /pkg/deadline/deadline_test.go: -------------------------------------------------------------------------------- 1 | package deadline 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | var ( 10 | errText = "xxx" 11 | ) 12 | 13 | func TestDeadline(t *testing.T) { 14 | d := New(10 * time.Millisecond) 15 | 16 | if err := d.Go(fiveMillisecondFunc); err != nil { 17 | t.Error(err) 18 | } 19 | 20 | if err := d.Go(twentyMillisecondFunc); err != ErrTimeout { 21 | t.Error(err) 22 | } 23 | 24 | if err := d.Go(errorFunc); err.Error() != errText { 25 | t.Error(err) 26 | } 27 | } 28 | 29 | func fiveMillisecondFunc(<-chan struct{}) error { 30 | time.Sleep(5 * time.Millisecond) 31 | return nil 32 | } 33 | 34 | func twentyMillisecondFunc(<-chan struct{}) error { 35 | time.Sleep(20 * time.Millisecond) 36 | return nil 37 | } 38 | 39 | func errorFunc(<-chan struct{}) error { 40 | return errors.New(errText) 41 | } 42 | -------------------------------------------------------------------------------- /docs/quickselect.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # quickselect 4 | `import "github.com/andy2046/gopie/pkg/quickselect"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package quickselect implements Quickselect. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [func Select(data []int, k int) int](#Select) 17 | 18 | 19 | #### Package files 20 | [quickselect.go](/src/github.com/andy2046/gopie/pkg/quickselect/quickselect.go) 21 | 22 | 23 | 24 | 25 | 26 | ## func [Select](/src/target/quickselect.go?s=103:137#L5) 27 | ``` go 28 | func Select(data []int, k int) int 29 | ``` 30 | Select selects the kth element. 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | - - - 40 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 41 | -------------------------------------------------------------------------------- /pkg/semaphore/semaphore_test.go: -------------------------------------------------------------------------------- 1 | package semaphore 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func work() { 10 | time.Sleep(100 * time.Millisecond) 11 | } 12 | 13 | func TestSemaphore(t *testing.T) { 14 | s := New(1) 15 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 16 | 17 | if err := s.Acquire(ctx); err != nil { 18 | t.Fatal(err) 19 | } 20 | work() 21 | if err := s.Release(ctx); err != nil { 22 | t.Fatal(err) 23 | } 24 | cancel() 25 | } 26 | 27 | func BenchmarkSemaphore(b *testing.B) { 28 | s := New(1) 29 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 30 | 31 | b.ResetTimer() 32 | for i := 0; i < b.N; i++ { 33 | if err := s.Acquire(ctx); err != nil { 34 | b.Fatal(err) 35 | } 36 | if err := s.Release(ctx); err != nil { 37 | b.Fatal(err) 38 | } 39 | } 40 | cancel() 41 | } 42 | -------------------------------------------------------------------------------- /pkg/subset/subset.go: -------------------------------------------------------------------------------- 1 | // Package subset implements deterministic subsetting. 2 | package subset 3 | 4 | /* 5 | https://landing.google.com/sre/book/chapters/load-balancing-datacenter.html 6 | */ 7 | 8 | import ( 9 | "math/rand" 10 | ) 11 | 12 | // Subset returns a subset of backends with size subsetSize. 13 | func Subset(backends []string, clientID, subsetSize int) []string { 14 | 15 | subsetCount := len(backends) / subsetSize 16 | 17 | // Group clients into rounds; each round uses the same shuffled list: 18 | round := clientID / subsetCount 19 | 20 | r := rand.New(rand.NewSource(int64(round))) 21 | r.Shuffle(len(backends), func(i, j int) { backends[i], backends[j] = backends[j], backends[i] }) 22 | 23 | // The subset id corresponding to the current client: 24 | subsetID := clientID % subsetCount 25 | 26 | start := subsetID * subsetSize 27 | return backends[start : start+subsetSize] 28 | } 29 | -------------------------------------------------------------------------------- /docs/subset.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # subset 4 | `import "github.com/andy2046/gopie/pkg/subset"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package subset implements deterministic subsetting. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [func Subset(backends []string, clientID, subsetSize int) []string](#Subset) 17 | 18 | 19 | #### Package files 20 | [subset.go](/src/github.com/andy2046/gopie/pkg/subset/subset.go) 21 | 22 | 23 | 24 | 25 | 26 | ## func [Subset](/src/target/subset.go?s=243:308#L13) 27 | ``` go 28 | func Subset(backends []string, clientID, subsetSize int) []string 29 | ``` 30 | Subset returns a subset of backends with size subsetSize. 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | - - - 40 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 41 | -------------------------------------------------------------------------------- /pkg/quicksort/quicksort.go: -------------------------------------------------------------------------------- 1 | // Package quicksort implements Quicksort. 2 | package quicksort 3 | 4 | // Sort sorts the slice. 5 | func Sort(data []int) { 6 | sort(data, 0, len(data)-1) 7 | } 8 | 9 | func sort(c []int, start, end int) { 10 | if end <= start { 11 | return 12 | } 13 | i, j := start, end+1 14 | // ensure: c[start] <= c[start+1] <= c[end] 15 | if c[start] > c[end] { 16 | c[start], c[end] = c[end], c[start] 17 | } 18 | if c[start+1] > c[end] { 19 | c[start+1], c[end] = c[end], c[start+1] 20 | } 21 | if c[start] > c[start+1] { 22 | c[start], c[start+1] = c[start+1], c[start] 23 | } 24 | comp := c[start] 25 | for { 26 | for ok := true; ok; ok = c[i] < comp { 27 | i++ 28 | } 29 | for ok := true; ok; ok = c[j] > comp { 30 | j-- 31 | } 32 | if j <= i { 33 | break 34 | } 35 | c[i], c[j] = c[j], c[i] 36 | } 37 | c[start], c[j] = c[j], c[start] 38 | sort(c, start, j-1) 39 | sort(c, j+1, end) 40 | } 41 | -------------------------------------------------------------------------------- /pkg/sequence/memsequence.go: -------------------------------------------------------------------------------- 1 | package sequence 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | ) 7 | 8 | // MemSeq is an implementation of in-memory Sequencer. 9 | type MemSeq struct { 10 | sync.Mutex 11 | current uint64 12 | machine uint64 13 | } 14 | 15 | var ( 16 | errLessThanToo = errors.New("`n` should not be less than 1") 17 | _ Sequencer = &MemSeq{} 18 | ) 19 | 20 | // NewMemSeq creates a MemSeq. 21 | func NewMemSeq(machineID uint64) *MemSeq { 22 | return &MemSeq{ 23 | machine: machineID, 24 | } 25 | } 26 | 27 | // Next ... 28 | func (m *MemSeq) Next() (uint64, error) { 29 | return m.NextN(1) 30 | } 31 | 32 | // NextN ... 33 | func (m *MemSeq) NextN(n int) (uint64, error) { 34 | if n < 1 { 35 | return 0, errLessThanToo 36 | } 37 | m.Lock() 38 | r := m.current 39 | m.current += uint64(n) 40 | m.Unlock() 41 | return r, nil 42 | } 43 | 44 | // MachineID ... 45 | func (m *MemSeq) MachineID() uint64 { 46 | return m.machine 47 | } 48 | -------------------------------------------------------------------------------- /pkg/mergesort/mergesort.go: -------------------------------------------------------------------------------- 1 | // Package mergesort implements Mergesort. 2 | package mergesort 3 | 4 | // Sort sorts the slice. 5 | func Sort(data []int) { 6 | aux := make([]int, len(data)) 7 | sort(data, aux) 8 | } 9 | 10 | func sort(a, aux []int) { 11 | n := len(a) 12 | for size := 1; size < n; size = 2 * size { 13 | for lo := 0; lo < n-size; lo += 2 * size { 14 | merge(a, aux, lo, lo+size-1, min(lo+size+size-1, n-1)) 15 | } 16 | } 17 | } 18 | 19 | func merge(a, aux []int, lo, mid, hi int) { 20 | for k := lo; k <= hi; k++ { 21 | aux[k] = a[k] 22 | } 23 | 24 | i, j := lo, mid+1 25 | 26 | for k := lo; k <= hi; k++ { 27 | if i > mid { 28 | a[k] = aux[j] 29 | j++ 30 | } else if j > hi { 31 | a[k] = aux[i] 32 | i++ 33 | } else if aux[j] <= aux[i] { 34 | a[k] = aux[j] 35 | j++ 36 | } else { 37 | a[k] = aux[i] 38 | i++ 39 | } 40 | } 41 | } 42 | 43 | func min(a, b int) int { 44 | if a < b { 45 | return a 46 | } 47 | return b 48 | } 49 | -------------------------------------------------------------------------------- /pkg/deadline/deadline.go: -------------------------------------------------------------------------------- 1 | // Package deadline implements Deadline pattern. 2 | package deadline 3 | 4 | import ( 5 | "errors" 6 | "time" 7 | ) 8 | 9 | // ErrTimeout is the error for deadline timeout. 10 | var ErrTimeout = errors.New("time out executing function") 11 | 12 | // Deadline represents the deadline. 13 | type Deadline struct { 14 | timeout time.Duration 15 | } 16 | 17 | // New returns a new Deadline with the provided timeout. 18 | func New(timeout time.Duration) *Deadline { 19 | return &Deadline{ 20 | timeout: timeout, 21 | } 22 | } 23 | 24 | // Go executes the provided function with a done channel as parameter to signal the timeout. 25 | func (d *Deadline) Go(fn func(<-chan struct{}) error) error { 26 | result, done := make(chan error), make(chan struct{}) 27 | 28 | go func() { 29 | result <- fn(done) 30 | }() 31 | 32 | select { 33 | case ret := <-result: 34 | return ret 35 | case <-time.After(d.timeout): 36 | close(done) 37 | return ErrTimeout 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pkg/heapsort/heapsort.go: -------------------------------------------------------------------------------- 1 | // Package heapsort implements Heapsort. 2 | package heapsort 3 | 4 | // Sort sorts the slice. 5 | func Sort(data []int) { 6 | sort(data) 7 | } 8 | 9 | func sort(a []int) { 10 | n := len(a) 11 | buildMaxHeap(a, n) 12 | sortDown(a, n) 13 | } 14 | 15 | func buildMaxHeap(a []int, n int) { 16 | for k := n / 2; k >= 1; k-- { 17 | sink(a, k, n) 18 | } 19 | } 20 | 21 | func sortDown(a []int, n int) { 22 | for n > 1 { 23 | swap(a, 1, n) 24 | n-- 25 | sink(a, 1, n) 26 | } 27 | } 28 | 29 | func sink(a []int, k, n int) { 30 | for 2*k <= n { 31 | j := 2 * k 32 | 33 | // is right key greater than left key 34 | if j < n && less(a, j, j+1) { 35 | j++ 36 | } 37 | 38 | // when both right and left child are not greater than parent 39 | if !less(a, k, j) { 40 | break 41 | } 42 | 43 | // moves the greater key up 44 | swap(a, k, j) 45 | 46 | k = j 47 | } 48 | } 49 | 50 | func less(a []int, i, j int) bool { 51 | return a[i-1] < a[j-1] 52 | } 53 | 54 | func swap(a []int, i, j int) { 55 | a[i-1], a[j-1] = a[j-1], a[i-1] 56 | } 57 | -------------------------------------------------------------------------------- /pkg/sequence/sequencer.go: -------------------------------------------------------------------------------- 1 | // Package sequence implements Iceflake sequence generator interface. 2 | // ```bash 3 | // Iceflake is the interface for snowflake similar sequence generator. 4 | // 5 | // Iceflake algorithm: 6 | // 7 | // +-------+--------------------+----------+ 8 | // | sign | delta milliseconds | sequence | 9 | // +-------+--------------------+----------+ 10 | // | 1 bit | 63-n bits | n bits | 11 | // 12 | // sequence (n bits) 13 | // The last custom n bits, represents sequence within the one millisecond. 14 | // 15 | // delta milliseconds (63-n bits) 16 | // The next 63-n bits, represents delta milliseconds since a custom epoch. 17 | // ``` 18 | package sequence 19 | 20 | // Sequencer is the interface for sequence generator. 21 | type Sequencer interface { 22 | // Next returns the next sequence. 23 | Next() (uint64, error) 24 | // NextN reserves the next `n` sequences and returns the first one, 25 | // `n` should not be less than 1. 26 | NextN(n int) (uint64, error) 27 | // MachineID returns the unique ID of the instance. 28 | MachineID() uint64 29 | } 30 | -------------------------------------------------------------------------------- /docs/singleton.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # singleton 4 | `import "github.com/andy2046/gopie/pkg/singleton"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package singleton provides a singleton implementation. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [type Instance](#Instance) 17 | * [func New() *Instance](#New) 18 | 19 | 20 | #### Package files 21 | [singleton.go](/src/github.com/andy2046/gopie/pkg/singleton/singleton.go) 22 | 23 | 24 | 25 | 26 | 27 | 28 | ## type [Instance](/src/target/singleton.go?s=131:191#L7) 29 | ``` go 30 | type Instance struct { 31 | Values map[interface{}]interface{} 32 | } 33 | ``` 34 | Instance is the singleton instance. 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | ### func [New](/src/target/singleton.go?s=281:301#L17) 43 | ``` go 44 | func New() *Instance 45 | ``` 46 | New returns the singleton instance. 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | - - - 57 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 58 | -------------------------------------------------------------------------------- /pkg/quickselect/quickselect_test.go: -------------------------------------------------------------------------------- 1 | package quickselect_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "math/rand" 8 | 9 | "github.com/andy2046/gopie/pkg/quickselect" 10 | "github.com/andy2046/gopie/pkg/quicksort" 11 | ) 12 | 13 | func TestQuickselect(t *testing.T) { 14 | size := 20 15 | selected := 10 16 | max := 999 17 | s := make([]int, 0, size) 18 | rand.Seed(time.Now().UTC().UnixNano()) 19 | for range make([]struct{}, size) { 20 | s = append(s, rand.Intn(max)) 21 | } 22 | t.Logf("before -> %v", s) 23 | quickselect.Select(s, selected) 24 | t.Logf("after -> %v", s) 25 | se := s[selected-1] 26 | t.Logf("%dth selected -> %v", selected, se) 27 | quicksort.Sort(s) 28 | so := s[selected-1] 29 | t.Logf("sorted -> %v", s) 30 | if se != so { 31 | t.Fatalf("expected %d, got %d", so, se) 32 | } 33 | } 34 | 35 | func BenchmarkQuickselect(b *testing.B) { 36 | size := 1000000 37 | selected := size - 10 38 | s := make([]int, 0, size) 39 | rand.Seed(time.Now().UTC().UnixNano()) 40 | for range make([]struct{}, size) { 41 | s = append(s, rand.Int()) 42 | } 43 | b.ResetTimer() 44 | quickselect.Select(s, selected) 45 | } 46 | -------------------------------------------------------------------------------- /pkg/subset/subset_test.go: -------------------------------------------------------------------------------- 1 | package subset 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | ) 7 | 8 | func TestSubset(t *testing.T) { 9 | subsetSize := 10 10 | clientSize := 300 11 | backendSize := 300 12 | loopC := make([]struct{}, clientSize) 13 | loopB := make([]struct{}, backendSize) 14 | results := make(map[string]int) 15 | min, max := clientSize, 1 16 | 17 | clients := make([]int, 0, clientSize) 18 | for i := range loopC { 19 | clients = append(clients, i) 20 | } 21 | 22 | for i := range loopC { 23 | backends := make([]string, 0, backendSize) 24 | for i := range loopB { 25 | backends = append(backends, strconv.Itoa(i)) 26 | } 27 | sets := Subset(backends, clients[i], subsetSize) 28 | for _, b := range sets { 29 | if _, ok := results[b]; !ok { 30 | results[b] = 1 31 | } else { 32 | results[b]++ 33 | } 34 | } 35 | } 36 | 37 | t.Log("results:") 38 | for k, v := range results { 39 | t.Logf("backend %s -> client count %d", k, v) 40 | if v > max { 41 | max = v 42 | } 43 | if v < min { 44 | min = v 45 | } 46 | } 47 | t.Logf("min -> %d max -> %d (max-min)/min -> %d%%", min, max, 100*(max-min)/min) 48 | } 49 | -------------------------------------------------------------------------------- /pkg/semaphore/semaphore.go: -------------------------------------------------------------------------------- 1 | // Package semaphore provides a semaphore implementation. 2 | package semaphore 3 | 4 | import ( 5 | "context" 6 | ) 7 | 8 | type ( 9 | // Semaphore is the semaphore implementation. 10 | Semaphore struct { 11 | cur chan struct{} 12 | } 13 | 14 | // ISemaphore is the semaphore interface. 15 | ISemaphore interface { 16 | Acquire(context.Context) error 17 | Release(context.Context) error 18 | } 19 | ) 20 | 21 | // Acquire acquires the semaphore. 22 | func (s *Semaphore) Acquire(ctx context.Context) error { 23 | select { 24 | case s.cur <- struct{}{}: 25 | return nil 26 | case <-ctx.Done(): 27 | return ctx.Err() 28 | } 29 | } 30 | 31 | // Release releases the semaphore. 32 | func (s *Semaphore) Release(ctx context.Context) error { 33 | select { 34 | case _ = <-s.cur: 35 | return nil 36 | case <-ctx.Done(): 37 | return ctx.Err() 38 | } 39 | } 40 | 41 | // New creates a new semaphore with given maximum concurrent access. 42 | func New(n int) ISemaphore { 43 | if n <= 0 { 44 | panic("the number of max concurrent access must be positive int") 45 | } 46 | return &Semaphore{ 47 | cur: make(chan struct{}, n), 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /pkg/latest/latest_test.go: -------------------------------------------------------------------------------- 1 | package latest 2 | 3 | import "fmt" 4 | 5 | func ExampleNew() { 6 | receiver := make(chan interface{}) 7 | sender := New(receiver) 8 | defer close(sender) 9 | 10 | sender <- 1 11 | 12 | fmt.Println("got", <-receiver) 13 | 14 | sender <- 2 15 | sender <- 3 16 | 17 | fmt.Println("got", <-receiver) 18 | fmt.Println("got", <-receiver) 19 | 20 | sender <- 4 21 | sender <- 5 22 | 23 | fmt.Println("got", <-receiver) 24 | fmt.Println("got", <-receiver) 25 | 26 | // Output: 27 | // got 1 28 | // got 3 29 | // got 3 30 | // got 5 31 | // got 5 32 | } 33 | 34 | func ExampleNewN() { 35 | receiver := make(chan []interface{}) 36 | n := 3 37 | sender := NewN(receiver, n) 38 | defer close(sender) 39 | 40 | sender <- 11 41 | 42 | fmt.Println("got", <-receiver) 43 | 44 | sender <- 12 45 | sender <- 13 46 | 47 | fmt.Println("got", <-receiver) 48 | fmt.Println("got", <-receiver) 49 | 50 | sender <- 14 51 | sender <- 15 52 | 53 | fmt.Println("got", <-receiver) 54 | fmt.Println("got", <-receiver) 55 | 56 | // Output: 57 | // got [ 11] 58 | // got [11 12 13] 59 | // got [11 12 13] 60 | // got [13 14 15] 61 | // got [13 14 15] 62 | } 63 | -------------------------------------------------------------------------------- /pkg/randomsequence/randomseq.go: -------------------------------------------------------------------------------- 1 | // Package randomsequence implements quadratic residues based random sequence. 2 | package randomsequence 3 | 4 | // https://preshing.com/20121224/how-to-generate-a-sequence-of-unique-random-integers 5 | 6 | // Random represents the random sequence. 7 | type Random struct { 8 | index uint32 9 | offset uint32 10 | } 11 | 12 | const ( 13 | prime uint32 = 4294967291 14 | primeBy2 uint32 = prime / 2 15 | ) 16 | 17 | // New creates a random sequence with the seed provided. 18 | func New(seedBase, seedOffset uint32) *Random { 19 | return &Random{ 20 | index: permute(permute(seedBase) + 0x682f0161), 21 | offset: permute(permute(seedOffset) + 0x46790905), 22 | } 23 | } 24 | 25 | // Next returns next random number. 26 | func (r *Random) Next() uint32 { 27 | i := r.index 28 | r.index++ 29 | return permute((permute(i) + r.offset) ^ 0x5bf03635) 30 | } 31 | 32 | func permute(x uint32) uint32 { 33 | if x >= prime { 34 | // The 5 integers in the range [4294967291, 2^32] are mapped to themselves. 35 | return x 36 | } 37 | 38 | residue := uint32(uint64(x) * uint64(x) % uint64(prime)) 39 | if x > primeBy2 { 40 | return prime - residue 41 | } 42 | return residue 43 | } 44 | -------------------------------------------------------------------------------- /pkg/ratelimit/sliding_window_test.go: -------------------------------------------------------------------------------- 1 | package ratelimit 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | "time" 7 | 8 | "github.com/go-redis/redis" 9 | ) 10 | 11 | func TestSlidingWindowLimiter(t *testing.T) { 12 | t.Skip("only works with a local running Redis server") 13 | opts := &redis.Options{ 14 | Addr: "localhost:6379", 15 | Password: "", // no password set 16 | DB: 0, // use default DB 17 | } 18 | key := "user_1" 19 | expire := 10 20 | var once sync.Once 21 | redStore, _ := NewRedisStore(opts) 22 | sLimiter := NewSlidingWindowLimiter(1, expire, redStore) 23 | 24 | t1 := t0.Add(200 * time.Millisecond) 25 | t2 := t0.Add(time.Duration(expire) * time.Second) 26 | cases := []struct { 27 | t time.Time 28 | n int 29 | ok bool 30 | }{ 31 | {t0, 1, true}, 32 | {t0, 1, false}, 33 | {t0, 1, false}, 34 | {t1, 1, false}, 35 | {t1, 1, false}, 36 | {t2, 1, true}, 37 | {t2, 1, false}, 38 | } 39 | 40 | for _, c := range cases { 41 | if c.t == t2 { 42 | once.Do(func() { 43 | time.Sleep(time.Duration(expire) * time.Second) 44 | }) 45 | } 46 | ok := sLimiter.AllowN(c.t, key, c.n) 47 | if ok != c.ok { 48 | t.Errorf("AllowN(%v, %v, %v) = %v want %v", 49 | c.t, key, c.n, ok, c.ok) 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /pkg/quickselect/quickselect.go: -------------------------------------------------------------------------------- 1 | // Package quickselect implements Quickselect. 2 | package quickselect 3 | 4 | // Select selects the kth element. 5 | func Select(data []int, k int) int { 6 | return do(data, 0, len(data)-1, k-1) 7 | } 8 | 9 | func do(a []int, left, right, k int) int { 10 | for { 11 | if right <= left+1 { 12 | if right == left+1 && a[right] < a[left] { 13 | swap(a, left, right) 14 | } 15 | return a[k] 16 | } 17 | 18 | middle := (left + right) >> 1 19 | swap(a, middle, left+1) 20 | 21 | if a[left] > a[right] { 22 | swap(a, left, right) 23 | } 24 | 25 | if a[left+1] > a[right] { 26 | swap(a, left+1, right) 27 | } 28 | 29 | if a[left] > a[left+1] { 30 | swap(a, left, left+1) 31 | } 32 | 33 | i, j := left+1, right 34 | pivot := a[left+1] 35 | 36 | for { 37 | for ok := true; ok; ok = a[i] < pivot { 38 | i++ 39 | } 40 | for ok := true; ok; ok = a[j] > pivot { 41 | j-- 42 | } 43 | 44 | if j < i { 45 | break 46 | } 47 | 48 | swap(a, i, j) 49 | } 50 | 51 | a[left+1] = a[j] 52 | a[j] = pivot 53 | 54 | if j >= k { 55 | right = j - 1 56 | } 57 | 58 | if j <= k { 59 | left = i 60 | } 61 | } 62 | } 63 | 64 | func swap(a []int, i, j int) { 65 | a[i], a[j] = a[j], a[i] 66 | } 67 | -------------------------------------------------------------------------------- /pkg/skiplist/types.go: -------------------------------------------------------------------------------- 1 | package skiplist 2 | 3 | import ( 4 | "math/rand" 5 | "sync" 6 | ) 7 | 8 | type ( 9 | elementNode struct { 10 | forward []*Element 11 | } 12 | 13 | // Element represents an element in the list. 14 | Element struct { 15 | elementNode 16 | key string 17 | value int64 18 | } 19 | 20 | // SkipList represents the list. 21 | SkipList struct { 22 | elementNode 23 | randSource rand.Source 24 | searchNodesCache []*elementNode 25 | probability float64 26 | probTable []float64 27 | maxLevel int 28 | length int 29 | mutex sync.RWMutex 30 | } 31 | ) 32 | 33 | // Key returns the given Element key. 34 | func (e *Element) Key() string { 35 | return e.key 36 | } 37 | 38 | // Value returns the given Element value. 39 | func (e *Element) Value() int64 { 40 | return e.value 41 | } 42 | 43 | // Next returns the adjacent next Element if existed or nil otherwise. 44 | func (e *Element) Next() *Element { 45 | return e.forward[0] 46 | } 47 | 48 | // NextLevel returns the adjacent next Element at provided level if existed or nil otherwise. 49 | func (e *Element) NextLevel(level int) *Element { 50 | if level >= len(e.forward) || level < 0 { 51 | return nil 52 | } 53 | 54 | return e.forward[level] 55 | } 56 | -------------------------------------------------------------------------------- /pkg/skiplist/typestr.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | package skiplist 4 | 5 | import ( 6 | "math/rand" 7 | "sync" 8 | ) 9 | 10 | type ( 11 | elementNodeS struct { 12 | forward []*ElementS 13 | } 14 | 15 | // ElementS represents an element in the list. 16 | ElementS struct { 17 | elementNodeS 18 | key string 19 | value string 20 | } 21 | 22 | // S represents the list. 23 | S struct { 24 | elementNodeS 25 | randSource rand.Source 26 | searchNodesCache []*elementNodeS 27 | probability float64 28 | probTable []float64 29 | maxLevel int 30 | length int 31 | mutex sync.RWMutex 32 | } 33 | ) 34 | 35 | // Key returns the given Element key. 36 | func (e *ElementS) Key() string { 37 | return e.key 38 | } 39 | 40 | // Value returns the given Element value. 41 | func (e *ElementS) Value() string { 42 | return e.value 43 | } 44 | 45 | // Next returns the adjacent next Element if existed or nil otherwise. 46 | func (e *ElementS) Next() *ElementS { 47 | return e.forward[0] 48 | } 49 | 50 | // NextLevel returns the adjacent next Element at provided level if existed or nil otherwise. 51 | func (e *ElementS) NextLevel(level int) *ElementS { 52 | if level >= len(e.forward) || level < 0 { 53 | return nil 54 | } 55 | 56 | return e.forward[level] 57 | } 58 | -------------------------------------------------------------------------------- /pkg/spinlock/spinlock_test.go: -------------------------------------------------------------------------------- 1 | package spinlock_test 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | "time" 7 | 8 | . "github.com/andy2046/gopie/pkg/spinlock" 9 | ) 10 | 11 | func TestLocker(t *testing.T) { 12 | threads, loops, count := 8, 1000, 0 13 | var wg sync.WaitGroup 14 | wg.Add(threads) 15 | 16 | l := New() 17 | start := time.Now() 18 | for i := 0; i < threads; i++ { 19 | go func() { 20 | for i := 0; i < loops; i++ { 21 | l.Lock() 22 | count++ 23 | if !l.IsLocked() { 24 | t.Error("expected to be locked") 25 | } 26 | l.Unlock() 27 | } 28 | wg.Done() 29 | }() 30 | } 31 | wg.Wait() 32 | 33 | duration := time.Since(start) 34 | t.Logf("duration: %4.2f Seconds\n", duration.Seconds()) 35 | if count != threads*loops { 36 | t.Errorf("expected %d got %d", threads*loops, count) 37 | } 38 | } 39 | 40 | func TestNoCopy(t *testing.T) { 41 | // go vet fails 42 | var l1 Locker 43 | l2 := l1 44 | var l3 = l1 45 | l2 = l1 46 | _, _ = l2, l3 47 | t.Log("go vet fails here") 48 | } 49 | 50 | func BenchmarkLock(b *testing.B) { 51 | l := New() 52 | for n := 0; n < b.N; n++ { 53 | l.Lock() 54 | l.Unlock() 55 | } 56 | } 57 | 58 | func BenchmarkMutex(b *testing.B) { 59 | var l sync.Mutex 60 | for n := 0; n < b.N; n++ { 61 | l.Lock() 62 | l.Unlock() 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /docs/latest.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # latest 4 | `import "github.com/andy2046/gopie/pkg/latest"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | * [Examples](#pkg-examples) 9 | 10 | ## Overview 11 | Package latest provides latest value. 12 | 13 | 14 | 15 | 16 | ## Index 17 | * [func New(receiver chan<- interface{}) chan<- interface{}](#New) 18 | * [func NewN(receiver chan<- []interface{}, n int) chan<- interface{}](#NewN) 19 | 20 | #### Examples 21 | * [New](#example_New) 22 | * [NewN](#example_NewN) 23 | 24 | #### Package files 25 | [latest.go](/src/github.com/andy2046/gopie/pkg/latest/latest.go) 26 | 27 | 28 | 29 | 30 | 31 | ## func [New](/src/target/latest.go?s=121:177#L5) 32 | ``` go 33 | func New(receiver chan<- interface{}) chan<- interface{} 34 | ``` 35 | New send latest value from returned chan to `receiver` chan. 36 | 37 | 38 | 39 | ## func [NewN](/src/target/latest.go?s=586:652#L34) 40 | ``` go 41 | func NewN(receiver chan<- []interface{}, n int) chan<- interface{} 42 | ``` 43 | NewN send latest `n` values from returned chan to `receiver` chan. 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | - - - 53 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 54 | -------------------------------------------------------------------------------- /pkg/sequence/memflake_test.go: -------------------------------------------------------------------------------- 1 | package sequence_test 2 | 3 | import ( 4 | . "github.com/andy2046/gopie/pkg/sequence" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestIceflake_Next(t *testing.T) { 10 | s := NewMemFlake(time.Time{}, 0, 0) 11 | if s == nil { 12 | t.Fatal("error: nil Iceflake") 13 | } 14 | test := tester{t, s} 15 | 16 | n0 := test.Next() 17 | for i := 0; i < 3; i++ { 18 | test.Next() 19 | } 20 | n := test.Next() 21 | if n <= n0 { 22 | t.Errorf("error: expected %d greater than %d", n, n0) 23 | } 24 | 25 | n0 = test.NextN(1000) 26 | if n0 <= n { 27 | t.Errorf("error: expected %d greater than %d", n0, n) 28 | } 29 | 30 | n = test.Next() 31 | if n <= n0 { 32 | t.Errorf("error: expected %d greater than %d", n, n0) 33 | } 34 | } 35 | 36 | func TestIceflake_Parallel(t *testing.T) { 37 | s := NewMemFlake(time.Time{}, 0, 0) 38 | test := tester{t, s} 39 | 40 | consumer := make(chan uint64) 41 | const numID = 10000 42 | generate := func() { 43 | for i := 0; i < numID; i++ { 44 | consumer <- test.Next() 45 | } 46 | } 47 | const numGenerator = 10 48 | for i := 0; i < numGenerator; i++ { 49 | go generate() 50 | } 51 | set := make(map[uint64]bool) 52 | for i := 0; i < numID*numGenerator; i++ { 53 | id := <-consumer 54 | if _, exist := set[id]; exist { 55 | t.Fatal("error: duplicated") 56 | } 57 | set[id] = true 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pkg/spinlock/spinlock.go: -------------------------------------------------------------------------------- 1 | // Package spinlock implements Spinlock. 2 | package spinlock 3 | 4 | import ( 5 | "runtime" 6 | "sync" 7 | "sync/atomic" 8 | ) 9 | 10 | // Locker is the Spinlock implementation. 11 | type Locker struct { 12 | noCopy noCopy 13 | lock uintptr 14 | } 15 | 16 | // New creates a Locker. 17 | func New() *Locker { 18 | return &Locker{} 19 | } 20 | 21 | // Lock wait in a loop to acquire the spinlock. 22 | func (l *Locker) Lock() { 23 | for !atomic.CompareAndSwapUintptr(&l.lock, 0, 1) { 24 | runtime.Gosched() 25 | } 26 | } 27 | 28 | // Unlock release the spinlock. 29 | func (l *Locker) Unlock() { 30 | atomic.StoreUintptr(&l.lock, 0) 31 | } 32 | 33 | // TryLock try to acquire the spinlock, 34 | // it returns true if succeed, false otherwise. 35 | func (l *Locker) TryLock() bool { 36 | return atomic.CompareAndSwapUintptr(&l.lock, 0, 1) 37 | } 38 | 39 | // IsLocked returns true if locked, false otherwise. 40 | func (l *Locker) IsLocked() bool { 41 | return atomic.LoadUintptr(&l.lock) == 1 42 | } 43 | 44 | // Locker returns a sync.Locker interface implementation. 45 | func (l *Locker) Locker() sync.Locker { 46 | return l 47 | } 48 | 49 | // noCopy may be embedded into structs which must not be copied 50 | // after the first use. 51 | type noCopy struct{} 52 | 53 | // Lock is a no-op used by -copylocks checker from `go vet`. 54 | func (*noCopy) Lock() {} 55 | func (*noCopy) Unlock() {} 56 | -------------------------------------------------------------------------------- /pkg/latest/latest.go: -------------------------------------------------------------------------------- 1 | // Package latest provides latest value. 2 | package latest 3 | 4 | // New send latest value from returned chan to `receiver` chan. 5 | func New(receiver chan<- interface{}) chan<- interface{} { 6 | sender := make(chan interface{}) 7 | 8 | go func() { 9 | var ( 10 | latest interface{} 11 | ok bool 12 | temp chan<- interface{} 13 | ) 14 | for { 15 | select { 16 | case latest, ok = <-sender: 17 | if !ok { 18 | return 19 | } 20 | if temp == nil { 21 | temp = receiver 22 | } 23 | continue 24 | case temp <- latest: 25 | break 26 | } 27 | } 28 | }() 29 | 30 | return sender 31 | } 32 | 33 | // NewN send latest `n` values from returned chan to `receiver` chan. 34 | func NewN(receiver chan<- []interface{}, n int) chan<- interface{} { 35 | if n < 1 { 36 | panic("n should be positive") 37 | } 38 | sender := make(chan interface{}) 39 | store := make([]interface{}, n) 40 | 41 | go func() { 42 | var ( 43 | latest interface{} 44 | ok bool 45 | temp chan<- []interface{} 46 | ) 47 | for { 48 | select { 49 | case latest, ok = <-sender: 50 | if !ok { 51 | return 52 | } 53 | store = append(store[1:], latest) 54 | 55 | if nil == temp { 56 | temp = receiver 57 | } 58 | continue 59 | case temp <- append(store[:0:0], store...): 60 | break 61 | } 62 | } 63 | }() 64 | 65 | return sender 66 | } 67 | -------------------------------------------------------------------------------- /docs/nocopy.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # nocopy 4 | `import "github.com/andy2046/gopie/pkg/nocopy"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package nocopy implements the interface for -copylocks checker from `go vet`. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [type NoCopy](#NoCopy) 17 | * [func (*NoCopy) Lock()](#NoCopy.Lock) 18 | * [func (*NoCopy) Unlock()](#NoCopy.Unlock) 19 | 20 | 21 | #### Package files 22 | [nocopy.go](/src/github.com/andy2046/gopie/pkg/nocopy/nocopy.go) 23 | 24 | 25 | 26 | 27 | 28 | 29 | ## type [NoCopy](/src/target/nocopy.go?s=185:205#L6) 30 | ``` go 31 | type NoCopy struct{} 32 | ``` 33 | NoCopy can be embedded into structs which must not be copied 34 | after the first use. 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | ### func (\*NoCopy) [Lock](/src/target/nocopy.go?s=268:289#L9) 46 | ``` go 47 | func (*NoCopy) Lock() 48 | ``` 49 | Lock is a no-op used by -copylocks checker from `go vet`. 50 | 51 | 52 | 53 | 54 | ### func (\*NoCopy) [Unlock](/src/target/nocopy.go?s=357:380#L12) 55 | ``` go 56 | func (*NoCopy) Unlock() 57 | ``` 58 | Unlock is not required by -copylocks checker from `go vet`. 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | - - - 68 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 69 | -------------------------------------------------------------------------------- /docs/randomsequence.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # randomsequence 4 | `import "github.com/andy2046/gopie/pkg/randomsequence"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package randomsequence implements quadratic residues based random sequence. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [type Random](#Random) 17 | * [func New(seedBase, seedOffset uint32) *Random](#New) 18 | * [func (r *Random) Next() uint32](#Random.Next) 19 | 20 | 21 | #### Package files 22 | [randomseq.go](/src/github.com/andy2046/gopie/pkg/randomsequence/randomseq.go) 23 | 24 | 25 | 26 | 27 | 28 | 29 | ## type [Random](/src/target/randomseq.go?s=232:284#L7) 30 | ``` go 31 | type Random struct { 32 | // contains filtered or unexported fields 33 | } 34 | ``` 35 | Random represents the random sequence. 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | ### func [New](/src/target/randomseq.go?s=413:458#L18) 44 | ``` go 45 | func New(seedBase, seedOffset uint32) *Random 46 | ``` 47 | New creates a random sequence with the seed provided. 48 | 49 | 50 | 51 | 52 | 53 | ### func (\*Random) [Next](/src/target/randomseq.go?s=624:654#L26) 54 | ``` go 55 | func (r *Random) Next() uint32 56 | ``` 57 | Next returns next random number. 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | - - - 67 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 68 | -------------------------------------------------------------------------------- /docs/base58.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # base58 4 | `import "github.com/andy2046/gopie/pkg/base58"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package base58 implements Base58 Encoder interface. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [Variables](#pkg-variables) 17 | * [func Decode(s string) (uint64, error)](#Decode) 18 | * [func Encode(n uint64) string](#Encode) 19 | * [func Reverse(b []byte)](#Reverse) 20 | 21 | 22 | #### Package files 23 | [base58.go](/src/github.com/andy2046/gopie/pkg/base58/base58.go) 24 | 25 | 26 | 27 | ## Variables 28 | ``` go 29 | var ( 30 | 31 | // ErrEmptyString for empty Base58 encoded string. 32 | ErrEmptyString = errors.New("base58: empty Base58 encoded string") 33 | ) 34 | ``` 35 | 36 | 37 | ## func [Decode](/src/target/base58.go?s=885:922#L49) 38 | ``` go 39 | func Decode(s string) (uint64, error) 40 | ``` 41 | Decode returns Base58 decoded unsigned int64. 42 | 43 | 44 | 45 | ## func [Encode](/src/target/base58.go?s=585:613#L33) 46 | ``` go 47 | func Encode(n uint64) string 48 | ``` 49 | Encode returns Base58 encoded string. 50 | 51 | 52 | 53 | ## func [Reverse](/src/target/base58.go?s=1202:1224#L69) 54 | ``` go 55 | func Reverse(b []byte) 56 | ``` 57 | Reverse your byte slice. 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | - - - 67 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 68 | -------------------------------------------------------------------------------- /pkg/drf/drf_test.go: -------------------------------------------------------------------------------- 1 | package drf_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/andy2046/gopie/pkg/drf" 7 | ) 8 | 9 | func TestDRF(t *testing.T) { 10 | nodeA := drf.NewNode(map[drf.Typ]float64{ 11 | drf.CPU: 1, 12 | drf.MEMORY: 4, 13 | }) 14 | nodeB := drf.NewNode(map[drf.Typ]float64{ 15 | drf.CPU: 3, 16 | drf.MEMORY: 1, 17 | }) 18 | cluster, _ := drf.New(map[drf.Typ]float64{ 19 | drf.CPU: 9, 20 | drf.MEMORY: 18, 21 | }, nodeA, nodeB) 22 | var err error 23 | for err == nil { 24 | err = cluster.NextTask() 25 | } 26 | t.Logf("Error from NextTask %v", err) 27 | t.Logf("Resource for cluster %v", cluster.Resource()) 28 | t.Logf("Consumed by cluster %v", cluster.Consumed()) 29 | allcNodeA := nodeA.Allocated() 30 | t.Logf("Allocated for node A %v", allcNodeA) 31 | if allcNodeA[drf.CPU] != 3 { 32 | t.Fatalf("got %v, want %v", allcNodeA[drf.CPU], 3) 33 | } 34 | allcNodeB := nodeB.Allocated() 35 | t.Logf("Allocated for node B %v", allcNodeB) 36 | if allcNodeB[drf.CPU] != 6 { 37 | t.Fatalf("got %v, want %v", allcNodeB[drf.CPU], 6) 38 | } 39 | } 40 | 41 | func BenchmarkDRF(b *testing.B) { 42 | nodeA := drf.NewNode(map[drf.Typ]float64{ 43 | drf.CPU: 1, 44 | drf.MEMORY: 4, 45 | }) 46 | nodeB := drf.NewNode(map[drf.Typ]float64{ 47 | drf.CPU: 3, 48 | drf.MEMORY: 1, 49 | }) 50 | cluster, _ := drf.New(map[drf.Typ]float64{ 51 | drf.CPU: 9, 52 | drf.MEMORY: 18, 53 | }, nodeA, nodeB) 54 | 55 | b.ReportAllocs() 56 | b.ResetTimer() 57 | for i := 0; i < b.N; i++ { 58 | cluster.NextTask() 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /pkg/base58/base58.go: -------------------------------------------------------------------------------- 1 | // Package base58 implements Base58 Encoder interface. 2 | package base58 3 | 4 | import ( 5 | "encoding/binary" 6 | "errors" 7 | "fmt" 8 | ) 9 | 10 | const ( 11 | source = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" 12 | ln = uint64(58) 13 | ) 14 | 15 | var ( 16 | encodeMap [58]byte 17 | decodeMap [256]int 18 | // ErrEmptyString for empty Base58 encoded string. 19 | ErrEmptyString = errors.New("base58: empty Base58 encoded string") 20 | ) 21 | 22 | func init() { 23 | for i := range decodeMap { 24 | decodeMap[i] = -1 25 | } 26 | for i := range source { 27 | encodeMap[i] = source[i] 28 | decodeMap[encodeMap[i]] = i 29 | } 30 | } 31 | 32 | // Encode returns Base58 encoded string. 33 | func Encode(n uint64) string { 34 | if n == 0 { 35 | return string(encodeMap[:1]) 36 | } 37 | 38 | b, div := make([]byte, 0, binary.MaxVarintLen64), uint64(0) 39 | for n > 0 { 40 | div = n / ln 41 | b = append(b, encodeMap[n-div*ln]) 42 | n = div 43 | } 44 | Reverse(b) 45 | return string(b) 46 | } 47 | 48 | // Decode returns Base58 decoded unsigned int64. 49 | func Decode(s string) (uint64, error) { 50 | if s == "" { 51 | return 0, ErrEmptyString 52 | } 53 | 54 | var ( 55 | n uint64 56 | c int 57 | ) 58 | for i := range s { 59 | c = decodeMap[s[i]] 60 | if c < 0 { 61 | return 0, fmt.Errorf("base58: invalid character <%s>", string(s[i])) 62 | } 63 | n = n*58 + uint64(c) 64 | } 65 | return n, nil 66 | } 67 | 68 | // Reverse your byte slice. 69 | func Reverse(b []byte) { 70 | for l, r := 0, len(b)-1; l < r; l, r = l+1, r-1 { 71 | b[l], b[r] = b[r], b[l] 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /pkg/pushsum/store.go: -------------------------------------------------------------------------------- 1 | package pushsum 2 | 3 | import "sync" 4 | 5 | type store struct { 6 | index map[uint64]uint64 // map key to ring index 7 | ring []valueWeight 8 | sync.RWMutex 9 | } 10 | 11 | func newStore(len uint32) *store { 12 | return &store{ 13 | index: make(map[uint64]uint64, len), 14 | ring: make([]valueWeight, len, len), 15 | } 16 | } 17 | 18 | func (s *store) get(key uint64) (uint64, valueWeight, bool) { 19 | s.RLock() 20 | pos, ok := s.index[key] 21 | if !ok { 22 | // not exist 23 | s.RUnlock() 24 | return 0, valueWeight{}, false 25 | } 26 | 27 | v := s.ring[pos] 28 | if v.key != key { 29 | // k deleted from index? 30 | s.RUnlock() 31 | return 0, valueWeight{}, false 32 | } 33 | 34 | s.RUnlock() 35 | return pos, v, true 36 | } 37 | 38 | func (s *store) compareAndSet(idx uint64, vw valueWeight) bool { 39 | s.Lock() 40 | if v := s.ring[idx]; v.key != vw.key { 41 | s.Unlock() 42 | return false 43 | } 44 | 45 | s.ring[idx] = vw 46 | s.Unlock() 47 | return true 48 | } 49 | 50 | func (s *store) update(key uint64, value, weight float64, inTransition bool) bool { 51 | s.Lock() 52 | defer s.Unlock() 53 | 54 | i, ok := s.index[key] 55 | if !ok { 56 | return false 57 | } 58 | 59 | vw := s.ring[i] 60 | if vw.key == key { 61 | vw.value += value 62 | vw.weight += weight 63 | vw.inTransition = inTransition 64 | s.ring[i] = vw 65 | } 66 | 67 | return true 68 | } 69 | 70 | func (s *store) insert(idx uint64, vw valueWeight) { 71 | s.Lock() 72 | // remove index for existing vw 73 | e := s.ring[idx] 74 | delete(s.index, e.key) 75 | 76 | s.ring[idx] = vw 77 | s.index[vw.key] = idx 78 | s.Unlock() 79 | } 80 | -------------------------------------------------------------------------------- /pkg/sequence/memsequence_test.go: -------------------------------------------------------------------------------- 1 | package sequence_test 2 | 3 | import ( 4 | . "github.com/andy2046/gopie/pkg/sequence" 5 | "testing" 6 | ) 7 | 8 | type tester struct { 9 | T *testing.T 10 | seq Sequencer 11 | } 12 | 13 | func (t tester) Next() uint64 { 14 | id, err := t.seq.Next() 15 | if err != nil { 16 | t.T.Fatal("error:", err) 17 | } 18 | return id 19 | } 20 | 21 | func (t tester) NextN(n int) uint64 { 22 | id, err := t.seq.NextN(n) 23 | if err != nil { 24 | t.T.Fatal("error:", err) 25 | } 26 | return id 27 | } 28 | 29 | func TestSequence_Next(t *testing.T) { 30 | s := NewMemSeq(0) 31 | test := tester{t, s} 32 | 33 | loop := uint64(3) 34 | for i := uint64(0); i < loop; i++ { 35 | test.Next() 36 | } 37 | n := test.Next() 38 | if n != loop { 39 | t.Errorf("error: expected %d got %d", loop, n) 40 | } 41 | 42 | n = test.NextN(1000) 43 | if n != loop+1 { 44 | t.Errorf("error: expected %d got %d", loop+1, n) 45 | } 46 | 47 | n = test.Next() 48 | if n != loop+1001 { 49 | t.Errorf("error: expected %d got %d", loop+1001, n) 50 | } 51 | } 52 | 53 | func TestSequence_Parallel(t *testing.T) { 54 | s := NewMemSeq(0) 55 | test := tester{t, s} 56 | 57 | consumer := make(chan uint64) 58 | const numID = 10000 59 | generate := func() { 60 | for i := 0; i < numID; i++ { 61 | consumer <- test.Next() 62 | } 63 | } 64 | const numGenerator = 10 65 | for i := 0; i < numGenerator; i++ { 66 | go generate() 67 | } 68 | set := make(map[uint64]bool) 69 | for i := 0; i < numID*numGenerator; i++ { 70 | id := <-consumer 71 | if _, exist := set[id]; exist { 72 | t.Fatal("error: duplicated") 73 | } 74 | set[id] = true 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /pkg/jumphash/jumphash.go: -------------------------------------------------------------------------------- 1 | // Package jumphash provides a jump consistent hash implementation. 2 | package jumphash 3 | 4 | import ( 5 | "hash/crc64" 6 | "io" 7 | ) 8 | 9 | var ( 10 | // hashCRC64 uses the 64-bit Cyclic Redundancy Check (CRC-64) with the ECMA polynomial. 11 | hashCRC64 = crc64.New(crc64.MakeTable(crc64.ECMA)) 12 | ) 13 | 14 | // Hash takes a key and the number of buckets, returns an integer in the range [0, buckets). 15 | // If the number of buckets is not greater than 1 then 1 is used. 16 | func Hash(key uint64, buckets int) int { 17 | var b, j int64 18 | 19 | if buckets <= 0 { 20 | buckets = 1 21 | } 22 | 23 | for j < int64(buckets) { 24 | b = j 25 | key = key*2862933555777941757 + 1 26 | j = int64(float64(b+1) * (float64(int64(1)<<31) / float64((key>>33)+1))) 27 | } 28 | 29 | return int(b) 30 | } 31 | 32 | // HashString takes string as key instead of integer and uses CRC-64 to generate key. 33 | func HashString(key string, buckets int) int { 34 | hashCRC64.Reset() 35 | _, err := io.WriteString(hashCRC64, key) 36 | if err != nil { 37 | panic(err) 38 | } 39 | return Hash(hashCRC64.Sum64(), buckets) 40 | } 41 | 42 | // Hasher represents a jump consistent Hasher using a string as key. 43 | type Hasher struct { 44 | n int32 45 | } 46 | 47 | // New returns a new instance of of Hasher. 48 | func New(n int) *Hasher { 49 | if n <= 0 { 50 | panic("the number of buckets must be positive int") 51 | } 52 | return &Hasher{int32(n)} 53 | } 54 | 55 | // N returns the number of buckets the hasher can assign to. 56 | func (h *Hasher) N() int { 57 | return int(h.n) 58 | } 59 | 60 | // Hash returns the integer hash for the given key. 61 | func (h *Hasher) Hash(key string) int { 62 | return HashString(key, int(h.n)) 63 | } 64 | -------------------------------------------------------------------------------- /docs/batch.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # batch 4 | `import "github.com/andy2046/gopie/pkg/batch"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package batch implements batch utility. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [type Batch](#Batch) 17 | * [func New(timeout time.Duration, count int, fn func([]interface{})) *Batch](#New) 18 | * [func (b *Batch) Batch(item interface{})](#Batch.Batch) 19 | * [func (b *Batch) Close()](#Batch.Close) 20 | 21 | 22 | #### Package files 23 | [batch.go](/src/github.com/andy2046/gopie/pkg/batch/batch.go) 24 | 25 | 26 | 27 | 28 | 29 | 30 | ## type [Batch](/src/target/batch.go?s=191:407#L14) 31 | ``` go 32 | type Batch struct { 33 | // contains filtered or unexported fields 34 | } 35 | ``` 36 | Batch process batch of work by time duration or batch count. 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | ### func [New](/src/target/batch.go?s=493:566#L27) 45 | ``` go 46 | func New(timeout time.Duration, count int, fn func([]interface{})) *Batch 47 | ``` 48 | New returns a Batch with time duration, batch count and batch function provided. 49 | 50 | 51 | 52 | 53 | 54 | ### func (\*Batch) [Batch](/src/target/batch.go?s=891:930#L46) 55 | ``` go 56 | func (b *Batch) Batch(item interface{}) 57 | ``` 58 | Batch batches the given item. 59 | 60 | 61 | 62 | 63 | ### func (\*Batch) [Close](/src/target/batch.go?s=1081:1104#L56) 64 | ``` go 65 | func (b *Batch) Close() 66 | ``` 67 | Close the batch. 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | - - - 77 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 78 | -------------------------------------------------------------------------------- /docs/deadline.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # deadline 4 | `import "github.com/andy2046/gopie/pkg/deadline"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package deadline implements Deadline pattern. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [Variables](#pkg-variables) 17 | * [type Deadline](#Deadline) 18 | * [func New(timeout time.Duration) *Deadline](#New) 19 | * [func (d *Deadline) Go(fn func(<-chan struct{}) error) error](#Deadline.Go) 20 | 21 | 22 | #### Package files 23 | [deadline.go](/src/github.com/andy2046/gopie/pkg/deadline/deadline.go) 24 | 25 | 26 | 27 | ## Variables 28 | ``` go 29 | var ErrTimeout = errors.New("time out executing function") 30 | ``` 31 | ErrTimeout is the error for deadline timeout. 32 | 33 | 34 | 35 | 36 | ## type [Deadline](/src/target/deadline.go?s=243:290#L13) 37 | ``` go 38 | type Deadline struct { 39 | // contains filtered or unexported fields 40 | } 41 | ``` 42 | Deadline represents the deadline. 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | ### func [New](/src/target/deadline.go?s=349:390#L18) 51 | ``` go 52 | func New(timeout time.Duration) *Deadline 53 | ``` 54 | New returns a new Deadline with the provided timeout. 55 | 56 | 57 | 58 | 59 | 60 | ### func (\*Deadline) [Go](/src/target/deadline.go?s=531:590#L25) 61 | ``` go 62 | func (d *Deadline) Go(fn func(<-chan struct{}) error) error 63 | ``` 64 | Go executes the provided function with a done channel as parameter to signal the timeout. 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | - - - 74 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 75 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | A collection of idiomatic patterns for Go language. 3 | 4 | ![gopie](./gopie.png "gopie") 5 | 6 | ## Patterns 7 | 8 | | Pattern | Description | Status | 9 | |:-------:|:----------- |:------:| 10 | | [Barrier](/docs/barrier.md) | Prevents a process from proceeding until all N processes reach to the barrier | ✔ | 11 | | [Deadline](/docs/deadline.md) | Implements Deadline pattern | ✔ | 12 | | [DRF](/docs/drf.md) | Implements Dominant Resource Fairness | ✔ | 13 | | [JumpHash](/docs/jumphash.md) | Provides a jump consistent hash implementation | ✔ | 14 | | [LRU](/docs/lru.md) | Implements a LRU cache | ✔ | 15 | | [Publish/Subscribe](/docs/pubsub.md) | Passes information to a collection of recipients who subscribed to a topic | ✔ | 16 | | [RingHash](/docs/ringhash.md) | Provides a ring hash implementation | ✔ | 17 | | [Semaphore](/docs/semaphore.md) | Allows controlling access to a common resource | ✔ | 18 | | [Singleton](/docs/singleton.md) | Restricts instantiation of a type to one object | ✔ | 19 | | [Subsetting](/docs/subset.md) | Implements client deterministic subsetting | ✔ | 20 | | [SkipList](/docs/skiplist.md) | Implements Skip List data structure | ✔ | 21 | | [BloomFilter](/docs/bloom.md) | Implements Bloom filter | ✔ | 22 | | [Count-Min Sketch](/docs/countminsketch.md) | Implements Count-Min Sketch | ✔ | 23 | | [HyperLogLog](/docs/hyperloglog.md) | Implements HyperLogLog cardinality estimation | ✔ | 24 | | [Circuit Breaker](/docs/breaker.md) | Implements Circuit Breaker | ✔ | 25 | | [Rate Limiter](/docs/ratelimit.md) | Implements Rate Limiter | ✔ | 26 | | [Bit Flag](/docs/bitflag.md) | Implements Bit Flag | ✔ | 27 | | [Base58](/docs/base58.md) | Implements Base58 Encoder | ✔ | 28 | | [Sequence](/docs/sequence.md) | Implements snowflake similar sequence generator | ✔ | 29 | -------------------------------------------------------------------------------- /pkg/jumphash/jumphash_test.go: -------------------------------------------------------------------------------- 1 | package jumphash_test 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | 7 | . "github.com/andy2046/gopie/pkg/jumphash" 8 | ) 9 | 10 | var jumphashTest = []struct { 11 | key uint64 12 | buckets int 13 | expected int 14 | }{ 15 | {1, 1, 0}, 16 | {42, 57, 43}, 17 | {0xDEAD10CC, 1, 0}, 18 | {0xDEAD10CC, 666, 361}, 19 | {256, 1024, 520}, 20 | {0, -10, 0}, 21 | } 22 | 23 | func TestHash(t *testing.T) { 24 | for _, v := range jumphashTest { 25 | h := Hash(v.key, v.buckets) 26 | if h != v.expected { 27 | t.Errorf("expected bucket for key=%d to be %d, got %d", 28 | v.key, v.expected, h) 29 | } 30 | } 31 | } 32 | 33 | var jumphashStringTest = []struct { 34 | key string 35 | buckets int 36 | expected int 37 | }{ 38 | {"localhost", 10, 6}, 39 | {"中国", 10, 1}, 40 | } 41 | 42 | func TestHashString(t *testing.T) { 43 | for _, v := range jumphashStringTest { 44 | h := HashString(v.key, v.buckets) 45 | if h != v.expected { 46 | t.Errorf("invalid bucket for key=%s, expected %d, got %d", 47 | strconv.Quote(v.key), v.expected, h) 48 | } 49 | } 50 | } 51 | 52 | func TestHasher(t *testing.T) { 53 | for _, v := range jumphashStringTest { 54 | hasher := New(v.buckets) 55 | h := hasher.Hash(v.key) 56 | if h != v.expected { 57 | t.Errorf("invalid bucket for key=%s, expected %d, got %d", 58 | strconv.Quote(v.key), v.expected, h) 59 | } 60 | } 61 | } 62 | 63 | func BenchmarkHashN100(b *testing.B) { benchmarkHash(b, 100) } 64 | func BenchmarkHashN1000(b *testing.B) { benchmarkHash(b, 1000) } 65 | func BenchmarkHashN10000(b *testing.B) { benchmarkHash(b, 10000) } 66 | 67 | func benchmarkHash(b *testing.B, n int) { 68 | h := New(n) 69 | for i := 0; i < b.N; i++ { 70 | if x := h.Hash(strconv.Itoa(i)); x > n { 71 | b.Fatal("invalid hash:", x) 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /pkg/bitflag/bitflag.go: -------------------------------------------------------------------------------- 1 | // Package bitflag implements bit flag. 2 | package bitflag 3 | 4 | // Flag is an 8 bits flag. 5 | type Flag byte 6 | 7 | const siz uint8 = 7 8 | 9 | // SetAll set all the flags. 10 | func (f *Flag) SetAll(opts ...Flag) { 11 | for _, o := range opts { 12 | *f |= o 13 | } 14 | } 15 | 16 | // ToggleAll toggle (XOR) all the flags. 17 | func (f *Flag) ToggleAll(opts ...Flag) { 18 | for _, o := range opts { 19 | *f ^= o 20 | } 21 | } 22 | 23 | // ClearAll clear all the flags. 24 | func (f *Flag) ClearAll(opts ...Flag) { 25 | for _, o := range opts { 26 | *f &^= o 27 | } 28 | } 29 | 30 | // AreAllSet check if all the flags are set. 31 | func (f Flag) AreAllSet(opts ...Flag) bool { 32 | for _, o := range opts { 33 | if f&o == 0 { 34 | return false 35 | } 36 | } 37 | return true 38 | } 39 | 40 | // IsAnySet check if any one flag is set. 41 | func (f Flag) IsAnySet(opts ...Flag) bool { 42 | for _, o := range opts { 43 | if f&o > 0 { 44 | return true 45 | } 46 | } 47 | return false 48 | } 49 | 50 | // IsSet check if the bit at `n` is set. 51 | // `n` should be less than `8`. 52 | func (f Flag) IsSet(n uint8) bool { 53 | if n > siz || f&(1< siz { 63 | return 64 | } 65 | *f |= (1 << n) 66 | } 67 | 68 | // Toggle toggle (XOR) a single bit at `n`. 69 | // `n` should be less than `8`. 70 | func (f *Flag) Toggle(n uint8) { 71 | if n > siz { 72 | return 73 | } 74 | *f ^= (1 << n) 75 | } 76 | 77 | // Clear clear a single bit at `n`. 78 | // `n` should be less than `8`. 79 | func (f *Flag) Clear(n uint8) { 80 | if n > siz { 81 | return 82 | } 83 | *f &^= (1 << n) 84 | } 85 | 86 | // Reset reset the flag. 87 | func (f *Flag) Reset() { 88 | *f = 0 89 | } 90 | -------------------------------------------------------------------------------- /pkg/batch/batch_test.go: -------------------------------------------------------------------------------- 1 | package batch 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestTime(t *testing.T) { 11 | var ( 12 | callcount int32 13 | wg sync.WaitGroup 14 | loop = 2 15 | ) 16 | wg.Add(loop) 17 | 18 | b := New(1*time.Second, 10, func(batch []interface{}) { 19 | atomic.AddInt32(&callcount, 1) 20 | if len(batch) != loop { 21 | t.Errorf("batch size got %v want %v", len(batch), loop) 22 | } 23 | }) 24 | defer b.Close() 25 | 26 | for i := 0; i < loop; i++ { 27 | go func(i int) { 28 | b.Batch(i) 29 | wg.Done() 30 | }(i) 31 | } 32 | 33 | wg.Wait() 34 | time.Sleep(2 * time.Second) 35 | 36 | cc := atomic.LoadInt32(&callcount) 37 | if cc != 1 { 38 | t.Errorf("callcount got %v want %v", cc, 1) 39 | } 40 | } 41 | 42 | func TestCount(t *testing.T) { 43 | var ( 44 | callcount int32 45 | wg sync.WaitGroup 46 | loop = 10 47 | ) 48 | wg.Add(loop) 49 | 50 | b := New(10*time.Second, 10, func(batch []interface{}) { 51 | atomic.AddInt32(&callcount, 1) 52 | if len(batch) != loop { 53 | t.Errorf("batch size got %v want %v", len(batch), loop) 54 | } 55 | }) 56 | defer b.Close() 57 | 58 | for i := 0; i < loop; i++ { 59 | go func(i int) { 60 | b.Batch(i) 61 | wg.Done() 62 | }(i) 63 | } 64 | 65 | wg.Wait() 66 | time.Sleep(1 * time.Second) 67 | 68 | cc := atomic.LoadInt32(&callcount) 69 | if cc != 1 { 70 | t.Errorf("callcount got %v want %v", cc, 1) 71 | } 72 | } 73 | 74 | func TestLoad(t *testing.T) { 75 | var ( 76 | wg sync.WaitGroup 77 | loop = 10000 78 | ) 79 | wg.Add(loop) 80 | 81 | b := New(1*time.Second, 100, func(batch []interface{}) { 82 | }) 83 | defer b.Close() 84 | 85 | for i := 0; i < loop; i++ { 86 | go func(i int) { 87 | b.Batch(i) 88 | wg.Done() 89 | }(i) 90 | } 91 | 92 | wg.Wait() 93 | time.Sleep(1 * time.Second) 94 | } 95 | -------------------------------------------------------------------------------- /pkg/dll/amr.go: -------------------------------------------------------------------------------- 1 | package dll 2 | 3 | import ( 4 | "sync/atomic" 5 | "unsafe" 6 | ) 7 | 8 | type ( 9 | atomicMarkableReference struct { 10 | pair *pair 11 | } 12 | 13 | pair struct { 14 | reference *Element 15 | mark bool 16 | } 17 | ) 18 | 19 | func newAtomicMarkableReference(initialRef *Element, initialMark bool) *atomicMarkableReference { 20 | return &atomicMarkableReference{ 21 | &pair{ 22 | reference: initialRef, 23 | mark: initialMark, 24 | }, 25 | } 26 | } 27 | 28 | func (amr *atomicMarkableReference) getPair() *pair { 29 | return (*pair)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&amr.pair)))) 30 | } 31 | 32 | func (amr *atomicMarkableReference) getReference() *Element { 33 | p := amr.getPair() 34 | return p.reference 35 | } 36 | 37 | func (amr *atomicMarkableReference) isMarked() bool { 38 | p := amr.getPair() 39 | return p.mark 40 | } 41 | 42 | func (amr *atomicMarkableReference) get() (bool, *Element) { 43 | p := amr.getPair() 44 | return p.mark, p.reference 45 | } 46 | 47 | func (amr *atomicMarkableReference) compareAndSet(expectedReference *Element, 48 | newReference *Element, 49 | expectedMark bool, 50 | newMark bool) bool { 51 | current := amr.getPair() 52 | val := &pair{newReference, newMark} 53 | 54 | return expectedReference == current.reference && 55 | expectedMark == current.mark && 56 | ((newReference == current.reference && 57 | newMark == current.mark) || 58 | amr.casPair(current, val)) 59 | } 60 | 61 | func (amr *atomicMarkableReference) tryMark(expectedReference *Element, newMark bool) bool { 62 | current := amr.getPair() 63 | val := &pair{expectedReference, newMark} 64 | return expectedReference == current.reference && 65 | (newMark == current.mark || 66 | amr.casPair(current, val)) 67 | } 68 | 69 | func (amr *atomicMarkableReference) casPair(cmp *pair, val *pair) bool { 70 | return atomic.CompareAndSwapPointer( 71 | (*unsafe.Pointer)(unsafe.Pointer(&amr.pair)), 72 | unsafe.Pointer(cmp), 73 | unsafe.Pointer(val)) 74 | } 75 | -------------------------------------------------------------------------------- /pkg/backoff/backoff_test.go: -------------------------------------------------------------------------------- 1 | package backoff 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | var result time.Duration 9 | 10 | func TestExponential(t *testing.T) { 11 | baseDelay, capDelay := 1*time.Second, 2*time.Second 12 | bk := New(baseDelay, capDelay, Exponential) 13 | v := bk.Backoff(0) 14 | if v != baseDelay { 15 | t.Errorf("want %v got %v", baseDelay, v) 16 | } 17 | 18 | for i := 1; i < 10; i++ { 19 | v = bk.Backoff(i) 20 | if v != capDelay { 21 | t.Errorf("want %v got %v", capDelay, v) 22 | } 23 | } 24 | 25 | v = bk.Backoff(0) 26 | if v != baseDelay { 27 | t.Errorf("want %v got %v", baseDelay, v) 28 | } 29 | } 30 | 31 | func TestFullJitter(t *testing.T) { 32 | baseDelay, capDelay := 1*time.Second, 2*time.Second 33 | bk := New(baseDelay, capDelay, FullJitter) 34 | result = bk.Backoff(0) 35 | 36 | for i := 1; i < 10; i++ { 37 | result = bk.Backoff(i) 38 | } 39 | } 40 | 41 | func TestEqualJitter(t *testing.T) { 42 | baseDelay, capDelay := 1*time.Second, 2*time.Second 43 | bk := New(baseDelay, capDelay, EqualJitter) 44 | result = bk.Backoff(0) 45 | 46 | for i := 1; i < 10; i++ { 47 | result = bk.Backoff(i) 48 | } 49 | } 50 | 51 | func TestDecorrelatedJitter(t *testing.T) { 52 | baseDelay, capDelay := 1*time.Second, 2*time.Second 53 | bk := New(baseDelay, capDelay, DecorrelatedJitter) 54 | result = bk.Backoff(0) 55 | 56 | for i := 1; i < 10; i++ { 57 | result = bk.Backoff(i) 58 | } 59 | } 60 | 61 | func TestMinDuration(t *testing.T) { 62 | testcases := []struct { 63 | x time.Duration 64 | y time.Duration 65 | want time.Duration 66 | }{ 67 | {1 * time.Second, 2 * time.Second, 1 * time.Second}, 68 | {1 * time.Second, 1 * time.Second, 1 * time.Second}, 69 | {1 * time.Millisecond, 2 * time.Millisecond, 1 * time.Millisecond}, 70 | {1 * time.Millisecond, 1 * time.Millisecond, 1 * time.Millisecond}, 71 | {1 * time.Minute, 1 * time.Hour, 1 * time.Minute}, 72 | } 73 | 74 | for _, c := range testcases { 75 | if r := minDuration(c.x, c.y); r != c.want { 76 | t.Errorf("want %v got %v", c.want, r) 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /pkg/drf/types.go: -------------------------------------------------------------------------------- 1 | package drf 2 | 3 | import ( 4 | "container/heap" 5 | "fmt" 6 | "sync" 7 | ) 8 | 9 | type ( 10 | // Typ represents resource type. 11 | Typ string 12 | 13 | // Node represents a Cluster Node. 14 | Node struct { 15 | index int 16 | allocated map[Typ]float64 17 | demand map[Typ]float64 18 | dShare float64 // Dominant Share 19 | mu sync.RWMutex 20 | } 21 | 22 | // A nodeQueue implements heap.Interface and holds Nodes. 23 | nodeQueue []*Node 24 | 25 | // DRF represents a DRF Cluster. 26 | DRF struct { 27 | clusterResource map[Typ]float64 28 | consumedResource map[Typ]float64 29 | nodes nodeQueue 30 | mu *sync.RWMutex 31 | } 32 | ) 33 | 34 | const ( 35 | // CPU resource. 36 | CPU = Typ("CPU") 37 | 38 | // MEMORY resource. 39 | MEMORY = Typ("MEMORY") 40 | ) 41 | 42 | var ( 43 | // ErrResourceSaturated when there is not enough resource to run next task. 44 | ErrResourceSaturated = fmt.Errorf("Fatal: resource has been saturated") 45 | 46 | // ErrEmptyResource when cluster resource is empty. 47 | ErrEmptyResource = fmt.Errorf("Fatal: empty cluster resource") 48 | 49 | // ErrEmptyNodes when cluster nodes is empty. 50 | ErrEmptyNodes = fmt.Errorf("Fatal: empty cluster nodes") 51 | 52 | // EmptyDRF is empty DRF. 53 | EmptyDRF = DRF{} 54 | ) 55 | 56 | func (nq nodeQueue) Len() int { return len(nq) } 57 | 58 | func (nq nodeQueue) Less(i, j int) bool { 59 | // Pop the lowest dShare 60 | return nq[i].dShare < nq[j].dShare 61 | } 62 | 63 | func (nq nodeQueue) Swap(i, j int) { 64 | nq[i], nq[j] = nq[j], nq[i] 65 | nq[i].index = i 66 | nq[j].index = j 67 | } 68 | 69 | func (nq *nodeQueue) Push(x interface{}) { 70 | n := len(*nq) 71 | item := x.(*Node) 72 | item.index = n 73 | *nq = append(*nq, item) 74 | } 75 | 76 | func (nq *nodeQueue) Pop() interface{} { 77 | old := *nq 78 | n := len(old) 79 | item := old[n-1] 80 | item.index = -1 // for safety 81 | *nq = old[0 : n-1] 82 | return item 83 | } 84 | 85 | // update modifies an Node in the queue. 86 | func (nq *nodeQueue) update(item *Node, dShare float64) { 87 | item.dShare = dShare 88 | heap.Fix(nq, item.index) 89 | } 90 | -------------------------------------------------------------------------------- /pkg/ratelimit/ratelimit_test.go: -------------------------------------------------------------------------------- 1 | package ratelimit 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | var ( 9 | t0 = time.Date(2017, time.August, 8, 8, 8, 8, 0, time.UTC) 10 | ) 11 | 12 | func TestEvery(t *testing.T) { 13 | defer func() { 14 | if r := recover(); r != nil { 15 | if r != "ratelimit: invalid time interval for Every" { 16 | t.Error("Every panic messgae error") 17 | } 18 | } 19 | }() 20 | 21 | cases := []struct { 22 | interval time.Duration 23 | l Limit 24 | }{ 25 | {1 * time.Second, Limit(1)}, 26 | {2 * time.Second, Limit(0.5)}, 27 | {4 * time.Second, Limit(0.25)}, 28 | {10 * time.Second, Limit(0.1)}, 29 | {1 * time.Millisecond, Limit(1e3)}, 30 | {10 * time.Millisecond, Limit(100)}, 31 | {-1 * time.Millisecond, Limit(-1)}, 32 | } 33 | for _, c := range cases { 34 | l := Every(c.interval) 35 | if l-c.l > 0.0001 { 36 | t.Errorf("Every(%v) = %v want %v", c.interval, l, c.l) 37 | } 38 | } 39 | } 40 | 41 | func TestAllow(t *testing.T) { 42 | t1 := t0.Add(100 * time.Millisecond) 43 | t2 := t0.Add(200 * time.Millisecond) 44 | cases := []struct { 45 | t time.Time 46 | n int 47 | ok bool 48 | }{ 49 | {t0, 1, true}, 50 | {t0, 1, false}, 51 | {t0, 1, false}, 52 | {t1, 1, true}, 53 | {t1, 1, false}, 54 | {t1, 1, false}, 55 | {t2, 2, false}, // exceeds burst 56 | {t2, 1, true}, 57 | {t2, 1, false}, 58 | } 59 | 60 | l := New(10, 1) 61 | for _, c := range cases { 62 | ok := l.AllowN(c.t, c.n) 63 | if ok != c.ok { 64 | t.Errorf("AllowN(%v, %v) = %v want %v", 65 | c.t, c.n, ok, c.ok) 66 | } 67 | } 68 | } 69 | 70 | func TestWait(t *testing.T) { 71 | cases := []struct { 72 | t time.Time 73 | n int 74 | nilErr bool 75 | }{ 76 | {t0, 1, true}, 77 | {t0, 2, false}, // exceeds burst 78 | {t0, 1, true}, 79 | {t0, 1, true}, 80 | } 81 | 82 | l := New(10, 1) 83 | for _, c := range cases { 84 | w, err := l.WaitN(c.t, c.n) 85 | delay := l.limit.durationFromTokens(float64(c.n)) 86 | if (c.nilErr && err != nil) || (!c.nilErr && err == nil) || w > delay { 87 | t.Errorf("WaitN(%v, %v) = %v, %v want %v, nilErr? %v", 88 | c.t, c.n, w, err, delay, c.nilErr) 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /pkg/base58/base58_test.go: -------------------------------------------------------------------------------- 1 | package base58_test 2 | 3 | import ( 4 | . "github.com/andy2046/gopie/pkg/base58" 5 | "math" 6 | "math/rand" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestEncode(t *testing.T) { 12 | testcases := map[uint64]string{ 13 | 0: "1", 14 | 32: "Z", 15 | 57: "z", 16 | math.MaxUint8: "5Q", 17 | math.MaxUint16: "LUv", 18 | math.MaxUint32: "7YXq9G", 19 | math.MaxUint64: "jpXCZedGfVQ", 20 | } 21 | for k, v := range testcases { 22 | r := Encode(k) 23 | if v != r { 24 | t.Errorf("expected %s got %s", v, r) 25 | } 26 | } 27 | } 28 | 29 | func TestDecode(t *testing.T) { 30 | _, err := Decode("") 31 | if err == nil { 32 | t.Fail() 33 | } 34 | _, err = Decode("0") 35 | if err == nil { 36 | t.Fail() 37 | } 38 | 39 | testcases := map[uint64]string{ 40 | 0: "1", 41 | 32: "Z", 42 | 57: "z", 43 | math.MaxUint8: "5Q", 44 | math.MaxUint16: "LUv", 45 | math.MaxUint32: "7YXq9G", 46 | math.MaxUint64: "jpXCZedGfVQ", 47 | } 48 | for k, v := range testcases { 49 | r, err := Decode(v) 50 | if err != nil { 51 | t.Fatal(err) 52 | } 53 | if k != r { 54 | t.Errorf("expected %d got %d", k, r) 55 | } 56 | } 57 | } 58 | 59 | func TestReverse(t *testing.T) { 60 | testcases := map[string]string{ 61 | "": "", 62 | "1": "1", 63 | "ABC": "CBA", 64 | "xyz": "zyx", 65 | } 66 | for k, v := range testcases { 67 | r := []byte(k) 68 | Reverse(r) 69 | if v != string(r) { 70 | t.Errorf("expected %s got %s", v, string(r)) 71 | } 72 | } 73 | } 74 | 75 | func BenchmarkEncode(b *testing.B) { 76 | s := rand.New(rand.NewSource(time.Now().UnixNano())) 77 | 78 | b.ReportAllocs() 79 | b.ResetTimer() 80 | for i := 0; i < b.N; i++ { 81 | Encode(uint64(s.Int63())) 82 | } 83 | } 84 | 85 | func BenchmarkDecode(b *testing.B) { 86 | arr := []string{"1", "Z", "z", "5Q", "LUv", "7YXq9G", "jpXCZedGfVQ"} 87 | ln := len(arr) 88 | s := rand.New(rand.NewSource(time.Now().UnixNano())) 89 | 90 | b.ReportAllocs() 91 | b.ResetTimer() 92 | for i := 0; i < b.N; i++ { 93 | _, err := Decode(arr[s.Intn(ln)]) 94 | if err != nil { 95 | b.Fatal(err) 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /docs/semaphore.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # semaphore 4 | `import "github.com/andy2046/gopie/pkg/semaphore"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package semaphore provides a semaphore implementation. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [type ISemaphore](#ISemaphore) 17 | * [func New(n int) ISemaphore](#New) 18 | * [type Semaphore](#Semaphore) 19 | * [func (s *Semaphore) Acquire(ctx context.Context) error](#Semaphore.Acquire) 20 | * [func (s *Semaphore) Release(ctx context.Context) error](#Semaphore.Release) 21 | 22 | 23 | #### Package files 24 | [semaphore.go](/src/github.com/andy2046/gopie/pkg/semaphore/semaphore.go) 25 | 26 | 27 | 28 | 29 | 30 | 31 | ## type [ISemaphore](/src/target/semaphore.go?s=242:333#L15) 32 | ``` go 33 | type ISemaphore interface { 34 | Acquire(context.Context) error 35 | Release(context.Context) error 36 | } 37 | ``` 38 | ISemaphore is the semaphore interface. 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | ### func [New](/src/target/semaphore.go?s=772:798#L42) 47 | ``` go 48 | func New(n int) ISemaphore 49 | ``` 50 | New creates a new semaphore with given maximum concurrent access. 51 | 52 | 53 | 54 | 55 | 56 | ## type [Semaphore](/src/target/semaphore.go?s=155:196#L10) 57 | ``` go 58 | type Semaphore struct { 59 | // contains filtered or unexported fields 60 | } 61 | ``` 62 | Semaphore is the semaphore implementation. 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | ### func (\*Semaphore) [Acquire](/src/target/semaphore.go?s=372:426#L22) 74 | ``` go 75 | func (s *Semaphore) Acquire(ctx context.Context) error 76 | ``` 77 | Acquire acquires the semaphore. 78 | 79 | 80 | 81 | 82 | ### func (\*Semaphore) [Release](/src/target/semaphore.go?s=559:613#L32) 83 | ``` go 84 | func (s *Semaphore) Release(ctx context.Context) error 85 | ``` 86 | Release releases the semaphore. 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | - - - 96 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 97 | -------------------------------------------------------------------------------- /docs/backoff.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # backoff 4 | `import "github.com/andy2046/gopie/pkg/backoff"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package backoff implements Backoff pattern. 11 | http://www.awsarchitectureblog.com/2015/03/backoff.html 12 | 13 | 14 | 15 | 16 | ## Index 17 | * [type Backoff](#Backoff) 18 | * [func New(baseDelay, capDelay time.Duration, strategy Strategy) *Backoff](#New) 19 | * [func (bk *Backoff) Backoff(retries int) time.Duration](#Backoff.Backoff) 20 | * [type Strategy](#Strategy) 21 | 22 | 23 | #### Package files 24 | [backoff.go](/src/github.com/andy2046/gopie/pkg/backoff/backoff.go) 25 | 26 | 27 | 28 | 29 | 30 | 31 | ## type [Backoff](/src/target/backoff.go?s=292:332#L18) 32 | ``` go 33 | type Backoff struct { 34 | // contains filtered or unexported fields 35 | } 36 | ``` 37 | Backoff holds the backoff logic. 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | ### func [New](/src/target/backoff.go?s=1474:1545#L75) 46 | ``` go 47 | func New(baseDelay, capDelay time.Duration, strategy Strategy) *Backoff 48 | ``` 49 | New returns a Backoff instance with provided delay and strategy. 50 | 51 | 52 | 53 | 54 | 55 | ### func (\*Backoff) [Backoff](/src/target/backoff.go?s=2114:2167#L104) 56 | ``` go 57 | func (bk *Backoff) Backoff(retries int) time.Duration 58 | ``` 59 | Backoff returns the amount of time to wait before the next retry. 60 | `retries` starts from zero. 61 | 62 | 63 | 64 | 65 | ## type [Strategy](/src/target/backoff.go?s=237:252#L15) 66 | ``` go 67 | type Strategy string 68 | ``` 69 | Strategy type. 70 | 71 | 72 | ``` go 73 | const ( 74 | Exponential Strategy = "Exponential" 75 | FullJitter Strategy = "FullJitter" 76 | EqualJitter Strategy = "EqualJitter" 77 | DecorrelatedJitter Strategy = "DecorrelatedJitter" 78 | ) 79 | ``` 80 | predefined Strategy types. 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | - - - 96 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 97 | -------------------------------------------------------------------------------- /docs/jumphash.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # jumphash 4 | `import "github.com/andy2046/gopie/pkg/jumphash"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package jumphash provides a jump consistent hash implementation. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [func Hash(key uint64, buckets int) int](#Hash) 17 | * [func HashString(key string, buckets int) int](#HashString) 18 | * [type Hasher](#Hasher) 19 | * [func New(n int) *Hasher](#New) 20 | * [func (h *Hasher) Hash(key string) int](#Hasher.Hash) 21 | * [func (h *Hasher) N() int](#Hasher.N) 22 | 23 | 24 | #### Package files 25 | [jumphash.go](/src/github.com/andy2046/gopie/pkg/jumphash/jumphash.go) 26 | 27 | 28 | 29 | 30 | 31 | ## func [Hash](/src/target/jumphash.go?s=427:465#L16) 32 | ``` go 33 | func Hash(key uint64, buckets int) int 34 | ``` 35 | Hash takes a key and the number of buckets, returns an integer in the range [0, buckets). 36 | If the number of buckets is not greater than 1 then 1 is used. 37 | 38 | 39 | 40 | ## func [HashString](/src/target/jumphash.go?s=775:819#L33) 41 | ``` go 42 | func HashString(key string, buckets int) int 43 | ``` 44 | HashString takes string as key instead of integer and uses CRC-64 to generate key. 45 | 46 | 47 | 48 | 49 | ## type [Hasher](/src/target/jumphash.go?s=1029:1060#L43) 50 | ``` go 51 | type Hasher struct { 52 | // contains filtered or unexported fields 53 | } 54 | ``` 55 | Hasher represents a jump consistent Hasher using a string as key. 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | ### func [New](/src/target/jumphash.go?s=1106:1129#L48) 64 | ``` go 65 | func New(n int) *Hasher 66 | ``` 67 | New returns a new instance of of Hasher. 68 | 69 | 70 | 71 | 72 | 73 | ### func (\*Hasher) [Hash](/src/target/jumphash.go?s=1391:1428#L61) 74 | ``` go 75 | func (h *Hasher) Hash(key string) int 76 | ``` 77 | Hash returns the integer hash for the given key. 78 | 79 | 80 | 81 | 82 | ### func (\*Hasher) [N](/src/target/jumphash.go?s=1292:1316#L56) 83 | ``` go 84 | func (h *Hasher) N() int 85 | ``` 86 | N returns the number of buckets the hasher can assign to. 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | - - - 96 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 97 | -------------------------------------------------------------------------------- /pkg/bitflag/bitflag_test.go: -------------------------------------------------------------------------------- 1 | package bitflag_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/andy2046/gopie/pkg/bitflag" 7 | ) 8 | 9 | const ( 10 | FLAG_A Flag = 1 << Flag(iota) 11 | FLAG_B 12 | FLAG_C 13 | FLAG_D 14 | ) 15 | 16 | func TestExample(t *testing.T) { 17 | 18 | var flag Flag 19 | 20 | flag.SetAll(FLAG_A) 21 | flag.SetAll(FLAG_B, FLAG_C) 22 | flag.SetAll(FLAG_C | FLAG_D) 23 | 24 | flag.Reset() 25 | 26 | flag.SetAll(FLAG_A, FLAG_B, FLAG_C, FLAG_D) 27 | 28 | flag.ClearAll(FLAG_A) 29 | 30 | flag.ClearAll(FLAG_B, FLAG_C) 31 | 32 | if flag.AreAllSet(FLAG_A) { 33 | t.Fatal("A") 34 | } 35 | 36 | if flag.AreAllSet(FLAG_B) { 37 | t.Fatal("B") 38 | } 39 | 40 | if flag.AreAllSet(FLAG_C) { 41 | t.Fatal("C") 42 | } 43 | 44 | if !flag.AreAllSet(FLAG_D) { 45 | t.Fatal("D") 46 | } 47 | } 48 | 49 | func TestSet(t *testing.T) { 50 | 51 | var flag Flag 52 | 53 | flag.SetAll(FLAG_A) 54 | flag.SetAll(FLAG_B) 55 | 56 | if !flag.AreAllSet(FLAG_A) { 57 | t.Fail() 58 | } 59 | 60 | if !flag.AreAllSet(FLAG_B) { 61 | t.Fail() 62 | } 63 | 64 | flag.SetAll(FLAG_A, FLAG_B) 65 | 66 | if !flag.AreAllSet(FLAG_A, FLAG_B) { 67 | t.Fail() 68 | } 69 | 70 | if !flag.IsAnySet(FLAG_B) { 71 | t.Fail() 72 | } 73 | } 74 | 75 | func TestClear(t *testing.T) { 76 | 77 | var flag Flag 78 | 79 | flag.SetAll(FLAG_A, FLAG_B, FLAG_C) 80 | 81 | if !flag.AreAllSet(FLAG_A, FLAG_B, FLAG_C) { 82 | t.Fail() 83 | } 84 | 85 | flag.ClearAll(FLAG_B) 86 | 87 | if flag.AreAllSet(FLAG_B) { 88 | t.Fail() 89 | } 90 | 91 | if !flag.AreAllSet(FLAG_A, FLAG_C) { 92 | t.Fail() 93 | } 94 | } 95 | 96 | func TestToggle(t *testing.T) { 97 | 98 | var flag Flag 99 | 100 | flag.SetAll(FLAG_A, FLAG_B, FLAG_C) 101 | 102 | if !flag.AreAllSet(FLAG_A, FLAG_B, FLAG_C) { 103 | t.Fail() 104 | } 105 | 106 | flag.ToggleAll(FLAG_B) 107 | 108 | if flag.AreAllSet(FLAG_B) { 109 | t.Fail() 110 | } 111 | 112 | if !flag.AreAllSet(FLAG_A, FLAG_C) { 113 | t.Fail() 114 | } 115 | } 116 | 117 | func TestN(t *testing.T) { 118 | 119 | var flag Flag 120 | 121 | flag.Set(1) 122 | 123 | if !flag.IsSet(1) { 124 | t.Fail() 125 | } 126 | 127 | flag.Toggle(1) 128 | 129 | if flag.IsSet(1) { 130 | t.Fail() 131 | } 132 | 133 | flag.Toggle(1) 134 | flag.Clear(1) 135 | 136 | if flag.IsSet(1) { 137 | t.Fail() 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /pkg/skiplist/skiplist_test.go: -------------------------------------------------------------------------------- 1 | package skiplist 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "sync" 7 | "testing" 8 | "unsafe" 9 | ) 10 | 11 | var testList *SkipList 12 | 13 | func init() { 14 | testList = New() 15 | 16 | for i := 0; i <= 10000000; i++ { 17 | testList.Set(strconv.Itoa(i), int64(i)) 18 | } 19 | 20 | var sl SkipList 21 | var el Element 22 | fmt.Printf("Sizeof(SkipList) = %v bytes Sizeof(Element) = %v bytes\n", unsafe.Sizeof(sl), unsafe.Sizeof(el)) 23 | fmt.Printf("Alignof(SkipList) = %v bytes Alignof(Element) = %v bytes\n", unsafe.Alignof(&sl), unsafe.Alignof(el)) 24 | } 25 | 26 | func TestGetSet(t *testing.T) { 27 | list := New() 28 | n := 1000000 29 | wg := &sync.WaitGroup{} 30 | wg.Add(2) 31 | 32 | go func() { 33 | for i := 0; i < n; i++ { 34 | list.Set(strconv.Itoa(i), int64(i)) 35 | } 36 | wg.Done() 37 | }() 38 | 39 | go func() { 40 | for i := 0; i < n; i++ { 41 | list.Get(strconv.Itoa(i)) 42 | } 43 | wg.Done() 44 | }() 45 | 46 | wg.Wait() 47 | if list.Len() != n { 48 | t.Fail() 49 | } 50 | } 51 | 52 | func TestCRUD(t *testing.T) { 53 | list := New() 54 | list.Set("A", 1) 55 | list.Set("B", 2) 56 | list.Set("C", 3) 57 | list.Set("D", 4) 58 | list.Set("E", 5) 59 | list.Set("C", 9) 60 | list.Remove("X") 61 | list.Remove("D") 62 | 63 | a := list.Get("A") 64 | b := list.Get("B") 65 | c := list.Get("C") 66 | d := list.Get("D") 67 | e := list.Get("E") 68 | x := list.Get("X") 69 | 70 | if a == nil || a.Key() != "A" || a.Value() != 1 { 71 | t.Fatal("wrong value", a) 72 | } 73 | if b == nil || b.Key() != "B" || b.Value() != 2 { 74 | t.Fatal("wrong value", b) 75 | } 76 | if c == nil || c.Key() != "C" || c.Value() != 9 { 77 | t.Fatal("wrong value", c) 78 | } 79 | if d != nil { 80 | t.Fatal("wrong value", d) 81 | } 82 | if e == nil || e.Key() != "E" || e.Value() != 5 { 83 | t.Fatal("wrong value", e) 84 | } 85 | if x != nil { 86 | t.Fatal("wrong value", x) 87 | } 88 | 89 | } 90 | 91 | func BenchmarkSet(b *testing.B) { 92 | b.ReportAllocs() 93 | list := New() 94 | 95 | for i := 0; i < b.N; i++ { 96 | list.Set(strconv.Itoa(i), int64(i)) 97 | } 98 | } 99 | 100 | func BenchmarkGet(b *testing.B) { 101 | b.ReportAllocs() 102 | 103 | for i := 0; i < b.N; i++ { 104 | e := testList.Get(strconv.Itoa(i)) 105 | if e == nil { 106 | b.Fatal("fail to Get") 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /docs/spinlock.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # spinlock 4 | `import "github.com/andy2046/gopie/pkg/spinlock"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package spinlock implements Spinlock. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [type Locker](#Locker) 17 | * [func New() *Locker](#New) 18 | * [func (l *Locker) IsLocked() bool](#Locker.IsLocked) 19 | * [func (l *Locker) Lock()](#Locker.Lock) 20 | * [func (l *Locker) Locker() sync.Locker](#Locker.Locker) 21 | * [func (l *Locker) TryLock() bool](#Locker.TryLock) 22 | * [func (l *Locker) Unlock()](#Locker.Unlock) 23 | 24 | 25 | #### Package files 26 | [spinlock.go](/src/github.com/andy2046/gopie/pkg/spinlock/spinlock.go) 27 | 28 | 29 | 30 | 31 | 32 | 33 | ## type [Locker](/src/target/spinlock.go?s=147:200#L11) 34 | ``` go 35 | type Locker struct { 36 | // contains filtered or unexported fields 37 | } 38 | ``` 39 | Locker is the Spinlock implementation. 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | ### func [New](/src/target/spinlock.go?s=227:245#L17) 48 | ``` go 49 | func New() *Locker 50 | ``` 51 | New creates a Locker. 52 | 53 | 54 | 55 | 56 | 57 | ### func (\*Locker) [IsLocked](/src/target/spinlock.go?s=747:779#L40) 58 | ``` go 59 | func (l *Locker) IsLocked() bool 60 | ``` 61 | IsLocked returns true if locked, false otherwise. 62 | 63 | 64 | 65 | 66 | ### func (\*Locker) [Lock](/src/target/spinlock.go?s=317:340#L22) 67 | ``` go 68 | func (l *Locker) Lock() 69 | ``` 70 | Lock wait in a loop to acquire the spinlock. 71 | 72 | 73 | 74 | 75 | ### func (\*Locker) [Locker](/src/target/spinlock.go?s=884:921#L45) 76 | ``` go 77 | func (l *Locker) Locker() sync.Locker 78 | ``` 79 | Locker returns a sync.Locker interface implementation. 80 | 81 | 82 | 83 | 84 | ### func (\*Locker) [TryLock](/src/target/spinlock.go?s=605:636#L35) 85 | ``` go 86 | func (l *Locker) TryLock() bool 87 | ``` 88 | TryLock try to acquire the spinlock, 89 | it returns true if succeed, false otherwise. 90 | 91 | 92 | 93 | 94 | ### func (\*Locker) [Unlock](/src/target/spinlock.go?s=453:478#L29) 95 | ``` go 96 | func (l *Locker) Unlock() 97 | ``` 98 | Unlock release the spinlock. 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | - - - 108 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 109 | -------------------------------------------------------------------------------- /pkg/lru/lru_test.go: -------------------------------------------------------------------------------- 1 | package lru 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | type simpleStruct struct { 9 | int 10 | string 11 | } 12 | 13 | type complexStruct struct { 14 | int 15 | simpleStruct 16 | } 17 | 18 | var getTests = []struct { 19 | name string 20 | keyToAdd interface{} 21 | keyToGet interface{} 22 | expectedOk bool 23 | }{ 24 | {"string_hit", "myKey", "myKey", true}, 25 | {"string_miss", "myKey", "nonsense", false}, 26 | {"simple_struct_hit", simpleStruct{1, "two"}, simpleStruct{1, "two"}, true}, 27 | {"simeple_struct_miss", simpleStruct{1, "two"}, simpleStruct{0, "noway"}, false}, 28 | {"complex_struct_hit", complexStruct{1, simpleStruct{2, "three"}}, 29 | complexStruct{1, simpleStruct{2, "three"}}, true}, 30 | } 31 | 32 | func TestGet(t *testing.T) { 33 | for _, tt := range getTests { 34 | lru := New(0) 35 | lru.Add(tt.keyToAdd, 1234) 36 | val, ok := lru.Get(tt.keyToGet) 37 | if ok != tt.expectedOk { 38 | t.Fatalf("%s: cache hit = %v; want %v", tt.name, ok, !ok) 39 | } else if ok && val != 1234 { 40 | t.Fatalf("%s expected get to return 1234 but got %v", tt.name, val) 41 | } 42 | } 43 | } 44 | 45 | func TestRemove(t *testing.T) { 46 | lru := New(0) 47 | lru.Add("myKey", 1234) 48 | if val, ok := lru.Get("myKey"); !ok { 49 | t.Fatal("TestRemove returned no match") 50 | } else if val != 1234 { 51 | t.Fatalf("TestRemove failed. Expected %d, got %v", 1234, val) 52 | } 53 | 54 | lru.Remove("myKey") 55 | if _, ok := lru.Get("myKey"); ok { 56 | t.Fatal("TestRemove returned a removed entry") 57 | } 58 | } 59 | 60 | func TestPurge(t *testing.T) { 61 | purgedKeys := make([]interface{}, 0) 62 | onPurgedFun := func(key interface{}, value interface{}) { 63 | purgedKeys = append(purgedKeys, key) 64 | } 65 | 66 | lru := New(20) 67 | lru.OnPurged = onPurgedFun 68 | for i := 0; i < 22; i++ { 69 | lru.Add(fmt.Sprintf("myKey%d", i), 1234) 70 | } 71 | 72 | if len(purgedKeys) != 2 { 73 | t.Fatalf("got %d evicted keys; want 2", len(purgedKeys)) 74 | } 75 | if purgedKeys[0] != interface{}("myKey0") { 76 | t.Fatalf("got %v in first evicted key; want %s", purgedKeys[0], "myKey0") 77 | } 78 | if purgedKeys[1] != interface{}("myKey1") { 79 | t.Fatalf("got %v in second evicted key; want %s", purgedKeys[1], "myKey1") 80 | } 81 | } 82 | 83 | func BenchmarkLRU(b *testing.B) { 84 | purgedKeys := make([]interface{}, 0) 85 | onPurgedFun := func(key interface{}, value interface{}) { 86 | purgedKeys = append(purgedKeys, key) 87 | } 88 | n, m := 20, 40 89 | lru := New(n) 90 | lru.OnPurged = onPurgedFun 91 | b.ResetTimer() 92 | for i := 0; i < b.N; i++ { 93 | lru.Add(fmt.Sprintf("myKey%d", i%m), 1234) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /pkg/pubsub/pubsub_test.go: -------------------------------------------------------------------------------- 1 | package pubsub 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | ) 8 | 9 | func TestNewTopic(t *testing.T) { 10 | psName := "pubsub-01" 11 | psNameExpected := "projects/" + psName 12 | ps := New(psName) 13 | if ps.Name() != psNameExpected { 14 | t.Error("wrong PubSub name, expected", psNameExpected, ", got", ps.Name()) 15 | } 16 | 17 | topic1, err := ps.NewTopic("topic-01", 10, 2) 18 | if err != nil { 19 | t.Error("NewTopic error", err) 20 | } 21 | 22 | sub11, _ := topic1.NewSubscription(2) 23 | sub11.Receive(func(m *Message) { 24 | t.Logf("sub11 Message -> %#v", *m) 25 | }) 26 | sub12, _ := topic1.NewSubscription(2) 27 | sub12.Receive(func(m *Message) { 28 | t.Logf("sub12 Message -> %#v", *m) 29 | }) 30 | 31 | err1 := topic1.Publish(context.Background(), &Message{ 32 | ID: "001", 33 | Data: []byte("data"), 34 | }) 35 | if err1 != nil { 36 | t.Logf("topic1.Publish error -> %#v", err1) 37 | } 38 | 39 | topic1.Delete() 40 | 41 | topic2, err := ps.NewTopic("topic-02", 10, 2) 42 | if err != nil { 43 | t.Error("NewTopic error", err) 44 | } 45 | 46 | sub2, _ := topic2.NewSubscription(2) 47 | sub2.Receive(func(m *Message) { 48 | t.Logf("sub2 Message -> %#v", *m) 49 | }) 50 | 51 | err2 := topic2.Publish(context.Background(), &Message{ 52 | ID: "002", 53 | Data: []byte("data"), 54 | }) 55 | if err2 != nil { 56 | t.Logf("topic2.Publish error -> %#v", err2) 57 | } 58 | 59 | t.Logf("Topics -> %#v", ps.Topics()) 60 | if fmt.Sprintf("%v", ps.Topics()) != fmt.Sprintf("%v", []string{"topic-02"}) { 61 | t.Error("Topics should return topic name") 62 | } 63 | 64 | topic2.Delete() 65 | } 66 | 67 | func TestNewSubscription(t *testing.T) { 68 | psName := "pubsub-02" 69 | psNameExpected := "projects/" + psName 70 | ps := New(psName) 71 | if ps.Name() != psNameExpected { 72 | t.Error("wrong PubSub name, expected", psNameExpected, ", got", ps.Name()) 73 | } 74 | 75 | topic1, err := ps.NewTopic("topic-01", 10, 2) 76 | if err != nil { 77 | t.Error("NewTopic error", err) 78 | } 79 | 80 | sub11, _ := topic1.NewSubscription(2) 81 | sub11.Receive(func(m *Message) { 82 | t.Logf("sub11 Message -> %#v", *m) 83 | }) 84 | sub12, _ := topic1.NewSubscription(2) 85 | sub12.Receive(func(m *Message) { 86 | t.Logf("sub12 Message -> %#v", *m) 87 | }) 88 | sub13, _ := topic1.NewSubscription(2) 89 | sub13.Receive(func(m *Message) { 90 | t.Logf("sub13 Message -> %#v", *m) 91 | }) 92 | 93 | err1 := topic1.Publish(context.Background(), &Message{ 94 | ID: "001", 95 | Data: []byte("data"), 96 | }) 97 | if err1 != nil { 98 | t.Logf("topic1.Publish error -> %#v", err1) 99 | } 100 | 101 | sub13.Delete() 102 | topic1.Delete() 103 | } 104 | -------------------------------------------------------------------------------- /pkg/sequence/memflake.go: -------------------------------------------------------------------------------- 1 | package sequence 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | // MemFlake is an implementation of in-memory Iceflake. 10 | type MemFlake struct { 11 | sync.Mutex 12 | startTime time.Time 13 | bitLenSequence uint8 14 | elapsedTime uint64 15 | startEpoch int64 16 | machineID uint64 17 | } 18 | 19 | var ( 20 | errLessThanOne = errors.New("`n` should not be less than 1") 21 | errTimeDrift = errors.New("time drifts too much") 22 | _ Iceflake = &MemFlake{} 23 | ) 24 | 25 | // NewMemFlake creates a MemFlake. 26 | func NewMemFlake(startTime time.Time, bitLenSequence uint8, machineID uint64) *MemFlake { 27 | if startTime.After(time.Now()) { 28 | // no future time 29 | return nil 30 | } 31 | if startTime.IsZero() { 32 | startTime = time.Date(2019, 10, 9, 0, 0, 0, 0, time.UTC) 33 | } 34 | startEpoch := toIceflakeTime(startTime) 35 | if startEpoch < 0 { 36 | // should not be too far away from now 37 | // should be after 1970/1/1 38 | return nil 39 | } 40 | if bitLenSequence < 10 || bitLenSequence > 21 { 41 | // keep control 42 | bitLenSequence = 18 43 | } 44 | 45 | return &MemFlake{ 46 | startTime: startTime, 47 | bitLenSequence: bitLenSequence, 48 | startEpoch: startEpoch, 49 | machineID: machineID, 50 | } 51 | } 52 | 53 | // Next ... 54 | func (m *MemFlake) Next() (uint64, error) { 55 | return m.NextN(1) 56 | } 57 | 58 | // NextN ... 59 | func (m *MemFlake) NextN(n int) (uint64, error) { 60 | if n < 1 { 61 | return 0, errLessThanOne 62 | } 63 | 64 | current := currentElapsedTime(m.startEpoch) 65 | if current < 0 { 66 | return 0, errTimeDrift 67 | } 68 | 69 | nextTime, un := uint64(m.toID(current)), uint64(n) 70 | m.Lock() 71 | defer m.Unlock() 72 | // [elapsedTime - n + 1, elapsedTime] will be returned 73 | if m.elapsedTime < nextTime { 74 | m.elapsedTime = nextTime + un - 1 75 | } else { 76 | m.elapsedTime += un 77 | } 78 | // return the first available timestamp 79 | return m.elapsedTime - un + 1, nil 80 | } 81 | 82 | // MachineID ... 83 | func (m *MemFlake) MachineID() uint64 { 84 | return m.machineID 85 | } 86 | 87 | // StartTime ... 88 | func (m *MemFlake) StartTime() time.Time { 89 | return m.startTime 90 | } 91 | 92 | // BitLenSequence ... 93 | func (m *MemFlake) BitLenSequence() uint8 { 94 | return m.bitLenSequence 95 | } 96 | 97 | func (m *MemFlake) toID(t int64) int64 { 98 | return t << m.bitLenSequence 99 | } 100 | 101 | func toIceflakeTime(t time.Time) int64 { 102 | return t.UTC().Unix() * 1000 // in milliseconds 103 | } 104 | 105 | func currentElapsedTime(startEpoch int64) int64 { 106 | return toIceflakeTime(time.Now()) - startEpoch 107 | } 108 | -------------------------------------------------------------------------------- /pkg/barrier/barrier.go: -------------------------------------------------------------------------------- 1 | // Package barrier provides a barrier implementation. 2 | package barrier 3 | 4 | import ( 5 | "context" 6 | "errors" 7 | "sync" 8 | ) 9 | 10 | // Barrier is a synchronizer that allows members to wait for each other. 11 | type Barrier struct { 12 | count int 13 | n int 14 | isBroken bool 15 | waitChan chan struct{} 16 | fallChan chan struct{} 17 | brokenChan chan struct{} 18 | mu sync.RWMutex 19 | } 20 | 21 | var ( 22 | // ErrBroken when barrier is broken. 23 | ErrBroken = errors.New("barrier is broken") 24 | ) 25 | 26 | // New returns a new barrier. 27 | func New(n int) *Barrier { 28 | if n <= 0 { 29 | panic("number of members must be positive int") 30 | } 31 | b := &Barrier{ 32 | n: n, 33 | waitChan: make(chan struct{}), 34 | brokenChan: make(chan struct{}), 35 | } 36 | return b 37 | } 38 | 39 | // Await waits until all members have called await on the barrier. 40 | func (b *Barrier) Await(ctx context.Context) error { 41 | select { 42 | case <-ctx.Done(): 43 | return ctx.Err() 44 | default: 45 | } 46 | 47 | b.mu.Lock() 48 | 49 | if b.isBroken { 50 | b.mu.Unlock() 51 | return ErrBroken 52 | } 53 | 54 | b.count++ 55 | waitChan := b.waitChan 56 | brokenChan := b.brokenChan 57 | count := b.count 58 | 59 | if count == b.n { 60 | b.reset(true) 61 | b.mu.Unlock() 62 | return nil 63 | } 64 | 65 | b.mu.Unlock() 66 | 67 | select { 68 | case <-waitChan: 69 | return nil 70 | case <-brokenChan: 71 | return ErrBroken 72 | case <-ctx.Done(): 73 | b.broke(true) 74 | return ctx.Err() 75 | } 76 | } 77 | 78 | func (b *Barrier) broke(toLock bool) { 79 | if toLock { 80 | b.mu.Lock() 81 | defer b.mu.Unlock() 82 | } 83 | 84 | if !b.isBroken { 85 | b.isBroken = true 86 | close(b.brokenChan) 87 | } 88 | } 89 | 90 | func (b *Barrier) reset(ok bool) { 91 | if ok { 92 | close(b.waitChan) 93 | } else if b.count > 0 { 94 | b.broke(false) 95 | } 96 | 97 | b.waitChan = make(chan struct{}) 98 | b.brokenChan = make(chan struct{}) 99 | b.count = 0 100 | b.isBroken = false 101 | } 102 | 103 | // Reset resets the barrier to initial state. 104 | func (b *Barrier) Reset() { 105 | b.mu.Lock() 106 | defer b.mu.Unlock() 107 | b.reset(false) 108 | } 109 | 110 | // N returns the number of members for the barrier. 111 | func (b *Barrier) N() int { 112 | return b.n 113 | } 114 | 115 | // NWaiting returns the number of members currently waiting at the barrier. 116 | func (b *Barrier) NWaiting() int { 117 | b.mu.RLock() 118 | defer b.mu.RUnlock() 119 | return b.count 120 | } 121 | 122 | // IsBroken returns true if the barrier is broken. 123 | func (b *Barrier) IsBroken() bool { 124 | b.mu.RLock() 125 | defer b.mu.RUnlock() 126 | return b.isBroken 127 | } 128 | -------------------------------------------------------------------------------- /pkg/batch/batch.go: -------------------------------------------------------------------------------- 1 | // Package batch implements batch utility. 2 | package batch 3 | 4 | import ( 5 | "log" 6 | "math" 7 | "runtime" 8 | "sync" 9 | "sync/atomic" 10 | "time" 11 | ) 12 | 13 | // Batch process batch of work by time duration or batch count. 14 | type Batch struct { 15 | timeout time.Duration 16 | lock sync.Mutex 17 | once sync.Once 18 | fn func([]interface{}) 19 | count uint32 20 | temp uint32 21 | input chan interface{} 22 | notify chan struct{} 23 | closer chan struct{} 24 | } 25 | 26 | // New returns a Batch with time duration, batch count and batch function provided. 27 | func New(timeout time.Duration, count int, fn func([]interface{})) *Batch { 28 | if count < 1 || count > math.MaxUint32 { 29 | panic("invalid batch count") 30 | } 31 | 32 | b := &Batch{ 33 | timeout: timeout, 34 | fn: fn, 35 | count: uint32(count), 36 | notify: make(chan struct{}), 37 | closer: make(chan struct{}), 38 | input: make(chan interface{}, count), 39 | } 40 | 41 | go b.batch() 42 | return b 43 | } 44 | 45 | // Batch batches the given item. 46 | func (b *Batch) Batch(item interface{}) { 47 | b.input <- item 48 | 49 | if atomic.AddUint32(&b.temp, 1) == b.count { 50 | atomic.StoreUint32(&b.temp, 0) 51 | b.notify <- struct{}{} 52 | } 53 | } 54 | 55 | // Close the batch. 56 | func (b *Batch) Close() { 57 | b.once.Do(func() { 58 | close(b.closer) 59 | }) 60 | } 61 | 62 | func (b *Batch) batch() { 63 | var ( 64 | batch []interface{} 65 | ch = make(chan struct{}) 66 | h = func() { 67 | d := append(batch[:0:0], batch...) 68 | batch = batch[:0] 69 | go func() { 70 | defer func() { 71 | if r := recover(); r != nil { 72 | var buf [4096]byte 73 | n := runtime.Stack(buf[:], false) 74 | log.Println(r) 75 | log.Println(string(buf[:n])) 76 | } 77 | }() 78 | 79 | b.fn(d) 80 | }() 81 | 82 | go b.counter(ch) 83 | } 84 | ) 85 | 86 | go b.counter(ch) 87 | 88 | for { 89 | select { 90 | case <-b.closer: 91 | return 92 | case <-ch: 93 | h() 94 | break 95 | default: 96 | } 97 | 98 | select { 99 | case <-b.closer: 100 | return 101 | case <-ch: 102 | h() 103 | break 104 | case item := <-b.input: 105 | batch = append(batch, item) 106 | continue 107 | } 108 | } 109 | } 110 | 111 | func (b *Batch) counter(ch chan struct{}) { 112 | timer := time.NewTimer(b.timeout) 113 | 114 | select { 115 | case <-b.closer: 116 | timer.Stop() 117 | return 118 | case <-timer.C: 119 | break 120 | case <-b.notify: 121 | break 122 | } 123 | 124 | if !timer.Stop() { 125 | select { 126 | case <-timer.C: 127 | default: 128 | } 129 | } 130 | 131 | select { 132 | case <-b.notify: 133 | default: 134 | } 135 | 136 | ch <- struct{}{} 137 | } 138 | -------------------------------------------------------------------------------- /pkg/fileutil/fileutil.go: -------------------------------------------------------------------------------- 1 | // Package fileutil implements some file utils. 2 | package fileutil 3 | 4 | import ( 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "sort" 10 | ) 11 | 12 | // ReadDirOperation represents read-directory operation. 13 | type ReadDirOperation struct { 14 | ext string 15 | } 16 | 17 | const ( 18 | // PrivateFileMode represents file read/write mode to owner. 19 | PrivateFileMode = 0600 20 | // PrivateDirMode represents file make/remove mode in the directory to owner. 21 | PrivateDirMode = 0700 22 | ) 23 | 24 | // WithExt filters file names by extension. 25 | func WithExt(ext string) func(*ReadDirOperation) { 26 | return func(op *ReadDirOperation) { op.ext = ext } 27 | } 28 | 29 | func (op *ReadDirOperation) applyOpts(opts []func(*ReadDirOperation)) { 30 | for _, opt := range opts { 31 | opt(op) 32 | } 33 | } 34 | 35 | // ReadDir returns the file names in the provided directory in order. 36 | func ReadDir(d string, opts ...func(*ReadDirOperation)) ([]string, error) { 37 | var err error 38 | op := &ReadDirOperation{} 39 | op.applyOpts(opts) 40 | 41 | dir, err := os.Open(d) 42 | if err != nil { 43 | return nil, err 44 | } 45 | defer dir.Close() 46 | 47 | names, err := dir.Readdirnames(-1) 48 | if err != nil { 49 | return nil, err 50 | } 51 | sort.Strings(names) 52 | 53 | if op.ext != "" { 54 | temp := make([]string, 0) 55 | for _, v := range names { 56 | if filepath.Ext(v) == op.ext { 57 | temp = append(temp, v) 58 | } 59 | } 60 | names = temp 61 | } 62 | return names, nil 63 | } 64 | 65 | // IsDirWriteable checks if dir is writable by writing and removing a file to dir. 66 | func IsDirWriteable(dir string) error { 67 | f := filepath.Join(dir, ".touch") 68 | if err := ioutil.WriteFile(f, []byte(""), PrivateFileMode); err != nil { 69 | return err 70 | } 71 | return os.Remove(f) 72 | } 73 | 74 | // TouchDirAll creates directories with 0700 permission if any directory 75 | // does not exist and ensures the provided directory is writable. 76 | func TouchDirAll(dir string) error { 77 | err := os.MkdirAll(dir, PrivateDirMode) 78 | if err != nil { 79 | return err 80 | } 81 | return IsDirWriteable(dir) 82 | } 83 | 84 | // CreateDirAll wraps TouchDirAll but returns error 85 | // if the deepest directory is not empty. 86 | func CreateDirAll(dir string) error { 87 | err := TouchDirAll(dir) 88 | if err == nil { 89 | var ns []string 90 | if ns, err = ReadDir(dir); err != nil { 91 | return err 92 | } 93 | if len(ns) != 0 { 94 | err = fmt.Errorf("expected %q to be empty, got %q", dir, ns) 95 | } 96 | } 97 | return err 98 | } 99 | 100 | // Exist returns true if a file or directory exists. 101 | func Exist(name string) bool { 102 | if _, err := os.Stat(name); os.IsNotExist(err) { 103 | return false 104 | } 105 | return true 106 | } 107 | -------------------------------------------------------------------------------- /pkg/bloom/bloombit.go: -------------------------------------------------------------------------------- 1 | package bloom 2 | 3 | import ( 4 | "math" 5 | 6 | "github.com/andy2046/bitmap" 7 | ) 8 | 9 | type ( 10 | bloomFilterBit struct { 11 | bitmap *bitmap.Bitmap // bloom filter bitmap 12 | k uint64 // number of hash functions 13 | n uint64 // number of elements in the bloom filter 14 | m uint64 // size of the bloom filter bits 15 | shift uint8 // the shift to get high/low bit fragments 16 | } 17 | ) 18 | 19 | // NewB creates standard bloom filter based on the provided m/k. 20 | // m is the size of bloom filter bits. 21 | // k is the number of hash functions. 22 | func NewB(m, k uint64) Bloom { 23 | mm, exponent := adjustM(m) 24 | return &bloomFilterBit{ 25 | bitmap: bitmap.New(mm), 26 | m: mm - 1, // x % 2^i = x & (2^i - 1) 27 | k: k, 28 | shift: 64 - exponent, 29 | } 30 | } 31 | 32 | // NewBGuess estimates m/k based on the provided n/p then creates standard bloom filter. 33 | // n is the estimated number of elements in the bloom filter. 34 | // p is the false positive probability. 35 | func NewBGuess(n uint64, p float64) Bloom { 36 | m, k := Guess(n, p) 37 | return NewB(m, k) 38 | } 39 | 40 | func (bf *bloomFilterBit) Add(entry []byte) { 41 | hash := sipHash(entry) 42 | h := hash >> bf.shift 43 | l := hash << bf.shift >> bf.shift 44 | for i := uint64(0); i < bf.k; i++ { 45 | bf.bitmap.SetBit((h+i*l)&bf.m, true) 46 | } 47 | bf.n++ 48 | } 49 | 50 | func (bf *bloomFilterBit) AddString(entry string) { 51 | bf.Add([]byte(entry)) 52 | } 53 | 54 | func (bf *bloomFilterBit) Exist(entry []byte) bool { 55 | hash := sipHash(entry) 56 | h := hash >> bf.shift 57 | l := hash << bf.shift >> bf.shift 58 | 59 | for i := uint64(0); i < bf.k; i++ { 60 | if !bf.bitmap.GetBit((h + i*l) & bf.m) { 61 | return false 62 | } 63 | } 64 | 65 | return true 66 | } 67 | 68 | func (bf *bloomFilterBit) ExistString(entry string) bool { 69 | return bf.Exist([]byte(entry)) 70 | } 71 | 72 | func (bf *bloomFilterBit) FalsePositive() float64 { 73 | return math.Pow((1 - math.Exp(-float64(bf.k*bf.n)/float64(bf.m))), 74 | float64(bf.k)) 75 | } 76 | 77 | func (bf *bloomFilterBit) GuessFalsePositive(n uint64) float64 { 78 | return math.Pow((1 - math.Exp(-float64(bf.k*n)/float64(bf.m))), 79 | float64(bf.k)) 80 | } 81 | 82 | func (bf *bloomFilterBit) M() uint64 { 83 | return bf.m + 1 84 | } 85 | 86 | func (bf *bloomFilterBit) K() uint64 { 87 | return bf.k 88 | } 89 | 90 | func (bf *bloomFilterBit) N() uint64 { 91 | return bf.n 92 | } 93 | 94 | func (bf *bloomFilterBit) Clear() { 95 | s := bf.bitmap.Size() 96 | for i := uint64(0); i < s; i++ { 97 | bf.bitmap.SetBit(i, false) 98 | } 99 | bf.n = 0 100 | } 101 | 102 | func (bf *bloomFilterBit) estimatedFillRatio() float64 { 103 | return 1 - math.Exp(-float64(bf.n)/math.Ceil(float64(bf.m)/float64(bf.k))) 104 | } 105 | -------------------------------------------------------------------------------- /docs/barrier.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # barrier 4 | `import "github.com/andy2046/gopie/pkg/barrier"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package barrier provides a barrier implementation. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [Variables](#pkg-variables) 17 | * [type Barrier](#Barrier) 18 | * [func New(n int) *Barrier](#New) 19 | * [func (b *Barrier) Await(ctx context.Context) error](#Barrier.Await) 20 | * [func (b *Barrier) IsBroken() bool](#Barrier.IsBroken) 21 | * [func (b *Barrier) N() int](#Barrier.N) 22 | * [func (b *Barrier) NWaiting() int](#Barrier.NWaiting) 23 | * [func (b *Barrier) Reset()](#Barrier.Reset) 24 | 25 | 26 | #### Package files 27 | [barrier.go](/src/github.com/andy2046/gopie/pkg/barrier/barrier.go) 28 | 29 | 30 | 31 | ## Variables 32 | ``` go 33 | var ( 34 | // ErrBroken when barrier is broken. 35 | ErrBroken = errors.New("barrier is broken") 36 | ) 37 | ``` 38 | 39 | 40 | 41 | ## type [Barrier](/src/target/barrier.go?s=185:360#L11) 42 | ``` go 43 | type Barrier struct { 44 | // contains filtered or unexported fields 45 | } 46 | ``` 47 | Barrier is a synchronizer that allows members to wait for each other. 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | ### func [New](/src/target/barrier.go?s=484:508#L27) 56 | ``` go 57 | func New(n int) *Barrier 58 | ``` 59 | New returns a new barrier. 60 | 61 | 62 | 63 | 64 | 65 | ### func (\*Barrier) [Await](/src/target/barrier.go?s=763:813#L40) 66 | ``` go 67 | func (b *Barrier) Await(ctx context.Context) error 68 | ``` 69 | Await waits until all members have called await on the barrier. 70 | 71 | 72 | 73 | 74 | ### func (\*Barrier) [IsBroken](/src/target/barrier.go?s=2069:2102#L123) 75 | ``` go 76 | func (b *Barrier) IsBroken() bool 77 | ``` 78 | IsBroken returns true if the barrier is broken. 79 | 80 | 81 | 82 | 83 | ### func (\*Barrier) [N](/src/target/barrier.go?s=1809:1834#L111) 84 | ``` go 85 | func (b *Barrier) N() int 86 | ``` 87 | N returns the number of members for the barrier. 88 | 89 | 90 | 91 | 92 | ### func (\*Barrier) [NWaiting](/src/target/barrier.go?s=1928:1960#L116) 93 | ``` go 94 | func (b *Barrier) NWaiting() int 95 | ``` 96 | NWaiting returns the number of members currently waiting at the barrier. 97 | 98 | 99 | 100 | 101 | ### func (\*Barrier) [Reset](/src/target/barrier.go?s=1676:1701#L104) 102 | ``` go 103 | func (b *Barrier) Reset() 104 | ``` 105 | Reset resets the barrier to initial state. 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | - - - 115 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 116 | -------------------------------------------------------------------------------- /pkg/bloom/bloombit_test.go: -------------------------------------------------------------------------------- 1 | package bloom_test 2 | 3 | import ( 4 | "encoding/binary" 5 | "testing" 6 | 7 | "github.com/andy2046/gopie/pkg/bloom" 8 | ) 9 | 10 | func TestBitBasic(t *testing.T) { 11 | f := bloom.NewB(1000, 4) 12 | e1 := []byte("Boss") 13 | e2 := []byte("Joke") 14 | e3 := []byte("Emotion") 15 | f.Add(e1) 16 | e3b := f.Exist(e3) 17 | e1a := f.Exist(e1) 18 | e2a := f.Exist(e2) 19 | f.Add(e3) 20 | e3a := f.Exist(e3) 21 | if !e1a { 22 | t.Errorf("%q should Exist.", e1) 23 | } 24 | if e2a { 25 | t.Errorf("%q should not Exist.", e2) 26 | } 27 | if e3b { 28 | t.Errorf("%q should not Exist the first time we check.", e3) 29 | } 30 | if !e3a { 31 | t.Errorf("%q should Exist the second time we check.", e3) 32 | } 33 | } 34 | 35 | func TestBitUint(t *testing.T) { 36 | f := bloom.NewB(1000, 4) 37 | n1 := make([]byte, 4) 38 | n2 := make([]byte, 4) 39 | n3 := make([]byte, 4) 40 | n4 := make([]byte, 4) 41 | binary.BigEndian.PutUint32(n1, 100) 42 | binary.BigEndian.PutUint32(n2, 101) 43 | binary.BigEndian.PutUint32(n3, 102) 44 | binary.BigEndian.PutUint32(n4, 103) 45 | f.Add(n1) 46 | n3b := f.Exist(n3) 47 | n1a := f.Exist(n1) 48 | n2a := f.Exist(n2) 49 | f.Add(n3) 50 | n3a := f.Exist(n3) 51 | n4a := f.Exist(n4) 52 | if !n1a { 53 | t.Errorf("%q should Exist.", n1) 54 | } 55 | if n2a { 56 | t.Errorf("%q should not Exist.", n2) 57 | } 58 | if n3b { 59 | t.Errorf("%q should not Exist the first time we check.", n3) 60 | } 61 | if !n3a { 62 | t.Errorf("%q should Exist the second time we check.", n3) 63 | } 64 | if n4a { 65 | t.Errorf("%q should not Exist.", n4) 66 | } 67 | } 68 | 69 | func TestBitString(t *testing.T) { 70 | f := bloom.NewBGuess(1000, 0.001) 71 | s1 := "Filter" 72 | s2 := "is" 73 | s3 := "in" 74 | s4 := "bloom" 75 | f.AddString(s1) 76 | s3b := f.ExistString(s3) 77 | s1a := f.ExistString(s1) 78 | s2a := f.ExistString(s2) 79 | f.AddString(s3) 80 | s3a := f.ExistString(s3) 81 | s4a := f.ExistString(s4) 82 | if !s1a { 83 | t.Errorf("%q should Exist.", s1) 84 | } 85 | if s2a { 86 | t.Errorf("%q should not Exist.", s2) 87 | } 88 | if s3b { 89 | t.Errorf("%q should not Exist the first time we check.", s3) 90 | } 91 | if !s3a { 92 | t.Errorf("%q should Exist the second time we check.", s3) 93 | } 94 | if s4a { 95 | t.Errorf("%q should not Exist.", s4) 96 | } 97 | } 98 | 99 | func TestBitM(t *testing.T) { 100 | f := bloom.NewB(1000, 4) 101 | if f.M() != 1024 { 102 | t.Errorf("M() %v is not correct", f.M()) 103 | } 104 | } 105 | 106 | func TestBitK(t *testing.T) { 107 | f := bloom.NewB(1000, 4) 108 | if f.K() != 4 { 109 | t.Errorf("K() %v is not correct", f.K()) 110 | } 111 | } 112 | 113 | func BenchmarkBitAddExist(b *testing.B) { 114 | f := bloom.NewBGuess(uint64(b.N), 0.0001) 115 | key := make([]byte, 8) 116 | b.ResetTimer() 117 | for i := 0; i < b.N; i++ { 118 | binary.BigEndian.PutUint64(key, uint64(i)) 119 | f.Add(key) 120 | f.Exist(key) 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /pkg/rwspinlock/rwspinlock_test.go: -------------------------------------------------------------------------------- 1 | package rwspinlock 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | ) 7 | 8 | func TestRLocker(t *testing.T) { 9 | threads, loops := 8, 1000 10 | var wg sync.WaitGroup 11 | wg.Add(threads) 12 | l := New().RLocker() 13 | 14 | for i := 0; i < threads; i++ { 15 | go func() { 16 | for i := 0; i < loops; i++ { 17 | l.Lock() 18 | l.Unlock() 19 | } 20 | wg.Done() 21 | }() 22 | } 23 | wg.Wait() 24 | } 25 | 26 | func TestLock(t *testing.T) { 27 | threads, loops, count := 8, 1000, 0 28 | var wg sync.WaitGroup 29 | wg.Add(threads) 30 | l := New() 31 | 32 | for i := 0; i < threads; i++ { 33 | go func() { 34 | for i := 0; i < loops; i++ { 35 | l.Lock() 36 | count++ 37 | l.Unlock() 38 | } 39 | wg.Done() 40 | }() 41 | } 42 | wg.Wait() 43 | 44 | if count != threads*loops { 45 | t.Errorf("expected %d got %d", threads*loops, count) 46 | } 47 | } 48 | 49 | func TestRLock(t *testing.T) { 50 | threads, loops := 8, 1000 51 | var wg sync.WaitGroup 52 | wg.Add(threads) 53 | l := New() 54 | 55 | for i := 0; i < threads; i++ { 56 | go func() { 57 | for i := 0; i < loops; i++ { 58 | l.RLock() 59 | l.RUnlock() 60 | } 61 | wg.Done() 62 | }() 63 | } 64 | wg.Wait() 65 | } 66 | 67 | func TestRWLock(t *testing.T) { 68 | threadsR, threadsW, loopsR, loopsW := 8, 2, 1000, 100 69 | var wg sync.WaitGroup 70 | wg.Add(threadsR + threadsW) 71 | l := New() 72 | 73 | for i := 0; i < threadsR; i++ { 74 | go func() { 75 | for i := 0; i < loopsR; i++ { 76 | l.RLock() 77 | l.RUnlock() 78 | } 79 | wg.Done() 80 | }() 81 | } 82 | 83 | for i := 0; i < threadsW; i++ { 84 | go func() { 85 | for i := 0; i < loopsW; i++ { 86 | l.Lock() 87 | l.Unlock() 88 | } 89 | wg.Done() 90 | }() 91 | } 92 | wg.Wait() 93 | } 94 | 95 | func BenchmarkRLock(b *testing.B) { 96 | l := New() 97 | for n := 0; n < b.N; n++ { 98 | l.RLock() 99 | l.RUnlock() 100 | } 101 | } 102 | 103 | func BenchmarkLock(b *testing.B) { 104 | l := New() 105 | for n := 0; n < b.N; n++ { 106 | l.Lock() 107 | l.Unlock() 108 | } 109 | } 110 | 111 | func BenchmarkRWLock(b *testing.B) { 112 | l := New() 113 | for n := 0; n < b.N; n++ { 114 | if n&3 == 0 { 115 | go func() { 116 | l.Lock() 117 | l.Unlock() 118 | }() 119 | } 120 | l.RLock() 121 | l.RUnlock() 122 | } 123 | } 124 | 125 | func BenchmarkRMutex(b *testing.B) { 126 | var l sync.RWMutex 127 | for n := 0; n < b.N; n++ { 128 | l.RLock() 129 | l.RUnlock() 130 | } 131 | } 132 | 133 | func BenchmarkMutex(b *testing.B) { 134 | var l sync.RWMutex 135 | for n := 0; n < b.N; n++ { 136 | l.Lock() 137 | l.Unlock() 138 | } 139 | } 140 | 141 | func BenchmarkRWMutex(b *testing.B) { 142 | var l sync.RWMutex 143 | for n := 0; n < b.N; n++ { 144 | if n&3 == 0 { 145 | go func() { 146 | l.Lock() 147 | l.Unlock() 148 | }() 149 | } 150 | l.RLock() 151 | l.RUnlock() 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /docs/log.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # log 4 | `import "github.com/andy2046/gopie/pkg/log"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package log implements a minimalistic Logger interface. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [Variables](#pkg-variables) 17 | * [type Config](#Config) 18 | * [type Level](#Level) 19 | * [type Logger](#Logger) 20 | * [func NewLogger(options ...Option) Logger](#NewLogger) 21 | * [type Option](#Option) 22 | 23 | 24 | #### Package files 25 | [log.go](/src/github.com/andy2046/gopie/pkg/log/log.go) 26 | 27 | 28 | 29 | ## Variables 30 | ``` go 31 | var ( 32 | // DefaultConfig is the default Logger Config. 33 | DefaultConfig = Config{ 34 | Level: INFO, 35 | Prefix: "", 36 | Flag: log.Ldate | log.Ltime, 37 | DebugHandler: os.Stdout, 38 | InfoHandler: os.Stdout, 39 | WarnHandler: os.Stdout, 40 | ErrorHandler: os.Stderr, 41 | } 42 | ) 43 | ``` 44 | 45 | 46 | 47 | ## type [Config](/src/target/log.go?s=580:760#L30) 48 | ``` go 49 | type Config struct { 50 | Level Level 51 | Prefix string 52 | Flag int 53 | DebugHandler io.Writer 54 | InfoHandler io.Writer 55 | WarnHandler io.Writer 56 | ErrorHandler io.Writer 57 | } 58 | ``` 59 | Config used to init Logger. 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | ## type [Level](/src/target/log.go?s=152:161#L13) 71 | ``` go 72 | type Level int 73 | ``` 74 | Level defines Logging level. 75 | 76 | 77 | ``` go 78 | const ( 79 | DEBUG Level = iota 80 | INFO 81 | WARN 82 | ERROR 83 | ) 84 | ``` 85 | Logging Levels. 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | ## type [Logger](/src/target/log.go?s=201:545#L16) 97 | ``` go 98 | type Logger interface { 99 | Debug(v ...interface{}) 100 | Info(v ...interface{}) 101 | Warn(v ...interface{}) 102 | Error(v ...interface{}) 103 | Debugf(format string, v ...interface{}) 104 | Infof(format string, v ...interface{}) 105 | Warnf(format string, v ...interface{}) 106 | Errorf(format string, v ...interface{}) 107 | SetLevel(l Level) 108 | LevelLogger(l Level) *log.Logger 109 | } 110 | ``` 111 | Logger is the Logging interface. 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | ### func [NewLogger](/src/target/log.go?s=1340:1380#L74) 120 | ``` go 121 | func NewLogger(options ...Option) Logger 122 | ``` 123 | NewLogger returns a new Logger. 124 | 125 | 126 | 127 | 128 | 129 | ## type [Option](/src/target/log.go?s=807:835#L41) 130 | ``` go 131 | type Option = func(*Config) error 132 | ``` 133 | Option applies config to Logger Config. 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | - - - 149 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 150 | -------------------------------------------------------------------------------- /pkg/fileutil/fileutil_test.go: -------------------------------------------------------------------------------- 1 | package fileutil_test 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "reflect" 9 | "strings" 10 | "testing" 11 | "time" 12 | 13 | . "github.com/andy2046/gopie/pkg/fileutil" 14 | ) 15 | 16 | func TestIsDirWriteable(t *testing.T) { 17 | tmpdir, err := ioutil.TempDir("", "foo") 18 | if err != nil { 19 | t.Fatalf("ioutil.TempDir error %v", err) 20 | } 21 | defer os.RemoveAll(tmpdir) 22 | if err = IsDirWriteable(tmpdir); err != nil { 23 | t.Fatalf("IsDirWriteable error %v", err) 24 | } 25 | } 26 | 27 | func TestCreateDirAll(t *testing.T) { 28 | tmpdir, err := ioutil.TempDir(os.TempDir(), "foo") 29 | if err != nil { 30 | t.Fatal(err) 31 | } 32 | defer os.RemoveAll(tmpdir) 33 | 34 | tmpdir2 := filepath.Join(tmpdir, "bar") 35 | if err = CreateDirAll(tmpdir2); err != nil { 36 | t.Fatal(err) 37 | } 38 | 39 | if err = ioutil.WriteFile(filepath.Join(tmpdir2, "test.txt"), 40 | []byte("test"), PrivateFileMode); err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | if err = CreateDirAll(tmpdir2); err == nil || !strings.Contains(err.Error(), "to be empty, got") { 45 | t.Fatalf("CreateDirAll error %v", err) 46 | } 47 | } 48 | 49 | func TestExist(t *testing.T) { 50 | fdir := filepath.Join(os.TempDir(), fmt.Sprint(time.Now().UnixNano())) 51 | os.RemoveAll(fdir) 52 | if err := os.Mkdir(fdir, 0666); err != nil { 53 | t.Skip(err) 54 | } 55 | defer os.RemoveAll(fdir) 56 | if !Exist(fdir) { 57 | t.Fatal("Exist expected to be true") 58 | } 59 | 60 | f, err := ioutil.TempFile(os.TempDir(), "fileutil") 61 | if err != nil { 62 | t.Skip(err) 63 | } 64 | f.Close() 65 | 66 | if !Exist(f.Name()) { 67 | t.Error("Exist expected to be true") 68 | } 69 | 70 | os.Remove(f.Name()) 71 | if Exist(f.Name()) { 72 | t.Error("Exist expected to be false") 73 | } 74 | } 75 | 76 | func TestReadDir(t *testing.T) { 77 | tmpdir, err := ioutil.TempDir("", "") 78 | defer os.RemoveAll(tmpdir) 79 | if err != nil { 80 | t.Fatalf("ioutil.TempDir error %v", err) 81 | } 82 | 83 | files := []string{"def", "abc", "xyz", "ghi"} 84 | for _, f := range files { 85 | fh, err := os.Create(filepath.Join(tmpdir, f)) 86 | if err != nil { 87 | t.Skip(err) 88 | } 89 | if err = fh.Close(); err != nil { 90 | t.Skip(err) 91 | } 92 | } 93 | fs, err := ReadDir(tmpdir) 94 | if err != nil { 95 | t.Fatalf("ReadDir error %v", err) 96 | } 97 | wfs := []string{"abc", "def", "ghi", "xyz"} 98 | if !reflect.DeepEqual(fs, wfs) { 99 | t.Fatalf("ReadDir error got %v, want %v", fs, wfs) 100 | } 101 | 102 | files = []string{"def.wal", "abc.wal", "xyz.wal", "ghi.wal"} 103 | for _, f := range files { 104 | fh, err := os.Create(filepath.Join(tmpdir, f)) 105 | if err != nil { 106 | t.Skip(err) 107 | } 108 | if err = fh.Close(); err != nil { 109 | t.Skip(err) 110 | } 111 | } 112 | fs, err = ReadDir(tmpdir, WithExt(".wal")) 113 | if err != nil { 114 | t.Fatalf("ReadDir error %v", err) 115 | } 116 | wfs = []string{"abc.wal", "def.wal", "ghi.wal", "xyz.wal"} 117 | if !reflect.DeepEqual(fs, wfs) { 118 | t.Fatalf("ReadDir error got %v, want %v", fs, wfs) 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /docs/lru.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # lru 4 | `import "github.com/andy2046/gopie/pkg/lru"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package lru implements a LRU cache. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [type Cache](#Cache) 17 | * [func New(maxEntries int) *Cache](#New) 18 | * [func (c *Cache) Add(key interface{}, value interface{})](#Cache.Add) 19 | * [func (c *Cache) Clear()](#Cache.Clear) 20 | * [func (c *Cache) Get(key interface{}) (value interface{}, ok bool)](#Cache.Get) 21 | * [func (c *Cache) Len() int](#Cache.Len) 22 | * [func (c *Cache) Remove(key interface{})](#Cache.Remove) 23 | * [func (c *Cache) RemoveOldest()](#Cache.RemoveOldest) 24 | 25 | 26 | #### Package files 27 | [lru.go](/src/github.com/andy2046/gopie/pkg/lru/lru.go) 28 | 29 | 30 | 31 | 32 | 33 | 34 | ## type [Cache](/src/target/lru.go?s=115:478#L10) 35 | ``` go 36 | type Cache struct { 37 | // MaxEntries is the maximum number of cache entries 38 | // before an item is purged. Zero means no limit. 39 | MaxEntries int 40 | 41 | // OnPurged specifies a function to be executed 42 | // when an entry is purged from the cache. 43 | OnPurged func(key interface{}, value interface{}) 44 | // contains filtered or unexported fields 45 | } 46 | ``` 47 | Cache is a LRU cache. 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | ### func [New](/src/target/lru.go?s=616:647#L30) 56 | ``` go 57 | func New(maxEntries int) *Cache 58 | ``` 59 | New creates a new cache, if maxEntries is zero, the cache has no limit. 60 | 61 | 62 | 63 | 64 | 65 | ### func (\*Cache) [Add](/src/target/lru.go?s=879:934#L42) 66 | ``` go 67 | func (c *Cache) Add(key interface{}, value interface{}) 68 | ``` 69 | Add adds value to the cache. 70 | 71 | 72 | 73 | 74 | ### func (\*Cache) [Clear](/src/target/lru.go?s=2527:2550#L131) 75 | ``` go 76 | func (c *Cache) Clear() 77 | ``` 78 | Clear purges all items from the cache. 79 | 80 | 81 | 82 | 83 | ### func (\*Cache) [Get](/src/target/lru.go?s=1353:1418#L63) 84 | ``` go 85 | func (c *Cache) Get(key interface{}) (value interface{}, ok bool) 86 | ``` 87 | Get looks up value by key from the cache. 88 | 89 | 90 | 91 | 92 | ### func (\*Cache) [Len](/src/target/lru.go?s=2363:2388#L120) 93 | ``` go 94 | func (c *Cache) Len() int 95 | ``` 96 | Len returns the number of items in the cache. 97 | 98 | 99 | 100 | 101 | ### func (\*Cache) [Remove](/src/target/lru.go?s=1654:1693#L78) 102 | ``` go 103 | func (c *Cache) Remove(key interface{}) 104 | ``` 105 | Remove removes the provided key from the cache. 106 | 107 | 108 | 109 | 110 | ### func (\*Cache) [RemoveOldest](/src/target/lru.go?s=1885:1915#L91) 111 | ``` go 112 | func (c *Cache) RemoveOldest() 113 | ``` 114 | RemoveOldest removes the oldest item from the cache. 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | - - - 124 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 125 | -------------------------------------------------------------------------------- /docs/spsc.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # spsc 4 | `import "github.com/andy2046/gopie/pkg/spsc"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package spsc implements a Single-Producer / Single-Consumer queue. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [Constants](#pkg-constants) 17 | * [type SPSC](#SPSC) 18 | * [func New(size uint32, batchSize ...uint32) *SPSC](#New) 19 | * [func (sp *SPSC) Close()](#SPSC.Close) 20 | * [func (sp *SPSC) Get(i interface{}) bool](#SPSC.Get) 21 | * [func (sp *SPSC) Offer(i interface{}) bool](#SPSC.Offer) 22 | * [func (sp *SPSC) Poll(i interface{}) bool](#SPSC.Poll) 23 | * [func (sp *SPSC) Put(i interface{})](#SPSC.Put) 24 | 25 | 26 | #### Package files 27 | [spsc.go](/src/github.com/andy2046/gopie/pkg/spsc/spsc.go) [util.go](/src/github.com/andy2046/gopie/pkg/spsc/util.go) 28 | 29 | 30 | ## Constants 31 | ``` go 32 | const ( 33 | // CacheLinePadSize is the size of OS CacheLine. 34 | CacheLinePadSize = unsafe.Sizeof(cpu.CacheLinePad{}) 35 | // DefaultMaxBatch is the default max batch size. 36 | DefaultMaxBatch uint32 = (1 << 8) - 1 37 | ) 38 | ``` 39 | 40 | 41 | 42 | 43 | ## type [SPSC](/src/target/spsc.go?s=398:888#L19) 44 | ``` go 45 | type SPSC struct { 46 | // contains filtered or unexported fields 47 | } 48 | ``` 49 | SPSC is the SPSC queue structure. 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | ### func [New](/src/target/spsc.go?s=936:984#L37) 58 | ``` go 59 | func New(size uint32, batchSize ...uint32) *SPSC 60 | ``` 61 | New create a new SPSC with bounded `size`. 62 | 63 | 64 | 65 | 66 | 67 | ### func (\*SPSC) [Close](/src/target/spsc.go?s=1320:1343#L50) 68 | ``` go 69 | func (sp *SPSC) Close() 70 | ``` 71 | Close the SPSC, it shall NOT be called before `Offer()` or `Put()`. 72 | 73 | 74 | 75 | 76 | ### func (\*SPSC) [Get](/src/target/spsc.go?s=2234:2273#L83) 77 | ``` go 78 | func (sp *SPSC) Get(i interface{}) bool 79 | ``` 80 | Get the value at the head of the queue to given variable, 81 | blocking, return false if the queue is closed. 82 | 83 | 84 | 85 | 86 | ### func (\*SPSC) [Offer](/src/target/spsc.go?s=2938:2979#L110) 87 | ``` go 88 | func (sp *SPSC) Offer(i interface{}) bool 89 | ``` 90 | Offer given variable at the tail of the queue, 91 | non-blocking, return false if the queue is full. 92 | 93 | 94 | 95 | 96 | ### func (\*SPSC) [Poll](/src/target/spsc.go?s=1503:1543#L56) 97 | ``` go 98 | func (sp *SPSC) Poll(i interface{}) bool 99 | ``` 100 | Poll the value at the head of the queue to given variable, 101 | non-blocking, return false if the queue is empty / closed. 102 | 103 | 104 | 105 | 106 | ### func (\*SPSC) [Put](/src/target/spsc.go?s=3583:3617#L133) 107 | ``` go 108 | func (sp *SPSC) Put(i interface{}) 109 | ``` 110 | Put given variable at the tail of the queue, 111 | blocking. 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | - - - 121 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 122 | -------------------------------------------------------------------------------- /docs/fileutil.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # fileutil 4 | `import "github.com/andy2046/gopie/pkg/fileutil"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package fileutil implements some file utils. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [Constants](#pkg-constants) 17 | * [func CreateDirAll(dir string) error](#CreateDirAll) 18 | * [func Exist(name string) bool](#Exist) 19 | * [func IsDirWriteable(dir string) error](#IsDirWriteable) 20 | * [func ReadDir(d string, opts ...func(*ReadDirOperation)) ([]string, error)](#ReadDir) 21 | * [func TouchDirAll(dir string) error](#TouchDirAll) 22 | * [func WithExt(ext string) func(*ReadDirOperation)](#WithExt) 23 | * [type ReadDirOperation](#ReadDirOperation) 24 | 25 | 26 | #### Package files 27 | [fileutil.go](/src/github.com/andy2046/gopie/pkg/fileutil/fileutil.go) 28 | 29 | 30 | ## Constants 31 | ``` go 32 | const ( 33 | // PrivateFileMode represents file read/write mode to owner. 34 | PrivateFileMode = 0600 35 | // PrivateDirMode represents file make/remove mode in the directory to owner. 36 | PrivateDirMode = 0700 37 | ) 38 | ``` 39 | 40 | 41 | 42 | ## func [CreateDirAll](/src/target/fileutil.go?s=1916:1951#L86) 43 | ``` go 44 | func CreateDirAll(dir string) error 45 | ``` 46 | CreateDirAll wraps TouchDirAll but returns error 47 | if the deepest directory is not empty. 48 | 49 | 50 | 51 | ## func [Exist](/src/target/fileutil.go?s=2233:2261#L101) 52 | ``` go 53 | func Exist(name string) bool 54 | ``` 55 | Exist returns true if a file or directory exists. 56 | 57 | 58 | 59 | ## func [IsDirWriteable](/src/target/fileutil.go?s=1352:1389#L66) 60 | ``` go 61 | func IsDirWriteable(dir string) error 62 | ``` 63 | IsDirWriteable checks if dir is writable by writing and removing a file to dir. 64 | 65 | 66 | 67 | ## func [ReadDir](/src/target/fileutil.go?s=767:840#L36) 68 | ``` go 69 | func ReadDir(d string, opts ...func(*ReadDirOperation)) ([]string, error) 70 | ``` 71 | ReadDir returns the file names in the provided directory in order. 72 | 73 | 74 | 75 | ## func [TouchDirAll](/src/target/fileutil.go?s=1680:1714#L76) 76 | ``` go 77 | func TouchDirAll(dir string) error 78 | ``` 79 | TouchDirAll creates directories with 0700 permission if any directory 80 | does not exist and ensures the provided directory is writable. 81 | 82 | 83 | 84 | ## func [WithExt](/src/target/fileutil.go?s=475:523#L25) 85 | ``` go 86 | func WithExt(ext string) func(*ReadDirOperation) 87 | ``` 88 | WithExt filters file names by extension. 89 | 90 | 91 | 92 | 93 | ## type [ReadDirOperation](/src/target/fileutil.go?s=186:230#L13) 94 | ``` go 95 | type ReadDirOperation struct { 96 | // contains filtered or unexported fields 97 | } 98 | ``` 99 | ReadDirOperation represents read-directory operation. 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | - - - 115 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 116 | -------------------------------------------------------------------------------- /pkg/countminsketch/countmin_test.go: -------------------------------------------------------------------------------- 1 | package countminsketch 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | ) 7 | 8 | func TestCount(t *testing.T) { 9 | cms, _ := NewGuess(0.001, 0.99) 10 | 11 | for i := 0; i < 100; i++ { 12 | cms.Add([]byte(strconv.Itoa(i))) 13 | } 14 | 15 | if count := cms.Count(); count != 100 { 16 | t.Errorf("expected 100, got %d", count) 17 | } 18 | } 19 | 20 | func TestEstimate(t *testing.T) { 21 | cms, _ := NewGuess(0.001, 0.99) 22 | cms.Add([]byte(`a`)) 23 | cms.Add([]byte(`b`), 1) 24 | cms.Add([]byte(`c`), 1) 25 | cms.Add([]byte(`b`), 1) 26 | 27 | if count := cms.Estimate([]byte(`a`)); count != 1 { 28 | t.Errorf("expected 1, got %d", count) 29 | } 30 | 31 | if count := cms.Estimate([]byte(`b`)); count != 2 { 32 | t.Errorf("expected 2, got %d", count) 33 | } 34 | 35 | if count := cms.Estimate([]byte(`c`)); count != 1 { 36 | t.Errorf("expected 1, got %d", count) 37 | } 38 | 39 | if count := cms.Estimate([]byte(`x`)); count != 0 { 40 | t.Errorf("expected 0, got %d", count) 41 | } 42 | } 43 | 44 | func TestMerge(t *testing.T) { 45 | cms, _ := NewGuess(0.001, 0.99) 46 | cms.Add([]byte(`a`)) 47 | cms.Add([]byte(`b`), 1) 48 | cms.Add([]byte(`c`), 1) 49 | cms.Add([]byte(`b`), 1) 50 | cms.Add([]byte(`d`), 1) 51 | 52 | other, _ := NewGuess(0.001, 0.99) 53 | other.Add([]byte(`b`), 1) 54 | other.Add([]byte(`c`), 1) 55 | other.Add([]byte(`b`), 1) 56 | 57 | if err := cms.Merge(other); err != nil { 58 | t.Error(err) 59 | } 60 | 61 | if count := cms.Estimate([]byte(`a`)); count != 1 { 62 | t.Errorf("expected 1, got %d", count) 63 | } 64 | 65 | if count := cms.Estimate([]byte(`b`)); count != 4 { 66 | t.Errorf("expected 4, got %d", count) 67 | } 68 | 69 | if count := cms.Estimate([]byte(`c`)); count != 2 { 70 | t.Errorf("expected 2, got %d", count) 71 | } 72 | 73 | if count := cms.Estimate([]byte(`d`)); count != 1 { 74 | t.Errorf("expected 1, got %d", count) 75 | } 76 | 77 | if count := cms.Estimate([]byte(`x`)); count != 0 { 78 | t.Errorf("expected 0, got %d", count) 79 | } 80 | } 81 | 82 | func TestReset(t *testing.T) { 83 | cms, _ := NewGuess(0.001, 0.99) 84 | cms.Add([]byte(`a`)) 85 | cms.Add([]byte(`b`), 1) 86 | cms.Add([]byte(`c`), 1) 87 | cms.Add([]byte(`b`), 1) 88 | cms.Add([]byte(`d`), 1) 89 | 90 | cms.Reset() 91 | 92 | for i := uint(0); i < cms.depth; i++ { 93 | for j := uint(0); j < cms.width; j++ { 94 | if x := cms.matrix[i][j]; x != 0 { 95 | t.Errorf("expected matrix to be empty, got %d", x) 96 | } 97 | } 98 | } 99 | } 100 | 101 | func BenchmarkAdd(b *testing.B) { 102 | b.StopTimer() 103 | cms, _ := NewGuess(0.001, 0.99) 104 | data := make([][]byte, b.N) 105 | for i := 0; i < b.N; i++ { 106 | data[i] = []byte(strconv.Itoa(i)) 107 | } 108 | b.StartTimer() 109 | 110 | for n := 0; n < b.N; n++ { 111 | cms.Add(data[n]) 112 | } 113 | } 114 | 115 | func BenchmarkCount(b *testing.B) { 116 | b.StopTimer() 117 | cms, _ := NewGuess(0.001, 0.99) 118 | data := make([][]byte, b.N) 119 | for i := 0; i < b.N; i++ { 120 | data[i] = []byte(strconv.Itoa(i)) 121 | cms.Add([]byte(strconv.Itoa(i))) 122 | } 123 | b.StartTimer() 124 | 125 | for n := 0; n < b.N; n++ { 126 | cms.Estimate(data[n]) 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /docs/hyperloglog.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # hyperloglog 4 | `import "github.com/andy2046/gopie/pkg/hyperloglog"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package hyperloglog implements HyperLogLog cardinality estimation. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [type HyperLogLog](#HyperLogLog) 17 | * [func New(m uint) (*HyperLogLog, error)](#New) 18 | * [func NewGuess(stdErr float64) (*HyperLogLog, error)](#NewGuess) 19 | * [func (h *HyperLogLog) Add(data []byte)](#HyperLogLog.Add) 20 | * [func (h *HyperLogLog) Count() uint64](#HyperLogLog.Count) 21 | * [func (h *HyperLogLog) Merge(other *HyperLogLog) error](#HyperLogLog.Merge) 22 | * [func (h *HyperLogLog) Reset()](#HyperLogLog.Reset) 23 | * [func (h *HyperLogLog) SetHash(hasher hash.Hash32)](#HyperLogLog.SetHash) 24 | 25 | 26 | #### Package files 27 | [hyperloglog.go](/src/github.com/andy2046/gopie/pkg/hyperloglog/hyperloglog.go) 28 | 29 | 30 | 31 | 32 | 33 | 34 | ## type [HyperLogLog](/src/target/hyperloglog.go?s=210:490#L12) 35 | ``` go 36 | type HyperLogLog struct { 37 | // contains filtered or unexported fields 38 | } 39 | ``` 40 | HyperLogLog probabilistic data struct for cardinality estimation. 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | ### func [New](/src/target/hyperloglog.go?s=737:775#L30) 49 | ``` go 50 | func New(m uint) (*HyperLogLog, error) 51 | ``` 52 | New creates a new HyperLogLog with `m` registers bucket. 53 | `m` should be a power of two. 54 | 55 | 56 | ### func [NewGuess](/src/target/hyperloglog.go?s=1088:1139#L45) 57 | ``` go 58 | func NewGuess(stdErr float64) (*HyperLogLog, error) 59 | ``` 60 | NewGuess creates a new HyperLogLog within the given standard error. 61 | 62 | 63 | 64 | 65 | 66 | ### func (\*HyperLogLog) [Add](/src/target/hyperloglog.go?s=1265:1303#L51) 67 | ``` go 68 | func (h *HyperLogLog) Add(data []byte) 69 | ``` 70 | Add adds the data to the set. 71 | 72 | 73 | 74 | 75 | ### func (\*HyperLogLog) [Count](/src/target/hyperloglog.go?s=1542:1578#L65) 76 | ``` go 77 | func (h *HyperLogLog) Count() uint64 78 | ``` 79 | Count returns the estimated cardinality of the set. 80 | 81 | 82 | 83 | 84 | ### func (\*HyperLogLog) [Merge](/src/target/hyperloglog.go?s=2112:2165#L90) 85 | ``` go 86 | func (h *HyperLogLog) Merge(other *HyperLogLog) error 87 | ``` 88 | Merge combines the HyperLogLog with the other. 89 | 90 | 91 | 92 | 93 | ### func (\*HyperLogLog) [Reset](/src/target/hyperloglog.go?s=2416:2445#L105) 94 | ``` go 95 | func (h *HyperLogLog) Reset() 96 | ``` 97 | Reset restores the HyperLogLog to its original state. 98 | 99 | 100 | 101 | 102 | ### func (\*HyperLogLog) [SetHash](/src/target/hyperloglog.go?s=2523:2572#L110) 103 | ``` go 104 | func (h *HyperLogLog) SetHash(hasher hash.Hash32) 105 | ``` 106 | SetHash sets the hashing function. 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | - - - 116 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 117 | -------------------------------------------------------------------------------- /pkg/lru/lru.go: -------------------------------------------------------------------------------- 1 | // Package lru implements a LRU cache. 2 | package lru 3 | 4 | import ( 5 | "container/list" 6 | "sync" 7 | ) 8 | 9 | // Cache is a LRU cache. 10 | type Cache struct { 11 | // MaxEntries is the maximum number of cache entries 12 | // before an item is purged. Zero means no limit. 13 | MaxEntries int 14 | 15 | // OnPurged specifies a function to be executed 16 | // when an entry is purged from the cache. 17 | OnPurged func(key interface{}, value interface{}) 18 | 19 | ll *list.List 20 | cache map[interface{}]*list.Element 21 | mu sync.RWMutex 22 | } 23 | 24 | type entry struct { 25 | key interface{} 26 | value interface{} 27 | } 28 | 29 | // New creates a new cache, if maxEntries is zero, the cache has no limit. 30 | func New(maxEntries int) *Cache { 31 | if maxEntries < 0 { 32 | panic("maxEntries can not be less than zero") 33 | } 34 | return &Cache{ 35 | MaxEntries: maxEntries, 36 | ll: list.New(), 37 | cache: make(map[interface{}]*list.Element), 38 | } 39 | } 40 | 41 | // Add adds value to the cache. 42 | func (c *Cache) Add(key interface{}, value interface{}) { 43 | c.mu.Lock() 44 | defer c.mu.Unlock() 45 | 46 | if c.cache == nil { 47 | c.cache = make(map[interface{}]*list.Element) 48 | c.ll = list.New() 49 | } 50 | if e, ok := c.cache[key]; ok { 51 | c.ll.MoveToFront(e) 52 | e.Value.(*entry).value = value 53 | return 54 | } 55 | ele := c.ll.PushFront(&entry{key, value}) 56 | c.cache[key] = ele 57 | if c.MaxEntries > 0 && c.ll.Len() > c.MaxEntries { 58 | c.removeOldest(false) 59 | } 60 | } 61 | 62 | // Get looks up value by key from the cache. 63 | func (c *Cache) Get(key interface{}) (value interface{}, ok bool) { 64 | c.mu.Lock() 65 | defer c.mu.Unlock() 66 | 67 | if c.cache == nil { 68 | return 69 | } 70 | if ele, hit := c.cache[key]; hit { 71 | c.ll.MoveToFront(ele) 72 | return ele.Value.(*entry).value, true 73 | } 74 | return 75 | } 76 | 77 | // Remove removes the provided key from the cache. 78 | func (c *Cache) Remove(key interface{}) { 79 | c.mu.Lock() 80 | defer c.mu.Unlock() 81 | 82 | if c.cache == nil { 83 | return 84 | } 85 | if ele, hit := c.cache[key]; hit { 86 | c.removeElement(ele) 87 | } 88 | } 89 | 90 | // RemoveOldest removes the oldest item from the cache. 91 | func (c *Cache) RemoveOldest() { 92 | c.removeOldest(true) 93 | } 94 | 95 | func (c *Cache) removeOldest(toLock bool) { 96 | if toLock { 97 | c.mu.Lock() 98 | defer c.mu.Unlock() 99 | } 100 | 101 | if c.cache == nil { 102 | return 103 | } 104 | ele := c.ll.Back() 105 | if ele != nil { 106 | c.removeElement(ele) 107 | } 108 | } 109 | 110 | func (c *Cache) removeElement(e *list.Element) { 111 | c.ll.Remove(e) 112 | kv := e.Value.(*entry) 113 | delete(c.cache, kv.key) 114 | if c.OnPurged != nil { 115 | c.OnPurged(kv.key, kv.value) 116 | } 117 | } 118 | 119 | // Len returns the number of items in the cache. 120 | func (c *Cache) Len() int { 121 | c.mu.RLock() 122 | defer c.mu.RUnlock() 123 | 124 | if c.cache == nil { 125 | return 0 126 | } 127 | return c.ll.Len() 128 | } 129 | 130 | // Clear purges all items from the cache. 131 | func (c *Cache) Clear() { 132 | c.mu.Lock() 133 | defer c.mu.Unlock() 134 | 135 | if c.OnPurged != nil { 136 | for _, e := range c.cache { 137 | kv := e.Value.(*entry) 138 | c.OnPurged(kv.key, kv.value) 139 | } 140 | } 141 | c.ll = nil 142 | c.cache = nil 143 | } 144 | -------------------------------------------------------------------------------- /pkg/multilane/multilane.go: -------------------------------------------------------------------------------- 1 | // Package multilane implements a concurrent blocking multiset. 2 | package multilane 3 | 4 | import ( 5 | "github.com/andy2046/gopie/pkg/spsc" 6 | "runtime" 7 | ) 8 | 9 | const ( 10 | defaultQSize uint32 = 1024 11 | defaultWidth uint32 = 8 12 | ) 13 | 14 | var defaultQueFunc = func(size uint32) Queue { 15 | return spsc.New(size) 16 | } 17 | 18 | type ( 19 | // Queue represents the queue with Get / Put / Close methods. 20 | Queue interface { 21 | Get(interface{}) bool 22 | Put(interface{}) 23 | Close() 24 | } 25 | 26 | // Config for MultiLane. 27 | Config struct { 28 | LaneWidth uint32 // number of queue for MultiLane 29 | QueueSize uint32 // size of each queue in MultiLane 30 | New func(uint32) Queue 31 | } 32 | 33 | // MultiLane represents the concurrent multiset. 34 | MultiLane struct { 35 | _ [spsc.CacheLinePadSize]byte 36 | lanes []Queue 37 | laneMask int64 38 | putCh chan int64 39 | getCh chan int64 40 | queSize uint32 41 | _ [4]byte 42 | } 43 | ) 44 | 45 | // New create a new MultiLane. 46 | func New(conf Config) *MultiLane { 47 | m := MultiLane{} 48 | qs, w, queFunc := defaultQSize, defaultWidth, defaultQueFunc 49 | if conf.New != nil { 50 | queFunc = conf.New 51 | } 52 | if conf.QueueSize > 7 { 53 | qs = conf.QueueSize 54 | } 55 | if conf.LaneWidth > 0 { 56 | w = conf.LaneWidth 57 | } 58 | m.queSize = nextPowerOf2(qs) 59 | m.lanes = make([]Queue, nextPowerOf2(w)) 60 | m.laneMask = int64(len(m.lanes) - 1) 61 | m.putCh = make(chan int64, len(m.lanes)) 62 | m.getCh = make(chan int64, len(m.lanes)) 63 | for i := range m.lanes { 64 | m.lanes[i] = queFunc(m.queSize) 65 | m.putCh <- int64(i) 66 | m.getCh <- int64(i) 67 | } 68 | return &m 69 | } 70 | 71 | // GetLane get the value at the provided lane to given variable, 72 | // blocking. 73 | func (m *MultiLane) GetLane(lane uint32, i interface{}) bool { 74 | return m.lanes[int64(lane)&m.laneMask].Get(i) 75 | } 76 | 77 | // PutLane put given variable at the provided lane, 78 | // blocking. 79 | func (m *MultiLane) PutLane(lane uint32, i interface{}) { 80 | m.lanes[int64(lane)&m.laneMask].Put(i) 81 | } 82 | 83 | // Get the value at one of the lanes pointed by the cursor to given variable, 84 | // blocking. 85 | func (m *MultiLane) Get(i interface{}) bool { 86 | for { 87 | select { 88 | case curs := <-m.getCh: 89 | r := m.lanes[curs&m.laneMask].Get(i) 90 | m.getCh <- curs 91 | return r 92 | default: 93 | runtime.Gosched() 94 | } 95 | } 96 | } 97 | 98 | // Put given variable at one of the lanes pointed by the cursor, 99 | // blocking. 100 | func (m *MultiLane) Put(i interface{}) { 101 | for { 102 | select { 103 | case curs := <-m.putCh: 104 | m.lanes[curs&m.laneMask].Put(i) 105 | m.putCh <- curs 106 | return 107 | default: 108 | runtime.Gosched() 109 | } 110 | } 111 | } 112 | 113 | // Close the MultiLane, it shall NOT be called before `Put()`. 114 | func (m *MultiLane) Close() { 115 | for _, l := range m.lanes { 116 | l.Close() 117 | } 118 | } 119 | 120 | // LaneWidth is the number of lanes. 121 | func (m *MultiLane) LaneWidth() uint32 { 122 | return uint32(m.laneMask + 1) 123 | } 124 | 125 | // QueueSize is the size of the underlying queue. 126 | func (m *MultiLane) QueueSize() uint32 { 127 | return m.queSize 128 | } 129 | 130 | func nextPowerOf2(v uint32) uint32 { 131 | v-- 132 | v |= v >> 1 133 | v |= v >> 2 134 | v |= v >> 4 135 | v |= v >> 8 136 | v |= v >> 16 137 | return v + 1 138 | } 139 | -------------------------------------------------------------------------------- /pkg/ringhash/ringhash_test.go: -------------------------------------------------------------------------------- 1 | package ringhash 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "testing" 7 | ) 8 | 9 | func TestAddNode(t *testing.T) { 10 | r := New() 11 | 12 | r.AddNode("127.0.0.1:80") 13 | if len(r.hashes) != r.replicas { 14 | t.Fatal("wrong vnodes number") 15 | } 16 | } 17 | 18 | func TestGetNode(t *testing.T) { 19 | r := New() 20 | 21 | r.AddNode("127.0.0.1:80") 22 | node, err := r.GetNode("127.0.0.1:80") 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | 27 | if node != "127.0.0.1:80" { 28 | t.Fatalf("wrong node, expected 127.0.0.1:80, got %v\n", node) 29 | } 30 | } 31 | 32 | func TestRemoveNode(t *testing.T) { 33 | r := New() 34 | 35 | r.AddNode("127.0.0.1:80") 36 | r.RemoveNode("127.0.0.1:80") 37 | 38 | if len(r.hashes) != 0 && len(r.hashKeyMap) != 0 { 39 | t.Fatal(("remove not working")) 40 | } 41 | } 42 | 43 | func TestGetLeastNode(t *testing.T) { 44 | option := func(c *Config) error { 45 | c.BalancingFactor = 1.02 46 | return nil 47 | } 48 | r := New(option) 49 | 50 | r.AddNode("127.0.0.1:80") 51 | r.AddNode("192.168.0.1:80") 52 | r.AddNode("10.0.0.1:80") 53 | 54 | for i := 0; i < 100; i++ { 55 | node, err := r.GetLeastNode("192.168.0.1:81") 56 | if err != nil { 57 | t.Fatal(err) 58 | } 59 | r.Add(node) 60 | } 61 | 62 | for k, v := range r.Loads() { 63 | if v > r.MaxLoad() { 64 | t.Fatalf("node %s is overloaded, %d > %d\n", k, v, r.MaxLoad()) 65 | } 66 | } 67 | fmt.Println("Max load per node ->", r.MaxLoad()) 68 | fmt.Println(r.Loads()) 69 | } 70 | 71 | func TestAddDone(t *testing.T) { 72 | r := New() 73 | 74 | r.AddNode("127.0.0.1:80") 75 | r.AddNode("192.168.0.1:80") 76 | 77 | node, err := r.GetLeastNode("192.168.0.1:81") 78 | if err != nil { 79 | t.Fatal(err) 80 | } 81 | 82 | r.Add(node) 83 | if r.keyLoadMap[node].Load != 1 { 84 | t.Fatalf("load for node %s should be 1\n", node) 85 | } 86 | 87 | r.Done(node) 88 | if r.keyLoadMap[node].Load != 0 { 89 | t.Fatalf("load for node %s should be 0\n", node) 90 | } 91 | } 92 | 93 | func BenchmarkGetNode(b *testing.B) { 94 | r := New() 95 | for i := 0; i < 10; i++ { 96 | r.AddNode("start" + strconv.Itoa(i)) 97 | } 98 | tt := []struct { 99 | key string 100 | }{ 101 | {"test"}, 102 | {"test1"}, 103 | {"test2"}, 104 | {"test3"}, 105 | {"test4"}, 106 | {"test5"}, 107 | } 108 | b.ResetTimer() 109 | for i := 0; i < b.N; i++ { 110 | t := tt[i%len(tt)] 111 | r.GetNode(t.key) 112 | } 113 | } 114 | 115 | func BenchmarkGetLeastNode(b *testing.B) { 116 | r := New() 117 | for i := 0; i < 10; i++ { 118 | r.AddNode("start" + strconv.Itoa(i)) 119 | } 120 | tt := []struct { 121 | key string 122 | }{ 123 | {"test"}, 124 | {"test1"}, 125 | {"test2"}, 126 | {"test3"}, 127 | {"test4"}, 128 | {"test5"}, 129 | } 130 | b.ResetTimer() 131 | for i := 0; i < b.N; i++ { 132 | t := tt[i%len(tt)] 133 | r.GetLeastNode(t.key) 134 | } 135 | } 136 | 137 | func BenchmarkAddRemoveNode(b *testing.B) { 138 | r := New() 139 | for i := 0; i < 10; i++ { 140 | r.AddNode("start" + strconv.Itoa(i)) 141 | } 142 | b.ResetTimer() 143 | for i := 0; i < b.N; i++ { 144 | r.AddNode("foo" + strconv.Itoa(i)) 145 | r.RemoveNode("foo" + strconv.Itoa(i)) 146 | } 147 | } 148 | 149 | func BenchmarkAddDone(b *testing.B) { 150 | r := New() 151 | for i := 0; i < 10; i++ { 152 | r.AddNode("start" + strconv.Itoa(i)) 153 | } 154 | b.ResetTimer() 155 | for i := 0; i < b.N; i++ { 156 | node, _ := r.GetLeastNode("start" + strconv.Itoa(i)) 157 | r.Add(node) 158 | r.Done(node) 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /pkg/bloom/bloomscale_test.go: -------------------------------------------------------------------------------- 1 | package bloom_test 2 | 3 | import ( 4 | "encoding/binary" 5 | "testing" 6 | 7 | "github.com/andy2046/gopie/pkg/bloom" 8 | ) 9 | 10 | func TestScaleBasic(t *testing.T) { 11 | f := bloom.NewS(0.01) 12 | e1 := []byte("Boss") 13 | e2 := []byte("Joke") 14 | e3 := []byte("Emotion") 15 | f.Add(e1) 16 | e3b := f.Exist(e3) 17 | e1a := f.Exist(e1) 18 | e2a := f.Exist(e2) 19 | f.Add(e3) 20 | e3a := f.Exist(e3) 21 | if !e1a { 22 | t.Errorf("%q should Exist.", e1) 23 | } 24 | if e2a { 25 | t.Errorf("%q should not Exist.", e2) 26 | } 27 | if e3b { 28 | t.Errorf("%q should not Exist the first time we check.", e3) 29 | } 30 | if !e3a { 31 | t.Errorf("%q should Exist the second time we check.", e3) 32 | } 33 | } 34 | 35 | func TestScaleUint(t *testing.T) { 36 | f := bloom.NewS(0.01) 37 | n1 := make([]byte, 4) 38 | n2 := make([]byte, 4) 39 | n3 := make([]byte, 4) 40 | n4 := make([]byte, 4) 41 | binary.BigEndian.PutUint32(n1, 10000) 42 | binary.BigEndian.PutUint32(n2, 10001) 43 | binary.BigEndian.PutUint32(n3, 10002) 44 | binary.BigEndian.PutUint32(n4, 10003) 45 | f.Add(n1) 46 | n3b := f.Exist(n3) 47 | n1a := f.Exist(n1) 48 | n2a := f.Exist(n2) 49 | f.Add(n3) 50 | n3a := f.Exist(n3) 51 | n4a := f.Exist(n4) 52 | if !n1a { 53 | t.Errorf("%q should Exist.", n1) 54 | } 55 | if n2a { 56 | t.Errorf("%q should not Exist.", n2) 57 | } 58 | if n3b { 59 | t.Errorf("%q should not Exist the first time we check.", n3) 60 | } 61 | if !n3a { 62 | t.Errorf("%q should Exist the second time we check.", n3) 63 | } 64 | if n4a { 65 | t.Errorf("%q should not Exist.", n4) 66 | } 67 | 68 | for i := uint32(1); i < 1000; i++ { 69 | buf := make([]byte, 4) 70 | binary.BigEndian.PutUint32(buf, i) 71 | f.Add(buf) 72 | } 73 | count := 0 74 | for i := uint32(1000); i < 4000; i++ { 75 | buf := make([]byte, 4) 76 | binary.BigEndian.PutUint32(buf, i) 77 | if f.Exist(buf) { 78 | count++ 79 | } 80 | } 81 | t.Logf("FP rate is %v", f.FalsePositive()) 82 | if f.FalsePositive() > 0.01 { 83 | t.Errorf("False Positive rate should not be > 0.01") 84 | } 85 | 86 | if count > 1 { 87 | t.Errorf("Actual FP %d greater than expected FP", count) 88 | } 89 | } 90 | 91 | func TestScaleString(t *testing.T) { 92 | f := bloom.NewS(0.01) 93 | s1 := "Filter" 94 | s2 := "is" 95 | s3 := "in" 96 | s4 := "bloom" 97 | f.AddString(s1) 98 | s3b := f.ExistString(s3) 99 | s1a := f.ExistString(s1) 100 | s2a := f.ExistString(s2) 101 | f.AddString(s3) 102 | s3a := f.ExistString(s3) 103 | s4a := f.ExistString(s4) 104 | if !s1a { 105 | t.Errorf("%q should Exist.", s1) 106 | } 107 | if s2a { 108 | t.Errorf("%q should not Exist.", s2) 109 | } 110 | if s3b { 111 | t.Errorf("%q should not Exist the first time we check.", s3) 112 | } 113 | if !s3a { 114 | t.Errorf("%q should Exist the second time we check.", s3) 115 | } 116 | if s4a { 117 | t.Errorf("%q should not Exist.", s4) 118 | } 119 | } 120 | 121 | func TestScaleM(t *testing.T) { 122 | f := bloom.NewS(0.01) 123 | if f.M() < 512 { 124 | t.Errorf("M() %v is not correct", f.M()) 125 | } 126 | } 127 | 128 | func TestScaleK(t *testing.T) { 129 | f := bloom.NewS(0.01) 130 | if f.K() < 3 { 131 | t.Errorf("K() %v is not correct", f.K()) 132 | } 133 | } 134 | 135 | func BenchmarkScaleAddExist(b *testing.B) { 136 | f := bloom.NewSGuess(uint64(b.N), 0.0001, 0.9) 137 | key := make([]byte, 8) 138 | b.ResetTimer() 139 | for i := 0; i < b.N; i++ { 140 | binary.BigEndian.PutUint64(key, uint64(i)) 141 | f.Add(key) 142 | f.Exist(key) 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /pkg/bloom/bloom_test.go: -------------------------------------------------------------------------------- 1 | package bloom_test 2 | 3 | import ( 4 | "encoding/binary" 5 | "testing" 6 | 7 | "github.com/andy2046/gopie/pkg/bloom" 8 | ) 9 | 10 | func TestBasic(t *testing.T) { 11 | f := bloom.New(1000, 4) 12 | e1 := []byte("Boss") 13 | e2 := []byte("Joke") 14 | e3 := []byte("Emotion") 15 | f.Add(e1) 16 | e3b := f.Exist(e3) 17 | e1a := f.Exist(e1) 18 | e2a := f.Exist(e2) 19 | f.Add(e3) 20 | e3a := f.Exist(e3) 21 | if !e1a { 22 | t.Errorf("%q should Exist.", e1) 23 | } 24 | if e2a { 25 | t.Errorf("%q should not Exist.", e2) 26 | } 27 | if e3b { 28 | t.Errorf("%q should not Exist the first time we check.", e3) 29 | } 30 | if !e3a { 31 | t.Errorf("%q should Exist the second time we check.", e3) 32 | } 33 | f.Remove(e1) 34 | e1a = f.Exist(e1) 35 | if e1a { 36 | t.Errorf("%q should be Removed.", e1) 37 | } 38 | } 39 | 40 | func TestUint(t *testing.T) { 41 | f := bloom.New(1000, 4) 42 | n1 := make([]byte, 4) 43 | n2 := make([]byte, 4) 44 | n3 := make([]byte, 4) 45 | n4 := make([]byte, 4) 46 | binary.BigEndian.PutUint32(n1, 100) 47 | binary.BigEndian.PutUint32(n2, 101) 48 | binary.BigEndian.PutUint32(n3, 102) 49 | binary.BigEndian.PutUint32(n4, 103) 50 | f.Add(n1) 51 | n3b := f.Exist(n3) 52 | n1a := f.Exist(n1) 53 | n2a := f.Exist(n2) 54 | f.Add(n3) 55 | n3a := f.Exist(n3) 56 | n4a := f.Exist(n4) 57 | if !n1a { 58 | t.Errorf("%q should Exist.", n1) 59 | } 60 | if n2a { 61 | t.Errorf("%q should not Exist.", n2) 62 | } 63 | if n3b { 64 | t.Errorf("%q should not Exist the first time we check.", n3) 65 | } 66 | if !n3a { 67 | t.Errorf("%q should Exist the second time we check.", n3) 68 | } 69 | if n4a { 70 | t.Errorf("%q should not Exist.", n4) 71 | } 72 | f.Remove(n1) 73 | n1a = f.Exist(n1) 74 | if n1a { 75 | t.Errorf("%q should be Removed.", n1) 76 | } 77 | } 78 | 79 | func TestString(t *testing.T) { 80 | f := bloom.NewGuess(1000, 0.001) 81 | s1 := "Filter" 82 | s2 := "is" 83 | s3 := "in" 84 | s4 := "bloom" 85 | f.AddString(s1) 86 | s3b := f.ExistString(s3) 87 | s1a := f.ExistString(s1) 88 | s2a := f.ExistString(s2) 89 | f.AddString(s3) 90 | s3a := f.ExistString(s3) 91 | s4a := f.ExistString(s4) 92 | if !s1a { 93 | t.Errorf("%q should Exist.", s1) 94 | } 95 | if s2a { 96 | t.Errorf("%q should not Exist.", s2) 97 | } 98 | if s3b { 99 | t.Errorf("%q should not Exist the first time we check.", s3) 100 | } 101 | if !s3a { 102 | t.Errorf("%q should Exist the second time we check.", s3) 103 | } 104 | if s4a { 105 | t.Errorf("%q should not Exist.", s4) 106 | } 107 | f.RemoveString(s1) 108 | s1a = f.ExistString(s1) 109 | if s1a { 110 | t.Errorf("%q should be Removed.", s1) 111 | } 112 | } 113 | 114 | func TestGuessFalsePositive(t *testing.T) { 115 | n, p := uint64(100000), float64(0.001) 116 | m, k := bloom.Guess(n, p) 117 | f := bloom.NewGuess(n, p) 118 | fp := f.GuessFalsePositive(n) 119 | t.Logf("m=%v k=%v n=%v p=%v fp=%v", m, k, n, p, fp) 120 | if fp > p { 121 | t.Errorf("False Positive too high") 122 | } 123 | } 124 | 125 | func TestM(t *testing.T) { 126 | f := bloom.New(1000, 4) 127 | if f.M() != 1024 { 128 | t.Errorf("M() %v is not correct", f.M()) 129 | } 130 | } 131 | 132 | func TestK(t *testing.T) { 133 | f := bloom.New(1000, 4) 134 | if f.K() != 4 { 135 | t.Errorf("K() %v is not correct", f.K()) 136 | } 137 | } 138 | 139 | func BenchmarkAddExist(b *testing.B) { 140 | f := bloom.NewGuess(uint64(b.N), 0.0001) 141 | key := make([]byte, 8) 142 | b.ResetTimer() 143 | for i := 0; i < b.N; i++ { 144 | binary.BigEndian.PutUint64(key, uint64(i)) 145 | f.Add(key) 146 | f.Exist(key) 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /docs/bitflag.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # bitflag 4 | `import "github.com/andy2046/gopie/pkg/bitflag"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package bitflag implements bit flag. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [type Flag](#Flag) 17 | * [func (f Flag) AreAllSet(opts ...Flag) bool](#Flag.AreAllSet) 18 | * [func (f *Flag) Clear(n uint8)](#Flag.Clear) 19 | * [func (f *Flag) ClearAll(opts ...Flag)](#Flag.ClearAll) 20 | * [func (f Flag) IsAnySet(opts ...Flag) bool](#Flag.IsAnySet) 21 | * [func (f Flag) IsSet(n uint8) bool](#Flag.IsSet) 22 | * [func (f *Flag) Reset()](#Flag.Reset) 23 | * [func (f *Flag) Set(n uint8)](#Flag.Set) 24 | * [func (f *Flag) SetAll(opts ...Flag)](#Flag.SetAll) 25 | * [func (f *Flag) Toggle(n uint8)](#Flag.Toggle) 26 | * [func (f *Flag) ToggleAll(opts ...Flag)](#Flag.ToggleAll) 27 | 28 | 29 | #### Package files 30 | [bitflag.go](/src/github.com/andy2046/gopie/pkg/bitflag/bitflag.go) 31 | 32 | 33 | 34 | 35 | 36 | 37 | ## type [Flag](/src/target/bitflag.go?s=84:98#L5) 38 | ``` go 39 | type Flag byte 40 | ``` 41 | Flag is an 8 bits flag. 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | ### func (Flag) [AreAllSet](/src/target/bitflag.go?s=515:557#L31) 53 | ``` go 54 | func (f Flag) AreAllSet(opts ...Flag) bool 55 | ``` 56 | AreAllSet check if all the flags are set. 57 | 58 | 59 | 60 | 61 | ### func (\*Flag) [Clear](/src/target/bitflag.go?s=1342:1371#L79) 62 | ``` go 63 | func (f *Flag) Clear(n uint8) 64 | ``` 65 | Clear clear a single bit at `n`. 66 | `n` should be less than `8`. 67 | 68 | 69 | 70 | 71 | ### func (\*Flag) [ClearAll](/src/target/bitflag.go?s=387:424#L24) 72 | ``` go 73 | func (f *Flag) ClearAll(opts ...Flag) 74 | ``` 75 | ClearAll clear all the flags. 76 | 77 | 78 | 79 | 80 | ### func (Flag) [IsAnySet](/src/target/bitflag.go?s=683:724#L41) 81 | ``` go 82 | func (f Flag) IsAnySet(opts ...Flag) bool 83 | ``` 84 | IsAnySet check if any one flag is set. 85 | 86 | 87 | 88 | 89 | ### func (Flag) [IsSet](/src/target/bitflag.go?s=880:913#L52) 90 | ``` go 91 | func (f Flag) IsSet(n uint8) bool 92 | ``` 93 | IsSet check if the bit at `n` is set. 94 | `n` should be less than `8`. 95 | 96 | 97 | 98 | 99 | ### func (\*Flag) [Reset](/src/target/bitflag.go?s=1445:1467#L87) 100 | ``` go 101 | func (f *Flag) Reset() 102 | ``` 103 | Reset reset the flag. 104 | 105 | 106 | 107 | 108 | ### func (\*Flag) [Set](/src/target/bitflag.go?s=1045:1072#L61) 109 | ``` go 110 | func (f *Flag) Set(n uint8) 111 | ``` 112 | Set set a single bit at `n`. 113 | `n` should be less than `8`. 114 | 115 | 116 | 117 | 118 | ### func (\*Flag) [SetAll](/src/target/bitflag.go?s=150:185#L10) 119 | ``` go 120 | func (f *Flag) SetAll(opts ...Flag) 121 | ``` 122 | SetAll set all the flags. 123 | 124 | 125 | 126 | 127 | ### func (\*Flag) [Toggle](/src/target/bitflag.go?s=1196:1226#L70) 128 | ``` go 129 | func (f *Flag) Toggle(n uint8) 130 | ``` 131 | Toggle toggle (XOR) a single bit at `n`. 132 | `n` should be less than `8`. 133 | 134 | 135 | 136 | 137 | ### func (\*Flag) [ToggleAll](/src/target/bitflag.go?s=271:309#L17) 138 | ``` go 139 | func (f *Flag) ToggleAll(opts ...Flag) 140 | ``` 141 | ToggleAll toggle (XOR) all the flags. 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | - - - 151 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 152 | -------------------------------------------------------------------------------- /pkg/bloom/bloomscale.go: -------------------------------------------------------------------------------- 1 | package bloom 2 | 3 | import ( 4 | "math" 5 | 6 | "github.com/andy2046/bitmap" 7 | ) 8 | 9 | type ( 10 | scalableBloomFilter struct { 11 | filterz []*bloomFilterBit // bloom filters list 12 | count uint64 // number of elements in the bloom filter 13 | n uint64 // estimated number of elements 14 | p float64 // target False Positive rate 15 | r float64 // optimal tightening ratio 16 | fillRatio float64 // fill ratio 17 | } 18 | ) 19 | 20 | const ( 21 | rDefault float64 = 0.8 22 | fillRatio float64 = 0.5 23 | ) 24 | 25 | // NewS creates scalable bloom filter based on the provided fpRate. 26 | // fpRate is the target False Positive probability. 27 | func NewS(fpRate float64) Bloom { 28 | return NewSGuess(10000, fpRate, rDefault) 29 | } 30 | 31 | // NewSGuess estimates m/k based on the provided n/p then creates scalable bloom filter. 32 | // n is the estimated number of elements in the bloom filter. 33 | // p is the false positive probability. 34 | // r is the optimal tightening ratio. 35 | func NewSGuess(n uint64, p, r float64) Bloom { 36 | m, k := Guess(n, p) 37 | mm, exponent := adjustM(m) 38 | 39 | sBF := scalableBloomFilter{ 40 | filterz: make([]*bloomFilterBit, 0, 1), 41 | r: r, 42 | fillRatio: fillRatio, 43 | p: p, 44 | n: n, 45 | } 46 | 47 | sBF.filterz = append(sBF.filterz, &bloomFilterBit{ 48 | bitmap: bitmap.New(mm), 49 | m: mm - 1, // x % 2^i = x & (2^i - 1) 50 | k: k, 51 | shift: 64 - exponent, 52 | }) 53 | return &sBF 54 | } 55 | 56 | func (bf *scalableBloomFilter) Add(entry []byte) { 57 | idx := len(bf.filterz) - 1 58 | if bf.filterz[idx].estimatedFillRatio() >= bf.fillRatio { 59 | fp := bf.p * math.Pow(bf.r, float64(len(bf.filterz))) 60 | m, k := Guess(bf.n, fp) 61 | mm, exponent := adjustM(m) 62 | bf.filterz = append(bf.filterz, &bloomFilterBit{ 63 | bitmap: bitmap.New(mm), 64 | m: mm - 1, // x % 2^i = x & (2^i - 1) 65 | k: k, 66 | shift: 64 - exponent, 67 | }) 68 | idx++ 69 | } 70 | bf.filterz[idx].Add(entry) 71 | bf.count++ 72 | } 73 | 74 | func (bf *scalableBloomFilter) AddString(entry string) { 75 | bf.Add([]byte(entry)) 76 | } 77 | 78 | func (bf *scalableBloomFilter) Exist(entry []byte) bool { 79 | for _, f := range bf.filterz { 80 | if f.Exist(entry) { 81 | return true 82 | } 83 | } 84 | return false 85 | } 86 | 87 | func (bf *scalableBloomFilter) ExistString(entry string) bool { 88 | return bf.Exist([]byte(entry)) 89 | } 90 | 91 | func (bf *scalableBloomFilter) FalsePositive() float64 { 92 | rez := 1.0 93 | for _, f := range bf.filterz { 94 | rez *= (1.0 - f.FalsePositive()) 95 | } 96 | return 1.0 - rez 97 | } 98 | 99 | func (bf *scalableBloomFilter) GuessFalsePositive(n uint64) float64 { 100 | rez := 1.0 101 | for _, f := range bf.filterz { 102 | rez *= (1.0 - f.GuessFalsePositive(n)) 103 | } 104 | return 1.0 - rez 105 | } 106 | 107 | func (bf *scalableBloomFilter) M() uint64 { 108 | m := uint64(0) 109 | for _, f := range bf.filterz { 110 | m += f.M() 111 | } 112 | return m 113 | } 114 | 115 | func (bf *scalableBloomFilter) K() uint64 { 116 | return bf.filterz[0].K() 117 | } 118 | 119 | func (bf *scalableBloomFilter) N() uint64 { 120 | return bf.count 121 | } 122 | 123 | func (bf *scalableBloomFilter) Clear() { 124 | for i := range bf.filterz { 125 | bf.filterz[i] = nil 126 | } 127 | bf.filterz = make([]*bloomFilterBit, 0, 1) 128 | m, k := Guess(bf.n, bf.p) 129 | mm, exponent := adjustM(m) 130 | bf.filterz = append(bf.filterz, &bloomFilterBit{ 131 | bitmap: bitmap.New(mm), 132 | m: mm - 1, // x % 2^i = x & (2^i - 1) 133 | k: k, 134 | shift: 64 - exponent, 135 | }) 136 | bf.count = 0 137 | } 138 | -------------------------------------------------------------------------------- /pkg/rwspinlock/rwspinlock.go: -------------------------------------------------------------------------------- 1 | // Package rwspinlock implements RWSpinLock. 2 | package rwspinlock 3 | 4 | import ( 5 | "runtime" 6 | "sync" 7 | "sync/atomic" 8 | ) 9 | 10 | // RWLocker is the RWSpinLock implementation. 11 | type RWLocker struct { 12 | noCopy noCopy 13 | // the writer bit is placed on the MSB, allowing 2^31 readers. 14 | // using recursion for the readers lock should be done with great caution, 15 | // if the lock is acquired for a thread and another thread acquire the writer lock, 16 | // reader recursion will lead to a deadlock. 17 | lock uint32 18 | } 19 | 20 | // New creates a RWLocker. 21 | func New() *RWLocker { 22 | return &RWLocker{} 23 | } 24 | 25 | // RLock locks rw for reading. 26 | func (rw *RWLocker) RLock() { 27 | for { 28 | // wait until there is no active writer. 29 | for atomic.LoadUint32(&rw.lock)&0x80000000 != 0 { 30 | runtime.Gosched() 31 | } 32 | 33 | o := atomic.LoadUint32(&rw.lock) & 0x7fffffff 34 | if atomic.CompareAndSwapUint32(&rw.lock, o, o+1) { 35 | return 36 | } 37 | } 38 | } 39 | 40 | // RUnlock undoes a single RLock call. 41 | func (rw *RWLocker) RUnlock() { 42 | atomic.AddUint32(&rw.lock, ^uint32(0)) 43 | } 44 | 45 | // TryRLock try to locks rw for reading, 46 | // it returns true if succeed, false otherwise. 47 | func (rw *RWLocker) TryRLock() bool { 48 | if atomic.LoadUint32(&rw.lock)&0x80000000 != 0 { 49 | return false 50 | } 51 | 52 | o := atomic.LoadUint32(&rw.lock) & 0x7fffffff 53 | return atomic.CompareAndSwapUint32(&rw.lock, o, o+1) 54 | } 55 | 56 | // Lock locks rw for writing. 57 | func (rw *RWLocker) Lock() { 58 | for { 59 | // wait until there is no active writer. 60 | for atomic.LoadUint32(&rw.lock)&0x80000000 != 0 { 61 | runtime.Gosched() 62 | } 63 | 64 | o := atomic.LoadUint32(&rw.lock) & 0x7fffffff 65 | n := o | 0x80000000 66 | 67 | if atomic.CompareAndSwapUint32(&rw.lock, o, n) { 68 | // wait for active readers to release locks. 69 | for atomic.LoadUint32(&rw.lock)&0x7fffffff != 0 { 70 | runtime.Gosched() 71 | } 72 | return 73 | } 74 | } 75 | } 76 | 77 | // Unlock unlocks rw for writing. 78 | func (rw *RWLocker) Unlock() { 79 | if atomic.LoadUint32(&rw.lock) != 0x80000000 { 80 | panic("Unlock") 81 | } 82 | atomic.StoreUint32(&rw.lock, 0) 83 | } 84 | 85 | // TryLock try to locks rw for writing, 86 | // it returns true if succeed, false otherwise. 87 | func (rw *RWLocker) TryLock() bool { 88 | if atomic.LoadUint32(&rw.lock)&0x80000000 != 0 { 89 | return false 90 | } 91 | 92 | o := atomic.LoadUint32(&rw.lock) & 0x7fffffff 93 | n := o | 0x80000000 94 | 95 | if !atomic.CompareAndSwapUint32(&rw.lock, o, n) { 96 | return false 97 | } 98 | 99 | // wait for active readers to release locks. 100 | for atomic.LoadUint32(&rw.lock)&0x7fffffff != 0 { 101 | runtime.Gosched() 102 | } 103 | return true 104 | } 105 | 106 | // IsLocked returns true if there is active writer, false otherwise. 107 | func (rw *RWLocker) IsLocked() bool { 108 | return atomic.LoadUint32(&rw.lock)&0x80000000 != 0 109 | } 110 | 111 | // IsRLocked returns true if there is active reader, false otherwise. 112 | func (rw *RWLocker) IsRLocked() bool { 113 | return atomic.LoadUint32(&rw.lock)&0x7fffffff != 0 114 | } 115 | 116 | // RLocker returns a sync.Locker interface implementation by calling rw.RLock and rw.RUnlock. 117 | func (rw *RWLocker) RLocker() sync.Locker { 118 | return (*rlocker)(rw) 119 | } 120 | 121 | type rlocker RWLocker 122 | 123 | func (r *rlocker) Lock() { (*RWLocker)(r).RLock() } 124 | func (r *rlocker) Unlock() { (*RWLocker)(r).RUnlock() } 125 | 126 | // noCopy may be embedded into structs which must not be copied 127 | // after the first use. 128 | type noCopy struct{} 129 | 130 | // Lock is a no-op used by -copylocks checker from `go vet`. 131 | func (*noCopy) Lock() {} 132 | func (*noCopy) Unlock() {} 133 | -------------------------------------------------------------------------------- /docs/tlv.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # tlv 4 | `import "github.com/andy2046/gopie/pkg/tlv"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package tlv implements Type-Length-Value encoding. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [type ByteSize](#ByteSize) 17 | * [type Codec](#Codec) 18 | * [type Reader](#Reader) 19 | * [func NewReader(reader io.Reader, codec *Codec) *Reader](#NewReader) 20 | * [func (r *Reader) Next() (*Record, error)](#Reader.Next) 21 | * [type Record](#Record) 22 | * [type Writer](#Writer) 23 | * [func NewWriter(w io.Writer, codec *Codec) *Writer](#NewWriter) 24 | * [func (w *Writer) Write(rec *Record) error](#Writer.Write) 25 | 26 | 27 | #### Package files 28 | [tlv.go](/src/github.com/andy2046/gopie/pkg/tlv/tlv.go) 29 | 30 | 31 | 32 | 33 | 34 | 35 | ## type [ByteSize](/src/target/tlv.go?s=321:333#L21) 36 | ``` go 37 | type ByteSize int 38 | ``` 39 | ByteSize is the size of a field in bytes. 40 | Used to define the size of the type and length field in a message. 41 | 42 | 43 | ``` go 44 | const ( 45 | Bytes1 ByteSize = 1 << iota 46 | Bytes2 47 | Bytes4 48 | Bytes8 49 | ) 50 | ``` 51 | ByteSize const. 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | ## type [Codec](/src/target/tlv.go?s=521:717#L30) 63 | ``` go 64 | type Codec struct { 65 | // TypeBytes defines the size in bytes of the message type field. 66 | TypeBytes ByteSize 67 | 68 | // LenBytes defines the size in bytes of the message length field. 69 | LenBytes ByteSize 70 | } 71 | ``` 72 | Codec is the configuration for a TLV encoding/decoding task. 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | ## type [Reader](/src/target/tlv.go?s=959:1012#L45) 84 | ``` go 85 | type Reader struct { 86 | // contains filtered or unexported fields 87 | } 88 | ``` 89 | Reader decodes records from TLV format using a Codec from the provided io.Reader. 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | ### func [NewReader](/src/target/tlv.go?s=1863:1917#L92) 98 | ``` go 99 | func NewReader(reader io.Reader, codec *Codec) *Reader 100 | ``` 101 | NewReader creates a new Reader. 102 | 103 | 104 | 105 | 106 | 107 | ### func (\*Reader) [Next](/src/target/tlv.go?s=2019:2059#L97) 108 | ``` go 109 | func (r *Reader) Next() (*Record, error) 110 | ``` 111 | Next reads a single Record from the io.Reader. 112 | 113 | 114 | 115 | 116 | ## type [Record](/src/target/tlv.go?s=403:453#L24) 117 | ``` go 118 | type Record struct { 119 | Payload []byte 120 | Type uint 121 | } 122 | ``` 123 | Record represents a record of data encoded in the TLV message. 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | ## type [Writer](/src/target/tlv.go?s=817:870#L39) 135 | ``` go 136 | type Writer struct { 137 | // contains filtered or unexported fields 138 | } 139 | ``` 140 | Writer encodes records into TLV format using a Codec and writes into the provided io.Writer. 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | ### func [NewWriter](/src/target/tlv.go?s=1051:1100#L52) 149 | ``` go 150 | func NewWriter(w io.Writer, codec *Codec) *Writer 151 | ``` 152 | NewWriter creates a new Writer. 153 | 154 | 155 | 156 | 157 | 158 | ### func (\*Writer) [Write](/src/target/tlv.go?s=1251:1292#L60) 159 | ``` go 160 | func (w *Writer) Write(rec *Record) error 161 | ``` 162 | Write encodes records into TLV format using a Codec and writes into the provided io.Writer. 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | - - - 172 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 173 | -------------------------------------------------------------------------------- /pkg/backoff/backoff.go: -------------------------------------------------------------------------------- 1 | // Package backoff implements Backoff pattern. 2 | // http://www.awsarchitectureblog.com/2015/03/backoff.html 3 | package backoff 4 | 5 | import ( 6 | crypto_rand "crypto/rand" 7 | "encoding/binary" 8 | "math" 9 | "math/rand" 10 | "time" 11 | ) 12 | 13 | type ( 14 | // Strategy type. 15 | Strategy string 16 | 17 | // Backoff holds the backoff logic. 18 | Backoff struct { 19 | strategy iStrategy 20 | } 21 | 22 | // iStrategy defines the methodology for backing off. 23 | iStrategy interface { 24 | // Backoff returns the amount of time to wait before the next retry. 25 | Backoff(retries int) time.Duration 26 | } 27 | 28 | // exponential Strategy. 29 | exponential struct { 30 | base 31 | } 32 | 33 | // fullJitter Strategy. 34 | fullJitter struct { 35 | base 36 | } 37 | 38 | // equalJitter Strategy. 39 | equalJitter struct { 40 | base 41 | } 42 | 43 | // decorrelatedJitter Strategy. 44 | decorrelatedJitter struct { 45 | base 46 | sleep time.Duration 47 | } 48 | 49 | base struct { 50 | // capDelay is the upper bound of backoff delay. 51 | capDelay time.Duration 52 | // baseDelay is the initial backoff delay. 53 | baseDelay time.Duration 54 | } 55 | ) 56 | 57 | // predefined Strategy types. 58 | const ( 59 | Exponential Strategy = "Exponential" 60 | FullJitter Strategy = "FullJitter" 61 | EqualJitter Strategy = "EqualJitter" 62 | DecorrelatedJitter Strategy = "DecorrelatedJitter" 63 | ) 64 | 65 | func init() { 66 | var b [8]byte 67 | _, err := crypto_rand.Read(b[:]) 68 | if err != nil { 69 | panic("fail to seed math/rand pkg with crypto/rand random number generator") 70 | } 71 | rand.Seed(int64(binary.LittleEndian.Uint64(b[:]))) 72 | } 73 | 74 | // New returns a Backoff instance with provided delay and strategy. 75 | func New(baseDelay, capDelay time.Duration, strategy Strategy) *Backoff { 76 | if capDelay < baseDelay { 77 | panic("capDelay must be greater than baseDelay") 78 | } 79 | 80 | var ( 81 | b = base{baseDelay: baseDelay, capDelay: capDelay} 82 | s iStrategy 83 | ) 84 | 85 | switch strategy { 86 | case Exponential: 87 | s = &exponential{b} 88 | case FullJitter: 89 | s = &fullJitter{b} 90 | case EqualJitter: 91 | s = &equalJitter{b} 92 | case DecorrelatedJitter: 93 | s = &decorrelatedJitter{base: b, sleep: baseDelay} 94 | default: 95 | panic("unknown strategy " + strategy) 96 | } 97 | 98 | bk := &Backoff{s} 99 | return bk 100 | } 101 | 102 | // Backoff returns the amount of time to wait before the next retry. 103 | // `retries` starts from zero. 104 | func (bk *Backoff) Backoff(retries int) time.Duration { 105 | return bk.strategy.Backoff(retries) 106 | } 107 | 108 | func (b base) expo(retries int) time.Duration { 109 | // min(cap, pow(2, n)*base) 110 | pow2n := math.Pow(2, float64(retries)) 111 | n := time.Duration(pow2n) * b.baseDelay 112 | return minDuration(b.capDelay, n) 113 | } 114 | 115 | func (ex *exponential) Backoff(retries int) time.Duration { 116 | return ex.expo(retries) 117 | } 118 | 119 | func (f *fullJitter) Backoff(retries int) time.Duration { 120 | n := f.expo(retries) 121 | return time.Duration(rand.Int63n(int64(n) + 1)) 122 | } 123 | 124 | func (e *equalJitter) Backoff(retries int) time.Duration { 125 | n := e.expo(retries) 126 | return n/2 + time.Duration(rand.Int63n(1+int64(n)/2)) 127 | } 128 | 129 | func (d *decorrelatedJitter) Backoff(retries int) time.Duration { 130 | if retries == 0 { 131 | d.sleep = d.baseDelay 132 | } 133 | 134 | // sleep = min(cap, random.uniform(base, sleep * 3)) 135 | base, sleep3 := int64(d.baseDelay), int64(d.sleep)*3 136 | n := base + rand.Int63n(sleep3-base+1) 137 | d.sleep = minDuration(d.capDelay, time.Duration(n)) 138 | return d.sleep 139 | } 140 | 141 | func minDuration(x, y time.Duration) time.Duration { 142 | if x < y { 143 | return x 144 | } 145 | return y 146 | } 147 | -------------------------------------------------------------------------------- /pkg/spsc/spsc_test.go: -------------------------------------------------------------------------------- 1 | package spsc_test 2 | 3 | import ( 4 | . "github.com/andy2046/gopie/pkg/spsc" 5 | "runtime" 6 | "sync" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func BenchmarkSPSC_NonBlocking(b *testing.B) { 12 | var sp = New(8192) 13 | var wg sync.WaitGroup 14 | wg.Add(2) 15 | type T struct { 16 | i int 17 | } 18 | var v = T{5} 19 | b.ResetTimer() 20 | go func(n int) { 21 | runtime.LockOSThread() 22 | for i := 0; i < n; i++ { 23 | for !sp.Offer(&v) { 24 | runtime.Gosched() 25 | } 26 | } 27 | wg.Done() 28 | }(b.N) 29 | 30 | go func(n int) { 31 | runtime.LockOSThread() 32 | var v *T 33 | for i := 0; i < n; i++ { 34 | for !sp.Poll(&v) { 35 | runtime.Gosched() 36 | } 37 | _ = *v 38 | } 39 | wg.Done() 40 | }(b.N) 41 | 42 | wg.Wait() 43 | } 44 | 45 | func BenchmarkSPSC_NonBlocking2(b *testing.B) { 46 | var sp = New(8192) 47 | var wg sync.WaitGroup 48 | wg.Add(2) 49 | type T struct { 50 | i int 51 | } 52 | p := runtime.GOMAXPROCS(1) 53 | var v = T{5} 54 | b.ResetTimer() 55 | go func(n int) { 56 | for i := 0; i < n; i++ { 57 | for !sp.Offer(&v) { 58 | runtime.Gosched() 59 | } 60 | } 61 | wg.Done() 62 | }(b.N) 63 | 64 | go func(n int) { 65 | var v *T 66 | for i := 0; i < n; i++ { 67 | for !sp.Poll(&v) { 68 | runtime.Gosched() 69 | } 70 | _ = *v 71 | } 72 | wg.Done() 73 | }(b.N) 74 | 75 | wg.Wait() 76 | runtime.GOMAXPROCS(p) 77 | } 78 | 79 | func BenchmarkSPSC_Blocking(b *testing.B) { 80 | var sp = New(8192) 81 | var wg sync.WaitGroup 82 | wg.Add(2) 83 | type T struct { 84 | i int 85 | } 86 | var v = T{5} 87 | b.ResetTimer() 88 | go func(n int) { 89 | runtime.LockOSThread() 90 | for i := 0; i < n; i++ { 91 | sp.Put(&v) 92 | } 93 | sp.Close() 94 | wg.Done() 95 | }(b.N) 96 | 97 | go func(n int) { 98 | runtime.LockOSThread() 99 | var v *T 100 | for i := 0; sp.Get(&v); i++ { 101 | _ = *v 102 | } 103 | wg.Done() 104 | }(b.N) 105 | 106 | wg.Wait() 107 | } 108 | 109 | func BenchmarkSPSC_Blocking2(b *testing.B) { 110 | var sp = New(8192) 111 | var wg sync.WaitGroup 112 | wg.Add(2) 113 | type T struct { 114 | i int 115 | } 116 | p := runtime.GOMAXPROCS(1) 117 | var v = T{5} 118 | b.ResetTimer() 119 | go func(n int) { 120 | for i := 0; i < b.N; i++ { 121 | sp.Put(&v) 122 | } 123 | sp.Close() 124 | wg.Done() 125 | }(b.N) 126 | 127 | go func(n int) { 128 | var v *T 129 | for i := 0; sp.Get(&v); i++ { 130 | _ = *v 131 | } 132 | wg.Done() 133 | }(b.N) 134 | 135 | wg.Wait() 136 | runtime.GOMAXPROCS(p) 137 | } 138 | 139 | func TestSPSC_NonBlocking(t *testing.T) { 140 | const N = 1000 141 | var sp = New(4) 142 | var wg sync.WaitGroup 143 | wg.Add(2) 144 | t1 := time.Now() 145 | go func(n int) { 146 | defer wg.Done() 147 | for i := 0; i < n; i++ { 148 | for !sp.Offer(int64(i)) { 149 | runtime.Gosched() 150 | } 151 | time.Sleep(1 * time.Microsecond) 152 | } 153 | }(N) 154 | go func(n int) { 155 | defer wg.Done() 156 | var v *int 157 | for i := 0; i < n; i++ { 158 | for !sp.Poll(&v) { 159 | runtime.Gosched() 160 | } 161 | if i != *v { 162 | t.Fatalf("Expected %d, but got %d", i, *v) 163 | panic(i) 164 | } 165 | } 166 | }(N) 167 | wg.Wait() 168 | t.Log(time.Since(t1)) 169 | } 170 | 171 | func TestSPSC_Blocking(t *testing.T) { 172 | const N = 1000 173 | var sp = New(4) 174 | var wg sync.WaitGroup 175 | wg.Add(2) 176 | t1 := time.Now() 177 | go func(n int) { 178 | defer wg.Done() 179 | for i := 0; i < n; i++ { 180 | sp.Put(int64(i)) 181 | time.Sleep(1 * time.Microsecond) 182 | } 183 | sp.Close() 184 | }(N) 185 | go func(n int) { 186 | defer wg.Done() 187 | var v *int 188 | for i := 0; sp.Get(&v); i++ { 189 | if i != *v { 190 | t.Fatalf("Expected %d, but got %d", i, *v) 191 | panic(i) 192 | } 193 | } 194 | }(N) 195 | wg.Wait() 196 | t.Log(time.Since(t1)) 197 | } 198 | -------------------------------------------------------------------------------- /pkg/tlv/tlv.go: -------------------------------------------------------------------------------- 1 | // Package tlv implements Type-Length-Value encoding. 2 | package tlv 3 | 4 | import ( 5 | "bytes" 6 | "encoding/binary" 7 | "io" 8 | ) 9 | 10 | // ByteSize const. 11 | const ( 12 | Bytes1 ByteSize = 1 << iota 13 | Bytes2 14 | Bytes4 15 | Bytes8 16 | ) 17 | 18 | type ( 19 | // ByteSize is the size of a field in bytes. 20 | // Used to define the size of the type and length field in a message. 21 | ByteSize int 22 | 23 | // Record represents a record of data encoded in the TLV message. 24 | Record struct { 25 | Payload []byte 26 | Type uint 27 | } 28 | 29 | // Codec is the configuration for a TLV encoding/decoding task. 30 | Codec struct { 31 | // TypeBytes defines the size in bytes of the message type field. 32 | TypeBytes ByteSize 33 | 34 | // LenBytes defines the size in bytes of the message length field. 35 | LenBytes ByteSize 36 | } 37 | 38 | // Writer encodes records into TLV format using a Codec and writes into the provided io.Writer. 39 | Writer struct { 40 | writer io.Writer 41 | codec *Codec 42 | } 43 | 44 | // Reader decodes records from TLV format using a Codec from the provided io.Reader. 45 | Reader struct { 46 | codec *Codec 47 | reader io.Reader 48 | } 49 | ) 50 | 51 | // NewWriter creates a new Writer. 52 | func NewWriter(w io.Writer, codec *Codec) *Writer { 53 | return &Writer{ 54 | codec: codec, 55 | writer: w, 56 | } 57 | } 58 | 59 | // Write encodes records into TLV format using a Codec and writes into the provided io.Writer. 60 | func (w *Writer) Write(rec *Record) error { 61 | err := writeUint(w.writer, w.codec.TypeBytes, rec.Type) 62 | if err != nil { 63 | return err 64 | } 65 | 66 | ulen := uint(len(rec.Payload)) 67 | err = writeUint(w.writer, w.codec.LenBytes, ulen) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | _, err = w.writer.Write(rec.Payload) 73 | return err 74 | } 75 | 76 | func writeUint(w io.Writer, b ByteSize, i uint) error { 77 | var num interface{} 78 | switch b { 79 | case Bytes1: 80 | num = uint8(i) 81 | case Bytes2: 82 | num = uint16(i) 83 | case Bytes4: 84 | num = uint32(i) 85 | case Bytes8: 86 | num = uint64(i) 87 | } 88 | return binary.Write(w, binary.LittleEndian, num) 89 | } 90 | 91 | // NewReader creates a new Reader. 92 | func NewReader(reader io.Reader, codec *Codec) *Reader { 93 | return &Reader{codec: codec, reader: reader} 94 | } 95 | 96 | // Next reads a single Record from the io.Reader. 97 | func (r *Reader) Next() (*Record, error) { 98 | // get type 99 | typeBytes := make([]byte, r.codec.TypeBytes) 100 | _, err := r.reader.Read(typeBytes) 101 | if err != nil { 102 | return nil, err 103 | } 104 | typ := readUint(typeBytes, r.codec.TypeBytes) 105 | 106 | // get len 107 | payloadLenBytes := make([]byte, r.codec.LenBytes) 108 | _, err = r.reader.Read(payloadLenBytes) 109 | if err != nil && err != io.EOF { 110 | return nil, err 111 | } 112 | payloadLen := readUint(payloadLenBytes, r.codec.LenBytes) 113 | 114 | if err == io.EOF && payloadLen != 0 { 115 | return nil, err 116 | } 117 | 118 | // get value 119 | v := make([]byte, payloadLen) 120 | _, err = r.reader.Read(v) 121 | if err != nil && err != io.EOF { 122 | return nil, err 123 | } 124 | 125 | return &Record{ 126 | Type: typ, 127 | Payload: v, 128 | }, nil 129 | 130 | } 131 | 132 | func readUint(b []byte, sz ByteSize) uint { 133 | reader := bytes.NewReader(b) 134 | switch sz { 135 | case Bytes1: 136 | var i uint8 137 | binary.Read(reader, binary.LittleEndian, &i) 138 | return uint(i) 139 | case Bytes2: 140 | var i uint16 141 | binary.Read(reader, binary.LittleEndian, &i) 142 | return uint(i) 143 | case Bytes4: 144 | var i uint32 145 | binary.Read(reader, binary.LittleEndian, &i) 146 | return uint(i) 147 | case Bytes8: 148 | var i uint64 149 | binary.Read(reader, binary.LittleEndian, &i) 150 | return uint(i) 151 | default: 152 | return 0 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /pkg/hyperloglog/hyperloglog.go: -------------------------------------------------------------------------------- 1 | // Package hyperloglog implements HyperLogLog cardinality estimation. 2 | package hyperloglog 3 | 4 | import ( 5 | "errors" 6 | "hash" 7 | "hash/fnv" 8 | "math" 9 | ) 10 | 11 | // HyperLogLog probabilistic data struct for cardinality estimation. 12 | type HyperLogLog struct { 13 | registers []uint8 // registers bucket 14 | m uint // number of registers 15 | b uint32 // number of bits to find registers bucket number 16 | alpha float64 // bias-correction constant 17 | hash hash.Hash32 // hash function 18 | } 19 | 20 | const ( 21 | exp32 float64 = 4294967296 22 | negexp32 float64 = -4294967296 23 | alpha16 float64 = 0.673 24 | alpha32 float64 = 0.697 25 | alpha64 float64 = 0.709 26 | ) 27 | 28 | // New creates a new HyperLogLog with `m` registers bucket. 29 | // `m` should be a power of two. 30 | func New(m uint) (*HyperLogLog, error) { 31 | if (m & (m - 1)) != 0 { 32 | m = adjustM(m) 33 | } 34 | 35 | return &HyperLogLog{ 36 | registers: make([]uint8, m), 37 | m: m, 38 | b: uint32(math.Ceil(math.Log2(float64(m)))), 39 | alpha: calculateAlpha(m), 40 | hash: fnv.New32(), 41 | }, nil 42 | } 43 | 44 | // NewGuess creates a new HyperLogLog within the given standard error. 45 | func NewGuess(stdErr float64) (*HyperLogLog, error) { 46 | m := math.Pow(1.04/stdErr, 2) 47 | return New(uint(math.Pow(2, math.Ceil(math.Log2(m))))) 48 | } 49 | 50 | // Add adds the data to the set. 51 | func (h *HyperLogLog) Add(data []byte) { 52 | var ( 53 | hash = h.calculateHash(data) 54 | k = 32 - h.b 55 | r = calculateConsecutiveZeros(hash, k) 56 | j = hash >> uint(k) 57 | ) 58 | 59 | if r > h.registers[j] { 60 | h.registers[j] = r 61 | } 62 | } 63 | 64 | // Count returns the estimated cardinality of the set. 65 | func (h *HyperLogLog) Count() uint64 { 66 | sum, m := 0.0, float64(h.m) 67 | for _, rv := range h.registers { 68 | sum += 1.0 / math.Pow(2.0, float64(rv)) 69 | } 70 | estimate := h.alpha * m * m / sum 71 | if estimate <= 5.0/2.0*m { 72 | // Small range correction 73 | v := 0 74 | for _, r := range h.registers { 75 | if r == 0 { 76 | v++ 77 | } 78 | } 79 | if v > 0 { 80 | estimate = m * math.Log(m/float64(v)) 81 | } 82 | } else if estimate > 1.0/30.0*exp32 { 83 | // Large range correction 84 | estimate = negexp32 * math.Log(1-estimate/exp32) 85 | } 86 | return uint64(estimate) 87 | } 88 | 89 | // Merge combines the HyperLogLog with the other. 90 | func (h *HyperLogLog) Merge(other *HyperLogLog) error { 91 | if h.m != other.m { 92 | return errors.New("registers bucket number must match") 93 | } 94 | 95 | for j, r := range other.registers { 96 | if r > h.registers[j] { 97 | h.registers[j] = r 98 | } 99 | } 100 | 101 | return nil 102 | } 103 | 104 | // Reset restores the HyperLogLog to its original state. 105 | func (h *HyperLogLog) Reset() { 106 | h.registers = make([]uint8, h.m) 107 | } 108 | 109 | // SetHash sets the hashing function. 110 | func (h *HyperLogLog) SetHash(hasher hash.Hash32) { 111 | h.hash = hasher 112 | } 113 | 114 | func (h *HyperLogLog) calculateHash(data []byte) uint32 { 115 | h.hash.Reset() 116 | h.hash.Write(data) 117 | sum := h.hash.Sum32() 118 | return sum 119 | } 120 | 121 | func calculateAlpha(m uint) float64 { 122 | var a float64 123 | switch m { 124 | case 16: 125 | a = alpha16 126 | case 32: 127 | a = alpha32 128 | case 64: 129 | a = alpha64 130 | default: 131 | a = 0.7213 / (1.0 + 1.079/float64(m)) 132 | } 133 | return a 134 | } 135 | 136 | // calculateConsecutiveZeros calculates the position of the rightmost 1-bit. 137 | func calculateConsecutiveZeros(val, max uint32) uint8 { 138 | r := uint32(1) 139 | for val&1 == 0 && r <= max { 140 | r++ 141 | val >>= 1 142 | } 143 | return uint8(r) 144 | } 145 | 146 | func adjustM(x uint) uint { 147 | m := uint(1) 148 | for m < x { 149 | m <<= 1 150 | } 151 | return m 152 | } 153 | -------------------------------------------------------------------------------- /docs/rwspinlock.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # rwspinlock 4 | `import "github.com/andy2046/gopie/pkg/rwspinlock"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package rwspinlock implements RWSpinLock. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [type RWLocker](#RWLocker) 17 | * [func New() *RWLocker](#New) 18 | * [func (rw *RWLocker) IsLocked() bool](#RWLocker.IsLocked) 19 | * [func (rw *RWLocker) IsRLocked() bool](#RWLocker.IsRLocked) 20 | * [func (rw *RWLocker) Lock()](#RWLocker.Lock) 21 | * [func (rw *RWLocker) RLock()](#RWLocker.RLock) 22 | * [func (rw *RWLocker) RLocker() sync.Locker](#RWLocker.RLocker) 23 | * [func (rw *RWLocker) RUnlock()](#RWLocker.RUnlock) 24 | * [func (rw *RWLocker) TryLock() bool](#RWLocker.TryLock) 25 | * [func (rw *RWLocker) TryRLock() bool](#RWLocker.TryRLock) 26 | * [func (rw *RWLocker) Unlock()](#RWLocker.Unlock) 27 | 28 | 29 | #### Package files 30 | [rwspinlock.go](/src/github.com/andy2046/gopie/pkg/rwspinlock/rwspinlock.go) 31 | 32 | 33 | 34 | 35 | 36 | 37 | ## type [RWLocker](/src/target/rwspinlock.go?s=157:480#L11) 38 | ``` go 39 | type RWLocker struct { 40 | // contains filtered or unexported fields 41 | } 42 | ``` 43 | RWLocker is the RWSpinLock implementation. 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | ### func [New](/src/target/rwspinlock.go?s=509:529#L21) 52 | ``` go 53 | func New() *RWLocker 54 | ``` 55 | New creates a RWLocker. 56 | 57 | 58 | 59 | 60 | 61 | ### func (\*RWLocker) [IsLocked](/src/target/rwspinlock.go?s=2444:2479#L107) 62 | ``` go 63 | func (rw *RWLocker) IsLocked() bool 64 | ``` 65 | IsLocked returns true if there is active writer, false otherwise. 66 | 67 | 68 | 69 | 70 | ### func (\*RWLocker) [IsRLocked](/src/target/rwspinlock.go?s=2607:2643#L112) 71 | ``` go 72 | func (rw *RWLocker) IsRLocked() bool 73 | ``` 74 | IsRLocked returns true if there is active reader, false otherwise. 75 | 76 | 77 | 78 | 79 | ### func (\*RWLocker) [Lock](/src/target/rwspinlock.go?s=1309:1335#L57) 80 | ``` go 81 | func (rw *RWLocker) Lock() 82 | ``` 83 | Lock locks rw for writing. 84 | 85 | 86 | 87 | 88 | ### func (\*RWLocker) [RLock](/src/target/rwspinlock.go?s=586:613#L26) 89 | ``` go 90 | func (rw *RWLocker) RLock() 91 | ``` 92 | RLock locks rw for reading. 93 | 94 | 95 | 96 | 97 | ### func (\*RWLocker) [RLocker](/src/target/rwspinlock.go?s=2795:2836#L117) 98 | ``` go 99 | func (rw *RWLocker) RLocker() sync.Locker 100 | ``` 101 | RLocker returns a sync.Locker interface implementation by calling rw.RLock and rw.RUnlock. 102 | 103 | 104 | 105 | 106 | ### func (\*RWLocker) [RUnlock](/src/target/rwspinlock.go?s=904:933#L41) 107 | ``` go 108 | func (rw *RWLocker) RUnlock() 109 | ``` 110 | RUnlock undoes a single RLock call. 111 | 112 | 113 | 114 | 115 | ### func (\*RWLocker) [TryLock](/src/target/rwspinlock.go?s=1994:2028#L87) 116 | ``` go 117 | func (rw *RWLocker) TryLock() bool 118 | ``` 119 | TryLock try to locks rw for writing, 120 | it returns true if succeed, false otherwise. 121 | 122 | 123 | 124 | 125 | ### func (\*RWLocker) [TryRLock](/src/target/rwspinlock.go?s=1068:1103#L47) 126 | ``` go 127 | func (rw *RWLocker) TryRLock() bool 128 | ``` 129 | TryRLock try to locks rw for reading, 130 | it returns true if succeed, false otherwise. 131 | 132 | 133 | 134 | 135 | ### func (\*RWLocker) [Unlock](/src/target/rwspinlock.go?s=1770:1798#L78) 136 | ``` go 137 | func (rw *RWLocker) Unlock() 138 | ``` 139 | Unlock unlocks rw for writing. 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | - - - 149 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 150 | -------------------------------------------------------------------------------- /docs/cached.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # cached 4 | `import "github.com/andy2046/gopie/pkg/cached"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package cached implements Caching Devil pattern. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [type Cache](#Cache) 17 | * [type Cacher](#Cacher) 18 | * [func New(interval time.Duration, cache Cache, lessor Lessor, teller TruthTeller) Cacher](#New) 19 | * [func (c Cacher) Read(key string) []byte](#Cacher.Read) 20 | * [type Lease](#Lease) 21 | * [type Lessor](#Lessor) 22 | * [type TruthTeller](#TruthTeller) 23 | 24 | 25 | #### Package files 26 | [cached.go](/src/github.com/andy2046/gopie/pkg/cached/cached.go) 27 | 28 | 29 | 30 | 31 | 32 | 33 | ## type [Cache](/src/target/cached.go?s=698:1537#L29) 34 | ``` go 35 | type Cache interface { 36 | // Get returns a list of []byte representing the values associated with the provided keys. 37 | // The value associated with a certain key will be in the same position in the returned list 38 | // as the key is in the keys list. 39 | Get(keys ...string) [][]byte 40 | 41 | // Set associates the provided value with the provided key in the cache layer. 42 | Set(key string, value []byte) error 43 | 44 | // AtomicAdd set the provided value for key if and only if the key has not already been set. 45 | // returns true if it succeeds, false otherwise. 46 | AtomicAdd(key string, value []byte) bool 47 | 48 | // AtomicCheckAndSet set the valueToSet for the provided key if and only if the key is currently associated with expectedValue. 49 | // returns true if it succeeds, false otherwise. 50 | AtomicCheckAndSet(key string, expectedValue, valueToSet []byte) bool 51 | } 52 | ``` 53 | Cache represents the Caching layer. 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | ## type [Cacher](/src/target/cached.go?s=1738:1839#L52) 65 | ``` go 66 | type Cacher struct { 67 | // contains filtered or unexported fields 68 | } 69 | ``` 70 | Cacher manages Caching Devil. 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | ### func [New](/src/target/cached.go?s=1951:2038#L62) 79 | ``` go 80 | func New(interval time.Duration, cache Cache, lessor Lessor, teller TruthTeller) Cacher 81 | ``` 82 | New returns a new Cacher. 83 | interval is the retry waiting time if another request is holding the lease. 84 | 85 | 86 | 87 | 88 | 89 | ### func (Cacher) [Read](/src/target/cached.go?s=2212:2251#L72) 90 | ``` go 91 | func (c Cacher) Read(key string) []byte 92 | ``` 93 | Read try to retrieve the value associated with the provided key. 94 | 95 | 96 | 97 | 98 | ## type [Lease](/src/target/cached.go?s=169:243#L8) 99 | ``` go 100 | type Lease interface { 101 | // Nonce is unique for each Lease. 102 | Nonce() string 103 | } 104 | ``` 105 | Lease is a per-cache-key lock preventing thundering herds and stale sets. 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | ## type [Lessor](/src/target/cached.go?s=278:655#L14) 117 | ``` go 118 | type Lessor interface { 119 | // NewLease create a new Lease. 120 | NewLease() Lease 121 | 122 | // FromValue construct a Lease from nonce value. 123 | FromValue(nonce []byte) (Lease, error) 124 | 125 | // MustFromValue is like FromValue but panics if nonce value is not a Lease. 126 | MustFromValue(nonce []byte) Lease 127 | 128 | // IsLease returns true if value is a Lease, false otherwise. 129 | IsLease(value []byte) bool 130 | } 131 | ``` 132 | Lessor is the Lease helper. 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | ## type [TruthTeller](/src/target/cached.go?s=1666:1701#L49) 144 | ``` go 145 | type TruthTeller func(key string) []byte 146 | ``` 147 | TruthTeller is the function to fetch the value associated with the looked up key 148 | from the source of truth data store. 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | - - - 164 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 165 | -------------------------------------------------------------------------------- /docs/multilane.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # multilane 4 | `import "github.com/andy2046/gopie/pkg/multilane"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package multilane implements a concurrent blocking multiset. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [type Config](#Config) 17 | * [type MultiLane](#MultiLane) 18 | * [func New(conf Config) *MultiLane](#New) 19 | * [func (m *MultiLane) Close()](#MultiLane.Close) 20 | * [func (m *MultiLane) Get(i interface{}) bool](#MultiLane.Get) 21 | * [func (m *MultiLane) GetLane(lane uint32, i interface{}) bool](#MultiLane.GetLane) 22 | * [func (m *MultiLane) LaneWidth() uint32](#MultiLane.LaneWidth) 23 | * [func (m *MultiLane) Put(i interface{})](#MultiLane.Put) 24 | * [func (m *MultiLane) PutLane(lane uint32, i interface{})](#MultiLane.PutLane) 25 | * [func (m *MultiLane) QueueSize() uint32](#MultiLane.QueueSize) 26 | * [type Queue](#Queue) 27 | 28 | 29 | #### Package files 30 | [multilane.go](/src/github.com/andy2046/gopie/pkg/multilane/multilane.go) 31 | 32 | 33 | 34 | 35 | 36 | 37 | ## type [Config](/src/target/multilane.go?s=454:609#L27) 38 | ``` go 39 | type Config struct { 40 | LaneWidth uint32 // number of queue for MultiLane 41 | QueueSize uint32 // size of each queue in MultiLane 42 | New func(uint32) Queue 43 | } 44 | ``` 45 | Config for MultiLane. 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | ## type [MultiLane](/src/target/multilane.go?s=662:839#L34) 57 | ``` go 58 | type MultiLane struct { 59 | // contains filtered or unexported fields 60 | } 61 | ``` 62 | MultiLane represents the concurrent multiset. 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | ### func [New](/src/target/multilane.go?s=874:906#L46) 71 | ``` go 72 | func New(conf Config) *MultiLane 73 | ``` 74 | New create a new MultiLane. 75 | 76 | 77 | 78 | 79 | 80 | ### func (\*MultiLane) [Close](/src/target/multilane.go?s=2432:2459#L114) 81 | ``` go 82 | func (m *MultiLane) Close() 83 | ``` 84 | Close the MultiLane, it shall NOT be called before `Put()`. 85 | 86 | 87 | 88 | 89 | ### func (\*MultiLane) [Get](/src/target/multilane.go?s=1897:1940#L85) 90 | ``` go 91 | func (m *MultiLane) Get(i interface{}) bool 92 | ``` 93 | Get the value at one of the lanes pointed by the cursor to given variable, 94 | blocking. 95 | 96 | 97 | 98 | 99 | ### func (\*MultiLane) [GetLane](/src/target/multilane.go?s=1527:1587#L73) 100 | ``` go 101 | func (m *MultiLane) GetLane(lane uint32, i interface{}) bool 102 | ``` 103 | GetLane get the value at the provided lane to given variable, 104 | blocking. 105 | 106 | 107 | 108 | 109 | ### func (\*MultiLane) [LaneWidth](/src/target/multilane.go?s=2546:2584#L121) 110 | ``` go 111 | func (m *MultiLane) LaneWidth() uint32 112 | ``` 113 | LaneWidth is the number of lanes. 114 | 115 | 116 | 117 | 118 | ### func (\*MultiLane) [Put](/src/target/multilane.go?s=2178:2216#L100) 119 | ``` go 120 | func (m *MultiLane) Put(i interface{}) 121 | ``` 122 | Put given variable at one of the lanes pointed by the cursor, 123 | blocking. 124 | 125 | 126 | 127 | 128 | ### func (\*MultiLane) [PutLane](/src/target/multilane.go?s=1705:1760#L79) 129 | ``` go 130 | func (m *MultiLane) PutLane(lane uint32, i interface{}) 131 | ``` 132 | PutLane put given variable at the provided lane, 133 | blocking. 134 | 135 | 136 | 137 | 138 | ### func (\*MultiLane) [QueueSize](/src/target/multilane.go?s=2671:2709#L126) 139 | ``` go 140 | func (m *MultiLane) QueueSize() uint32 141 | ``` 142 | QueueSize is the size of the underlying queue. 143 | 144 | 145 | 146 | 147 | ## type [Queue](/src/target/multilane.go?s=352:425#L20) 148 | ``` go 149 | type Queue interface { 150 | Get(interface{}) bool 151 | Put(interface{}) 152 | Close() 153 | } 154 | ``` 155 | Queue represents the queue with Get / Put / Close methods. 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | - - - 171 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 172 | -------------------------------------------------------------------------------- /docs/bloom.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # bloom 4 | `import "github.com/andy2046/gopie/pkg/bloom"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package bloom implements a Bloom filter. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [func Guess(n uint64, p float64) (m, k uint64)](#Guess) 17 | * [type Bloom](#Bloom) 18 | * [func NewB(m, k uint64) Bloom](#NewB) 19 | * [func NewBGuess(n uint64, p float64) Bloom](#NewBGuess) 20 | * [func NewS(fpRate float64) Bloom](#NewS) 21 | * [func NewSGuess(n uint64, p, r float64) Bloom](#NewSGuess) 22 | * [type CountingBloom](#CountingBloom) 23 | * [func New(m, k uint64) CountingBloom](#New) 24 | * [func NewGuess(n uint64, p float64) CountingBloom](#NewGuess) 25 | 26 | 27 | #### Package files 28 | [bloom.go](/src/github.com/andy2046/gopie/pkg/bloom/bloom.go) [bloombit.go](/src/github.com/andy2046/gopie/pkg/bloom/bloombit.go) [bloomscale.go](/src/github.com/andy2046/gopie/pkg/bloom/bloomscale.go) [siphash.go](/src/github.com/andy2046/gopie/pkg/bloom/siphash.go) 29 | 30 | 31 | 32 | 33 | 34 | ## func [Guess](/src/target/bloom.go?s=1743:1788#L68) 35 | ``` go 36 | func Guess(n uint64, p float64) (m, k uint64) 37 | ``` 38 | Guess estimates m/k based on the provided n/p. 39 | 40 | 41 | 42 | 43 | ## type [Bloom](/src/target/bloom.go?s=127:341#L10) 44 | ``` go 45 | type Bloom interface { 46 | Add([]byte) 47 | AddString(string) 48 | Exist([]byte) bool 49 | ExistString(string) bool 50 | FalsePositive() float64 51 | GuessFalsePositive(uint64) float64 52 | M() uint64 53 | K() uint64 54 | N() uint64 55 | Clear() 56 | } 57 | ``` 58 | Bloom is the standard bloom filter. 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | ### func [NewB](/src/target/bloombit.go?s=535:563#L22) 67 | ``` go 68 | func NewB(m, k uint64) Bloom 69 | ``` 70 | NewB creates standard bloom filter based on the provided m/k. 71 | m is the size of bloom filter bits. 72 | k is the number of hash functions. 73 | 74 | 75 | ### func [NewBGuess](/src/target/bloombit.go?s=925:966#L35) 76 | ``` go 77 | func NewBGuess(n uint64, p float64) Bloom 78 | ``` 79 | NewBGuess estimates m/k based on the provided n/p then creates standard bloom filter. 80 | n is the estimated number of elements in the bloom filter. 81 | p is the false positive probability. 82 | 83 | 84 | ### func [NewS](/src/target/bloomscale.go?s=638:669#L27) 85 | ``` go 86 | func NewS(fpRate float64) Bloom 87 | ``` 88 | NewS creates scalable bloom filter based on the provided fpRate. 89 | fpRate is the target False Positive probability. 90 | 91 | 92 | ### func [NewSGuess](/src/target/bloomscale.go?s=947:991#L35) 93 | ``` go 94 | func NewSGuess(n uint64, p, r float64) Bloom 95 | ``` 96 | NewSGuess estimates m/k based on the provided n/p then creates scalable bloom filter. 97 | n is the estimated number of elements in the bloom filter. 98 | p is the false positive probability. 99 | r is the optimal tightening ratio. 100 | 101 | 102 | 103 | 104 | 105 | ## type [CountingBloom](/src/target/bloom.go?s=483:559#L25) 106 | ``` go 107 | type CountingBloom interface { 108 | Bloom 109 | Remove([]byte) 110 | RemoveString(string) 111 | } 112 | ``` 113 | CountingBloom is the bloom filter which allows deletion of entries. 114 | Take note that an 16-bit counter is maintained for each entry. 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | ### func [New](/src/target/bloom.go?s=1203:1238#L49) 123 | ``` go 124 | func New(m, k uint64) CountingBloom 125 | ``` 126 | New creates counting bloom filter based on the provided m/k. 127 | m is the size of bloom filter bits. 128 | k is the number of hash functions. 129 | 130 | 131 | ### func [NewGuess](/src/target/bloom.go?s=1600:1648#L62) 132 | ``` go 133 | func NewGuess(n uint64, p float64) CountingBloom 134 | ``` 135 | NewGuess estimates m/k based on the provided n/p then creates counting bloom filter. 136 | n is the estimated number of elements in the bloom filter. 137 | p is the false positive probability. 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | - - - 148 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 149 | -------------------------------------------------------------------------------- /pkg/countminsketch/countmin.go: -------------------------------------------------------------------------------- 1 | // Package countminsketch implements Count-Min Sketch. 2 | package countminsketch 3 | 4 | import ( 5 | "encoding/binary" 6 | "errors" 7 | "hash" 8 | "hash/fnv" 9 | "math" 10 | ) 11 | 12 | // CountMinSketch struct. 13 | type CountMinSketch struct { 14 | matrix [][]uint64 // count matrix 15 | width uint // matrix width 16 | depth uint // matrix depth 17 | count uint64 // total number of items added 18 | hash hash.Hash64 // hash function 19 | } 20 | 21 | // For a sketch matrix w x d with total sum of all counts N, 22 | // the estimate has error at most 2N/w, with probability at least 1-(1/2)^d. 23 | 24 | // New returns new Count-Min Sketch with the given `width` and `depth`. 25 | func New(width, depth uint) (*CountMinSketch, error) { 26 | if width < 1 || depth < 1 { 27 | return nil, errors.New("Dimensions must be positive") 28 | } 29 | 30 | matrix := make([][]uint64, depth) 31 | for i := uint(0); i < depth; i++ { 32 | matrix[i] = make([]uint64, width) 33 | } 34 | 35 | return &CountMinSketch{ 36 | matrix: matrix, 37 | width: width, 38 | depth: depth, 39 | hash: fnv.New64(), 40 | }, nil 41 | } 42 | 43 | // NewGuess returns new Count-Min Sketch with the given error rate `epsilon` and confidence `delta`. 44 | func NewGuess(epsilon, delta float64) (*CountMinSketch, error) { 45 | if epsilon <= 0 || epsilon >= 1 { 46 | return nil, errors.New("epsilon must be in range (0, 1)") 47 | } 48 | if delta <= 0 || delta >= 1 { 49 | return nil, errors.New("delta must be in range (0, 1)") 50 | } 51 | 52 | width, depth := uint(math.Ceil(math.E/epsilon)), 53 | uint(math.Ceil(math.Log(1-delta)/math.Log(0.5))) 54 | 55 | return New(width, depth) 56 | } 57 | 58 | // Count returns the number of items added to the sketch. 59 | func (c *CountMinSketch) Count() uint64 { 60 | return c.count 61 | } 62 | 63 | // Add add the `data` to the sketch. `count` default to 1. 64 | func (c *CountMinSketch) Add(data []byte, count ...uint64) { 65 | cnt := uint64(1) 66 | if len(count) > 0 { 67 | cnt = count[0] 68 | } 69 | 70 | lower, upper := hashn(data, c.hash) 71 | 72 | for i := uint(0); i < c.depth; i++ { 73 | c.matrix[i][(uint(lower)+uint(upper)*i)%c.width] += cnt 74 | } 75 | 76 | c.count += cnt 77 | } 78 | 79 | // AddString add the `data` string to the sketch. `count` default to 1. 80 | func (c *CountMinSketch) AddString(data string, count ...uint64) { 81 | c.Add([]byte(data), count...) 82 | } 83 | 84 | // Estimate estimate the frequency of the `data`. 85 | func (c *CountMinSketch) Estimate(data []byte) uint64 { 86 | var ( 87 | lower, upper = hashn(data, c.hash) 88 | count uint64 89 | ) 90 | 91 | for i := uint(0); i < c.depth; i++ { 92 | j := (uint(lower) + uint(upper)*i) % c.width 93 | if i == 0 || c.matrix[i][j] < count { 94 | count = c.matrix[i][j] 95 | } 96 | } 97 | 98 | return count 99 | } 100 | 101 | // EstimateString estimate the frequency of the `data` string. 102 | func (c *CountMinSketch) EstimateString(data string) uint64 { 103 | return c.Estimate([]byte(data)) 104 | } 105 | 106 | // Reset reset the sketch to its original state. 107 | func (c *CountMinSketch) Reset() { 108 | matrix := make([][]uint64, c.depth) 109 | for i := uint(0); i < c.depth; i++ { 110 | matrix[i] = make([]uint64, c.width) 111 | } 112 | 113 | c.matrix = matrix 114 | c.count = 0 115 | } 116 | 117 | // Merge combines the sketch with another. 118 | func (c *CountMinSketch) Merge(other *CountMinSketch) error { 119 | if c.depth != other.depth { 120 | return errors.New("matrix depth must match") 121 | } 122 | 123 | if c.width != other.width { 124 | return errors.New("matrix width must match") 125 | } 126 | 127 | for i := uint(0); i < c.depth; i++ { 128 | for j := uint(0); j < c.width; j++ { 129 | c.matrix[i][j] += other.matrix[i][j] 130 | } 131 | } 132 | 133 | c.count += other.count 134 | return nil 135 | } 136 | 137 | // Depth returns the matrix depth. 138 | func (c *CountMinSketch) Depth() uint { 139 | return c.depth 140 | } 141 | 142 | // Width returns the matrix width. 143 | func (c *CountMinSketch) Width() uint { 144 | return c.width 145 | } 146 | 147 | func hashn(data []byte, hasher hash.Hash64) (uint32, uint32) { 148 | hasher.Reset() 149 | hasher.Write(data) 150 | sum := hasher.Sum(nil) 151 | return binary.BigEndian.Uint32(sum[4:8]), binary.BigEndian.Uint32(sum[0:4]) 152 | } 153 | -------------------------------------------------------------------------------- /docs/dll.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # dll 4 | `import "github.com/andy2046/gopie/pkg/dll"` 5 | 6 | * [Overview](#pkg-overview) 7 | * [Index](#pkg-index) 8 | 9 | ## Overview 10 | Package dll provides a lock-free implementation of doubly linked list. 11 | 12 | 13 | 14 | 15 | ## Index 16 | * [type Element](#Element) 17 | * [type List](#List) 18 | * [func New() *List](#New) 19 | * [func (l *List) Empty() bool](#List.Empty) 20 | * [func (l *List) Init() *List](#List.Init) 21 | * [func (l *List) Next(node *Element) *Element](#List.Next) 22 | * [func (l *List) PopLeft() interface{}](#List.PopLeft) 23 | * [func (l *List) PopRight() interface{}](#List.PopRight) 24 | * [func (l *List) Prev(node *Element) *Element](#List.Prev) 25 | * [func (l *List) PushLeft(v interface{}) *Element](#List.PushLeft) 26 | * [func (l *List) PushRight(v interface{}) *Element](#List.PushRight) 27 | * [func (l *List) Remove(e *Element) interface{}](#List.Remove) 28 | 29 | 30 | #### Package files 31 | [amr.go](/src/github.com/andy2046/gopie/pkg/dll/amr.go) [dll.go](/src/github.com/andy2046/gopie/pkg/dll/dll.go) 32 | 33 | 34 | 35 | 36 | 37 | 38 | ## type [Element](/src/target/dll.go?s=232:422#L12) 39 | ``` go 40 | type Element struct { 41 | 42 | // The value stored with this element. 43 | Value interface{} 44 | // contains filtered or unexported fields 45 | } 46 | ``` 47 | Element is an element of a linked list. 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | ## type [List](/src/target/dll.go?s=137:185#L6) 59 | ``` go 60 | type List struct { 61 | // contains filtered or unexported fields 62 | } 63 | ``` 64 | List represents a doubly linked list. 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | ### func [New](/src/target/dll.go?s=462:478#L22) 73 | ``` go 74 | func New() *List 75 | ``` 76 | New returns an initialized list. 77 | 78 | 79 | 80 | 81 | 82 | ### func (\*List) [Empty](/src/target/dll.go?s=995:1022#L42) 83 | ``` go 84 | func (l *List) Empty() bool 85 | ``` 86 | Empty returns true if list l is empty, false otherwise 87 | 88 | 89 | 90 | 91 | ### func (\*List) [Init](/src/target/dll.go?s=546:573#L25) 92 | ``` go 93 | func (l *List) Init() *List 94 | ``` 95 | Init initializes or clears list l. 96 | 97 | 98 | 99 | 100 | ### func (\*List) [Next](/src/target/dll.go?s=5438:5481#L258) 101 | ``` go 102 | func (l *List) Next(node *Element) *Element 103 | ``` 104 | Next returns the next list element or nil. 105 | 106 | 107 | 108 | 109 | ### func (\*List) [PopLeft](/src/target/dll.go?s=2722:2758#L133) 110 | ``` go 111 | func (l *List) PopLeft() interface{} 112 | ``` 113 | PopLeft returns the first element of list l or nil if the list is empty. 114 | 115 | 116 | 117 | 118 | ### func (\*List) [PopRight](/src/target/dll.go?s=3389:3426#L164) 119 | ``` go 120 | func (l *List) PopRight() interface{} 121 | ``` 122 | PopRight returns the last element of list l or nil if the list is empty. 123 | 124 | 125 | 126 | 127 | ### func (\*List) [Prev](/src/target/dll.go?s=6059:6102#L291) 128 | ``` go 129 | func (l *List) Prev(node *Element) *Element 130 | ``` 131 | Prev returns the previous list element or nil. 132 | 133 | 134 | 135 | 136 | ### func (\*List) [PushLeft](/src/target/dll.go?s=3921:3968#L189) 137 | ``` go 138 | func (l *List) PushLeft(v interface{}) *Element 139 | ``` 140 | PushLeft inserts a new element e with value v at the front of list l and returns e. 141 | 142 | 143 | 144 | 145 | ### func (\*List) [PushRight](/src/target/dll.go?s=4478:4526#L214) 146 | ``` go 147 | func (l *List) PushRight(v interface{}) *Element 148 | ``` 149 | PushRight inserts a new element e with value v at the back of list l and returns e. 150 | 151 | 152 | 153 | 154 | ### func (\*List) [Remove](/src/target/dll.go?s=1283:1328#L55) 155 | ``` go 156 | func (l *List) Remove(e *Element) interface{} 157 | ``` 158 | Remove removes e from l if e is an element of list l. 159 | It returns the element value e.Value if succeed, nil otherwise 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | - - - 169 | Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) 170 | -------------------------------------------------------------------------------- /pkg/pushsum/pushsum_test.go: -------------------------------------------------------------------------------- 1 | package pushsum 2 | 3 | import ( 4 | "math" 5 | "net" 6 | "sync" 7 | "testing" 8 | "time" 9 | 10 | "github.com/andy2046/tik" 11 | ) 12 | 13 | type ( 14 | netAddr struct { 15 | addr string 16 | } 17 | 18 | gossiper struct{} 19 | 20 | scheduler struct { 21 | tk *tik.Ticker 22 | } 23 | 24 | reader struct { 25 | receiver chan float64 26 | sender chan<- float64 27 | } 28 | ) 29 | 30 | var ( 31 | g = gossiper{} 32 | 33 | node0 = netAddr{"0"} 34 | node1 = netAddr{"1"} 35 | node2 = netAddr{"2"} 36 | 37 | r0 *reader 38 | r1 *reader 39 | r2 *reader 40 | 41 | ps0 *PushSum 42 | ps1 *PushSum 43 | ps2 *PushSum 44 | ) 45 | 46 | func TestPushSum(t *testing.T) { 47 | threshold := float64(0.5) 48 | key := uint64(2020) 49 | wg := &sync.WaitGroup{} 50 | wg.Add(3) 51 | 52 | r0 = newReader() 53 | r1 = newReader() 54 | r2 = newReader() 55 | 56 | cfg0 := Config{ 57 | Scheduler: newScheduler(), 58 | ValueReader: r0, 59 | Gossiper: g, 60 | IntervalInMS: 100, 61 | UpdateSteps: 1, 62 | StoreLen: 8, 63 | } 64 | cfg1 := Config{ 65 | Scheduler: newScheduler(), 66 | ValueReader: r1, 67 | Gossiper: g, 68 | IntervalInMS: 100, 69 | UpdateSteps: 1, 70 | StoreLen: 8, 71 | } 72 | cfg2 := Config{ 73 | Scheduler: newScheduler(), 74 | ValueReader: r2, 75 | Gossiper: g, 76 | IntervalInMS: 100, 77 | UpdateSteps: 1, 78 | StoreLen: 8, 79 | } 80 | 81 | ps0 = New(node0, []net.Addr{node1, node2}, cfg0) 82 | ps1 = New(node1, []net.Addr{node0, node2}, cfg1) 83 | ps2 = New(node2, []net.Addr{node1, node0}, cfg2) 84 | 85 | defer func() { 86 | ps0.Close() 87 | ps1.Close() 88 | ps2.Close() 89 | 90 | r0.close() 91 | r1.close() 92 | r2.close() 93 | }() 94 | 95 | msg := Message{Key: key, Value: 0, Weight: 0} 96 | ps0.OnMessage(msg) 97 | ps1.OnMessage(msg) 98 | ps2.OnMessage(msg) 99 | 100 | // init with 15/25/35 then add 30 to each 101 | // average should be 55 102 | update(r0, key, 15, wg) 103 | update(r1, key, 25, wg) 104 | update(r2, key, 35, wg) 105 | 106 | wg.Wait() 107 | time.Sleep(1000 * time.Millisecond) 108 | 109 | v0, err := ps0.Estimate(key) 110 | if err != nil { 111 | t.Error(err) 112 | } 113 | 114 | v1, err := ps1.Estimate(key) 115 | if err != nil { 116 | t.Error(err) 117 | } 118 | 119 | v2, err := ps2.Estimate(key) 120 | if err != nil { 121 | t.Error(err) 122 | } 123 | 124 | if math.Abs(v0-v1) > threshold || 125 | math.Abs(v2-v1) > threshold || 126 | math.Abs(v0-v2) > threshold { 127 | t.Fail() 128 | } 129 | 130 | t.Logf("v0: %f v1: %f v2: %f", v0, v1, v2) 131 | } 132 | 133 | func update(r *reader, k uint64, v float64, wg *sync.WaitGroup) { 134 | go func() { 135 | i := float64(0) 136 | for i < 3 { 137 | r.update(k, v+15*i) 138 | time.Sleep(100 * time.Millisecond) 139 | i++ 140 | } 141 | wg.Done() 142 | }() 143 | } 144 | 145 | func (na netAddr) Network() string { 146 | return "pushsum" 147 | } 148 | 149 | func (na netAddr) String() string { 150 | return na.addr 151 | } 152 | 153 | func (g gossiper) Gossip(addr net.Addr, msg Message) { 154 | switch addr.String() { 155 | case "0": 156 | ps0.OnMessage(msg) 157 | case "1": 158 | ps1.OnMessage(msg) 159 | case "2": 160 | ps2.OnMessage(msg) 161 | default: 162 | } 163 | } 164 | 165 | func (s scheduler) Schedule(interval uint64, cb func()) { 166 | _ = s.tk.Schedule(interval, cb) 167 | } 168 | 169 | func (s scheduler) Close() { 170 | s.tk.Close() 171 | } 172 | 173 | func newScheduler() scheduler { 174 | return scheduler{ 175 | tk: tik.New(), 176 | } 177 | } 178 | 179 | func newReader() *reader { 180 | rcv := make(chan float64) 181 | snd := latest(rcv) 182 | snd <- 0 183 | return &reader{ 184 | receiver: rcv, 185 | sender: snd, 186 | } 187 | } 188 | 189 | func (r *reader) Read(key uint64) float64 { 190 | return <-r.receiver 191 | } 192 | 193 | func (r *reader) update(key uint64, v float64) { 194 | r.sender <- v 195 | } 196 | 197 | func (r *reader) close() { 198 | close(r.sender) 199 | } 200 | 201 | func latest(receiver chan<- float64) chan<- float64 { 202 | sender := make(chan float64) 203 | 204 | go func() { 205 | var ( 206 | latest float64 207 | ok bool 208 | temp chan<- float64 209 | ) 210 | for { 211 | select { 212 | case latest, ok = <-sender: 213 | if !ok { 214 | return 215 | } 216 | if temp == nil { 217 | temp = receiver 218 | } 219 | continue 220 | case temp <- latest: 221 | break 222 | } 223 | } 224 | }() 225 | 226 | return sender 227 | } 228 | --------------------------------------------------------------------------------