├── skl ├── lf │ ├── skl.go │ ├── doc │ │ └── Mikhail.pdf │ ├── types.go │ ├── list_test.go │ ├── arena.go │ └── list.go ├── type.go ├── skl_iterator.go ├── skl_test.go └── skl.go ├── .gitignore ├── README.md ├── rpheap ├── doc │ └── rank pairing heap.pdf ├── README.md ├── rank_pairing_heap_test.go └── rank_pairing_heap.go ├── xorlayer ├── doc │ └── xor-overlay-topology-saso17.pdf ├── xtm_test.go ├── bucket.go └── xtm.go ├── diskqueue ├── README.md ├── meta_test.go ├── monotime.go ├── conf.go ├── qfile_size_reader.go ├── diskqueue_test.go ├── meta.go └── qfile.go ├── crypto ├── signature │ ├── hash.go │ ├── signature_test.go │ └── signature.go ├── vrf │ ├── vrf.go │ └── p256 │ │ ├── unmarshal.go │ │ ├── p256_test.go │ │ └── p256.go ├── claim │ ├── data │ │ ├── test.rsa.pub │ │ └── test.rsa │ ├── jwt.go │ ├── claim_test.go │ ├── signer.go │ └── verifier.go └── wnaf │ ├── rl_test.go │ ├── lr_test.go │ ├── rl.go │ └── lr.go ├── math ├── divisor_test.go ├── lambda_test.go ├── pi_test.go ├── phi_test.go ├── partition_test.go ├── phi.go ├── partition.go ├── polynomial_test.go ├── pi.go ├── polynomial.go ├── lambda.go └── divisor.go ├── bytes ├── realloc.go ├── aligned_test.go └── aligned.go ├── monitor └── init.go ├── bytearena_test.go ├── signal └── signal.go ├── port └── tcp.go ├── lf └── mcas │ ├── api.go │ ├── api_test.go │ ├── ccas.go │ ├── arena.go │ └── mcas.go ├── wm ├── max.go ├── offset.go └── max_test.go ├── concurrent └── bucket.go ├── hack.go ├── hack_amd64.s ├── mutex ├── cmutex.go ├── crwmutex.go ├── tmutex.go └── crwmutex_test.go ├── deadlock ├── README.md ├── detector_test.go ├── mutex.go ├── rwmutex.go ├── semaphore.go └── detector.go ├── hash.go ├── hack_amd64.go ├── logger ├── instance.go ├── dev.go ├── prod.go └── zap.go ├── mmr ├── hasher.go ├── file_hash_store.go ├── README.md ├── util.go └── util_test.go ├── mapped └── file_test.go ├── closer ├── naive.go └── strict.go ├── mmap_test.go ├── uuid_test.go ├── bytearena.go ├── uuid.go ├── workerpool.go ├── go.mod ├── misc.go ├── mmap.go ├── metrics └── metrics.go ├── sort ├── kofn.go └── kofn_test.go ├── parallel └── parallel.go ├── leetcode_test.go ├── reflect_test.go ├── reflect.go └── osc └── osc.go /skl/lf/skl.go: -------------------------------------------------------------------------------- 1 | package lf 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .DS_Store 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # util 2 | 3 | useful toolkit -------------------------------------------------------------------------------- /skl/lf/doc/Mikhail.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhiqiangxu/util/HEAD/skl/lf/doc/Mikhail.pdf -------------------------------------------------------------------------------- /rpheap/doc/rank pairing heap.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhiqiangxu/util/HEAD/rpheap/doc/rank pairing heap.pdf -------------------------------------------------------------------------------- /xorlayer/doc/xor-overlay-topology-saso17.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhiqiangxu/util/HEAD/xorlayer/doc/xor-overlay-topology-saso17.pdf -------------------------------------------------------------------------------- /diskqueue/README.md: -------------------------------------------------------------------------------- 1 | # diskqueue 2 | 3 | yet another diskqueue, it has better interface design than [`go-diskqueue`](https://github.com/nsqio/go-diskqueue) -------------------------------------------------------------------------------- /crypto/signature/hash.go: -------------------------------------------------------------------------------- 1 | package signature 2 | 3 | import ( 4 | "crypto/sha512" 5 | "hash" 6 | ) 7 | 8 | func getHasher() hash.Hash { 9 | return sha512.New() 10 | } 11 | -------------------------------------------------------------------------------- /skl/lf/types.go: -------------------------------------------------------------------------------- 1 | package lf 2 | 3 | type list interface { 4 | Contains(k []byte) bool 5 | Get(k []byte) (v []byte, exists bool) 6 | Insert(k, v []byte) (isNew bool, err error) 7 | Delete(k []byte) bool 8 | } 9 | -------------------------------------------------------------------------------- /diskqueue/meta_test.go: -------------------------------------------------------------------------------- 1 | package diskqueue 2 | 3 | import ( 4 | "testing" 5 | "unsafe" 6 | 7 | "gotest.tools/assert" 8 | ) 9 | 10 | func TestMeta(t *testing.T) { 11 | size := unsafe.Sizeof(FileMeta{}) 12 | assert.Assert(t, size == 40) 13 | } 14 | -------------------------------------------------------------------------------- /math/divisor_test.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/assert" 7 | ) 8 | 9 | func TestDivCount(t *testing.T) { 10 | assert.Equal(t, DivCount(10), 4) 11 | assert.Equal(t, DivCount(144), 15) 12 | 13 | assert.Equal(t, AbelGroups(72).Uint64(), uint64(6)) 14 | } 15 | -------------------------------------------------------------------------------- /math/lambda_test.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/assert" 7 | ) 8 | 9 | func TestLambda(t *testing.T) { 10 | assert.Equal(t, Lambda(35), 12) 11 | assert.Equal(t, Lambda(34), 16) 12 | assert.Equal(t, Lambda(33), 10) 13 | assert.Equal(t, Lambda(16), 4) 14 | } 15 | -------------------------------------------------------------------------------- /crypto/vrf/vrf.go: -------------------------------------------------------------------------------- 1 | package vrf 2 | 3 | // Signer for vrf 4 | type Signer interface { 5 | Hash(alpha []byte) (beta [32]byte, proof []byte, err error) 6 | Public() Verifier 7 | } 8 | 9 | // Verifier for vrf 10 | type Verifier interface { 11 | Verify(alpha, proof []byte, beta [32]byte) (valid bool, err error) 12 | } 13 | -------------------------------------------------------------------------------- /bytes/realloc.go: -------------------------------------------------------------------------------- 1 | package bytes 2 | 3 | // Realloc is like realloc of c. 4 | func Realloc(b []byte, n int) []byte { 5 | newSize := len(b) + n 6 | if cap(b) < newSize { 7 | bs := make([]byte, len(b), newSize) 8 | copy(bs, b) 9 | return bs 10 | } 11 | 12 | // slice b has capability to store n bytes 13 | return b 14 | } 15 | -------------------------------------------------------------------------------- /monitor/init.go: -------------------------------------------------------------------------------- 1 | package monitor 2 | 3 | // this module registers metrics and pprof to http.DefaultServeMux 4 | 5 | import ( 6 | "net/http" 7 | // enable pprof 8 | _ "net/http/pprof" 9 | 10 | "github.com/prometheus/client_golang/prometheus/promhttp" 11 | ) 12 | 13 | func init() { 14 | http.Handle("/metrics", promhttp.Handler()) 15 | } 16 | -------------------------------------------------------------------------------- /math/pi_test.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/assert" 7 | ) 8 | 9 | func TestPi(t *testing.T) { 10 | assert.Equal(t, Pi(5), 3) 11 | assert.Equal(t, Pi(4), 2) 12 | assert.Equal(t, Pi(6), 3) 13 | } 14 | 15 | func TestR2(t *testing.T) { 16 | assert.Equal(t, R2(5), 1) 17 | assert.Equal(t, R2(6), 1) 18 | } 19 | -------------------------------------------------------------------------------- /math/phi_test.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/assert" 7 | ) 8 | 9 | func TestPhi(t *testing.T) { 10 | assert.Equal(t, Phi(10), 4) 11 | assert.Equal(t, Phi(11), 10) 12 | assert.Equal(t, Phi(12), 4) 13 | assert.Equal(t, Phi(14), 6) 14 | assert.Equal(t, Phi(15), 8) 15 | assert.Equal(t, Phi(229), 228) 16 | assert.Equal(t, Phi(242), 110) 17 | } 18 | -------------------------------------------------------------------------------- /bytearena_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/assert" 7 | ) 8 | 9 | func TestByteArena(t *testing.T) { 10 | a := NewByteArena(512, 16384) 11 | n := 10 12 | bytes := a.AllocBytes(n) 13 | assert.Assert(t, len(bytes) == n && cap(bytes) == n) 14 | 15 | cp := 10 16 | bytes = make([]byte, cp) 17 | assert.Assert(t, cap(bytes) == cp && cap(bytes[5:]) == cp-5) 18 | } 19 | -------------------------------------------------------------------------------- /signal/signal.go: -------------------------------------------------------------------------------- 1 | package signal 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | ) 7 | 8 | // SetupHandler will call handler when signal received 9 | func SetupHandler(handler func(os.Signal), s ...os.Signal) { 10 | sigCh := make(chan os.Signal, 1) 11 | 12 | if len(s) == 0 { 13 | panic("signal missing") 14 | } 15 | 16 | signal.Notify(sigCh, s...) 17 | 18 | go func() { 19 | for sig := range sigCh { 20 | handler(sig) 21 | } 22 | }() 23 | } 24 | -------------------------------------------------------------------------------- /diskqueue/monotime.go: -------------------------------------------------------------------------------- 1 | package diskqueue 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | var ( 9 | timeLock sync.Mutex 10 | lastTime int64 11 | ) 12 | 13 | // NowNano returns monotonic nano time 14 | func NowNano() int64 { 15 | timeLock.Lock() 16 | 17 | thisTime := time.Now().UnixNano() 18 | 19 | if thisTime <= lastTime { 20 | thisTime = lastTime + 1 21 | } 22 | 23 | lastTime = thisTime 24 | 25 | timeLock.Unlock() 26 | 27 | return thisTime 28 | } 29 | -------------------------------------------------------------------------------- /port/tcp.go: -------------------------------------------------------------------------------- 1 | package port 2 | 3 | import "net" 4 | 5 | // ChooseTCP returns a useable tcp port 6 | func ChooseTCP() (p uint16, err error) { 7 | l, err := net.Listen("tcp", ":0") 8 | if err != nil { 9 | return 10 | } 11 | 12 | p = uint16(l.Addr().(*net.TCPAddr).Port) 13 | 14 | err = l.Close() 15 | return 16 | } 17 | 18 | // ListenTCP returns a new tcp listener 19 | func ListenTCP() (l net.Listener, err error) { 20 | l, err = net.Listen("tcp", ":0") 21 | return 22 | } 23 | -------------------------------------------------------------------------------- /crypto/claim/data/test.rsa.pub: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwmHUWUTLSCBe7YTIAQam 3 | ZvLfDYJ4JhgbvVRjqr54aNMlPqT+wA1EQrWhYnmJwYy+8JbQh0KJ8HOFHzHiQbEZ 4 | UgkNupq0xiLoDFZ0PrpSixtChgVermcfYfaj0gFxjF4TVpoHVWSHslji80ALiht4 5 | jfXlhU7NJGNK8VFETuuoQMJIl+h+sD0BjWvZfUvN7dTYDYX448Ho1gEc5qlnEsQE 6 | dI0UJ+QKm76wb3kq7zg5NnVicfsd990qysIFIqtvX/faym7U8HW7cFgUI3NjN3Yl 7 | xGPPsWpO79NMMPGIUTjNalSxDOW1Wlu5sEtD4VdXXiQJnykmQ9zaczCjjDrLBYFO 8 | LQIDAQAB 9 | -----END PUBLIC KEY----- 10 | -------------------------------------------------------------------------------- /math/partition_test.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/assert" 7 | ) 8 | 9 | func TestPartition(t *testing.T) { 10 | p1 := Partition(1) 11 | p2 := Partition(2) 12 | p3 := Partition(3) 13 | p4 := Partition(4) 14 | p5 := Partition(5) 15 | p6 := Partition(6) 16 | 17 | assert.Equal(t, p1.Uint64(), uint64(1)) 18 | assert.Equal(t, p2.Uint64(), uint64(2)) 19 | assert.Equal(t, p3.Uint64(), uint64(3)) 20 | assert.Equal(t, p4.Uint64(), uint64(5)) 21 | assert.Equal(t, p5.Uint64(), uint64(7)) 22 | assert.Equal(t, p6.Uint64(), uint64(11)) 23 | } 24 | -------------------------------------------------------------------------------- /math/phi.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | // Phi returns # of coprime integers of n 4 | func Phi(n int) int { 5 | if n == 0 { 6 | panic("invalid input") 7 | } 8 | if n == 1 { 9 | return 1 10 | } 11 | // store numbers in range [2,n] 12 | numbers := make([]int, n-1) 13 | for i := range numbers { 14 | numbers[i] = i + 2 15 | } 16 | 17 | // sieve method 18 | for i, v := range numbers { 19 | if v == i+2 { 20 | // v is a prime 21 | for j := i; j <= n-2; j += v { 22 | numbers[j] /= v 23 | numbers[j] *= v - 1 24 | } 25 | } 26 | } 27 | 28 | return numbers[n-2] 29 | } 30 | -------------------------------------------------------------------------------- /lf/mcas/api.go: -------------------------------------------------------------------------------- 1 | package mcas 2 | 3 | import "unsafe" 4 | 5 | // CompareAndSwap for multiple pointer type variables 6 | func CompareAndSwap(a []*unsafe.Pointer, e []unsafe.Pointer, n []unsafe.Pointer) (swapped bool) { 7 | d := are.putMCDesc() 8 | *d = mcDesc{a: a, e: e, n: n, s: undecided} 9 | /* Memory locations must be sorted into address order. */ 10 | d.sortAddr() 11 | swapped = d.mcasHelp() 12 | return 13 | } 14 | 15 | // Read for a mcas consistent view 16 | func Read(a *unsafe.Pointer) (v unsafe.Pointer) { 17 | 18 | for v = ccasRead(a); isMCDesc(v); ccasRead(a) { 19 | mcfromPointer(v).mcasHelp() 20 | } 21 | 22 | return 23 | } 24 | -------------------------------------------------------------------------------- /bytes/aligned_test.go: -------------------------------------------------------------------------------- 1 | package bytes 2 | 3 | import ( 4 | "testing" 5 | "unsafe" 6 | 7 | "gotest.tools/assert" 8 | ) 9 | 10 | func TestAligned(t *testing.T) { 11 | s1 := AlignedTo8(1) 12 | assert.Assert(t, uintptr(unsafe.Pointer(&s1[0]))%8 == 0 && len(s1) == 1) 13 | s2 := AlignedTo4(1) 14 | assert.Assert(t, uintptr(unsafe.Pointer(&s2[0]))%4 == 0 && len(s2) == 1) 15 | } 16 | 17 | func BenchmarkAligned(b *testing.B) { 18 | sizes := []uint32{10, 20, 30, 40} 19 | b.RunParallel(func(pb *testing.PB) { 20 | for pb.Next() { 21 | for _, size := range sizes { 22 | _ = make([]byte, size) 23 | // _ = AlignedTo8(size) 24 | } 25 | } 26 | 27 | }) 28 | } 29 | -------------------------------------------------------------------------------- /skl/type.go: -------------------------------------------------------------------------------- 1 | /*Package skl provides a non-concurrent skiplist implementation*/ 2 | package skl 3 | 4 | // SkipList for skl interface 5 | // TODO extend key type when go supports generic 6 | type SkipList interface { 7 | Add(key int64, value interface{}) 8 | Get(key int64) (value interface{}, ok bool) 9 | Remove(key int64) 10 | Head() (key int64, value interface{}, ok bool) 11 | Length() int 12 | NewIterator() SkipListIterator 13 | } 14 | 15 | // SkipListIterator for skl iterator 16 | type SkipListIterator interface { 17 | SeekGE(key int64) (ok bool) 18 | First() (ok bool) 19 | Next() (ok bool) 20 | Valid() bool 21 | Key() int64 22 | KeyValue() (int64, interface{}) 23 | } 24 | -------------------------------------------------------------------------------- /math/partition.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | import "math/big" 4 | 5 | func Partition(n int) *big.Int { 6 | if n <= 0 { 7 | panic("invalid input") 8 | } 9 | 10 | // Euler's method 11 | var current, next *Polynomial 12 | for i := 1; i <= n; i++ { 13 | coefficients := make([]*big.Int, n+1) 14 | for j := 0; j <= n; j++ { 15 | if j%i == 0 { 16 | coefficients[j] = big.NewInt(1) 17 | } else { 18 | coefficients[j] = big.NewInt(0) 19 | } 20 | } 21 | 22 | next = NewPolynomialWithMaxOrder(n, coefficients) 23 | if current == nil { 24 | current = next 25 | } else { 26 | current = current.Mul(next) 27 | } 28 | } 29 | 30 | return current.coefficients[n] 31 | } 32 | -------------------------------------------------------------------------------- /wm/max.go: -------------------------------------------------------------------------------- 1 | package wm 2 | 3 | import ( 4 | "context" 5 | 6 | "golang.org/x/sync/semaphore" 7 | ) 8 | 9 | // Max watermark model 10 | type Max struct { 11 | sema *semaphore.Weighted 12 | } 13 | 14 | // NewMax is ctor for Max 15 | func NewMax(max int64) *Max { 16 | m := &Max{sema: semaphore.NewWeighted(max)} 17 | return m 18 | } 19 | 20 | // Enter returns nil if not canceled 21 | func (m *Max) Enter(ctx context.Context) error { 22 | return m.sema.Acquire(ctx, 1) 23 | } 24 | 25 | // TryEnter returns true if succeed 26 | func (m *Max) TryEnter() bool { 27 | return m.sema.TryAcquire(1) 28 | } 29 | 30 | // Exit should only be called if Enter returns nil 31 | func (m *Max) Exit() { 32 | m.sema.Release(1) 33 | } 34 | -------------------------------------------------------------------------------- /concurrent/bucket.go: -------------------------------------------------------------------------------- 1 | package concurrent 2 | 3 | type Bucket[K any, E any] struct { 4 | hash func(K) uint32 5 | elements []E 6 | } 7 | 8 | func NewBucket[K any, E any](buckets int, genFunc func() E, hash func(K) uint32) *Bucket[K, E] { 9 | elements := make([]E, 0, buckets) 10 | for i := 0; i < buckets; i++ { 11 | elements = append(elements, genFunc()) 12 | } 13 | 14 | return &Bucket[K, E]{elements: elements, hash: hash} 15 | } 16 | 17 | func (b *Bucket[K, E]) Element(key K) E { 18 | idx := b.hash(key) 19 | return b.elements[int(idx)%len(b.elements)] 20 | } 21 | 22 | func (b *Bucket[K, E]) Range(fn func(i int, e E)) { 23 | for i := 0; i < len(b.elements); i++ { 24 | fn(i, b.elements[i]) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /hack.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "reflect" 5 | "unsafe" 6 | ) 7 | 8 | // Slice does no copy to change string to slice 9 | func Slice(s string) (b []byte) { 10 | pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) 11 | pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) 12 | pbytes.Data = pstring.Data 13 | pbytes.Len = pstring.Len 14 | pbytes.Cap = pstring.Len 15 | return 16 | } 17 | 18 | // String does no copy to change slice to string 19 | // use your own risk 20 | func String(b []byte) (s string) { 21 | pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) 22 | pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) 23 | pstring.Data = pbytes.Data 24 | pstring.Len = pbytes.Len 25 | return 26 | } 27 | -------------------------------------------------------------------------------- /hack_amd64.s: -------------------------------------------------------------------------------- 1 | #include "go_asm.h" 2 | // Copyright 2014 The Go Authors. All rights reserved. 3 | // Use of this source code is governed by a BSD-style 4 | // license that can be found in the LICENSE file. 5 | 6 | #ifdef GOARCH_arm 7 | #define LR R14 8 | #endif 9 | 10 | #ifdef GOARCH_amd64 11 | #define get_tls(r) MOVQ TLS, r 12 | #define g(r) 0(r)(TLS*1) 13 | #endif 14 | 15 | #ifdef GOARCH_amd64p32 16 | #define get_tls(r) MOVL TLS, r 17 | #define g(r) 0(r)(TLS*1) 18 | #endif 19 | 20 | #ifdef GOARCH_386 21 | #define get_tls(r) MOVL TLS, r 22 | #define g(r) 0(r)(TLS*1) 23 | #endif 24 | #include "funcdata.h" 25 | #include "textflag.h" 26 | 27 | 28 | // func getg() *g 29 | TEXT ·getg(SB),NOSPLIT,$0-0 30 | get_tls(CX) 31 | MOVQ g(CX), AX 32 | MOVQ AX, ret+0(FP) 33 | RET 34 | -------------------------------------------------------------------------------- /mutex/cmutex.go: -------------------------------------------------------------------------------- 1 | package mutex 2 | 3 | import ( 4 | "context" 5 | 6 | "golang.org/x/sync/semaphore" 7 | ) 8 | 9 | // CMutex implements a cancelable mutex (in fact also a try-able mutex) 10 | type CMutex struct { 11 | sema *semaphore.Weighted 12 | } 13 | 14 | // NewCMutex is ctor for CMutex 15 | func NewCMutex() *CMutex { 16 | return &CMutex{sema: semaphore.NewWeighted(1)} 17 | } 18 | 19 | // Lock with context 20 | func (m *CMutex) Lock(ctx context.Context) (err error) { 21 | err = m.sema.Acquire(ctx, 1) 22 | return 23 | } 24 | 25 | // Unlock should only be called after a successful Lock 26 | func (m *CMutex) Unlock() { 27 | m.sema.Release(1) 28 | } 29 | 30 | // TryLock returns true if lock acquired 31 | func (m *CMutex) TryLock() bool { 32 | return m.sema.TryAcquire(1) 33 | } 34 | -------------------------------------------------------------------------------- /deadlock/README.md: -------------------------------------------------------------------------------- 1 | # deadlock 2 | 3 | This package implements a deadlock detector with easy to prove correctness, just replace `sync.Mutex` and `sync.RWMutex` with `deadlock.NewMutex()` and `deadlock.NewRWMutex()` and you're ready to go!!! 4 | 5 | When something wrong happend, panic will happen. 6 | 7 | Just catch the `panic` and handle it with `ParsePanicError`, like following: 8 | 9 | ```golang 10 | defer func() { 11 | panicErr := recover() 12 | errDeadlock, errUsage := ParsePanicError(panicErr) 13 | }() 14 | 15 | ``` 16 | 17 | If everything went well, both `errDeadlock` and `errUsage` should be nil. 18 | 19 | If deadlock happend, `errDeadlock` is non nil. 20 | 21 | If usage problem happens, like the same goroutine calls `Locks` on the same `Mutex` multiple times, `errUsage` is non nil. -------------------------------------------------------------------------------- /bytes/aligned.go: -------------------------------------------------------------------------------- 1 | package bytes 2 | 3 | import "unsafe" 4 | 5 | const ( 6 | // Align4Mask for 4 bytes boundary 7 | Align4Mask = 3 8 | // Align8Mask for 8 bytes boundary 9 | Align8Mask = 7 10 | ) 11 | 12 | // AlignedTo4 returns a byte slice aligned to 4 bytes boundary 13 | func AlignedTo4(n uint32) []byte { 14 | buf := make([]byte, int(n)+Align4Mask) 15 | buf0Alignment := uint32(uintptr(unsafe.Pointer(&buf[0]))) & uint32(Align4Mask) 16 | buf = buf[buf0Alignment : buf0Alignment+n] 17 | return buf 18 | } 19 | 20 | // AlignedTo8 returns a byte slice aligned to 8 bytes boundary 21 | func AlignedTo8(n uint32) []byte { 22 | buf := make([]byte, int(n)+Align8Mask) 23 | // return buf 24 | buf0Alignment := uint32(uintptr(unsafe.Pointer(&buf[0]))) & uint32(Align8Mask) 25 | buf = buf[buf0Alignment : buf0Alignment+n] 26 | return buf 27 | } 28 | -------------------------------------------------------------------------------- /rpheap/README.md: -------------------------------------------------------------------------------- 1 | # rpheap 2 | 3 | An easy to use rank pairing heap, which takes **O(1)** for `Insert`, `FindMin`, `Meld`, and **O(log n)** for `DeleteMin`. Rank pairing heap is one of the most efficient heaps so far! 4 | 5 | ```golang 6 | 7 | package rpheap 8 | 9 | import ( 10 | "sort" 11 | "testing" 12 | 13 | "gotest.tools/assert" 14 | "github.com/zhiqiangxu/rpheap" 15 | ) 16 | 17 | func TestRPHeap(t *testing.T) { 18 | heap := rpheap.New() 19 | numbers := []int{10, 4, 3, 2, 5, 1} 20 | for _, number := range numbers { 21 | rpheap.Insert(int64(number)) 22 | } 23 | 24 | sort.Ints(numbers) 25 | 26 | for _, number := range numbers { 27 | m := heap.DeleteMin() 28 | assert.Assert(t, int64(number) == m, "number:%v m:%v", number, m) 29 | } 30 | 31 | assert.Assert(t, heap.Size() == 0, "heap not empty") 32 | } 33 | 34 | ``` -------------------------------------------------------------------------------- /diskqueue/conf.go: -------------------------------------------------------------------------------- 1 | package diskqueue 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | // CustomDecoder for customized packets 10 | type CustomDecoder func(context.Context, *QfileSizeReader) (bool, []byte, error) 11 | 12 | // Conf for diskqueue 13 | type Conf struct { 14 | Directory string 15 | WriteBatch int 16 | WriteMmap bool 17 | MaxMsgSize int 18 | CustomDecoder CustomDecoder 19 | MaxPutting int 20 | EnableWriteBuffer bool 21 | MaxFileSize int64 22 | PersistDuration time.Duration // GC works at qfile granularity 23 | // below only valid when EnableWriteBuffer is true 24 | // unit: second 25 | CommitInterval int 26 | WriteBufferPool *sync.Pool 27 | // below are modified internally for cache 28 | writeBufferPool *sync.Pool 29 | customDecoder bool 30 | } 31 | -------------------------------------------------------------------------------- /skl/lf/list_test.go: -------------------------------------------------------------------------------- 1 | package lf 2 | 3 | import ( 4 | "testing" 5 | 6 | "reflect" 7 | 8 | "gotest.tools/assert" 9 | ) 10 | 11 | func TestList(t *testing.T) { 12 | l := NewList(1024) 13 | isNew, err := l.Insert([]byte("a"), []byte("a")) 14 | assert.Assert(t, isNew && err == nil) 15 | 16 | v, exists := l.Get([]byte("a")) 17 | assert.Assert(t, exists && reflect.DeepEqual(v, []byte("a"))) 18 | 19 | isNew, err = l.Insert([]byte("a"), []byte("b")) 20 | assert.Assert(t, !isNew && err == nil) 21 | 22 | v, exists = l.Get([]byte("a")) 23 | assert.Assert(t, exists && reflect.DeepEqual(v, []byte("b"))) 24 | 25 | isNew, err = l.Insert([]byte("b"), []byte("b")) 26 | assert.Assert(t, isNew && err == nil) 27 | 28 | deleted := l.Delete([]byte("a")) 29 | assert.Assert(t, deleted) 30 | v, exists = l.Get([]byte("a")) 31 | assert.Assert(t, !exists && v == nil) 32 | } 33 | -------------------------------------------------------------------------------- /hash.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "hash/crc32" 5 | "hash/fnv" 6 | ) 7 | 8 | // CRC32 computes the Castagnoli CRC32 of the given data. 9 | func CRC32(data []byte) (sum32 uint32, err error) { 10 | hash := crc32.New(crc32.MakeTable(crc32.Castagnoli)) 11 | if _, err = hash.Write(data); err != nil { 12 | return 13 | } 14 | sum32 = hash.Sum32() 15 | return 16 | } 17 | 18 | // FNV32 returns a new 32-bit FNV-1 hash.Hash 19 | func FNV32(data []byte) (sum32 uint32, err error) { 20 | hash := fnv.New32() 21 | if _, err = hash.Write(data); err != nil { 22 | return 23 | } 24 | sum32 = hash.Sum32() 25 | return 26 | } 27 | 28 | // FNV32a returns a new 32-bit FNV-1a hash.Hash 29 | func FNV32a(data []byte) (sum32 uint32, err error) { 30 | hash := fnv.New32() 31 | if _, err = hash.Write(data); err != nil { 32 | return 33 | } 34 | sum32 = hash.Sum32() 35 | return 36 | } 37 | -------------------------------------------------------------------------------- /hack_amd64.go: -------------------------------------------------------------------------------- 1 | //go:build amd64 2 | // +build amd64 3 | 4 | package util 5 | 6 | func getg() *g 7 | 8 | type stack struct { 9 | lo uintptr 10 | hi uintptr 11 | } 12 | type g struct { 13 | // Stack parameters. 14 | // stack describes the actual stack memory: [stack.lo, stack.hi). 15 | // stackguard0 is the stack pointer compared in the Go stack growth prologue. 16 | // It is stack.lo+StackGuard normally, but can be StackPreempt to trigger a preemption. 17 | // stackguard1 is the stack pointer compared in the C stack growth prologue. 18 | // It is stack.lo+StackGuard on g0 and gsignal stacks. 19 | // It is ~0 on other goroutine stacks, to trigger a call to morestackc (and crash). 20 | stack stack // offset known to runtime/cgo 21 | } 22 | 23 | // StackSize calculates current stack size 24 | func StackSize() uintptr { 25 | g := getg() 26 | return g.stack.hi - g.stack.lo 27 | } 28 | -------------------------------------------------------------------------------- /math/polynomial_test.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | import ( 4 | "math/big" 5 | "testing" 6 | 7 | "gotest.tools/assert" 8 | ) 9 | 10 | func intArray2BigArray(coefficients []int) []*big.Int { 11 | result := make([]*big.Int, len(coefficients)) 12 | for i, c := range coefficients { 13 | result[i] = big.NewInt(int64(c)) 14 | } 15 | return result 16 | } 17 | 18 | func TestPolynomial(t *testing.T) { 19 | 20 | // euler formula to calculate Partition(5) 21 | p1 := NewPolynomial(intArray2BigArray([]int{1, 1, 1, 1, 1, 1})) 22 | p2 := NewPolynomial(intArray2BigArray([]int{1, 0, 1, 0, 1})) 23 | p3 := NewPolynomial(intArray2BigArray([]int{1, 0, 0, 1})) 24 | p4 := NewPolynomial(intArray2BigArray([]int{1, 0, 0, 0, 1})) 25 | p5 := NewPolynomial(intArray2BigArray([]int{1, 0, 0, 0, 0, 1})) 26 | 27 | p := p1.Mul(p2).Mul(p3).Mul(p4).Mul(p5) 28 | assert.Equal(t, p.coefficients[5].Uint64(), uint64(7)) 29 | t.Log(p) 30 | } 31 | -------------------------------------------------------------------------------- /skl/skl_iterator.go: -------------------------------------------------------------------------------- 1 | package skl 2 | 3 | type sklIter struct { 4 | s *skl 5 | node *element 6 | } 7 | 8 | func (it *sklIter) First() (ok bool) { 9 | nele := it.s.links[0].next 10 | if nele != nil { 11 | ok = true 12 | it.node = nele 13 | } 14 | return 15 | } 16 | 17 | func (it *sklIter) SeekGE(key int64) (ok bool) { 18 | 19 | prevs := it.s.getPrevLinks(key) 20 | 21 | ele := prevs[0].next 22 | if ele != nil { 23 | ok = true 24 | it.node = ele 25 | } 26 | 27 | return 28 | } 29 | 30 | func (it *sklIter) Next() (ok bool) { 31 | ele := it.node.links[0].next 32 | if ele != nil { 33 | ok = true 34 | it.node = ele 35 | } 36 | return 37 | } 38 | 39 | func (it *sklIter) Valid() bool { 40 | return it.node != nil 41 | } 42 | 43 | func (it *sklIter) Key() int64 { 44 | return it.node.key 45 | } 46 | 47 | func (it *sklIter) KeyValue() (int64, interface{}) { 48 | return it.node.key, it.node.value 49 | } 50 | -------------------------------------------------------------------------------- /logger/instance.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "fmt" 5 | 6 | "go.uber.org/zap" 7 | ) 8 | 9 | // EnvType for environment type 10 | type EnvType int 11 | 12 | const ( 13 | // Prod for production environment (default) 14 | Prod EnvType = iota 15 | // Dev for develop environment 16 | Dev 17 | ) 18 | 19 | // Env for chosen environment 20 | var Env EnvType 21 | 22 | var ( 23 | logger *zap.Logger 24 | ) 25 | 26 | // SetInstance for set logger instance 27 | func SetInstance(zl *zap.Logger) { 28 | logger = zl 29 | } 30 | 31 | // Instance for chosen logger 32 | // you can change the actual instance by either: 33 | // 1. change Env 34 | // 2. call SetInstance 35 | func Instance() *zap.Logger { 36 | if logger != nil { 37 | return logger 38 | } 39 | switch Env { 40 | case Prod: 41 | return ProdInstance() 42 | case Dev: 43 | return DevInstance() 44 | default: 45 | panic(fmt.Sprintf("unknown EnvType:%v", Env)) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /crypto/signature/signature_test.go: -------------------------------------------------------------------------------- 1 | package signature 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/assert" 7 | ) 8 | 9 | func TestSignature(t *testing.T) { 10 | pri, pub, err := GenerateKeypair() 11 | 12 | assert.Assert(t, err == nil) 13 | 14 | msg := []byte("hello msg") 15 | sig, err := Sign(pri, msg) 16 | assert.Assert(t, err == nil) 17 | 18 | ok, err := Verify(pub, msg, sig) 19 | assert.Assert(t, err == nil && ok) 20 | 21 | sigbytes, err := sig.Marshal() 22 | assert.Assert(t, err == nil) 23 | var sig2 Signature 24 | err = sig2.Unmarshal(sigbytes) 25 | assert.Assert(t, err == nil) 26 | ok, err = Verify(pub, msg, &sig2) 27 | assert.Assert(t, err == nil && ok) 28 | } 29 | 30 | func BenchmarkSignature(b *testing.B) { 31 | pri, pub, _ := GenerateKeypair() 32 | msg := []byte("hello msg") 33 | b.RunParallel(func(pb *testing.PB) { 34 | for pb.Next() { 35 | sig, _ := Sign(pri, msg) 36 | 37 | Verify(pub, msg, sig) 38 | } 39 | 40 | }) 41 | } 42 | -------------------------------------------------------------------------------- /crypto/claim/jwt.go: -------------------------------------------------------------------------------- 1 | package claim 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/dgrijalva/jwt-go" 8 | ) 9 | 10 | // JWT is both signer and verifier 11 | type JWT struct { 12 | expire time.Duration 13 | method jwt.SigningMethod 14 | secret []byte 15 | } 16 | 17 | // NewJWT is ctor for JWT 18 | func NewJWT(expire time.Duration, secret []byte) (j *JWT, err error) { 19 | signingAlgorithm := "HS256" 20 | method := jwt.GetSigningMethod(signingAlgorithm) 21 | if method == nil { 22 | err = fmt.Errorf("invalid signingAlgorithm:%s", method) 23 | return 24 | } 25 | j = &JWT{expire: expire, method: method, secret: secret} 26 | return 27 | } 28 | 29 | // Sign claims 30 | func (j *JWT) Sign(values map[string]interface{}) (tokenString string, err error) { 31 | 32 | tokenString, err = sign(values, j.expire, j.method, j.secret) 33 | 34 | return 35 | } 36 | 37 | // Verify claims 38 | func (j *JWT) Verify(tokenString string) (ok bool, values map[string]interface{}) { 39 | ok, values = verify(tokenString, j.method, j.secret) 40 | return 41 | } 42 | -------------------------------------------------------------------------------- /mmr/hasher.go: -------------------------------------------------------------------------------- 1 | package mmr 2 | 3 | import "crypto/sha256" 4 | 5 | // HashType can be replaced by https://github.com/zhiqiangxu/gg 6 | type HashType [32]byte 7 | 8 | // Hasher for mmr 9 | type Hasher interface { 10 | Empty() HashType 11 | Leaf(data []byte) HashType 12 | Node(left, right HashType) HashType 13 | } 14 | 15 | type hasher32 struct { 16 | leafPrefix []byte 17 | nodePrefix []byte 18 | } 19 | 20 | // NewHasher creates a Hasher 21 | func NewHasher(leafPrefix, nodePrefix []byte) Hasher { 22 | return &hasher32{leafPrefix: leafPrefix, nodePrefix: nodePrefix} 23 | } 24 | 25 | func (self *hasher32) Empty() HashType { 26 | return sha256.Sum256(nil) 27 | } 28 | 29 | func (self *hasher32) Leaf(leaf []byte) HashType { 30 | data := append([]byte{}, self.leafPrefix...) 31 | data = append(data, leaf...) 32 | return sha256.Sum256(data) 33 | } 34 | 35 | func (self *hasher32) Node(left, right HashType) HashType { 36 | data := append([]byte{}, self.nodePrefix...) 37 | data = append(data, left[:]...) 38 | data = append(data, right[:]...) 39 | return sha256.Sum256(data) 40 | } 41 | -------------------------------------------------------------------------------- /lf/mcas/api_test.go: -------------------------------------------------------------------------------- 1 | package mcas 2 | 3 | import ( 4 | "testing" 5 | "unsafe" 6 | 7 | "gotest.tools/assert" 8 | ) 9 | 10 | func TestMCAS(t *testing.T) { 11 | v1 := 1 12 | v2 := 2 13 | v3 := 3 14 | v4 := 4 15 | 16 | // try to compare and swap p1 and p2 atomically 17 | 18 | // p1 and p2 initially points to v1 and v2 19 | p1 := unsafe.Pointer(&v1) 20 | p2 := unsafe.Pointer(&v2) 21 | 22 | a := []*unsafe.Pointer{&p1, &p2} 23 | e := []unsafe.Pointer{unsafe.Pointer(&v1), unsafe.Pointer(&v2)} 24 | n := []unsafe.Pointer{unsafe.Pointer(&v3), unsafe.Pointer(&v4)} 25 | 26 | swapped := CompareAndSwap(a, e, n) 27 | assert.Assert(t, swapped) 28 | 29 | // assert that p1 and p2 should be swapped to v3 and v4 30 | p1v := Read(&p1) 31 | p2v := Read(&p2) 32 | assert.Assert(t, p1v == unsafe.Pointer(&v3) && p2v == unsafe.Pointer(&v4)) 33 | assert.Assert(t, p1 == p1v && p2 == p2v) 34 | 35 | swapped = CompareAndSwap(a, e, n) 36 | assert.Assert(t, !swapped) 37 | 38 | p1v = Read(&p1) 39 | p2v = Read(&p2) 40 | assert.Assert(t, p1v == unsafe.Pointer(&v3) && p2v == unsafe.Pointer(&v4)) 41 | } 42 | -------------------------------------------------------------------------------- /mapped/file_test.go: -------------------------------------------------------------------------------- 1 | package mapped 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | "testing" 7 | 8 | "gotest.tools/assert" 9 | ) 10 | 11 | func TestFile(t *testing.T) { 12 | 13 | fileName := "/tmp/test_file" 14 | os.Remove(fileName) 15 | f, err := CreateFile(fileName, 64000, false, nil) 16 | assert.Assert(t, err == nil) 17 | 18 | mtime, err := f.LastModified() 19 | assert.Assert(t, err == nil) 20 | mtime2, err := f.LastModified() 21 | assert.Assert(t, err == nil && mtime2 == mtime) 22 | 23 | wbytes := []byte("123") 24 | n, err := f.Write(wbytes) 25 | assert.Assert(t, err == nil && n == len(wbytes)) 26 | 27 | mtime2, err = f.LastModified() 28 | assert.Assert(t, err == nil && mtime2 != mtime) 29 | 30 | rbytes := make([]byte, len(wbytes)) 31 | n, err = f.Read(0, rbytes) 32 | assert.Assert(t, err == nil && n == len(wbytes) && bytes.Equal(wbytes, rbytes)) 33 | err = f.Close() 34 | assert.Assert(t, err == nil) 35 | 36 | f, err = CreateFile(fileName, 64000, false, nil) 37 | // 已经存在 38 | assert.Assert(t, err != nil) 39 | os.Remove(fileName) 40 | // 已删除 41 | f, err = OpenFile(fileName, 64000, os.O_RDWR, false, nil) 42 | assert.Assert(t, err != nil) 43 | } 44 | -------------------------------------------------------------------------------- /deadlock/detector_test.go: -------------------------------------------------------------------------------- 1 | package deadlock 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestDetector(t *testing.T) { 9 | var ( 10 | lock1, lock2 Mutex 11 | ) 12 | lock1.Init() 13 | lock2.Init() 14 | 15 | var ( 16 | errDL *ErrorDeadlock 17 | errUsage *ErrorUsage 18 | ) 19 | 20 | go func() { 21 | lock1.Lock() 22 | 23 | time.Sleep(time.Millisecond * 200) 24 | 25 | defer func() { 26 | panicErr := recover() 27 | errDL, errUsage = ParsePanicError(panicErr) 28 | }() 29 | 30 | lock2.Lock() 31 | 32 | select {} 33 | }() 34 | 35 | time.Sleep(time.Millisecond * 100) 36 | 37 | go func() { 38 | lock2.Lock() 39 | 40 | lock1.Lock() 41 | }() 42 | 43 | time.Sleep(time.Millisecond * 300) 44 | if errDL == nil { 45 | t.FailNow() 46 | } 47 | if errUsage != nil { 48 | t.FailNow() 49 | } 50 | 51 | } 52 | 53 | func TestMap(t *testing.T) { 54 | m := make(map[int]int) 55 | 56 | m[1]++ 57 | 58 | if m[1] != 1 { 59 | t.FailNow() 60 | } 61 | 62 | m[1]-- 63 | m[1]-- 64 | 65 | if m[1] != -1 { 66 | t.FailNow() 67 | } 68 | 69 | var nilMap map[int]int 70 | if _, ok := nilMap[1]; ok { 71 | t.FailNow() 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /crypto/wnaf/rl_test.go: -------------------------------------------------------------------------------- 1 | package wnaf 2 | 3 | import ( 4 | "math/big" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | func TestToWnafRL(t *testing.T) { 10 | testCases := []struct { 11 | D *big.Int 12 | Wnaf []int64 13 | W uint8 14 | }{ 15 | // quoted from https://rd.springer.com/content/pdf/10.1007/978-3-540-68914-0_26.pdf Page 11 16 | { 17 | D: big.NewInt(0b11101001100100010101110101010111), 18 | Wnaf: []int64{7, 0, 0, 0, 5, 0, 0, 0, -3, 0, 0, 0, 0, -5, 0, 0, 0, -7, 0, 0, 0, -3, 0, 0, 0, 5, 0, 0, 0, 7}, 19 | W: 4, 20 | }, 21 | { 22 | D: big.NewInt(0b11101001100100010101110101010111), 23 | Wnaf: []int64{-1, 0, 0, 3, 0, 0, -3, 0, 0, -1, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, -3, 0, 0, 0, 0, 1}, 24 | W: 3, 25 | }, 26 | { 27 | D: big.NewInt(0), 28 | Wnaf: []int64{0}, 29 | W: 8, 30 | }, 31 | { 32 | D: big.NewInt(1 << 8), 33 | Wnaf: []int64{0, 0, 0, 0, 0, 0, 0, 0, 1}, 34 | W: 8, 35 | }, 36 | } 37 | 38 | for _, testCase := range testCases { 39 | ret := ToWnafRL(testCase.D, testCase.W) 40 | if !reflect.DeepEqual(ret, testCase.Wnaf) { 41 | t.Fatalf("actual %v vs expected %v, w %d", ret, testCase.Wnaf, testCase.W) 42 | } 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /crypto/wnaf/lr_test.go: -------------------------------------------------------------------------------- 1 | package wnaf 2 | 3 | import ( 4 | "math/big" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | func TestToWnafLR(t *testing.T) { 10 | testCases := []struct { 11 | D *big.Int 12 | Wnaf []int64 13 | W uint8 14 | }{ 15 | // quoted from https://rd.springer.com/content/pdf/10.1007/978-3-540-68914-0_26.pdf Page 11 16 | { 17 | D: big.NewInt(0b11101001100100010101110101010111), 18 | Wnaf: []int64{0, 0, 7, 0, 0, 0, 5, 0, 0, 0, 0, -7, 0, 0, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, -2}, 19 | W: 4, 20 | }, 21 | { 22 | D: big.NewInt(0b11101001100100010101110101010111), 23 | Wnaf: []int64{2, 0, 0, 0, -3, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 3, 0, -1, 0, 0, 0, 0, -3, 0, 0, 3, 0, -1, 0, 0, 0, -2}, 24 | W: 3, 25 | }, 26 | { 27 | D: big.NewInt(0b1110), 28 | Wnaf: []int64{0, 0, 7, 0}, 29 | W: 4, 30 | }, 31 | { 32 | D: big.NewInt(0b111), 33 | Wnaf: []int64{0, 0, 7}, 34 | W: 4, 35 | }, 36 | } 37 | 38 | for _, testCase := range testCases { 39 | ret := ToWnafLR(testCase.D, testCase.W) 40 | if !reflect.DeepEqual(ret, testCase.Wnaf) { 41 | t.Fatalf("actual %v vs expected %v, w %d", ret, testCase.Wnaf, testCase.W) 42 | } 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /math/pi.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | // Pi returns # of primes LTE than n 4 | func Pi(n int) int { 5 | if n <= 1 { 6 | return 0 7 | } 8 | 9 | // store numbers in range [2,n] 10 | numbers := make([]int, n-1) 11 | for i := range numbers { 12 | numbers[i] = i + 2 13 | } 14 | 15 | count := 0 16 | // sieve method 17 | for i, v := range numbers { 18 | if v == i+2 { 19 | count++ 20 | // v is a prime 21 | for j := i + v; j <= n-2; j += v { 22 | numbers[j] = 0 23 | } 24 | } 25 | } 26 | 27 | return count 28 | 29 | } 30 | 31 | // R2 returns # of ordered prime pairs with sum n 32 | func R2(n int) int { 33 | if n <= 1 { 34 | return 0 35 | } 36 | 37 | // store numbers in range [2,n] 38 | numbers := make([]int, n-1) 39 | for i := range numbers { 40 | numbers[i] = i + 2 41 | } 42 | 43 | // sieve method 44 | for i, v := range numbers { 45 | if v == i+2 { 46 | // v is a prime 47 | for j := i + v; j <= n-2; j += v { 48 | numbers[j] = 0 49 | } 50 | } 51 | } 52 | 53 | count := 0 54 | for i, v := range numbers { 55 | if v == i+2 { 56 | // v is a prime 57 | j := n - v - 2 58 | if j >= 0 && numbers[j] > 0 { 59 | count++ 60 | numbers[j] = 0 61 | } 62 | } 63 | } 64 | return count 65 | } 66 | -------------------------------------------------------------------------------- /xorlayer/xtm_test.go: -------------------------------------------------------------------------------- 1 | package xorlayer 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "sort" 8 | 9 | "math/rand" 10 | 11 | "gotest.tools/assert" 12 | ) 13 | 14 | func TestXTM(t *testing.T) { 15 | nodeID := NodeID(1) 16 | k, h := 4, 1 17 | x := NewXTM(k, h, nodeID, nil) 18 | 19 | assert.Assert(t, x.getBucketIdx(nodeID) == bitSize) 20 | 21 | total := 10000 22 | cookie := uint64(0) 23 | neighbours := make(map[NodeID]bool) 24 | for i := 0; i < total; i++ { 25 | n := NodeID(rand.Uint64()) 26 | if n == nodeID { 27 | continue 28 | } 29 | if neighbours[n] { 30 | continue 31 | } 32 | neighbours[n] = true 33 | x.AddNeighbour(n, cookie) 34 | } 35 | 36 | kclosest := x.KClosest(nodeID) 37 | sort.Slice(kclosest, func(i, j int) bool { 38 | return kclosest[i] < kclosest[j] 39 | }) 40 | 41 | neighbourSlice := make([]NodeID, 0, len(neighbours)) 42 | for n := range neighbours { 43 | neighbourSlice = append(neighbourSlice, n) 44 | } 45 | sort.Slice(neighbourSlice, func(i, j int) bool { 46 | return neighbourSlice[i]^nodeID < neighbourSlice[j]^nodeID 47 | }) 48 | 49 | expected := []NodeID{nodeID} 50 | expected = append(expected, neighbourSlice[0:k-1]...) 51 | assert.Assert(t, reflect.DeepEqual(kclosest, expected)) 52 | } 53 | -------------------------------------------------------------------------------- /closer/naive.go: -------------------------------------------------------------------------------- 1 | package closer 2 | 3 | import "sync" 4 | 5 | // Naive holds the two things we need to close a goroutine and wait for it to finish: a chan 6 | // to tell the goroutine to shut down, and a WaitGroup with which to wait for it to finish shutting 7 | // down. 8 | // The happens before relationship between Add and Wait is guaranteed by the caller side. 9 | type Naive struct { 10 | waiting sync.WaitGroup 11 | done chan struct{} 12 | } 13 | 14 | // NewNaive is ctor for Naive 15 | func NewNaive() *Naive { 16 | return &Naive{done: make(chan struct{})} 17 | } 18 | 19 | // WaitGroupRef returns the reference of WaitGroup 20 | func (n *Naive) WaitGroupRef() *sync.WaitGroup { 21 | return &n.waiting 22 | } 23 | 24 | // Add delta to WaitGroup 25 | func (n *Naive) Add(delta int) { 26 | n.waiting.Add(delta) 27 | } 28 | 29 | // ClosedSignal gets signaled when Wait() is called. 30 | func (n *Naive) ClosedSignal() <-chan struct{} { 31 | return n.done 32 | } 33 | 34 | // Done calls Done() on the WaitGroup. 35 | func (n *Naive) Done() { 36 | n.waiting.Done() 37 | } 38 | 39 | // SignalAndWait closes chan and wait on the WaitGroup. 40 | // Call it more than once will panic 41 | func (n *Naive) SignalAndWait() { 42 | close(n.done) 43 | n.waiting.Wait() 44 | } 45 | -------------------------------------------------------------------------------- /diskqueue/qfile_size_reader.go: -------------------------------------------------------------------------------- 1 | package diskqueue 2 | 3 | import "context" 4 | 5 | type qfileSizeReaderInterface interface { 6 | Read(ctx context.Context, sizeBytes []byte) (err error) 7 | NextOffset() int64 8 | } 9 | 10 | var _ qfileSizeReaderInterface = (*QfileSizeReader)(nil) 11 | 12 | // QfileSizeReader for read qfile by size 13 | type QfileSizeReader struct { 14 | qf *qfile 15 | fileOffset int64 16 | isLatest bool 17 | } 18 | 19 | // if ctx is nil, won't wait for commit 20 | func (r *QfileSizeReader) Read(ctx context.Context, sizeBytes []byte) (err error) { 21 | 22 | _, err = r.qf.mappedFile.ReadRLocked(r.fileOffset, sizeBytes) 23 | if err != nil { 24 | if !r.isLatest { 25 | return 26 | } 27 | if ctx == nil { 28 | // nil means don't wait 29 | return 30 | } 31 | err = r.qf.q.wm.Wait(ctx, r.qf.startOffset+r.fileOffset+int64(len(sizeBytes))) 32 | if err != nil { 33 | return 34 | } 35 | _, err = r.qf.mappedFile.ReadRLocked(r.fileOffset, sizeBytes) 36 | if err != nil { 37 | // 说明换文件了 38 | return 39 | } 40 | } 41 | 42 | r.fileOffset += int64(len(sizeBytes)) 43 | return 44 | } 45 | 46 | // NextOffset returns the next offset to read from 47 | func (r *QfileSizeReader) NextOffset() int64 { 48 | return r.qf.startOffset + r.fileOffset 49 | } 50 | -------------------------------------------------------------------------------- /crypto/wnaf/rl.go: -------------------------------------------------------------------------------- 1 | package wnaf 2 | 3 | import ( 4 | "math/big" 5 | ) 6 | 7 | // FYI https://en.wikipedia.org/wiki/Elliptic_curve_point_multiplication#w-ary_non-adjacent_form_(wNAF)_method 8 | // https://cs.uwaterloo.ca/~shallit/Papers/bbr.pdf 9 | 10 | var ( 11 | zero = big.NewInt(0) 12 | one = big.NewInt(1) 13 | two = big.NewInt(2) 14 | ) 15 | 16 | // ToWnafRL converts a big int to wnaf form from right to left. 17 | // the returned wnaf "bits" are in little endian. 18 | // each "bit" is in range [-2^(w-1), 2^(w-1)) 19 | func ToWnafRL(d *big.Int, w uint8) (wnaf []int64) { 20 | if d.Cmp(zero) < 0 { 21 | panic("negative d not supported") 22 | } 23 | if w > 64 { 24 | panic("w > 64 not supported") 25 | } 26 | 27 | mod := big.NewInt(0).Exp(two, big.NewInt(int64(w)), nil) 28 | halfMod := big.NewInt(0).Div(mod, two) 29 | 30 | for { 31 | if d.Cmp(zero) <= 0 { 32 | break 33 | } 34 | if big.NewInt(0).Mod(d, two).Uint64() == 1 { 35 | // mods 36 | di := big.NewInt(0).Mod(d, mod) 37 | if di.Cmp(halfMod) >= 0 { 38 | di.Sub(di, mod) 39 | } 40 | 41 | d.Sub(d, di) 42 | wnaf = append(wnaf, di.Int64()) 43 | } else { 44 | wnaf = append(wnaf, 0) 45 | } 46 | 47 | d.Div(d, two) 48 | } 49 | 50 | if len(wnaf) == 0 { 51 | wnaf = append(wnaf, 0) 52 | } 53 | 54 | return 55 | } 56 | -------------------------------------------------------------------------------- /mmap_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "syscall" 8 | 9 | "gotest.tools/assert" 10 | ) 11 | 12 | func TestMmap(t *testing.T) { 13 | fileName := "/tmp/test_util.txt" 14 | os.Remove(fileName) 15 | f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) 16 | assert.Assert(t, err == nil) 17 | defer func() { 18 | f.Close() 19 | os.Remove(fileName) 20 | }() 21 | 22 | size := 64000 23 | n, err := f.Write(make([]byte, size)) 24 | assert.Assert(t, n == size && err == nil) 25 | bytes, err := Mmap(f, true, int64(size)) 26 | assert.Assert(t, err == nil && len(bytes) == size) 27 | bytes[0] = 1 28 | assert.Assert(t, MSync(bytes, 1, syscall.MS_SYNC) == nil) 29 | assert.Assert(t, MSync(bytes, 1, syscall.MS_SYNC) == nil) // call twice is ok 30 | assert.Assert(t, MLock(bytes, 1) == nil) 31 | assert.Assert(t, MUnlock(bytes, 1) == nil) 32 | err = Madvise(bytes, true) 33 | assert.Assert(t, err == nil) 34 | err = Madvise(bytes, false) 35 | assert.Assert(t, err == nil) 36 | err = Munmap(bytes) 37 | assert.Assert(t, err == nil) 38 | err = Madvise(bytes, false) 39 | assert.Assert(t, err != nil) 40 | err = Munmap(bytes) 41 | assert.Assert(t, err != nil) 42 | 43 | bytes = make([]byte, 10) 44 | err = Munmap(bytes) 45 | assert.Assert(t, err != nil) 46 | 47 | } 48 | -------------------------------------------------------------------------------- /uuid_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "testing" 5 | 6 | "golang.org/x/exp/rand" 7 | "gotest.tools/assert" 8 | ) 9 | 10 | func TestUUID(t *testing.T) { 11 | 12 | assert.Assert(t, PoorManUUID(true)%2 == 1) 13 | assert.Assert(t, PoorManUUID(false)%2 == 0) 14 | assert.Assert(t, FastRandN(1) == 0) 15 | for i := 2; i < 100; i++ { 16 | assert.Assert(t, FastRandN(uint32(i)) < uint32(i)) 17 | } 18 | 19 | // test empty interface 20 | var i interface{} 21 | 22 | type test struct { 23 | a int 24 | b string 25 | } 26 | 27 | var s test 28 | s.a = 1 29 | s.b = "1" 30 | 31 | i = s 32 | s.a = 2 33 | s.b = "2" 34 | assert.Assert(t, i.(test).a == 1) 35 | 36 | var i2 interface{} 37 | 38 | i2 = i 39 | 40 | i = s 41 | s.a = 3 42 | s.b = "3" 43 | 44 | assert.Assert(t, i.(test).a == 2 && i2.(test).a == 1) 45 | 46 | // // this will error 47 | // i2.(test).a = 3 48 | 49 | // test slice 50 | { 51 | encID := make([]byte, 0, 10) 52 | _ = append(encID, 'a') 53 | assert.Assert(t, len(encID) == 0 && cap(encID) == 10) 54 | } 55 | 56 | } 57 | 58 | func BenchmarkFastRand(b *testing.B) { 59 | for i := 0; i < b.N; i++ { 60 | FastRand() 61 | } 62 | } 63 | 64 | func BenchmarkPCG(b *testing.B) { 65 | r := rand.PCGSource{} 66 | for i := 0; i < b.N; i++ { 67 | r.Uint64() 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /deadlock/mutex.go: -------------------------------------------------------------------------------- 1 | package deadlock 2 | 3 | import ( 4 | "context" 5 | "unsafe" 6 | ) 7 | 8 | // Mutex is like sync.Mutex but with builtin deadlock detecting ability 9 | type Mutex struct { 10 | sema *Weighted 11 | } 12 | 13 | // NewMutex is ctor for Mutex 14 | func NewMutex() *Mutex { 15 | m := &Mutex{} 16 | m.Init() 17 | return m 18 | } 19 | 20 | // Init for embeded usage 21 | func (m *Mutex) Init() { 22 | m.sema = NewWeighted(1, m) 23 | } 24 | 25 | // Lock with context 26 | func (m *Mutex) Lock() (err error) { 27 | err = m.sema.Acquire(context.Background(), 1) 28 | return 29 | } 30 | 31 | // Unlock should only be called after a successful Lock 32 | func (m *Mutex) Unlock() { 33 | m.sema.Release(1) 34 | } 35 | 36 | // TryLock returns true if lock acquired 37 | func (m *Mutex) TryLock() bool { 38 | return m.sema.TryAcquire(1) 39 | } 40 | 41 | func (m *Mutex) onAcquiredLocked(n int64) { 42 | d.onAcquiredLocked(uint64(uintptr(unsafe.Pointer(m))), true) 43 | } 44 | 45 | func (m *Mutex) onWaitLocked(n int64) { 46 | d.onWaitLocked(uint64(uintptr(unsafe.Pointer(m))), true) 47 | } 48 | 49 | func (m *Mutex) onWaitCanceledLocked(n int64) { 50 | panic("this should never happen") 51 | } 52 | 53 | func (m *Mutex) onReleaseLocked(n int64) { 54 | d.onReleaseLocked(uint64(uintptr(unsafe.Pointer(m))), true) 55 | } 56 | -------------------------------------------------------------------------------- /logger/dev.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "sync/atomic" 7 | "unsafe" 8 | 9 | "go.uber.org/zap" 10 | "go.uber.org/zap/zapcore" 11 | ) 12 | 13 | var ( 14 | devLoggerPtr unsafe.Pointer 15 | devMU sync.Mutex 16 | ) 17 | 18 | // DevInstance returns the instance for develop environment 19 | func DevInstance() *zap.Logger { 20 | devLogger := atomic.LoadPointer(&devLoggerPtr) 21 | if devLogger != nil { 22 | return (*zap.Logger)(devLogger) 23 | } 24 | 25 | devMU.Lock() 26 | defer devMU.Unlock() 27 | devLogger = atomic.LoadPointer(&devLoggerPtr) 28 | if devLogger != nil { 29 | return (*zap.Logger)(devLogger) 30 | } 31 | 32 | encoderConfig := zap.NewDevelopmentEncoderConfig() 33 | 34 | zconf := zap.Config{ 35 | DisableCaller: true, 36 | DisableStacktrace: true, 37 | Level: zap.NewAtomicLevelAt(zapcore.DebugLevel), 38 | Development: true, 39 | Encoding: "json", 40 | EncoderConfig: encoderConfig, 41 | OutputPaths: []string{"stdout"}, 42 | ErrorOutputPaths: []string{"stderr"}, 43 | } 44 | 45 | devLoggerType, err := New(zconf) 46 | if err != nil { 47 | panic(fmt.Sprintf("DevInstance New:%v", err)) 48 | } 49 | 50 | atomic.StorePointer(&devLoggerPtr, unsafe.Pointer(devLoggerType)) 51 | 52 | return devLoggerType 53 | } 54 | -------------------------------------------------------------------------------- /bytearena.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // ByteArena for reduce GC pressure, not concurrent safe 4 | type ByteArena struct { 5 | alloc []byte 6 | chunkAllocMinSize int 7 | chunkAllocMaxSize int 8 | } 9 | 10 | // NewByteArena is ctor for ByteArena 11 | func NewByteArena(chunkAllocMinSize, chunkAllocMaxSize int) *ByteArena { 12 | return &ByteArena{chunkAllocMinSize: chunkAllocMinSize, chunkAllocMaxSize: chunkAllocMaxSize} 13 | } 14 | 15 | // AllocBytes for allocate bytes 16 | func (a *ByteArena) AllocBytes(n int) (bytes []byte) { 17 | if cap(a.alloc)-len(a.alloc) < n { 18 | bytes = a.reserveOrAlloc(n) 19 | if bytes != nil { 20 | return 21 | } 22 | } 23 | 24 | pos := len(a.alloc) 25 | bytes = a.alloc[pos : pos+n : pos+n] 26 | a.alloc = a.alloc[:pos+n] 27 | 28 | return 29 | } 30 | 31 | // UnsafeReset for reuse 32 | func (a *ByteArena) UnsafeReset() { 33 | a.alloc = a.alloc[:0] 34 | } 35 | 36 | func (a *ByteArena) reserveOrAlloc(n int) (bytes []byte) { 37 | 38 | allocSize := cap(a.alloc) * 2 39 | if allocSize < a.chunkAllocMinSize { 40 | allocSize = a.chunkAllocMinSize 41 | } else if allocSize > a.chunkAllocMaxSize { 42 | allocSize = a.chunkAllocMaxSize 43 | } 44 | if allocSize <= n { 45 | bytes = make([]byte, 0, n) 46 | return 47 | } 48 | 49 | a.alloc = make([]byte, 0, allocSize) 50 | return 51 | } 52 | -------------------------------------------------------------------------------- /uuid.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "math" 5 | _ "unsafe" // required by go:linkname 6 | ) 7 | 8 | // FYI: https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ 9 | // https://github.com/golang/go/blob/b5c66de0892d0e9f3f59126eeebc31070e79143b/src/runtime/stubs.go#L115 10 | 11 | // FastRand returns a lock free uint32 value. 12 | //go:linkname FastRand runtime.fastrand 13 | func FastRand() uint32 14 | 15 | // FastRandN returns a random number in [0, n) 16 | //go:linkname FastRandN runtime.fastrandn 17 | func FastRandN(uint32) uint32 18 | 19 | // FastRand64 returns a random uint64 without lock 20 | func FastRand64() (result uint64) { 21 | a := FastRand() 22 | b := FastRand() 23 | result = uint64(a)<<32 + uint64(b) 24 | return 25 | } 26 | 27 | // FastRand64N returns, as an uint64, a pseudo-random number in [0,n). 28 | func FastRand64N(n uint64) uint64 { 29 | v := FastRand64() 30 | if n&(n-1) == 0 { // n is power of two, can mask 31 | return v & (n - 1) 32 | } 33 | return v % n 34 | } 35 | 36 | // PoorManUUID generate a uint64 uuid 37 | func PoorManUUID(client bool) (result uint64) { 38 | 39 | result = PoorManUUID2() 40 | if client { 41 | result |= 1 //odd for client 42 | } else { 43 | result &= math.MaxUint64 - 1 //even for server 44 | } 45 | return 46 | } 47 | 48 | // PoorManUUID2 doesn't care whether client/server side 49 | func PoorManUUID2() uint64 { 50 | return FastRand64() 51 | } 52 | -------------------------------------------------------------------------------- /logger/prod.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "sync/atomic" 7 | "unsafe" 8 | 9 | "go.uber.org/zap" 10 | "go.uber.org/zap/zapcore" 11 | ) 12 | 13 | var ( 14 | prodLoggerPtr unsafe.Pointer 15 | prodMU sync.Mutex 16 | ) 17 | 18 | // ProdInstance returns the instance for production environment 19 | func ProdInstance() *zap.Logger { 20 | prodLogger := atomic.LoadPointer(&prodLoggerPtr) 21 | if prodLogger != nil { 22 | return (*zap.Logger)(prodLogger) 23 | } 24 | 25 | prodMU.Lock() 26 | defer prodMU.Unlock() 27 | prodLogger = atomic.LoadPointer(&prodLoggerPtr) 28 | if prodLogger != nil { 29 | return (*zap.Logger)(prodLogger) 30 | } 31 | 32 | encoderConfig := zap.NewProductionEncoderConfig() 33 | encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder 34 | encoderConfig.EncodeDuration = zapcore.StringDurationEncoder 35 | 36 | zconf := zap.Config{ 37 | DisableCaller: true, 38 | DisableStacktrace: true, 39 | Level: zap.NewAtomicLevelAt(zapcore.InfoLevel), 40 | Development: false, 41 | Encoding: "json", 42 | EncoderConfig: encoderConfig, 43 | OutputPaths: []string{"stdout"}, 44 | ErrorOutputPaths: []string{"stderr"}, 45 | } 46 | 47 | prodLoggerType, err := New(zconf) 48 | if err != nil { 49 | panic(fmt.Sprintf("ProdInstance New:%v", err)) 50 | } 51 | 52 | atomic.StorePointer(&prodLoggerPtr, unsafe.Pointer(prodLoggerType)) 53 | 54 | return prodLoggerType 55 | } 56 | -------------------------------------------------------------------------------- /mutex/crwmutex.go: -------------------------------------------------------------------------------- 1 | package mutex 2 | 3 | import ( 4 | "context" 5 | 6 | "golang.org/x/sync/semaphore" 7 | ) 8 | 9 | const rwmutexMaxReaders = 1 << 30 10 | 11 | // CRWMutex implements a cancelable rwmutex (in fact also a try-able rwmutex) 12 | type CRWMutex struct { 13 | sema *semaphore.Weighted 14 | } 15 | 16 | // NewCRWMutex is ctor for CRWMutex 17 | func NewCRWMutex() *CRWMutex { 18 | rw := &CRWMutex{} 19 | rw.Init() 20 | return rw 21 | } 22 | 23 | // Init for embeded usage 24 | func (rw *CRWMutex) Init() { 25 | rw.sema = semaphore.NewWeighted(rwmutexMaxReaders) 26 | } 27 | 28 | // Lock with context 29 | func (rw *CRWMutex) Lock(ctx context.Context) (err error) { 30 | err = rw.sema.Acquire(ctx, rwmutexMaxReaders) 31 | return 32 | } 33 | 34 | // Unlock should only be called after a successful Lock 35 | func (rw *CRWMutex) Unlock() { 36 | rw.sema.Release(rwmutexMaxReaders) 37 | return 38 | } 39 | 40 | // RLock with context 41 | func (rw *CRWMutex) RLock(ctx context.Context) (err error) { 42 | err = rw.sema.Acquire(ctx, 1) 43 | return 44 | } 45 | 46 | // RUnlock should only be called after a successful RLock 47 | func (rw *CRWMutex) RUnlock() { 48 | rw.sema.Release(1) 49 | } 50 | 51 | // TryLock returns true if lock acquired 52 | func (rw *CRWMutex) TryLock() bool { 53 | return rw.sema.TryAcquire(rwmutexMaxReaders) 54 | } 55 | 56 | // TryRLock returns true if rlock acquired 57 | func (rw *CRWMutex) TryRLock() bool { 58 | return rw.sema.TryAcquire(1) 59 | } 60 | -------------------------------------------------------------------------------- /workerpool.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "errors" 5 | "runtime" 6 | "sync" 7 | 8 | "github.com/zhiqiangxu/util/logger" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | // WorkerPool is pool of workers 13 | type WorkerPool struct { 14 | wg sync.WaitGroup 15 | doneChan chan struct{} 16 | workChan chan func() 17 | once sync.Once 18 | } 19 | 20 | // NewWorkerPool is ctor for WorkerPool 21 | func NewWorkerPool() *WorkerPool { 22 | wp := &WorkerPool{doneChan: make(chan struct{}), workChan: make(chan func())} 23 | wp.Start() 24 | return wp 25 | } 26 | 27 | // Start worker pool 28 | func (wp *WorkerPool) Start() { 29 | n := runtime.NumCPU() 30 | for i := 0; i < n; i++ { 31 | GoFunc(&wp.wg, func() { 32 | defer func() { 33 | err := recover() 34 | if err != nil { 35 | logger.Instance().Error("workerPool", zap.Any("err", err)) 36 | } 37 | }() 38 | for { 39 | select { 40 | case f := <-wp.workChan: 41 | f() 42 | case <-wp.doneChan: 43 | return 44 | } 45 | } 46 | }) 47 | } 48 | } 49 | 50 | // Close the worker pool 51 | func (wp *WorkerPool) Close() { 52 | wp.once.Do(func() { 53 | close(wp.doneChan) 54 | wp.wg.Wait() 55 | }) 56 | } 57 | 58 | var ( 59 | // ErrWorkerPoolClosed when run on closed pool 60 | ErrWorkerPoolClosed = errors.New("workerPool closed") 61 | ) 62 | 63 | // Run a task 64 | func (wp *WorkerPool) Run(f func()) error { 65 | select { 66 | case wp.workChan <- f: 67 | return nil 68 | case <-wp.doneChan: 69 | return ErrWorkerPoolClosed 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /lf/mcas/ccas.go: -------------------------------------------------------------------------------- 1 | package mcas 2 | 3 | import ( 4 | "sync/atomic" 5 | "unsafe" 6 | ) 7 | 8 | type ccDesc struct { 9 | a *unsafe.Pointer 10 | e unsafe.Pointer 11 | n unsafe.Pointer 12 | sp *uint32 13 | } 14 | 15 | func ccasRead(a *unsafe.Pointer) (v unsafe.Pointer) { 16 | for v = atomic.LoadPointer(a); isCCDesc(v); v = atomic.LoadPointer(a) { 17 | ccfromPointer(v).ccasHelp() 18 | } 19 | return 20 | } 21 | 22 | func isCCDesc(v unsafe.Pointer) bool { 23 | return uintptr(v)&addrMask == ccDescAddr 24 | } 25 | 26 | func ccfromPointer(v unsafe.Pointer) *ccDesc { 27 | ptr := uintptr(v) 28 | ptr = ptr & ^uintptr(addrMask) 29 | return (*ccDesc)(are.getPointer(ptr)) 30 | } 31 | 32 | func (d *ccDesc) toPointer() unsafe.Pointer { 33 | return unsafe.Pointer(uintptr(unsafe.Pointer(d)) + uintptr(ccDescAddr)) 34 | } 35 | 36 | func ccas(a *unsafe.Pointer, e, n unsafe.Pointer, sp *uint32) (ok, swapped, isn bool) { 37 | d := are.putCCDesc() 38 | *d = ccDesc{a: a, e: e, n: n, sp: sp} 39 | var v unsafe.Pointer 40 | for !atomic.CompareAndSwapPointer(d.a, d.e, d.toPointer()) { 41 | v = atomic.LoadPointer(d.a) 42 | if !isCCDesc(v) { 43 | return 44 | } 45 | ccfromPointer(v).ccasHelp() 46 | } 47 | ok = true 48 | swapped, isn = d.ccasHelp() 49 | return 50 | } 51 | 52 | func (d *ccDesc) ccasHelp() (swapped, isn bool) { 53 | s := atomic.LoadUint32(d.sp) 54 | var v unsafe.Pointer 55 | if s == undecided { 56 | isn = true 57 | v = d.n 58 | } else { 59 | v = d.e 60 | } 61 | swapped = atomic.CompareAndSwapPointer(d.a, d.toPointer(), v) 62 | return 63 | } 64 | -------------------------------------------------------------------------------- /lf/mcas/arena.go: -------------------------------------------------------------------------------- 1 | package mcas 2 | 3 | import ( 4 | "sync/atomic" 5 | "unsafe" 6 | 7 | "github.com/zhiqiangxu/util/bytes" 8 | ) 9 | 10 | type arena struct { 11 | offset uint32 12 | buf []byte 13 | } 14 | 15 | const ( 16 | mcDescSize = uint32(unsafe.Sizeof(mcDesc{})) 17 | ccDescSize = uint32(unsafe.Sizeof(ccDesc{})) 18 | ) 19 | 20 | func newArena(size uint32) *arena { 21 | return &arena{buf: bytes.AlignedTo8(size)} 22 | } 23 | 24 | func (a *arena) alloc(size uint32) (offset uint32) { 25 | if int(size) > len(a.buf) { 26 | panic("size > buf size") 27 | } 28 | 29 | // Pad the allocation with enough bytes to ensure pointer alignment. 30 | l := uint32(size + bytes.Align8Mask) 31 | 32 | try: 33 | n := atomic.AddUint32(&a.offset, l) 34 | if int(n) > len(a.buf) { 35 | if atomic.CompareAndSwapUint32(&a.offset, 0, l) { 36 | n = l 37 | goto final 38 | } 39 | goto try 40 | } 41 | 42 | final: 43 | // Return the aligned offset. 44 | offset = (n - l + uint32(bytes.Align8Mask)) & ^uint32(bytes.Align8Mask) 45 | return 46 | } 47 | 48 | func (a *arena) getPointer(ptr uintptr) unsafe.Pointer { 49 | offset := ptr - uintptr(unsafe.Pointer(&a.buf[0])) 50 | return unsafe.Pointer(&a.buf[offset]) 51 | } 52 | 53 | func (a *arena) putMCDesc() *mcDesc { 54 | offset := a.alloc(mcDescSize) 55 | return (*mcDesc)(unsafe.Pointer(&a.buf[offset])) 56 | } 57 | 58 | func (a *arena) putCCDesc() *ccDesc { 59 | offset := a.alloc(ccDescSize) 60 | return (*ccDesc)(unsafe.Pointer(&a.buf[offset])) 61 | } 62 | 63 | var are *arena 64 | 65 | func init() { 66 | // enough for 10w concurrency 67 | are = newArena(1024 * 1024) 68 | } 69 | -------------------------------------------------------------------------------- /mutex/tmutex.go: -------------------------------------------------------------------------------- 1 | package mutex 2 | 3 | import "sync/atomic" 4 | 5 | // TMutex is a mutual exclusion primitive that implements TryLock in addition 6 | // to Lock and Unlock. 7 | type TMutex struct { 8 | v int32 9 | ch chan struct{} 10 | } 11 | 12 | // New is ctor for TMutex 13 | func New() *TMutex { 14 | return &TMutex{v: 1, ch: make(chan struct{}, 1)} 15 | } 16 | 17 | // Lock has the same semantics as normal Mutex.Lock 18 | func (m *TMutex) Lock() { 19 | // Uncontended case. 20 | if atomic.AddInt32(&m.v, -1) == 0 { 21 | return 22 | } 23 | 24 | for { 25 | 26 | // if v < 0, someone is already contending, just wait for lock release 27 | // otherwise, SwapInt32 can only return one of -1, 0, 1 28 | // 1 means no contention 29 | // -1 means someone is already contending 30 | // 0 means no one else is contending but the lock hasn't been released 31 | // so for -1 or 0, just wait for lock release 32 | if v := atomic.LoadInt32(&m.v); v >= 0 && atomic.SwapInt32(&m.v, -1) == 1 { 33 | return 34 | } 35 | 36 | // Wait for the mutex to be released before trying again. 37 | <-m.ch 38 | } 39 | } 40 | 41 | // TryLock returns true on success 42 | func (m *TMutex) TryLock() bool { 43 | v := atomic.LoadInt32(&m.v) 44 | if v <= 0 { 45 | return false 46 | } 47 | return atomic.CompareAndSwapInt32(&m.v, 1, 0) 48 | } 49 | 50 | // Unlock has the same semantics as normal Mutex.Unlock 51 | func (m *TMutex) Unlock() { 52 | if atomic.SwapInt32(&m.v, 1) == 0 { 53 | // There were no pending waiters. 54 | return 55 | } 56 | 57 | // Wake some waiter up. 58 | select { 59 | case m.ch <- struct{}{}: 60 | default: 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /skl/skl_test.go: -------------------------------------------------------------------------------- 1 | package skl 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/assert" 7 | ) 8 | 9 | func TestSKL(t *testing.T) { 10 | // test SkipListIterator 11 | skl := NewSkipList() 12 | { 13 | it := skl.NewIterator() 14 | ok := it.First() 15 | assert.Assert(t, !ok) 16 | } 17 | total := 10 18 | for j := 0; j < 2; j++ { 19 | for i := 0; i < total; i++ { 20 | skl.Add(int64(i), i) 21 | } 22 | } 23 | 24 | assert.Assert(t, skl.Length() == total) 25 | 26 | it := skl.NewIterator() 27 | ok := it.First() 28 | assert.Assert(t, ok) 29 | for i := 0; i < total; i++ { 30 | assert.Assert(t, it.Valid()) 31 | k, v := it.KeyValue() 32 | assert.Assert(t, k == int64(i) && v == i) 33 | if i == total-1 { 34 | assert.Assert(t, !it.Next()) 35 | } else { 36 | assert.Assert(t, it.Next()) 37 | } 38 | } 39 | 40 | for j := 1; j < total-1; j++ { 41 | ok = it.SeekGE(int64(j)) 42 | assert.Assert(t, ok) 43 | for i := j; i < total; i++ { 44 | assert.Assert(t, it.Valid()) 45 | k, v := it.KeyValue() 46 | assert.Assert(t, k == int64(i) && v == i) 47 | if i == total-1 { 48 | assert.Assert(t, !it.Next()) 49 | } else { 50 | assert.Assert(t, it.Next()) 51 | } 52 | } 53 | } 54 | 55 | ok = it.SeekGE(int64(total + 1)) 56 | assert.Assert(t, !ok) 57 | } 58 | 59 | func BenchmarkSKL(b *testing.B) { 60 | skl := NewSkipList() 61 | for i := 0; i < b.N; i++ { 62 | i64 := int64(i) 63 | skl.Add(i64, i) 64 | skl.Get(i64) 65 | } 66 | } 67 | 68 | func BenchmarkMap(b *testing.B) { 69 | m := make(map[int64]interface{}) 70 | for i := 0; i < b.N; i++ { 71 | i64 := int64(i) 72 | m[i64] = i 73 | _ = m[i64] 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /crypto/claim/claim_test.go: -------------------------------------------------------------------------------- 1 | package claim 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | "time" 7 | 8 | "reflect" 9 | 10 | "github.com/dgrijalva/jwt-go" 11 | "gotest.tools/assert" 12 | ) 13 | 14 | func Test(t *testing.T) { 15 | 16 | // test signer and verifier 17 | { 18 | // openssl genrsa -out test.rsa 2048 19 | signBytes, err := ioutil.ReadFile("data/test.rsa") 20 | assert.Assert(t, err == nil) 21 | // openssl rsa -in test.rsa -pubout > test.rsa.pub 22 | verifyBytes, err := ioutil.ReadFile("data/test.rsa.pub") 23 | assert.Assert(t, err == nil) 24 | 25 | signKey, err := jwt.ParseRSAPrivateKeyFromPEM(signBytes) 26 | assert.Assert(t, err == nil) 27 | verifyKey, err := jwt.ParseRSAPublicKeyFromPEM(verifyBytes) 28 | assert.Assert(t, err == nil) 29 | 30 | s, err := NewSigner(time.Hour, signKey) 31 | assert.Assert(t, err == nil) 32 | 33 | v, err := NewVerifier(verifyKey) 34 | assert.Assert(t, err == nil) 35 | 36 | values := map[string]interface{}{ 37 | "c1": "v1", 38 | "c2": "v2", 39 | } 40 | 41 | tokenString, err := s.Sign(values) 42 | assert.Assert(t, err == nil, err) 43 | 44 | ok, valuesVerified := v.Verify(tokenString) 45 | assert.Assert(t, ok && reflect.DeepEqual(valuesVerified, values)) 46 | } 47 | 48 | // test jwt 49 | { 50 | j, err := NewJWT(time.Hour*24, []byte("sec")) 51 | assert.Assert(t, err == nil) 52 | 53 | values := map[string]interface{}{ 54 | "c1": "v1", 55 | "c2": "v2", 56 | } 57 | tokenString, err := j.Sign(values) 58 | assert.Assert(t, err == nil) 59 | 60 | ok, valuesVerified := j.Verify(tokenString) 61 | assert.Assert(t, ok && reflect.DeepEqual(valuesVerified, values)) 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /crypto/claim/signer.go: -------------------------------------------------------------------------------- 1 | package claim 2 | 3 | import ( 4 | "crypto/rsa" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/dgrijalva/jwt-go" 9 | ) 10 | 11 | // Signer for claimer 12 | type Signer struct { 13 | expire time.Duration 14 | method jwt.SigningMethod 15 | signKey *rsa.PrivateKey 16 | } 17 | 18 | // NewSigner is ctor for Signer 19 | func NewSigner(expire time.Duration, signKey *rsa.PrivateKey) (s *Signer, err error) { 20 | signingAlgorithm := "RS256" 21 | method := jwt.GetSigningMethod(signingAlgorithm) 22 | if method == nil { 23 | err = fmt.Errorf("invalid signingAlgorithm:%s", method) 24 | return 25 | } 26 | s = &Signer{expire: expire, method: method, signKey: signKey} 27 | return 28 | } 29 | 30 | const ( 31 | // ExpireATKey for expire_at 32 | ExpireATKey = "expire_at" 33 | // CreatedKey for created 34 | CreatedKey = "created" 35 | ) 36 | 37 | // Sign claims 38 | func (s *Signer) Sign(values map[string]interface{}) (tokenString string, err error) { 39 | 40 | tokenString, err = sign(values, s.expire, s.method, s.signKey) 41 | return 42 | } 43 | 44 | func sign(values map[string]interface{}, expire time.Duration, method jwt.SigningMethod, signKey interface{}) (tokenString string, err error) { 45 | claims := jwt.MapClaims{ 46 | ExpireATKey: time.Now().Add(expire).Unix(), 47 | CreatedKey: time.Now().Unix(), 48 | } 49 | for k, v := range values { 50 | if _, ok := claims[k]; ok { 51 | err = fmt.Errorf("%s is reserved for claims", k) 52 | return 53 | } 54 | claims[k] = v 55 | } 56 | token := jwt.NewWithClaims(method, claims) 57 | // Sign and get the complete encoded token as a string using the secret 58 | tokenString, err = token.SignedString(signKey) 59 | return 60 | } 61 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/zhiqiangxu/util 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/dgrijalva/jwt-go v3.2.1-0.20210802184156-9742bd7fca1c+incompatible 7 | github.com/go-kit/kit v0.9.0 8 | github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 9 | github.com/prometheus/client_golang v1.2.1 10 | github.com/rs/zerolog v1.28.0 11 | github.com/zhiqiangxu/rpheap v0.0.0-20191222053847-9002d7e5a1a1 12 | go.uber.org/zap v1.13.0 13 | golang.org/x/exp v0.0.0-20200320212757-167ffe94c325 14 | golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a 15 | golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b 16 | gotest.tools v2.2.0+incompatible 17 | ) 18 | 19 | require ( 20 | github.com/BurntSushi/toml v0.3.1 // indirect 21 | github.com/VividCortex/gohistogram v1.0.0 // indirect 22 | github.com/beorn7/perks v1.0.1 // indirect 23 | github.com/cespare/xxhash/v2 v2.1.0 // indirect 24 | github.com/golang/protobuf v1.3.2 // indirect 25 | github.com/google/go-cmp v0.3.1 // indirect 26 | github.com/mattn/go-colorable v0.1.12 // indirect 27 | github.com/mattn/go-isatty v0.0.14 // indirect 28 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 29 | github.com/pkg/errors v0.9.1 // indirect 30 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect 31 | github.com/prometheus/common v0.7.0 // indirect 32 | github.com/prometheus/procfs v0.0.5 // indirect 33 | go.uber.org/atomic v1.5.0 // indirect 34 | go.uber.org/multierr v1.3.0 // indirect 35 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee // indirect 36 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de // indirect 37 | golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa // indirect 38 | honnef.co/go/tools v0.0.1-2019.2.3 // indirect 39 | ) 40 | -------------------------------------------------------------------------------- /rpheap/rank_pairing_heap_test.go: -------------------------------------------------------------------------------- 1 | package rpheap 2 | 3 | import ( 4 | "sort" 5 | "testing" 6 | 7 | "gotest.tools/assert" 8 | ) 9 | 10 | func TestRPHeap(t *testing.T) { 11 | rpheap := &Heap{} 12 | numbers := []int{10, 4, 3, 2, 5, 1} 13 | for _, number := range numbers { 14 | rpheap.Insert(int64(number)) 15 | } 16 | 17 | sort.Ints(numbers) 18 | 19 | for _, number := range numbers { 20 | m := rpheap.DeleteMin() 21 | assert.Assert(t, int64(number) == m, "number:%v m:%v", number, m) 22 | } 23 | 24 | assert.Assert(t, rpheap.Size() == 0, "rpheap not empty") 25 | 26 | runTestMeld([]int{2, 8, 5, 7}, []int{4, 9, 6}, t) 27 | runTestMeld([]int{4, 9, 6}, []int{2, 8, 5, 7}, t) 28 | runTestMeld([]int{2}, []int{4, 9, 6}, t) 29 | runTestMeld([]int{2, 8, 5, 7}, []int{4}, t) 30 | runTestMeld([]int{2, 8, 5, 7}, []int{}, t) 31 | runTestMeld([]int{}, []int{4, 9, 6}, t) 32 | 33 | } 34 | 35 | func runTestMeld(arr1, arr2 []int, t *testing.T) { 36 | ans := append(arr1, arr2...) 37 | sort.Ints(ans) 38 | 39 | rpheap1 := &Heap{} 40 | rpheap2 := &Heap{} 41 | for _, number := range arr1 { 42 | rpheap1.Insert(int64(number)) 43 | } 44 | assert.Assert(t, rpheap1.Size() == len(arr1), "rpheap1 size not match") 45 | for _, number := range arr2 { 46 | rpheap2.Insert(int64(number)) 47 | } 48 | assert.Assert(t, rpheap2.Size() == len(arr2), "rpheap2 size not match") 49 | 50 | rpheap1.Meld(rpheap2) 51 | 52 | assert.Assert(t, rpheap2.Size() == 0, "rpheap2 not empty") 53 | assert.Assert(t, rpheap1.Size() == len(ans), "rpheap1 size not match") 54 | for _, number := range ans { 55 | m := rpheap1.DeleteMin() 56 | assert.Assert(t, int64(number) == m, "number:%v m:%v", number, m) 57 | } 58 | 59 | assert.Assert(t, rpheap1.Size() == 0, "rpheap1 not empty") 60 | } 61 | -------------------------------------------------------------------------------- /crypto/claim/data/test.rsa: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAwmHUWUTLSCBe7YTIAQamZvLfDYJ4JhgbvVRjqr54aNMlPqT+ 3 | wA1EQrWhYnmJwYy+8JbQh0KJ8HOFHzHiQbEZUgkNupq0xiLoDFZ0PrpSixtChgVe 4 | rmcfYfaj0gFxjF4TVpoHVWSHslji80ALiht4jfXlhU7NJGNK8VFETuuoQMJIl+h+ 5 | sD0BjWvZfUvN7dTYDYX448Ho1gEc5qlnEsQEdI0UJ+QKm76wb3kq7zg5NnVicfsd 6 | 990qysIFIqtvX/faym7U8HW7cFgUI3NjN3YlxGPPsWpO79NMMPGIUTjNalSxDOW1 7 | Wlu5sEtD4VdXXiQJnykmQ9zaczCjjDrLBYFOLQIDAQABAoIBAF2gn5bMAxpftTm0 8 | 3l1YJaiqzXOFZ3f53sqoLkWc7wSWx3bQJIC/IyT6tZxoLElVENdz3Ud5blWqXvGy 9 | pme79RVFlR5PzEkvoRPoRJ++QWoNnFJPndZlh4E5Opk55saUvb3V/ThNbush01cV 10 | fzlrl/rQ+Vfnp/oh9YQc/8j/y5A86+B7uP0cblKv51VwF4n2aw8J2Lk01/pgusuF 11 | OAkPJpRkUtj6UlPNN052dOCOhsURW23Y+LsX1aKGl0MUYIRxkaLRuoBkfd9XiTbn 12 | grLIPaTvV3w5VSBkAyeIwbHpZkjZo1fP+hOQfYuEar3JwnANuLgvpALpIbw2/+Zc 13 | Ht551qUCgYEA/hENtM/R3wgZlXkuvHZow6EN/FX0QH4upteQOga2Db0cUVKuvoZ3 14 | mWVZzMfj9UoRiPJ8P/A+Y+d1TViQkjZEMzKPje/SZ+4N/EsX2hyy0MwL6PCynroi 15 | nT1D+2FvMQ89A73syWuXAPV/3dAY9P0QAXYkVhdhCp35FLyddoeXxIMCgYEAw9yB 16 | OpyFOGhy6D5YmhtimW2B7ELTAi5PjWvM9MeM08aCV02tS2J6W96VUjU3xe8Qsks9 17 | xh0XGFr5JAZQhWzVXb4r51S6VuiEdmatcnW0wVjfp6mfWzSqzrq/kYmezlWHo29u 18 | 2rHy7hNQBfDl45fXjSLkna2W5rc6uD+y9PRPA48CgYAL5Mh7em8cvchU0wN5FyH4 19 | ZoW9FlkE2+NbNCQwUzgalIvK0tATY0AeEjyN9Z2aUglYG+HtLFX/X0qCgG7kKV4v 20 | 92H5Bw0WiHJnKSzIrSdB1wuHsBzsiVItgjy4e+s0pL36et2m4D6FcPi3bZJ9hYtG 21 | Rj2VOir00EhkzB+i+N4F+QKBgDfUTf3gCdVnaIGs6+KZtcRYQ7WN4sn3W3PLa7PG 22 | ge0H47VxDdm0K36uds/lHY3NNGwuKGClgCTQIb5UEnXF7Rf4klkxfKAzTTbW09AR 23 | n0GRzMblAeiknZ+p+dtJaDwWg0BiDi5GlSGp3aCj51O1v5qafKI5jg8b4aANgDx0 24 | u2UfAoGBAKJ3Xqic/DN4JgFW+2lFdcbCHG8fxAJCYtrV6BbwduTqLa6VlrNNJrot 25 | vPxy/i/aQdahG4wu1HwIuTEO+fe3swaUdSutcpU2ll2P9i3CYTFJkE3DUEhzxxJL 26 | Y3Tb0X0LK7HmqYvNpANDZKNM7dhYpucj46ut0iZhYhUuS7sP/5qt 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /math/polynomial.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | import ( 4 | "fmt" 5 | "math/big" 6 | "strings" 7 | ) 8 | 9 | // Polynomial is used for educational purpose only, not optimized for sparse polynomials 10 | type Polynomial struct { 11 | maxOrder int 12 | coefficients []*big.Int 13 | } 14 | 15 | func NewPolynomialWithMaxOrder(maxOrder int, coefficients []*big.Int) *Polynomial { 16 | return &Polynomial{maxOrder: maxOrder, coefficients: coefficients} 17 | } 18 | 19 | func NewPolynomial(coefficients []*big.Int) *Polynomial { 20 | return &Polynomial{coefficients: coefficients} 21 | } 22 | 23 | var zero = big.NewInt(0) 24 | var one = big.NewInt(1) 25 | 26 | func (p1 *Polynomial) String() string { 27 | var parts []string 28 | for i, c := range p1.coefficients { 29 | if c.Cmp(zero) != 0 { 30 | if i == 0 { 31 | parts = append(parts, c.String()) 32 | } else { 33 | if c.Cmp(one) == 0 { 34 | parts = append(parts, fmt.Sprintf("x^%d", i)) 35 | } else { 36 | parts = append(parts, fmt.Sprintf("%sx^%d", c.String(), i)) 37 | } 38 | } 39 | } 40 | } 41 | 42 | return strings.Join(parts, " + ") 43 | } 44 | 45 | func (p1 *Polynomial) Mul(p2 *Polynomial) *Polynomial { 46 | dim := len(p1.coefficients) + len(p2.coefficients) - 1 47 | if dim <= 0 { 48 | return &Polynomial{} 49 | } 50 | 51 | if p1.maxOrder > 0 && dim > p1.maxOrder+1 { 52 | dim = p1.maxOrder + 1 53 | } 54 | coefficients := make([]*big.Int, dim) 55 | for i, c1 := range p1.coefficients { 56 | for j, c2 := range p2.coefficients { 57 | if p1.maxOrder > 0 && p1.maxOrder < i+j { 58 | break 59 | } 60 | c3 := coefficients[i+j] 61 | if c3 == nil { 62 | c3 = big.NewInt(0) 63 | } 64 | 65 | coefficients[i+j] = c3.Add(c3, big.NewInt(0).Mul(c1, c2)) 66 | } 67 | } 68 | 69 | return &Polynomial{maxOrder: p1.maxOrder, coefficients: coefficients} 70 | } 71 | -------------------------------------------------------------------------------- /crypto/vrf/p256/unmarshal.go: -------------------------------------------------------------------------------- 1 | package p256 2 | 3 | import ( 4 | "crypto/elliptic" 5 | "errors" 6 | "math/big" 7 | ) 8 | 9 | var ( 10 | errInvalidData = errors.New("invalid data") 11 | ) 12 | 13 | // Unmarshal a compressed point in the form specified in section 4.3.6 of ANSI X9.62. 14 | func Unmarshal(curve elliptic.Curve, data []byte) (x, y *big.Int, err error) { 15 | if (data[0] &^ 1) != 2 { 16 | err = errInvalidData 17 | return 18 | } 19 | 20 | byteLen := (curve.Params().BitSize + 7) >> 3 21 | if len(data) != 1+byteLen { 22 | err = errInvalidData 23 | return 24 | } 25 | 26 | // Based on Routine 2.2.4 in NIST Mathematical routines paper 27 | params := curve.Params() 28 | tx := new(big.Int).SetBytes(data[1 : 1+byteLen]) 29 | y2 := y2(params, tx) 30 | sqrt := defaultSqrt 31 | ty := sqrt(y2, params.P) 32 | if ty == nil { 33 | // "y^2" is not a square: invalid point 34 | err = errInvalidData 35 | return 36 | } 37 | 38 | var y2c big.Int 39 | y2c.Mul(ty, ty).Mod(&y2c, params.P) 40 | if y2c.Cmp(y2) != 0 { 41 | // sqrt(y2)^2 != y2: invalid point 42 | err = errInvalidData 43 | return 44 | } 45 | 46 | if ty.Bit(0) != uint(data[0]&1) { 47 | ty.Sub(params.P, ty) 48 | } 49 | 50 | x, y = tx, ty // valid point: return it 51 | return 52 | } 53 | 54 | // Use the curve equation to calculate y² given x. 55 | // only applies to curves of the form y² = x³ - 3x + b. 56 | func y2(curve *elliptic.CurveParams, x *big.Int) *big.Int { 57 | // y² = x³ - 3x + b 58 | x3 := new(big.Int).Mul(x, x) 59 | x3.Mul(x3, x) 60 | 61 | threeX := new(big.Int).Lsh(x, 1) 62 | threeX.Add(threeX, x) 63 | 64 | x3.Sub(x3, threeX) 65 | x3.Add(x3, curve.B) 66 | x3.Mod(x3, curve.P) 67 | return x3 68 | } 69 | 70 | func defaultSqrt(x, p *big.Int) *big.Int { 71 | var r big.Int 72 | if nil == r.ModSqrt(x, p) { 73 | return nil // x is not a square 74 | } 75 | return &r 76 | } 77 | -------------------------------------------------------------------------------- /deadlock/rwmutex.go: -------------------------------------------------------------------------------- 1 | package deadlock 2 | 3 | import ( 4 | "context" 5 | "unsafe" 6 | ) 7 | 8 | const rwmutexMaxReaders = 1 << 30 9 | 10 | // RWMutex is like sync.RWMutex but with builtin deadlock detecting ability 11 | type RWMutex struct { 12 | sema *Weighted 13 | } 14 | 15 | // NewRWMutex is ctor for RWMutex 16 | func NewRWMutex() *RWMutex { 17 | rw := &RWMutex{} 18 | rw.Init() 19 | return rw 20 | } 21 | 22 | // Init for embeded usage 23 | func (rw *RWMutex) Init() { 24 | rw.sema = NewWeighted(rwmutexMaxReaders, rw) 25 | } 26 | 27 | // Lock for write lock 28 | func (rw *RWMutex) Lock() { 29 | rw.sema.Acquire(context.Background(), rwmutexMaxReaders) 30 | return 31 | } 32 | 33 | // Unlock should only be called after a successful Lock 34 | func (rw *RWMutex) Unlock() { 35 | rw.sema.Release(rwmutexMaxReaders) 36 | return 37 | } 38 | 39 | // RLock for read lock 40 | func (rw *RWMutex) RLock() { 41 | rw.sema.Acquire(context.Background(), 1) 42 | } 43 | 44 | // RUnlock should only be called after a successful RLock 45 | func (rw *RWMutex) RUnlock() { 46 | rw.sema.Release(1) 47 | } 48 | 49 | // TryLock returns true if lock acquired 50 | func (rw *RWMutex) TryLock() bool { 51 | return rw.sema.TryAcquire(rwmutexMaxReaders) 52 | } 53 | 54 | // TryRLock returns true if rlock acquired 55 | func (rw *RWMutex) TryRLock() bool { 56 | return rw.sema.TryAcquire(1) 57 | } 58 | 59 | func (rw *RWMutex) onAcquiredLocked(n int64) { 60 | d.onAcquiredLocked(uint64(uintptr(unsafe.Pointer(rw))), n == rwmutexMaxReaders) 61 | } 62 | 63 | func (rw *RWMutex) onWaitLocked(n int64) { 64 | d.onWaitLocked(uint64(uintptr(unsafe.Pointer(rw))), n == rwmutexMaxReaders) 65 | } 66 | 67 | func (rw *RWMutex) onWaitCanceledLocked(n int64) { 68 | panic("this should never happen") 69 | } 70 | 71 | func (rw *RWMutex) onReleaseLocked(n int64) { 72 | d.onReleaseLocked(uint64(uintptr(unsafe.Pointer(rw))), n == rwmutexMaxReaders) 73 | } 74 | -------------------------------------------------------------------------------- /crypto/vrf/p256/p256_test.go: -------------------------------------------------------------------------------- 1 | package p256 2 | 3 | import ( 4 | "crypto/rand" 5 | "testing" 6 | ) 7 | 8 | func TestH1(t *testing.T) { 9 | for i := 0; i < 10000; i++ { 10 | m := make([]byte, 100) 11 | if _, err := rand.Read(m); err != nil { 12 | t.Fatalf("Failed generating random message: %v", err) 13 | } 14 | x, y, err := H1(m) 15 | if err != nil { 16 | t.Errorf("H1(%v)=%v, want curve point", m, x) 17 | } 18 | if got := curve.Params().IsOnCurve(x, y); !got { 19 | t.Errorf("H1(%v)=[%v, %v], is not on curve", m, x, y) 20 | } 21 | } 22 | } 23 | 24 | func TestH2(t *testing.T) { 25 | l := 32 26 | for i := 0; i < 10000; i++ { 27 | m := make([]byte, 100) 28 | if _, err := rand.Read(m); err != nil { 29 | t.Fatalf("Failed generating random message: %v", err) 30 | } 31 | x := H2(m) 32 | if got := len(x.Bytes()); got < 1 || got > l { 33 | t.Errorf("len(h2(%v)) = %v, want: 1 <= %v <= %v", m, got, got, l) 34 | } 35 | } 36 | } 37 | 38 | func TestVRF(t *testing.T) { 39 | k, pk, err := GeneratePair() 40 | if err != nil { 41 | t.Fatalf("Failed generating key pairs: %v", err) 42 | } 43 | 44 | m1 := []byte("data1") 45 | m2 := []byte("data2") 46 | m3 := []byte("data2") 47 | beta1, proof1, err := k.Hash(m1) 48 | if err != nil { 49 | t.Fatalf("Hash failed: %v", err) 50 | } 51 | beta2, proof2, err := k.Hash(m2) 52 | if err != nil { 53 | t.Fatalf("Hash failed: %v", err) 54 | } 55 | beta3, proof3, err := k.Hash(m3) 56 | if err != nil { 57 | t.Fatalf("Hash failed: %v", err) 58 | } 59 | for _, tc := range []struct { 60 | m []byte 61 | beta [32]byte 62 | proof []byte 63 | valid bool 64 | }{ 65 | {m1, beta1, proof1, true}, 66 | {m2, beta2, proof2, true}, 67 | {m3, beta3, proof3, true}, 68 | {m3, beta3, proof2, true}, 69 | {m3, beta3, proof1, false}, 70 | } { 71 | valid, err := pk.Verify(tc.m, tc.proof, tc.beta) 72 | if err != nil { 73 | t.Fatalf("Verify failed: %v", err) 74 | } 75 | if got, want := valid, tc.valid; got != want { 76 | t.Errorf("Verify(%s, %x): %v, want %v", tc.m, tc.proof, got, want) 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /misc.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | 8 | "github.com/zhiqiangxu/util/logger" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | // GoFunc runs a goroutine under WaitGroup 13 | func GoFunc(routinesGroup *sync.WaitGroup, f func()) { 14 | routinesGroup.Add(1) 15 | go func() { 16 | defer routinesGroup.Done() 17 | f() 18 | }() 19 | } 20 | 21 | // TryUntilSuccess will try f until success 22 | func TryUntilSuccess(f func() bool, duration time.Duration) { 23 | var r bool 24 | for { 25 | r = f() 26 | if r { 27 | return 28 | } 29 | time.Sleep(duration) 30 | } 31 | } 32 | 33 | // RunWithRecovery for run exec, calls recoverFn when panic 34 | func RunWithRecovery(exec func(), recoverFn func(r interface{})) { 35 | defer func() { 36 | r := recover() 37 | if r != nil { 38 | logger.Instance().Error("panic in the recoverable goroutine", 39 | zap.Reflect("r", r), 40 | zap.Stack("stack trace")) 41 | 42 | if recoverFn != nil { 43 | recoverFn(r) 44 | } 45 | } 46 | }() 47 | exec() 48 | } 49 | 50 | // RunWithRetry will run the f with backoff and retry. 51 | // retryCnt: Max retry count 52 | // backoff: When run f failed, it will sleep backoff * triedCount time.Millisecond. 53 | // Function f should have two return value. The first one is an bool which indicate if the err if retryable. 54 | // The second is if the f meet any error. 55 | func RunWithRetry(retryCnt int, backoff uint64, f func() (bool, error)) (err error) { 56 | var retryAble bool 57 | for i := 1; i <= retryCnt; i++ { 58 | retryAble, err = f() 59 | if err == nil || !retryAble { 60 | return 61 | } 62 | sleepTime := time.Duration(backoff*uint64(i)) * time.Millisecond 63 | time.Sleep(sleepTime) 64 | } 65 | return 66 | } 67 | 68 | // RunWithCancel for run a job with cancel-ability 69 | func RunWithCancel(ctx context.Context, f, cancel func()) { 70 | var wg sync.WaitGroup 71 | 72 | doneCh := make(chan struct{}) 73 | GoFunc(&wg, func() { 74 | f() 75 | close(doneCh) 76 | }) 77 | 78 | select { 79 | case <-ctx.Done(): 80 | cancel() 81 | case <-doneCh: 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /crypto/claim/verifier.go: -------------------------------------------------------------------------------- 1 | package claim 2 | 3 | import ( 4 | "crypto/rsa" 5 | "encoding/json" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/dgrijalva/jwt-go" 10 | ) 11 | 12 | // Verifier for claims 13 | type Verifier struct { 14 | method jwt.SigningMethod 15 | verifyKey *rsa.PublicKey 16 | } 17 | 18 | // NewVerifier is ctor for Verifier 19 | func NewVerifier(verifyKey *rsa.PublicKey) (v *Verifier, err error) { 20 | signingAlgorithm := "RS256" 21 | method := jwt.GetSigningMethod(signingAlgorithm) 22 | if method == nil { 23 | err = fmt.Errorf("invalid signingAlgorithm:%s", method) 24 | return 25 | } 26 | 27 | v = &Verifier{method: method, verifyKey: verifyKey} 28 | return 29 | } 30 | 31 | // Verify claims 32 | func (v *Verifier) Verify(tokenString string) (ok bool, values map[string]interface{}) { 33 | ok, values = verify(tokenString, v.method, v.verifyKey) 34 | return 35 | } 36 | 37 | // GetInt64 for retrieve claim value as int64 38 | func GetInt64(value interface{}) (int64Value int64) { 39 | switch exp := value.(type) { 40 | case float64: 41 | int64Value = int64(exp) 42 | case json.Number: 43 | int64Value, _ = exp.Int64() 44 | } 45 | return 46 | } 47 | 48 | func verify(tokenString string, method jwt.SigningMethod, verifyKey interface{}) (ok bool, values map[string]interface{}) { 49 | token, err := jwt.Parse(tokenString, func(token *jwt.Token) (key interface{}, err error) { 50 | if method != token.Method { 51 | err = fmt.Errorf("invalid signingAlgorithm:%s", token.Method.Alg()) 52 | return 53 | } 54 | key = verifyKey 55 | return 56 | }) 57 | if err != nil || !token.Valid { 58 | return 59 | } 60 | 61 | claims, ok := token.Claims.(jwt.MapClaims) 62 | if !ok { 63 | return 64 | } 65 | 66 | expireAt, exists := claims[ExpireATKey] 67 | if !exists { 68 | ok = false 69 | return 70 | } 71 | 72 | ok = verifyExp(GetInt64(expireAt)) 73 | if !ok { 74 | return 75 | } 76 | 77 | values = (map[string]interface{})(claims) 78 | delete(values, ExpireATKey) 79 | delete(values, CreatedKey) 80 | return 81 | } 82 | 83 | func verifyExp(exp int64) (ok bool) { 84 | nowSecond := time.Now().Unix() 85 | ok = exp > nowSecond 86 | return 87 | } 88 | -------------------------------------------------------------------------------- /mmr/file_hash_store.go: -------------------------------------------------------------------------------- 1 | package mmr 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "os" 7 | "unsafe" 8 | ) 9 | 10 | type fileHashStore struct { 11 | fileName string 12 | file *os.File 13 | } 14 | 15 | // NewFileHashStore returns a HashStore implement in file 16 | func NewFileHashStore(name string, treeSize uint64) (hs HashStore, err error) { 17 | f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, 0755) 18 | if err != nil { 19 | return 20 | } 21 | store := &fileHashStore{ 22 | fileName: name, 23 | file: f, 24 | } 25 | 26 | hashCount := getStoredHashCount(treeSize) 27 | size := int64(hashCount) * int64(unsafe.Sizeof(HashType{})) 28 | 29 | err = store.checkConsistence(size) 30 | if err != nil { 31 | return 32 | } 33 | 34 | _, err = store.file.Seek(size, io.SeekStart) 35 | if err != nil { 36 | return 37 | } 38 | 39 | hs = store 40 | return 41 | } 42 | 43 | var ( 44 | errStoredHashLessThanExpected = errors.New("stored hashes are less than expected") 45 | ) 46 | 47 | func (self *fileHashStore) checkConsistence(fileSize int64) (err error) { 48 | 49 | stat, err := self.file.Stat() 50 | if err != nil { 51 | return 52 | } 53 | if stat.Size() < fileSize { 54 | err = errStoredHashLessThanExpected 55 | return 56 | } 57 | 58 | return 59 | } 60 | 61 | func (self *fileHashStore) Append(hash []HashType) error { 62 | buf := make([]byte, 0, len(hash)*int(unsafe.Sizeof(HashType{}))) 63 | for _, h := range hash { 64 | buf = append(buf, h[:]...) 65 | } 66 | _, err := self.file.Write(buf) 67 | return err 68 | } 69 | 70 | func (self *fileHashStore) Flush() error { 71 | return self.file.Sync() 72 | } 73 | 74 | func (self *fileHashStore) Close() { 75 | self.file.Close() 76 | } 77 | 78 | func (self *fileHashStore) GetHash(pos uint64) (h HashType, err error) { 79 | h = unknownHash 80 | _, err = self.file.ReadAt(h[:], int64(pos)*int64(unsafe.Sizeof(HashType{}))) 81 | if err != nil { 82 | return 83 | } 84 | 85 | return 86 | } 87 | 88 | func getStoredHashCount(treeSize uint64) int64 { 89 | subtreesize := getMoutainSizes(treeSize) 90 | sum := int64(0) 91 | for _, v := range subtreesize { 92 | sum += int64(v) 93 | } 94 | 95 | return sum 96 | } 97 | -------------------------------------------------------------------------------- /mmr/README.md: -------------------------------------------------------------------------------- 1 | # merkle mountain range 2 | 3 | Merkle Mountain Ranges (MMR) decomposes an arbitory number of leaves into a seiries of perfectly balanced merkle trees, MMR is strictly append-only, elements are added from the left to the right, adding a parent as soon as 2 children exist, filling up the range accordingly. 4 | 5 | This illustrates a range with 11 inserted leaves and total size 19, where each node is annotated with its order of insertion: 6 | 7 | ``` 8 | Height 9 | 10 | 3 14 11 | / \ 12 | / \ 13 | / \ 14 | / \ 15 | 2 6 13 16 | / \ / \ 17 | 1 2 5 9 12 17 18 | / \ / \ / \ / \ / \ 19 | 0 0 1 3 4 7 8 10 11 15 16 18 20 | ``` 21 | 22 | ## find inclusion proof visually 23 | 24 | Suppose the path from root to the leaf `tn` is `(root, t1, t2, ... tn)`, then the inclusion proof is chosen by replacing `ti` with it's sibling. 25 | 26 | ``` 27 | hash 28 | / \ 29 | / \ 30 | / \ 31 | / \ 32 | / \ 33 | k l 34 | / \ / \ 35 | / \ / \ 36 | / \ / \ 37 | g h i j 38 | / \ / \ / \ 39 | a b c d e f 40 | 41 | ``` 42 | 43 | The path from root to `a` is `(hash, k, g, a)`, so the inclusion proof is `(b, h, l)`. 44 | 45 | ## find consistency proof visually 46 | 47 | The consistency proof first has to prove the existence of the old root, and then prove the inclusion of the old root in the new root. So the first step is to identify the old root, then take the peak `P` of rightest merkle montain, then find the inclusion proof of `P` visually; If `P` equals to the old root(old tree size is power of two), the consistency proof is just `InclusionProof(P)`; If `P` doesn't equal to the old root(old tree size is not power of two), the consistency proof is `P || InclusionProof(P)`. 48 | 49 | Using the method, it's not hard to see that `ConsistencyProot(c)` is `[c,d,g,i]`, `ConsistencyProot(d)` is `[l]`, `ConsistencyProot(f)` is `[i, j, k]`, 50 | -------------------------------------------------------------------------------- /mmap.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package util 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | "unsafe" 9 | 10 | "golang.org/x/sys/unix" 11 | ) 12 | 13 | // Mmap uses the mmap system call to memory-map a file. If writable is true, 14 | // memory protection of the pages is set so that they may be written to as well. 15 | func Mmap(fd *os.File, writable bool, size int64) ([]byte, error) { 16 | mtype := unix.PROT_READ 17 | if writable { 18 | mtype |= unix.PROT_WRITE 19 | } 20 | return unix.Mmap(int(fd.Fd()), 0, int(size), mtype, unix.MAP_SHARED) 21 | } 22 | 23 | // Munmap unmaps a previously mapped slice. 24 | func Munmap(b []byte) error { 25 | return unix.Munmap(b) 26 | } 27 | 28 | // Madvise uses the madvise system call to give advise about the use of memory 29 | // when using a slice that is memory-mapped to a file. Set the readahead flag to 30 | // false if page references are expected in random order. 31 | func Madvise(b []byte, readahead bool) error { 32 | flags := unix.MADV_NORMAL 33 | if !readahead { 34 | flags = unix.MADV_RANDOM 35 | } 36 | return madvise(b, flags) 37 | } 38 | 39 | // This is required because the unix package does not support the madvise system call on OS X. 40 | func madvise(b []byte, advice int) (err error) { 41 | _, _, e1 := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&b[0])), 42 | uintptr(len(b)), uintptr(advice)) 43 | if e1 != 0 { 44 | err = e1 45 | } 46 | return 47 | } 48 | 49 | // MSync for flush mmaped bytes 50 | func MSync(b []byte, length int64, flags int) (err error) { 51 | _, _, e1 := syscall.Syscall(syscall.SYS_MSYNC, 52 | uintptr(unsafe.Pointer(&b[0])), uintptr(length), uintptr(flags)) 53 | if e1 != 0 { 54 | err = e1 55 | } 56 | 57 | return 58 | } 59 | 60 | // MLock mmaped bytes 61 | func MLock(b []byte, length int) (err error) { 62 | _, _, e1 := syscall.Syscall(syscall.SYS_MLOCK, 63 | uintptr(unsafe.Pointer(&b[0])), uintptr(length), 0) 64 | if e1 != 0 { 65 | err = e1 66 | } 67 | 68 | return 69 | } 70 | 71 | // MUnlock mmaped bytes 72 | func MUnlock(b []byte, length int) (err error) { 73 | _, _, e1 := syscall.Syscall(syscall.SYS_MUNLOCK, 74 | uintptr(unsafe.Pointer(&b[0])), uintptr(length), 0) 75 | if e1 != 0 { 76 | err = e1 77 | } 78 | 79 | return 80 | } 81 | -------------------------------------------------------------------------------- /closer/strict.go: -------------------------------------------------------------------------------- 1 | package closer 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | "sync/atomic" 7 | ) 8 | 9 | // Strict closer is a sync.WaitGroup with state. 10 | // It guarantees no Add with positive delta will ever succeed after Wait. 11 | // Wait can only be called once, but Add and Wait can be called concurrently. 12 | // The happens before relationship between Add and Wait is taken care of automatically. 13 | type Strict struct { 14 | mu sync.RWMutex 15 | cond *sync.Cond 16 | closed uint32 17 | counter int32 18 | done chan struct{} 19 | } 20 | 21 | // NewStrict is ctor for Strict 22 | func NewStrict() *Strict { 23 | s := &Strict{done: make(chan struct{})} 24 | s.cond = sync.NewCond(&s.mu) 25 | return s 26 | } 27 | 28 | var ( 29 | errAlreadyClosed = errors.New("closer already closed") 30 | ) 31 | 32 | // Add delta to wait group 33 | // Trying to Add positive delta after Wait will return non nil error 34 | func (s *Strict) Add(delta int) (err error) { 35 | if delta > 0 { 36 | if s.HasBeenClosed() { 37 | err = errAlreadyClosed 38 | return 39 | } 40 | 41 | s.mu.RLock() 42 | if s.closed != 0 { 43 | s.mu.RUnlock() 44 | err = errAlreadyClosed 45 | return 46 | } 47 | } 48 | 49 | counter := atomic.AddInt32(&s.counter, int32(delta)) 50 | 51 | if delta > 0 { 52 | s.mu.RUnlock() 53 | } 54 | 55 | if counter == 0 { 56 | s.mu.RLock() 57 | if s.HasBeenClosed() { 58 | s.cond.Signal() 59 | } 60 | s.mu.RUnlock() 61 | } 62 | 63 | return 64 | } 65 | 66 | // HasBeenClosed tells whether closed 67 | func (s *Strict) HasBeenClosed() bool { 68 | return atomic.LoadUint32(&s.closed) != 0 69 | } 70 | 71 | // ClosedSignal gets signaled when Wait() is called. 72 | func (s *Strict) ClosedSignal() <-chan struct{} { 73 | return s.done 74 | } 75 | 76 | // Done decrements the WaitGroup counter by one. 77 | func (s *Strict) Done() { 78 | s.Add(-1) 79 | } 80 | 81 | // SignalAndWait updates closed and blocks until the WaitGroup counter is zero. 82 | // Call it more than once will panic 83 | func (s *Strict) SignalAndWait() { 84 | close(s.done) 85 | 86 | s.mu.Lock() 87 | 88 | atomic.StoreUint32(&s.closed, 1) 89 | 90 | for atomic.LoadInt32(&s.counter) != 0 { 91 | s.cond.Wait() 92 | } 93 | 94 | s.mu.Unlock() 95 | } 96 | -------------------------------------------------------------------------------- /mutex/crwmutex_test.go: -------------------------------------------------------------------------------- 1 | package mutex 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestCRWMutex(t *testing.T) { 10 | crwm := NewCRWMutex() 11 | // test RLock/RUnlock 12 | err := crwm.RLock(context.Background()) 13 | if err != nil { 14 | t.FailNow() 15 | } 16 | 17 | crwm.RUnlock() 18 | 19 | // test Lock/Unlock 20 | err = crwm.Lock(context.Background()) 21 | if err != nil { 22 | t.FailNow() 23 | } 24 | crwm.Unlock() 25 | 26 | // test TryLock/Unlock 27 | ok := crwm.TryLock() 28 | if !ok { 29 | t.FailNow() 30 | } 31 | crwm.Unlock() 32 | 33 | // test TryRLock/Unlock 34 | ok = crwm.TryRLock() 35 | if !ok { 36 | t.FailNow() 37 | } 38 | crwm.RUnlock() 39 | 40 | // test RLock/canceled Lock/RUnlock 41 | err = crwm.RLock(context.Background()) 42 | if err != nil { 43 | t.FailNow() 44 | } 45 | 46 | ctx, cancelFunc := context.WithTimeout(context.Background(), time.Millisecond*300) 47 | defer cancelFunc() 48 | err = crwm.Lock(ctx) 49 | if err == nil { 50 | t.FailNow() 51 | } 52 | 53 | crwm.RUnlock() 54 | 55 | } 56 | 57 | func TestSemaphoreBug(t *testing.T) { 58 | crwm := NewCRWMutex() 59 | // hold 1 read lock 60 | crwm.RLock(context.Background()) 61 | 62 | go func() { 63 | ctx, cancelFunc := context.WithTimeout(context.Background(), time.Millisecond*300) 64 | defer cancelFunc() 65 | // start a Lock request that will giveup after 300ms 66 | err := crwm.Lock(ctx) 67 | if err == nil { 68 | t.FailNow() 69 | } 70 | }() 71 | 72 | // sleep 100ms, long enough for the Lock request to be queued 73 | time.Sleep(time.Millisecond * 100) 74 | // this channel will be closed if the following RLock succeeded 75 | doneCh := make(chan struct{}) 76 | go func() { 77 | // try to grab a read lock, it will be queued after the Lock request 78 | // but should be notified when the Lock request is canceled 79 | // this doesn't happen because there's a bug in semaphore 80 | err := crwm.RLock(context.Background()) 81 | if err != nil { 82 | t.FailNow() 83 | } 84 | crwm.RUnlock() 85 | close(doneCh) 86 | }() 87 | 88 | // because of the bug in semaphore, doneCh is never closed 89 | select { 90 | case <-doneCh: 91 | case <-time.After(time.Second): 92 | t.FailNow() 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /lf/mcas/mcas.go: -------------------------------------------------------------------------------- 1 | package mcas 2 | 3 | import ( 4 | "sort" 5 | "sync/atomic" 6 | "unsafe" 7 | ) 8 | 9 | type mcDesc struct { 10 | a []*unsafe.Pointer 11 | e []unsafe.Pointer 12 | n []unsafe.Pointer 13 | s uint32 14 | } 15 | 16 | const ( 17 | undecided uint32 = iota 18 | failed 19 | successful 20 | ) 21 | 22 | func isMCDesc(v unsafe.Pointer) bool { 23 | return uintptr(v)&addrMask == mcDescAddr 24 | } 25 | 26 | const ( 27 | mcDescAddr = 1 28 | ccDescAddr = 2 29 | addrMask = 3 30 | ) 31 | 32 | func mcfromPointer(v unsafe.Pointer) *mcDesc { 33 | ptr := uintptr(v) 34 | ptr = ptr & ^uintptr(addrMask) 35 | return (*mcDesc)(are.getPointer(ptr)) 36 | } 37 | 38 | func (d *mcDesc) toPointer() unsafe.Pointer { 39 | return unsafe.Pointer(uintptr(unsafe.Pointer(d)) + uintptr(mcDescAddr)) 40 | } 41 | 42 | func (d *mcDesc) sortAddr() { 43 | sort.Slice(d.a, func(i, j int) bool { 44 | return uintptr(unsafe.Pointer(d.a[i])) < uintptr(unsafe.Pointer(d.a[j])) 45 | }) 46 | } 47 | 48 | func (d *mcDesc) status() uint32 { 49 | return atomic.LoadUint32(&d.s) 50 | } 51 | 52 | func (d *mcDesc) mcasHelp() (suc bool) { 53 | ds := failed 54 | var ( 55 | v unsafe.Pointer 56 | ) 57 | /* PHASE 1: Attempt to acquire each location in turn. */ 58 | for i := range d.a { 59 | for { 60 | ccas(d.a[i], d.e[i], d.toPointer(), &d.s) 61 | v = atomic.LoadPointer(d.a[i]) 62 | if v == d.toPointer() { 63 | break 64 | } 65 | 66 | if isCCDesc(v) { 67 | ccfromPointer(v).ccasHelp() 68 | continue 69 | } else if isMCDesc(v) { 70 | mcfromPointer(v).mcasHelp() 71 | continue 72 | } 73 | 74 | // v is plain pointer value 75 | 76 | if v == d.e[i] && d.status() == undecided { 77 | continue 78 | } 79 | 80 | goto decision_point 81 | } 82 | } 83 | 84 | ds = successful 85 | 86 | decision_point: 87 | 88 | atomic.CompareAndSwapUint32(&d.s, undecided, ds) 89 | 90 | /* PHASE 2: Release each location that we hold. */ 91 | suc = atomic.LoadUint32(&d.s) == successful 92 | for i := range d.a { 93 | if suc { 94 | atomic.CompareAndSwapPointer(d.a[i], d.toPointer(), d.n[i]) 95 | } else { 96 | atomic.CompareAndSwapPointer(d.a[i], d.toPointer(), d.e[i]) 97 | } 98 | } 99 | 100 | return 101 | } 102 | -------------------------------------------------------------------------------- /math/lambda.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | // FYI : https://www.math.sinica.edu.tw/www/file_upload/summer/crypt2017/data/2015/[20150707][carmichael].pdf 4 | 5 | func Lambda(n int) int { 6 | if n == 0 { 7 | panic("invalid input") 8 | } 9 | if n == 1 { 10 | return 1 11 | } 12 | 13 | factors := PrimeFactors(n) 14 | 15 | var parts []int 16 | for prime, power := range factors { 17 | switch { 18 | case prime == 2: 19 | if power == 1 || power == 2 { 20 | parts = append(parts, PowInts(2, power-1)) 21 | } else { 22 | parts = append(parts, PowInts(2, power-2)) 23 | } 24 | default: 25 | parts = append(parts, PowInts(prime, power)-PowInts(prime, power-1)) 26 | } 27 | } 28 | 29 | switch { 30 | case len(parts) == 1: 31 | return parts[0] 32 | case len(parts) == 2: 33 | return LCM(parts[0], parts[1]) 34 | default: 35 | return LCM(parts[0], parts[1], parts[2:]...) 36 | } 37 | } 38 | 39 | // greatest common divisor (GCD) via Euclidean algorithm 40 | func GCD(a, b int) int { 41 | for b != 0 { 42 | t := b 43 | b = a % b 44 | a = t 45 | } 46 | return a 47 | } 48 | 49 | // find Least Common Multiple (LCM) via GCD 50 | func LCM(a, b int, integers ...int) int { 51 | result := a * b / GCD(a, b) 52 | 53 | for i := 0; i < len(integers); i++ { 54 | result = LCM(result, integers[i]) 55 | } 56 | 57 | return result 58 | } 59 | 60 | // Assumption: n >= 0 61 | func PowInts(x, n int) int { 62 | if n == 0 { 63 | return 1 64 | } 65 | 66 | v := 1 67 | for n != 0 { 68 | if n&1 == 1 { 69 | v *= x 70 | } 71 | x *= x 72 | n /= 2 73 | } 74 | 75 | return v 76 | } 77 | 78 | // Get all prime factors of a given number n 79 | func PrimeFactors(n int) (pfs map[int]int) { 80 | // Get the number of 2s that divide n 81 | 82 | pfs = make(map[int]int) 83 | for n%2 == 0 { 84 | pfs[2] += 1 85 | n = n / 2 86 | } 87 | 88 | // n must be odd at this point. so we can skip one element 89 | // (note i = i + 2) 90 | for i := 3; i*i <= n; i = i + 2 { 91 | // while i divides n, append i and divide n 92 | for n%i == 0 { 93 | pfs[i] += 1 94 | n = n / i 95 | } 96 | } 97 | 98 | // This condition is to handle the case when n is a prime number 99 | // greater than 2 100 | if n > 2 { 101 | pfs[n] += 1 102 | } 103 | 104 | return 105 | } 106 | -------------------------------------------------------------------------------- /skl/lf/arena.go: -------------------------------------------------------------------------------- 1 | package lf 2 | 3 | import ( 4 | "errors" 5 | "sync/atomic" 6 | "unsafe" 7 | 8 | "github.com/zhiqiangxu/util/bytes" 9 | ) 10 | 11 | // Arena is lock free 12 | type Arena struct { 13 | n uint32 14 | buf []byte 15 | } 16 | 17 | var ( 18 | // ErrOOM used by Arena 19 | ErrOOM = errors.New("oom") 20 | ) 21 | 22 | // NewArena is ctor for Arena 23 | func NewArena(n uint32) *Arena { 24 | 25 | // offset 0 is reserved for nil value 26 | out := &Arena{n: 1, buf: bytes.AlignedTo8(n)} 27 | 28 | return out 29 | } 30 | 31 | func (a *Arena) putKV(k, v []byte) (koff, voff uint32, err error) { 32 | lk := uint32(len(k)) 33 | lv := uint32(len(v)) 34 | l := lk + lv 35 | n := atomic.AddUint32(&a.n, l) 36 | if int(n) > len(a.buf) { 37 | err = ErrOOM 38 | return 39 | } 40 | 41 | koff = n - l 42 | copy(a.buf[koff:koff+lk], k) 43 | voff = koff + lk 44 | copy(a.buf[voff:voff+lv], v) 45 | return 46 | } 47 | 48 | func (a *Arena) putBytes(b []byte) (offset uint32, err error) { 49 | l := uint32(len(b)) 50 | n := atomic.AddUint32(&a.n, l) 51 | if int(n) > len(a.buf) { 52 | err = ErrOOM 53 | return 54 | } 55 | 56 | offset = n - l 57 | copy(a.buf[offset:n], b) 58 | return 59 | 60 | } 61 | 62 | const ( 63 | nodeAlign = int(unsafe.Sizeof(uint64(0))) - 1 64 | ) 65 | 66 | func (a *Arena) putListNode() (offset uint32, err error) { 67 | 68 | // Pad the allocation with enough bytes to ensure pointer alignment. 69 | l := uint32(ListNodeSize + nodeAlign) 70 | n := atomic.AddUint32(&a.n, l) 71 | if int(n) > len(a.buf) { 72 | err = ErrOOM 73 | return 74 | } 75 | 76 | // Return the aligned offset. 77 | offset = (n - l + uint32(nodeAlign)) & ^uint32(nodeAlign) 78 | return 79 | } 80 | 81 | func (a *Arena) getBytes(offset uint32, size uint16) []byte { 82 | if offset == 0 { 83 | return nil 84 | } 85 | 86 | return a.buf[offset : offset+uint32(size)] 87 | } 88 | func (a *Arena) getListNode(offset uint32) *listNode { 89 | if offset == 0 { 90 | return nil 91 | } 92 | 93 | return (*listNode)(unsafe.Pointer(&a.buf[offset])) 94 | } 95 | 96 | func (a *Arena) getListNodeOffset(n *listNode) uint32 { 97 | if n == nil { 98 | return 0 99 | } 100 | 101 | offset := uintptr(unsafe.Pointer(n)) - uintptr(unsafe.Pointer(&a.buf[0])) 102 | return uint32(offset) 103 | } 104 | -------------------------------------------------------------------------------- /crypto/signature/signature.go: -------------------------------------------------------------------------------- 1 | package signature 2 | 3 | import ( 4 | "crypto" 5 | "crypto/ecdsa" 6 | "crypto/elliptic" 7 | "crypto/rand" 8 | "errors" 9 | "fmt" 10 | "math/big" 11 | ) 12 | 13 | // Signature ... 14 | type Signature struct { 15 | r *big.Int 16 | s *big.Int 17 | } 18 | 19 | // Marshal ... 20 | func (sig *Signature) Marshal() (bytes []byte, err error) { 21 | size := (curve.Params().BitSize + 7) >> 3 22 | bytes = make([]byte, size*2) 23 | 24 | r := sig.r.Bytes() 25 | s := sig.s.Bytes() 26 | copy(bytes[size-len(r):], r) 27 | copy(bytes[size*2-len(s):], s) 28 | return 29 | } 30 | 31 | // Unmarshal ... 32 | func (sig *Signature) Unmarshal(data []byte) (err error) { 33 | length := len(data) 34 | if length&1 != 0 { 35 | err = errors.New("invalid length") 36 | return 37 | } 38 | 39 | sig.r = new(big.Int).SetBytes(data[0 : length/2]) 40 | sig.s = new(big.Int).SetBytes(data[length/2:]) 41 | return 42 | } 43 | 44 | // Sign ... 45 | func Sign(privateKey crypto.PrivateKey, msg []byte) (sig *Signature, err error) { 46 | switch key := privateKey.(type) { 47 | case *ecdsa.PrivateKey: 48 | h := getHasher() 49 | _, err = h.Write(msg) 50 | if err != nil { 51 | return 52 | } 53 | digest := h.Sum(nil) 54 | 55 | var r, s *big.Int 56 | r, s, err = ecdsa.Sign(rand.Reader, key, digest) 57 | if err != nil { 58 | return 59 | } 60 | sig = &Signature{r: r, s: s} 61 | default: 62 | err = fmt.Errorf("only ecdsa supported") 63 | } 64 | return 65 | } 66 | 67 | // Verify ... 68 | func Verify(publicKey crypto.PublicKey, msg []byte, sig *Signature) (ok bool, err error) { 69 | switch key := publicKey.(type) { 70 | case *ecdsa.PublicKey: 71 | h := getHasher() 72 | _, err = h.Write(msg) 73 | if err != nil { 74 | return 75 | } 76 | digest := h.Sum(nil) 77 | 78 | ok = ecdsa.Verify(key, digest, sig.r, sig.s) 79 | default: 80 | err = fmt.Errorf("only ecdsa supported") 81 | } 82 | return 83 | } 84 | 85 | var ( 86 | curve = elliptic.P256() 87 | ) 88 | 89 | // GenerateKeypair ... 90 | func GenerateKeypair() (privateKey crypto.PrivateKey, publicKey crypto.PublicKey, err error) { 91 | key, err := ecdsa.GenerateKey(curve, rand.Reader) 92 | if err != nil { 93 | return 94 | } 95 | 96 | privateKey = key 97 | publicKey = &key.PublicKey 98 | return 99 | 100 | } 101 | -------------------------------------------------------------------------------- /xorlayer/bucket.go: -------------------------------------------------------------------------------- 1 | package xorlayer 2 | 3 | import ( 4 | "container/list" 5 | 6 | "github.com/zhiqiangxu/util/sort" 7 | ) 8 | 9 | type bucket struct { 10 | m map[NodeID]*list.Element 11 | l *list.List 12 | } 13 | 14 | type nodeWithCookie struct { 15 | N NodeID 16 | Cookie uint64 17 | } 18 | 19 | func newBucket() *bucket { 20 | return &bucket{m: make(map[NodeID]*list.Element), l: list.New()} 21 | } 22 | 23 | func (b *bucket) insert(n NodeID, cookie uint64) { 24 | e := b.m[n] 25 | if e != nil { 26 | e.Value.(*nodeWithCookie).Cookie = cookie 27 | b.l.MoveToFront(e) 28 | } else { 29 | nwc := &nodeWithCookie{ 30 | N: n, 31 | Cookie: cookie, 32 | } 33 | e = b.l.PushFront(nwc) 34 | b.m[n] = e 35 | } 36 | } 37 | 38 | func (b *bucket) remove(n NodeID, cookie uint64) (removed bool) { 39 | e := b.m[n] 40 | if e != nil { 41 | if e.Value.(*nodeWithCookie).Cookie != cookie { 42 | return 43 | } 44 | removed = true 45 | b.l.Remove(e) 46 | delete(b.m, n) 47 | } 48 | return 49 | } 50 | 51 | func (b *bucket) reduceTo(max int) { 52 | for b.size() > max { 53 | e := b.l.Back() 54 | nwc := e.Value.(*nodeWithCookie) 55 | delete(b.m, nwc.N) 56 | b.l.Remove(e) 57 | } 58 | } 59 | 60 | func (b *bucket) refresh(n NodeID) (exists bool) { 61 | e := b.m[n] 62 | if e != nil { 63 | b.l.MoveToFront(e) 64 | exists = true 65 | } 66 | return 67 | } 68 | 69 | func (b *bucket) size() int { 70 | return len(b.m) 71 | } 72 | 73 | func (b *bucket) appendXClosest(r []NodeID, x int, target NodeID) []NodeID { 74 | if b.size() <= x { 75 | return b.appendAll(r) 76 | } 77 | 78 | // find k closest NodeID to target 79 | all := make([]NodeID, 0, b.size()) 80 | for n := range b.m { 81 | all = append(all, n) 82 | } 83 | 84 | kclosest := sort.KSmallest(all, x, func(j, k int) int { 85 | dj := all[j] ^ target 86 | dk := all[k] ^ target 87 | switch { 88 | case dj < dk: 89 | return -1 90 | case dj > dk: 91 | return 1 92 | } 93 | 94 | return 0 95 | }).([]NodeID) 96 | 97 | for _, n := range kclosest { 98 | r = append(r, n) 99 | } 100 | return r 101 | } 102 | 103 | func (b *bucket) appendAll(r []NodeID) []NodeID { 104 | for n := range b.m { 105 | r = append(r, n) 106 | } 107 | return r 108 | } 109 | 110 | func (b *bucket) oldest() nodeWithCookie { 111 | return *b.l.Back().Value.(*nodeWithCookie) 112 | } 113 | -------------------------------------------------------------------------------- /math/divisor.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | import "math/big" 4 | 5 | func DivCount(n int) int { 6 | 7 | if n <= 0 { 8 | panic("invalid input") 9 | } 10 | if n == 1 { 11 | return 1 12 | } 13 | 14 | // store numbers in range [2,n] 15 | numbers := make([]bool, n-1) 16 | for i := range numbers { 17 | numbers[i] = true 18 | } 19 | 20 | // sieve method for prime calculation 21 | for i, v := range numbers { 22 | p := i + 2 23 | if v { 24 | for j := i + p; j <= n-2; j += p { 25 | numbers[j] = false 26 | } 27 | } 28 | // if n can factor, then there's at least one <= sqrt(n) 29 | if p*p >= n { 30 | break 31 | } 32 | } 33 | 34 | // Traversing through all prime numbers 35 | total := 1 36 | for i, v := range numbers { 37 | if v { 38 | // calculate number of divisor 39 | // with formula total div = 40 | // (p1+1) * (p2+1) *.....* (pn+1) 41 | // where n = (a1^p1)*(a2^p2).... 42 | // *(an^pn) ai being prime divisor 43 | // for n and pi are their respective 44 | // power in factorization 45 | count := 0 46 | p := i + 2 47 | for n%p == 0 { 48 | n /= p 49 | count++ 50 | } 51 | if count > 0 { 52 | total *= 1 + count 53 | } 54 | } 55 | } 56 | 57 | return total 58 | } 59 | 60 | // AbelGroups returns the # of abel groups of order n 61 | // it's based on the fundamental theory of algebra that 62 | // every finitely generated abel group is isomorphic to 63 | // direct sum of prime-power cyclic groups. 64 | func AbelGroups(n int) *big.Int { 65 | if n <= 0 { 66 | panic("invalid input") 67 | } 68 | if n == 1 { 69 | return big.NewInt(1) 70 | } 71 | 72 | // store numbers in range [2,n] 73 | numbers := make([]bool, n-1) 74 | for i := range numbers { 75 | numbers[i] = true 76 | } 77 | 78 | // sieve method for prime calculation 79 | for i, v := range numbers { 80 | p := i + 2 81 | if v { 82 | for j := i + p; j <= n-2; j += p { 83 | numbers[j] = false 84 | } 85 | } 86 | // if n can factor, then there's at least one <= sqrt(n) 87 | if p*p >= n { 88 | break 89 | } 90 | } 91 | 92 | // Traversing through all prime numbers 93 | total := big.NewInt(1) 94 | for i, v := range numbers { 95 | if v { 96 | count := 0 97 | p := i + 2 98 | for n%p == 0 { 99 | n /= p 100 | count++ 101 | } 102 | if count > 0 { 103 | total.Mul(total, Partition(count)) 104 | } 105 | } 106 | } 107 | 108 | return total 109 | } 110 | -------------------------------------------------------------------------------- /wm/offset.go: -------------------------------------------------------------------------------- 1 | package wm 2 | 3 | import ( 4 | "context" 5 | "sync/atomic" 6 | 7 | "github.com/zhiqiangxu/rpheap" 8 | ) 9 | 10 | type wait struct { 11 | expectOffset int64 12 | waiter chan struct{} 13 | } 14 | 15 | // Offset watermark model 16 | type Offset struct { 17 | doneOffset int64 18 | waitCh chan wait 19 | doneCh chan struct{} 20 | heap *rpheap.Heap 21 | } 22 | 23 | // NewOffset is ctor for Offset 24 | func NewOffset() *Offset { 25 | o := &Offset{waitCh: make(chan wait), doneCh: make(chan struct{}, 1), heap: rpheap.New()} 26 | go o.process() 27 | return o 28 | } 29 | 30 | // Done for update doneOffset 31 | func (o *Offset) Done(offset int64) { 32 | for { 33 | doneOffset := atomic.LoadInt64(&o.doneOffset) 34 | if doneOffset >= offset { 35 | return 36 | } 37 | 38 | if atomic.CompareAndSwapInt64(&o.doneOffset, doneOffset, offset) { 39 | break 40 | } 41 | } 42 | 43 | select { 44 | case o.doneCh <- struct{}{}: 45 | default: 46 | } 47 | } 48 | 49 | // Wait for doneOffset>= expectOffset 50 | func (o *Offset) Wait(ctx context.Context, expectOffset int64) error { 51 | if atomic.LoadInt64(&o.doneOffset) >= expectOffset { 52 | return nil 53 | } 54 | 55 | waiter := make(chan struct{}) 56 | select { 57 | case <-ctx.Done(): 58 | return ctx.Err() 59 | case o.waitCh <- wait{expectOffset: expectOffset, waiter: waiter}: 60 | select { 61 | case <-ctx.Done(): 62 | return ctx.Err() 63 | case <-waiter: 64 | return nil 65 | } 66 | } 67 | } 68 | 69 | func (o *Offset) process() { 70 | 71 | waits := make(map[int64][]chan struct{}) 72 | saveWait := func(w wait) { 73 | ws := waits[w.expectOffset] 74 | if ws == nil { 75 | o.heap.Insert(w.expectOffset) 76 | waits[w.expectOffset] = []chan struct{}{w.waiter} 77 | } else { 78 | waits[w.expectOffset] = append(ws, w.waiter) 79 | } 80 | 81 | } 82 | notifyUntil := func(doneOffset int64) { 83 | for { 84 | if o.heap.Size() == 0 { 85 | return 86 | } 87 | minOffset := o.heap.FindMin() 88 | if minOffset <= doneOffset { 89 | for _, w := range waits[minOffset] { 90 | close(w) 91 | } 92 | delete(waits, minOffset) 93 | o.heap.DeleteMin() 94 | } else { 95 | break 96 | } 97 | } 98 | } 99 | for { 100 | select { 101 | case w := <-o.waitCh: 102 | doneOffset := atomic.LoadInt64(&o.doneOffset) 103 | if w.expectOffset <= doneOffset { 104 | close(w.waiter) 105 | } else { 106 | saveWait(w) 107 | } 108 | case <-o.doneCh: 109 | doneOffset := atomic.LoadInt64(&o.doneOffset) 110 | 111 | // notify all waiters until doneOffset 112 | notifyUntil(doneOffset) 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /mmr/util.go: -------------------------------------------------------------------------------- 1 | package mmr 2 | 3 | import ( 4 | "math/bits" 5 | ) 6 | 7 | // minPeakHeight equals position of the lowest 1 bit 8 | // zero based height 9 | func minPeakHeight(size uint64) int { 10 | return bits.TrailingZeros64(size) 11 | } 12 | 13 | func maxPeakHeight(size uint64) int { 14 | highBit := 64 - bits.LeadingZeros64(size) 15 | return highBit - 1 16 | } 17 | 18 | // peakCount equals the number of 1 bit 19 | func peakCount(size uint64) int { 20 | return bits.OnesCount64(size) 21 | } 22 | 23 | func bagPeaks(hasher Hasher, peaks []HashType) HashType { 24 | l := len(peaks) 25 | accum := peaks[l-1] 26 | 27 | for i := l - 2; i >= 0; i-- { 28 | accum = hasher.Node(peaks[i], accum) 29 | } 30 | return accum 31 | } 32 | 33 | // leftPeak returns left peak 34 | // 1 based index 35 | // precondition: size >= 1 36 | func leftPeak(size uint64) uint64 { 37 | highBit := 64 - bits.LeadingZeros64(size) 38 | return uint64(1<> 1 { 53 | id = id * 2 54 | if size%2 == 1 { 55 | moutainSizes[i] = id - 1 56 | i-- 57 | } 58 | } 59 | 60 | return moutainSizes 61 | } 62 | 63 | // getMoutainPeaks returns moutain peaks from left to right 64 | // 1 based index 65 | func getMoutainPeaks(size uint64) []uint64 { 66 | nPeaks := bits.OnesCount64(size) 67 | peakPos := make([]uint64, nPeaks, nPeaks) 68 | for i, id := nPeaks-1, uint64(1); size != 0; size = size >> 1 { 69 | id = id * 2 70 | if size%2 == 1 { 71 | peakPos[i] = id - 1 72 | i-- 73 | } 74 | } 75 | 76 | for i := 1; i < nPeaks; i++ { 77 | peakPos[i] += peakPos[i-1] 78 | } 79 | 80 | return peakPos 81 | } 82 | 83 | func proofLength(index, size uint64) int { 84 | length := 0 85 | lastNode := size - 1 86 | for lastNode > 0 { 87 | if index%2 == 1 || index < lastNode { 88 | length += 1 89 | } 90 | index /= 2 91 | lastNode /= 2 92 | } 93 | return length 94 | } 95 | 96 | var defaultHasher = NewHasher([]byte{0}, []byte{1}) 97 | 98 | // ComputeRoot ... 99 | func ComputeRoot(hashes []HashType) (root HashType) { 100 | if len(hashes) == 0 { 101 | return 102 | } 103 | 104 | mmr := NewMMR(0, nil, defaultHasher, nil) 105 | for _, hash := range hashes { 106 | mmr.PushHash(hash, false) 107 | } 108 | 109 | root = mmr.Root() 110 | 111 | return 112 | } 113 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "sync" 5 | 6 | kitmetrics "github.com/go-kit/kit/metrics" 7 | 8 | kitprometheus "github.com/go-kit/kit/metrics/prometheus" 9 | stdprometheus "github.com/prometheus/client_golang/prometheus" 10 | ) 11 | 12 | var ( 13 | 14 | // metric maps 15 | 16 | counterMap = make(map[string]kitmetrics.Counter) 17 | cl sync.Mutex 18 | 19 | gaugeMap = make(map[string]kitmetrics.Gauge) 20 | gl sync.Mutex 21 | 22 | histMap = make(map[string]kitmetrics.Histogram) 23 | hl sync.Mutex 24 | ) 25 | 26 | // RegisterCounter registers a counter 27 | func RegisterCounter(name string, labels []string) kitmetrics.Counter { 28 | cl.Lock() 29 | defer cl.Unlock() 30 | 31 | _, ok := counterMap[name] 32 | if ok { 33 | panic("registered the same counter twice") 34 | } 35 | 36 | counter := kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{Name: name}, labels) 37 | counterMap[name] = counter 38 | return counter 39 | } 40 | 41 | // GetCounter get a Counter by name 42 | func GetCounter(name string) kitmetrics.Counter { 43 | cl.Lock() 44 | defer cl.Unlock() 45 | 46 | counter, ok := counterMap[name] 47 | if ok { 48 | return counter 49 | } 50 | 51 | panic("unregistered counter") 52 | 53 | } 54 | 55 | // RegisterGauge registers a gauge 56 | func RegisterGauge(name string, labels []string) kitmetrics.Gauge { 57 | gl.Lock() 58 | defer gl.Unlock() 59 | 60 | _, ok := gaugeMap[name] 61 | if ok { 62 | panic("registered the same gauge twice") 63 | } 64 | 65 | gauge := kitprometheus.NewGaugeFrom(stdprometheus.GaugeOpts{Name: name}, labels) 66 | gaugeMap[name] = gauge 67 | return gauge 68 | } 69 | 70 | // GetGauge get a Gauge by name 71 | func GetGauge(name string) kitmetrics.Gauge { 72 | gl.Lock() 73 | defer gl.Unlock() 74 | 75 | gauge, ok := gaugeMap[name] 76 | if ok { 77 | return gauge 78 | } 79 | 80 | panic("unregistered gauge") 81 | 82 | } 83 | 84 | var Objectives map[float64]float64 85 | 86 | // RegisterHist registers a hist 87 | func RegisterHist(name string, labels []string) kitmetrics.Histogram { 88 | hl.Lock() 89 | defer hl.Unlock() 90 | 91 | _, ok := histMap[name] 92 | if ok { 93 | panic("registered the same gauge twice") 94 | } 95 | 96 | objectives := Objectives 97 | if objectives == nil { 98 | objectives = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001} 99 | } 100 | hist := kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{Name: name, Objectives: objectives}, labels) 101 | histMap[name] = hist 102 | return hist 103 | } 104 | 105 | // GetHist get a Hist by name 106 | func GetHist(name string) kitmetrics.Histogram { 107 | hl.Lock() 108 | defer hl.Unlock() 109 | 110 | hist, ok := histMap[name] 111 | if ok { 112 | return hist 113 | } 114 | 115 | panic("unregistered gauge") 116 | 117 | } 118 | -------------------------------------------------------------------------------- /sort/kofn.go: -------------------------------------------------------------------------------- 1 | package sort 2 | 3 | import "reflect" 4 | 5 | // KSmallest for k smallest 6 | // mutates ns 7 | // not sorted 8 | func KSmallest(slice interface{}, k int, cmp func(j, k int) int) interface{} { 9 | 10 | v := reflect.ValueOf(slice) 11 | i := v.Len() / 2 12 | 13 | pos := PartitionLT(slice, i, cmp) + 1 14 | if pos == k { 15 | return v.Slice(0, k).Interface() 16 | } 17 | 18 | if pos < k { 19 | return reflect.AppendSlice( 20 | v.Slice(0, pos), 21 | reflect.ValueOf(KLargest(v.Slice(pos+1, v.Len()).Interface(), k-pos, cmp)), 22 | ).Interface() 23 | } 24 | 25 | // pos > k 26 | 27 | return KLargest(v.Slice(0, pos-1).Interface(), k, cmp) 28 | } 29 | 30 | // KLargest for k largest 31 | // mutates ns 32 | func KLargest(slice interface{}, k int, cmp func(j, k int) int) interface{} { 33 | v := reflect.ValueOf(slice) 34 | i := v.Len() / 2 35 | 36 | pos := PartitionGT(slice, i, cmp) + 1 37 | if pos == k { 38 | return v.Slice(0, k).Interface() 39 | } 40 | 41 | if pos < k { 42 | return reflect.AppendSlice( 43 | v.Slice(0, pos), 44 | reflect.ValueOf(KLargest(v.Slice(pos+1, v.Len()).Interface(), k-pos, cmp)), 45 | ).Interface() 46 | } 47 | 48 | // pos > k 49 | 50 | return KLargest(v.Slice(0, pos-1).Interface(), k, cmp) 51 | } 52 | 53 | // PartitionLT partitions array by i-th element 54 | // mutates ns so that all values less than i-th element are on the left 55 | // assume values are distinct 56 | // returns the pos of i-th element 57 | func PartitionLT(slice interface{}, i int, cmp func(j, k int) int) (pos int) { 58 | v := reflect.ValueOf(slice) 59 | swp := reflect.Swapper(slice) 60 | 61 | size := v.Len() 62 | for j := 0; j < size; j++ { 63 | if cmp(j, i) < 0 { 64 | pos++ 65 | } 66 | } 67 | 68 | if i != pos { 69 | swp(i, pos) 70 | } 71 | 72 | ri := pos + 1 73 | if ri == size { 74 | return 75 | } 76 | for li := 0; li < pos; li++ { 77 | if cmp(li, pos) > 0 { 78 | for { 79 | if cmp(ri, pos) < 0 { 80 | swp(li, ri) 81 | ri++ 82 | break 83 | } else { 84 | ri++ 85 | } 86 | } 87 | if ri == size { 88 | return 89 | } 90 | } 91 | 92 | } 93 | 94 | return 95 | } 96 | 97 | // PartitionGT places larger ones on the left 98 | func PartitionGT(slice interface{}, i int, cmp func(j, k int) int) (pos int) { 99 | v := reflect.ValueOf(slice) 100 | swp := reflect.Swapper(slice) 101 | 102 | size := v.Len() 103 | for j := 0; j < size; j++ { 104 | if cmp(j, i) > 0 { 105 | pos++ 106 | } 107 | } 108 | 109 | if i != pos { 110 | swp(i, pos) 111 | } 112 | 113 | ri := pos + 1 114 | if ri == size { 115 | return 116 | } 117 | for li := 0; li < pos; li++ { 118 | if cmp(li, pos) < 0 { 119 | for { 120 | if cmp(ri, pos) > 0 { 121 | swp(li, ri) 122 | ri++ 123 | break 124 | } else { 125 | ri++ 126 | } 127 | } 128 | if ri == size { 129 | return 130 | } 131 | } 132 | 133 | } 134 | 135 | return 136 | } 137 | -------------------------------------------------------------------------------- /logger/zap.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "errors" 5 | "sort" 6 | "time" 7 | 8 | "go.uber.org/zap" 9 | "go.uber.org/zap/zapcore" 10 | ) 11 | 12 | var errEncodingNotSupported = errors.New("encoding not supported") 13 | 14 | // New is similar to Config.Build except that info and error logs are separated 15 | // only json/console encoder is supported (zap doesn't provide a way to refer to other encoders) 16 | func New(cfg zap.Config) (logger *zap.Logger, err error) { 17 | 18 | sink, errSink, err := openSinks(cfg) 19 | if err != nil { 20 | return 21 | } 22 | 23 | var encoder zapcore.Encoder 24 | switch cfg.Encoding { 25 | case "json": 26 | encoder = zapcore.NewJSONEncoder(cfg.EncoderConfig) 27 | case "console": 28 | encoder = zapcore.NewConsoleEncoder(cfg.EncoderConfig) 29 | default: 30 | err = errEncodingNotSupported 31 | return 32 | } 33 | 34 | logLevel := cfg.Level.Level() 35 | stdoutPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { 36 | return lvl >= logLevel && lvl < zapcore.ErrorLevel 37 | }) 38 | stderrPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { 39 | return lvl >= zapcore.ErrorLevel 40 | }) 41 | 42 | core := zapcore.NewTee( 43 | zapcore.NewCore(encoder, sink, stdoutPriority), 44 | zapcore.NewCore(encoder, errSink, stderrPriority), 45 | ) 46 | 47 | return zap.New(core, buildOptions(cfg, errSink)...), nil 48 | } 49 | 50 | func openSinks(cfg zap.Config) (zapcore.WriteSyncer, zapcore.WriteSyncer, error) { 51 | sink, closeOut, err := zap.Open(cfg.OutputPaths...) 52 | if err != nil { 53 | return nil, nil, err 54 | } 55 | errSink, _, err := zap.Open(cfg.ErrorOutputPaths...) 56 | if err != nil { 57 | closeOut() 58 | return nil, nil, err 59 | } 60 | return sink, errSink, nil 61 | } 62 | 63 | func buildOptions(cfg zap.Config, errSink zapcore.WriteSyncer) []zap.Option { 64 | opts := []zap.Option{zap.ErrorOutput(errSink)} 65 | 66 | if cfg.Development { 67 | opts = append(opts, zap.Development()) 68 | } 69 | 70 | if !cfg.DisableCaller { 71 | opts = append(opts, zap.AddCaller()) 72 | } 73 | 74 | stackLevel := zap.ErrorLevel 75 | if cfg.Development { 76 | stackLevel = zap.WarnLevel 77 | } 78 | if !cfg.DisableStacktrace { 79 | opts = append(opts, zap.AddStacktrace(stackLevel)) 80 | } 81 | 82 | if cfg.Sampling != nil { 83 | opts = append(opts, zap.WrapCore(func(core zapcore.Core) zapcore.Core { 84 | return zapcore.NewSampler(core, time.Second, int(cfg.Sampling.Initial), int(cfg.Sampling.Thereafter)) 85 | })) 86 | } 87 | 88 | if len(cfg.InitialFields) > 0 { 89 | fs := make([]zap.Field, 0, len(cfg.InitialFields)) 90 | keys := make([]string, 0, len(cfg.InitialFields)) 91 | for k := range cfg.InitialFields { 92 | keys = append(keys, k) 93 | } 94 | sort.Strings(keys) 95 | for _, k := range keys { 96 | fs = append(fs, zap.Any(k, cfg.InitialFields[k])) 97 | } 98 | opts = append(opts, zap.Fields(fs...)) 99 | } 100 | 101 | return opts 102 | } 103 | -------------------------------------------------------------------------------- /sort/kofn_test.go: -------------------------------------------------------------------------------- 1 | package sort 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/assert" 7 | ) 8 | 9 | func TestPartition(t *testing.T) { 10 | 11 | { 12 | data := [][]int{ 13 | []int{5, 4, 3, 2, 1}, 14 | []int{1, 2, 3, 4, 5}, 15 | []int{1, 3, 2, 5, 4}, 16 | } 17 | 18 | for _, ns := range data { 19 | i := 3 20 | v := ns[i] 21 | cmp := func(i, j int) int { 22 | return ns[i] - ns[j] 23 | } 24 | pos := PartitionLT(ns, i, cmp) 25 | verifyLT(t, ns, v, pos) 26 | 27 | } 28 | } 29 | 30 | { 31 | data := [][]int{ 32 | []int{5, 4, 3, 2, 1}, 33 | []int{1, 2, 3, 4, 5}, 34 | []int{1, 3, 2, 5, 4}, 35 | } 36 | 37 | for _, ns := range data { 38 | i := 3 39 | v := ns[i] 40 | cmp := func(i, j int) int { 41 | return ns[i] - ns[j] 42 | } 43 | pos := PartitionGT(ns, i, cmp) 44 | verifyGT(t, ns, v, pos) 45 | } 46 | } 47 | 48 | { 49 | data := [][]int{ 50 | []int{5, 4, 3, 2, 1}, 51 | []int{1, 2, 3, 4, 5}, 52 | []int{1, 3, 2, 5, 4}, 53 | } 54 | 55 | for _, ns := range data { 56 | cmp := func(i, j int) int { 57 | return ns[i] - ns[j] 58 | } 59 | ks := KSmallest(ns, 3, cmp) 60 | verifyKS(t, ks.([]int), ns, 3) 61 | } 62 | } 63 | 64 | { 65 | data := [][]int{ 66 | []int{5, 4, 3, 2, 1}, 67 | []int{1, 2, 3, 4, 5}, 68 | []int{1, 3, 2, 5, 4}, 69 | } 70 | 71 | for _, ns := range data { 72 | cmp := func(i, j int) int { 73 | return ns[i] - ns[j] 74 | } 75 | kl := KLargest(ns, 3, cmp) 76 | verifyKL(t, kl.([]int), ns, 3) 77 | } 78 | } 79 | 80 | } 81 | 82 | func verifyKL(t *testing.T, ks, ns []int, k int) { 83 | assert.Assert(t, len(ks) == k, "len(ks) != k") 84 | m := make(map[int]bool) 85 | for _, v := range ns { 86 | m[v] = true 87 | } 88 | 89 | for _, v := range ks { 90 | delete(m, v) 91 | } 92 | 93 | for _, v1 := range ks { 94 | for v2 := range m { 95 | assert.Assert(t, v2 < v1, "v2>v1") 96 | } 97 | } 98 | } 99 | 100 | func verifyKS(t *testing.T, ks, ns []int, k int) { 101 | assert.Assert(t, len(ks) == k, "len(ks) != k") 102 | m := make(map[int]bool) 103 | for _, v := range ns { 104 | m[v] = true 105 | } 106 | 107 | for _, v := range ks { 108 | delete(m, v) 109 | } 110 | 111 | for _, v1 := range ks { 112 | for v2 := range m { 113 | assert.Assert(t, v2 > v1, "v2 pos { 124 | assert.Assert(t, ns[i] > v, "right element not larger") 125 | } 126 | } 127 | } 128 | 129 | func verifyGT(t *testing.T, ns []int, v int, pos int) { 130 | assert.Assert(t, v == ns[pos], "v != ns[pos]") 131 | for i := 0; i < len(ns); i++ { 132 | if i < pos { 133 | assert.Assert(t, ns[i] > v, "left element not larger") 134 | } else if i > pos { 135 | assert.Assert(t, ns[i] < v, "right element not smaller") 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /mmr/util_test.go: -------------------------------------------------------------------------------- 1 | package mmr 2 | 3 | import ( 4 | "crypto/sha256" 5 | "math" 6 | "math/bits" 7 | "testing" 8 | 9 | "gotest.tools/assert" 10 | ) 11 | 12 | func TestInclusionProof(t *testing.T) { 13 | size := uint64(0b00111110) 14 | 15 | completeSize := uint64(0b10000) 16 | assert.Assert(t, minPeakHeight(completeSize) == 4 && maxPeakHeight(completeSize) == 4) 17 | assert.Assert(t, minPeakHeight(size) == 1) 18 | assert.Assert(t, maxPeakHeight(size) == 5) 19 | assert.Assert(t, peakCount(size) == 5) 20 | assert.Assert(t, leftPeak(size) == 2*0b100000-1 && leftPeakLeaf(size) == 0b100000) 21 | 22 | sizes := getMoutainSizes(size) 23 | assert.Assert(t, 24 | sizes[0] == 2*0b100000-1 && 25 | sizes[1] == 2*0b10000-1 && 26 | sizes[2] == 2*0b1000-1 && 27 | sizes[3] == 2*0b100-1 && 28 | sizes[4] == 2*0b10-1) 29 | 30 | indices := getMoutainPeaks(size) 31 | assert.Assert(t, 32 | indices[0] == 2*0b100000-1 && 33 | indices[1] == 2*0b100000-1+2*0b10000-1 && 34 | indices[2] == 2*0b100000-1+2*0b10000-1+2*0b1000-1 && 35 | indices[3] == 2*0b100000-1+2*0b10000-1+2*0b1000-1+2*0b100-1 && 36 | indices[4] == 2*0b100000-1+2*0b10000-1+2*0b1000-1+2*0b100-1+2*0b10-1) 37 | 38 | store, err := NewFileHashStore("/tmp/hs.log", 0) 39 | assert.Assert(t, err == nil) 40 | defer store.Close() 41 | 42 | mmr := NewMMR(0, nil, NewHasher([]byte{0}, []byte{1}), store) 43 | h1 := sha256.Sum256([]byte{1}) 44 | mmr.PushHash(h1, false) 45 | h1Idx := mmr.Size() - 1 46 | 47 | h2 := sha256.Sum256([]byte{2}) 48 | ap2 := mmr.PushHash(h2, true) 49 | h2Idx := mmr.Size() - 1 50 | assert.Assert(t, len(ap2) == proofLength(h2Idx, mmr.Size()) && ap2[0] == h1) 51 | 52 | rootHash2 := mmr.Root() 53 | 54 | h3 := sha256.Sum256([]byte{3}) 55 | ap3 := mmr.PushHash(h3, true) 56 | h3Idx := mmr.Size() - 1 57 | assert.Assert(t, len(ap3) == proofLength(h3Idx, mmr.Size())) 58 | 59 | assert.Assert(t, ComputeRoot([]HashType{h1, h2, h3}) == mmr.Root()) 60 | 61 | // h2's proof is returned by Push 62 | err = mmr.VerifyInclusion(h2, rootHash2, h2Idx, h2Idx+1, ap2) 63 | assert.Assert(t, err == nil) 64 | 65 | // h3's proof is returned by Push 66 | err = mmr.VerifyInclusion(h3, mmr.Root(), h3Idx, h3Idx+1, ap3) 67 | assert.Assert(t, err == nil) 68 | 69 | // generate proof for h1 wrt current root 70 | proof, err := mmr.InclusionProof(h1Idx, mmr.Size()) 71 | assert.Assert(t, err == nil) 72 | err = mmr.VerifyInclusion(h1, mmr.Root(), h1Idx, mmr.Size(), proof) 73 | assert.Assert(t, err == nil) 74 | 75 | // test getMoutainPeaks 76 | peaks := getMoutainPeaks(8) 77 | assert.Assert(t, len(peaks) == 1 && peaks[0] == 15) 78 | 79 | } 80 | 81 | func TestConsistencyProof(t *testing.T) { 82 | store, err := NewFileHashStore("/tmp/hs.log", 0) 83 | assert.Assert(t, err == nil) 84 | defer store.Close() 85 | 86 | m := NewMMR(0, nil, NewHasher([]byte{0}, []byte{1}), store) 87 | 88 | n := uint64(7) 89 | for i := uint64(0); i < n; i++ { 90 | h := sha256.Sum256([]byte{byte(i + 1)}) 91 | m.PushHash(h, false) 92 | } 93 | 94 | cmp := []int{3, 2, 4, 1, 4, 3, 0} 95 | for i := uint64(0); i < n; i++ { 96 | proof, err := m.ConsistencyProof(uint64(i+1), n) 97 | assert.Assert(t, err == nil && len(proof) == cmp[i]) 98 | } 99 | } 100 | 101 | func TestBits(t *testing.T) { 102 | assert.Assert(t, bits.Len32(256) == 9) 103 | assert.Assert(t, bits.Len32(0) == 0) 104 | assert.Assert(t, bits.Len32(1) == 1) 105 | assert.Assert(t, math.Ceil(math.Log2(float64(256))) == 8) 106 | assert.Assert(t, 1<<8 == 256) 107 | } 108 | -------------------------------------------------------------------------------- /diskqueue/diskqueue_test.go: -------------------------------------------------------------------------------- 1 | package diskqueue 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "testing" 7 | 8 | "gotest.tools/assert" 9 | ) 10 | 11 | func TestQueue(t *testing.T) { 12 | conf := Conf{Directory: "/tmp/dq", WriteMmap: true} 13 | q, err := New(conf) 14 | assert.Assert(t, err == nil) 15 | 16 | testData := []byte("abcd") 17 | 18 | // test Read 19 | n := 1000 20 | var offsets []int64 21 | for i := 0; i < n; i++ { 22 | offset, err := q.Put(testData) 23 | offsets = append(offsets, offset) 24 | assert.Assert(t, err == nil) 25 | 26 | readData, err := q.Read(nil, offset) 27 | // fmt.Println(string(readData)) 28 | assert.Assert(t, err == nil && bytes.Equal(readData, testData), "%v:%v", err, i) 29 | assert.Assert(t, q.FileMeta(0).MsgCount == uint64(i+1)) 30 | } 31 | 32 | assert.Assert(t, q.NumFiles() == 1 && q.FileMeta(0).MsgCount == uint64(n)) 33 | 34 | // test StreamOffsetRead 35 | offsetCh := make(chan int64) 36 | ch, err := q.StreamOffsetRead(offsetCh) 37 | go func() { 38 | for i := 0; i < n; i++ { 39 | offsetCh <- offsets[i] 40 | } 41 | }() 42 | for i := 0; i < n; i++ { 43 | streamData, ok := <-ch 44 | assert.Assert(t, bytes.Equal(streamData.Bytes, testData) && streamData.Offset == offsets[i], "i:%v ok:%v streamData.Offset:%v offsets[i]:%v", i, ok, streamData.Offset, offsets[i]) 45 | } 46 | close(offsetCh) 47 | 48 | // test StreamRead 49 | ch, err = q.StreamRead(context.Background(), 0) 50 | assert.Assert(t, err == nil) 51 | for i := 0; i < n; i++ { 52 | streamData := <-ch 53 | assert.Assert(t, bytes.Equal(streamData.Bytes, testData)) 54 | } 55 | 56 | n, err = q.GC() 57 | assert.Assert(t, err == nil && n == 0) 58 | 59 | err = q.Delete() 60 | assert.Assert(t, err == nil) 61 | 62 | } 63 | 64 | func TestFixedQueue(t *testing.T) { 65 | fixedSizeMsg := []byte("abcd") 66 | 67 | conf := Conf{Directory: "/tmp/dq", WriteMmap: true, CustomDecoder: func(ctx context.Context, r *QfileSizeReader) (otherFile bool, data []byte, err error) { 68 | data = make([]byte, len(fixedSizeMsg)) 69 | err = r.Read(ctx, data) 70 | return 71 | }} 72 | q, err := New(conf) 73 | assert.Assert(t, err == nil) 74 | defer func() { 75 | err = q.Delete() 76 | assert.Assert(t, err == nil) 77 | }() 78 | 79 | // test Read 80 | n := 1000 81 | var offsets []int64 82 | for i := 0; i < n; i++ { 83 | offset, err := q.Put(fixedSizeMsg) 84 | assert.Assert(t, err == nil) 85 | offsets = append(offsets, offset) 86 | 87 | readData, err := q.Read(nil, offset) 88 | // fmt.Println(string(readData)) 89 | assert.Assert(t, err == nil && bytes.Equal(readData, fixedSizeMsg), "%v:%v:%v", err, i, offset) 90 | assert.Assert(t, q.FileMeta(0).MsgCount == uint64(i+1)) 91 | } 92 | 93 | assert.Assert(t, q.NumFiles() == 1 && q.FileMeta(0).MsgCount == uint64(n)) 94 | 95 | // test StreamOffsetRead 96 | offsetCh := make(chan int64) 97 | ch, err := q.StreamOffsetRead(offsetCh) 98 | go func() { 99 | for i := 0; i < n; i++ { 100 | offsetCh <- offsets[i] 101 | } 102 | }() 103 | for i := 0; i < n; i++ { 104 | streamData, ok := <-ch 105 | assert.Assert(t, bytes.Equal(streamData.Bytes, fixedSizeMsg) && streamData.Offset == offsets[i], "%v %v", i, ok) 106 | } 107 | close(offsetCh) 108 | 109 | // test StreamRead 110 | ch, err = q.StreamRead(context.Background(), 0) 111 | assert.Assert(t, err == nil) 112 | for i := 0; i < n; i++ { 113 | streamData := <-ch 114 | assert.Assert(t, bytes.Equal(streamData.Bytes, fixedSizeMsg)) 115 | } 116 | 117 | n, err = q.GC() 118 | assert.Assert(t, err == nil && n == 0) 119 | 120 | } 121 | -------------------------------------------------------------------------------- /parallel/parallel.go: -------------------------------------------------------------------------------- 1 | package parallel 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "sync/atomic" 8 | "time" 9 | 10 | zlog "github.com/rs/zerolog/log" 11 | ) 12 | 13 | type Result struct { 14 | R interface{} 15 | I int 16 | } 17 | 18 | func First[R any](ctx context.Context, workers int, handleFunc func(ctx context.Context, workerIdx int) (R, error), cd time.Duration) (r R, i int, err error) { 19 | if workers == 0 { 20 | err = fmt.Errorf("workers == 0") 21 | return 22 | } 23 | 24 | if workers == 1 { 25 | for { 26 | r, err = handleFunc(ctx, 0) 27 | if err == nil { 28 | return 29 | } 30 | 31 | select { 32 | case <-ctx.Done(): 33 | return 34 | default: 35 | } 36 | time.Sleep(cd) 37 | } 38 | } 39 | 40 | resultCh := make(chan Result, 1) 41 | doneCh := make(chan struct{}) 42 | for i := 0; i < workers; i++ { 43 | go func(i int) { 44 | var ( 45 | r R 46 | err error 47 | ) 48 | for { 49 | r, err = handleFunc(ctx, i) 50 | if err != nil { 51 | select { 52 | case <-doneCh: 53 | return 54 | case <-ctx.Done(): 55 | return 56 | default: 57 | } 58 | time.Sleep(cd) 59 | continue 60 | } 61 | 62 | select { 63 | case resultCh <- Result{R: r, I: i}: 64 | default: 65 | } 66 | return 67 | } 68 | 69 | }(i) 70 | } 71 | 72 | select { 73 | case result := <-resultCh: 74 | r = result.R.(R) 75 | i = result.I 76 | close(doneCh) 77 | case <-ctx.Done(): 78 | err = ctx.Err() 79 | } 80 | 81 | return 82 | } 83 | 84 | type subTask struct { 85 | from, to int 86 | } 87 | 88 | func All(ctx context.Context, total, unit, workers int, handleFunc func(ctx context.Context, workerIdx int, from int, to int) error, dispatchCB func(int, int), retry int, cd time.Duration) { 89 | if workers == 0 { 90 | panic("workers == 0") 91 | } 92 | 93 | taskChan := make(chan subTask, workers) 94 | doneCh := make(chan struct{}) 95 | 96 | var ( 97 | wg sync.WaitGroup 98 | doneCount int64 99 | ) 100 | wg.Add(workers) 101 | for i := 0; i < workers; i++ { 102 | go func(i int) { 103 | defer wg.Done() 104 | 105 | var ( 106 | err error 107 | task subTask 108 | ) 109 | for { 110 | select { 111 | case task = <-taskChan: 112 | for k := 0; k < retry; k++ { 113 | err = handleFunc(ctx, i, task.from, task.to) 114 | if err != nil { 115 | if k == retry-1 { 116 | break 117 | } 118 | zlog.Warn().Int("i", i).Err(err).Msg("handleFunc") 119 | time.Sleep(cd) 120 | continue 121 | } 122 | if atomic.AddInt64(&doneCount, int64(task.to-task.from)) == int64(total) { 123 | close(doneCh) 124 | return 125 | } 126 | break 127 | } 128 | if err != nil { 129 | // switch a worker and try again 130 | select { 131 | case taskChan <- task: 132 | case <-ctx.Done(): 133 | return 134 | } 135 | time.Sleep(cd) 136 | } 137 | 138 | case <-ctx.Done(): 139 | return 140 | case <-doneCh: 141 | return 142 | } 143 | } 144 | }(i) 145 | } 146 | 147 | for from := 0; from < total; from += unit { 148 | to := from + unit 149 | if to > total { 150 | to = total 151 | } 152 | if dispatchCB != nil { 153 | dispatchCB(from, to) 154 | } 155 | select { 156 | case taskChan <- subTask{from: from, to: to}: 157 | case <-ctx.Done(): 158 | return 159 | } 160 | } 161 | 162 | select { 163 | case <-doneCh: 164 | case <-ctx.Done(): 165 | } 166 | 167 | wg.Wait() 168 | return 169 | } 170 | -------------------------------------------------------------------------------- /leetcode_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/assert" 7 | ) 8 | 9 | func TestMaxLengthOfUniqueSubslice(t *testing.T) { 10 | { 11 | s1 := "abcdee" 12 | 13 | assert.Assert(t, MaxLengthOfUniqueSubstring(s1) == 5) 14 | } 15 | 16 | { 17 | s2 := "abcdaef" 18 | 19 | assert.Assert(t, MaxLengthOfUniqueSubstring(s2) == 6) 20 | } 21 | } 22 | 23 | func TestManacherFallback(t *testing.T) { 24 | assert.Assert(t, ManacherFallback("") == "") 25 | assert.Assert(t, ManacherFallback("abcd") == "a") 26 | assert.Assert(t, ManacherFallback("babcd") == "bab") 27 | assert.Assert(t, ManacherFallback("cbabcd") == "cbabc") 28 | assert.Assert(t, ManacherFallback("cbaabcd") == "cbaabc") 29 | 30 | assert.Assert(t, ManacherWithFallback("") == "") 31 | assert.Assert(t, ManacherWithFallback("abcd") == "a") 32 | assert.Assert(t, ManacherWithFallback("babcd") == "bab") 33 | assert.Assert(t, ManacherWithFallback("cbabcd") == "cbabc") 34 | assert.Assert(t, ManacherWithFallback("cbaabcd") == "cbaabc") 35 | 36 | } 37 | 38 | func TestReverseDigits(t *testing.T) { 39 | assert.Assert(t, ReverseDigits(123) == 321) 40 | assert.Assert(t, ReverseDigits(-123) == -321) 41 | } 42 | 43 | func TestIsPalindrome(t *testing.T) { 44 | assert.Assert(t, IsPalindrome(121)) 45 | assert.Assert(t, IsPalindrome(1122332211)) 46 | assert.Assert(t, !IsPalindrome(123)) 47 | assert.Assert(t, IsPalindrome(0)) 48 | } 49 | 50 | func TestPatternMatchAllRec(t *testing.T) { 51 | assert.Assert(t, !PatternMatchAllRec("ss", "s")) 52 | assert.Assert(t, PatternMatchAllRec("ss", "s*")) 53 | assert.Assert(t, PatternMatchAllRec("ss", "s*s")) 54 | assert.Assert(t, PatternMatchAllRec("ss", ".*s")) 55 | assert.Assert(t, PatternMatchAllRec("ss", ".*")) 56 | assert.Assert(t, PatternMatchAllRec("aab", "c*a*b")) 57 | assert.Assert(t, !PatternMatchAllRec("mississippi", "mis*is*p*.")) 58 | assert.Assert(t, PatternMatchAllRec("", ".*")) 59 | assert.Assert(t, PatternMatchAllRec("", ".*a*")) 60 | 61 | } 62 | 63 | func TestPatternMatchAllTD(t *testing.T) { 64 | assert.Assert(t, !PatternMatchAllTD("ss", "s")) 65 | assert.Assert(t, PatternMatchAllTD("ss", "s*")) 66 | assert.Assert(t, PatternMatchAllTD("ss", "s*s")) 67 | assert.Assert(t, PatternMatchAllTD("ss", ".*s")) 68 | assert.Assert(t, PatternMatchAllTD("ss", ".*")) 69 | assert.Assert(t, PatternMatchAllTD("aab", "c*a*b")) 70 | assert.Assert(t, !PatternMatchAllTD("mississippi", "mis*is*p*.")) 71 | assert.Assert(t, PatternMatchAllTD("", ".*a*")) 72 | } 73 | 74 | func TestPatternMatchAllBU(t *testing.T) { 75 | assert.Assert(t, !PatternMatchAllBU("ss", "s")) 76 | assert.Assert(t, PatternMatchAllBU("ss", "s*")) 77 | assert.Assert(t, PatternMatchAllBU("ss", "s*s")) 78 | assert.Assert(t, PatternMatchAllBU("ss", ".*s")) 79 | assert.Assert(t, PatternMatchAllBU("ss", ".*")) 80 | assert.Assert(t, PatternMatchAllBU("aab", "c*a*b")) 81 | assert.Assert(t, !PatternMatchAllBU("mississippi", "mis*is*p*.")) 82 | assert.Assert(t, PatternMatchAllBU("", ".*a*")) 83 | } 84 | 85 | func TestFindOnceNum(t *testing.T) { 86 | assert.Assert(t, FindOnceNum([]int{102, 101, 102}) == 101) 87 | assert.Assert(t, FindOnceNum([]int{999, 999, 102}) == 102) 88 | } 89 | 90 | func TestMinCoveringSubstr(t *testing.T) { 91 | assert.Assert(t, MinCoveringSubstr("ADOBECODEBANC", "ABC") == "BANC") 92 | assert.Assert(t, MinCoveringSubstr("ADOBECODEBAC", "ABC") == "BAC") 93 | assert.Assert(t, MinCoveringSubstr("", "ABC") == "") 94 | } 95 | 96 | func TestLongestConsecutive(t *testing.T) { 97 | n, len := LongestConsecutive([]int{100, 4, 200, 1, 3, 2}) 98 | assert.Assert(t, n == 1 && len == 4) 99 | } 100 | -------------------------------------------------------------------------------- /crypto/wnaf/lr.go: -------------------------------------------------------------------------------- 1 | package wnaf 2 | 3 | import ( 4 | "math/big" 5 | "math/bits" 6 | ) 7 | 8 | // FYI https://rd.springer.com/content/pdf/10.1007/978-3-540-68914-0_26.pdf 9 | 10 | // the returned wnaf "bits" are in big endian and in modified wNAF* version 11 | func ToWnafLR(d *big.Int, w uint8) (wnaf []int64) { 12 | if d.Cmp(zero) < 0 { 13 | panic("negative d not supported") 14 | } 15 | if w >= 64 { 16 | panic("w >= 64 not supported") 17 | } 18 | 19 | n := d.BitLen() 20 | if n == 0 { 21 | return []int64{0} 22 | } 23 | 24 | i := n - 1 25 | c := 0 26 | 27 | mod := big.NewInt(0).Exp(two, big.NewInt(int64(w)), nil) 28 | w_all1 := big.NewInt(0).Sub(mod, one) 29 | appendWnaf := func(r int64, w uint8) { 30 | // the caller ensures that |r| is < 2^(w-1) 31 | 32 | var abs uint64 33 | if r > 0 { 34 | abs = uint64(r) 35 | } else { 36 | abs = uint64(-r) 37 | } 38 | nz := bits.TrailingZeros64(abs) 39 | // append w-1-nz 0 to wnaf 40 | wnaf = append(wnaf, make([]int64, w-1-uint8(nz))...) 41 | wnaf = append(wnaf, r>>nz) 42 | // append nz 0 to wnaf 43 | wnaf = append(wnaf, make([]int64, nz)...) 44 | } 45 | 46 | for { 47 | if i < -1 { 48 | break 49 | } 50 | 51 | var di uint 52 | // when i < 0, di is 0 53 | if i >= 0 { 54 | di = d.Bit(i) 55 | } 56 | 57 | switch c { 58 | case 0: 59 | if di == 0 { 60 | if i < 0 { 61 | return 62 | } 63 | 64 | wnaf = append(wnaf, 0) 65 | i -= 1 66 | continue 67 | } 68 | 69 | // here it's guaranteed that di == 1 70 | j := i - int(w) + 1 71 | if j < 0 || d.Bit(j) == 0 { 72 | if j < 0 { 73 | mod := big.NewInt(0).Exp(two, big.NewInt(int64(i+1)), nil) 74 | r := big.NewInt(0).Mod(d, mod).Uint64() 75 | appendWnaf(int64(r), uint8(i+1)) 76 | 77 | i = -1 78 | } else { 79 | r := big.NewInt(0).Mod(big.NewInt(0).Rsh(d, uint(j)), mod).Uint64() 80 | appendWnaf(int64(r), w) 81 | 82 | i -= int(w) 83 | } 84 | continue 85 | } 86 | 87 | // here it's guaranteed that j >= 0 and dj == 1 88 | 89 | r := big.NewInt(0).Mod(big.NewInt(0).Rsh(d, uint(j)), mod) 90 | if r.Cmp(w_all1) == 0 { 91 | // all w bits are 1 92 | wnaf = append(wnaf, 2) 93 | // append w-1 0 to wnaf 94 | zeros := make([]int64, w-1) 95 | wnaf = append(wnaf, zeros...) 96 | 97 | i -= int(w) 98 | c = -2 99 | } else { 100 | // with 0 between i and j 101 | 102 | rPlus1 := r.Uint64() + 1 103 | appendWnaf(int64(rPlus1), w) 104 | 105 | i -= int(w) 106 | c = -2 107 | } 108 | case -2: 109 | if di == 1 { 110 | wnaf = append(wnaf, 0) 111 | i -= 1 112 | // c doesn't change 113 | continue 114 | } 115 | 116 | // here it's guaranteed that di == 0 117 | j := i - int(w) + 1 118 | if j < 0 || d.Bit(j) == 0 { 119 | 120 | if j < 0 { 121 | if i < 0 { 122 | wnaf = append(wnaf, -2) 123 | return 124 | } else { 125 | mod := big.NewInt(0).Exp(two, big.NewInt(int64(i+1)), nil) 126 | r := big.NewInt(0).Mod(d, mod) 127 | appendWnaf(-int64(big.NewInt(0).Sub(mod, r).Uint64()), uint8(i+1)) 128 | i = -1 129 | } 130 | 131 | } else { 132 | r := big.NewInt(0).Mod(big.NewInt(0).Rsh(d, uint(j)), mod) 133 | appendWnaf(-int64(big.NewInt(0).Sub(mod, r).Uint64()), w) 134 | 135 | i -= int(w) 136 | } 137 | 138 | c = 0 139 | continue 140 | } 141 | 142 | // here it's guaranteed that j >=0 and dj == 1 143 | 144 | r := big.NewInt(0).Mod(big.NewInt(0).Rsh(d, uint(j)), mod) 145 | appendWnaf(-int64(big.NewInt(0).Sub(mod, big.NewInt(0).Add(r, one)).Uint64()), w) // w < 64, so the type cast is safe 146 | 147 | i -= int(w) 148 | // c doesn't change 149 | } 150 | 151 | } 152 | 153 | return 154 | } 155 | -------------------------------------------------------------------------------- /reflect_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "encoding/json" 5 | "reflect" 6 | "testing" 7 | 8 | "gotest.tools/assert" 9 | ) 10 | 11 | func TestReflect(t *testing.T) { 12 | var f func() int 13 | { 14 | ReplaceFuncVar(&f, func([]reflect.Value) []reflect.Value { 15 | return []reflect.Value{reflect.ValueOf(30)} 16 | }) 17 | 18 | result := f() 19 | assert.Assert(t, result == 30) 20 | } 21 | 22 | { 23 | fv := reflect.ValueOf(&f) 24 | ReplaceFuncVar(fv, func([]reflect.Value) []reflect.Value { 25 | return []reflect.Value{reflect.ValueOf(31)} 26 | }) 27 | 28 | result := f() 29 | assert.Assert(t, result == 31) 30 | } 31 | 32 | s := &struct { 33 | F func() int 34 | I int 35 | }{} 36 | 37 | { 38 | ReplaceFuncVar(&s.F, func([]reflect.Value) []reflect.Value { 39 | return []reflect.Value{reflect.ValueOf(40)} 40 | }) 41 | 42 | result := s.F() 43 | assert.Assert(t, result == 40) 44 | } 45 | 46 | { 47 | sv := reflect.ValueOf(s) 48 | ReplaceFuncVar(reflect.Indirect(sv).Field(0), func([]reflect.Value) []reflect.Value { 49 | return []reflect.Value{reflect.ValueOf(41)} 50 | }) 51 | 52 | result := s.F() 53 | assert.Assert(t, result == 41) 54 | } 55 | 56 | { 57 | sfields := StructFieldValues(s, func(_ string, field reflect.Value) bool { 58 | return field.Kind() == reflect.Int 59 | }) 60 | assert.Assert(t, len(sfields) == 1) 61 | sfields["I"].Set(reflect.ValueOf(20)) 62 | assert.Assert(t, s.I == 20) 63 | } 64 | 65 | { 66 | vf := Func2Value(f) 67 | ret := vf.Call(nil) 68 | assert.Assert(t, ret[0].Interface().(int) == 31) 69 | } 70 | 71 | { 72 | inTypes := FuncInputTypes(testTarget) 73 | assert.Assert(t, len(inTypes) == 2 && inTypes[0].Kind() == reflect.Int && inTypes[1].Kind() == reflect.String) 74 | 75 | outTypes := FuncOutputTypes(testTarget) 76 | assert.Assert(t, len(outTypes) == 1 && outTypes[0].Kind() == reflect.Slice) 77 | } 78 | 79 | { 80 | stringType := TypeByPointer((*string)(nil)) 81 | assert.Assert(t, stringType == reflect.ValueOf("").Type()) 82 | 83 | is := InstanceByType(stringType) 84 | _, ok := is.(string) 85 | assert.Assert(t, ok) 86 | isPtr := InstancePtrByType(stringType) 87 | _, ok = isPtr.(*string) 88 | assert.Assert(t, ok) 89 | } 90 | 91 | var t2 TestType 92 | { 93 | methods := ScanMethods(t2) 94 | _, ok := methods["M1"] 95 | assert.Assert(t, len(methods) == 1 && ok) 96 | 97 | methods = ScanMethods(&t2) 98 | _, ok = methods["M2"] 99 | assert.Assert(t, len(methods) == 3 && ok, "%v %v", len(methods), ok) 100 | } 101 | 102 | { 103 | s := "abc" 104 | sv := reflect.ValueOf(s) 105 | sptr := InstancePtrByClone(sv) 106 | sp, ok := sptr.(*string) 107 | assert.Assert(t, ok && *sp == "abc") 108 | *sp = "def" 109 | sp, ok = sptr.(*string) 110 | assert.Assert(t, s == "abc" && ok && *sp == "def") 111 | } 112 | 113 | { 114 | var itf interface{} 115 | itf = t2 116 | _, ok := itf.(interface{ M2() }) 117 | assert.Assert(t, !ok) 118 | itf = &itf 119 | _, ok = itf.(interface{ M2() }) 120 | assert.Assert(t, !ok) 121 | itf = &t2 122 | _, ok = itf.(interface{ M2() }) 123 | assert.Assert(t, ok) 124 | } 125 | 126 | { 127 | a := JSON{A: 1} 128 | b := &JSON{A: 1} 129 | bytes1, _ := json.Marshal(a) 130 | bytes2, _ := json.Marshal(b) 131 | assert.Assert(t, string(bytes1) == string(bytes2)) 132 | } 133 | 134 | } 135 | 136 | type JSON struct { 137 | A int 138 | } 139 | 140 | func testTarget(int, string) []int { 141 | return nil 142 | } 143 | 144 | type TestType struct { 145 | Base 146 | } 147 | 148 | type Base struct { 149 | } 150 | 151 | func (t TestType) M1() { 152 | 153 | } 154 | 155 | func (t TestType) m1() { 156 | 157 | } 158 | 159 | func (t *TestType) M2() { 160 | } 161 | 162 | func (b *Base) OK() { 163 | 164 | } 165 | -------------------------------------------------------------------------------- /reflect.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "reflect" 5 | ) 6 | 7 | // ReplaceFuncVar for replace funcVar with fn 8 | func ReplaceFuncVar(funcVarPtr interface{}, fn func(in []reflect.Value) (out []reflect.Value)) { 9 | 10 | v, ok := funcVarPtr.(reflect.Value) 11 | if !ok { 12 | v = reflect.ValueOf(funcVarPtr) 13 | } 14 | 15 | v = reflect.Indirect(v) 16 | 17 | if v.Kind() != reflect.Func { 18 | panic("funcVarPtr must point to a func") 19 | } 20 | 21 | v.Set(reflect.MakeFunc(v.Type(), fn)) 22 | } 23 | 24 | // Func2Value wraps a func with reflect.Value 25 | func Func2Value(fun interface{}) reflect.Value { 26 | v := reflect.ValueOf(fun) 27 | if v.Kind() != reflect.Func { 28 | panic("fun must be a func") 29 | } 30 | return v 31 | } 32 | 33 | // FuncInputTypes for retrieve func input types 34 | func FuncInputTypes(fun interface{}) (result []reflect.Type) { 35 | fv, ok := fun.(reflect.Value) 36 | if !ok { 37 | fv = reflect.ValueOf(fun) 38 | } 39 | 40 | if fv.Kind() != reflect.Func { 41 | panic("fun must be a func") 42 | } 43 | 44 | tp := fv.Type() 45 | n := tp.NumIn() 46 | for i := 0; i < n; i++ { 47 | result = append(result, tp.In(i)) 48 | } 49 | 50 | return 51 | } 52 | 53 | // FuncOutputTypes for retrieve func output types 54 | func FuncOutputTypes(fun interface{}) (result []reflect.Type) { 55 | fv, ok := fun.(reflect.Value) 56 | if !ok { 57 | fv = reflect.ValueOf(fun) 58 | } 59 | 60 | if fv.Kind() != reflect.Func { 61 | panic("fun must be a func") 62 | } 63 | 64 | tp := fv.Type() 65 | n := tp.NumOut() 66 | for i := 0; i < n; i++ { 67 | result = append(result, tp.Out(i)) 68 | } 69 | 70 | return 71 | } 72 | 73 | // TypeByPointer for retrieve reflect.Type by a pointer value 74 | func TypeByPointer(tp interface{}) reflect.Type { 75 | return reflect.TypeOf(tp).Elem() 76 | } 77 | 78 | // InstanceByType returns a instance of reflect.Type wrapped in interface{} 79 | func InstanceByType(t reflect.Type) interface{} { 80 | return reflect.New(t).Elem().Interface() 81 | } 82 | 83 | // InstancePtrByType returns a pointer to instance 84 | func InstancePtrByType(t reflect.Type) interface{} { 85 | return reflect.New(t).Interface() 86 | } 87 | 88 | // InstancePtrByClone creates an instance ptr by clone 89 | func InstancePtrByClone(v reflect.Value) interface{} { 90 | cv := reflect.New(v.Type()) 91 | cv.Elem().Set(v) 92 | return cv.Interface() 93 | } 94 | 95 | // StructFieldValues for manipulate field values 96 | func StructFieldValues(s interface{}, filter func(name string, f reflect.Value) bool) (fields map[string]reflect.Value) { 97 | v, ok := s.(reflect.Value) 98 | if !ok { 99 | v = reflect.ValueOf(s) 100 | } 101 | v = reflect.Indirect(v) 102 | 103 | fields = make(map[string]reflect.Value) 104 | t := v.Type() 105 | count := v.NumField() 106 | for i := 0; i < count; i++ { 107 | field := v.Field(i) 108 | name := t.Field(i).Name 109 | if filter == nil { 110 | fields[name] = field 111 | } else if filter(name, field) { 112 | fields[name] = field 113 | } 114 | } 115 | 116 | return 117 | } 118 | 119 | // StructFields for general struct field info 120 | func StructFields(s interface{}) (fields []reflect.StructField) { 121 | v, ok := s.(reflect.Value) 122 | if !ok { 123 | v = reflect.ValueOf(s) 124 | } 125 | v = reflect.Indirect(v) 126 | 127 | t := v.Type() 128 | count := v.NumField() 129 | for i := 0; i < count; i++ { 130 | field := t.Field(i) 131 | fields = append(fields, field) 132 | } 133 | 134 | return 135 | } 136 | 137 | // ScanMethods for scan methods of s 138 | func ScanMethods(s interface{}) (methods map[string]reflect.Value) { 139 | v, ok := s.(reflect.Value) 140 | if !ok { 141 | v = reflect.ValueOf(s) 142 | } 143 | 144 | methods = make(map[string]reflect.Value) 145 | t := v.Type() 146 | count := v.NumMethod() 147 | for i := 0; i < count; i++ { 148 | methods[t.Method(i).Name] = v.Method(i) 149 | } 150 | 151 | return 152 | } 153 | -------------------------------------------------------------------------------- /osc/osc.go: -------------------------------------------------------------------------------- 1 | package osc 2 | 3 | import "fmt" 4 | 5 | // this package implements a generic online schema change process 6 | 7 | // SchemaState is the state for schema elements. 8 | type SchemaState byte 9 | 10 | const ( 11 | // StateAbsent means this schema element is absent and can't be used. 12 | StateAbsent SchemaState = iota 13 | // StateDeleteOnly means we can only delete items for this schema element. 14 | StateDeleteOnly 15 | // StateWriteOnly means we can use any write operation on this schema element, 16 | // but outer can't read the changed data. 17 | StateWriteOnly 18 | // StateWriteReorganization means we are re-organizing whole data after write only state. 19 | StateWriteReorganization 20 | // StateDeleteReorganization means we are re-organizing whole data after delete only state. 21 | StateDeleteReorganization 22 | // StatePublic means this schema element is ok for all write and read operations. 23 | StatePublic 24 | ) 25 | 26 | // AddSchemaChange for add schema change 27 | // each method should block until the state has been synced, or error 28 | type AddSchemaChange interface { 29 | GetState() SchemaState 30 | EnterDeleteOnly() error 31 | EnterWriteOnly() error 32 | EnterReorgAfterWriteOnly() error 33 | EnterPublic() error 34 | } 35 | 36 | // DeleteSchemaChange for delete schema change 37 | // each method should block until the state has been synced, or error 38 | type DeleteSchemaChange interface { 39 | GetState() SchemaState 40 | EnterWriteOnly() error 41 | EnterDeleteOnly() error 42 | EnterReorgAfterDeleteOnly() error 43 | EnterAbsent() error 44 | } 45 | 46 | const ( 47 | errInvalidState = "invalid state %d" 48 | ) 49 | 50 | // StepAdd for a single add schema state transition 51 | func StepAdd(asc AddSchemaChange) (err error) { 52 | state := asc.GetState() 53 | switch state { 54 | case StateAbsent: 55 | err = asc.EnterDeleteOnly() 56 | return 57 | case StateDeleteOnly: 58 | err = asc.EnterWriteOnly() 59 | return 60 | case StateWriteOnly: 61 | err = asc.EnterReorgAfterWriteOnly() 62 | return 63 | case StateWriteReorganization: 64 | err = asc.EnterPublic() 65 | return 66 | case StatePublic: 67 | return 68 | default: 69 | err = fmt.Errorf(errInvalidState, state) 70 | return 71 | } 72 | } 73 | 74 | // StepDelete for a single delete schema state transition 75 | func StepDelete(dsc DeleteSchemaChange) (err error) { 76 | state := dsc.GetState() 77 | switch state { 78 | case StatePublic: 79 | err = dsc.EnterWriteOnly() 80 | return 81 | case StateWriteOnly: 82 | err = dsc.EnterDeleteOnly() 83 | return 84 | case StateDeleteOnly: 85 | err = dsc.EnterReorgAfterDeleteOnly() 86 | return 87 | case StateDeleteReorganization: 88 | err = dsc.EnterAbsent() 89 | return 90 | case StateAbsent: 91 | return 92 | default: 93 | err = fmt.Errorf(errInvalidState, state) 94 | return 95 | } 96 | } 97 | 98 | // StartAdd for start osc add process 99 | func StartAdd(asc AddSchemaChange) (err error) { 100 | for asc.GetState() != StatePublic { 101 | err = StepAdd(asc) 102 | if err != nil { 103 | return 104 | } 105 | } 106 | return 107 | } 108 | 109 | // StartDelete for start osc delete process 110 | func StartDelete(dsc DeleteSchemaChange) (err error) { 111 | for dsc.GetState() != StateAbsent { 112 | err = StepDelete(dsc) 113 | if err != nil { 114 | return 115 | } 116 | } 117 | return 118 | } 119 | 120 | // String implements fmt.Stringer interface. 121 | func (s SchemaState) String() string { 122 | switch s { 123 | case StateAbsent: 124 | return "absent" 125 | case StateDeleteOnly: 126 | return "delete only" 127 | case StateWriteOnly: 128 | return "write only" 129 | case StateWriteReorganization: 130 | return "write reorganization" 131 | case StateDeleteReorganization: 132 | return "delete reorganization" 133 | case StatePublic: 134 | return "public" 135 | default: 136 | return fmt.Sprintf(errInvalidState, s) 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /skl/skl.go: -------------------------------------------------------------------------------- 1 | package skl 2 | 3 | import ( 4 | "math" 5 | 6 | "github.com/zhiqiangxu/util" 7 | ) 8 | 9 | const ( 10 | // DefaultMaxLevel for skl 11 | DefaultMaxLevel int = 18 12 | // DefaultProbability for skl 13 | DefaultProbability float64 = 1 / math.E 14 | ) 15 | 16 | type link struct { 17 | next *element 18 | } 19 | 20 | type element struct { 21 | links []link 22 | key int64 23 | value interface{} 24 | } 25 | 26 | type skl struct { 27 | links []link 28 | maxLevel int 29 | length int 30 | probability float64 31 | probTable []uint32 32 | prevLinksCache []*link 33 | } 34 | 35 | // NewSkipList creates a new SkipList 36 | func NewSkipList() SkipList { 37 | return NewSkipListWithMaxLevel(DefaultMaxLevel) 38 | } 39 | 40 | // NewSkipListWithMaxLevel creates a new SkipList with specified maxLevel 41 | func NewSkipListWithMaxLevel(maxLevel int) SkipList { 42 | return &skl{ 43 | links: make([]link, maxLevel), 44 | maxLevel: maxLevel, 45 | probability: DefaultProbability, 46 | probTable: probabilityTable(DefaultProbability, maxLevel), 47 | prevLinksCache: make([]*link, maxLevel), 48 | } 49 | } 50 | 51 | func probabilityTable(probability float64, maxLevel int) (table []uint32) { 52 | for i := 0; i < maxLevel; i++ { 53 | prob := math.Pow(probability, float64(i)) 54 | table = append(table, uint32(prob*math.MaxUint32)) 55 | } 56 | return table 57 | } 58 | 59 | func (s *skl) Add(key int64, value interface{}) { 60 | // prev := s.links 61 | // for i := s.maxLevel - 1; i >= 0; i-- { 62 | // s.findSpliceForLevel(key, prev, i) 63 | // } 64 | 65 | prevs := s.getPrevLinks(key) 66 | ele := prevs[0].next 67 | if ele != nil && ele.key <= key { 68 | ele.value = value 69 | return 70 | } 71 | 72 | ele = &element{ 73 | links: make([]link, s.randLevel()), 74 | key: key, 75 | value: value, 76 | } 77 | 78 | for i := range ele.links { 79 | ele.links[i].next = prevs[i].next 80 | prevs[i].next = ele 81 | } 82 | 83 | s.length++ 84 | } 85 | 86 | func (s *skl) Length() int { 87 | return s.length 88 | } 89 | 90 | func (s *skl) randLevel() (level int) { 91 | 92 | r := util.FastRand() 93 | 94 | level = 1 95 | for level < s.maxLevel && r < s.probTable[level] { 96 | level++ 97 | } 98 | return 99 | } 100 | 101 | // 找到每一层上毗邻于该key对应元素之前的links 102 | // 返回的是*link，所以可以原地更新 103 | func (s *skl) getPrevLinks(key int64) []*link { 104 | var prev = s.links 105 | var current *element 106 | 107 | prevs := s.prevLinksCache 108 | for i := s.maxLevel - 1; i >= 0; i-- { 109 | current = prev[i].next 110 | 111 | for current != nil && current.key < key { 112 | prev = current.links 113 | current = prev[i].next 114 | } 115 | 116 | prevs[i] = &prev[i] 117 | } 118 | 119 | return prevs 120 | } 121 | 122 | // func (s *skl) findSpliceForLevel(key int64, prev []link, level0based int) (before, next *element) { 123 | 124 | // current := prev[level0based].next 125 | // for current != nil && current.key < key { 126 | // prev = current.links 127 | // current = prev[level0based].next 128 | // } 129 | 130 | // before = &prev[level0based] 131 | 132 | // if current != nil { 133 | // if current.key == key { 134 | // next = before 135 | // } else { 136 | // next = ¤t.links[level0based] 137 | // } 138 | // } 139 | 140 | // return 141 | // } 142 | 143 | func (s *skl) Get(key int64) (value interface{}, ok bool) { 144 | prev := s.links 145 | var current *element 146 | for i := s.maxLevel - 1; i >= 0; i-- { 147 | current = prev[i].next 148 | for current != nil && current.key < key { 149 | prev = current.links 150 | current = current.links[i].next 151 | } 152 | } 153 | 154 | if current != nil && current.key <= key { 155 | return current.value, true 156 | } 157 | 158 | return nil, false 159 | } 160 | 161 | func (s *skl) Remove(key int64) { 162 | prevs := s.getPrevLinks(key) 163 | if ele := prevs[0].next; ele != nil && ele.key <= key { 164 | 165 | for i, l := range ele.links { 166 | prevs[i].next = l.next 167 | } 168 | s.length-- 169 | } 170 | } 171 | 172 | func (s *skl) Head() (key int64, value interface{}, ok bool) { 173 | nele := s.links[0].next 174 | if nele != nil { 175 | key = nele.key 176 | value = nele.value 177 | ok = true 178 | } 179 | return 180 | } 181 | 182 | func (s *skl) NewIterator() SkipListIterator { 183 | return &sklIter{s: s} 184 | } 185 | -------------------------------------------------------------------------------- /deadlock/semaphore.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package deadlock provides a deadlock detector based on semaphore 6 | package deadlock // import "golang.org/x/sync/semaphore" 7 | 8 | import ( 9 | "container/list" 10 | "context" 11 | "sync" 12 | ) 13 | 14 | type waiter struct { 15 | n int64 16 | ready chan<- struct{} // Closed when semaphore acquired. 17 | } 18 | 19 | type callback interface { 20 | onAcquiredLocked(n int64) 21 | onWaitLocked(n int64) 22 | onWaitCanceledLocked(n int64) 23 | onReleaseLocked(n int64) 24 | } 25 | 26 | // NewWeighted creates a new weighted semaphore with the given 27 | // maximum combined weight for concurrent access. 28 | func NewWeighted(n int64, cb callback) *Weighted { 29 | w := &Weighted{size: n, cb: cb} 30 | return w 31 | } 32 | 33 | // Weighted provides a way to bound concurrent access to a resource. 34 | // The callers can request access with a given weight. 35 | type Weighted struct { 36 | size int64 37 | cur int64 38 | mu sync.Mutex 39 | waiters list.List 40 | cb callback 41 | } 42 | 43 | // Acquire acquires the semaphore with a weight of n, blocking until resources 44 | // are available or ctx is done. On success, returns nil. On failure, returns 45 | // ctx.Err() and leaves the semaphore unchanged. 46 | // 47 | // If ctx is already done, Acquire may still succeed without blocking. 48 | func (s *Weighted) Acquire(ctx context.Context, n int64) error { 49 | s.mu.Lock() 50 | if s.size-s.cur >= n && s.waiters.Len() == 0 { 51 | s.cur += n 52 | s.cb.onAcquiredLocked(n) 53 | s.mu.Unlock() 54 | return nil 55 | } 56 | 57 | if n > s.size { 58 | // Don't make other Acquire calls block on one that's doomed to fail. 59 | s.mu.Unlock() 60 | panic("semaphore: acquire more than size") 61 | } 62 | 63 | ready := make(chan struct{}) 64 | w := waiter{n: n, ready: ready} 65 | elem := s.waiters.PushBack(w) 66 | s.cb.onWaitLocked(n) 67 | s.mu.Unlock() 68 | 69 | select { 70 | case <-ctx.Done(): 71 | err := ctx.Err() 72 | s.mu.Lock() 73 | select { 74 | case <-ready: 75 | // Acquired the semaphore after we were canceled. Rather than trying to 76 | // fix up the queue, just pretend we didn't notice the cancelation. 77 | err = nil 78 | default: 79 | isFront := s.waiters.Front() == elem 80 | s.waiters.Remove(elem) 81 | if isFront && s.size > s.cur { 82 | s.notifyWaiters() 83 | } 84 | s.cb.onWaitCanceledLocked(n) 85 | } 86 | s.mu.Unlock() 87 | return err 88 | 89 | case <-ready: 90 | return nil 91 | } 92 | } 93 | 94 | // TryAcquire acquires the semaphore with a weight of n without blocking. 95 | // On success, returns true. On failure, returns false and leaves the semaphore unchanged. 96 | func (s *Weighted) TryAcquire(n int64) bool { 97 | s.mu.Lock() 98 | success := s.size-s.cur >= n && s.waiters.Len() == 0 99 | if success { 100 | s.cb.onAcquiredLocked(n) 101 | s.cur += n 102 | } 103 | s.mu.Unlock() 104 | return success 105 | } 106 | 107 | // Release releases the semaphore with a weight of n. 108 | func (s *Weighted) Release(n int64) { 109 | s.mu.Lock() 110 | s.cur -= n 111 | if s.cur < 0 { 112 | s.mu.Unlock() 113 | panic("semaphore: released more than held") 114 | } 115 | s.cb.onReleaseLocked(n) 116 | s.notifyWaiters() 117 | s.mu.Unlock() 118 | } 119 | 120 | func (s *Weighted) notifyWaiters() { 121 | for { 122 | next := s.waiters.Front() 123 | if next == nil { 124 | break // No more waiters blocked. 125 | } 126 | 127 | w := next.Value.(waiter) 128 | if s.size-s.cur < w.n { 129 | // Not enough tokens for the next waiter. We could keep going (to try to 130 | // find a waiter with a smaller request), but under load that could cause 131 | // starvation for large requests; instead, we leave all remaining waiters 132 | // blocked. 133 | // 134 | // Consider a semaphore used as a read-write lock, with N tokens, N 135 | // readers, and one writer. Each reader can Acquire(1) to obtain a read 136 | // lock. The writer can Acquire(N) to obtain a write lock, excluding all 137 | // of the readers. If we allow the readers to jump ahead in the queue, 138 | // the writer will starve — there is always one token available for every 139 | // reader. 140 | break 141 | } 142 | 143 | s.cur += w.n 144 | s.waiters.Remove(next) 145 | s.cb.onAcquiredLocked(w.n) 146 | close(w.ready) 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /rpheap/rank_pairing_heap.go: -------------------------------------------------------------------------------- 1 | package rpheap 2 | 3 | type node struct { 4 | item int64 5 | left, next *node // 对于root，next用于循环链表，指向下一棵半树；对于非root，next指向右儿子 6 | rank int 7 | } 8 | 9 | // Heap for rank pairing heap 10 | // zero value is an empty heap 11 | type Heap struct { 12 | head *node // 指向循环链表的头部 13 | size int 14 | } 15 | 16 | // New is ctor for Heap 17 | func New() *Heap { 18 | return &Heap{} 19 | } 20 | 21 | type heapInterface interface { 22 | Insert(val int64) 23 | FindMin() int64 24 | // 传入的heap将清零 25 | Meld(*Heap) 26 | DeleteMin() int64 27 | Size() int 28 | Clear() 29 | } 30 | 31 | var _ heapInterface = (*Heap)(nil) 32 | 33 | // Insert into the heap 34 | func (h *Heap) Insert(val int64) { 35 | ptr := &node{item: val} 36 | h.insertRoot(ptr) 37 | h.size++ 38 | } 39 | 40 | func (h *Heap) insertRoot(ptr *node /*ptr 是根节点*/) { 41 | if h.head == nil { // 第一棵半树 42 | h.head = ptr 43 | ptr.next = ptr // 循环链表 44 | } else { 45 | // 先把ptr串到header后面 46 | ptr.next = h.head.next 47 | h.head.next = ptr 48 | // 如果ptr更小，将header指向ptr即可，这里体现了循环链表的灵活性 49 | if ptr.item < h.head.item { 50 | h.head = ptr 51 | } 52 | } 53 | } 54 | 55 | // FindMin from the heap 56 | func (h *Heap) FindMin() int64 { 57 | if h.head == nil { 58 | panic("FindMin on empty heap") 59 | } 60 | return h.head.item 61 | } 62 | 63 | // Meld two heaps 64 | func (h *Heap) Meld(a *Heap) { 65 | if h.head == nil { 66 | h.head = a.head 67 | h.size = a.size 68 | a.Clear() 69 | return 70 | } 71 | if a.head == nil { 72 | return 73 | } 74 | 75 | // 两个heap都非空 76 | if h.head.item < a.head.item { 77 | merge(h, a) 78 | } else { 79 | merge(a, h) 80 | h.head = a.head 81 | h.size = a.size 82 | } 83 | 84 | a.Clear() 85 | return 86 | } 87 | 88 | func merge(a, b *Heap) { 89 | // 前置条件是a、b非空 90 | // 将b往a合并， 循环链表需要对长度为1的情况特殊处理下 91 | if a.size == 1 { 92 | ptr := a.head 93 | ptr.next = nil 94 | if ptr.left != nil || ptr.rank != 0 { 95 | panic("size 1 heap with left or non-zero rank") 96 | } 97 | b.insertRoot(ptr) 98 | b.size++ 99 | a.head = b.head 100 | a.size = b.size 101 | return 102 | } else if b.size == 1 { 103 | ptr := b.head 104 | ptr.next = nil 105 | if ptr.left != nil || ptr.rank != 0 { 106 | panic("size 1 heap with left or non-zero rank") 107 | } 108 | a.insertRoot(ptr) 109 | a.size++ 110 | return 111 | } 112 | 113 | // 两个链表长度都大于1 114 | // 将b的第二个元素串到a的第一个元素后面 115 | // 同时将a原先的第二个元素串到b的第一个元素后面 116 | // 这样b的第二个元素最终会回到b的第一个元素，然后再到a原先的第二个元素，直到回到a的头部 117 | // 便把两个链表合并了 118 | a.head.next, b.head.next = b.head.next, a.head.next 119 | a.size += b.size 120 | 121 | } 122 | 123 | // DeleteMin from the heap 124 | func (h *Heap) DeleteMin() int64 { 125 | if h.head == nil { 126 | panic("DeleteMin on empty heap") 127 | } 128 | 129 | h.size-- 130 | ret := h.head.item 131 | 132 | bucket := make([]*node, h.maxBucketSize()) 133 | // 沿着root的唯一左child一路往右（包括左child本身），将root所在的树分解成若干棵半树，并对相同rank的树进行合并 134 | for ptr := h.head.left; ptr != nil; { 135 | // 先记录下右child，因为ptr要作为独立半树进行合并，next必须置为空 136 | nextPtr := ptr.next 137 | ptr.next = nil 138 | mergeSubTreeByRank(bucket, ptr) 139 | ptr = nextPtr 140 | } 141 | // 遍历所有半树串成的循环链表，首棵树除外 142 | for ptr := h.head.next; ptr != h.head; { 143 | nextPtr := ptr.next 144 | ptr.next = nil 145 | mergeSubTreeByRank(bucket, ptr) 146 | ptr = nextPtr 147 | } 148 | 149 | // 将bucket中的所有半树插入到空heap 150 | h.head = nil 151 | for _, ptr := range bucket { 152 | if ptr != nil { 153 | h.insertRoot(ptr) 154 | } 155 | } 156 | return ret 157 | } 158 | 159 | func mergeSubTreeByRank(bucket []*node, ptr *node /*ptr 是根节点*/) { 160 | for bucket[ptr.rank] != nil { 161 | rank := ptr.rank 162 | ptr = link(ptr, bucket[rank]) 163 | bucket[rank] = nil 164 | } 165 | bucket[ptr.rank] = ptr 166 | } 167 | 168 | func link(a *node, b *node) *node { 169 | // 前置条件是a、b都非空 170 | var winner, loser *node 171 | // 以a、b中较小的根作为合并后的半树的根 172 | if a.item < b.item { 173 | winner = a 174 | loser = b 175 | } else { 176 | winner = b 177 | loser = a 178 | } 179 | 180 | // 将胜出半树的左child挂到胜败半树的右child上（此前右child为空） 181 | loser.next = winner.left 182 | winner.left = loser 183 | // 在此之前a、b的rank已经保证相等 184 | winner.rank++ 185 | return winner 186 | } 187 | 188 | func (h *Heap) maxBucketSize() int { 189 | // 对于一n个元素的rpheap，rank最多是log n 190 | // 对于type 1或type 2的rpheap，rank上限更低，这里简单起见按log n算 191 | bit, cnt := 0, h.size 192 | for cnt > 1 { 193 | cnt /= 2 194 | bit++ 195 | } 196 | return bit + 1 // [0, rank]共有rank+1个值 197 | } 198 | 199 | // Size of the heap 200 | func (h *Heap) Size() int { 201 | return h.size 202 | } 203 | 204 | // Clear the heap 205 | func (h *Heap) Clear() { 206 | h.head = nil 207 | h.size = 0 208 | } 209 | -------------------------------------------------------------------------------- /xorlayer/xtm.go: -------------------------------------------------------------------------------- 1 | package xorlayer 2 | 3 | import ( 4 | "context" 5 | "math/bits" 6 | "sync" 7 | "time" 8 | "unsafe" 9 | 10 | "github.com/zhiqiangxu/util/sort" 11 | ) 12 | 13 | // NodeID can be replace by https://github.com/zhiqiangxu/gg 14 | type NodeID uint64 15 | 16 | const ( 17 | bitSize = int(unsafe.Sizeof(NodeID(0)) * 8) 18 | ) 19 | 20 | // Callback is used by XTM 21 | type Callback interface { 22 | Ping(ctx context.Context, nodeID NodeID) error 23 | } 24 | 25 | // XTM manages xor topology 26 | type XTM struct { 27 | sync.RWMutex 28 | k int 29 | h int 30 | theta int // for delimiting close region 31 | id NodeID 32 | buckets []*bucket 33 | cb Callback 34 | } 35 | 36 | // NewXTM is ctor for XTM 37 | // caller is responsible for dealing with duplicate NodeID 38 | func NewXTM(k, h int, id NodeID, cb Callback) *XTM { 39 | buckets := make([]*bucket, bitSize) 40 | for i := range buckets { 41 | buckets[i] = newBucket() 42 | } 43 | x := &XTM{k: k, h: h, theta: -1, id: id, buckets: buckets, cb: cb} 44 | return x 45 | } 46 | 47 | // AddNeighbours is batch for AddNeighbour 48 | func (x *XTM) AddNeighbours(ns []NodeID, cookies []uint64) { 49 | x.Lock() 50 | 51 | var unlocked bool 52 | 53 | for i, n := range ns { 54 | unlocked = x.addNeighbourLocked(n, cookies[i]) 55 | if unlocked { 56 | unlocked = false 57 | x.Lock() 58 | } 59 | } 60 | 61 | if !unlocked { 62 | x.Unlock() 63 | } 64 | } 65 | 66 | // AddNeighbour tries to add NodeID with cookie to kbucket 67 | func (x *XTM) AddNeighbour(n NodeID, cookie uint64) { 68 | x.Lock() 69 | 70 | if !x.addNeighbourLocked(n, cookie) { 71 | x.Unlock() 72 | } 73 | } 74 | 75 | const ( 76 | pingTimeout = time.Second * 2 77 | ) 78 | 79 | func (x *XTM) addNeighbourLocked(n NodeID, cookie uint64) (unlocked bool) { 80 | i := x.getBucketIdx(n) 81 | 82 | if i >= bitSize { 83 | return 84 | } 85 | 86 | bucket := x.buckets[i] 87 | exists := bucket.refresh(n) 88 | if exists { 89 | return 90 | } 91 | 92 | if i <= x.theta { 93 | // at most |x.k + x.h| 94 | if bucket.size() < (x.k + x.h) { 95 | bucket.insert(n, cookie) 96 | } else { 97 | if x.cb != nil { 98 | oldest := bucket.oldest() 99 | x.Unlock() 100 | ctx, cancelFunc := context.WithTimeout(context.Background(), pingTimeout) 101 | defer cancelFunc() 102 | 103 | err := x.cb.Ping(ctx, oldest.N) 104 | if err != nil { 105 | x.Lock() 106 | if bucket.remove(oldest.N, oldest.Cookie) { 107 | bucket.insert(n, cookie) 108 | } 109 | } else { 110 | unlocked = true 111 | } 112 | } 113 | } 114 | } else { 115 | // no limit 116 | bucket.insert(n, cookie) 117 | 118 | // increment theta if necessary 119 | 120 | for { 121 | if x.buckets[x.theta+1].size() >= (x.k + x.h) { 122 | total := 0 123 | for j := x.theta + 2; j < bitSize; j++ { 124 | total += x.buckets[j].size() 125 | } 126 | if total >= (x.k + x.h) { 127 | x.theta++ 128 | x.buckets[x.theta].reduceTo(x.k + x.h) 129 | } else { 130 | break 131 | } 132 | } else { 133 | break 134 | } 135 | } 136 | 137 | } 138 | 139 | return 140 | } 141 | 142 | func (x *XTM) delNeighbourLocked(n NodeID, cookie uint64) { 143 | i := x.getBucketIdx(n) 144 | if i >= bitSize { 145 | return 146 | } 147 | 148 | x.buckets[i].remove(n, cookie) 149 | 150 | if i <= x.theta { 151 | // decrement theta if necessary 152 | if x.buckets[i].size() < x.k { 153 | x.theta = i - 1 154 | } 155 | } 156 | 157 | } 158 | 159 | // zero based index 160 | // 0 for all NodeID that's different from id from the first bit, all with the same prefix 1 bit (2^^63) 161 | // 1 for all NodeID that's different from id from the second bit, all with the same prefix 2 bit (2^^62) 162 | // ... 163 | // 63 for all NodeID that's different from id from the 64th bit, all with the same prefix 64 bit(2^^0) 164 | // 64 means n == id 165 | func (x *XTM) getBucketIdx(n NodeID) int { 166 | xor := uint64(n ^ x.id) 167 | lz := bits.LeadingZeros64(xor) 168 | 169 | return lz 170 | } 171 | 172 | // DelNeighbours is batch for DelNeighbour 173 | func (x *XTM) DelNeighbours(ns []NodeID, cookies []uint64) { 174 | x.Lock() 175 | defer x.Unlock() 176 | 177 | for i, n := range ns { 178 | x.delNeighbourLocked(n, cookies[i]) 179 | } 180 | } 181 | 182 | // DelNeighbour by NodeID and cookie 183 | func (x *XTM) DelNeighbour(n NodeID, cookie uint64) { 184 | x.Lock() 185 | defer x.Unlock() 186 | 187 | x.delNeighbourLocked(n, cookie) 188 | } 189 | 190 | // NeighbourCount returns total neighbour count 191 | func (x *XTM) NeighbourCount() (total int) { 192 | x.RLock() 193 | defer x.RUnlock() 194 | 195 | for _, bucket := range x.buckets { 196 | total += bucket.size() 197 | } 198 | return total 199 | } 200 | 201 | // KClosest returns k-closest nodes to target 202 | func (x *XTM) KClosest(target NodeID) (ns []NodeID) { 203 | x.RLock() 204 | defer x.RUnlock() 205 | 206 | ns = make([]NodeID, 0, x.k) 207 | 208 | i := x.getBucketIdx(target) 209 | if i >= bitSize { 210 | ns = append(ns, x.id) 211 | remain := x.k - 1 212 | for j := bitSize - 1; j >= 0; j-- { 213 | bucket := x.buckets[j] 214 | ns = bucket.appendXClosest(ns, remain, target) 215 | remain = x.k - len(ns) 216 | if remain == 0 { 217 | return 218 | } 219 | } 220 | return 221 | } 222 | 223 | ns = x.buckets[i].appendXClosest(ns, x.k, target) 224 | remain := x.k - len(ns) 225 | if remain == 0 { 226 | return 227 | } 228 | 229 | // search i+1, i+2, ... etc 230 | var right []NodeID 231 | for j := i + 1; j < bitSize; j++ { 232 | right = x.buckets[i].appendAll(right) 233 | } 234 | right = append(right, x.id) 235 | 236 | kclosest := sort.KSmallest(right, remain, func(j, k int) int { 237 | dj := right[j] ^ target 238 | dk := right[k] ^ target 239 | switch { 240 | case dj < dk: 241 | return -1 242 | case dj > dk: 243 | return 1 244 | } 245 | 246 | return 0 247 | }).([]NodeID) 248 | for _, n := range kclosest { 249 | ns = append(ns, n) 250 | } 251 | 252 | remain = x.k - len(ns) 253 | if remain == 0 { 254 | return 255 | } 256 | 257 | // search i-1, i-2, ... etc 258 | for j := i - 1; j >= 0; j-- { 259 | bucket := x.buckets[j] 260 | ns = bucket.appendXClosest(ns, remain, target) 261 | remain = x.k - len(ns) 262 | if remain == 0 { 263 | return 264 | } 265 | } 266 | 267 | return 268 | } 269 | -------------------------------------------------------------------------------- /diskqueue/meta.go: -------------------------------------------------------------------------------- 1 | package diskqueue 2 | 3 | import ( 4 | "encoding/binary" 5 | "sync" 6 | "unsafe" 7 | 8 | "os" 9 | "path/filepath" 10 | 11 | "github.com/zhiqiangxu/util" 12 | "github.com/zhiqiangxu/util/logger" 13 | "github.com/zhiqiangxu/util/mapped" 14 | "go.uber.org/zap" 15 | ) 16 | 17 | type queueMetaROInterface interface { 18 | NumFiles() int 19 | Stat() QueueMeta 20 | FileMeta(idx int) FileMeta 21 | } 22 | 23 | type queueMetaInterface interface { 24 | queueMetaROInterface 25 | Init() error 26 | AddFile(f FileMeta) 27 | UpdateFileStat(idx, n int, endOffset, endTime int64) 28 | LocateFile(readOffset int64) int 29 | UpdateMinValidIndex(minValidIndex uint32) 30 | Sync() error 31 | Close() error 32 | } 33 | 34 | var _ queueMetaInterface = (*queueMeta)(nil) 35 | 36 | const ( 37 | maxSizeForMeta = 1024 * 1024 38 | ) 39 | 40 | var ( 41 | reservedHeaderSize = util.Max(256 /*should be enough*/, int(unsafe.Sizeof(QueueMeta{}))) 42 | ) 43 | 44 | // FileMeta for a single file 45 | type FileMeta struct { 46 | StartOffset int64 // inclusive 47 | EndOffset int64 // exclusive 48 | StartTime int64 49 | EndTime int64 50 | MsgCount uint64 51 | } 52 | 53 | // QueueMeta for the queue 54 | type QueueMeta struct { 55 | FileCount uint32 56 | MinValidIndex uint32 57 | } 58 | 59 | type queueMeta struct { 60 | mu sync.RWMutex 61 | conf *Conf 62 | mappedFile *mapped.File 63 | mappedBytes []byte 64 | } 65 | 66 | // NewQueueMeta is ctor for queueMeta 67 | func newQueueMeta(conf *Conf) *queueMeta { 68 | return &queueMeta{conf: conf} 69 | } 70 | 71 | const ( 72 | metaFile = "qm" 73 | ) 74 | 75 | // Init either load or creates the meta file 76 | func (m *queueMeta) Init() (err error) { 77 | path := filepath.Join(m.conf.Directory, metaFile) 78 | if _, err := os.Stat(path); os.IsNotExist(err) { 79 | m.mappedFile, err = mapped.CreateFile(path, maxSizeForMeta, true, nil) 80 | } else { 81 | m.mappedFile, err = mapped.OpenFile(path, maxSizeForMeta, os.O_RDWR, true, nil) 82 | } 83 | if err != nil { 84 | return 85 | } 86 | 87 | err = m.mappedFile.MLock() 88 | if err != nil { 89 | return 90 | } 91 | m.mappedBytes = m.mappedFile.MappedBytes() 92 | 93 | return nil 94 | } 95 | 96 | func (m *queueMeta) NumFiles() int { 97 | m.mu.RLock() 98 | defer m.mu.RUnlock() 99 | 100 | return int(binary.BigEndian.Uint32(m.mappedBytes)) 101 | } 102 | 103 | func (m *queueMeta) Stat() QueueMeta { 104 | m.mu.RLock() 105 | defer m.mu.RUnlock() 106 | 107 | return QueueMeta{FileCount: binary.BigEndian.Uint32(m.mappedBytes), MinValidIndex: binary.BigEndian.Uint32(m.mappedBytes[4:])} 108 | } 109 | 110 | func (m *queueMeta) FileMeta(idx int) (fm FileMeta) { 111 | m.mu.RLock() 112 | defer m.mu.RUnlock() 113 | 114 | nFiles := int(binary.BigEndian.Uint32(m.mappedBytes)) 115 | if idx >= nFiles { 116 | logger.Instance().Fatal("FileMeta idx over size", zap.Int("idx", idx), zap.Int("nFiles", nFiles)) 117 | } 118 | 119 | offset := reservedHeaderSize + int(unsafe.Sizeof(FileMeta{}))*idx 120 | startOffset := int64(binary.BigEndian.Uint64(m.mappedBytes[offset:])) 121 | endOffset := int64(binary.BigEndian.Uint64(m.mappedBytes[offset+8:])) 122 | startTime := int64(binary.BigEndian.Uint64(m.mappedBytes[offset+16:])) 123 | endTime := int64(binary.BigEndian.Uint64(m.mappedBytes[offset+24:])) 124 | msgCount := binary.BigEndian.Uint64(m.mappedBytes[offset+32:]) 125 | 126 | fm = FileMeta{StartOffset: startOffset, EndOffset: endOffset, StartTime: startTime, EndTime: endTime, MsgCount: msgCount} 127 | return 128 | 129 | } 130 | 131 | func (m *queueMeta) AddFile(f FileMeta) { 132 | m.mu.Lock() 133 | defer m.mu.Unlock() 134 | 135 | nFiles := binary.BigEndian.Uint32(m.mappedBytes) 136 | binary.BigEndian.PutUint32(m.mappedBytes, nFiles+1) 137 | offset := reservedHeaderSize + int(unsafe.Sizeof(FileMeta{}))*int(nFiles) 138 | binary.BigEndian.PutUint64(m.mappedBytes[offset:], uint64(f.StartOffset)) 139 | binary.BigEndian.PutUint64(m.mappedBytes[offset+8:], uint64(f.EndOffset)) 140 | binary.BigEndian.PutUint64(m.mappedBytes[offset+16:], uint64(f.StartTime)) 141 | binary.BigEndian.PutUint64(m.mappedBytes[offset+24:], uint64(f.EndTime)) 142 | 143 | } 144 | 145 | func (m *queueMeta) UpdateFileStat(idx, n int, endOffset, endTime int64) { 146 | m.mu.Lock() 147 | defer m.mu.Unlock() 148 | 149 | nFiles := int(binary.BigEndian.Uint32(m.mappedBytes)) 150 | if idx >= nFiles { 151 | logger.Instance().Fatal("UpdateFileStat idx over size", zap.Int("idx", idx), zap.Int("nFiles", nFiles)) 152 | } 153 | 154 | offset := reservedHeaderSize + int(unsafe.Sizeof(FileMeta{}))*idx 155 | endOffset0 := int64(binary.BigEndian.Uint64(m.mappedBytes[offset+8:])) 156 | startTime0 := int64(binary.BigEndian.Uint64(m.mappedBytes[offset+16:])) 157 | endTime0 := int64(binary.BigEndian.Uint64(m.mappedBytes[offset+24:])) 158 | msgCount0 := binary.BigEndian.Uint64(m.mappedBytes[offset+32:]) 159 | 160 | if endOffset > endOffset0 { 161 | binary.BigEndian.PutUint64(m.mappedBytes[offset+8:], uint64(endOffset)) 162 | } 163 | 164 | if endTime < startTime0 { 165 | binary.BigEndian.PutUint64(m.mappedBytes[offset+16:], uint64(endTime)) 166 | } 167 | 168 | if endTime > endTime0 { 169 | binary.BigEndian.PutUint64(m.mappedBytes[offset+24:], uint64(endTime)) 170 | } 171 | 172 | binary.BigEndian.PutUint64(m.mappedBytes[offset+32:], msgCount0+uint64(n)) 173 | 174 | return 175 | 176 | } 177 | 178 | func (m *queueMeta) LocateFile(readOffset int64) int { 179 | 180 | m.mu.RLock() 181 | defer m.mu.RUnlock() 182 | 183 | start := int(binary.BigEndian.Uint32(m.mappedBytes[4:])) 184 | end := int(binary.BigEndian.Uint32(m.mappedBytes) - 1) 185 | 186 | // 二分查找 187 | for start <= end { 188 | target := start + (end-start)/2 // 防止溢出 189 | offset := reservedHeaderSize + int(unsafe.Sizeof(FileMeta{}))*target 190 | startOffset := int64(binary.BigEndian.Uint64(m.mappedBytes[offset:])) 191 | endOffset := int64(binary.BigEndian.Uint64(m.mappedBytes[offset+8:])) 192 | switch { 193 | case startOffset <= readOffset && endOffset > readOffset: 194 | return target 195 | case readOffset < startOffset: 196 | end = target - 1 197 | case readOffset >= endOffset: 198 | start = target + 1 199 | } 200 | } 201 | 202 | return -1 203 | 204 | } 205 | 206 | func (m *queueMeta) UpdateMinValidIndex(minValidIndex uint32) { 207 | m.mu.Lock() 208 | binary.BigEndian.PutUint32(m.mappedBytes[4:], minValidIndex) 209 | m.mu.Unlock() 210 | } 211 | 212 | func (m *queueMeta) Sync() error { 213 | return m.mappedFile.Sync() 214 | } 215 | 216 | func (m *queueMeta) Close() error { 217 | m.mappedBytes = nil 218 | return m.mappedFile.Close() 219 | } 220 | -------------------------------------------------------------------------------- /deadlock/detector.go: -------------------------------------------------------------------------------- 1 | package deadlock 2 | 3 | import ( 4 | "runtime" 5 | "sync" 6 | 7 | "github.com/petermattis/goid" 8 | "github.com/zhiqiangxu/util" 9 | ) 10 | 11 | type detector struct { 12 | mu sync.Mutex 13 | ownerResouces map[int64]map[uint64]bool 14 | resouceOwners map[uint64]*resourceOwner 15 | waitForMap map[int64]*waitForResource 16 | } 17 | 18 | type resourceOwner struct { 19 | wgid int64 20 | rgids map[int64]int 21 | } 22 | 23 | type waitForResource struct { 24 | resourceID uint64 25 | w bool 26 | } 27 | 28 | func newDetector() *detector { 29 | return &detector{ 30 | ownerResouces: make(map[int64]map[uint64]bool), 31 | resouceOwners: make(map[uint64]*resourceOwner), 32 | waitForMap: make(map[int64]*waitForResource), 33 | } 34 | } 35 | 36 | func (d *detector) onAcquiredLocked(resourceID uint64, w bool) { 37 | gid := goid.Get() 38 | 39 | d.mu.Lock() 40 | defer d.mu.Unlock() 41 | 42 | // update ownerResouces 43 | ownedResources := d.ownerResouces[gid] 44 | if ownedResources == nil { 45 | ownedResources = make(map[uint64]bool) 46 | d.ownerResouces[gid] = ownedResources 47 | } 48 | ownedResources[resourceID] = w 49 | 50 | // update resouceOwners 51 | resourceOwners := d.resouceOwners[resourceID] 52 | if resourceOwners == nil { 53 | resourceOwners = &resourceOwner{} 54 | d.resouceOwners[resourceID] = resourceOwners 55 | } 56 | if w { 57 | if resourceOwners.wgid != 0 { 58 | panic("write lock holding by more than one owners") 59 | } 60 | resourceOwners.wgid = gid 61 | } else { 62 | rgids := resourceOwners.rgids 63 | if rgids == nil { 64 | rgids = make(map[int64]int) 65 | resourceOwners.rgids = rgids 66 | } 67 | rgids[gid]++ 68 | } 69 | 70 | // update waitForMap 71 | delete(d.waitForMap, gid) 72 | } 73 | 74 | // ErrorDeadlock contains deadlock info 75 | type ErrorDeadlock struct { 76 | SourceParty Party 77 | OwnerParty Party 78 | Stack string 79 | } 80 | 81 | // ErrorUsage for incorrect lock usage 82 | type ErrorUsage struct { 83 | Msg string 84 | Stack string 85 | } 86 | 87 | // Party for one side of deadlock 88 | type Party struct { 89 | GID int64 90 | ResourceID uint64 91 | W bool 92 | } 93 | 94 | func getCallStack() string { 95 | const size = 64 << 10 96 | buf := make([]byte, size) 97 | buf = buf[:runtime.Stack(buf, false)] 98 | return util.String(buf) 99 | } 100 | 101 | // ParsePanicError returns non nil ErrorDeadlock if deadlock happend 102 | // the ErrorUsage is non nil for lock usage problems 103 | func ParsePanicError(panicErr interface{}) (edl *ErrorDeadlock, errUsage *ErrorUsage) { 104 | if panicErr == nil { 105 | return 106 | } 107 | 108 | if panicErrStr, ok := panicErr.(string); ok { 109 | errUsage = &ErrorUsage{ 110 | Msg: panicErrStr, 111 | Stack: getCallStack(), 112 | } 113 | return 114 | } 115 | 116 | if panicErrDL, ok := panicErr.(*ErrorDeadlock); ok { 117 | panicErrDL.Stack = getCallStack() 118 | edl = panicErrDL 119 | return 120 | } 121 | 122 | panic("bug happened") 123 | } 124 | 125 | func (d *detector) onWaitLocked(resourceID uint64, w bool) { 126 | gid := goid.Get() 127 | 128 | d.mu.Lock() 129 | defer d.mu.Unlock() 130 | 131 | if d.waitForMap[gid] != nil { 132 | panic("waiting for multiple resources") 133 | } 134 | 135 | resourceOwners := d.resouceOwners[resourceID] 136 | if resourceOwners == nil { 137 | panic("waiting for a resource with no owner") 138 | } 139 | if resourceOwners.wgid == 0 && len(resourceOwners.rgids) == 0 { 140 | panic("waiting for a resource with no owner") 141 | } 142 | 143 | // detect deadlock 144 | var err *ErrorDeadlock 145 | // check deadlock with write lock owner 146 | if resourceOwners.wgid != 0 { 147 | err = d.doDetect(gid, resourceOwners.wgid) 148 | if err != nil { 149 | err.OwnerParty = Party{GID: resourceOwners.wgid, ResourceID: resourceID, W: true} 150 | panic(err) 151 | } 152 | } 153 | // check deadlock with read lock owner 154 | for rgid := range resourceOwners.rgids { 155 | err = d.doDetect(gid, rgid) 156 | if err != nil { 157 | err.OwnerParty = Party{GID: rgid, ResourceID: resourceID, W: false} 158 | panic(err) 159 | } 160 | } 161 | 162 | d.waitForMap[gid] = &waitForResource{resourceID: resourceID, w: w} 163 | } 164 | 165 | func (d *detector) doDetect(sourceGID, ownerGID int64) (err *ErrorDeadlock) { 166 | waitingForResource := d.waitForMap[ownerGID] 167 | if waitingForResource == nil { 168 | return 169 | } 170 | 171 | resourceOwners := d.resouceOwners[waitingForResource.resourceID] 172 | if resourceOwners == nil || (resourceOwners.wgid == 0 && len(resourceOwners.rgids) == 0) { 173 | panic("waiting for a resource with no owner") 174 | } 175 | 176 | if resourceOwners.wgid != 0 { 177 | if resourceOwners.wgid == sourceGID { 178 | err = &ErrorDeadlock{SourceParty: Party{ 179 | GID: sourceGID, 180 | ResourceID: waitingForResource.resourceID, 181 | W: true, 182 | }} 183 | return 184 | } 185 | err = d.doDetect(sourceGID, resourceOwners.wgid) 186 | if err != nil { 187 | return 188 | } 189 | } 190 | 191 | for rgid := range resourceOwners.rgids { 192 | if rgid == sourceGID { 193 | err = &ErrorDeadlock{ 194 | SourceParty: Party{ 195 | GID: sourceGID, 196 | ResourceID: waitingForResource.resourceID, 197 | W: false, 198 | }} 199 | return 200 | } 201 | err = d.doDetect(sourceGID, rgid) 202 | if err != nil { 203 | return 204 | } 205 | } 206 | return 207 | } 208 | 209 | func (d *detector) onReleaseLocked(resourceID uint64, w bool) { 210 | gid := goid.Get() 211 | 212 | d.mu.Lock() 213 | defer d.mu.Unlock() 214 | 215 | // update ownerResouces 216 | ownedResources := d.ownerResouces[gid] 217 | if ownedResources == nil { 218 | panic("releasing a lock not owned") 219 | } 220 | if _, exists := ownedResources[resourceID]; !exists { 221 | panic("releasing a lock not owned") 222 | } 223 | delete(ownedResources, resourceID) 224 | if len(ownedResources) == 0 { 225 | delete(d.ownerResouces, gid) 226 | } 227 | 228 | // update resouceOwners 229 | resourceOwners := d.resouceOwners[resourceID] 230 | if resourceOwners == nil { 231 | panic("releasing a lock not owned") 232 | } 233 | if w { 234 | if resourceOwners.wgid != gid { 235 | panic("releasing a lock not owned") 236 | } 237 | resourceOwners.wgid = 0 238 | if len(resourceOwners.rgids) == 0 { 239 | delete(d.resouceOwners, resourceID) 240 | } 241 | } else { 242 | if _, exists := resourceOwners.rgids[gid]; !exists { 243 | panic("releasing a lock not owned") 244 | } 245 | 246 | resourceOwners.rgids[gid]-- 247 | if resourceOwners.rgids[gid] == 0 { 248 | delete(resourceOwners.rgids, gid) 249 | if len(resourceOwners.rgids) == 0 && resourceOwners.wgid == 0 { 250 | delete(d.resouceOwners, resourceID) 251 | } 252 | } else if resourceOwners.rgids[gid] < 0 { 253 | panic("releasing a read lock too many times") 254 | } 255 | } 256 | } 257 | 258 | var d *detector 259 | 260 | func init() { 261 | d = newDetector() 262 | } 263 | -------------------------------------------------------------------------------- /wm/max_test.go: -------------------------------------------------------------------------------- 1 | package wm 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "runtime" 7 | "sync" 8 | "sync/atomic" 9 | "testing" 10 | "time" 11 | 12 | "golang.org/x/sync/semaphore" 13 | ) 14 | 15 | func TestBasicEnter(t *testing.T) { 16 | m := NewMax(1) 17 | 18 | m.Enter(context.Background()) 19 | 20 | // Try blocking lock the mutex from a different goroutine. This must 21 | // not block because the mutex is held. 22 | ch := make(chan struct{}, 1) 23 | go func() { 24 | m.Enter(context.Background()) 25 | ch <- struct{}{} 26 | m.Exit() 27 | ch <- struct{}{} 28 | }() 29 | 30 | select { 31 | case <-ch: 32 | t.Fatalf("Lock succeeded on locked mutex") 33 | case <-time.After(100 * time.Millisecond): 34 | } 35 | 36 | // Unlock the mutex and make sure that the goroutine waiting on Lock() 37 | // unblocks and succeeds. 38 | m.Exit() 39 | 40 | select { 41 | case <-ch: 42 | case <-time.After(100 * time.Millisecond): 43 | t.Fatalf("Lock failed to acquire unlocked mutex") 44 | } 45 | 46 | // Make sure we can lock and unlock again. 47 | m.Enter(context.Background()) 48 | m.Exit() 49 | } 50 | 51 | func TestTryEnter(t *testing.T) { 52 | m := NewMax(1) 53 | 54 | // Try to lock. It should succeed. 55 | if !m.TryEnter() { 56 | t.Fatalf("TryEnter failed on unlocked mutex") 57 | } 58 | 59 | // Try to lock again, it should now fail. 60 | if m.TryEnter() { 61 | t.Fatalf("TryEnter succeeded on locked mutex") 62 | } 63 | 64 | // Try blocking lock the mutex from a different goroutine. This must 65 | // not block because the mutex is held. 66 | ch := make(chan struct{}, 1) 67 | go func() { 68 | m.Enter(context.Background()) 69 | ch <- struct{}{} 70 | m.Exit() 71 | }() 72 | 73 | select { 74 | case <-ch: 75 | t.Fatalf("Lock succeeded on locked mutex") 76 | case <-time.After(100 * time.Millisecond): 77 | } 78 | 79 | // Unlock the mutex and make sure that the goroutine waiting on Lock() 80 | // unblocks and succeeds. 81 | m.Exit() 82 | 83 | select { 84 | case <-ch: 85 | case <-time.After(100 * time.Millisecond): 86 | t.Fatalf("Lock failed to acquire unlocked mutex") 87 | } 88 | } 89 | 90 | func TestMutualExclusion(t *testing.T) { 91 | m := NewMax(1) 92 | 93 | // Test mutual exclusion by running "gr" goroutines concurrently, and 94 | // have each one increment a counter "iters" times within the critical 95 | // section established by the mutex. 96 | // 97 | // If at the end the counter is not gr * iters, then we know that 98 | // goroutines ran concurrently within the critical section. 99 | // 100 | // If one of the goroutines doesn't complete, it's likely a bug that 101 | // causes to it to wait forever. 102 | const gr = 100 103 | const iters = 100000 104 | v := 0 105 | var wg sync.WaitGroup 106 | for i := 0; i < gr; i++ { 107 | wg.Add(1) 108 | go func() { 109 | for j := 0; j < iters; j++ { 110 | m.Enter(context.Background()) 111 | v++ 112 | m.Exit() 113 | } 114 | wg.Done() 115 | }() 116 | } 117 | 118 | wg.Wait() 119 | 120 | if v != gr*iters { 121 | t.Fatalf("Bad count: got %v, want %v", v, gr*iters) 122 | } 123 | } 124 | 125 | func TestMutualExclusionWithTryEnter(t *testing.T) { 126 | m := NewMax(1) 127 | 128 | // Similar to the previous, with the addition of some goroutines that 129 | // only increment the count if TryEnter succeeds. 130 | const gr = 100 131 | const iters = 100000 132 | total := int64(gr * iters) 133 | var tryTotal int64 134 | v := int64(0) 135 | var wg sync.WaitGroup 136 | for i := 0; i < gr; i++ { 137 | wg.Add(2) 138 | go func() { 139 | for j := 0; j < iters; j++ { 140 | m.Enter(context.Background()) 141 | v++ 142 | m.Exit() 143 | } 144 | wg.Done() 145 | }() 146 | go func() { 147 | local := int64(0) 148 | for j := 0; j < iters; j++ { 149 | if m.TryEnter() { 150 | v++ 151 | m.Exit() 152 | local++ 153 | } 154 | } 155 | atomic.AddInt64(&tryTotal, local) 156 | wg.Done() 157 | }() 158 | } 159 | 160 | wg.Wait() 161 | 162 | t.Logf("tryTotal = %d", tryTotal) 163 | total += tryTotal 164 | 165 | if v != total { 166 | t.Fatalf("Bad count: got %v, want %v", v, total) 167 | } 168 | } 169 | 170 | // BenchmarkMax is equivalent to TestMutualExclusion, with the following 171 | // differences: 172 | // 173 | // - The number of goroutines is variable, with the maximum value depending on 174 | // GOMAXPROCS. 175 | // 176 | // - The number of iterations per benchmark is controlled by the benchmarking 177 | // framework. 178 | // 179 | // - Care is taken to ensure that all goroutines participating in the benchmark 180 | // have been created before the benchmark begins. 181 | func BenchmarkMax(b *testing.B) { 182 | for n, max := 1, 4*runtime.GOMAXPROCS(0); n > 0 && n <= max; n *= 2 { 183 | b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { 184 | m := NewMax(1) 185 | 186 | var ready sync.WaitGroup 187 | begin := make(chan struct{}) 188 | var end sync.WaitGroup 189 | for i := 0; i < n; i++ { 190 | ready.Add(1) 191 | end.Add(1) 192 | go func() { 193 | ready.Done() 194 | <-begin 195 | for j := 0; j < b.N; j++ { 196 | m.Enter(context.Background()) 197 | m.Exit() 198 | } 199 | end.Done() 200 | }() 201 | } 202 | 203 | ready.Wait() 204 | b.ResetTimer() 205 | close(begin) 206 | end.Wait() 207 | }) 208 | } 209 | } 210 | 211 | func BenchmarkSyncMutex(b *testing.B) { 212 | for n, max := 1, 4*runtime.GOMAXPROCS(0); n > 0 && n <= max; n *= 2 { 213 | b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { 214 | var m sync.Mutex 215 | 216 | var ready sync.WaitGroup 217 | begin := make(chan struct{}) 218 | var end sync.WaitGroup 219 | for i := 0; i < n; i++ { 220 | ready.Add(1) 221 | end.Add(1) 222 | go func() { 223 | ready.Done() 224 | <-begin 225 | for j := 0; j < b.N; j++ { 226 | m.Lock() 227 | m.Unlock() 228 | } 229 | end.Done() 230 | }() 231 | } 232 | 233 | ready.Wait() 234 | b.ResetTimer() 235 | close(begin) 236 | end.Wait() 237 | }) 238 | } 239 | } 240 | 241 | func BenchmarkChan(b *testing.B) { 242 | for n, max := 1, 4*runtime.GOMAXPROCS(0); n > 0 && n <= max; n *= 2 { 243 | b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { 244 | ch := make(chan struct{}, 1) 245 | 246 | var ready sync.WaitGroup 247 | begin := make(chan struct{}) 248 | var end sync.WaitGroup 249 | for i := 0; i < n; i++ { 250 | ready.Add(1) 251 | end.Add(1) 252 | go func() { 253 | ready.Done() 254 | <-begin 255 | for j := 0; j < b.N; j++ { 256 | ch <- struct{}{} 257 | <-ch 258 | } 259 | end.Done() 260 | }() 261 | } 262 | 263 | ready.Wait() 264 | b.ResetTimer() 265 | close(begin) 266 | end.Wait() 267 | }) 268 | } 269 | } 270 | 271 | func BenchmarkSemaphore(b *testing.B) { 272 | for n, max := 1, 4*runtime.GOMAXPROCS(0); n > 0 && n <= max; n *= 2 { 273 | b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { 274 | sema := semaphore.NewWeighted(1) 275 | 276 | var ready sync.WaitGroup 277 | begin := make(chan struct{}) 278 | var end sync.WaitGroup 279 | for i := 0; i < n; i++ { 280 | ready.Add(1) 281 | end.Add(1) 282 | go func() { 283 | ready.Done() 284 | <-begin 285 | for j := 0; j < b.N; j++ { 286 | sema.Acquire(context.Background(), 1) 287 | sema.Release(1) 288 | } 289 | end.Done() 290 | }() 291 | } 292 | 293 | ready.Wait() 294 | b.ResetTimer() 295 | close(begin) 296 | end.Wait() 297 | }) 298 | } 299 | } 300 | -------------------------------------------------------------------------------- /crypto/vrf/p256/p256.go: -------------------------------------------------------------------------------- 1 | // Package p256 implements a verifiable random function using curve p256. 2 | package p256 3 | 4 | import ( 5 | "bytes" 6 | "crypto/ecdsa" 7 | "crypto/elliptic" 8 | "crypto/hmac" 9 | "crypto/rand" 10 | "crypto/sha256" 11 | "crypto/sha512" 12 | "crypto/x509" 13 | "encoding/binary" 14 | "encoding/pem" 15 | "errors" 16 | "math/big" 17 | 18 | "github.com/zhiqiangxu/util/crypto/vrf" 19 | ) 20 | 21 | // PublicKey for vrf 22 | type PublicKey struct { 23 | *ecdsa.PublicKey 24 | } 25 | 26 | // PrivateKey for vrf 27 | type PrivateKey struct { 28 | *ecdsa.PrivateKey 29 | } 30 | 31 | var ( 32 | curve = elliptic.P256() 33 | params = curve.Params() 34 | ) 35 | 36 | // H1 hashes m to a curve point 37 | func H1(m []byte) (x, y *big.Int, err error) { 38 | h := sha512.New() 39 | var i uint32 40 | byteLen := (params.BitSize + 7) >> 3 41 | for x == nil && i < 100 { 42 | // TODO: Use a NIST specified DRBG. 43 | h.Reset() 44 | binary.Write(h, binary.BigEndian, i) 45 | h.Write(m) 46 | r := []byte{2} // Set point encoding to "compressed", y=0. 47 | r = h.Sum(r) 48 | x, y, err = Unmarshal(curve, r[:byteLen+1]) 49 | i++ 50 | } 51 | return 52 | } 53 | 54 | var one = big.NewInt(1) 55 | 56 | // H2 hashes m to an integer [1,N-1] 57 | func H2(m []byte) *big.Int { 58 | // NIST SP 800-90A § A.5.1: Simple discard method. 59 | byteLen := (params.BitSize + 7) >> 3 60 | h := sha512.New() 61 | for i := uint32(0); ; i++ { 62 | // TODO: Use a NIST specified DRBG. 63 | h.Reset() 64 | binary.Write(h, binary.BigEndian, i) 65 | h.Write(m) 66 | b := h.Sum(nil) 67 | k := new(big.Int).SetBytes(b[:byteLen]) 68 | if k.Cmp(new(big.Int).Sub(params.N, one)) == -1 { 69 | return k.Add(k, one) 70 | } 71 | } 72 | } 73 | 74 | // Hash converts alpha to beta+proof 75 | func (k PrivateKey) Hash(alpha []byte) (beta [32]byte, proof []byte, err error) { 76 | r, _, _, err := elliptic.GenerateKey(curve, rand.Reader) 77 | if err != nil { 78 | return 79 | } 80 | 81 | ri := new(big.Int).SetBytes(r) 82 | 83 | // H = H1(m) 84 | Hx, Hy, err := H1(alpha) 85 | if err != nil { 86 | return 87 | } 88 | 89 | // VRF_k(m) = [k]H 90 | sHx, sHy := curve.ScalarMult(Hx, Hy, k.D.Bytes()) 91 | vrf := elliptic.Marshal(curve, sHx, sHy) // 65 bytes. 92 | 93 | // G is the base point 94 | // s = H2(G, H, [k]G, VRF, [r]G, [r]H) 95 | rGx, rGy := curve.ScalarBaseMult(r) 96 | rHx, rHy := curve.ScalarMult(Hx, Hy, r) 97 | var b bytes.Buffer 98 | b.Write(elliptic.Marshal(curve, params.Gx, params.Gy)) 99 | b.Write(elliptic.Marshal(curve, Hx, Hy)) 100 | b.Write(elliptic.Marshal(curve, k.PublicKey.X, k.PublicKey.Y)) 101 | b.Write(vrf) 102 | b.Write(elliptic.Marshal(curve, rGx, rGy)) 103 | b.Write(elliptic.Marshal(curve, rHx, rHy)) 104 | s := H2(b.Bytes()) 105 | 106 | // t = r−s*k mod N 107 | t := new(big.Int).Sub(ri, new(big.Int).Mul(s, k.D)) 108 | t.Mod(t, params.N) 109 | 110 | // beta = H(vrf) 111 | beta = sha256.Sum256(vrf) 112 | 113 | // Write s, t, and vrf to a proof blob. Also write leading zeros before s and t 114 | // if needed. 115 | var buf bytes.Buffer 116 | buf.Write(make([]byte, 32-len(s.Bytes()))) 117 | buf.Write(s.Bytes()) 118 | buf.Write(make([]byte, 32-len(t.Bytes()))) 119 | buf.Write(t.Bytes()) 120 | buf.Write(vrf) 121 | proof = buf.Bytes() 122 | 123 | return 124 | } 125 | 126 | // Public returns the corresponding public key as Verifier. 127 | func (k PrivateKey) Public() vrf.Verifier { 128 | return PublicKey{PublicKey: &k.PublicKey} 129 | } 130 | 131 | // Verify that beta+proof is generated from m by PrivateKey 132 | func (pk PublicKey) Verify(m, proof []byte, beta [32]byte) (valid bool, err error) { 133 | // verifier checks that s == H2(m, [t]G + [s]([k]G), [t]H1(m) + [s]VRF_k(m)) 134 | if got, want := len(proof), 64+65; got != want { 135 | return 136 | } 137 | 138 | // Parse proof into s, t, and vrf. 139 | s := proof[0:32] 140 | t := proof[32:64] 141 | vrf := proof[64 : 64+65] 142 | 143 | uHx, uHy := elliptic.Unmarshal(curve, vrf) 144 | if uHx == nil { 145 | return 146 | } 147 | 148 | // [t]G + [s]([k]G) = [t+ks]G 149 | tGx, tGy := curve.ScalarBaseMult(t) 150 | ksGx, ksGy := curve.ScalarMult(pk.X, pk.Y, s) 151 | tksGx, tksGy := curve.Add(tGx, tGy, ksGx, ksGy) 152 | 153 | // H = H1(m) 154 | // [t]H + [s]VRF = [t+ks]H 155 | Hx, Hy, err := H1(m) 156 | if err != nil { 157 | return 158 | } 159 | tHx, tHy := curve.ScalarMult(Hx, Hy, t) 160 | sHx, sHy := curve.ScalarMult(uHx, uHy, s) 161 | tksHx, tksHy := curve.Add(tHx, tHy, sHx, sHy) 162 | 163 | // H2(G, H, [k]G, VRF, [t]G + [s]([k]G), [t]H + [s]VRF) 164 | // = H2(G, H, [k]G, VRF, [t+ks]G, [t+ks]H) 165 | // = H2(G, H, [k]G, VRF, [r]G, [r]H) 166 | var b bytes.Buffer 167 | b.Write(elliptic.Marshal(curve, params.Gx, params.Gy)) 168 | b.Write(elliptic.Marshal(curve, Hx, Hy)) 169 | b.Write(elliptic.Marshal(curve, pk.X, pk.Y)) 170 | b.Write(vrf) 171 | b.Write(elliptic.Marshal(curve, tksGx, tksGy)) 172 | b.Write(elliptic.Marshal(curve, tksHx, tksHy)) 173 | h2 := H2(b.Bytes()) 174 | 175 | // Left pad h2 with zeros if needed. This will ensure that h2 is padded 176 | // the same way s is. 177 | var buf bytes.Buffer 178 | buf.Write(make([]byte, 32-len(h2.Bytes()))) 179 | buf.Write(h2.Bytes()) 180 | 181 | if !hmac.Equal(s, buf.Bytes()) { 182 | return 183 | } 184 | 185 | valid = true 186 | return 187 | } 188 | 189 | // GeneratePair generates a fresh signer/verifier pair for this VRF 190 | func GeneratePair() (k vrf.Signer, pk vrf.Verifier, err error) { 191 | key, err := ecdsa.GenerateKey(curve, rand.Reader) 192 | if err != nil { 193 | return 194 | } 195 | 196 | k, pk = PrivateKey{PrivateKey: key}, PublicKey{PublicKey: &key.PublicKey} 197 | return 198 | } 199 | 200 | var ( 201 | errPointNotOnCurve = errors.New("point not on curve") 202 | errNoPEMFound = errors.New("no pem found") 203 | errWrongKeyType = errors.New("wrong key type") 204 | ) 205 | 206 | // NewVRFSigner creates a signer object from a private key. 207 | func NewVRFSigner(key *ecdsa.PrivateKey) (k vrf.Signer, err error) { 208 | if *(key.Params()) != *curve.Params() { 209 | err = errPointNotOnCurve 210 | return 211 | } 212 | if !curve.IsOnCurve(key.X, key.Y) { 213 | err = errPointNotOnCurve 214 | return 215 | } 216 | k = PrivateKey{PrivateKey: key} 217 | return 218 | } 219 | 220 | // NewVRFVerifier creates a verifier object from a public key. 221 | func NewVRFVerifier(pubkey *ecdsa.PublicKey) (pk vrf.Verifier, err error) { 222 | if *(pubkey.Params()) != *curve.Params() { 223 | err = errPointNotOnCurve 224 | return 225 | } 226 | if !curve.IsOnCurve(pubkey.X, pubkey.Y) { 227 | err = errPointNotOnCurve 228 | return 229 | } 230 | pk = PublicKey{PublicKey: pubkey} 231 | return 232 | } 233 | 234 | // NewVRFSignerFromRawKey returns the private key from a raw private key bytes. 235 | func NewVRFSignerFromRawKey(b []byte) (k vrf.Signer, err error) { 236 | key, err := x509.ParseECPrivateKey(b) 237 | if err != nil { 238 | return 239 | } 240 | k, err = NewVRFSigner(key) 241 | return 242 | } 243 | 244 | // NewVRFSignerFromPEM creates a vrf private key from a PEM data structure. 245 | func NewVRFSignerFromPEM(b []byte) (k vrf.Signer, err error) { 246 | p, _ := pem.Decode(b) 247 | if p == nil { 248 | err = errNoPEMFound 249 | return 250 | } 251 | k, err = NewVRFSignerFromRawKey(p.Bytes) 252 | return 253 | } 254 | 255 | // NewVRFVerifierFromRawKey returns the public key from a raw public key bytes. 256 | func NewVRFVerifierFromRawKey(b []byte) (pk vrf.Verifier, err error) { 257 | key, err := x509.ParsePKIXPublicKey(b) 258 | if err != nil { 259 | return 260 | } 261 | pubkey, ok := key.(*ecdsa.PublicKey) 262 | if !ok { 263 | err = errWrongKeyType 264 | return 265 | } 266 | pk, err = NewVRFVerifier(pubkey) 267 | return 268 | } 269 | -------------------------------------------------------------------------------- /skl/lf/list.go: -------------------------------------------------------------------------------- 1 | package lf 2 | 3 | import ( 4 | "bytes" 5 | "sync/atomic" 6 | "unsafe" 7 | ) 8 | 9 | // List is a lock free sorted singly linked list 10 | type List struct { 11 | head listNode 12 | arena *Arena 13 | } 14 | 15 | var _ list = (*List)(nil) 16 | 17 | // NewListWithArena with specified Arena 18 | func NewListWithArena(arena *Arena) *List { 19 | l := &List{arena: arena} 20 | 21 | l.head.head = true 22 | return l 23 | } 24 | 25 | // NewList with arenaSize 26 | func NewList(arenaSize uint32) *List { 27 | arena := NewArena(arenaSize) 28 | return NewListWithArena(arena) 29 | } 30 | 31 | // ListNodeSize is the size of ListNode 32 | const ListNodeSize = int(unsafe.Sizeof(listNode{})) 33 | 34 | const ( 35 | bitMask = ^uint32(0x03) 36 | markBit = uint32(1) 37 | flagBit = uint32(2) 38 | ) 39 | 40 | type listNode struct { 41 | // Multiple parts of the value are encoded as a single uint64 so that it 42 | // can be atomically loaded and stored: 43 | // value offset: uint32 (bits 0-31) 44 | // value size : uint16 (bits 32-63) 45 | value uint64 46 | backlink uint32 // points to the prev node 47 | succ uint32 // contains a next pointer, a mark bit and a flag bit. 48 | keyOffset uint32 // Immutable. No need to lock to access key. 49 | keySize uint16 // Immutable. No need to lock to access key. 50 | head bool 51 | } 52 | 53 | // Contains checks whether k is in list 54 | func (l *List) Contains(k []byte) bool { 55 | current, _ := l.searchFrom(k, l.headNode(), true) 56 | return current.Compare(l.arena, k) == 0 57 | } 58 | 59 | // Get v by k if exists 60 | // v is readonly 61 | func (l *List) Get(k []byte) (v []byte, exists bool) { 62 | current, _ := l.searchFrom(k, l.headNode(), true) 63 | if current.Compare(l.arena, k) == 0 { 64 | exists = true 65 | v = current.Value(l.arena) 66 | } 67 | return 68 | } 69 | 70 | func (l *List) headNode() *listNode { 71 | return &l.head 72 | } 73 | 74 | // Insert attempts to insert a new node with the supplied key and value. 75 | func (l *List) Insert(k, v []byte) (isNew bool, err error) { 76 | prev, next := l.searchFrom(k, l.headNode(), true) 77 | 78 | var voffset uint32 79 | 80 | if prev.Compare(l.arena, k) == 0 { 81 | 82 | voffset, err = l.arena.putBytes(v) 83 | if err != nil { 84 | return 85 | } 86 | prev.UpdateValue(voffset, uint16(len(v))) 87 | return 88 | } 89 | 90 | node, err := newListNode(l.arena, k, v) 91 | if err != nil { 92 | return 93 | } 94 | nodeOffset := l.arena.getListNodeOffset(node) 95 | 96 | for { 97 | prevSucc := prev.Succ() 98 | // If the predecessor is flagged, help 99 | // the corresponding deletion to complete. 100 | if prevSucc&flagBit != 0 { 101 | l.helpFlagged(prev, l.arena.getListNode(prevSucc&bitMask)) 102 | } else { 103 | node.succ = l.arena.getListNodeOffset(next) 104 | // Insertion attempt. 105 | if atomic.CompareAndSwapUint32(&prev.succ, node.succ, nodeOffset) { 106 | // Successful insertion. 107 | isNew = true 108 | return 109 | } 110 | 111 | // Failure. 112 | 113 | // Failure due to flagging. 114 | if prev.Flagged() { 115 | l.helpFlagged(prev, prev.Next(l.arena)) 116 | } 117 | // Possibly a failure due to marking. Traverse a 118 | // chain of backlinks to reach an unmarked node. 119 | for prev.Marked() { 120 | prev = l.arena.getListNode(prev.GetBacklist()) 121 | } 122 | } 123 | 124 | prev, next = l.searchFrom(k, prev, true) 125 | if prev.Compare(l.arena, k) == 0 { 126 | prev.UpdateValue(voffset, uint16(len(v))) 127 | return 128 | } 129 | } 130 | } 131 | 132 | // Delete sttempts to delete a node with the supplied key 133 | func (l *List) Delete(k []byte) bool { 134 | prev, del := l.searchFrom(k, l.headNode(), false) 135 | if del == nil || del.Compare(l.arena, k) != 0 { 136 | return false 137 | } 138 | 139 | prev, flagged := l.tryFlag(prev, del) 140 | if prev != nil { 141 | l.helpFlagged(prev, del) 142 | } 143 | 144 | return flagged 145 | } 146 | 147 | // finds two consecutive nodes n1 and n2 148 | // pre condition: 149 | // node.key < k 150 | // if equal is true: 151 | // n1.key <= k < n2.key. 152 | // if equal is false: 153 | // n1.key < k <= n2.key. 154 | func (l *List) searchFrom(k []byte, node *listNode, equal bool) (current, next *listNode) { 155 | 156 | var cmpFunc func(n *listNode) bool 157 | if equal { 158 | cmpFunc = func(n *listNode) bool { 159 | return n.Compare(l.arena, k) <= 0 160 | } 161 | } else { 162 | cmpFunc = func(n *listNode) bool { 163 | return n.Compare(l.arena, k) < 0 164 | } 165 | } 166 | 167 | current = node 168 | next = node.Next(l.arena) 169 | for next != nil && cmpFunc(next) { 170 | for { 171 | nextSuc := next.Succ() 172 | currentSuc := current.Succ() 173 | currentNext := l.arena.getListNode(currentSuc & bitMask) 174 | 175 | // Ensure that either next node is unmarked, 176 | // or both curr node and next node are 177 | // marked and curr node was marked earlier. 178 | if nextSuc&markBit == 1 && (currentSuc&markBit == 0 || currentNext != next) { 179 | if currentNext == next { 180 | l.helpMarked(current, next) 181 | } 182 | next = currentNext 183 | } else { 184 | break 185 | } 186 | } 187 | 188 | if next != nil && cmpFunc(next) { 189 | current = next 190 | next = current.Next(l.arena) 191 | } 192 | 193 | } 194 | return 195 | } 196 | 197 | // Attempts to physically delete the marked 198 | // node del node and unflag prev node. 199 | func (l *List) helpMarked(prev, del *listNode) { 200 | next := del.Succ() & bitMask 201 | atomic.CompareAndSwapUint32(&prev.succ, l.arena.getListNodeOffset(del)+flagBit, next) 202 | } 203 | 204 | // Attempts to flag the predecessor of target node. P rev node is the last node known to be the predecessor. 205 | func (l *List) tryFlag(prev, target *listNode) (n *listNode, flagged bool) { 206 | 207 | for { 208 | // predecessor is already flagged 209 | if prev.Flagged() { 210 | n = prev 211 | return 212 | } 213 | targetOffset := l.arena.getListNodeOffset(target) 214 | if atomic.CompareAndSwapUint32(&prev.succ, targetOffset, targetOffset+flagBit) { 215 | // c&s was successful 216 | n = prev 217 | flagged = true 218 | return 219 | } 220 | 221 | if prev.Flagged() { 222 | // failure due to flagging 223 | n = prev 224 | return 225 | } 226 | 227 | // possibly failure due to marking 228 | for prev.Marked() { 229 | prev = l.arena.getListNode(prev.GetBacklist()) 230 | } 231 | 232 | var del *listNode 233 | prev, del = l.searchFrom(target.Key(l.arena), prev, false) 234 | // target_node was deleted from the list 235 | if del != target { 236 | return 237 | } 238 | } 239 | 240 | } 241 | 242 | // Attempts to mark the node del node. 243 | func (l *List) tryMark(del *listNode) { 244 | for !del.Marked() { 245 | next := del.Succ() & bitMask 246 | swapped := atomic.CompareAndSwapUint32(&del.succ, next, next+markBit) 247 | if !swapped { 248 | if del.Flagged() { 249 | l.helpFlagged(del, l.arena.getListNode(next)) 250 | } 251 | } 252 | } 253 | } 254 | 255 | // Attempts to mark and physically delete node del node, 256 | // which is the successor of the flagged node prev node. 257 | func (l *List) helpFlagged(prev, del *listNode) { 258 | del.SetBacklist(l.arena.getListNodeOffset(prev)) 259 | 260 | l.tryMark(del) 261 | l.helpMarked(prev, del) 262 | } 263 | 264 | func newListNode(arena *Arena, k, v []byte) (n *listNode, err error) { 265 | koff, voff, err := arena.putKV(k, v) 266 | if err != nil { 267 | return 268 | } 269 | noff, err := arena.putListNode() 270 | if err != nil { 271 | return 272 | } 273 | n = arena.getListNode(noff) 274 | n.keyOffset = koff 275 | n.keySize = uint16(len(k)) 276 | n.value = encodeValue(voff, uint16(len(v))) 277 | return 278 | } 279 | 280 | func encodeValue(valOffset uint32, valSize uint16) uint64 { 281 | return uint64(valSize)<<32 | uint64(valOffset) 282 | } 283 | 284 | func decodeValue(value uint64) (valOffset uint32, valSize uint16) { 285 | valSize = uint16(value >> 32) 286 | valOffset = uint32(value) 287 | return 288 | } 289 | 290 | func (n *listNode) Flagged() bool { 291 | return atomic.LoadUint32(&n.succ)&flagBit != 0 292 | } 293 | 294 | func (n *listNode) Marked() bool { 295 | return atomic.LoadUint32(&n.succ)&markBit != 0 296 | } 297 | 298 | func (n *listNode) Succ() uint32 { 299 | return atomic.LoadUint32(&n.succ) 300 | } 301 | 302 | func (n *listNode) Key(arena *Arena) []byte { 303 | return arena.getBytes(n.keyOffset, n.keySize) 304 | } 305 | 306 | func (n *listNode) Value(arena *Arena) []byte { 307 | v := atomic.LoadUint64(&n.value) 308 | voff, vsize := decodeValue(v) 309 | return arena.getBytes(voff, vsize) 310 | } 311 | 312 | func (n *listNode) UpdateValue(offset uint32, size uint16) { 313 | value := encodeValue(offset, size) 314 | atomic.StoreUint64(&n.value, value) 315 | } 316 | 317 | func (n *listNode) SetBacklist(prevOffset uint32) { 318 | atomic.StoreUint32(&n.backlink, prevOffset) 319 | } 320 | 321 | func (n *listNode) GetBacklist() uint32 { 322 | return atomic.LoadUint32(&n.backlink) 323 | } 324 | 325 | func (n *listNode) Next(arena *Arena) *listNode { 326 | succ := n.Succ() 327 | return arena.getListNode(succ & bitMask) 328 | } 329 | 330 | func (n *listNode) Compare(arena *Arena, k []byte) int { 331 | if n.head { 332 | return -1 333 | } 334 | return bytes.Compare(arena.getBytes(n.keyOffset, n.keySize), k) 335 | } 336 | -------------------------------------------------------------------------------- /diskqueue/qfile.go: -------------------------------------------------------------------------------- 1 | package diskqueue 2 | 3 | import ( 4 | "context" 5 | "encoding/binary" 6 | "fmt" 7 | "net" 8 | "os" 9 | "path/filepath" 10 | "sync" 11 | "sync/atomic" 12 | 13 | "github.com/zhiqiangxu/util/logger" 14 | "github.com/zhiqiangxu/util/mapped" 15 | "go.uber.org/zap" 16 | ) 17 | 18 | type refCountInterface interface { 19 | IncrRef() int32 20 | DecrRef() int32 21 | } 22 | type qfileInterface interface { 23 | refCountInterface 24 | Shrink() error 25 | writeBuffers(buffs *net.Buffers) (int64, error) 26 | WrotePosition() int64 27 | DoneWrite() int64 28 | Commit() int64 29 | Read(ctx context.Context, offset int64) ([]byte, error) 30 | StreamRead(ctx context.Context, offset int64, ch chan<- StreamBytes) (bool, error) 31 | StreamOffsetRead(ctx context.Context, offset int64, offsetCh <-chan int64, ch chan<- StreamBytes) (bool, int64, error) 32 | Sync() error 33 | Close() error 34 | } 35 | 36 | // qfile has no write-write races, but has read-write races 37 | type qfile struct { 38 | ref int32 39 | q *Queue 40 | idx int 41 | startOffset int64 42 | mappedFile *mapped.File 43 | notLatest bool 44 | readLockedFunc func(ctx context.Context, r *QfileSizeReader) (otherFile bool, startOffset int64, dataBytes []byte, err error) 45 | } 46 | 47 | const ( 48 | qfSubDir = "qf" 49 | qfileDefaultSize = 1024 * 1024 * 1024 50 | ) 51 | 52 | var _ qfileInterface = (*qfile)(nil) 53 | 54 | func qfilePath(startOffset int64, conf *Conf) string { 55 | return filepath.Join(conf.Directory, qfSubDir, fmt.Sprintf("%020d", startOffset)) 56 | } 57 | 58 | func openQfile(q *Queue, idx int, isLatest bool) (qf *qfile, err error) { 59 | fm := q.meta.FileMeta(idx) 60 | 61 | qf = &qfile{q: q, idx: idx, startOffset: fm.StartOffset, ref: 1} 62 | var pool *sync.Pool 63 | if isLatest { 64 | pool = q.writeBufferPool() 65 | } 66 | qf.mappedFile, err = mapped.OpenFile(qfilePath(fm.StartOffset, &q.conf), int64(fm.EndOffset-fm.StartOffset), os.O_RDWR, q.conf.WriteMmap, pool) 67 | if err != nil { 68 | return 69 | } 70 | qf.init() 71 | 72 | return 73 | } 74 | 75 | func createQfile(q *Queue, idx int, startOffset int64) (qf *qfile, err error) { 76 | qf = &qfile{q: q, idx: idx, startOffset: startOffset, ref: 1} 77 | var pool *sync.Pool 78 | if q.conf.EnableWriteBuffer { 79 | pool = q.writeBufferPool() 80 | } 81 | qf.mappedFile, err = mapped.CreateFile(qfilePath(startOffset, &q.conf), q.conf.MaxFileSize, q.conf.WriteMmap, pool) 82 | if err != nil { 83 | return 84 | } 85 | qf.init() 86 | 87 | if q.meta.NumFiles() != idx { 88 | logger.Instance().Fatal("createQfile idx != NumFiles", zap.Int("NumFiles", q.meta.NumFiles()), zap.Int("idx", idx)) 89 | } 90 | 91 | nowNano := NowNano() 92 | q.meta.AddFile(FileMeta{StartOffset: startOffset, EndOffset: startOffset, StartTime: nowNano, EndTime: nowNano}) 93 | 94 | return 95 | } 96 | 97 | func (qf *qfile) init() { 98 | if qf.q.conf.customDecoder { 99 | qf.readLockedFunc = qf.readLockedCustom 100 | } else { 101 | qf.readLockedFunc = qf.readLockedDefault 102 | } 103 | } 104 | 105 | func (qf *qfile) IncrRef() int32 { 106 | return atomic.AddInt32(&qf.ref, 1) 107 | } 108 | 109 | func (qf *qfile) DecrRef() (newRef int32) { 110 | newRef = atomic.AddInt32(&qf.ref, -1) 111 | if newRef > 0 { 112 | return 113 | } 114 | 115 | err := qf.Close() 116 | if err != nil { 117 | logger.Instance().Error("qf.Close", zap.Error(err)) 118 | } 119 | 120 | err = qf.remove() 121 | if err != nil { 122 | logger.Instance().Error("qf.remove", zap.Error(err)) 123 | } 124 | 125 | return 126 | } 127 | 128 | func (qf *qfile) writeBuffers(buffs *net.Buffers) (n int64, err error) { 129 | n, err = qf.mappedFile.WriteBuffers(buffs) 130 | return 131 | // n, err = buffs.WriteTo(qf.mappedFile) 132 | // return 133 | } 134 | 135 | func (qf *qfile) WrotePosition() int64 { 136 | return qf.startOffset + qf.mappedFile.GetWrotePosition() 137 | } 138 | 139 | func (qf *qfile) DoneWrite() int64 { 140 | return qf.startOffset + qf.mappedFile.DoneWrite() 141 | } 142 | 143 | func (qf *qfile) Commit() int64 { 144 | return qf.startOffset + qf.mappedFile.Commit() 145 | } 146 | 147 | // isLatest can be called concurrently :) 148 | func (qf *qfile) isLatest() bool { 149 | if qf.notLatest { 150 | return false 151 | } 152 | isLatest := qf.idx == qf.q.meta.NumFiles()-1 153 | if !isLatest { 154 | qf.notLatest = true 155 | } 156 | return isLatest 157 | } 158 | 159 | func (qf *qfile) readLockedCustom(ctx context.Context, r *QfileSizeReader) (otherFile bool, startOffset int64, dataBytes []byte, err error) { 160 | 161 | startOffset = r.NextOffset() 162 | otherFile, dataBytes, err = qf.q.conf.CustomDecoder(ctx, r) 163 | if err == mapped.ErrReadBeyond && !qf.isLatest() { 164 | otherFile = true 165 | } 166 | return 167 | } 168 | 169 | func (qf *qfile) readLockedDefault(ctx context.Context, r *QfileSizeReader) (otherFile bool, startOffset int64, dataBytes []byte, err error) { 170 | 171 | startOffset = r.NextOffset() 172 | var sizeBytes [sizeLength]byte 173 | err = r.Read(ctx, sizeBytes[:]) 174 | if err != nil { 175 | if err == mapped.ErrReadBeyond && !qf.isLatest() { 176 | otherFile = true 177 | } 178 | return 179 | } 180 | 181 | size := int(binary.BigEndian.Uint32(sizeBytes[:])) 182 | if size > qf.q.conf.MaxMsgSize { 183 | err = errInvalidOffset 184 | return 185 | } 186 | 187 | dataBytes = make([]byte, size) 188 | err = r.Read(ctx, dataBytes) 189 | return 190 | } 191 | 192 | func (qf *qfile) calcFileOffset(offset int64) (fileOffset int64, err error) { 193 | fileOffset = offset - qf.startOffset 194 | if fileOffset < 0 { 195 | logger.Instance().Error("calcFileOffset negative fileOffset", zap.Int64("offset", offset), zap.Int64("startOffset", qf.startOffset)) 196 | err = errInvalidOffset 197 | return 198 | } 199 | 200 | return 201 | } 202 | 203 | func (qf *qfile) Read(ctx context.Context, offset int64) (data []byte, err error) { 204 | fileOffset, err := qf.calcFileOffset(offset) 205 | if err != nil { 206 | return 207 | } 208 | 209 | qf.mappedFile.RLock() 210 | defer qf.mappedFile.RUnlock() 211 | 212 | r := qf.getSizeReader(fileOffset) 213 | _, _, data, err = qf.readLockedFunc(ctx, r) 214 | qf.putSizeReader(r) 215 | return 216 | } 217 | 218 | var sizeReaderPool = sync.Pool{ 219 | New: func() interface{} { 220 | return &QfileSizeReader{} 221 | }, 222 | } 223 | 224 | func (qf *qfile) getSizeReader(fileOffset int64) *QfileSizeReader { 225 | r := sizeReaderPool.Get().(*QfileSizeReader) 226 | r.qf = qf 227 | r.fileOffset = fileOffset 228 | r.isLatest = qf.isLatest() 229 | return r 230 | } 231 | 232 | func (qf *qfile) putSizeReader(r *QfileSizeReader) { 233 | r.qf = nil 234 | sizeReaderPool.Put(r) 235 | } 236 | 237 | // when StreamRead returns , err is guaranteed not nil 238 | func (qf *qfile) StreamRead(ctx context.Context, offset int64, ch chan<- StreamBytes) (otherFile bool, err error) { 239 | fileOffset, err := qf.calcFileOffset(offset) 240 | if err != nil { 241 | logger.Instance().Fatal("calcFileOffset err", zap.Int64("offset", offset), zap.Int64("startOffset", qf.startOffset)) 242 | return 243 | } 244 | 245 | qf.mappedFile.RLock() 246 | defer qf.mappedFile.RUnlock() 247 | 248 | r := qf.getSizeReader(fileOffset) 249 | defer qf.putSizeReader(r) 250 | 251 | var ( 252 | dataBytes []byte 253 | startOffset int64 254 | ) 255 | 256 | for { 257 | 258 | otherFile, startOffset, dataBytes, err = qf.readLockedFunc(ctx, r) 259 | if err != nil { 260 | return 261 | } 262 | 263 | select { 264 | case ch <- StreamBytes{Bytes: dataBytes, Offset: startOffset}: 265 | case <-ctx.Done(): 266 | err = ctx.Err() 267 | return 268 | } 269 | 270 | } 271 | 272 | } 273 | 274 | func (qf *qfile) StreamOffsetRead(ctx context.Context, offset int64, offsetCh <-chan int64, ch chan<- StreamBytes) (otherFile bool, lastOffset int64, err error) { 275 | fileOffset, err := qf.calcFileOffset(offset) 276 | if err != nil { 277 | logger.Instance().Fatal("calcFileOffset err", zap.Int64("offset", offset), zap.Int64("startOffset", qf.startOffset)) 278 | return 279 | } 280 | 281 | qf.mappedFile.RLock() 282 | defer qf.mappedFile.RUnlock() 283 | 284 | r := qf.getSizeReader(fileOffset) 285 | defer func() { 286 | lastOffset = r.fileOffset + qf.startOffset 287 | qf.putSizeReader(r) 288 | }() 289 | 290 | var ( 291 | dataBytes []byte 292 | nextOffset int64 293 | ok bool 294 | startOffset int64 295 | ) 296 | 297 | for { 298 | 299 | otherFile, startOffset, dataBytes, err = qf.readLockedFunc(ctx, r) 300 | if err != nil { 301 | return 302 | } 303 | 304 | select { 305 | case ch <- StreamBytes{Bytes: dataBytes, Offset: startOffset}: 306 | case <-ctx.Done(): 307 | err = ctx.Err() 308 | return 309 | } 310 | 311 | select { 312 | case nextOffset, ok = <-offsetCh: 313 | if !ok { 314 | err = errOffsetChClosed 315 | return 316 | } 317 | r.fileOffset = nextOffset - qf.startOffset 318 | if r.fileOffset < 0 { 319 | err = errInvalidOffset 320 | otherFile = true 321 | return 322 | } 323 | case <-ctx.Done(): 324 | err = ctx.Err() 325 | return 326 | } 327 | 328 | } 329 | } 330 | 331 | func (qf *qfile) Shrink() error { 332 | return qf.mappedFile.Shrink() 333 | } 334 | 335 | func (qf *qfile) Sync() error { 336 | return qf.mappedFile.Sync() 337 | } 338 | 339 | func (qf *qfile) Close() error { 340 | return qf.mappedFile.Close() 341 | } 342 | 343 | func (qf *qfile) remove() (err error) { 344 | err = qf.mappedFile.Remove() 345 | return 346 | } 347 | --------------------------------------------------------------------------------