├── natx ├── README.md ├── natx.go ├── msgpack_enc.go ├── go.mod ├── defaultApp_test.go ├── defaultApp.go ├── natc.go └── go.sum ├── redisx ├── go.mod ├── go.sum ├── redis_test.go ├── mutex.go └── redis.go ├── nsqx ├── go.mod ├── go.sum ├── producer.go └── consumer.go ├── httpx ├── go.mod ├── go.sum ├── README.md ├── sign_test.go └── sign.go ├── codec ├── README.md ├── codec.go └── codec_test.go ├── go.mod ├── check.sh ├── log ├── go.mod ├── zap_test.go ├── zap.go └── go.sum ├── pretty.go ├── sign ├── hmac_test.go └── hmac.go ├── etcdx ├── readme.md ├── v3 │ ├── lock_test.go │ ├── README.md │ ├── kvstore_test.go │ ├── client_test.go │ ├── go.mod │ ├── kvstore.go │ ├── lock.go │ ├── client.go │ └── go.sum ├── sync │ ├── go.mod │ ├── go.sum │ ├── sync_test.go │ └── sync.go ├── master │ ├── go.mod │ ├── go.sum │ └── master.go └── discovery │ ├── go.mod │ ├── worker.go │ ├── go.sum │ └── master.go ├── .gitignore ├── hash ├── ketama │ ├── ketama_test.go │ └── ketama.go ├── hash.go ├── murmurhash3 │ ├── mmhash3_test.go │ └── mmhash3.go └── cityhash │ └── cityhash.go ├── crypto ├── tripledes_b_test.go ├── descbc_test.go ├── tripledes_test.go ├── PKCS.go ├── descbc.go ├── aes256cbc.go ├── aes256cbc_test.go ├── aes.go └── tripledes.go ├── go.sum ├── timerx ├── wheel │ ├── timer_test.go │ └── timer.go └── minheap │ ├── timer_test.go │ └── timer.go ├── pprof ├── pprof_http_test.go └── pprof_http.go ├── random └── string.go ├── container ├── pqueue │ ├── README.md │ ├── priority_queue.go │ └── priority_queue_test.go ├── fifo │ ├── fifo.go │ └── fifo_test.go ├── blocking_queue │ ├── blocking_queue_test.go │ └── blocking_queue.go ├── bitset │ ├── bitset_test.go │ └── bitset.go ├── bloom │ ├── bloom9_test.go │ └── bloom9.go └── omap │ ├── omap_test.go │ └── omap.go ├── pool ├── xbytes │ ├── writer.go │ └── bytes.go ├── allocator │ ├── alloc.go │ └── alloc_test.go ├── xtime │ └── xtime.go └── xbufio │ └── buffio.go ├── LICENSE ├── referral ├── refferal_test.go └── referral.go ├── validator └── validator.go └── README.md /natx/README.md: -------------------------------------------------------------------------------- 1 | # natx 2 | 3 | 基于nat的request/reply 封装，每一个request/reply需要实现NatXAPP接口。默认使用msgp编码，其他默认参数可修改。 -------------------------------------------------------------------------------- /redisx/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/axengine/utils/redisx 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/garyburd/redigo v1.6.4 7 | github.com/pkg/errors v0.9.1 8 | ) 9 | -------------------------------------------------------------------------------- /nsqx/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/axengine/utils/nsqx 2 | 3 | go 1.20 4 | 5 | require github.com/nsqio/go-nsq v1.1.0 6 | 7 | require github.com/golang/snappy v0.0.1 // indirect 8 | -------------------------------------------------------------------------------- /nsqx/go.sum: -------------------------------------------------------------------------------- 1 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 2 | github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY= 3 | -------------------------------------------------------------------------------- /redisx/go.sum: -------------------------------------------------------------------------------- 1 | github.com/garyburd/redigo v1.6.4/go.mod h1:rTb6epsqigu3kYKBnaF028A7Tf/Aw5s0cqA47doKKqw= 2 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 3 | -------------------------------------------------------------------------------- /httpx/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/axengine/utils/httpx 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/axengine/utils v0.0.0-20230307013359-98101aacbccd 7 | github.com/shopspring/decimal v1.3.1 8 | ) 9 | -------------------------------------------------------------------------------- /codec/README.md: -------------------------------------------------------------------------------- 1 | # codec []byte编解码方法 2 | 3 | 使用基本的位运算对数据进行编解码，编码后的数据是同大小的[]byte。提供三种方式，可以自行选择和改造。 4 | 5 | codec提供了一种简单的思路对数据进行混淆，可以使用在app与server时间数据混淆，前提是app代码不被逆向和重放。 6 | 7 | - ENCODE_BIT_NOT：按位取反 8 | - ENCODE_BYTE_RVS：逆序 9 | - ENCODE_LOOP_XOR：环形异或 -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/axengine/utils 2 | 3 | go 1.25.0 4 | 5 | require ( 6 | github.com/pkg/errors v0.9.2-0.20201214064552-5dd12d0cfe7f 7 | golang.org/x/crypto v0.36.0 8 | ) 9 | 10 | require golang.org/x/sys v0.31.0 // indirect 11 | -------------------------------------------------------------------------------- /check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 查找所有.go文件 4 | for file in $(find . -name "*.go"); do 5 | # 获取文件所在目录的路径 6 | dir=$(dirname $file) 7 | # 切换到该目录 8 | echo $dir 9 | cd $dir 10 | # 执行 go build 命令 11 | go build 12 | # 返回上一级目录 13 | cd - 14 | done -------------------------------------------------------------------------------- /log/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/axengine/utils/log 2 | 3 | go 1.20 4 | 5 | require ( 6 | go.uber.org/zap v1.24.0 7 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 8 | ) 9 | 10 | require ( 11 | go.uber.org/atomic v1.7.0 // indirect 12 | go.uber.org/multierr v1.6.0 // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /natx/natx.go: -------------------------------------------------------------------------------- 1 | package natx 2 | 3 | import "context" 4 | 5 | type Handle func(subj string, req interface{}) interface{} 6 | 7 | type NatXAPP interface { 8 | Request(context.Context, string, interface{}) (interface{}, error) 9 | Subscribe(string, Handle) error 10 | QueueSubscribe(string, Handle) error 11 | Publish(string, interface{}) error 12 | } 13 | -------------------------------------------------------------------------------- /pretty.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | ) 7 | 8 | func JsonPretty(v interface{}) string { 9 | bz, _ := json.MarshalIndent(v, "", " ") 10 | return string(bz) 11 | } 12 | 13 | func JsonPrettyToStdout(v interface{}) { 14 | en := json.NewEncoder(os.Stdout) 15 | en.SetIndent("", " ") 16 | _ = en.Encode(v) 17 | } 18 | -------------------------------------------------------------------------------- /sign/hmac_test.go: -------------------------------------------------------------------------------- 1 | package sign 2 | 3 | import ( 4 | "crypto/hmac" 5 | "crypto/sha1" 6 | "encoding/base64" 7 | "fmt" 8 | "testing" 9 | ) 10 | 11 | func TestName(t *testing.T) { 12 | hmacHash := hmac.New(sha1.New, []byte("xxx")) 13 | hmacHash.Write([]byte("hello world")) 14 | fmt.Println(base64.StdEncoding.EncodeToString(hmacHash.Sum(nil))) 15 | } 16 | -------------------------------------------------------------------------------- /etcdx/readme.md: -------------------------------------------------------------------------------- 1 | # sync 2 | * sync Mutex 3 | * sync New的时候会连接ETCD，失败返回nil；LockBlocking时去set key，直到set成功或发生错误；UnLock时删除key； 4 | * LockBlocking是阻塞的，如果key一直没有expire或delete，会永远阻塞,因此set key时应正确设置ttl； 5 | * Lock是非阻塞的，如果已经加锁会立即返回失败 6 | * 多个任务去获取同一把锁，会先后获取到锁 7 | 8 | # Master 9 | * Master采用锁机制申请为主，默认为slave，直到成功或发生错误才返回; 10 | * 一但变为Master则一直为Master，知道程序coredump 11 | 12 | # discovery -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | 16 | # IDE 17 | .idea 18 | .vscode -------------------------------------------------------------------------------- /httpx/go.sum: -------------------------------------------------------------------------------- 1 | github.com/axengine/utils v0.0.0-20230307013359-98101aacbccd h1:2SaVyDqZGgbR7O5iRx5pgBKJjj0P1jI1kb5gAAC+xzk= 2 | github.com/axengine/utils v0.0.0-20230307013359-98101aacbccd/go.mod h1:PaloEMMrWUPnpGTnWkYp/r61lnJNjzpHp3OjUss6w3I= 3 | github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= 4 | github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= 5 | -------------------------------------------------------------------------------- /hash/ketama/ketama_test.go: -------------------------------------------------------------------------------- 1 | package ketama 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | ) 7 | 8 | func Benchmark_Hash(b *testing.B) { 9 | ring := NewRing(255) 10 | ring.AddNode("node1", 1) 11 | ring.AddNode("node2", 1) 12 | ring.AddNode("node3", 1) 13 | ring.AddNode("node4", 1) 14 | ring.AddNode("node5", 1) 15 | b.StartTimer() 16 | for i := 0; i < b.N; i++ { 17 | ring.Hash(strconv.Itoa(i)) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /etcdx/v3/lock_test.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | ) 7 | 8 | func TestLockUnlock(t *testing.T) { 9 | mutex := NewLock(_cli_, "/notary/txhash/") 10 | if err := mutex.Lock(context.Background(), 1); err != nil { 11 | t.Fatal(err) 12 | } 13 | t.Log("lock success") 14 | if err := mutex.Unlock(context.Background()); err != nil { 15 | t.Fatal(err) 16 | } 17 | t.Log("unlock success") 18 | } 19 | -------------------------------------------------------------------------------- /etcdx/sync/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/axengine/utils/etcdx/sync 2 | 3 | go 1.23.0 4 | 5 | require go.etcd.io/etcd/client/v2 v2.305.7 6 | 7 | require ( 8 | github.com/coreos/go-semver v0.3.0 // indirect 9 | github.com/json-iterator/go v1.1.11 // indirect 10 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect 11 | github.com/modern-go/reflect2 v1.0.1 // indirect 12 | go.etcd.io/etcd/api/v3 v3.5.7 // indirect 13 | go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect 14 | ) 15 | -------------------------------------------------------------------------------- /natx/msgpack_enc.go: -------------------------------------------------------------------------------- 1 | package natx 2 | 3 | import ( 4 | msgpack "github.com/vmihailenco/msgpack/v4" 5 | ) 6 | 7 | const ( 8 | MSGP_ENCODER = "msgp" 9 | ) 10 | 11 | type MsgpEncoder struct { 12 | // Empty 13 | } 14 | 15 | // Encode 16 | func (ge *MsgpEncoder) Encode(subject string, v interface{}) ([]byte, error) { 17 | return msgpack.Marshal(v) 18 | } 19 | 20 | // Decode 21 | func (ge *MsgpEncoder) Decode(subject string, data []byte, vPtr interface{}) (err error) { 22 | return msgpack.Unmarshal(data, vPtr) 23 | } 24 | -------------------------------------------------------------------------------- /crypto/tripledes_b_test.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "encoding/base64" 5 | "testing" 6 | ) 7 | 8 | func Benchmark_TripleDesECBEncrypt(b *testing.B) { 9 | for i := 0; i < b.N; i++ { 10 | TripleDesECBEncrypt([]byte("Hello World!"), "www.huhutong.com") 11 | } 12 | } 13 | 14 | func Benchmark_TripleDesECBDecrypt(b *testing.B) { 15 | for i := 0; i < b.N; i++ { 16 | encryped, _ := base64.StdEncoding.DecodeString("ZXzboHCq6vON+5xjAgzb2Q==") 17 | TripleDesECBDecrypt(encryped, "www.huhutong.com") 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/pkg/errors v0.9.2-0.20201214064552-5dd12d0cfe7f h1:lJqhwddJVYAkyp72a4pwzMClI20xTwL7miDdm2W/KBM= 2 | github.com/pkg/errors v0.9.2-0.20201214064552-5dd12d0cfe7f/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 3 | golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= 4 | golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= 5 | golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= 6 | golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 7 | -------------------------------------------------------------------------------- /etcdx/master/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/axengine/utils/etcdx/master 2 | 3 | go 1.20 4 | 5 | require ( 6 | go.etcd.io/etcd/client/v2 v2.305.7 7 | golang.org/x/net v0.38.0 8 | ) 9 | 10 | require ( 11 | github.com/coreos/go-semver v0.3.0 // indirect 12 | github.com/json-iterator/go v1.1.11 // indirect 13 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect 14 | github.com/modern-go/reflect2 v1.0.1 // indirect 15 | go.etcd.io/etcd/api/v3 v3.5.7 // indirect 16 | go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /etcdx/discovery/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/axengine/utils/etcdx/discovery 2 | 3 | go 1.20 4 | 5 | require ( 6 | go.etcd.io/etcd/client/v2 v2.305.7 7 | golang.org/x/net v0.38.0 8 | ) 9 | 10 | require ( 11 | github.com/coreos/go-semver v0.3.0 // indirect 12 | github.com/json-iterator/go v1.1.11 // indirect 13 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect 14 | github.com/modern-go/reflect2 v1.0.1 // indirect 15 | go.etcd.io/etcd/api/v3 v3.5.7 // indirect 16 | go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /nsqx/producer.go: -------------------------------------------------------------------------------- 1 | package nsqx 2 | 3 | import ( 4 | "github.com/nsqio/go-nsq" 5 | ) 6 | 7 | const ( 8 | USER_AGENT = "go agent v0.1" 9 | ) 10 | 11 | type Producer struct { 12 | producer *nsq.Producer 13 | } 14 | 15 | func NewProducer(nsqdTcpAddr string) *Producer { 16 | var p Producer 17 | cfg := nsq.NewConfig() 18 | cfg.UserAgent = USER_AGENT 19 | producer, err := nsq.NewProducer(nsqdTcpAddr, cfg) 20 | if err != nil { 21 | panic(err) 22 | return &p 23 | } 24 | p.producer = producer 25 | return &p 26 | } 27 | 28 | func (p *Producer) Publish(topic string, msg []byte) error { 29 | return p.producer.Publish(topic, msg) 30 | } 31 | -------------------------------------------------------------------------------- /timerx/wheel/timer_test.go: -------------------------------------------------------------------------------- 1 | package timer 2 | 3 | import ( 4 | "fmt" 5 | // "sync/atomic" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | //var tt *Timer 11 | 12 | func fn(t int, arg interface{}) { 13 | fmt.Println(time.Now().Format("2006-01-02 15:04:05")) 14 | fmt.Println("type:", t, " arg:", arg) 15 | } 16 | 17 | func TestTimer(t *testing.T) { 18 | timer := New(time.Millisecond * 10) 19 | // tt = timer 20 | //fmt.Println(timer) 21 | timer.NewTimer(time.Millisecond*time.Duration(1000*1), fn, 14, 15) 22 | go timer.Start() 23 | T.NewTimer(time.Millisecond*time.Duration(3000*1), fn, 12, 13) 24 | time.Sleep(time.Second * 100) 25 | 26 | } 27 | -------------------------------------------------------------------------------- /crypto/descbc_test.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "bytes" 5 | "encoding/hex" 6 | "fmt" 7 | "testing" 8 | ) 9 | 10 | func TestDESEnDeCrypt(t *testing.T) { 11 | key := []byte("12345678") 12 | plaintext := []byte("hello world!") 13 | 14 | ciphertext, err := DESCBCPCSK5Encrypt(key, plaintext) 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | fmt.Println(hex.EncodeToString(ciphertext)) 19 | 20 | plaintextnew, err := DESCBCPCSK5Decrypt(key, ciphertext) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | if !bytes.Equal(plaintextnew, plaintext) { 25 | t.Fatal("encrypt or decrypt failed") 26 | } 27 | fmt.Println(string(plaintextnew)) 28 | } 29 | -------------------------------------------------------------------------------- /log/zap_test.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | "testing" 6 | ) 7 | 8 | // type 9 | 10 | func Test_logLevel(t *testing.T) { 11 | Logger.Debug("debug", zap.String("k", "v")) 12 | Logger.Info("info", zap.String("k", "v")) 13 | Logger.Warn("warn", zap.String("k", "v")) 14 | Logger.Error("error", zap.String("k", "v")) 15 | } 16 | 17 | func Benchmark_logs(b *testing.B) { 18 | for i := 0; i < b.N; i++ { 19 | Logger.Debug("debug", zap.Int("i", i)) 20 | if i%10 == 0 { 21 | Logger.Info("info", zap.Int("i", i)) 22 | } 23 | if i%100 == 0 { 24 | Logger.Warn("warn", zap.Int("i", i)) 25 | } 26 | if i%1000 == 0 { 27 | Logger.Error("error", zap.Int("i", i)) 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /natx/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/axengine/utils/natx 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/nats-io/nats.go v1.36.0 9 | github.com/vmihailenco/msgpack/v4 v4.3.12 10 | ) 11 | 12 | require ( 13 | github.com/golang/protobuf v1.5.0 // indirect 14 | github.com/klauspost/compress v1.18.0 // indirect 15 | github.com/nats-io/nkeys v0.4.10 // indirect 16 | github.com/nats-io/nuid v1.0.1 // indirect 17 | github.com/vmihailenco/tagparser v0.1.1 // indirect 18 | golang.org/x/crypto v0.36.0 // indirect 19 | golang.org/x/net v0.38.0 // indirect 20 | golang.org/x/sys v0.31.0 // indirect 21 | google.golang.org/appengine v1.6.5 // indirect 22 | google.golang.org/protobuf v1.36.5 // indirect 23 | ) 24 | -------------------------------------------------------------------------------- /etcdx/v3/README.md: -------------------------------------------------------------------------------- 1 | # etcd permission configuration 2 | 3 | etcdctl --cert="./certs/client.pem" --key="./certs/client-key.pem" --cacert="./certs/ca.pem" --endpoints=https://192.168.1.51:2379,https://192.168.1.53:2379,https://192.168.1.105:2379 user add root 4 | user add root 5 | # 000000 6 | user grant-role root root 7 | 8 | Add a role 9 | role add role-bridge 10 | role grant-permission role-bridge readwrite /yala/bridge/ --prefix=true 11 | 12 | Add users 13 | user add notary0 #000000 14 | user add notary1 15 | user add notary2 16 | 17 | Character bindings 18 | user grant-role notary0 role-bridge 19 | user grant-role notary1 role-bridge 20 | user grant-role notary2 role-bridge 21 | 22 | Enable username and password verification 23 | auth enable -------------------------------------------------------------------------------- /pprof/pprof_http_test.go: -------------------------------------------------------------------------------- 1 | package pprof 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/signal" 7 | "sync" 8 | "syscall" 9 | "testing" 10 | ) 11 | 12 | func TestMustStart(t *testing.T) { 13 | MustStart(":6060") 14 | select {} 15 | } 16 | 17 | func TestStart(t *testing.T) { 18 | if err := Start(":6060"); err != nil { 19 | t.Fatal(err) 20 | } 21 | select {} 22 | } 23 | 24 | func TestStartAsync(t *testing.T) { 25 | ctx, cancel := context.WithCancel(context.Background()) 26 | 27 | var wg sync.WaitGroup 28 | 29 | wg.Add(1) 30 | go StartAsync(ctx, &wg, ":6060") 31 | 32 | sigChan := make(chan os.Signal, 1) 33 | signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) 34 | <-sigChan 35 | 36 | cancel() 37 | wg.Wait() 38 | } 39 | -------------------------------------------------------------------------------- /crypto/tripledes_test.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "encoding/base64" 5 | "testing" 6 | ) 7 | 8 | func Test_TripleDesECBEncrypt(t *testing.T) { 9 | out, err := TripleDesECBEncrypt([]byte("Hello World!"), "www.huhutong.com") 10 | if err != nil { 11 | t.Error(err) 12 | } 13 | dst := base64.StdEncoding.EncodeToString(out) 14 | if dst != "ZXzboHCq6vON+5xjAgzb2Q==" { 15 | t.Error("加密失败") 16 | } 17 | } 18 | 19 | func Test_TripleDesECBDecrypt(t *testing.T) { 20 | encryped, _ := base64.StdEncoding.DecodeString("ZXzboHCq6vON+5xjAgzb2Q==") 21 | out, err := TripleDesECBDecrypt(encryped, "www.huhutong.com") 22 | if err != nil { 23 | t.Error(err) 24 | } 25 | if string(out) != "Hello World!" { 26 | t.Error("解密失败") 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /hash/hash.go: -------------------------------------------------------------------------------- 1 | package hash 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/base64" 6 | "encoding/hex" 7 | "fmt" 8 | "golang.org/x/crypto/sha3" 9 | ) 10 | 11 | func Keccak256Hex(in []byte) string { 12 | sha := sha3.NewLegacyKeccak256() 13 | sha.Write(in) 14 | sum := sha.Sum(nil) 15 | return hex.EncodeToString(sum) 16 | } 17 | 18 | func Keccak256Base64(in []byte) string { 19 | sha := sha3.NewLegacyKeccak256() 20 | sha.Write(in) 21 | sum := sha.Sum(nil) 22 | return base64.StdEncoding.EncodeToString(sum) 23 | } 24 | 25 | func MD5Lower(b []byte) string { 26 | sha := md5.New() 27 | sha.Write(b) 28 | return fmt.Sprintf("%x", sha.Sum(nil)) 29 | } 30 | 31 | func MD5Upper(b []byte) string { 32 | sha := md5.New() 33 | sha.Write(b) 34 | return fmt.Sprintf("%X", sha.Sum(nil)) 35 | } 36 | -------------------------------------------------------------------------------- /random/string.go: -------------------------------------------------------------------------------- 1 | package random 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | func RandDigits(length int) string { 11 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 12 | codes := make([]string, 0) 13 | for i := 0; i < length; i++ { 14 | codes = append(codes, fmt.Sprintf("%d", r.Intn(10))) 15 | } 16 | return strings.Join(codes, "") 17 | } 18 | 19 | func RandAlphaAndDigits(length int, lower ...bool) string { 20 | str := "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 21 | bytes := []byte(str) 22 | result := make([]byte, length) 23 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 24 | for i := 0; i < length; i++ { 25 | result[i] = bytes[r.Intn(len(bytes))] 26 | } 27 | if len(lower) > 0 && lower[0] { 28 | return strings.ToLower(string(result)) 29 | } 30 | return string(result) 31 | } 32 | -------------------------------------------------------------------------------- /container/pqueue/README.md: -------------------------------------------------------------------------------- 1 | 非线程安全的优先队列（golang标准库中的heap是小根堆） 2 | 3 | ~~~ go 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | "github.com/gansidui/priority_queue" 9 | ) 10 | 11 | type Node struct { 12 | priority int 13 | value int 14 | } 15 | 16 | func (this *Node) Less(other interface{}) bool { 17 | return this.priority < other.(*Node).priority 18 | } 19 | 20 | func main() { 21 | q := priority_queue.New() 22 | 23 | q.Push(&Node{priority: 8, value: 1}) 24 | q.Push(&Node{priority: 7, value: 2}) 25 | q.Push(&Node{priority: 9, value: 3}) 26 | 27 | x := q.Top().(*Node) 28 | fmt.Println(x.priority, x.value) 29 | 30 | for q.Len() > 0 { 31 | x = q.Pop().(*Node) 32 | fmt.Println(x.priority, x.value) 33 | } 34 | 35 | // output: 36 | // 7 2 37 | 38 | // 7 2 39 | // 8 1 40 | // 9 3 41 | } 42 | 43 | ~~~ 44 | 45 | 46 | ##LICENSE 47 | 48 | MIT -------------------------------------------------------------------------------- /httpx/README.md: -------------------------------------------------------------------------------- 1 | # httpx 2 | 3 | ## APISign API参数签名验证 4 | - 1. rawStr eg: GET http://example.com/hello?n=1&a=2 Key["n","a"]-ASC Sort["a","n"] GetParam(a) a=2&n=1 param key string attaches the methods 5 | - 2. other request http method，for Content-Type: application/json {"n":"m","a":2} Key ASC Sort,param key string attaches the methods => {"a":2,"n":"m"} => a=2&n=m 6 | - 3. rawStr+timestamp => a=2&n=m1626167650 (1626167650 is unix timestamp), verify sign time valid（default 10s） 7 | - 4. Sign Method： Method(rawStr+timestamp, secretKey) signed text encode [Base64, Hex(default)] 8 | // Method=[HMAC-SHA256,HMAC-SHA1] Encode=[Base64,Hex] Default = HMAC-SHA256-HEX 9 | - 5. default: signStr=Hex(HMAC-SHA256(rawStr+timestamp,secretKey)) 10 | - 6. Sign http request Header X-Signature=accessKey:signStr:timestamp (: split elem) 11 | 12 | ### USEAGE 13 | see test example -------------------------------------------------------------------------------- /pool/xbytes/writer.go: -------------------------------------------------------------------------------- 1 | package xbytes 2 | 3 | type Writer struct { 4 | n int 5 | buf []byte 6 | } 7 | 8 | func NewWriterSize(n int) *Writer { 9 | return &Writer{buf: make([]byte, n)} 10 | } 11 | 12 | func (w *Writer) Size() int { 13 | return len(w.buf) 14 | } 15 | 16 | func (w *Writer) Reset() { 17 | w.n = 0 18 | } 19 | 20 | func (w *Writer) Buffer() []byte { 21 | return w.buf[:w.n] 22 | } 23 | 24 | func (w *Writer) Peek(n int) []byte { 25 | var buf []byte 26 | w.grow(n) 27 | buf = w.buf[w.n : w.n+n] 28 | w.n += n 29 | return buf 30 | } 31 | 32 | func (w *Writer) Write(p []byte) { 33 | w.grow(len(p)) 34 | w.n += copy(w.buf[w.n:], p) 35 | } 36 | 37 | func (w *Writer) grow(n int) { 38 | var buf []byte 39 | if w.n+n < len(w.buf) { 40 | return 41 | } 42 | buf = make([]byte, 2*len(w.buf)+n) 43 | copy(buf, w.buf[:w.n]) 44 | w.buf = buf 45 | return 46 | } 47 | -------------------------------------------------------------------------------- /etcdx/v3/kvstore_test.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestGetSet(t *testing.T) { 10 | kvstore := NewKvStore(_cli_) 11 | if err := kvstore.Set(context.Background(), "key", "value", 3); err != nil { 12 | t.Fatal(err) 13 | } 14 | 15 | if value, err := kvstore.Get(context.Background(), "key"); err != nil { 16 | t.Fatal(err) 17 | } else if value != "value" { 18 | t.Fatal("not equal") 19 | } 20 | } 21 | 22 | func TestSetExpire(t *testing.T) { 23 | kvstore := NewKvStore(_cli_) 24 | if err := kvstore.Set(context.Background(), "key", "value", 3); err != nil { 25 | t.Fatal(err) 26 | } 27 | 28 | time.Sleep(time.Second * 4) 29 | 30 | if value, err := kvstore.Get(context.Background(), "key"); err != nil { 31 | t.Fatal(err) 32 | } else if value == "value" { 33 | t.Fatal("not expire:", value) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /crypto/PKCS.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import "bytes" 4 | 5 | /* 6 | https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.1.0/com.ibm.zos.v2r1.csfb400/pkcspad.htm 7 | Value of clear text length (mod 16) Number of padding bytes added Value of each padding byte 8 | 0 16 0x10 9 | 1 15 0x0F 10 | 2 14 0x0E 11 | 3 13 0x0D 12 | 4 12 0x0C 13 | 5 11 0x0B 14 | 6 10 0x0A 15 | 7 9 0x09 16 | 8 8 0x08 17 | 9 7 0x07 18 | 10 6 0x06 19 | 11 5 0x05 20 | 12 4 0x04 21 | 13 3 0x03 22 | 14 2 0x02 23 | 15 1 0x01 24 | */ 25 | 26 | func PKCSPadding(ciphertext []byte, blockSize int) []byte { 27 | padding := blockSize - len(ciphertext)%blockSize 28 | padtext := bytes.Repeat([]byte{byte(padding)}, padding) 29 | return append(ciphertext, padtext...) 30 | } 31 | 32 | func PKCSUnPadding(origData []byte) []byte { 33 | length := len(origData) 34 | unpadding := int(origData[length-1]) 35 | return origData[:(length - unpadding)] 36 | } 37 | -------------------------------------------------------------------------------- /container/fifo/fifo.go: -------------------------------------------------------------------------------- 1 | package fifo 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type FIFO struct { 8 | front int 9 | rear int 10 | size int 11 | mu sync.Mutex 12 | queue []interface{} 13 | } 14 | 15 | func New(size int) *FIFO { 16 | return &FIFO{ 17 | front: 0, 18 | rear: 0, 19 | size: size, 20 | queue: make([]interface{}, size), 21 | } 22 | } 23 | 24 | func (f *FIFO) Enqueue(i interface{}) bool { 25 | f.mu.Lock() 26 | if (f.rear+1)%(f.size+1) == f.front { 27 | f.mu.Unlock() 28 | return false 29 | } 30 | 31 | f.queue[f.rear] = i 32 | f.rear = (f.rear + 1) % (f.size + 1) 33 | f.mu.Unlock() 34 | return true 35 | } 36 | 37 | func (f *FIFO) Dequeue() interface{} { 38 | f.mu.Lock() 39 | if f.rear == f.front { 40 | f.mu.Unlock() 41 | return nil 42 | } 43 | x := f.queue[f.front] 44 | f.queue[f.front] = nil 45 | f.front = (f.front + 1) % (f.size + 1) 46 | f.mu.Unlock() 47 | return x 48 | } 49 | -------------------------------------------------------------------------------- /redisx/redis_test.go: -------------------------------------------------------------------------------- 1 | package redisx 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | var rds *Rds 8 | 9 | func init() { 10 | rds = New("192.168.8.145:6379", "12345678", 15) 11 | } 12 | 13 | func TestMutex_Lock(t *testing.T) { 14 | mu := rds.NewMutex("abc") 15 | ok, err := mu.Lock(3) 16 | if err != nil { 17 | t.Fatal(err) 18 | } 19 | 20 | if !ok { 21 | t.Fatal(ok) 22 | } 23 | defer mu.Unlock() 24 | 25 | { 26 | _, err := mu.Lock(3) 27 | if err == nil { 28 | t.Fatal("error lock") 29 | } 30 | 31 | } 32 | } 33 | 34 | func TestMutex_Set(t *testing.T) { 35 | err := rds.Set("abc", []byte("abvc")) 36 | if err != nil { 37 | t.Fatal(err) 38 | } 39 | 40 | } 41 | 42 | func TestMutex_SetNX(t *testing.T) { 43 | err := rds.SetNX("abc", []byte("abvc")) 44 | if err != nil { 45 | t.Fatal(err) 46 | } 47 | 48 | } 49 | 50 | func TestMutex_Get(t *testing.T) { 51 | v, err := rds.Get("abc") 52 | if err != nil { 53 | t.Fatal(err) 54 | } 55 | t.Log(string(v)) 56 | 57 | } 58 | -------------------------------------------------------------------------------- /redisx/mutex.go: -------------------------------------------------------------------------------- 1 | package redisx 2 | 3 | import ( 4 | "github.com/garyburd/redigo/redis" 5 | "github.com/pkg/errors" 6 | ) 7 | 8 | type Mutex struct { 9 | key string 10 | conn redis.Conn 11 | } 12 | 13 | // NewMutex 14 | // 返回互斥量Mutex 15 | func (p *Rds) NewMutex(key string) *Mutex { 16 | return &Mutex{ 17 | key: key, 18 | conn: p.pool.Get(), 19 | } 20 | } 21 | 22 | func (mu *Mutex) Key() string { 23 | return "lock_" + mu.key 24 | } 25 | 26 | func (mu *Mutex) Lock(ttl int64) (bool, error) { 27 | conn := mu.conn 28 | reply, err := redis.String(conn.Do("SET", mu.Key(), mu.Key(), "EX", ttl, "NX")) 29 | if err != nil { 30 | conn.Close() 31 | return false, errors.Wrap(err, "lock failed") 32 | } 33 | 34 | if reply != "OK" { 35 | conn.Close() 36 | return false, errors.New("lock failed " + reply) 37 | } 38 | return true, nil 39 | } 40 | 41 | func (mu *Mutex) Unlock() (err error) { 42 | conn := mu.conn 43 | _, err = conn.Do("del", mu.Key()) 44 | conn.Close() 45 | return 46 | } 47 | -------------------------------------------------------------------------------- /etcdx/v3/client_test.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | clientv3 "go.etcd.io/etcd/client/v3" 8 | ) 9 | 10 | var _cli_ *clientv3.Client 11 | 12 | func TestMain(m *testing.M) { 13 | cli, err := NewClient("./certs/ca.pem", "./certs/client.pem", "./certs/client-key.pem", 14 | "root", "000000", []string{"https://192.168.1.51:2379", "https://192.168.1.53:2379", "https://192.168.1.105:2379"}) 15 | if err != nil { 16 | panic(err) 17 | } 18 | _cli_ = cli 19 | os.Exit(m.Run()) 20 | } 21 | 22 | func TestNewClient(t *testing.T) { 23 | _, err := NewClient("./certs/ca.pem", "./certs/client.pem", "./certs/client-key.pem", 24 | "root", "000000", []string{"https://192.168.1.51:2379", "https://192.168.1.53:2379", "https://192.168.1.105:2379"}) 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | // Can be different from TLS and passwords 29 | _, err = NewClient("", "", "", "", "", []string{"http://127.0.0.1:2379"}) 30 | if err != nil { 31 | t.Fatal(err) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /crypto/descbc.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "crypto/cipher" 5 | "crypto/des" 6 | ) 7 | 8 | func DESCBCPCSK5Encrypt(key []byte, plaintext []byte) ([]byte, error) { 9 | block, err := des.NewCipher(key) 10 | if err != nil { 11 | return nil, err 12 | } 13 | plaintext = PKCSPadding(plaintext, block.BlockSize()) 14 | // 创建CBC模式的加密器 15 | iv := key // 使用密钥作为初始向量 16 | mode := cipher.NewCBCEncrypter(block, iv) 17 | ciphertext := make([]byte, len(plaintext)) 18 | mode.CryptBlocks(ciphertext, plaintext) 19 | return ciphertext, err 20 | } 21 | 22 | func DESCBCPCSK5Decrypt(key []byte, ciphertext []byte) ([]byte, error) { 23 | block, err := des.NewCipher(key) 24 | if err != nil { 25 | return nil, err 26 | } 27 | // 创建CBC模式的加密器 28 | iv := key // 使用密钥作为初始向量 29 | mode := cipher.NewCBCDecrypter(block, iv) 30 | // 解密数据 31 | plaintext := make([]byte, len(ciphertext)) 32 | mode.CryptBlocks(plaintext, ciphertext) 33 | // 去除填充数据 34 | plaintext = PKCSUnPadding(plaintext) 35 | return plaintext, err 36 | } 37 | -------------------------------------------------------------------------------- /etcdx/v3/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/axengine/utils/etcdx/v3 2 | 3 | go 1.23.4 4 | 5 | require go.etcd.io/etcd/client/v3 v3.5.18 6 | 7 | require ( 8 | github.com/coreos/go-semver v0.3.0 // indirect 9 | github.com/coreos/go-systemd/v22 v22.3.2 // indirect 10 | github.com/gogo/protobuf v1.3.2 // indirect 11 | github.com/golang/protobuf v1.5.4 // indirect 12 | go.etcd.io/etcd/api/v3 v3.5.18 // indirect 13 | go.etcd.io/etcd/client/pkg/v3 v3.5.18 // indirect 14 | go.uber.org/atomic v1.7.0 // indirect 15 | go.uber.org/multierr v1.6.0 // indirect 16 | go.uber.org/zap v1.17.0 // indirect 17 | golang.org/x/net v0.38.0 // indirect 18 | golang.org/x/sys v0.31.0 // indirect 19 | golang.org/x/text v0.23.0 // indirect 20 | google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect 21 | google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect 22 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect 23 | google.golang.org/grpc v1.59.0 // indirect 24 | google.golang.org/protobuf v1.33.0 // indirect 25 | ) 26 | -------------------------------------------------------------------------------- /etcdx/v3/kvstore.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | 6 | clientv3 "go.etcd.io/etcd/client/v3" 7 | ) 8 | 9 | type KvStore struct { 10 | cli *clientv3.Client 11 | 12 | lease clientv3.Lease 13 | } 14 | 15 | func NewKvStore(cli *clientv3.Client) *KvStore { 16 | return &KvStore{cli: cli, lease: clientv3.NewLease(cli)} 17 | } 18 | 19 | func (kv *KvStore) Set(ctx context.Context, key, value string, ttl int) error { 20 | leaseResp, err := kv.lease.Grant(ctx, int64(ttl)) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | // Use the lease id to put the key-value pair into etcd 26 | putResp, err := kv.cli.Put(ctx, key, value, clientv3.WithLease(leaseResp.ID)) 27 | if err != nil { 28 | return err 29 | } 30 | _ = putResp 31 | return nil 32 | } 33 | 34 | func (kv *KvStore) Get(ctx context.Context, key string) (string, error) { 35 | // Check if the key-value pair still exists 36 | getResp, err := kv.cli.Get(ctx, key) 37 | if err != nil { 38 | return "", err 39 | } 40 | if len(getResp.Kvs) == 0 { 41 | return "", nil 42 | } 43 | return string(getResp.Kvs[0].Value), nil 44 | } 45 | -------------------------------------------------------------------------------- /crypto/aes256cbc.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "crypto/aes" 5 | "crypto/cipher" 6 | "errors" 7 | ) 8 | 9 | func AES256CBCPKCS0Encrypt(src []byte, iv []byte, key []byte) ([]byte, error) { 10 | block, err := aes.NewCipher(key) 11 | if err != nil { 12 | return nil, err 13 | } 14 | src = PKCSPadding(src, block.BlockSize()) 15 | srcLen := len(src) 16 | if srcLen%block.BlockSize() != 0 { 17 | return nil, errors.New("Need a multiple of the blocksize") 18 | } 19 | dst := make([]byte, srcLen) 20 | mode := cipher.NewCBCEncrypter(block, iv) 21 | mode.CryptBlocks(dst, src) 22 | return dst, nil 23 | } 24 | 25 | func AES256CBCPKCS0Decrypt(src []byte, iv []byte, key []byte) ([]byte, error) { 26 | block, err := aes.NewCipher(key) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | srcLen := len(src) 32 | if srcLen%block.BlockSize() != 0 { 33 | return nil, errors.New("crypto/cipher: input not full blocks") 34 | } 35 | dst := make([]byte, srcLen) 36 | 37 | mode := cipher.NewCBCDecrypter(block, iv) 38 | 39 | mode.CryptBlocks(dst, src) 40 | dst = PKCSUnPadding(dst) 41 | return dst, nil 42 | } 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 axengine 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /natx/defaultApp_test.go: -------------------------------------------------------------------------------- 1 | package natx 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func Test_defaultApp_ReqReply(t *testing.T) { 11 | app := &DefaultAPP{ 12 | cli: NewNATClient([]string{"http://192.168.8.101:4222", "http://192.168.8.121:4222", "http://192.168.8.141:4222"}), 13 | } 14 | testTopic := "xyz" 15 | app.Subscribe(testTopic, app.defaultHandle) 16 | 17 | resp, err := app.Request(context.Background(), testTopic, &DefaultAPPReq{ 18 | Arg1: 456, 19 | Arg2: "alimmm", 20 | }) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | ret := resp.(*DefaultAPPResp) 25 | fmt.Println("客户端：", ret.Arg1, " ", ret.Arg2) 26 | } 27 | 28 | func Test_defaultApp_Publish(t *testing.T) { 29 | app := &DefaultAPP{ 30 | cli: NewNATClient([]string{"http://192.168.8.101:4222", "http://192.168.8.121:4222", "http://192.168.8.141:4222"}), 31 | } 32 | testTopic := "xyz" 33 | app.Subscribe(testTopic, app.defaultHandle) 34 | 35 | err := app.Publish(testTopic, &DefaultAPPReq{ 36 | Arg1: 456, 37 | Arg2: "alimmm", 38 | }) 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | time.Sleep(time.Second * 3) 43 | } 44 | -------------------------------------------------------------------------------- /referral/refferal_test.go: -------------------------------------------------------------------------------- 1 | package referral 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "testing" 7 | ) 8 | 9 | func init() { 10 | //SetBase("ETN6BGQF7P5IK3MUAR4HV8S2DJZX9WYL") 11 | //SetLength(8) 12 | } 13 | 14 | func TestEncode(t *testing.T) { 15 | for i := 1; i < 10000000; i++ { 16 | code := Encode(uint64(i)) 17 | num := Decode(code) 18 | if num != uint64(i) { 19 | t.Fatal(errors.New("error")) 20 | } 21 | if i%100 == 0 { 22 | fmt.Println(code, num) 23 | } 24 | } 25 | } 26 | 27 | func TestDecode(t *testing.T) { 28 | code := Encode(1) 29 | fmt.Println(code) 30 | id := Decode(code) 31 | fmt.Println(id) 32 | 33 | } 34 | 35 | func TestDecode2(t *testing.T) { 36 | code := "NIBCN6" 37 | id := Decode(code) 38 | fmt.Println(id) 39 | } 40 | 41 | func TestEncode2(t *testing.T) { 42 | id := 1 43 | code := Encode(uint64(id)) 44 | fmt.Println("code", code) 45 | } 46 | 47 | func TestDumpKey(t *testing.T) { 48 | keys := make(map[string]struct{}, 100000000) 49 | for i := 1; i < 1000000000; i++ { 50 | code := Encode(uint64(i)) 51 | _, ok := keys[code] 52 | if ok { 53 | t.Fatal("dump key") 54 | } 55 | keys[code] = struct{}{} 56 | if Decode(code) != uint64(i) { 57 | t.Fatal("decode exception") 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /timerx/minheap/timer_test.go: -------------------------------------------------------------------------------- 1 | package timer 2 | 3 | import ( 4 | "log" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestTimer(t *testing.T) { 10 | timer := NewTimer(100) 11 | tds := make([]*TimerData, 100) 12 | for i := 0; i < 100; i++ { 13 | tds[i] = timer.Add(time.Duration(i)*time.Second+5*time.Minute, nil, nil) 14 | } 15 | printTimer(timer) 16 | for i := 0; i < 100; i++ { 17 | log.Printf("td: %s, %s, %d", tds[i].Key, tds[i].ExpireString(), tds[i].index) 18 | timer.Del(tds[i]) 19 | } 20 | printTimer(timer) 21 | for i := 0; i < 100; i++ { 22 | tds[i] = timer.Add(time.Duration(i)*time.Second+5*time.Minute, nil, nil) 23 | } 24 | printTimer(timer) 25 | for i := 0; i < 100; i++ { 26 | timer.Del(tds[i]) 27 | } 28 | printTimer(timer) 29 | timer.Add(time.Second, someecho, "testarg") 30 | time.Sleep(time.Second * 2) 31 | if len(timer.timers) != 0 { 32 | t.FailNow() 33 | } 34 | } 35 | 36 | func someecho(arg interface{}) { 37 | log.Println(arg) 38 | } 39 | 40 | func printTimer(timer *Timer) { 41 | log.Printf("----------timers: %d ----------", len(timer.timers)) 42 | for i := 0; i < len(timer.timers); i++ { 43 | log.Printf("timer: %s, %s, index: %d", timer.timers[i].Key, timer.timers[i].ExpireString(), timer.timers[i].index) 44 | } 45 | log.Printf("--------------------") 46 | } 47 | -------------------------------------------------------------------------------- /crypto/aes256cbc_test.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "testing" 7 | ) 8 | 9 | var ( 10 | srcStr = `964432afc9c34739a06d6b877b7033b0217e96fa3e2db71896e708fd2bbc5a35` 11 | dstStr = `ns6Mi2GyaFJJ8Z3HiUwLs3kjShAqQepSyKRFzNv1FViXgOpccwFH6Gab1MSkyZ25OSoCWuQV1a5c+pHG/ZnJnCcoGqfPbkWeMcLpaLYhh0M=` 12 | ivStr = `ri34iHc5LzgiWAhw` 13 | keyStr = `dZgfIU0XsfRzUFbOVRI39LSytTXs4Mvs` 14 | ) 15 | 16 | func TestAES256_CBC_PKCS0Encrpt(t *testing.T) { 17 | src := []byte(srcStr) 18 | //iv, _ := base64.StdEncoding.DecodeString(ivStr) 19 | iv := []byte(ivStr) 20 | key := []byte(keyStr) 21 | dst, err := AES256CBCPKCS0Encrypt(src, iv, key) 22 | if err != nil { 23 | t.Error(err) 24 | } 25 | fmt.Println(base64.StdEncoding.EncodeToString(dst)) 26 | if base64.StdEncoding.EncodeToString(dst) != dstStr { 27 | //t.Error("error encrpt ", base64.StdEncoding.EncodeToString(dst), "\n", dst) 28 | } 29 | } 30 | 31 | func TestAES256_CBC_PKCS0Decrpt(t *testing.T) { 32 | src, _ := base64.StdEncoding.DecodeString(dstStr) 33 | iv, _ := base64.StdEncoding.DecodeString(ivStr) 34 | key := []byte(keyStr) 35 | dst, err := AES256CBCPKCS0Decrypt(src, iv, key) 36 | if err != nil { 37 | t.Error(err) 38 | } 39 | if srcStr != string(dst) { 40 | t.Error("error decrpt ", string(dst), "\n", dst) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /crypto/aes.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "crypto/aes" 5 | "crypto/cipher" 6 | "errors" 7 | ) 8 | 9 | // AESEncrypt AES加密 初始向量16字节空 PKCS5 CBC 10 | // 入参:src 待加密[]byte 11 | // key:密钥[]byte 16/24/32 12 | // 返回:加密后[]byte 13 | func AESEncrypt(src, key []byte) ([]byte, error) { 14 | block, err := aes.NewCipher(key) 15 | if err != nil { 16 | return nil, err 17 | } 18 | src = PKCSPadding(src, block.BlockSize()) 19 | srcLen := len(src) 20 | if srcLen%block.BlockSize() != 0 { 21 | return nil, errors.New("Need a multiple of the blocksize") 22 | } 23 | dst := make([]byte, srcLen) 24 | iv := make([]byte, 16) 25 | mode := cipher.NewCBCEncrypter(block, iv) 26 | mode.CryptBlocks(dst, src) 27 | return dst, nil 28 | } 29 | 30 | // AESDecrypt AES解密 初始向量16字节空 PKCS5 CBC 31 | // 入参:src 已加密[]byte 32 | // key:密钥[]byte 16/24/32 33 | // 返回:解密后[]byte 34 | func AESDecrypt(src, key []byte) ([]byte, error) { 35 | block, err := aes.NewCipher(key) 36 | if err != nil { 37 | return nil, err 38 | } 39 | srcLen := len(src) 40 | if srcLen%block.BlockSize() != 0 { 41 | return nil, errors.New("crypto/cipher: input not full blocks") 42 | } 43 | dst := make([]byte, srcLen) 44 | iv := make([]byte, 16) 45 | mode := cipher.NewCBCDecrypter(block, iv) 46 | mode.CryptBlocks(dst, src) 47 | return PKCSPadding(dst, block.BlockSize()), nil 48 | } 49 | -------------------------------------------------------------------------------- /etcdx/v3/lock.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | 6 | clientv3 "go.etcd.io/etcd/client/v3" 7 | "go.etcd.io/etcd/client/v3/concurrency" 8 | ) 9 | 10 | type Lock struct { 11 | cli *clientv3.Client 12 | key string 13 | 14 | session *concurrency.Session 15 | mutex *concurrency.Mutex 16 | } 17 | 18 | func NewLock(cli *clientv3.Client, key string) *Lock { 19 | return &Lock{cli: cli, key: key} 20 | } 21 | 22 | // Lock TTL is a lease, which will be automatically renewed, and the program must ensure that Unlock can be called after Lock, 23 | // otherwise other programs cannot obtain the lock, unless the program exits, the network is disconnected, etc., and the lease is not actively renewed 24 | func (l *Lock) Lock(ctx context.Context, ttl int) error { 25 | // Create a session for lock management 26 | session, err := concurrency.NewSession(l.cli, concurrency.WithTTL(ttl), concurrency.WithContext(ctx)) 27 | if err != nil { 28 | return err 29 | } 30 | l.session = session 31 | 32 | // Create a lock 33 | mutex := concurrency.NewMutex(session, l.key) 34 | if err := mutex.Lock(ctx); err != nil { 35 | return err 36 | } 37 | l.mutex = mutex 38 | return nil 39 | } 40 | 41 | func (l *Lock) Unlock(ctx context.Context) error { 42 | defer l.session.Close() 43 | if err := l.mutex.Unlock(ctx); err != nil { 44 | return err 45 | } 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /log/zap.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | "go.uber.org/zap/zapcore" 6 | "gopkg.in/natefinch/lumberjack.v2" 7 | ) 8 | 9 | type ( 10 | aboveDebug struct{} 11 | aboveWarn struct{} 12 | ) 13 | 14 | var Logger *zap.Logger 15 | 16 | func (l aboveDebug) Enabled(lv zapcore.Level) bool { 17 | return lv >= zapcore.DebugLevel 18 | } 19 | 20 | func (l aboveWarn) Enabled(lv zapcore.Level) bool { 21 | return lv >= zapcore.WarnLevel 22 | } 23 | 24 | // init 初始化默认Logger 25 | func init() { 26 | var encoder zapcore.Encoder 27 | encoder = zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()) 28 | wDebug := zapcore.AddSync(&lumberjack.Logger{ 29 | Filename: "./log/debug.log", 30 | MaxSize: 100, 31 | MaxBackups: 50, 32 | MaxAge: 365, 33 | LocalTime: true, 34 | Compress: true, 35 | }) 36 | 37 | wError := zapcore.AddSync(&lumberjack.Logger{ 38 | Filename: "./log/error.log", 39 | MaxSize: 100, 40 | MaxBackups: 10, 41 | MaxAge: 365, 42 | LocalTime: true, 43 | Compress: true, 44 | }) 45 | 46 | coreDebug := zapcore.NewCore( 47 | encoder, 48 | wDebug, 49 | aboveDebug{}, 50 | ) 51 | 52 | coreError := zapcore.NewCore( 53 | encoder, 54 | wError, 55 | aboveWarn{}, 56 | ) 57 | 58 | Logger = zap.New(zapcore.NewTee(coreDebug, coreError), zap.AddCaller(), zap.AddStacktrace(zap.DPanicLevel)) 59 | } 60 | -------------------------------------------------------------------------------- /container/blocking_queue/blocking_queue_test.go: -------------------------------------------------------------------------------- 1 | package blocking_queue 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func Test_Base(t *testing.T) { 10 | bqueue := New(0) 11 | for i := 0; i < 10; i++ { 12 | bqueue.PushBack(i) 13 | } 14 | 15 | for i := 0; i < 10; i++ { 16 | v := bqueue.PopFront() 17 | if i != v.(int) { 18 | t.Error("not equal") 19 | } 20 | } 21 | } 22 | 23 | func Benchmark_Put(b *testing.B) { 24 | al := New(0) 25 | for i := 0; i < b.N; i++ { 26 | al.PushBack(i) 27 | } 28 | 29 | for i := 0; i < b.N; i++ { 30 | al.PopFront() 31 | } 32 | } 33 | 34 | func Test_GetPut(t *testing.T) { 35 | al := New(0) 36 | go func() { 37 | v := al.PopFront() 38 | fmt.Println("1 got ", v.(int), " at ", time.Now()) 39 | }() 40 | go func() { 41 | v := al.PopFront() 42 | fmt.Println("2 got ", v.(int), " at ", time.Now()) 43 | }() 44 | time.Sleep(time.Second) 45 | 46 | al.PushBack(1) 47 | al.PushBack(2) 48 | time.Sleep(time.Second * 3) 49 | } 50 | 51 | func Test_PutGet(t *testing.T) { 52 | al := New(5) 53 | 54 | time.AfterFunc(time.Second*3, func() { 55 | for { 56 | v := al.PopFront() 57 | fmt.Println("got ", v.(int), " at ", time.Now()) 58 | } 59 | }) 60 | for i := 0; i < 10; i++ { 61 | al.PushBack(i) 62 | fmt.Println("put ", i, " at ", time.Now()) 63 | } 64 | 65 | time.Sleep(time.Second * 10) 66 | } 67 | -------------------------------------------------------------------------------- /etcdx/v3/client.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "os" 7 | "time" 8 | 9 | clientv3 "go.etcd.io/etcd/client/v3" 10 | ) 11 | 12 | func NewClient(cacertFile, certFile, keyFile string, username, password string, endpoints []string) (*clientv3.Client, error) { 13 | var tlsConfig *tls.Config 14 | if cacertFile != "" && certFile != "" && keyFile != "" { 15 | caCert, err := os.ReadFile(cacertFile) 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | clientCert, err := os.ReadFile(certFile) 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | clientKey, err := os.ReadFile(keyFile) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | // Create a certificate pool and add CA certificates 31 | caCertPool := x509.NewCertPool() 32 | caCertPool.AppendCertsFromPEM(caCert) 33 | 34 | cert, err := tls.X509KeyPair(clientCert, clientKey) 35 | if err != nil { 36 | return nil, err 37 | } 38 | // Create a TLS configuration 39 | tlsConfig = &tls.Config{ 40 | RootCAs: caCertPool, 41 | Certificates: []tls.Certificate{ 42 | cert, 43 | }, 44 | } 45 | } 46 | 47 | cli, err := clientv3.New(clientv3.Config{ 48 | Endpoints: endpoints, 49 | DialTimeout: 5 * time.Second, 50 | TLS: tlsConfig, 51 | Username: username, 52 | Password: password, 53 | }) 54 | if err != nil { 55 | return nil, err 56 | } 57 | return cli, nil 58 | } 59 | -------------------------------------------------------------------------------- /container/bitset/bitset_test.go: -------------------------------------------------------------------------------- 1 | package bitset 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestBitSet_Add(t *testing.T) { 10 | bitset := NewBitSet() 11 | var key uint32 = 123851235 12 | bitset.Add(key) 13 | if !bitset.IsExist(key) { 14 | t.Error("key can not found in the bitset") 15 | } 16 | bitset.Del(key) 17 | if bitset.IsExist(key) { 18 | t.Error("key should not found in the bitset") 19 | } 20 | 21 | key = 0 22 | bitset.Add(key) 23 | if !bitset.IsExist(0) { 24 | t.Error("key can not found in the bitset") 25 | } 26 | } 27 | func BenchmarkBitSet_Add(b *testing.B) { 28 | bitset := NewBitSet() 29 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 30 | for i := 0; i < b.N; i++ { 31 | key := uint32(r.Int31()) 32 | bitset.Add(key) 33 | ok := bitset.IsExist(key) 34 | if !ok { 35 | b.Errorf("第%9d计算,出现数据不存在,key=%9d \n", i, key) 36 | break 37 | } 38 | 39 | } 40 | 41 | } 42 | func BenchmarkBitSet_IsExist(b *testing.B) { 43 | bitset := NewBitSet() 44 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 45 | var max uint32 = 1024 * 1024 * 1024 46 | for i := uint32(0); i < max; i++ { 47 | key := uint32(i) 48 | bitset.Add(key) 49 | } 50 | b.ResetTimer() 51 | for i := 0; i < b.N; i++ { 52 | key := uint32(r.Int31()) 53 | ok := bitset.IsExist(key) 54 | if (ok && key >= max) || (!ok && key < max) { 55 | b.Errorf("第%9d运算,发现存放的key:%9d不符合逻辑,max:%9d,ok=%v \n", i, key, max, ok) 56 | break 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /etcdx/discovery/worker.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "log" 7 | "runtime" 8 | "time" 9 | 10 | "go.etcd.io/etcd/client/v2" 11 | ) 12 | 13 | type Worker struct { 14 | Name string 15 | IP string 16 | KeysAPI client.KeysAPI 17 | } 18 | 19 | // workerInfo is the service register information to etcdx 20 | type WorkerInfo struct { 21 | Name string 22 | IP string 23 | CPU int 24 | } 25 | 26 | func NewWorker(name, IP string, endpoints []string) *Worker { 27 | cfg := client.Config{ 28 | Endpoints: endpoints, 29 | Transport: client.DefaultTransport, 30 | HeaderTimeoutPerRequest: time.Second, 31 | } 32 | 33 | etcdClient, err := client.New(cfg) 34 | if err != nil { 35 | log.Fatal("Error: cannot connec to etcdx:", err) 36 | } 37 | 38 | w := &Worker{ 39 | Name: name, 40 | IP: IP, 41 | KeysAPI: client.NewKeysAPI(etcdClient), 42 | } 43 | return w 44 | } 45 | 46 | func (w *Worker) HeartBeat() { 47 | api := w.KeysAPI 48 | 49 | for { 50 | info := &WorkerInfo{ 51 | Name: w.Name, 52 | IP: w.IP, 53 | CPU: runtime.NumCPU(), 54 | } 55 | 56 | key := "workers/" + w.Name 57 | value, _ := json.Marshal(info) 58 | 59 | _, err := api.Set(context.Background(), key, string(value), &client.SetOptions{ 60 | TTL: time.Second * 10, 61 | }) 62 | if err != nil { 63 | log.Println("Error update workerInfo:", err) 64 | } 65 | time.Sleep(time.Second * 3) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /pprof/pprof_http.go: -------------------------------------------------------------------------------- 1 | package pprof 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "net/http" 7 | _ "net/http/pprof" 8 | "sync" 9 | "time" 10 | 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | func MustStart(listen string) { 15 | go func() { 16 | if err := http.ListenAndServe(listen, nil); err != nil { 17 | panic(errors.Wrap(err, "pprof bind "+listen)) 18 | } 19 | }() 20 | } 21 | 22 | func Start(listen string) error { 23 | var ch chan error 24 | go func() { 25 | if err := http.ListenAndServe(listen, nil); err != nil { 26 | ch <- errors.Wrap(err, "pprof bind "+listen) 27 | return 28 | } 29 | ch <- nil 30 | }() 31 | 32 | select { 33 | case err := <-ch: 34 | return err 35 | case <-time.After(time.Second): 36 | return nil 37 | } 38 | } 39 | 40 | // StartAsync run pprof http server async 41 | // usage:go Start(ctx,wg,listen) 42 | func StartAsync(ctx context.Context, wg *sync.WaitGroup, listen string) { 43 | server := &http.Server{ 44 | Addr: listen, 45 | Handler: http.DefaultServeMux, 46 | } 47 | wg.Go(func() { 48 | if err := server.ListenAndServe(); err != nil { 49 | log.Println("start pprof server failed", err) 50 | } 51 | }) 52 | 53 | <-ctx.Done() 54 | 55 | shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) 56 | defer shutdownCancel() 57 | if err := server.Shutdown(shutdownCtx); err != nil { 58 | log.Println("shutdown pprof server failed", err) 59 | } 60 | log.Println("pprof server stopped") 61 | } 62 | -------------------------------------------------------------------------------- /codec/codec.go: -------------------------------------------------------------------------------- 1 | package codec 2 | 3 | const ( 4 | ENCODE_DEFAULT = iota 5 | ENCODE_BIT_NOT 6 | ENCODE_BYTE_RVS 7 | ENCODE_LOOP_XOR 8 | ) 9 | 10 | func Encode(tp int, buff []byte) { 11 | switch tp { 12 | case ENCODE_BIT_NOT: 13 | { 14 | for i := 0; i < len(buff); i++ { 15 | buff[i] = ^buff[i] 16 | } 17 | } 18 | case ENCODE_BYTE_RVS: 19 | { 20 | length := len(buff) 21 | if len(buff)%2 != 0 { 22 | length -= 1 23 | } 24 | for i := 0; i < length; i += 2 { 25 | buff[i], buff[i+1] = buff[i+1], buff[i] 26 | } 27 | } 28 | case ENCODE_LOOP_XOR: 29 | { 30 | length := len(buff) 31 | if length <= 1 { 32 | break 33 | } 34 | for i := 0; i < length-1; i++ { 35 | buff[i+1] ^= buff[i] 36 | } 37 | buff[0] ^= buff[length-1] 38 | } 39 | case ENCODE_DEFAULT: 40 | default: 41 | 42 | } 43 | } 44 | 45 | func Decode(tp int, buff []byte) { 46 | switch tp { 47 | case ENCODE_BIT_NOT: 48 | { 49 | for i := 0; i < len(buff); i++ { 50 | buff[i] = ^buff[i] 51 | } 52 | } 53 | case ENCODE_BYTE_RVS: 54 | { 55 | length := len(buff) 56 | if len(buff)%2 != 0 { 57 | length -= 1 58 | } 59 | for i := 0; i < length; i += 2 { 60 | buff[i], buff[i+1] = buff[i+1], buff[i] 61 | } 62 | } 63 | case ENCODE_LOOP_XOR: 64 | { 65 | length := len(buff) 66 | if length <= 1 { 67 | break 68 | } 69 | buff[0] ^= buff[length-1] 70 | for i := length - 1; i > 0; i-- { 71 | buff[i] ^= buff[i-1] 72 | } 73 | } 74 | case ENCODE_DEFAULT: 75 | default: 76 | 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /sign/hmac.go: -------------------------------------------------------------------------------- 1 | package sign 2 | 3 | import ( 4 | "crypto/hmac" 5 | "crypto/sha1" 6 | "crypto/sha256" 7 | "crypto/sha512" 8 | "encoding/base64" 9 | "encoding/hex" 10 | ) 11 | 12 | // HMACSha1B64 HMAC&SHA1->base64 13 | func HMACSha1B64(raw []byte, key []byte) string { 14 | hmacHash := hmac.New(sha1.New, key) 15 | hmacHash.Write(raw) 16 | return base64.StdEncoding.EncodeToString(hmacHash.Sum(nil)) 17 | } 18 | 19 | // HMACSha256B64 HMAC&SHA256->base64 20 | func HMACSha256B64(raw []byte, key []byte) string { 21 | hmacHash := hmac.New(sha256.New, key) 22 | hmacHash.Write(raw) 23 | return base64.StdEncoding.EncodeToString(hmacHash.Sum(nil)) 24 | } 25 | 26 | // HMACSha512B64 HMAC&SHA512->base64 27 | func HMACSha512B64(raw []byte, key []byte) string { 28 | hmacHash := hmac.New(sha512.New, key) 29 | hmacHash.Write(raw) 30 | return base64.StdEncoding.EncodeToString(hmacHash.Sum(nil)) 31 | } 32 | 33 | // HMACSha1Hex HMAC&SHA1->hex 34 | func HMACSha1Hex(raw []byte, key []byte) string { 35 | hmacHash := hmac.New(sha1.New, key) 36 | hmacHash.Write(raw) 37 | return hex.EncodeToString(hmacHash.Sum(nil)) 38 | } 39 | 40 | // HMACSha256Hex HMAC&SHA256->hex 41 | func HMACSha256Hex(raw []byte, key []byte) string { 42 | hmacHash := hmac.New(sha256.New, key) 43 | hmacHash.Write(raw) 44 | return hex.EncodeToString(hmacHash.Sum(nil)) 45 | } 46 | 47 | // HMACSha512Hex HMAC&SHA512->hex 48 | func HMACSha512Hex(raw []byte, key []byte) string { 49 | hmacHash := hmac.New(sha512.New, key) 50 | hmacHash.Write(raw) 51 | return hex.EncodeToString(hmacHash.Sum(nil)) 52 | } 53 | -------------------------------------------------------------------------------- /container/fifo/fifo_test.go: -------------------------------------------------------------------------------- 1 | package fifo 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestFIFO_Enqueue(t *testing.T) { 9 | f := New(3) 10 | for i := 0; i < 3; i++ { 11 | b := f.Enqueue(i) 12 | if !b { 13 | t.Fatal("enqueue false i:", i) 14 | } 15 | fmt.Println("h:", f.front) 16 | fmt.Println("t:", f.rear) 17 | fmt.Println("d:", f.queue) 18 | } 19 | 20 | b := f.Enqueue(1) 21 | if b { 22 | t.Fatal("not full check:") 23 | } 24 | fmt.Println(b) 25 | } 26 | 27 | func TestFIFO_Dequeue(t *testing.T) { 28 | f := New(3) 29 | for i := 0; i < 3; i++ { 30 | b := f.Enqueue(i) 31 | if !b { 32 | t.Fatal("enqueue false i:", i) 33 | } 34 | } 35 | fmt.Println("h:", f.front) 36 | fmt.Println("t:", f.rear) 37 | fmt.Println("d:", f.queue) 38 | for i := 0; i < 3; i++ { 39 | x := f.Dequeue() 40 | if x == nil { 41 | t.Fatal("dequeue failed") 42 | } 43 | fmt.Println(x) 44 | } 45 | x := f.Dequeue() 46 | if x != nil { 47 | t.Fatal("dequeue failed") 48 | } 49 | fmt.Println(x) 50 | } 51 | 52 | func BenchmarkFIFO_Put(b *testing.B) { 53 | var max = int(10000000) 54 | f := New(max) 55 | for i := 0; i < b.N; i++ { 56 | ok := f.Enqueue(i) 57 | if i < max && !ok { 58 | b.Fatal(ok) 59 | } 60 | } 61 | } 62 | 63 | func BenchmarkFIFO_Get(b *testing.B) { 64 | var max = int(10000000) 65 | f := New(max) 66 | for i := 0; i < max; i++ { 67 | ok := f.Enqueue(i) 68 | if !ok { 69 | b.Fatal(ok) 70 | } 71 | } 72 | 73 | for i := 0; i < b.N; i++ { 74 | x := f.Dequeue() 75 | if i < max && x == nil { 76 | b.Fatal("dequeue failed") 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /codec/codec_test.go: -------------------------------------------------------------------------------- 1 | package codec 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func Test_ENCODE_BIT_NOT(t *testing.T) { 9 | str := "hello world" 10 | fmt.Println("length is ", len(str)) 11 | buff := []byte(str) 12 | Encode(ENCODE_BIT_NOT, buff) 13 | fmt.Println("after encode is:", buff) 14 | Decode(ENCODE_BIT_NOT, buff) 15 | if string(buff) != str { 16 | t.Fatal("not equal") 17 | } 18 | } 19 | 20 | func Test_ENCODE_BYTE_RVS(t *testing.T) { 21 | str := "hello world" 22 | fmt.Println("length is ", len(str)) 23 | buff := []byte(str) 24 | Encode(ENCODE_BYTE_RVS, buff) 25 | fmt.Println("after encode is:", buff) 26 | Decode(ENCODE_BYTE_RVS, buff) 27 | if string(buff) != str { 28 | t.Fatal("not equal") 29 | } 30 | } 31 | 32 | func Test_ENCODE_LOOP_XOR(t *testing.T) { 33 | str := "hello world" 34 | fmt.Println("length is ", len(str)) 35 | buff := []byte(str) 36 | Encode(ENCODE_LOOP_XOR, buff) 37 | fmt.Println("after encode is:", buff) 38 | Decode(ENCODE_LOOP_XOR, buff) 39 | if string(buff) != str { 40 | t.Fatal("not equal") 41 | } 42 | } 43 | 44 | func Test_ENCODE_LOOP_XOR_N(t *testing.T) { 45 | str := "hello world" 46 | fmt.Println("length is ", len(str)) 47 | buff := []byte(str) 48 | n := 10240 49 | for i := 0; i < n; i++ { 50 | Encode(ENCODE_LOOP_XOR, buff) 51 | } 52 | 53 | for i := 0; i < n; i++ { 54 | Decode(ENCODE_LOOP_XOR, buff) 55 | } 56 | fmt.Printf("after %v\n", string(buff)) 57 | } 58 | 59 | func Benchmark_ENCODE_LOOP_XOR(b *testing.B) { 60 | buff := make([]byte, 1024) 61 | for i := 0; i < b.N; i++ { 62 | Encode(ENCODE_LOOP_XOR, buff) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /container/bloom/bloom9_test.go: -------------------------------------------------------------------------------- 1 | package bloom 2 | 3 | import ( 4 | "math/big" 5 | "testing" 6 | ) 7 | 8 | func TestBloom(t *testing.T) { 9 | positive := []string{ 10 | "testtest", 11 | "test", 12 | "hallo", 13 | "other", 14 | } 15 | negative := []string{ 16 | "tes", 17 | "lo", 18 | } 19 | 20 | var bloom Bloom 21 | for _, data := range positive { 22 | bloom.Add(new(big.Int).SetBytes([]byte(data))) 23 | } 24 | 25 | for _, data := range positive { 26 | if !bloom.TestBytes([]byte(data)) { 27 | t.Error("expected", data, "to test true") 28 | } 29 | } 30 | for _, data := range negative { 31 | if bloom.TestBytes([]byte(data)) { 32 | t.Error("did not expect", data, "to test true") 33 | } 34 | } 35 | 36 | b := bloom.Bytes() 37 | 38 | var bloom2 Bloom 39 | bloom2.SetBytes(b) 40 | for _, data := range positive { 41 | if !bloom.TestBytes([]byte(data)) { 42 | t.Error("expected", data, "to test true") 43 | } 44 | } 45 | for _, data := range negative { 46 | if bloom.TestBytes([]byte(data)) { 47 | t.Error("did not expect", data, "to test true") 48 | } 49 | } 50 | 51 | // 52 | bs, err := bloom.MarshalText() 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | 57 | var bloom3 Bloom 58 | err = bloom3.UnmarshalText(bs) 59 | if err != nil { 60 | t.Fatal(err) 61 | } 62 | for _, data := range positive { 63 | if !bloom.TestBytes([]byte(data)) { 64 | t.Error("expected", data, "to test true") 65 | } 66 | } 67 | for _, data := range negative { 68 | if bloom.TestBytes([]byte(data)) { 69 | t.Error("did not expect", data, "to test true") 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /container/pqueue/priority_queue.go: -------------------------------------------------------------------------------- 1 | package pqueue 2 | 3 | import ( 4 | "container/heap" 5 | ) 6 | 7 | type Interface interface { 8 | Less(other interface{}) bool 9 | } 10 | 11 | type sorter []Interface 12 | 13 | // Implement heap.Interface: Push, Pop, Len, Less, Swap 14 | func (s *sorter) Push(x interface{}) { 15 | *s = append(*s, x.(Interface)) 16 | } 17 | 18 | func (s *sorter) Pop() interface{} { 19 | n := len(*s) 20 | if n > 0 { 21 | x := (*s)[n-1] 22 | *s = (*s)[0 : n-1] 23 | return x 24 | } 25 | return nil 26 | } 27 | 28 | func (s *sorter) Len() int { 29 | return len(*s) 30 | } 31 | 32 | func (s *sorter) Less(i, j int) bool { 33 | return (*s)[i].Less((*s)[j]) 34 | } 35 | 36 | func (s *sorter) Swap(i, j int) { 37 | (*s)[i], (*s)[j] = (*s)[j], (*s)[i] 38 | } 39 | 40 | // Define priority queue struct 41 | type PriorityQueue struct { 42 | s *sorter 43 | } 44 | 45 | func New() *PriorityQueue { 46 | q := &PriorityQueue{s: new(sorter)} 47 | heap.Init(q.s) 48 | return q 49 | } 50 | 51 | func (q *PriorityQueue) Push(x Interface) { 52 | heap.Push(q.s, x) 53 | } 54 | 55 | func (q *PriorityQueue) Pop() Interface { 56 | return heap.Pop(q.s).(Interface) 57 | } 58 | 59 | func (q *PriorityQueue) Top() Interface { 60 | if len(*q.s) > 0 { 61 | return (*q.s)[0].(Interface) 62 | } 63 | return nil 64 | } 65 | 66 | func (q *PriorityQueue) Fix(x Interface, i int) { 67 | (*q.s)[i] = x 68 | heap.Fix(q.s, i) 69 | } 70 | 71 | func (q *PriorityQueue) Remove(i int) Interface { 72 | return heap.Remove(q.s, i).(Interface) 73 | } 74 | 75 | func (q *PriorityQueue) Len() int { 76 | return q.s.Len() 77 | } 78 | -------------------------------------------------------------------------------- /container/bitset/bitset.go: -------------------------------------------------------------------------------- 1 | // github.com/alex023/basekit/tree/master/container/bitset 2 | package bitset 3 | 4 | import "math" 5 | 6 | const ( 7 | max_value = math.MaxUint32 8 | shift = 5 9 | mask = 0x1F 10 | ) 11 | 12 | //var defaultBitSet = NewBitSet() 13 | 14 | // BitSet 是位图索引的基本数据结构。用一个bit位来标记某个元素对应的value，而key即是这个元素。由于采用bit为单位来存储数据，因此在可以大大的节省存储空间 15 | // 通过该结构提供的方法,可用于排序、查重。对于包括所有uint32的数字的key（约40亿）,存储所有的整形,只需要其1/32的容量,即500M即可。 16 | type BitSet struct { 17 | //保存实际的bit数据 18 | data []uint32 19 | //指示该BitSet的bit容量 20 | bitsize uint32 21 | //该bitmap被设置1的最大位数 22 | maxpos uint32 23 | } 24 | 25 | // 默认的位图索引创建方法,按照uint32的最大范围开辟500M内存空间以作计算使用。 26 | func NewBitSet() *BitSet { 27 | return &BitSet{ 28 | data: make([]uint32, max_value>>shift), 29 | bitsize: max_value, 30 | } 31 | } 32 | 33 | // 根据指定的值创建位图索引,分配适合的空间,但输入值不得大于uint32的最大值。。 34 | func NewBitSetByMax(maxnum uint32) *BitSet { 35 | if maxnum == 0 || maxnum > max_value { 36 | maxnum = max_value 37 | } 38 | return &BitSet{ 39 | data: make([]uint32, maxnum>>shift), 40 | bitsize: maxnum, 41 | } 42 | } 43 | 44 | // 添加给定key 45 | func (bitset *BitSet) Add(key uint32) { 46 | index, pos := key>>shift, key&mask 47 | bitset.data[index] |= (1 << (pos)) 48 | if bitset.maxpos < key { 49 | bitset.maxpos = key 50 | } 51 | } 52 | 53 | // 删除给定key 54 | func (bitset *BitSet) Del(key uint32) { 55 | index, pos := key>>shift, key&mask 56 | bitset.data[index] &^= (1 << (pos)) 57 | } 58 | 59 | // 判断给定key是否存在 60 | func (BitSet *BitSet) IsExist(key uint32) bool { 61 | index, pos := key>>shift, key&mask 62 | return (BitSet.data[index]>>pos)&0x01 == 1 63 | } 64 | -------------------------------------------------------------------------------- /container/blocking_queue/blocking_queue.go: -------------------------------------------------------------------------------- 1 | package blocking_queue 2 | 3 | import ( 4 | "container/list" 5 | "math" 6 | "sync" 7 | ) 8 | 9 | // 阻塞队列 10 | type BlockingQueue struct { 11 | cap int 12 | list *list.List 13 | mu *sync.RWMutex 14 | cond *sync.Cond 15 | } 16 | 17 | func New(cap int) *BlockingQueue { 18 | var al BlockingQueue 19 | al.mu = new(sync.RWMutex) 20 | al.cond = sync.NewCond(al.mu) 21 | al.list = list.New() 22 | al.cap = cap 23 | if al.cap <= 0 { 24 | al.cap = math.MaxInt32 25 | } 26 | return &al 27 | } 28 | 29 | func (p *BlockingQueue) Empty() bool { 30 | p.mu.RLock() 31 | defer p.mu.RUnlock() 32 | return p.list.Len() == 0 33 | } 34 | 35 | func (p *BlockingQueue) Size() int { 36 | p.mu.RLock() 37 | defer p.mu.RUnlock() 38 | return p.list.Len() 39 | } 40 | 41 | func (p *BlockingQueue) Front() interface{} { 42 | p.mu.RLock() 43 | defer p.mu.RUnlock() 44 | if p.list.Front() != nil { 45 | return p.list.Front().Value 46 | } 47 | return nil 48 | } 49 | 50 | func (p *BlockingQueue) Back() interface{} { 51 | p.mu.RLock() 52 | defer p.mu.RUnlock() 53 | if p.list.Back() != nil { 54 | return p.list.Back().Value 55 | } 56 | return nil 57 | } 58 | 59 | func (p *BlockingQueue) PushBack(e interface{}) { 60 | p.mu.Lock() 61 | defer p.mu.Unlock() 62 | if p.list.Len() >= p.cap { 63 | p.cond.Wait() 64 | } 65 | p.list.PushBack(e) 66 | p.cond.Signal() 67 | } 68 | 69 | func (p *BlockingQueue) PopFront() interface{} { 70 | p.mu.Lock() 71 | defer p.mu.Unlock() 72 | if p.list.Len() == 0 { 73 | p.cond.Wait() 74 | } 75 | e := p.list.Front() 76 | p.cond.Signal() 77 | return p.list.Remove(e) 78 | } 79 | -------------------------------------------------------------------------------- /validator/validator.go: -------------------------------------------------------------------------------- 1 | package validator 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | func IsE164(in string) bool { 8 | e164RegexString := "^\\+[1-9]?[0-9]{7,14}$" 9 | e164Regex := regexp.MustCompile(e164RegexString) 10 | return e164Regex.MatchString(in) 11 | } 12 | 13 | func IsEmail(in string) bool { 14 | emailRegexString := "^(?:(?:(?:(?:[a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+(?:\\.([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+)*)|(?:(?:\\x22)(?:(?:(?:(?:\\x20|\\x09)*(?:\\x0d\\x0a))?(?:\\x20|\\x09)+)?(?:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f]|\\x21|[\\x23-\\x5b]|[\\x5d-\\x7e]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(?:(?:[\\x01-\\x09\\x0b\\x0c\\x0d-\\x7f]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}]))))*(?:(?:(?:\\x20|\\x09)*(?:\\x0d\\x0a))?(\\x20|\\x09)+)?(?:\\x22))))@(?:(?:(?:[a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(?:(?:[a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])(?:[a-zA-Z]|\\d|-|\\.|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*(?:[a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.)+(?:(?:[a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(?:(?:[a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])(?:[a-zA-Z]|\\d|-|\\.|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*(?:[a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.?$" 15 | emailRegex := regexp.MustCompile(emailRegexString) 16 | return emailRegex.MatchString(in) 17 | } 18 | -------------------------------------------------------------------------------- /hash/murmurhash3/mmhash3_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The Go Authors. All rights reserved. 2 | // refer https://github.com/gwenn/murmurhash3 3 | 4 | package murmurhash3 5 | 6 | import ( 7 | "hash" 8 | "testing" 9 | ) 10 | 11 | const testDataSize = 40 12 | 13 | func TestMurmur3A(t *testing.T) { 14 | expected := uint32(3127628307) 15 | hash := Murmur3A([]byte("test"), 0) 16 | if hash != expected { 17 | t.Errorf("Expected %d but was %d for Murmur3A\n", expected, hash) 18 | } 19 | } 20 | 21 | func TestMurmur3C(t *testing.T) { 22 | expected := []uint32{1862463280, 1426881896, 1426881896, 1426881896} 23 | hash := Murmur3C([]byte("test"), 0) 24 | for i, e := range expected { 25 | if hash[i] != e { 26 | t.Errorf("Expected %d but was %d for Murmur3C[%d]\n", e, hash[i], i) 27 | } 28 | } 29 | } 30 | 31 | func TestMurmur3F(t *testing.T) { 32 | expected := []uint64{12429135405209477533, 11102079182576635266} 33 | hash := Murmur3F([]byte("test"), 0) 34 | for i, e := range expected { 35 | if hash[i] != e { 36 | t.Errorf("Expected %d but was %d for Murmur3F[%d]\n", e, hash[i], i) 37 | } 38 | } 39 | } 40 | 41 | func Benchmark3A(b *testing.B) { 42 | benchmark(b, NewMurmur3A()) 43 | } 44 | func Benchmark3C(b *testing.B) { 45 | benchmark(b, NewMurmur3C()) 46 | } 47 | func Benchmark3F(b *testing.B) { 48 | benchmark(b, NewMurmur3F()) 49 | } 50 | 51 | func benchmark(b *testing.B, h hash.Hash) { 52 | b.ResetTimer() 53 | b.SetBytes(testDataSize) 54 | data := make([]byte, testDataSize) 55 | for i := range data { 56 | data[i] = byte(i + 'a') 57 | } 58 | 59 | b.StartTimer() 60 | for todo := b.N; todo != 0; todo-- { 61 | h.Reset() 62 | h.Write(data) 63 | h.Sum(nil) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /log/go.sum: -------------------------------------------------------------------------------- 1 | github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 6 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 7 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 8 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 9 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 10 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 11 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 12 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 13 | go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= 14 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 15 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 16 | go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= 17 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= 18 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= 19 | gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= 20 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 21 | -------------------------------------------------------------------------------- /pool/xbytes/bytes.go: -------------------------------------------------------------------------------- 1 | package xbytes 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type Buffer struct { 8 | buf []byte 9 | next *Buffer // next free buffer 10 | } 11 | 12 | func (b *Buffer) Bytes() []byte { 13 | return b.buf 14 | } 15 | 16 | // Pool is a buffer pool. 17 | type Pool struct { 18 | lock sync.Mutex 19 | free *Buffer 20 | max int 21 | num int 22 | size int 23 | } 24 | 25 | // NewPool new a memory buffer pool struct. 26 | func NewPool(num, size int) (p *Pool) { 27 | p = new(Pool) 28 | p.init(num, size) 29 | return 30 | } 31 | 32 | // Init init the memory buffer. 33 | func (p *Pool) Init(num, size int) { 34 | p.init(num, size) 35 | return 36 | } 37 | 38 | // init init the memory buffer. 39 | func (p *Pool) init(num, size int) { 40 | p.num = num 41 | p.size = size 42 | p.max = num * size 43 | p.grow() 44 | } 45 | 46 | // grow grow the memory buffer size, and update free pointer. 47 | func (p *Pool) grow() { 48 | var ( 49 | i int 50 | b *Buffer 51 | bs []Buffer 52 | buf []byte 53 | ) 54 | buf = make([]byte, p.max) 55 | bs = make([]Buffer, p.num) 56 | p.free = &bs[0] 57 | b = p.free 58 | for i = 1; i < p.num; i++ { 59 | b.buf = buf[(i-1)*p.size : i*p.size] 60 | b.next = &bs[i] 61 | b = b.next 62 | } 63 | b.buf = buf[(i-1)*p.size : i*p.size] 64 | b.next = nil 65 | return 66 | } 67 | 68 | // Get get a free memory buffer. 69 | func (p *Pool) Get() (b *Buffer) { 70 | p.lock.Lock() 71 | if b = p.free; b == nil { 72 | p.grow() 73 | b = p.free 74 | } 75 | p.free = b.next 76 | p.lock.Unlock() 77 | return 78 | } 79 | 80 | // Put put back a memory buffer to free. 81 | func (p *Pool) Put(b *Buffer) { 82 | p.lock.Lock() 83 | b.next = p.free 84 | p.free = b 85 | p.lock.Unlock() 86 | return 87 | } 88 | -------------------------------------------------------------------------------- /pool/allocator/alloc.go: -------------------------------------------------------------------------------- 1 | // from github.com/xtaci/smux 2 | package allocator 3 | 4 | import ( 5 | "sync" 6 | 7 | "github.com/pkg/errors" 8 | ) 9 | 10 | var defaultAllocator *Allocator 11 | 12 | func init() { 13 | defaultAllocator = NewAllocator() 14 | } 15 | 16 | // Allocator for incoming frames, optimized to prevent overwriting after zeroing 17 | type Allocator struct { 18 | buffers []sync.Pool 19 | } 20 | 21 | // NewAllocator initiates a []byte allocator for frames less than 65536 bytes, 22 | // the waste(memory fragmentation) of space allocation is guaranteed to be 23 | // no more than 50%. 24 | func NewAllocator() *Allocator { 25 | alloc := new(Allocator) 26 | alloc.buffers = make([]sync.Pool, 17) // 1B -> 64K 27 | for k := range alloc.buffers { 28 | i := k 29 | alloc.buffers[k].New = func() interface{} { 30 | return make([]byte, 1< 65536 { 39 | return nil 40 | } 41 | 42 | bits := msb(size) 43 | if size == 1< 65536 || cap(buf) != 1<>= 1 65 | for size > 0 { 66 | size >>= 1 67 | pos++ 68 | } 69 | return pos 70 | } 71 | -------------------------------------------------------------------------------- /crypto/tripledes.go: -------------------------------------------------------------------------------- 1 | //package crypto 2 | //DES和3DES加解密 3 | package crypto 4 | 5 | import ( 6 | "crypto/des" 7 | "errors" 8 | ) 9 | 10 | //TripleDesECBEncrypt 3des加密 使用加密模式为ECB 填充方式为pkcs5 11 | //入参:待加密原始数据 字符串key 12 | //返回值:加密后的base64字符串 13 | func TripleDesECBEncrypt(origData []byte, key string) ([]byte, error) { 14 | bKey := transKey(key) 15 | block, err := des.NewTripleDESCipher(bKey) 16 | if err != nil { 17 | return nil, err 18 | } 19 | bs := block.BlockSize() 20 | origData = PKCSPadding(origData, bs) 21 | if len(origData)%bs != 0 { 22 | return nil, errors.New("Need a multiple of the blocksize") 23 | } 24 | out := make([]byte, len(origData)) 25 | dst := out 26 | for len(origData) > 0 { 27 | block.Encrypt(dst, origData[:bs]) 28 | origData = origData[bs:] 29 | dst = dst[bs:] 30 | } 31 | return out, nil 32 | } 33 | 34 | //TripleDesECBDecrypt 3des解密 35 | func TripleDesECBDecrypt(crypted []byte, key string) ([]byte, error) { 36 | bKey := transKey(key) 37 | block, err := des.NewTripleDESCipher(bKey) 38 | if err != nil { 39 | return nil, err 40 | } 41 | bs := block.BlockSize() 42 | if len(crypted)%bs != 0 { 43 | return nil, errors.New("crypto/cipher: input not full blocks") 44 | } 45 | out := make([]byte, len(crypted)) 46 | dst := out 47 | for len(crypted) > 0 { 48 | block.Decrypt(dst, crypted[:bs]) 49 | crypted = crypted[bs:] 50 | dst = dst[bs:] 51 | } 52 | out = PKCSPadding(out, block.BlockSize()) 53 | return out, nil 54 | } 55 | 56 | //0填充 57 | func transKey(key string) []byte { 58 | //key只取24位 59 | bKey := make([]byte, 24) 60 | keys := []byte(key) 61 | length := len(keys) 62 | if length == 24 { 63 | copy(bKey, keys) 64 | } else if length < 24 { 65 | copy(bKey, keys) 66 | for i := length; i < length; i++ { 67 | bKey[i] = 0 68 | } 69 | } else { 70 | copy(bKey, keys[:24]) 71 | } 72 | return bKey 73 | } 74 | -------------------------------------------------------------------------------- /referral/referral.go: -------------------------------------------------------------------------------- 1 | package referral 2 | 3 | import ( 4 | "errors" 5 | "strings" 6 | ) 7 | 8 | var ( 9 | base = "E8S2DZX9WYLTN6BGQF7P5IK3MJUAR4HV" 10 | decimal = 32 11 | pad = "C" 12 | length = 6 13 | ) 14 | 15 | func SetBase(b string) { 16 | b = strings.ToUpper(strings.TrimSpace(b)) 17 | if len(b) <= 0 { 18 | return 19 | } 20 | decimal = len(base) 21 | base = b 22 | } 23 | 24 | func SetPad(p string) error { 25 | p = strings.ToUpper(p) 26 | if strings.Contains(base, p) { 27 | return errors.New("pad should not exists in base") 28 | } 29 | pad = p 30 | return nil 31 | } 32 | 33 | func SetLength(n int) { 34 | length = n 35 | } 36 | 37 | // Encode uid to referral code 38 | func Encode(uid uint64) (referral string) { 39 | id := uid 40 | mod := uint64(0) 41 | for id != 0 { 42 | mod = id % uint64(decimal) 43 | id = id / uint64(decimal) 44 | referral += string(base[mod]) 45 | } 46 | resLen := len(referral) 47 | if resLen < length { 48 | referral += pad 49 | for i := 0; i < length-resLen-1; i++ { 50 | referral += string(base[(int(uid)+i)%decimal]) 51 | } 52 | } 53 | return 54 | } 55 | 56 | // Decode code to uid 57 | func Decode(referral string) (id uint64) { 58 | lenCode := len(referral) 59 | baseArr := []byte(base) // string decimal to byte array 60 | baseRev := make(map[byte]int) // decimal data key to map 61 | for k, v := range baseArr { 62 | baseRev[v] = k 63 | } 64 | // find cover char addr 65 | isPad := strings.Index(referral, pad) 66 | if isPad != -1 { 67 | lenCode = isPad 68 | } 69 | r := 0 70 | for i := 0; i < lenCode; i++ { 71 | // if cover char , continue 72 | if string(referral[i]) == pad { 73 | continue 74 | } 75 | index, ok := baseRev[referral[i]] 76 | if !ok { 77 | return 0 78 | } 79 | b := uint64(1) 80 | for j := 0; j < r; j++ { 81 | b *= uint64(decimal) 82 | } 83 | id += uint64(index) * b 84 | r++ 85 | } 86 | return id 87 | } 88 | -------------------------------------------------------------------------------- /pool/allocator/alloc_test.go: -------------------------------------------------------------------------------- 1 | package allocator 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | ) 7 | 8 | func TestAllocGet(t *testing.T) { 9 | alloc := NewAllocator() 10 | if alloc.Get(0) != nil { 11 | t.Fatal(0) 12 | } 13 | if len(alloc.Get(1)) != 1 { 14 | t.Fatal(1) 15 | } 16 | if len(alloc.Get(2)) != 2 { 17 | t.Fatal(2) 18 | } 19 | if len(alloc.Get(3)) != 3 || cap(alloc.Get(3)) != 4 { 20 | t.Fatal(3) 21 | } 22 | if len(alloc.Get(4)) != 4 { 23 | t.Fatal(4) 24 | } 25 | if len(alloc.Get(1023)) != 1023 || cap(alloc.Get(1023)) != 1024 { 26 | t.Fatal(1023) 27 | } 28 | if len(alloc.Get(1024)) != 1024 { 29 | t.Fatal(1024) 30 | } 31 | if len(alloc.Get(65536)) != 65536 { 32 | t.Fatal(65536) 33 | } 34 | if alloc.Get(65537) != nil { 35 | t.Fatal(65537) 36 | } 37 | } 38 | 39 | func TestAllocPut(t *testing.T) { 40 | alloc := NewAllocator() 41 | if err := alloc.Put(nil); err == nil { 42 | t.Fatal("put nil misbehavior") 43 | } 44 | if err := alloc.Put(make([]byte, 3, 3)); err == nil { 45 | t.Fatal("put elem:3 []bytes misbehavior") 46 | } 47 | if err := alloc.Put(make([]byte, 4, 4)); err != nil { 48 | t.Fatal("put elem:4 []bytes misbehavior") 49 | } 50 | if err := alloc.Put(make([]byte, 1023, 1024)); err != nil { 51 | t.Fatal("put elem:1024 []bytes misbehavior") 52 | } 53 | if err := alloc.Put(make([]byte, 65536, 65536)); err != nil { 54 | t.Fatal("put elem:65536 []bytes misbehavior") 55 | } 56 | if err := alloc.Put(make([]byte, 65537, 65537)); err == nil { 57 | t.Fatal("put elem:65537 []bytes misbehavior") 58 | } 59 | } 60 | 61 | func TestAllocPutThenGet(t *testing.T) { 62 | alloc := NewAllocator() 63 | data := alloc.Get(4) 64 | alloc.Put(data) 65 | newData := alloc.Get(4) 66 | if cap(data) != cap(newData) { 67 | t.Fatal("different cap while alloc.Get()") 68 | } 69 | } 70 | 71 | func BenchmarkMSB(b *testing.B) { 72 | for i := 0; i < b.N; i++ { 73 | msb(rand.Int()) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /hash/ketama/ketama.go: -------------------------------------------------------------------------------- 1 | package ketama 2 | 3 | import ( 4 | "crypto/sha1" 5 | "sort" 6 | "strconv" 7 | ) 8 | 9 | const ( 10 | // TODO you can modify this get more virtual node 11 | Base = 255 12 | ) 13 | 14 | type node struct { 15 | node string 16 | hash uint 17 | } 18 | 19 | type tickArray []node 20 | 21 | func (p tickArray) Len() int { return len(p) } 22 | func (p tickArray) Less(i, j int) bool { return p[i].hash < p[j].hash } 23 | func (p tickArray) Swap(i, j int) { p[i], p[j] = p[j], p[i] } 24 | func (p tickArray) Sort() { sort.Sort(p) } 25 | 26 | type HashRing struct { 27 | defaultSpots int 28 | ticks tickArray 29 | length int 30 | } 31 | 32 | func NewRing(n int) (h *HashRing) { 33 | h = new(HashRing) 34 | h.defaultSpots = n 35 | return 36 | } 37 | 38 | // Adds a new node to a hash ring 39 | // n: name of the server 40 | // s: multiplier for default number of ticks (useful when one cache node has more resources, like RAM, than another) 41 | func (h *HashRing) AddNode(n string, s int) { 42 | tSpots := h.defaultSpots * s 43 | hash := sha1.New() 44 | for i := 1; i <= tSpots; i++ { 45 | hash.Write([]byte(n + ":" + strconv.Itoa(i))) 46 | hashBytes := hash.Sum(nil) 47 | 48 | n := &node{ 49 | node: n, 50 | hash: uint(hashBytes[19]) | uint(hashBytes[18])<<8 | uint(hashBytes[17])<<16 | uint(hashBytes[16])<<24, 51 | } 52 | 53 | h.ticks = append(h.ticks, *n) 54 | hash.Reset() 55 | } 56 | } 57 | 58 | func (h *HashRing) Bake() { 59 | h.ticks.Sort() 60 | h.length = len(h.ticks) 61 | } 62 | 63 | func (h *HashRing) Hash(s string) string { 64 | hash := sha1.New() 65 | hash.Write([]byte(s)) 66 | hashBytes := hash.Sum(nil) 67 | v := uint(hashBytes[19]) | uint(hashBytes[18])<<8 | uint(hashBytes[17])<<16 | uint(hashBytes[16])<<24 68 | i := sort.Search(h.length, func(i int) bool { return h.ticks[i].hash >= v }) 69 | 70 | if i == h.length { 71 | i = 0 72 | } 73 | 74 | return h.ticks[i].node 75 | } 76 | -------------------------------------------------------------------------------- /natx/defaultApp.go: -------------------------------------------------------------------------------- 1 | package natx 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "strings" 8 | ) 9 | 10 | type DefaultAPP struct { 11 | cli *NATClient 12 | } 13 | 14 | type DefaultAPPReq struct { 15 | Arg1 int 16 | Arg2 string 17 | } 18 | 19 | type DefaultAPPResp struct { 20 | Arg1 int 21 | Arg2 string 22 | } 23 | 24 | func (app *DefaultAPP) Request(ctx context.Context, subj string, req interface{}) (interface{}, error) { 25 | var resp DefaultAPPResp 26 | err := app.cli.conn.RequestWithContext(context.Background(), SUBJ_PRE+subj, req, &resp) 27 | return &resp, err 28 | } 29 | 30 | func (app *DefaultAPP) Subscribe(subj string, handle Handle) error { 31 | f := func(subj, reply string, req *DefaultAPPReq) { 32 | go func() { 33 | subj = strings.TrimPrefix(subj, SUBJ_PRE) 34 | resp := handle(subj, req) 35 | if reply != "" { 36 | if err := app.cli.conn.Publish(reply, resp); err != nil { 37 | log.Println(err, " reply is ", reply) 38 | } 39 | } 40 | }() 41 | } 42 | 43 | _, err := app.cli.conn.Subscribe(SUBJ_PRE+subj, f) 44 | return err 45 | } 46 | 47 | func (app *DefaultAPP) QueueSubscribe(subj string, handle Handle) error { 48 | f := func(subj, reply string, req *DefaultAPPReq) { 49 | go func() { 50 | subj = strings.TrimPrefix(subj, SUBJ_PRE) 51 | resp := handle(subj, req) 52 | if reply != "" { 53 | if err := app.cli.conn.Publish(reply, resp); err != nil { 54 | log.Println(err, " reply is ", reply) 55 | } 56 | } 57 | }() 58 | } 59 | 60 | _, err := app.cli.conn.QueueSubscribe(SUBJ_PRE+subj, QUEUE_PRE+subj, f) 61 | return err 62 | } 63 | 64 | func (app *DefaultAPP) Publish(subj string, data interface{}) error { 65 | return app.cli.conn.Publish(SUBJ_PRE+subj, data) 66 | } 67 | 68 | func (app *DefaultAPP) defaultHandle(subj string, req interface{}) interface{} { 69 | fmt.Println("服务端：", subj) 70 | r := req.(*DefaultAPPReq) 71 | fmt.Println("服务端：", r.Arg1, " ", r.Arg2) 72 | return &DefaultAPPResp{ 73 | Arg1: 123, 74 | Arg2: "hello world", 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /nsqx/consumer.go: -------------------------------------------------------------------------------- 1 | package nsqx 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/nsqio/go-nsq" 7 | ) 8 | 9 | type dispatcher struct { 10 | topic string 11 | handle func(string, interface{}) 12 | } 13 | 14 | type consumerDispatcher struct { 15 | F *dispatcher 16 | C *nsq.Consumer 17 | } 18 | 19 | type TopicDiscoverer struct { 20 | topics map[string]*consumerDispatcher 21 | cfg *nsq.Config 22 | } 23 | 24 | // NewTopicDiscoverer 生成多个主题发现的消费者 25 | // topics 订阅主题列表 26 | // channel channel名称 27 | // maxInFlight NSQD在有maxInFlight*25条消息时向下推送 28 | // lookupdHTTPAddrs NSQLOOKUPs的http地址 29 | // handle 消息处理句柄 30 | // PS:对channel不同的topic应该生成不同的消费者 31 | func NewTopicDiscoverer(topics []string, channel string, maxInFlight int, lookupdHTTPAddrs []string, handle func(string, interface{})) *TopicDiscoverer { 32 | var discoverer TopicDiscoverer 33 | discoverer.topics = make(map[string]*consumerDispatcher) 34 | 35 | cfg := nsq.NewConfig() 36 | cfg.UserAgent = USER_AGENT 37 | cfg.MaxInFlight = maxInFlight 38 | 39 | discoverer.cfg = cfg 40 | for _, v := range topics { 41 | cd, err := newConsumerDispatcher(v, channel, cfg, lookupdHTTPAddrs, handle) 42 | if err != nil { 43 | log.Fatal(err) 44 | } 45 | discoverer.topics[v] = cd 46 | } 47 | return &discoverer 48 | } 49 | 50 | func newConsumerDispatcher(topic string, channel string, cfg *nsq.Config, lookupdHTTPAddrs []string, handle func(string, interface{})) (*consumerDispatcher, error) { 51 | var err error 52 | r := newDispatcher(topic, handle) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | consumer, err := nsq.NewConsumer(topic, channel, cfg) 58 | if err != nil { 59 | return nil, err 60 | } 61 | consumer.AddHandler(r) 62 | err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs) 63 | if err != nil { 64 | log.Fatal(err) 65 | } 66 | return &consumerDispatcher{ 67 | C: consumer, 68 | F: r, 69 | }, nil 70 | } 71 | 72 | func newDispatcher(topic string, handle func(string, interface{})) *dispatcher { 73 | var disp dispatcher 74 | disp.topic = topic 75 | disp.handle = handle 76 | return &disp 77 | } 78 | 79 | // HandleMessage 消息处理句柄 实现接口Handler 80 | func (p *dispatcher) HandleMessage(msg *nsq.Message) error { 81 | p.handle(p.topic, msg.Body) 82 | return nil 83 | } 84 | -------------------------------------------------------------------------------- /natx/natc.go: -------------------------------------------------------------------------------- 1 | package natx 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "strings" 7 | "time" 8 | 9 | "github.com/nats-io/nats.go" 10 | ) 11 | 12 | // 默认为subj和queue名称加前缀 13 | const ( 14 | SUBJ_PRE = "NCSUBJ-" 15 | QUEUE_PRE = "NCQUE-" 16 | ) 17 | 18 | type NATClient struct { 19 | conn *nats.EncodedConn 20 | } 21 | 22 | func NewNATClient(endpoints []string) *NATClient { 23 | opts := nats.Options{ 24 | Servers: endpoints, 25 | AllowReconnect: true, 26 | MaxReconnect: 10, 27 | ReconnectWait: 5 * time.Second, 28 | Timeout: 3 * time.Second, 29 | PingInterval: 30 * time.Second, 30 | } 31 | nc, err := opts.Connect() 32 | if err != nil { 33 | panic(err) 34 | } 35 | nats.RegisterEncoder(MSGP_ENCODER, &MsgpEncoder{}) 36 | c, err := nats.NewEncodedConn(nc, MSGP_ENCODER) 37 | if err != nil { 38 | panic(err) 39 | } 40 | return &NATClient{ 41 | conn: c, 42 | } 43 | } 44 | 45 | func (p *NATClient) Destroy() { 46 | p.conn.Close() 47 | } 48 | 49 | func (p *NATClient) Request(ctx context.Context, subj string, req interface{}, resp interface{}) error { 50 | return p.conn.RequestWithContext(ctx, SUBJ_PRE+subj, req, &resp) 51 | } 52 | 53 | func (p *NATClient) Publish(subj string, data interface{}) error { 54 | return p.conn.Publish(SUBJ_PRE+subj, data) 55 | } 56 | 57 | func (p *NATClient) Subscribe(subj string, handle Handle) error { 58 | f := func(subj, reply string, req interface{}) { 59 | go func() { 60 | subj = strings.TrimPrefix(subj, SUBJ_PRE) 61 | resp := handle(subj, req) 62 | if reply != "" { 63 | if err := p.conn.Publish(reply, resp); err != nil { 64 | log.Println(err, " reply is ", reply) 65 | } 66 | } 67 | }() 68 | } 69 | 70 | _, err := p.conn.Subscribe(SUBJ_PRE+subj, f) 71 | return err 72 | } 73 | 74 | func (p *NATClient) QueueSubscribe(subj string, handle Handle) error { 75 | f := func(subj, reply string, req interface{}) { 76 | go func() { 77 | subj = strings.TrimPrefix(subj, SUBJ_PRE) 78 | resp := handle(subj, req) 79 | if reply != "" { 80 | if err := p.conn.Publish(reply, resp); err != nil { 81 | log.Println(err, " reply is ", reply) 82 | } 83 | } 84 | }() 85 | } 86 | 87 | _, err := p.conn.QueueSubscribe(SUBJ_PRE+subj, QUEUE_PRE+subj, f) 88 | return err 89 | } 90 | -------------------------------------------------------------------------------- /httpx/sign_test.go: -------------------------------------------------------------------------------- 1 | package httpx 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | var ( 14 | exampleAccessKey = "exampleAccessKey" 15 | exampleAccessSecret = "exampleAccessSecret" 16 | exampleTTL = int64(60) 17 | exampleMethod = HmacSha256Hex 18 | ) 19 | 20 | func testHttpServer(sign *APISign) { 21 | http.HandleFunc("/sign", func(writer http.ResponseWriter, request *http.Request) { 22 | switch request.Method { 23 | case "GET": 24 | msg := request.Method + "\n" 25 | err := sign.Verify(request, "Authorization") 26 | if err != nil { 27 | msg += err.Error() 28 | } 29 | writer.WriteHeader(200) 30 | writer.Write([]byte(msg)) 31 | case "POST", "PUT": 32 | msg := request.Method + "\n" 33 | err := sign.Verify(request, "Authorization") 34 | if err != nil { 35 | msg += err.Error() 36 | } 37 | 38 | // 读出Body 39 | bodyData, _ := io.ReadAll(request.Body) 40 | defer request.Body.Close() 41 | 42 | writer.WriteHeader(200) 43 | writer.Write([]byte(msg + "\n")) 44 | writer.Write(bodyData) 45 | } 46 | }) 47 | http.ListenAndServe(":8080", http.DefaultServeMux) 48 | } 49 | 50 | func TestAPISignAndVerify(t *testing.T) { 51 | sn := NewAPISign(exampleAccessKey, exampleAccessSecret, exampleTTL, exampleMethod) 52 | go func() { 53 | testHttpServer(sn) 54 | }() 55 | time.Sleep(time.Second) 56 | path := "http://localhost:8080/sign?2=2&1=1" 57 | pointVal := float32(40.5) 58 | type Add struct { 59 | Address string `json:"address"` 60 | No int 61 | } 62 | b := struct { 63 | Name string 64 | Age int 65 | Balance float64 66 | Point *float32 67 | Adds []Add 68 | }{Name: "nice", Age: 5, Balance: 102.22222, Point: &pointVal, Adds: []Add{{Address: "XX", No: 88}, {Address: "AA", No: 77}}} 69 | byts, err := json.Marshal(b) 70 | if err != nil { 71 | t.Fatal(err) 72 | } 73 | 74 | req, err := http.NewRequest("POST", path, bytes.NewReader(byts)) 75 | if err != nil { 76 | t.Fatal(err) 77 | } 78 | req.Header.Set("Content-Type", "application/json") 79 | 80 | deadline := time.Now().Unix() + sn.TTL 81 | 82 | signature, err := sn.Sign(req, deadline) 83 | if err != nil { 84 | t.Fatal(err) 85 | } 86 | 87 | req.Header.Set("Authorization", fmt.Sprintf("%s:%s:%d", 88 | exampleAccessKey, signature, deadline)) 89 | resp, err := http.DefaultClient.Do(req) 90 | if err != nil { 91 | t.Fatal(err) 92 | } 93 | defer resp.Body.Close() 94 | respVal, _ := io.ReadAll(resp.Body) 95 | fmt.Println(string(respVal)) 96 | } 97 | -------------------------------------------------------------------------------- /etcdx/discovery/go.sum: -------------------------------------------------------------------------------- 1 | github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= 2 | github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 7 | github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= 8 | github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 9 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 10 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 11 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 12 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 13 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 14 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 15 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 16 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 17 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 18 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 19 | go.etcd.io/etcd/api/v3 v3.5.7 h1:sbcmosSVesNrWOJ58ZQFitHMdncusIifYcrBfwrlJSY= 20 | go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA= 21 | go.etcd.io/etcd/client/pkg/v3 v3.5.7 h1:y3kf5Gbp4e4q7egZdn5T7W9TSHUvkClN6u+Rq9mEOmg= 22 | go.etcd.io/etcd/client/pkg/v3 v3.5.7/go.mod h1:o0Abi1MK86iad3YrWhgUsbGx1pmTS+hrORWc2CamuhY= 23 | go.etcd.io/etcd/client/v2 v2.305.7 h1:AELPkjNR3/igjbO7CjyF1fPuVPjrblliiKj+Y6xSGOU= 24 | go.etcd.io/etcd/client/v2 v2.305.7/go.mod h1:GQGT5Z3TBuAQGvgPfhR7VPySu/SudxmEkRq9BgzFU6s= 25 | golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= 26 | golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= 27 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 28 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= 29 | -------------------------------------------------------------------------------- /etcdx/master/go.sum: -------------------------------------------------------------------------------- 1 | github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= 2 | github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 7 | github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= 8 | github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 9 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 10 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 11 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 12 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 13 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 14 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 15 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 16 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 17 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 18 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 19 | go.etcd.io/etcd/api/v3 v3.5.7 h1:sbcmosSVesNrWOJ58ZQFitHMdncusIifYcrBfwrlJSY= 20 | go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA= 21 | go.etcd.io/etcd/client/pkg/v3 v3.5.7 h1:y3kf5Gbp4e4q7egZdn5T7W9TSHUvkClN6u+Rq9mEOmg= 22 | go.etcd.io/etcd/client/pkg/v3 v3.5.7/go.mod h1:o0Abi1MK86iad3YrWhgUsbGx1pmTS+hrORWc2CamuhY= 23 | go.etcd.io/etcd/client/v2 v2.305.7 h1:AELPkjNR3/igjbO7CjyF1fPuVPjrblliiKj+Y6xSGOU= 24 | go.etcd.io/etcd/client/v2 v2.305.7/go.mod h1:GQGT5Z3TBuAQGvgPfhR7VPySu/SudxmEkRq9BgzFU6s= 25 | golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= 26 | golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= 27 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 28 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= 29 | -------------------------------------------------------------------------------- /etcdx/discovery/master.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "log" 7 | "time" 8 | 9 | "go.etcd.io/etcd/client/v2" 10 | ) 11 | 12 | type Master struct { 13 | members map[string]*Member 14 | KeysAPI client.KeysAPI 15 | } 16 | 17 | // Member is a client machine 18 | type Member struct { 19 | InGroup bool 20 | IP string 21 | Name string 22 | CPU int 23 | } 24 | 25 | func NewMaster(endpoints []string) *Master { 26 | cfg := client.Config{ 27 | Endpoints: endpoints, 28 | Transport: client.DefaultTransport, 29 | HeaderTimeoutPerRequest: time.Second, 30 | } 31 | 32 | etcdClient, err := client.New(cfg) 33 | if err != nil { 34 | log.Fatal("Error: cannot connec to etcdx:", err) 35 | } 36 | 37 | master := &Master{ 38 | members: make(map[string]*Member), 39 | KeysAPI: client.NewKeysAPI(etcdClient), 40 | } 41 | go master.WatchWorkers() 42 | return master 43 | } 44 | 45 | func (m *Master) AddWorker(info *WorkerInfo) { 46 | member := &Member{ 47 | InGroup: true, 48 | IP: info.IP, 49 | Name: info.Name, 50 | CPU: info.CPU, 51 | } 52 | m.members[member.Name] = member 53 | } 54 | 55 | func (m *Master) UpdateWorker(info *WorkerInfo) { 56 | member := m.members[info.Name] 57 | member.InGroup = true 58 | } 59 | 60 | func NodeToWorkerInfo(node *client.Node) *WorkerInfo { 61 | log.Println(node.Value) 62 | info := &WorkerInfo{} 63 | err := json.Unmarshal([]byte(node.Value), info) 64 | if err != nil { 65 | log.Print(err) 66 | } 67 | return info 68 | } 69 | 70 | func (m *Master) WatchWorkers() { 71 | api := m.KeysAPI 72 | watcher := api.Watcher("workers/", &client.WatcherOptions{ 73 | Recursive: true, 74 | }) 75 | for { 76 | res, err := watcher.Next(context.Background()) 77 | if err != nil { 78 | log.Println("Error watch workers:", err) 79 | break 80 | } 81 | if res.Action == "expire" { 82 | info := NodeToWorkerInfo(res.PrevNode) 83 | log.Println("Expire worker ", info.Name) 84 | member, ok := m.members[info.Name] 85 | if ok { 86 | member.InGroup = false 87 | } 88 | } else if res.Action == "set" { 89 | info := NodeToWorkerInfo(res.Node) 90 | if _, ok := m.members[info.Name]; ok { 91 | log.Println("Update worker ", info.Name) 92 | m.UpdateWorker(info) 93 | } else { 94 | log.Println("Add worker ", info.Name) 95 | m.AddWorker(info) 96 | } 97 | } else if res.Action == "delete" { 98 | info := NodeToWorkerInfo(res.PrevNode) 99 | log.Println("Delete worker ", info.Name) 100 | delete(m.members, info.Name) 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /etcdx/sync/go.sum: -------------------------------------------------------------------------------- 1 | github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= 2 | github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 7 | github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= 8 | github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 9 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 10 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 11 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 12 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 13 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 14 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 15 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 16 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 17 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 18 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 19 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 20 | go.etcd.io/etcd/api/v3 v3.5.7 h1:sbcmosSVesNrWOJ58ZQFitHMdncusIifYcrBfwrlJSY= 21 | go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA= 22 | go.etcd.io/etcd/client/pkg/v3 v3.5.7 h1:y3kf5Gbp4e4q7egZdn5T7W9TSHUvkClN6u+Rq9mEOmg= 23 | go.etcd.io/etcd/client/pkg/v3 v3.5.7/go.mod h1:o0Abi1MK86iad3YrWhgUsbGx1pmTS+hrORWc2CamuhY= 24 | go.etcd.io/etcd/client/v2 v2.305.7 h1:AELPkjNR3/igjbO7CjyF1fPuVPjrblliiKj+Y6xSGOU= 25 | go.etcd.io/etcd/client/v2 v2.305.7/go.mod h1:GQGT5Z3TBuAQGvgPfhR7VPySu/SudxmEkRq9BgzFU6s= 26 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 27 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 28 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= 29 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 30 | -------------------------------------------------------------------------------- /etcdx/master/master.go: -------------------------------------------------------------------------------- 1 | package master 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "time" 7 | 8 | "go.etcd.io/etcd/client/v2" 9 | ) 10 | 11 | type Master struct { 12 | key string //as key 13 | id string //as value 14 | kapi client.KeysAPI //etcdx kapi 15 | defaultTTL time.Duration // 16 | defaultHeartbeat time.Duration // 17 | } 18 | 19 | func NewMaster(key, id string, endpoints []string) *Master { 20 | cfg := client.Config{ 21 | Endpoints: endpoints, 22 | Transport: client.DefaultTransport, 23 | HeaderTimeoutPerRequest: time.Second, 24 | } 25 | 26 | etcdClient, err := client.New(cfg) 27 | if err != nil { 28 | log.Fatal("Error: cannot connec to etcdx:", err) 29 | return nil 30 | } 31 | return &Master{ 32 | kapi: client.NewKeysAPI(etcdClient), 33 | key: key, 34 | id: id, 35 | defaultTTL: time.Second * 5, 36 | defaultHeartbeat: time.Second * 3, 37 | } 38 | } 39 | 40 | func (m *Master) heartbeat() { 41 | tk := time.NewTicker(m.defaultHeartbeat) 42 | for { 43 | select { 44 | case <-tk.C: 45 | log.Println("heartbeat...k=", m.key, " v=", m.id) 46 | if _, err := m.kapi.Set(context.Background(), m.key, m.id, &client.SetOptions{ 47 | TTL: m.defaultTTL, 48 | }); err != nil { 49 | log.Println("heartbeat err:", err) 50 | } 51 | } 52 | } 53 | } 54 | 55 | // Apply for master block until success or occur an error 56 | func (m *Master) Apply() error { 57 | setOptions := &client.SetOptions{ 58 | PrevExist: client.PrevNoExist, 59 | TTL: m.defaultTTL, 60 | } 61 | for { 62 | resp, err := m.kapi.Set(context.Background(), m.key, m.id, setOptions) 63 | if err == nil { 64 | go m.heartbeat() 65 | return nil 66 | } 67 | 68 | e, ok := err.(client.Error) 69 | if !ok { 70 | return err 71 | } 72 | if e.Code != client.ErrorCodeNodeExist { 73 | return err 74 | } 75 | resp, err = m.kapi.Get(context.Background(), m.key, nil) 76 | if err != nil { 77 | return err 78 | } 79 | log.Println("Apply failed,watch ", m.key) 80 | watcherOptions := &client.WatcherOptions{ 81 | AfterIndex: resp.Index, 82 | Recursive: false, 83 | } 84 | watcher := m.kapi.Watcher(m.key, watcherOptions) 85 | for { 86 | resp, err = watcher.Next(context.Background()) 87 | if err != nil { 88 | return err 89 | } 90 | if resp.Action == "set" { 91 | log.Println("watching event:", resp.Action, " key:", m.key, " value:", resp.Node.Value) 92 | } 93 | if resp.Action == "delete" || resp.Action == "expire" { 94 | log.Println("watching event:", resp.Action, " key:", m.key, " value:", resp.PrevNode.Value) 95 | break 96 | } 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /container/bloom/bloom9.go: -------------------------------------------------------------------------------- 1 | // bloom https://github.com/ethereum/go-ethereum/core/types/bloom9.go 2 | package bloom 3 | 4 | import ( 5 | "encoding/hex" 6 | "errors" 7 | "fmt" 8 | "golang.org/x/crypto/sha3" 9 | "math/big" 10 | ) 11 | 12 | type bytesBacked interface { 13 | Bytes() []byte 14 | } 15 | 16 | const ( 17 | // BloomByteLength represents the number of bytes used in a header log bloom. 18 | BloomByteLength = 256 19 | 20 | // BloomBitLength represents the number of bits used in a header log bloom. 21 | BloomBitLength = 8 * BloomByteLength 22 | ) 23 | 24 | // Bloom represents a 2048 bit bloom filter. 25 | type Bloom [BloomByteLength]byte 26 | 27 | // BytesToBloom converts a byte slice to a bloom filter. 28 | // It panics if b is not of suitable size. 29 | func BytesToBloom(b []byte) Bloom { 30 | var bloom Bloom 31 | bloom.SetBytes(b) 32 | return bloom 33 | } 34 | 35 | // SetBytes sets the content of b to the given bytes. 36 | // It panics if d is not of suitable size. 37 | func (b *Bloom) SetBytes(d []byte) { 38 | if len(b) < len(d) { 39 | panic(fmt.Sprintf("bloom bytes too big %d %d", len(b), len(d))) 40 | } 41 | copy(b[BloomByteLength-len(d):], d) 42 | } 43 | 44 | // Add adds d to the filter. Future calls of Test(d) will return true. 45 | func (b *Bloom) Add(d *big.Int) { 46 | bin := new(big.Int).SetBytes(b[:]) 47 | bin.Or(bin, bloom9(d.Bytes())) 48 | b.SetBytes(bin.Bytes()) 49 | } 50 | 51 | // Big converts b to a big integer. 52 | func (b Bloom) Big() *big.Int { 53 | return new(big.Int).SetBytes(b[:]) 54 | } 55 | 56 | func (b Bloom) Bytes() []byte { 57 | return b[:] 58 | } 59 | 60 | func (b Bloom) Test(test *big.Int) bool { 61 | return BloomLookup(b, test) 62 | } 63 | 64 | func (b Bloom) TestBytes(test []byte) bool { 65 | return b.Test(new(big.Int).SetBytes(test)) 66 | 67 | } 68 | 69 | // MarshalText encodes b as a hex string with 0x prefix. 70 | func (b Bloom) MarshalText() ([]byte, error) { 71 | result := make([]byte, len(b)*2+2) 72 | copy(result, `0x`) 73 | hex.Encode(result[2:], b[:]) 74 | return result, nil 75 | } 76 | 77 | // UnmarshalText b as a hex string with 0x prefix. 78 | func (b *Bloom) UnmarshalText(input []byte) error { 79 | if len(input) != 514 { 80 | return errors.New("err bloom hex") 81 | } 82 | _, err := hex.Decode(b[:], input[2:]) 83 | return err 84 | } 85 | 86 | func bloom9(b []byte) *big.Int { 87 | b = sha3.NewLegacyKeccak256().Sum(b) 88 | r := new(big.Int) 89 | for i := 0; i < 6; i += 2 { 90 | t := big.NewInt(1) 91 | b := (uint(b[i+1]) + (uint(b[i]) << 8)) & 2047 92 | r.Or(r, t.Lsh(t, b)) 93 | } 94 | return r 95 | } 96 | 97 | var Bloom9 = bloom9 98 | 99 | func BloomLookup(bin Bloom, topic bytesBacked) bool { 100 | bloom := bin.Big() 101 | cmp := bloom9(topic.Bytes()) 102 | 103 | return bloom.And(bloom, cmp).Cmp(cmp) == 0 104 | } 105 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # utils 2 | 3 | ``` 4 | ├── codec # 一种数据混淆编码方法 5 | │   ├── codec.go 6 | │   ├── codec_test.go 7 | │   └── README.md 8 | ├── container # 常用容器 9 | │   ├── bitset 10 | │   │   ├── bitset.go 11 | │   │   └── bitset_test.go 12 | │   ├── blocking_queue 13 | │   │   ├── blocking_queue.go 14 | │   │   └── blocking_queue_test.go 15 | │   ├── bloom 16 | │   │   ├── bloom9.go 17 | │   │   └── bloom9_test.go 18 | │   ├── fifo 19 | │   │   ├── fifo.go 20 | │   │   └── fifo_test.go 21 | │   ├── omap 22 | │   │   ├── omap.go 23 | │   │   └── omap_test.go 24 | │   └── pqueue 25 | │   ├── priority_queue.go 26 | │   ├── priority_queue_test.go 27 | │   └── README.md 28 | ├── crypto # AES,DES加密解密 29 | │   ├── aes256cbc.go 30 | │   ├── aes256cbc_test.go 31 | │   ├── aes.go 32 | │   ├── PKCS.go 33 | │   ├── tripledes_b_test.go 34 | │   ├── tripledes.go 35 | │   └── tripledes_test.go 36 | ├── etcdx # 基于etch的服务发行、选主、分布式锁 37 | │   ├── discovery 38 | │   │   ├── go.mod 39 | │   │   ├── master.go 40 | │   │   └── worker.go 41 | │   ├── master 42 | │   │   ├── go.mod 43 | │   │   └── master.go 44 | │   ├── readme.md 45 | │   └── sync 46 | │   ├── go.mod 47 | │   ├── sync.go 48 | │   └── sync_test.go 49 | ├── hash # 常用hash方法和一致性hash 50 | │   ├── cityhash 51 | │   │   ├── cityhash.go 52 | │   │   └── cityhash_test.go 53 | │   ├── hash.go 54 | │   ├── ketama 55 | │   │   ├── ketama.go 56 | │   │   └── ketama_test.go 57 | │   └── murmurhash3 58 | │   ├── mmhash3.go 59 | │   └── mmhash3_test.go 60 | ├── httpx # http sign 61 | │   ├── go.mod 62 | │   ├── README.md 63 | │   ├── sign.go 64 | │   └── sign_test.go 65 | ├── log # 封装基于zap.Logger的日志 66 | │   ├── go.mod 67 | │   ├── zap.go 68 | │   └── zap_test.go 69 | ├── natx # nat使用封装 70 | │   ├── defaultApp.go 71 | │   ├── defaultApp_test.go 72 | │   ├── go.mod 73 | │   ├── msgpack_enc.go 74 | │   ├── natc.go 75 | │   ├── natx.go 76 | │   └── README.md 77 | ├── nsqx # nsq使用封装 78 | │   ├── consumer.go 79 | │   ├── go.mod 80 | │   └── producer.go 81 | ├── pool # 内存分配与优化 82 | │   ├── allocator 83 | │   │   ├── alloc.go 84 | │   │   └── alloc_test.go 85 | │   ├── xbufio 86 | │   │   └── buffio.go 87 | │   ├── xbytes 88 | │   │   ├── bytes.go 89 | │   │   └── writer.go 90 | │   └── xtime # 基于内存小根堆定时器，扩展了timer func方法:func(interface{}) 91 | │   └── xtime.go 92 | ├── pprof # http pprof 93 | │   ├── pprof_http.go 94 | │   └── pprof_http_test.go 95 | ├── random # 随机字符串 96 | │   └── string.go 97 | ├── README.md 98 | ├── redisx # redis常用封装和乐观锁 99 | │   ├── go.mod 100 | │   ├── mutex.go 101 | │   ├── redis.go 102 | │   └── redis_test.go 103 | ├── referral # 邀请码 104 | │   ├── referral.go 105 | │   └── refferal_test.go 106 | ├── sign # hmac签名 107 | │   ├── hmac.go 108 | │   └── hmac_test.go 109 | ├── timerx # 定时器 110 | │   ├── minheap # 小根堆 111 | │   │   ├── timer.go 112 | │   │   └── timer_test.go 113 | │   └── wheel # 时间轮 114 | │   ├── timer.go 115 | │   └── timer_test.go 116 | └── validator # 参数验证 117 | └── validator.go 118 | ``` -------------------------------------------------------------------------------- /redisx/redis.go: -------------------------------------------------------------------------------- 1 | package redisx 2 | 3 | import ( 4 | "errors" 5 | "github.com/garyburd/redigo/redis" 6 | "time" 7 | ) 8 | 9 | type Rds struct { 10 | pool *redis.Pool 11 | } 12 | 13 | func New(dial, password string, db int) *Rds { 14 | pool := initPool(dial, password, db) 15 | return &Rds{ 16 | pool: pool, 17 | } 18 | } 19 | 20 | func initPool(dial, password string, db int) *redis.Pool { 21 | return &redis.Pool{ 22 | MaxIdle: 100, 23 | IdleTimeout: 120 * time.Second, 24 | Dial: func() (redis.Conn, error) { 25 | c, err := redis.Dial("tcp", dial) 26 | if err != nil { 27 | return nil, err 28 | } 29 | if _, err := c.Do("AUTH", password); err != nil { 30 | c.Close() 31 | return nil, err 32 | } 33 | if _, err := c.Do("SELECT", db); err != nil { 34 | c.Close() 35 | return nil, err 36 | } 37 | return c, err 38 | }, 39 | TestOnBorrow: func(c redis.Conn, t time.Time) error { 40 | if time.Since(t) < time.Minute { 41 | return nil 42 | } 43 | _, err := c.Do("PING") 44 | return err 45 | }, 46 | } 47 | } 48 | 49 | // Lock 锁住key 50 | func (p *Rds) Lock(key string, ttl int) error { 51 | conn := p.pool.Get() 52 | defer conn.Close() 53 | mu := &Mutex{ 54 | key: key, 55 | } 56 | 57 | reply, err := redis.String(conn.Do("SET", mu.Key(), mu.Key(), "EX", ttl, "NX")) 58 | if err != nil { 59 | return err 60 | } 61 | if reply != "OK" { 62 | return errors.New("xxx" + reply) 63 | } 64 | return err 65 | } 66 | 67 | // UnLock 解锁key 68 | func (p *Rds) UnLock(key string) (err error) { 69 | conn := p.pool.Get() 70 | defer conn.Close() 71 | mu := &Mutex{ 72 | key: key, 73 | } 74 | _, err = conn.Do("del", mu.Key()) 75 | return 76 | } 77 | 78 | func (p *Rds) Get(key string) (v []byte, err error) { 79 | conn := p.pool.Get() 80 | defer conn.Close() 81 | v, err = redis.Bytes(conn.Do("GET", key)) 82 | if err == redis.ErrNil { 83 | return v, nil 84 | } 85 | return 86 | } 87 | 88 | func (p *Rds) Set(key string, value []byte) (err error) { 89 | conn := p.pool.Get() 90 | defer conn.Close() 91 | _, err = redis.String(conn.Do("SET", key, value)) 92 | return 93 | } 94 | 95 | func (p *Rds) SetNX(k string, v interface{}) (err error) { 96 | conn := p.pool.Get() 97 | defer conn.Close() 98 | _, err = redis.Int64(conn.Do("SETNX", k, v)) 99 | return 100 | } 101 | 102 | func (p *Rds) SetEX(k string, expire int, v interface{}) (err error) { 103 | conn := p.pool.Get() 104 | defer conn.Close() 105 | _, err = redis.Bytes(conn.Do("SETEX", k, expire, v)) 106 | return 107 | } 108 | 109 | func (p *Rds) Del(k string) (err error) { 110 | conn := p.pool.Get() 111 | defer conn.Close() 112 | _, err = conn.Do("DEL", k) 113 | return 114 | } 115 | 116 | func (p *Rds) Keys() (keys []string, err error) { 117 | conn := p.pool.Get() 118 | defer conn.Close() 119 | keys, err = redis.Strings(conn.Do("KEYS", "*")) 120 | return 121 | } 122 | 123 | func (p *Rds) FlushDB() (err error) { 124 | conn := p.pool.Get() 125 | defer conn.Close() 126 | _, err = conn.Do("FLUSHDB") 127 | return 128 | } 129 | -------------------------------------------------------------------------------- /natx/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 3 | github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 4 | github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= 5 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 6 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 7 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 8 | github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 9 | github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 10 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 11 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 12 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 13 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 14 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 15 | github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU= 16 | github.com/nats-io/nats.go v1.36.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= 17 | github.com/nats-io/nkeys v0.4.10 h1:glmRrpCmYLHByYcePvnTBEAwawwapjCPMjy2huw20wc= 18 | github.com/nats-io/nkeys v0.4.10/go.mod h1:OjRrnIKnWBFl+s4YK5ChQfvHP2fxqZexrKJoVVyWB3U= 19 | github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= 20 | github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= 21 | github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= 22 | github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= 23 | github.com/vmihailenco/tagparser v0.1.1 h1:quXMXlA39OCbd2wAdTsGDlK9RkOk6Wuw+x37wVyIuWY= 24 | github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= 25 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 26 | golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= 27 | golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= 28 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 29 | golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 30 | golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= 31 | golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= 32 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 33 | golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= 34 | golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 35 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 36 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 37 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 38 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 39 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 40 | google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= 41 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 42 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 43 | google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= 44 | google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 45 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 46 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 47 | -------------------------------------------------------------------------------- /etcdx/sync/sync_test.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "testing" 7 | "time" 8 | 9 | "go.etcd.io/etcd/client/v2" 10 | ) 11 | 12 | func newKeysAPI(machines []string) client.KeysAPI { 13 | cfg := client.Config{ 14 | Endpoints: machines, 15 | Transport: client.DefaultTransport, 16 | HeaderTimeoutPerRequest: time.Second, 17 | } 18 | 19 | c, err := client.New(cfg) 20 | if err != nil { 21 | return nil 22 | } 23 | 24 | return client.NewKeysAPI(c) 25 | } 26 | 27 | func checkKeyExists(key string, kapi client.KeysAPI) bool { 28 | // Get the already node's value. 29 | _, err := kapi.Get(context.TODO(), key, nil) 30 | if err != nil { 31 | return false 32 | } 33 | return true 34 | } 35 | 36 | func TestMutex(t *testing.T) { 37 | log.SetFlags(log.Ltime | log.Ldate | log.Lshortfile) 38 | lockKey := "/etcdsync" 39 | machines := []string{"http://127.0.0.1:2379"} 40 | kapi := newKeysAPI(machines) 41 | m, err := New(lockKey, 60, machines) 42 | if err != nil { 43 | t.Error(err) 44 | return 45 | } 46 | if m == nil { 47 | t.Errorf("New Mutex ERROR") 48 | } 49 | err = m.Lock() 50 | if err != nil { 51 | t.Errorf("failed") 52 | } 53 | 54 | if checkKeyExists(lockKey, kapi) == false { 55 | t.Errorf("The mutex have been locked but the key node does not exists.") 56 | t.Fail() 57 | } 58 | //do something here 59 | 60 | err = m.Unlock() 61 | if err != nil { 62 | t.Errorf("failed") 63 | } 64 | 65 | _, err = m.kapi.Get(context.Background(), lockKey, nil) 66 | if e, ok := err.(client.Error); !ok { 67 | t.Errorf("Get key %v failed from etcdx", lockKey) 68 | } else if e.Code != client.ErrorCodeKeyNotFound { 69 | t.Errorf("ERROR %v", err) 70 | } 71 | } 72 | 73 | func TestLockConcurrently(t *testing.T) { 74 | slice := make([]int, 0, 3) 75 | lockKey := "/etcd_sync" 76 | machines := []string{"http://127.0.0.1:2379"} 77 | kapi := newKeysAPI(machines) 78 | m1, err := New(lockKey, 60, machines) 79 | m2, err := New(lockKey, 60, machines) 80 | m3, err := New(lockKey, 60, machines) 81 | if err != nil { 82 | t.Error(err) 83 | return 84 | } 85 | if m1 == nil || m2 == nil || m3 == nil { 86 | t.Errorf("New Mutex ERROR") 87 | } 88 | m1.Lock() 89 | if checkKeyExists(lockKey, kapi) == false { 90 | t.Errorf("The mutex have been locked but the key node does not exists.") 91 | t.Fail() 92 | } 93 | ch1 := make(chan bool) 94 | go func() { 95 | ch2 := make(chan bool) 96 | m2.Lock() 97 | if checkKeyExists(lockKey, kapi) == false { 98 | t.Errorf("The mutex have been locked but the key node does not exists.") 99 | t.Fail() 100 | } 101 | go func() { 102 | m3.Lock() 103 | if checkKeyExists(lockKey, kapi) == false { 104 | t.Errorf("The mutex have been locked but the key node does not exists.") 105 | t.Fail() 106 | } 107 | slice = append(slice, 2) 108 | m3.Unlock() 109 | ch2 <- true 110 | }() 111 | slice = append(slice, 1) 112 | time.Sleep(1 * time.Second) 113 | m2.Unlock() 114 | <-ch2 115 | ch1 <- true 116 | }() 117 | slice = append(slice, 0) 118 | time.Sleep(1 * time.Second) 119 | m1.Unlock() 120 | <-ch1 121 | if len(slice) != 3 { 122 | t.Fail() 123 | } 124 | for n, i := range slice { 125 | if n != i { 126 | t.Fail() 127 | } 128 | } 129 | } 130 | 131 | func TestLockTimeout(t *testing.T) { 132 | slice := make([]int, 0, 2) 133 | m1, err := New("key", 2, []string{"http://127.0.0.1:2379"}) 134 | m2, err := New("key", 2, []string{"http://127.0.0.1:2379"}) 135 | if err != nil { 136 | t.Error(err) 137 | return 138 | } 139 | m1.Lock() 140 | ch := make(chan bool) 141 | go func() { 142 | m2.Lock() 143 | slice = append(slice, 1) 144 | m2.Unlock() 145 | ch <- true 146 | }() 147 | slice = append(slice, 0) 148 | <-ch 149 | for n, i := range slice { 150 | if n != i { 151 | t.Fail() 152 | } 153 | } 154 | } 155 | 156 | func TestRefreshLockTTL(t *testing.T) { 157 | lockKey := "/etcd_sync" 158 | machines := []string{"http://127.0.0.1:2379"} 159 | kapi := newKeysAPI(machines) 160 | m, err := New(lockKey, 10, machines) 161 | if err != nil { 162 | t.Error(err) 163 | return 164 | } 165 | m.Lock() 166 | if checkKeyExists(lockKey, kapi) == false { 167 | t.Errorf("The mutex have been refreshed but the key node does not exists.") 168 | t.Fail() 169 | } 170 | time.Sleep(5 * time.Second) 171 | m.RefreshLockTTL(10 * time.Second) 172 | time.Sleep(5 * time.Second) 173 | if checkKeyExists(lockKey, kapi) == false { 174 | t.Errorf("The mutex's TTL has been refreshed but the key node still expired.") 175 | t.Fail() 176 | } 177 | m.Unlock() 178 | } 179 | -------------------------------------------------------------------------------- /container/pqueue/priority_queue_test.go: -------------------------------------------------------------------------------- 1 | package pqueue 2 | 3 | import ( 4 | "math/rand" 5 | "sort" 6 | "testing" 7 | ) 8 | 9 | type Int int 10 | 11 | func (this Int) Less(other interface{}) bool { 12 | return this < other.(Int) 13 | } 14 | 15 | type IntSorter []Int 16 | 17 | func (s *IntSorter) Len() int { 18 | return len(*s) 19 | } 20 | 21 | func (s *IntSorter) Less(i, j int) bool { 22 | return (*s)[i] < (*s)[j] 23 | } 24 | 25 | func (s *IntSorter) Swap(i, j int) { 26 | (*s)[i], (*s)[j] = (*s)[j], (*s)[i] 27 | } 28 | 29 | func TestInt(t *testing.T) { 30 | q := New() 31 | 32 | if q.Len() != 0 { 33 | t.Fatal() 34 | } 35 | 36 | q.Push(Int(-1)) 37 | for i := 0; i < 998; i++ { 38 | q.Push(Int(rand.Intn(100))) 39 | } 40 | q.Push(Int(5201314)) 41 | 42 | s := new(IntSorter) 43 | n := 1000 44 | for q.Len() > 0 { 45 | if q.Len() != n { 46 | t.Fatal() 47 | } 48 | n-- 49 | 50 | x := q.Top() 51 | y := q.Pop() 52 | if x != y || x.(Int) != y.(Int) { 53 | t.Fatal() 54 | } 55 | 56 | *s = append(*s, x.(Int)) 57 | } 58 | 59 | if (*s)[0] != -1 || (*s)[999] != 5201314 { 60 | t.Fatal() 61 | } 62 | 63 | if !sort.IsSorted(s) { 64 | t.Fatal() 65 | } 66 | 67 | q = New() 68 | 69 | if q.Len() != 0 { 70 | t.Fatal() 71 | } 72 | } 73 | 74 | func TestFixAndRemove(t *testing.T) { 75 | q := New() 76 | q.Push(Int(1)) 77 | q.Push(Int(3)) 78 | q.Push(Int(2)) 79 | q.Push(Int(4)) 80 | 81 | if (*q.s)[0].(Int) != 1 || (*q.s)[1].(Int) != 3 || (*q.s)[2].(Int) != 2 || (*q.s)[3].(Int) != 4 { 82 | t.Fatal() 83 | } 84 | 85 | q.Fix(Int(5), 1) 86 | 87 | if (*q.s)[0].(Int) != 1 || (*q.s)[1].(Int) != 4 || (*q.s)[2].(Int) != 2 || (*q.s)[3].(Int) != 5 { 88 | t.Fatal() 89 | } 90 | 91 | a, b, c, d := q.Pop(), q.Pop(), q.Pop(), q.Pop() 92 | if a.(Int) != 1 || b.(Int) != 2 || c.(Int) != 4 || d.(Int) != 5 { 93 | t.Fatal() 94 | } 95 | 96 | if q.Len() != 0 { 97 | t.Fatal() 98 | } 99 | 100 | q.Push(Int(8)) 101 | q.Push(Int(6)) 102 | q.Push(Int(7)) 103 | q.Push(Int(9)) 104 | 105 | // println((*q.s)[0].(Int), (*q.s)[1].(Int), (*q.s)[2].(Int), (*q.s)[3].(Int)) 106 | 107 | if (*q.s)[0].(Int) != 6 || (*q.s)[1].(Int) != 8 || (*q.s)[2].(Int) != 7 || (*q.s)[3].(Int) != 9 { 108 | t.Fatal() 109 | } 110 | 111 | if q.Top().(Int) != 6 { 112 | t.Fatal() 113 | } 114 | 115 | q.Remove(0) 116 | 117 | if q.Top().(Int) != 7 { 118 | t.Fatal() 119 | } 120 | 121 | a, b, c = q.Pop(), q.Pop(), q.Pop() 122 | if a.(Int) != 7 || b.(Int) != 8 || c.(Int) != 9 { 123 | t.Fatal() 124 | } 125 | } 126 | 127 | type Node struct { 128 | priority int 129 | value int 130 | } 131 | 132 | func NewNode(p, v int) *Node { 133 | return &Node{priority: p} 134 | } 135 | 136 | func (this *Node) Less(other interface{}) bool { 137 | return this.priority < other.(*Node).priority 138 | } 139 | 140 | func TestStruct(t *testing.T) { 141 | q := New() 142 | for i := 0; i < 1000; i++ { 143 | q.Push(NewNode(rand.Intn(100000), i)) 144 | } 145 | 146 | n := 1000 147 | for q.Len() > 0 { 148 | if q.Len() != n { 149 | t.Fatal() 150 | } 151 | n-- 152 | 153 | x := q.Top().(*Node) 154 | y := q.Pop().(*Node) 155 | 156 | if x.priority != y.priority || x.value != y.value { 157 | t.Fatal() 158 | } 159 | } 160 | } 161 | 162 | func BenchmarkPush1(b *testing.B) { 163 | b.StopTimer() 164 | q := New() 165 | b.StartTimer() 166 | for i := 0; i < 100000; i++ { 167 | q.Push(NewNode(rand.Intn(100), i)) 168 | } 169 | } 170 | 171 | func BenchmarkPush2(b *testing.B) { 172 | b.StopTimer() 173 | q := New() 174 | b.StartTimer() 175 | for i := 0; i < 500000; i++ { 176 | q.Push(NewNode(rand.Intn(100), i)) 177 | } 178 | } 179 | 180 | func BenchmarkPush3(b *testing.B) { 181 | b.StopTimer() 182 | q := New() 183 | b.StartTimer() 184 | for i := 0; i < 1000000; i++ { 185 | q.Push(NewNode(rand.Intn(100), i)) 186 | } 187 | } 188 | 189 | func BenchmarkPop1(b *testing.B) { 190 | b.StopTimer() 191 | q := New() 192 | for i := 0; i < 100000; i++ { 193 | q.Push(NewNode(rand.Intn(100), i)) 194 | } 195 | 196 | b.StartTimer() 197 | for q.Len() > 0 { 198 | q.Pop() 199 | } 200 | } 201 | 202 | func BenchmarkPop2(b *testing.B) { 203 | b.StopTimer() 204 | q := New() 205 | for i := 0; i < 500000; i++ { 206 | q.Push(NewNode(rand.Intn(100), i)) 207 | } 208 | 209 | b.StartTimer() 210 | for q.Len() > 0 { 211 | q.Pop() 212 | } 213 | } 214 | 215 | func BenchmarkPop3(b *testing.B) { 216 | b.StopTimer() 217 | q := New() 218 | for i := 0; i < 1000000; i++ { 219 | q.Push(NewNode(rand.Intn(100), i)) 220 | } 221 | 222 | b.StartTimer() 223 | for q.Len() > 0 { 224 | q.Pop() 225 | } 226 | } 227 | -------------------------------------------------------------------------------- /timerx/wheel/timer.go: -------------------------------------------------------------------------------- 1 | // timer 基于时间轮的定时器服务，精度为10ms 2 | // refer:http://blog.csdn.net/yueguanghaidao/article/details/46290539 3 | // 修改内容：为定时器增加类型和参数属性，修改回调函数类型，更通用 4 | // 使用方法 5 | // tm = timer.New(timerx.Millisecond * 10) 6 | // go tm.Start() 7 | // tm.NewTimer(timerx.Second*60, ProcTimer, TIMER_TYPE_DEFAULT, "") 8 | // func ProcTimer(tp int, arg interface{}){switch tp { case TIMER_TYPE_DEFAULT:}} 9 | 10 | package timer 11 | 12 | import ( 13 | "container/list" 14 | "fmt" 15 | "sync" 16 | "time" 17 | ) 18 | 19 | //referer https://github.com/cloudwu/skynet/blob/master/skynet-src/skynet_timer.c 20 | 21 | const ( 22 | TIME_NEAR_SHIFT = 8 23 | TIME_NEAR = 1 << TIME_NEAR_SHIFT 24 | TIME_LEVEL_SHIFT = 6 25 | TIME_LEVEL = 1 << TIME_LEVEL_SHIFT 26 | TIME_NEAR_MASK = TIME_NEAR - 1 27 | TIME_LEVEL_MASK = TIME_LEVEL - 1 28 | ) 29 | 30 | type Timer struct { 31 | near [TIME_NEAR]*list.List //256 32 | t [4][TIME_LEVEL]*list.List //64 33 | sync.Mutex 34 | time uint32 35 | tick time.Duration 36 | quit chan struct{} 37 | } 38 | 39 | type Node struct { 40 | expire uint32 41 | t int 42 | arg interface{} 43 | f func(int, interface{}) 44 | } 45 | 46 | var T *Timer 47 | 48 | func init() { 49 | timer := New(time.Millisecond * 10) 50 | T = timer 51 | //fmt.Println("start timer server...") 52 | go T.Start() 53 | } 54 | 55 | func (n *Node) String() string { 56 | return fmt.Sprintf("Node:expire,%d,t:%d,arg:%s", n.expire, n.t, n.arg) 57 | } 58 | 59 | func New(d time.Duration) *Timer { 60 | t := new(Timer) 61 | t.time = 0 62 | t.tick = d 63 | t.quit = make(chan struct{}) 64 | 65 | var i, j int 66 | for i = 0; i < TIME_NEAR; i++ { 67 | t.near[i] = list.New() 68 | } 69 | 70 | for i = 0; i < 4; i++ { 71 | for j = 0; j < TIME_LEVEL; j++ { 72 | t.t[i][j] = list.New() 73 | } 74 | } 75 | 76 | return t 77 | } 78 | 79 | func (t *Timer) addNode(n *Node) { 80 | expire := n.expire 81 | current := t.time 82 | if (expire | TIME_NEAR_MASK) == (current | TIME_NEAR_MASK) { 83 | t.near[expire&TIME_NEAR_MASK].PushBack(n) 84 | } else { 85 | var i uint32 86 | var mask uint32 = TIME_NEAR << TIME_LEVEL_SHIFT 87 | for i = 0; i < 3; i++ { 88 | if (expire | (mask - 1)) == (current | (mask - 1)) { 89 | break 90 | } 91 | mask <<= TIME_LEVEL_SHIFT 92 | } 93 | 94 | t.t[i][(expire>>(TIME_NEAR_SHIFT+i*TIME_LEVEL_SHIFT))&TIME_LEVEL_MASK].PushBack(n) 95 | } 96 | 97 | } 98 | 99 | func (t *Timer) NewTimer(d time.Duration, f func(int, interface{}), tp int, arg interface{}) *Node { 100 | n := new(Node) 101 | n.f = f 102 | n.t = tp 103 | n.arg = arg 104 | t.Lock() 105 | n.expire = uint32(d/t.tick) + t.time 106 | t.addNode(n) 107 | t.Unlock() 108 | return n 109 | } 110 | 111 | func (t *Timer) String() string { 112 | return fmt.Sprintf("Timer:timerx:%d, tick:%s", t.time, t.tick) 113 | } 114 | 115 | func dispatchList(front *list.Element) { 116 | for e := front; e != nil; e = e.Next() { 117 | node := e.Value.(*Node) 118 | go node.f(node.t, node.arg) 119 | } 120 | } 121 | 122 | func (t *Timer) moveList(level, idx int) { 123 | vec := t.t[level][idx] 124 | front := vec.Front() 125 | vec.Init() 126 | for e := front; e != nil; e = e.Next() { 127 | node := e.Value.(*Node) 128 | t.addNode(node) 129 | } 130 | } 131 | 132 | func (t *Timer) shift() { 133 | t.Lock() 134 | var mask uint32 = TIME_NEAR 135 | t.time++ 136 | ct := t.time 137 | if ct == 0 { 138 | t.moveList(3, 0) 139 | } else { 140 | time := ct >> TIME_NEAR_SHIFT 141 | var i int = 0 142 | for (ct & (mask - 1)) == 0 { 143 | idx := int(time & TIME_LEVEL_MASK) 144 | if idx != 0 { 145 | t.moveList(i, idx) 146 | break 147 | } 148 | mask <<= TIME_LEVEL_SHIFT 149 | time >>= TIME_LEVEL_SHIFT 150 | i++ 151 | } 152 | } 153 | t.Unlock() 154 | } 155 | 156 | func (t *Timer) execute() { 157 | t.Lock() 158 | idx := t.time & TIME_NEAR_MASK 159 | vec := t.near[idx] 160 | if vec.Len() > 0 { 161 | front := vec.Front() 162 | vec.Init() 163 | t.Unlock() 164 | // dispatch_list don't need lock 165 | dispatchList(front) 166 | return 167 | } 168 | 169 | t.Unlock() 170 | } 171 | 172 | func (t *Timer) update() { 173 | // try to dispatch timeout 0 (rare condition) 174 | t.execute() 175 | 176 | // shift timerx first, and then dispatch timer message 177 | t.shift() 178 | 179 | t.execute() 180 | 181 | } 182 | 183 | func (t *Timer) Start() { 184 | tick := time.NewTicker(t.tick) 185 | defer tick.Stop() 186 | for { 187 | select { 188 | case <-tick.C: 189 | t.update() 190 | case <-t.quit: 191 | return 192 | } 193 | } 194 | } 195 | 196 | func (t *Timer) Stop() { 197 | close(t.quit) 198 | } 199 | -------------------------------------------------------------------------------- /etcdx/sync/sync.go: -------------------------------------------------------------------------------- 1 | // fork from github.com/zieckey/etcdsync 2 | package sync 3 | 4 | import ( 5 | "context" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "os" 10 | "sync" 11 | "time" 12 | 13 | "go.etcd.io/etcd/client/v2" 14 | ) 15 | 16 | const ( 17 | defaultTTL = 60 18 | defaultTry = 3 19 | deleteAction = "delete" 20 | expireAction = "expire" 21 | ) 22 | 23 | // A Mutex is a mutual exclusion lock which is distributed across a cluster. 24 | type Mutex struct { 25 | key string 26 | id string // The identity of the caller 27 | client client.Client 28 | kapi client.KeysAPI 29 | ctx context.Context 30 | ttl time.Duration 31 | mutex *sync.Mutex 32 | logger io.Writer 33 | } 34 | 35 | // New creates a Mutex with the given key which must be the same 36 | // across the cluster nodes. 37 | // machines are the ectd cluster addresses 38 | func New(key string, ttl int, machines []string) (*Mutex, error) { 39 | cfg := client.Config{ 40 | Endpoints: machines, 41 | Transport: client.DefaultTransport, 42 | HeaderTimeoutPerRequest: time.Second, 43 | } 44 | 45 | c, err := client.New(cfg) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | hostname, err := os.Hostname() 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | if len(key) == 0 || len(machines) == 0 { 56 | return nil, errors.New("wrong lock key or empty machines") 57 | } 58 | 59 | if key[0] != '/' { 60 | key = "/" + key 61 | } 62 | 63 | if ttl < 1 { 64 | ttl = defaultTTL 65 | } 66 | 67 | return &Mutex{ 68 | key: key, 69 | id: fmt.Sprintf("%v-%v-%v", hostname, os.Getpid(), time.Now().Format("20060102-15:04:05.999999999")), 70 | client: c, 71 | kapi: client.NewKeysAPI(c), 72 | ctx: context.TODO(), 73 | ttl: time.Second * time.Duration(ttl), 74 | mutex: new(sync.Mutex), 75 | }, nil 76 | } 77 | 78 | // Lock locks m. 79 | // If the lock is already in use, return err. 80 | func (m *Mutex) Lock() (err error) { 81 | m.mutex.Lock() 82 | return m.lock_failfast() 83 | } 84 | 85 | // LockNonBlock locks m. 86 | // If the lock is already in use, the calling goroutine 87 | // blocks until the mutex is available. 88 | func (m *Mutex) LockBlocking() (err error) { 89 | m.mutex.Lock() 90 | for try := 1; try <= defaultTry; try++ { 91 | err = m.lock() 92 | if err == nil { 93 | return nil 94 | } 95 | 96 | m.debug("Lock node %v ERROR %v", m.key, err) 97 | if try < defaultTry { 98 | m.debug("Try to lock node %v again", m.key) 99 | } 100 | } 101 | return err 102 | } 103 | 104 | func (m *Mutex) lock() (err error) { 105 | m.debug("Trying to create a node : key=%v", m.key) 106 | setOptions := &client.SetOptions{ 107 | PrevExist: client.PrevNoExist, 108 | TTL: m.ttl, 109 | } 110 | for { 111 | resp, err := m.kapi.Set(m.ctx, m.key, m.id, setOptions) 112 | if err == nil { 113 | m.debug("Create node %v OK [%q]", m.key, resp) 114 | return nil 115 | } 116 | m.debug("Create node %v failed [%v]", m.key, err) 117 | e, ok := err.(client.Error) 118 | if !ok { 119 | return err 120 | } 121 | 122 | if e.Code != client.ErrorCodeNodeExist { 123 | return err 124 | } 125 | 126 | // Get the already node's value. 127 | resp, err = m.kapi.Get(m.ctx, m.key, nil) 128 | if err != nil { 129 | m.debug("Get node err [%v]", err) 130 | continue 131 | } 132 | m.debug("Get node %v OK", m.key) 133 | watcherOptions := &client.WatcherOptions{ 134 | AfterIndex: resp.Index, 135 | Recursive: false, 136 | } 137 | watcher := m.kapi.Watcher(m.key, watcherOptions) 138 | for { 139 | m.debug("Watching %v ...", m.key) 140 | resp, err = watcher.Next(m.ctx) 141 | if err != nil { 142 | return err 143 | } 144 | 145 | m.debug("Received an event : %q", resp) 146 | if resp.Action == deleteAction || resp.Action == expireAction { 147 | // break this for-loop, and try to create the node again. 148 | break 149 | } 150 | } 151 | } 152 | return err 153 | } 154 | 155 | func (m *Mutex) lock_failfast() error { 156 | m.debug("Trying to create a node : key=%v", m.key) 157 | setOptions := &client.SetOptions{ 158 | PrevExist: client.PrevNoExist, 159 | TTL: m.ttl, 160 | } 161 | 162 | resp, err := m.kapi.Set(m.ctx, m.key, m.id, setOptions) 163 | if err == nil { 164 | m.debug("Create node %v OK [%q]", m.key, resp) 165 | return nil 166 | } 167 | 168 | return err 169 | } 170 | 171 | // Unlock unlocks m. 172 | // It is a run-timerx error if m is not locked on entry to Unlock. 173 | // 174 | // A locked Mutex is not associated with a particular goroutine. 175 | // It is allowed for one goroutine to lock a Mutex and then 176 | // arrange for another goroutine to unlock it. 177 | func (m *Mutex) Unlock() (err error) { 178 | defer m.mutex.Unlock() 179 | for i := 1; i <= defaultTry; i++ { 180 | var resp *client.Response 181 | resp, err = m.kapi.Delete(m.ctx, m.key, nil) 182 | if err == nil { 183 | m.debug("Delete %v OK", m.key) 184 | return nil 185 | } 186 | m.debug("Delete %v falied: %q", m.key, resp) 187 | e, ok := err.(client.Error) 188 | if ok && e.Code == client.ErrorCodeKeyNotFound { 189 | return nil 190 | } 191 | } 192 | return err 193 | } 194 | 195 | func (m *Mutex) RefreshLockTTL(ttl time.Duration) (err error) { 196 | setOptions := &client.SetOptions{ 197 | PrevExist: client.PrevExist, 198 | TTL: ttl, 199 | } 200 | resp, err := m.kapi.Set(m.ctx, m.key, m.id, setOptions) 201 | if err != nil { 202 | m.debug("Refresh ttl of %v failed [%q]", m.key, resp) 203 | } else { 204 | m.debug("Refresh ttl of %v OK", m.key) 205 | } 206 | 207 | return err 208 | } 209 | 210 | func (m *Mutex) debug(format string, v ...interface{}) { 211 | if m.logger != nil { 212 | m.logger.Write([]byte(m.id)) 213 | m.logger.Write([]byte(" ")) 214 | m.logger.Write([]byte(fmt.Sprintf(format, v...))) 215 | m.logger.Write([]byte("\n")) 216 | } 217 | } 218 | 219 | func (m *Mutex) SetDebugLogger(w io.Writer) { 220 | m.logger = w 221 | } 222 | -------------------------------------------------------------------------------- /pool/xtime/xtime.go: -------------------------------------------------------------------------------- 1 | // xtime 是基于小根堆的定时器服务 2 | package xtime 3 | 4 | import ( 5 | "log" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | const ( 11 | Debug = false 12 | ) 13 | 14 | const ( 15 | timerFormat = "2006-01-02 15:04:05" 16 | infiniteDuration = time.Duration(1<<63 - 1) 17 | ) 18 | 19 | var ( 20 | timerLazyDelay = 300 * time.Millisecond 21 | ) 22 | 23 | type TimerData struct { 24 | Key string 25 | expire time.Time 26 | fn func() 27 | index int 28 | next *TimerData 29 | } 30 | 31 | func (td *TimerData) Delay() time.Duration { 32 | return td.expire.Sub(time.Now()) 33 | } 34 | 35 | func (td *TimerData) ExpireString() string { 36 | return td.expire.Format(timerFormat) 37 | } 38 | 39 | type Timer struct { 40 | lock sync.Mutex 41 | free *TimerData 42 | timers []*TimerData 43 | signal *time.Timer 44 | num int 45 | } 46 | 47 | // A heap must be initialized before any of the heap operations 48 | // can be used. Init is idempotent with respect to the heap invariants 49 | // and may be called whenever the heap invariants may have been invalidated. 50 | // Its complexity is O(n) where n = h.Len(). 51 | // 52 | func NewTimer(num int) (t *Timer) { 53 | t = new(Timer) 54 | t.init(num) 55 | return t 56 | } 57 | 58 | // Init init the timer. 59 | func (t *Timer) Init(num int) { 60 | t.init(num) 61 | } 62 | 63 | func (t *Timer) init(num int) { 64 | t.signal = time.NewTimer(infiniteDuration) 65 | t.timers = make([]*TimerData, 0, num) 66 | t.num = num 67 | t.grow() 68 | go t.start() 69 | } 70 | 71 | func (t *Timer) grow() { 72 | var ( 73 | i int 74 | td *TimerData 75 | tds = make([]TimerData, t.num) 76 | ) 77 | t.free = &(tds[0]) 78 | td = t.free 79 | for i = 1; i < t.num; i++ { 80 | td.next = &(tds[i]) 81 | td = td.next 82 | } 83 | td.next = nil 84 | return 85 | } 86 | 87 | // get get a free timer data. 88 | func (t *Timer) get() (td *TimerData) { 89 | if td = t.free; td == nil { 90 | t.grow() 91 | td = t.free 92 | } 93 | t.free = td.next 94 | return 95 | } 96 | 97 | // put put back a timer data. 98 | func (t *Timer) put(td *TimerData) { 99 | td.fn = nil 100 | td.next = t.free 101 | t.free = td 102 | } 103 | 104 | // Push pushes the element x onto the heap. The complexity is 105 | // O(log(n)) where n = h.Len(). 106 | func (t *Timer) Add(expire time.Duration, fn func()) (td *TimerData) { 107 | t.lock.Lock() 108 | td = t.get() 109 | td.expire = time.Now().Add(expire) 110 | td.fn = fn 111 | t.add(td) 112 | t.lock.Unlock() 113 | return 114 | } 115 | 116 | // Del removes the element at index i from the heap. 117 | // The complexity is O(log(n)) where n = h.Len(). 118 | func (t *Timer) Del(td *TimerData) { 119 | t.lock.Lock() 120 | t.del(td) 121 | t.put(td) 122 | t.lock.Unlock() 123 | return 124 | } 125 | 126 | // Push pushes the element x onto the heap. The complexity is 127 | // O(log(n)) where n = h.Len(). 128 | func (t *Timer) add(td *TimerData) { 129 | var d time.Duration 130 | td.index = len(t.timers) 131 | // add to the minheap last node 132 | t.timers = append(t.timers, td) 133 | t.up(td.index) 134 | if td.index == 0 { 135 | // if first node, signal start goroutine 136 | d = td.Delay() 137 | t.signal.Reset(d) 138 | if Debug { 139 | log.Println("timer: add reset delay %d ms", int64(d)/int64(time.Millisecond)) 140 | } 141 | } 142 | if Debug { 143 | log.Println("timer: push item key: %s, expire: %s, index: %d", td.Key, td.ExpireString(), td.index) 144 | } 145 | return 146 | } 147 | 148 | func (t *Timer) del(td *TimerData) { 149 | var ( 150 | i = td.index 151 | last = len(t.timers) - 1 152 | ) 153 | if i < 0 || i > last || t.timers[i] != td { 154 | // already remove, usually by expire 155 | if Debug { 156 | log.Println("timer del i: %d, last: %d, %p", i, last, td) 157 | } 158 | return 159 | } 160 | if i != last { 161 | t.swap(i, last) 162 | t.down(i, last) 163 | t.up(i) 164 | } 165 | // remove item is the last node 166 | t.timers[last].index = -1 // for safety 167 | t.timers = t.timers[:last] 168 | if Debug { 169 | log.Println("timer: remove item key: %s, expire: %s, index: %d", td.Key, td.ExpireString(), td.index) 170 | } 171 | return 172 | } 173 | 174 | // Set update timer data. 175 | func (t *Timer) Set(td *TimerData, expire time.Duration) { 176 | t.lock.Lock() 177 | t.del(td) 178 | td.expire = time.Now().Add(expire) 179 | t.add(td) 180 | t.lock.Unlock() 181 | return 182 | } 183 | 184 | // start start the timer. 185 | func (t *Timer) start() { 186 | for { 187 | t.expire() 188 | <-t.signal.C 189 | } 190 | } 191 | 192 | // expire removes the minimum element (according to Less) from the heap. 193 | // The complexity is O(log(n)) where n = max. 194 | // It is equivalent to Del(0). 195 | func (t *Timer) expire() { 196 | var ( 197 | fn func() 198 | td *TimerData 199 | d time.Duration 200 | ) 201 | t.lock.Lock() 202 | for { 203 | if len(t.timers) == 0 { 204 | d = infiniteDuration 205 | if Debug { 206 | log.Println("timer: no other instance") 207 | } 208 | break 209 | } 210 | td = t.timers[0] 211 | if d = td.Delay(); d > 0 { 212 | break 213 | } 214 | fn = td.fn 215 | // let caller put back 216 | t.del(td) 217 | t.lock.Unlock() 218 | if fn == nil { 219 | log.Println("expire timer no fn") 220 | } else { 221 | if Debug { 222 | log.Println("timer key: %s, expire: %s, index: %d expired, call fn", td.Key, td.ExpireString(), td.index) 223 | } 224 | fn() 225 | } 226 | t.lock.Lock() 227 | } 228 | t.signal.Reset(d) 229 | if Debug { 230 | log.Println("timer: expier reset delay %d ms", int64(d)/int64(time.Millisecond)) 231 | } 232 | t.lock.Unlock() 233 | return 234 | } 235 | 236 | func (t *Timer) up(j int) { 237 | for { 238 | i := (j - 1) / 2 // parent 239 | if i == j || !t.less(j, i) { 240 | break 241 | } 242 | t.swap(i, j) 243 | j = i 244 | } 245 | } 246 | 247 | func (t *Timer) down(i, n int) { 248 | for { 249 | j1 := 2*i + 1 250 | if j1 >= n || j1 < 0 { // j1 < 0 after int overflow 251 | break 252 | } 253 | j := j1 // left child 254 | if j2 := j1 + 1; j2 < n && !t.less(j1, j2) { 255 | j = j2 // = 2*i + 2 // right child 256 | } 257 | if !t.less(j, i) { 258 | break 259 | } 260 | t.swap(i, j) 261 | i = j 262 | } 263 | } 264 | 265 | func (t *Timer) less(i, j int) bool { 266 | return t.timers[i].expire.Before(t.timers[j].expire) 267 | } 268 | 269 | func (t *Timer) swap(i, j int) { 270 | t.timers[i], t.timers[j] = t.timers[j], t.timers[i] 271 | t.timers[i].index = i 272 | t.timers[j].index = j 273 | } 274 | -------------------------------------------------------------------------------- /timerx/minheap/timer.go: -------------------------------------------------------------------------------- 1 | // 基于小根堆的定时器 2 | // 在github.com\Terry-Mao\goim\libs\time基础上 3 | // 增加超时函数参数，去掉log4go 4 | package timer 5 | 6 | import ( 7 | "log" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | const ( 13 | Debug = false 14 | ) 15 | 16 | const ( 17 | timerFormat = "2006-01-02 15:04:05" 18 | infiniteDuration = time.Duration(1<<63 - 1) 19 | ) 20 | 21 | var ( 22 | timerLazyDelay = 300 * time.Millisecond 23 | ) 24 | 25 | type TimerData struct { 26 | Key string 27 | expire time.Time 28 | fn func(interface{}) 29 | arg interface{} 30 | index int 31 | next *TimerData 32 | } 33 | 34 | func (td *TimerData) Delay() time.Duration { 35 | return td.expire.Sub(time.Now()) 36 | } 37 | 38 | func (td *TimerData) ExpireString() string { 39 | return td.expire.Format(timerFormat) 40 | } 41 | 42 | type Timer struct { 43 | lock sync.Mutex 44 | free *TimerData 45 | timers []*TimerData 46 | signal *time.Timer 47 | num int 48 | } 49 | 50 | // A heap must be initialized before any of the heap operations 51 | // can be used. Init is idempotent with respect to the heap invariants 52 | // and may be called whenever the heap invariants may have been invalidated. 53 | // Its complexity is O(n) where n = h.Len(). 54 | // 55 | func NewTimer(num int) (t *Timer) { 56 | t = new(Timer) 57 | t.init(num) 58 | return t 59 | } 60 | 61 | // Init init the timer. 62 | func (t *Timer) Init(num int) { 63 | t.init(num) 64 | } 65 | 66 | func (t *Timer) init(num int) { 67 | t.signal = time.NewTimer(infiniteDuration) 68 | t.timers = make([]*TimerData, 0, num) 69 | t.num = num 70 | t.grow() 71 | go t.start() 72 | } 73 | 74 | func (t *Timer) grow() { 75 | var ( 76 | i int 77 | td *TimerData 78 | tds = make([]TimerData, t.num) 79 | ) 80 | t.free = &(tds[0]) 81 | td = t.free 82 | for i = 1; i < t.num; i++ { 83 | td.next = &(tds[i]) 84 | td = td.next 85 | } 86 | td.next = nil 87 | return 88 | } 89 | 90 | // get get a free timer data. 91 | func (t *Timer) get() (td *TimerData) { 92 | if td = t.free; td == nil { 93 | t.grow() 94 | td = t.free 95 | } 96 | t.free = td.next 97 | return 98 | } 99 | 100 | // put put back a timer data. 101 | func (t *Timer) put(td *TimerData) { 102 | td.fn = nil 103 | td.next = t.free 104 | t.free = td 105 | } 106 | 107 | // Push pushes the element x onto the heap. The complexity is 108 | // O(log(n)) where n = h.Len(). 109 | func (t *Timer) Add(expire time.Duration, fn func(interface{}), arg interface{}) (td *TimerData) { 110 | t.lock.Lock() 111 | td = t.get() 112 | td.expire = time.Now().Add(expire) 113 | td.fn = fn 114 | td.arg = arg 115 | t.add(td) 116 | t.lock.Unlock() 117 | return 118 | } 119 | 120 | // Del removes the element at index i from the heap. 121 | // The complexity is O(log(n)) where n = h.Len(). 122 | func (t *Timer) Del(td *TimerData) { 123 | t.lock.Lock() 124 | t.del(td) 125 | t.put(td) 126 | t.lock.Unlock() 127 | return 128 | } 129 | 130 | // Push pushes the element x onto the heap. The complexity is 131 | // O(log(n)) where n = h.Len(). 132 | func (t *Timer) add(td *TimerData) { 133 | var d time.Duration 134 | td.index = len(t.timers) 135 | // add to the minheap last node 136 | t.timers = append(t.timers, td) 137 | t.up(td.index) 138 | if td.index == 0 { 139 | // if first node, signal start goroutine 140 | d = td.Delay() 141 | t.signal.Reset(d) 142 | if Debug { 143 | log.Printf("timer: add reset delay %d ms", int64(d)/int64(time.Millisecond)) 144 | } 145 | } 146 | if Debug { 147 | log.Printf("timer: push item key: %s, expire: %s, index: %d", td.Key, td.ExpireString(), td.index) 148 | } 149 | return 150 | } 151 | 152 | func (t *Timer) del(td *TimerData) { 153 | var ( 154 | i = td.index 155 | last = len(t.timers) - 1 156 | ) 157 | if i < 0 || i > last || t.timers[i] != td { 158 | // already remove, usually by expire 159 | if Debug { 160 | log.Printf("timer del i: %d, last: %d, %p", i, last, td) 161 | } 162 | return 163 | } 164 | if i != last { 165 | t.swap(i, last) 166 | t.down(i, last) 167 | t.up(i) 168 | } 169 | // remove item is the last node 170 | t.timers[last].index = -1 // for safety 171 | t.timers = t.timers[:last] 172 | if Debug { 173 | log.Printf("timer: remove item key: %s, expire: %s, index: %d", td.Key, td.ExpireString(), td.index) 174 | } 175 | return 176 | } 177 | 178 | // Set update timer data. 179 | func (t *Timer) Set(td *TimerData, expire time.Duration) { 180 | t.lock.Lock() 181 | t.del(td) 182 | td.expire = time.Now().Add(expire) 183 | t.add(td) 184 | t.lock.Unlock() 185 | return 186 | } 187 | 188 | // start start the timer. 189 | func (t *Timer) start() { 190 | for { 191 | t.expire() 192 | <-t.signal.C 193 | } 194 | } 195 | 196 | // expire removes the minimum element (according to Less) from the heap. 197 | // The complexity is O(log(n)) where n = max. 198 | // It is equivalent to Del(0). 199 | func (t *Timer) expire() { 200 | var ( 201 | fn func(interface{}) 202 | arg interface{} 203 | td *TimerData 204 | d time.Duration 205 | ) 206 | t.lock.Lock() 207 | for { 208 | if len(t.timers) == 0 { 209 | d = infiniteDuration 210 | if Debug { 211 | log.Printf("timer: no other instance") 212 | } 213 | break 214 | } 215 | td = t.timers[0] 216 | if d = td.Delay(); d > 0 { 217 | break 218 | } 219 | fn = td.fn 220 | arg = td.arg 221 | // let caller put back 222 | t.del(td) 223 | t.lock.Unlock() 224 | if fn == nil { 225 | log.Printf("[WARN]expire timer no fn") 226 | } else { 227 | if Debug { 228 | log.Printf("timer key: %s, expire: %s, index: %d expired, call fn", td.Key, td.ExpireString(), td.index) 229 | } 230 | fn(arg) 231 | } 232 | t.lock.Lock() 233 | } 234 | t.signal.Reset(d) 235 | if Debug { 236 | log.Printf("timer: expier reset delay %d ms", int64(d)/int64(time.Millisecond)) 237 | } 238 | t.lock.Unlock() 239 | return 240 | } 241 | 242 | func (t *Timer) up(j int) { 243 | for { 244 | i := (j - 1) / 2 // parent 245 | if i == j || !t.less(j, i) { 246 | break 247 | } 248 | t.swap(i, j) 249 | j = i 250 | } 251 | } 252 | 253 | func (t *Timer) down(i, n int) { 254 | for { 255 | j1 := 2*i + 1 256 | if j1 >= n || j1 < 0 { // j1 < 0 after int overflow 257 | break 258 | } 259 | j := j1 // left child 260 | if j2 := j1 + 1; j2 < n && !t.less(j1, j2) { 261 | j = j2 // = 2*i + 2 // right child 262 | } 263 | if !t.less(j, i) { 264 | break 265 | } 266 | t.swap(i, j) 267 | i = j 268 | } 269 | } 270 | 271 | func (t *Timer) less(i, j int) bool { 272 | return t.timers[i].expire.Before(t.timers[j].expire) 273 | } 274 | 275 | func (t *Timer) swap(i, j int) { 276 | t.timers[i], t.timers[j] = t.timers[j], t.timers[i] 277 | t.timers[i].index = i 278 | t.timers[j].index = j 279 | } 280 | -------------------------------------------------------------------------------- /httpx/sign.go: -------------------------------------------------------------------------------- 1 | package httpx 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | "net/url" 11 | "sort" 12 | "strconv" 13 | "strings" 14 | "time" 15 | 16 | "github.com/axengine/utils/sign" 17 | "github.com/shopspring/decimal" 18 | ) 19 | 20 | // APISign API Param Sign 21 | // 1. rawStr eg: GET http://example.com/hello?n=1&a=2 Key["n","a"]-ASC Sort["a","n"] GetParam(a) a=2&n=1 param key string attaches the methods 22 | // 2. other request http method，for Content-Type: application/json {"n":"m","a":2} Key ASC Sort,param key string attaches the methods => {"a":2,"n":"m"} => a=2&n=m 23 | // 3. rawStr+timestamp => a=2&n=m1626167650 (1626167650 is unix timestamp), verify sign time valid（default 10s） 24 | // 4. Sign Method： Method(rawStr+timestamp, secretKey) signed text encode [Base64, Hex(default)] 25 | // Method=[HMAC-SHA256,HMAC-SHA1] Encode=[Base64,Hex] Default = HMAC-SHA256-HEX 26 | // 5. default: signStr=Hex(HMAC-SHA256(rawStr+timestamp,secretKey)) 27 | // 6. Sign http request Header X-Signature=accessKey:signStr:timestamp (: split elem) 28 | type APISign struct { 29 | AccessKey string 30 | AccessSecret string 31 | TTL int64 32 | Method 33 | } 34 | 35 | type Method string 36 | 37 | const ( 38 | HmacSha256 Method = "HMAC-SHA256-BASE64" 39 | HmacSha1 Method = "HMAC-SHA1-BASE64" 40 | HmacSha1Hex Method = "HMAC-SHA1-HEX" 41 | HmacSha256Hex Method = "HMAC-SHA256-HEX" 42 | ) 43 | 44 | func NewAPISign(accessKey, accessSecret string, ttl int64, method Method) *APISign { 45 | return &APISign{ 46 | AccessKey: accessKey, 47 | AccessSecret: accessSecret, 48 | TTL: ttl, 49 | Method: method, 50 | } 51 | } 52 | 53 | // Verify param sign result verify 54 | // req:the http.Request 55 | // authHeaderName:header name,like X-Signature,value:accessKey:signature:deadline 56 | func (p *APISign) Verify(req *http.Request, authHeaderName string) error { 57 | authStr := req.Header.Get(authHeaderName) 58 | if authStr == "" { 59 | return fmt.Errorf("signature header required:%s", authHeaderName) 60 | } 61 | ss := strings.Split(authStr, ":") 62 | if len(ss) != 3 { 63 | return fmt.Errorf("signature header invalid:%s", authHeaderName) 64 | } 65 | accessKey, signature, deadlines := ss[0], ss[1], ss[2] 66 | 67 | deadline, err := strconv.ParseInt(deadlines, 10, 64) 68 | if err != nil || deadline < time.Now().Unix() { 69 | return errors.New("signature expired") 70 | } 71 | if accessKey != p.AccessKey { 72 | return errors.New("invalid AccessKey") 73 | } 74 | 75 | raw, err := p.ToSignRaw(req) 76 | if err != nil { 77 | return err 78 | } 79 | raw = fmt.Sprintf("%s%d", raw, deadline) 80 | checkSignature := signHash(p.Method, []byte(raw), []byte(p.AccessSecret)) 81 | if checkSignature != signature { 82 | return fmt.Errorf("sign method invalid raw:%s", raw) 83 | } 84 | return nil 85 | } 86 | 87 | // Sign sign and return signature,deadline is validity period of signature 88 | func (p *APISign) Sign(req *http.Request, deadline int64) (string, error) { 89 | raw, err := p.ToSignRaw(req) 90 | if err != nil { 91 | return "", err 92 | } 93 | if deadline == 0 { 94 | deadline = time.Now().Unix() + p.TTL 95 | } 96 | raw = fmt.Sprintf("%s%d", raw, deadline) 97 | return signHash(p.Method, []byte(raw), []byte(p.AccessSecret)), nil 98 | } 99 | 100 | // ToSignRaw 从req解析并生成已排序待签名参数字符串 101 | // 如果contentType是JSON，只取JSON参数 102 | // 其他：取URL PARAMS和BODY PARAMS 103 | func (p *APISign) ToSignRaw(req *http.Request) (string, error) { 104 | var raw string 105 | contentType := req.Header.Get("Content-Type") 106 | switch { 107 | case strings.Contains(contentType, "application/json"): 108 | bz, err := io.ReadAll(req.Body) 109 | if err != nil { 110 | _ = req.Body.Close() 111 | return "", err 112 | } 113 | _ = req.Body.Close() 114 | req.Body = io.NopCloser(bytes.NewBuffer(bz)) 115 | 116 | var reqBody = make(requestBodyMap) 117 | if err := json.Unmarshal(bz, &reqBody); err != nil { 118 | return "", err 119 | } 120 | raw, _ = reqBody.SortToString("&") 121 | default: 122 | if err := req.ParseForm(); err != nil { 123 | return "", err 124 | } 125 | 126 | if req.Form != nil && len(req.Form) > 0 { 127 | var paramNames []string 128 | for k := range req.Form { 129 | paramNames = append(paramNames, k) 130 | } 131 | sort.Strings(paramNames) 132 | 133 | var query []string 134 | for _, k := range paramNames { 135 | query = append(query, url.QueryEscape(k)+"="+url.QueryEscape(req.Form.Get(k))) 136 | } 137 | raw = strings.Join(query, "&") 138 | } 139 | } 140 | return raw, nil 141 | } 142 | 143 | // signHash hash and encode 144 | func signHash(method Method, rawStr, secretKey []byte) (hash string) { 145 | switch method { 146 | case HmacSha1: 147 | hash = sign.HMACSha1B64(rawStr, secretKey) 148 | case HmacSha256: 149 | hash = sign.HMACSha256B64(rawStr, secretKey) 150 | case HmacSha1Hex: 151 | hash = sign.HMACSha1Hex(rawStr, secretKey) 152 | case HmacSha256Hex: 153 | hash = sign.HMACSha256Hex(rawStr, secretKey) 154 | default: 155 | hash = sign.HMACSha256Hex(rawStr, secretKey) 156 | } 157 | return 158 | } 159 | 160 | type requestBodyMap map[string]interface{} 161 | 162 | // SortToString request body param sort format 163 | func (r requestBodyMap) SortToString(separator string) (string, error) { 164 | if len(r) == 0 { 165 | return "", nil 166 | } 167 | kvs := make(KvSlice, 0) 168 | for k, v := range r { 169 | kvs = append(kvs, Kv{Key: k, Value: v}) 170 | } 171 | 172 | sort.Sort(kvs) 173 | var s = make([]string, 0, len(kvs)) 174 | for _, v := range kvs { 175 | switch v.Value.(type) { 176 | case float64: 177 | s = append(s, fmt.Sprintf("%s=%s", v.Key, decimal.NewFromFloat(v.Value.(float64)).String())) 178 | case float32: 179 | s = append(s, fmt.Sprintf("%s=%s", v.Key, decimal.NewFromFloat(float64(v.Value.(float32))).String())) 180 | case *float64: 181 | s = append(s, fmt.Sprintf("%s=%s", v.Key, decimal.NewFromFloat(*v.Value.(*float64)).String())) 182 | case *float32: 183 | s = append(s, fmt.Sprintf("%s=%s", v.Key, decimal.NewFromFloat(float64(*v.Value.(*float32))).String())) 184 | case string: 185 | s = append(s, fmt.Sprintf("%s=%s", v.Key, v.Value)) 186 | case *string: 187 | s = append(s, fmt.Sprintf("%s=%s", v.Key, *v.Value.(*string))) 188 | default: 189 | buf := make([]byte, 0) 190 | buffer := bytes.NewBuffer(buf) 191 | if err := json.NewEncoder(buffer).Encode(v.Value); err != nil { 192 | return "", err 193 | } 194 | s = append(s, fmt.Sprintf("%s=%s", v.Key, string(r.trimNewline(buffer.Bytes())))) 195 | } 196 | } 197 | return strings.Join(s, separator), nil 198 | } 199 | 200 | func (r requestBodyMap) trimNewline(buf []byte) []byte { 201 | if i := len(buf) - 1; i >= 0 { 202 | if buf[i] == '\n' { 203 | buf = buf[:i] 204 | } 205 | } 206 | return buf 207 | } 208 | 209 | type Kv struct { 210 | Key string 211 | Value interface{} 212 | } 213 | type KvSlice []Kv 214 | 215 | func (s KvSlice) Len() int { return len(s) } 216 | func (s KvSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 217 | func (s KvSlice) Less(i, j int) bool { return s[i].Key < s[j].Key } 218 | -------------------------------------------------------------------------------- /etcdx/v3/go.sum: -------------------------------------------------------------------------------- 1 | github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= 2 | github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 3 | github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= 4 | github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 9 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 10 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 11 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 12 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 13 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 14 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 15 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 16 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 17 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 18 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 19 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 20 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 21 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 22 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 23 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 24 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 25 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 26 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 27 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 28 | go.etcd.io/etcd/api/v3 v3.5.18 h1:Q4oDAKnmwqTo5lafvB+afbgCDF7E35E4EYV2g+FNGhs= 29 | go.etcd.io/etcd/api/v3 v3.5.18/go.mod h1:uY03Ob2H50077J7Qq0DeehjM/A9S8PhVfbQ1mSaMopU= 30 | go.etcd.io/etcd/client/pkg/v3 v3.5.18 h1:mZPOYw4h8rTk7TeJ5+3udUkfVGBqc+GCjOJYd68QgNM= 31 | go.etcd.io/etcd/client/pkg/v3 v3.5.18/go.mod h1:BxVf2o5wXG9ZJV+/Cu7QNUiJYk4A29sAhoI5tIRsCu4= 32 | go.etcd.io/etcd/client/v3 v3.5.18 h1:nvvYmNHGumkDjZhTHgVU36A9pykGa2K4lAJ0yY7hcXA= 33 | go.etcd.io/etcd/client/v3 v3.5.18/go.mod h1:kmemwOsPU9broExyhYsBxX4spCTDX3yLgPMWtpBXG6E= 34 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 35 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 36 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 37 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 38 | go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= 39 | go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= 40 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 41 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 42 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 43 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 44 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 45 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 46 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 47 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 48 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 49 | golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= 50 | golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= 51 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 52 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 53 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 54 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 55 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 56 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 57 | golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= 58 | golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 59 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 60 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 61 | golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= 62 | golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= 63 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 64 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 65 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 66 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 67 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 68 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 69 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 70 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 71 | google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= 72 | google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= 73 | google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= 74 | google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= 75 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= 76 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= 77 | google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= 78 | google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= 79 | google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= 80 | google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 81 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 82 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 83 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 84 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 85 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 86 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 87 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 88 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 89 | -------------------------------------------------------------------------------- /container/omap/omap_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2011-12 Qtrac Ltd. 2 | // 3 | // This program or package and any associated files are licensed under the 4 | // Apache License, Version 2.0 (the "License"); you may not use these files 5 | // except in compliance with the License. You can get a copy of the License 6 | // at: http://www.apache.org/licenses/LICENSE-2.0. 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | // The tests here are very incomplete and just to show examples of how it 15 | // can be done. 16 | package omap 17 | 18 | import ( 19 | "math/rand" 20 | "sort" 21 | "strings" 22 | "testing" 23 | "time" 24 | ) 25 | 26 | func TestStringKeyOMapInsertion(t *testing.T) { 27 | wordForWord := NewCaseFoldedKeyed() 28 | for _, word := range []string{"one", "Two", "THREE", "four", "Five"} { 29 | wordForWord.Insert(word, word) 30 | } 31 | var words []string 32 | wordForWord.Do(func(_, value interface{}) { 33 | words = append(words, value.(string)) 34 | }) 35 | actual, expected := strings.Join(words, ""), "FivefouroneTHREETwo" 36 | if actual != expected { 37 | t.Errorf("%q != %q", actual, expected) 38 | } 39 | } 40 | func TestMap_FindMinKey(t *testing.T) { 41 | intMap := NewIntKeyed() 42 | // 随机生成100长度的随机正整数,将其排序后,注入有序map 43 | r := rand.New(rand.NewSource(time.Now().Unix())) 44 | ints := make([]int, 100) 45 | for i := 0; i < 100; i++ { 46 | ints[i] = r.Intn(1000) 47 | } 48 | sort.Ints(ints) 49 | for _, number := range ints { 50 | intMap.Insert(number, number*2) 51 | } 52 | 53 | key, _, found := intMap.First() 54 | if !found || key != ints[0] { 55 | t.Error("Cann‘t found the mininal key for it's value") 56 | } 57 | } 58 | func TestMap_FindMaxKey(t *testing.T) { 59 | intMap := NewIntKeyed() 60 | // 随机生成100长度的随机正整数,将其排序后,注入有序map 61 | r := rand.New(rand.NewSource(time.Now().Unix())) 62 | ints := make([]int, 100) 63 | for i := 0; i < 100; i++ { 64 | ints[i] = r.Intn(1000) 65 | } 66 | sort.Ints(ints) 67 | for _, number := range ints { 68 | intMap.Insert(number, number*2) 69 | } 70 | 71 | key, _, found := intMap.Latest() 72 | if !found || key != ints[100-1] { 73 | t.Error("Cann‘t found the maxinal key for it's value") 74 | } 75 | } 76 | func TestOMap_FindIntKey(t *testing.T) { 77 | intMap := NewIntKeyed() 78 | for _, number := range []int{9, 1, 8, 2, 7, 3, 6, 4, 5, 0} { 79 | intMap.Insert(number, number*10) 80 | } 81 | for _, number := range []int{0, 1, 5, 8, 9} { 82 | value, found := intMap.Find(number) 83 | if !found { 84 | t.Errorf("failed to find %d", number) 85 | } 86 | actual, expected := value.(int), number*10 87 | if actual != expected { 88 | t.Errorf("value is %d should be %d", actual, expected) 89 | } 90 | } 91 | for _, number := range []int{-1, -21, 10, 11, 148} { 92 | _, found := intMap.Find(number) 93 | if found { 94 | t.Errorf("should not have found %d", number) 95 | } 96 | } 97 | } 98 | 99 | func TestOMap_DeleteIntKey(t *testing.T) { 100 | intMap := NewIntKeyed() 101 | for _, number := range []int{9, 1, 8, 2, 7, 3, 6, 4, 5, 0} { 102 | intMap.Insert(number, number*10) 103 | } 104 | if intMap.Len() != 10 { 105 | t.Errorf("map len %d should be 10", intMap.Len()) 106 | } 107 | length := 9 108 | for i, number := range []int{0, 1, 5, 8, 9} { 109 | if deleted := intMap.Delete(number); !deleted { 110 | t.Errorf("failed to delete %d", number) 111 | } 112 | if intMap.Len() != length-i { 113 | t.Errorf("map len %d should be %d", intMap.Len(), length-i) 114 | } 115 | } 116 | for _, number := range []int{-1, -21, 10, 11, 148} { 117 | if deleted := intMap.Delete(number); deleted { 118 | t.Errorf("should not have deleted nonexistent %d", number) 119 | } 120 | } 121 | if intMap.Len() != 5 { 122 | t.Errorf("map len %d should be 5", intMap.Len()) 123 | } 124 | } 125 | 126 | func TestOMap_DeleteIntKey2(t *testing.T) { 127 | var ( 128 | size = 10000 129 | numbers = make([]int, size) 130 | ) 131 | intMap := NewIntKeyed() 132 | for i := 0; i < size; i++ { 133 | numbers[i] = i * 2 134 | intMap.Insert(numbers[i], i*10) 135 | } 136 | 137 | if intMap.Len() != size { 138 | t.Errorf("map len %d should be %v", intMap.Len(), size) 139 | } 140 | for i := size - 1; i >= 0; i-- { 141 | intMap.Delete(numbers[i]) 142 | } 143 | if intMap.Len() != 0 { 144 | t.Errorf("map len %d should be 0", intMap.Len()) 145 | } 146 | } 147 | 148 | type Point struct { 149 | ActionTime time.Time 150 | Key int 151 | } 152 | 153 | var ( 154 | r = rand.New(rand.NewSource(time.Now().Unix())) 155 | randscope = 10 * 1000 * 1000 * 1000 156 | now = time.Now() 157 | ) 158 | 159 | func getRandPoint() Point { 160 | return Point{ 161 | ActionTime: now.Add(time.Duration(int64(r.Intn(randscope)))), 162 | Key: rand.Int(), 163 | } 164 | } 165 | func less(a, b interface{}) bool { 166 | α, β := a.(Point), b.(Point) 167 | if !α.ActionTime.Equal(β.ActionTime) { 168 | return α.ActionTime.Before(β.ActionTime) 169 | } 170 | return α.Key < β.Key 171 | } 172 | func TestOMap_FindStructKey(t *testing.T) { 173 | var ( 174 | number = 1000000 175 | ) 176 | points := make([]Point, number) 177 | 178 | for i := 0; i < number; i++ { 179 | points[i] = getRandPoint() 180 | } 181 | pointMap := New(less) 182 | for i := 0; i < number; i++ { 183 | pointMap.Insert(points[i], i) 184 | } 185 | if pointMap.Len() != number { 186 | t.Error("数据加入有丢失") 187 | } 188 | 189 | for i := 0; i < number; i++ { 190 | nodevalue, founded := pointMap.Find(points[i]) 191 | value := nodevalue.(int) 192 | if !founded || value != i { 193 | t.Errorf("查找数据失败,key=%+v \n", points[i]) 194 | break 195 | } 196 | } 197 | } 198 | 199 | // TODO:this test cannot pass,bacause some delete will cause other key change the same 200 | func TestOMap_DelStructKey(t *testing.T) { 201 | t.Skip() 202 | var ( 203 | size = 1000 204 | ) 205 | points := make([]Point, size) 206 | for i := 0; i < size; i++ { 207 | points[i] = getRandPoint() 208 | } 209 | pointMap := New(less) 210 | for i := 0; i < size; i++ { 211 | pointMap.Insert(points[i], i) //这里值，仅仅只是一个填充，无实际意义 212 | } 213 | if pointMap.Len() != size { 214 | t.Errorf("map len %d should be %v", pointMap.Len(), size) 215 | 216 | } 217 | 218 | for i := 0; i < size; i++ { 219 | _, founded := pointMap.Find(points[i]) 220 | if !founded { 221 | t.Errorf("查找数据失败,key=%+v,value=%v \n", points[i-1], i-1) 222 | 223 | t.Errorf("查找数据失败,key=%+v,value=%v \n", points[i], i) 224 | //break 225 | } 226 | deleted := pointMap.Delete(points[i]) 227 | if !deleted { 228 | t.Errorf("删除数据失败,key=%+v,value=%v \n", points[i], i) 229 | break 230 | } 231 | } 232 | for i := 0; i < pointMap.Len(); i++ { 233 | f, v, ok := pointMap.First() 234 | if ok { 235 | t.Logf("savive遗留数据,key=%+v ,value=%+v \n", f, v) 236 | } 237 | } 238 | } 239 | 240 | // 测试逆序插入，能否按从小到大的顺序导出与清理 241 | func TestOMap_DelStructFirst(t *testing.T) { 242 | var ( 243 | num = 100000 244 | zerotime time.Time 245 | points = make([]Point, num) 246 | ) 247 | ordermap := New(less) 248 | for i := 0; i < num; i++ { 249 | //生成从大到小逆序的点位 250 | points[i] = Point{ 251 | ActionTime: zerotime.Add(time.Duration(int64(num - i))), 252 | Key: num - i, 253 | } 254 | ordermap.Insert(points[i], i) 255 | if v, finded := ordermap.Find(points[i]); !finded { 256 | t.Errorf("cannot find :key=%+v | value=%+v\n", points[i], v) 257 | } 258 | } 259 | //clean ordermap by minkey,one by one 260 | for i := 0; i < num; i++ { 261 | k, v, founded := ordermap.First() 262 | if !founded { 263 | t.Error("获取最小值出错") 264 | break 265 | } 266 | key, _ := k.(Point) 267 | deleted := ordermap.Delete(k) 268 | if founded != deleted || founded == false { 269 | t.Errorf("key=%+v | value = %+v \n", k, v) 270 | break 271 | } 272 | if key.Key != points[num-i-1].Key || !key.ActionTime.Equal(points[num-i-1].ActionTime) { 273 | t.Errorf("数据解析顺序有误,ordermap.minKey=%v,points[%v]=%v\n", key, i, points[num-i-1]) 274 | } 275 | } 276 | if ordermap.Len() != 0 { 277 | t.Error("omap is not clean") 278 | } 279 | } 280 | func TestPassing(t *testing.T) { 281 | intMap := NewIntKeyed() 282 | intMap.Insert(7, 7) 283 | passMap(intMap, t) 284 | } 285 | 286 | func passMap(m *Map, t *testing.T) { 287 | for _, number := range []int{9, 3, 6, 4, 5, 0} { 288 | m.Insert(number, number) 289 | } 290 | if m.Len() != 7 { 291 | t.Errorf("should have %d items", 7) 292 | } 293 | } 294 | 295 | // Thanks to Russ Cox for improving these benchmarks 296 | func BenchmarkOMapFindSuccess(b *testing.B) { 297 | b.StopTimer() // Don't timerx creation and population 298 | intMap := NewIntKeyed() 299 | for i := 0; i < 1e6; i++ { 300 | intMap.Insert(i, i) 301 | } 302 | b.StartTimer() // Time the Find() method succeeding 303 | for i := 0; i < b.N; i++ { 304 | intMap.Find(i % 1e6) 305 | } 306 | } 307 | 308 | func BenchmarkOMapFindFailure(b *testing.B) { 309 | b.StopTimer() // Don't timerx creation and population 310 | intMap := NewIntKeyed() 311 | for i := 0; i < 1e6; i++ { 312 | intMap.Insert(2*i, i) 313 | } 314 | b.StartTimer() // Time the Find() method failing 315 | for i := 0; i < b.N; i++ { 316 | intMap.Find(2*(i%1e6) + 1) 317 | } 318 | } 319 | -------------------------------------------------------------------------------- /hash/murmurhash3/mmhash3.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The Go Authors. All rights reserved. 2 | // refer https://github.com/gwenn/murmurhash3 3 | 4 | package murmurhash3 5 | 6 | import ( 7 | "encoding/binary" 8 | "hash" 9 | ) 10 | 11 | type ( 12 | murmurhash3A uint32 13 | murmurhash3C uint32 14 | murmurhash3F uint64 15 | ) 16 | 17 | func NewMurmur3A() hash.Hash32 { 18 | var m murmurhash3A 19 | return &m 20 | } 21 | 22 | func (m *murmurhash3A) Reset() { *m = 0 } 23 | 24 | func (m *murmurhash3A) Size() int { 25 | return 4 26 | } 27 | 28 | func (m *murmurhash3A) BlockSize() int { 29 | return 4 30 | } 31 | 32 | func (m *murmurhash3A) Write(p []byte) (n int, err error) { 33 | *m = murmurhash3A(Murmur3A(p, uint32(*m))) 34 | return len(p), nil 35 | } 36 | 37 | func (m *murmurhash3A) Sum32() uint32 { 38 | return uint32(*m) 39 | } 40 | 41 | func (m *murmurhash3A) Sum(in []byte) []byte { 42 | v := uint32(*m) 43 | return append(in, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) 44 | } 45 | 46 | func NewMurmur3C() hash.Hash32 { 47 | var m murmurhash3C 48 | return &m 49 | } 50 | 51 | func (m *murmurhash3C) Reset() { *m = 0 } 52 | 53 | func (m *murmurhash3C) Size() int { 54 | return 4 55 | } 56 | 57 | func (m *murmurhash3C) BlockSize() int { 58 | return 16 59 | } 60 | 61 | func (m *murmurhash3C) Write(p []byte) (n int, err error) { 62 | *m = murmurhash3C(Murmur3C(p, uint32(*m))[0]) 63 | return len(p), nil 64 | } 65 | 66 | func (m *murmurhash3C) Sum32() uint32 { 67 | return uint32(*m) 68 | } 69 | 70 | func (m *murmurhash3C) Sum(in []byte) []byte { 71 | v := uint32(*m) 72 | return append(in, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) 73 | } 74 | 75 | func NewMurmur3F() hash.Hash64 { 76 | var m murmurhash3F 77 | return &m 78 | } 79 | 80 | func (m *murmurhash3F) Reset() { *m = 0 } 81 | 82 | func (m *murmurhash3F) Size() int { 83 | return 8 84 | } 85 | 86 | func (m *murmurhash3F) BlockSize() int { 87 | return 16 88 | } 89 | 90 | func (m *murmurhash3F) Write(p []byte) (n int, err error) { 91 | *m = murmurhash3F(Murmur3F(p, uint64(*m))[0]) 92 | return len(p), nil 93 | } 94 | 95 | func (m *murmurhash3F) Sum64() uint64 { 96 | return uint64(*m) 97 | } 98 | 99 | func (m *murmurhash3F) Sum(in []byte) []byte { 100 | v := uint64(*m) 101 | return append(in, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) 102 | } 103 | 104 | func rotl32(x uint32, r uint8) uint32 { 105 | return (x << r) | (x >> (32 - r)) 106 | } 107 | 108 | func rotl64(x uint64, r uint8) uint64 { 109 | return (x << r) | (x >> (64 - r)) 110 | } 111 | 112 | func fmix32(h uint32) uint32 { 113 | h ^= h >> 16 114 | h *= 0x85ebca6b 115 | h ^= h >> 13 116 | h *= 0xc2b2ae35 117 | h ^= h >> 16 118 | 119 | return h 120 | } 121 | 122 | func fmix64(h uint64) uint64 { 123 | h ^= h >> 33 124 | h *= 0xff51afd7ed558ccd 125 | h ^= h >> 33 126 | h *= 0xc4ceb9fe1a85ec53 127 | h ^= h >> 33 128 | 129 | return h 130 | } 131 | 132 | // MurmurHash3 for x86, 32-bit (MurmurHash3_x86_32) 133 | func Murmur3A(key []byte, seed uint32) uint32 { 134 | nblocks := len(key) / 4 135 | var h1 = seed 136 | 137 | var c1 uint32 = 0xcc9e2d51 138 | var c2 uint32 = 0x1b873593 139 | 140 | // body 141 | for i := 0; i < nblocks; i++ { 142 | k1 := binary.LittleEndian.Uint32(key[i*4:]) // TODO Validate 143 | 144 | k1 *= c1 145 | k1 = rotl32(k1, 15) 146 | k1 *= c2 147 | 148 | h1 ^= k1 149 | h1 = rotl32(h1, 13) 150 | h1 = h1*5 + 0xe6546b64 151 | } 152 | 153 | // tail 154 | var tail = key[nblocks*4:] // TODO Validate 155 | var k1 uint32 156 | switch len(key) & 3 { 157 | case 3: 158 | k1 ^= uint32(tail[2]) << 16 159 | fallthrough 160 | case 2: 161 | k1 ^= uint32(tail[1]) << 8 162 | fallthrough 163 | case 1: 164 | k1 ^= uint32(tail[0]) 165 | k1 *= c1 166 | k1 = rotl32(k1, 15) 167 | k1 *= c2 168 | h1 ^= k1 169 | } 170 | 171 | //finalization 172 | h1 ^= uint32(len(key)) 173 | 174 | h1 = fmix32(h1) 175 | 176 | return h1 177 | } 178 | 179 | // MurmurHash3 for x86, 128-bit (MurmurHash3_x86_128) 180 | func Murmur3C(key []byte, seed uint32) [4]uint32 { 181 | nblocks := len(key) / 16 182 | var h1 = seed 183 | var h2 = seed 184 | var h3 = seed 185 | var h4 = seed 186 | 187 | var c1 uint32 = 0x239b961b 188 | var c2 uint32 = 0xab0e9789 189 | var c3 uint32 = 0x38b34ae5 190 | var c4 uint32 = 0xa1e38b93 191 | 192 | // body 193 | for i := 0; i < nblocks; i++ { 194 | k1 := binary.LittleEndian.Uint32(key[(i*4+0)*4:]) // TODO Validate 195 | k2 := binary.LittleEndian.Uint32(key[(i*4+1)*4:]) 196 | k3 := binary.LittleEndian.Uint32(key[(i*4+2)*4:]) 197 | k4 := binary.LittleEndian.Uint32(key[(i*4+3)*4:]) 198 | 199 | k1 *= c1 200 | k1 = rotl32(k1, 15) 201 | k1 *= c2 202 | h1 ^= k1 203 | 204 | h1 = rotl32(h1, 19) 205 | h1 += h2 206 | h1 = h1*5 + 0x561ccd1b 207 | 208 | k2 *= c2 209 | k2 = rotl32(k2, 16) 210 | k2 *= c3 211 | h2 ^= k2 212 | 213 | h2 = rotl32(h2, 17) 214 | h2 += h3 215 | h2 = h2*5 + 0x0bcaa747 216 | 217 | k3 *= c3 218 | k3 = rotl32(k3, 17) 219 | k3 *= c4 220 | h3 ^= k3 221 | 222 | h3 = rotl32(h3, 15) 223 | h3 += h4 224 | h3 = h3*5 + 0x96cd1c35 225 | 226 | k4 *= c4 227 | k4 = rotl32(k4, 18) 228 | k4 *= c1 229 | h4 ^= k4 230 | 231 | h4 = rotl32(h4, 13) 232 | h4 += h1 233 | h4 = h4*5 + 0x32ac3b17 234 | } 235 | 236 | // tail 237 | var tail = key[nblocks*16:] // TODO Validate 238 | var k1 uint32 239 | var k2 uint32 240 | var k3 uint32 241 | var k4 uint32 242 | switch len(key) & 15 { 243 | case 15: 244 | k4 ^= uint32(tail[14]) << 16 245 | fallthrough 246 | case 14: 247 | k4 ^= uint32(tail[13]) << 8 248 | fallthrough 249 | case 13: 250 | k4 ^= uint32(tail[12]) << 0 251 | k4 *= c4 252 | k4 = rotl32(k4, 18) 253 | k4 *= c1 254 | h4 ^= k4 255 | fallthrough 256 | case 12: 257 | k3 ^= uint32(tail[11]) << 24 258 | fallthrough 259 | case 11: 260 | k3 ^= uint32(tail[10]) << 16 261 | fallthrough 262 | case 10: 263 | k3 ^= uint32(tail[9]) << 8 264 | fallthrough 265 | case 9: 266 | k3 ^= uint32(tail[8]) << 0 267 | k3 *= c3 268 | k3 = rotl32(k3, 17) 269 | k3 *= c4 270 | h3 ^= k3 271 | fallthrough 272 | case 8: 273 | k2 ^= uint32(tail[7]) << 24 274 | fallthrough 275 | case 7: 276 | k2 ^= uint32(tail[6]) << 16 277 | fallthrough 278 | case 6: 279 | k2 ^= uint32(tail[5]) << 8 280 | fallthrough 281 | case 5: 282 | k2 ^= uint32(tail[4]) << 0 283 | k2 *= c2 284 | k2 = rotl32(k2, 16) 285 | k2 *= c3 286 | h2 ^= k2 287 | fallthrough 288 | case 4: 289 | k1 ^= uint32(tail[3]) << 24 290 | fallthrough 291 | case 3: 292 | k1 ^= uint32(tail[2]) << 16 293 | fallthrough 294 | case 2: 295 | k1 ^= uint32(tail[1]) << 8 296 | fallthrough 297 | case 1: 298 | k1 ^= uint32(tail[0]) << 0 299 | k1 *= c1 300 | k1 = rotl32(k1, 15) 301 | k1 *= c2 302 | h1 ^= k1 303 | } 304 | 305 | //finalization 306 | h1 ^= uint32(len(key)) 307 | h2 ^= uint32(len(key)) 308 | h3 ^= uint32(len(key)) 309 | h4 ^= uint32(len(key)) 310 | 311 | h1 += h2 312 | h1 += h3 313 | h1 += h4 314 | h2 += h1 315 | h3 += h1 316 | h4 += h1 317 | 318 | h1 = fmix32(h1) 319 | h2 = fmix32(h2) 320 | h3 = fmix32(h3) 321 | h4 = fmix32(h4) 322 | 323 | h1 += h2 324 | h1 += h3 325 | h1 += h4 326 | h2 += h1 327 | h3 += h1 328 | h4 += h1 329 | 330 | return [4]uint32{h1, h2, h3, h4} 331 | } 332 | 333 | // MurmurHash3 for x64, 128-bit (MurmurHash3_x64_128) 334 | func Murmur3F(key []byte, seed uint64) [2]uint64 { 335 | nblocks := len(key) / 16 336 | var h1 = seed 337 | var h2 = seed 338 | 339 | var c1 uint64 = 0x87c37b91114253d5 340 | var c2 uint64 = 0x4cf5ad432745937f 341 | 342 | // body 343 | for i := 0; i < nblocks; i++ { 344 | k1 := binary.LittleEndian.Uint64(key[(i*2+0)*8:]) // TODO Validate 345 | k2 := binary.LittleEndian.Uint64(key[(i*2+1)*8:]) 346 | 347 | k1 *= c1 348 | k1 = rotl64(k1, 31) 349 | k1 *= c2 350 | h1 ^= k1 351 | 352 | h1 = rotl64(h1, 27) 353 | h1 += h2 354 | h1 = h1*5 + 0x52dce729 355 | 356 | k2 *= c2 357 | k2 = rotl64(k2, 33) 358 | k2 *= c1 359 | h2 ^= k2 360 | 361 | h2 = rotl64(h2, 31) 362 | h2 += h1 363 | h2 = h2*5 + 0x38495ab5 364 | } 365 | 366 | // tail 367 | var tail = key[nblocks*16:] // TODO Validate 368 | var k1 uint64 369 | var k2 uint64 370 | switch len(key) & 15 { 371 | case 15: 372 | k2 ^= uint64(tail[14]) << 48 373 | fallthrough 374 | case 14: 375 | k2 ^= uint64(tail[13]) << 40 376 | fallthrough 377 | case 13: 378 | k2 ^= uint64(tail[12]) << 32 379 | fallthrough 380 | case 12: 381 | k2 ^= uint64(tail[11]) << 24 382 | fallthrough 383 | case 11: 384 | k2 ^= uint64(tail[10]) << 16 385 | fallthrough 386 | case 10: 387 | k2 ^= uint64(tail[9]) << 8 388 | fallthrough 389 | case 9: 390 | k2 ^= uint64(tail[8]) << 0 391 | k2 *= c2 392 | k2 = rotl64(k2, 33) 393 | k2 *= c1 394 | h2 ^= k2 395 | fallthrough 396 | case 8: 397 | k1 ^= uint64(tail[7]) << 56 398 | fallthrough 399 | case 7: 400 | k1 ^= uint64(tail[6]) << 48 401 | fallthrough 402 | case 6: 403 | k1 ^= uint64(tail[5]) << 40 404 | fallthrough 405 | case 5: 406 | k1 ^= uint64(tail[4]) << 32 407 | fallthrough 408 | case 4: 409 | k1 ^= uint64(tail[3]) << 24 410 | fallthrough 411 | case 3: 412 | k1 ^= uint64(tail[2]) << 16 413 | fallthrough 414 | case 2: 415 | k1 ^= uint64(tail[1]) << 8 416 | fallthrough 417 | case 1: 418 | k1 ^= uint64(tail[0]) << 0 419 | k1 *= c1 420 | k1 = rotl64(k1, 31) 421 | k1 *= c2 422 | h1 ^= k1 423 | } 424 | 425 | //finalization 426 | h1 ^= uint64(len(key)) 427 | h2 ^= uint64(len(key)) 428 | 429 | h1 += h2 430 | h2 += h1 431 | 432 | h1 = fmix64(h1) 433 | h2 = fmix64(h2) 434 | 435 | h1 += h2 436 | h2 += h1 437 | 438 | return [2]uint64{h1, h2} 439 | } 440 | -------------------------------------------------------------------------------- /pool/xbufio/buffio.go: -------------------------------------------------------------------------------- 1 | // Copyright 2009 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package bufio implements buffered I/O. It wraps an io.Reader or io.Writer 6 | // object, creating another object (Reader or Writer) that also implements 7 | // the interface but provides buffering and some help for textual I/O. 8 | package xbufio 9 | 10 | import ( 11 | "errors" 12 | "io" 13 | ) 14 | 15 | const ( 16 | defaultBufSize = 4096 17 | ) 18 | 19 | var ( 20 | ErrInvalidUnreadByte = errors.New("bufio: invalid use of UnreadByte") 21 | ErrInvalidUnreadRune = errors.New("bufio: invalid use of UnreadRune") 22 | ErrBufferFull = errors.New("bufio: buffer full") 23 | ErrNegativeCount = errors.New("bufio: negative count") 24 | ) 25 | 26 | // Buffered input. 27 | 28 | // Reader implements buffering for an io.Reader object. 29 | type Reader struct { 30 | buf []byte 31 | rd io.Reader // reader provided by the client 32 | r, w int // buf read and write positions 33 | err error 34 | } 35 | 36 | const minReadBufferSize = 16 37 | const maxConsecutiveEmptyReads = 100 38 | 39 | // NewReaderSize returns a new Reader whose buffer has at least the specified 40 | // size. If the argument io.Reader is already a Reader with large enough 41 | // size, it returns the underlying Reader. 42 | func NewReaderSize(rd io.Reader, size int) *Reader { 43 | // Is it already a Reader? 44 | b, ok := rd.(*Reader) 45 | if ok && len(b.buf) >= size { 46 | return b 47 | } 48 | if size < minReadBufferSize { 49 | size = minReadBufferSize 50 | } 51 | r := new(Reader) 52 | r.reset(make([]byte, size), rd) 53 | return r 54 | } 55 | 56 | // NewReader returns a new Reader whose buffer has the default size. 57 | func NewReader(rd io.Reader) *Reader { 58 | return NewReaderSize(rd, defaultBufSize) 59 | } 60 | 61 | // Reset discards any buffered data, resets all state, and switches 62 | // the buffered reader to read from r. 63 | func (b *Reader) Reset(r io.Reader) { 64 | b.reset(b.buf, r) 65 | } 66 | 67 | // ResetBuffer discards any buffered data, resets all state, and switches 68 | // the buffered reader to read from r. 69 | func (b *Reader) ResetBuffer(r io.Reader, buf []byte) { 70 | b.reset(buf, r) 71 | } 72 | 73 | func (b *Reader) reset(buf []byte, r io.Reader) { 74 | *b = Reader{ 75 | buf: buf, 76 | rd: r, 77 | } 78 | } 79 | 80 | var errNegativeRead = errors.New("bufio: reader returned negative count from Read") 81 | 82 | // fill reads a new chunk into the buffer. 83 | func (b *Reader) fill() { 84 | // Slide existing data to beginning. 85 | if b.r > 0 { 86 | copy(b.buf, b.buf[b.r:b.w]) 87 | b.w -= b.r 88 | b.r = 0 89 | } 90 | 91 | if b.w >= len(b.buf) { 92 | panic("bufio: tried to fill full buffer") 93 | } 94 | 95 | // Read new data: try a limited number of times. 96 | for i := maxConsecutiveEmptyReads; i > 0; i-- { 97 | n, err := b.rd.Read(b.buf[b.w:]) 98 | if n < 0 { 99 | panic(errNegativeRead) 100 | } 101 | b.w += n 102 | if err != nil { 103 | b.err = err 104 | return 105 | } 106 | if n > 0 { 107 | return 108 | } 109 | } 110 | b.err = io.ErrNoProgress 111 | } 112 | 113 | func (b *Reader) readErr() error { 114 | err := b.err 115 | b.err = nil 116 | return err 117 | } 118 | 119 | // Peek returns the next n bytes without advancing the reader. The bytes stop 120 | // being valid at the next read call. If Peek returns fewer than n bytes, it 121 | // also returns an error explaining why the read is short. The error is 122 | // ErrBufferFull if n is larger than b's buffer size. 123 | func (b *Reader) Peek(n int) ([]byte, error) { 124 | if n < 0 { 125 | return nil, ErrNegativeCount 126 | } 127 | if n > len(b.buf) { 128 | return nil, ErrBufferFull 129 | } 130 | // 0 <= n <= len(b.buf) 131 | for b.w-b.r < n && b.err == nil { 132 | b.fill() // b.w-b.r < len(b.buf) => buffer is not full 133 | } 134 | 135 | var err error 136 | if avail := b.w - b.r; avail < n { 137 | // not enough data in buffer 138 | n = avail 139 | err = b.readErr() 140 | if err == nil { 141 | err = ErrBufferFull 142 | } 143 | } 144 | return b.buf[b.r : b.r+n], err 145 | } 146 | 147 | // Pop returns the next n bytes with advancing the reader. The bytes stop 148 | // being valid at the next read call. If Pop returns fewer than n bytes, it 149 | // also returns an error explaining why the read is short. The error is 150 | // ErrBufferFull if n is larger than b's buffer size. 151 | func (b *Reader) Pop(n int) ([]byte, error) { 152 | if d, err := b.Peek(n); err == nil { 153 | b.r += n 154 | return d, err 155 | } else { 156 | return nil, err 157 | } 158 | } 159 | 160 | // Discard skips the next n bytes, returning the number of bytes discarded. 161 | // 162 | // If Discard skips fewer than n bytes, it also returns an error. 163 | // If 0 <= n <= b.Buffered(), Discard is guaranteed to succeed without 164 | // reading from the underlying io.Reader. 165 | func (b *Reader) Discard(n int) (discarded int, err error) { 166 | if n < 0 { 167 | return 0, ErrNegativeCount 168 | } 169 | if n == 0 { 170 | return 171 | } 172 | remain := n 173 | for { 174 | skip := b.Buffered() 175 | if skip == 0 { 176 | b.fill() 177 | skip = b.Buffered() 178 | } 179 | if skip > remain { 180 | skip = remain 181 | } 182 | b.r += skip 183 | remain -= skip 184 | if remain == 0 { 185 | return n, nil 186 | } 187 | if b.err != nil { 188 | return n - remain, b.readErr() 189 | } 190 | } 191 | } 192 | 193 | // Read reads data into p. 194 | // It returns the number of bytes read into p. 195 | // It calls Read at most once on the underlying Reader, 196 | // hence n may be less than len(p). 197 | // At EOF, the count will be zero and err will be io.EOF. 198 | func (b *Reader) Read(p []byte) (n int, err error) { 199 | n = len(p) 200 | if n == 0 { 201 | return 0, b.readErr() 202 | } 203 | if b.r == b.w { 204 | if b.err != nil { 205 | return 0, b.readErr() 206 | } 207 | if len(p) >= len(b.buf) { 208 | // Large read, empty buffer. 209 | // Read directly into p to avoid copy. 210 | n, b.err = b.rd.Read(p) 211 | if n < 0 { 212 | panic(errNegativeRead) 213 | } 214 | return n, b.readErr() 215 | } 216 | b.fill() // buffer is empty 217 | if b.r == b.w { 218 | return 0, b.readErr() 219 | } 220 | } 221 | 222 | // copy as much as we can 223 | n = copy(p, b.buf[b.r:b.w]) 224 | b.r += n 225 | return n, nil 226 | } 227 | 228 | // Buffered returns the number of bytes that can be read from the current buffer. 229 | func (b *Reader) Buffered() int { return b.w - b.r } 230 | 231 | // buffered output 232 | 233 | // Writer implements buffering for an io.Writer object. 234 | // If an error occurs writing to a Writer, no more data will be 235 | // accepted and all subsequent writes will return the error. 236 | // After all data has been written, the client should call the 237 | // Flush method to guarantee all data has been forwarded to 238 | // the underlying io.Writer. 239 | type Writer struct { 240 | err error 241 | buf []byte 242 | n int 243 | wr io.Writer 244 | } 245 | 246 | // NewWriterSize returns a new Writer whose buffer has at least the specified 247 | // size. If the argument io.Writer is already a Writer with large enough 248 | // size, it returns the underlying Writer. 249 | func NewWriterSize(w io.Writer, size int) *Writer { 250 | // Is it already a Writer? 251 | b, ok := w.(*Writer) 252 | if ok && len(b.buf) >= size { 253 | return b 254 | } 255 | if size <= 0 { 256 | size = defaultBufSize 257 | } 258 | return &Writer{ 259 | buf: make([]byte, size), 260 | wr: w, 261 | } 262 | } 263 | 264 | // NewWriter returns a new Writer whose buffer has the default size. 265 | func NewWriter(w io.Writer) *Writer { 266 | return NewWriterSize(w, defaultBufSize) 267 | } 268 | 269 | // Reset discards any unflushed buffered data, clears any error, and 270 | // resets b to write its output to w. 271 | func (b *Writer) Reset(w io.Writer) { 272 | b.err = nil 273 | b.n = 0 274 | b.wr = w 275 | } 276 | 277 | // ResetBuffer discards any unflushed buffered data, clears any error, and 278 | // resets b to write its output to w. 279 | func (b *Writer) ResetBuffer(w io.Writer, buf []byte) { 280 | b.buf = buf 281 | b.err = nil 282 | b.n = 0 283 | b.wr = w 284 | } 285 | 286 | // Flush writes any buffered data to the underlying io.Writer. 287 | func (b *Writer) Flush() error { 288 | err := b.flush() 289 | return err 290 | } 291 | 292 | func (b *Writer) flush() error { 293 | if b.err != nil { 294 | return b.err 295 | } 296 | if b.n == 0 { 297 | return nil 298 | } 299 | n, err := b.wr.Write(b.buf[0:b.n]) 300 | if n < b.n && err == nil { 301 | err = io.ErrShortWrite 302 | } 303 | if err != nil { 304 | if n > 0 && n < b.n { 305 | copy(b.buf[0:b.n-n], b.buf[n:b.n]) 306 | } 307 | b.n -= n 308 | b.err = err 309 | return err 310 | } 311 | b.n = 0 312 | return nil 313 | } 314 | 315 | // Available returns how many bytes are unused in the buffer. 316 | func (b *Writer) Available() int { return len(b.buf) - b.n } 317 | 318 | // Buffered returns the number of bytes that have been written into the current buffer. 319 | func (b *Writer) Buffered() int { return b.n } 320 | 321 | // Write writes the contents of p into the buffer. 322 | // It returns the number of bytes written. 323 | // If nn < len(p), it also returns an error explaining 324 | // why the write is short. 325 | func (b *Writer) Write(p []byte) (nn int, err error) { 326 | for len(p) > b.Available() && b.err == nil { 327 | var n int 328 | if b.Buffered() == 0 { 329 | // Large write, empty buffer. 330 | // Write directly from p to avoid copy. 331 | n, b.err = b.wr.Write(p) 332 | } else { 333 | n = copy(b.buf[b.n:], p) 334 | b.n += n 335 | b.flush() 336 | } 337 | nn += n 338 | p = p[n:] 339 | } 340 | if b.err != nil { 341 | return nn, b.err 342 | } 343 | n := copy(b.buf[b.n:], p) 344 | b.n += n 345 | nn += n 346 | return nn, nil 347 | } 348 | 349 | // Peek returns the next n bytes with advancing the writer. The bytes stop 350 | // being used at the next write call. If Peek returns fewer than n bytes, it 351 | // also returns an error explaining why the read is short. The error is 352 | // ErrBufferFull if n is larger than b's buffer size. 353 | func (b *Writer) Peek(n int) ([]byte, error) { 354 | if n < 0 { 355 | return nil, ErrNegativeCount 356 | } 357 | if n > len(b.buf) { 358 | return nil, ErrBufferFull 359 | } 360 | for b.Available() < n && b.err == nil { 361 | b.flush() 362 | } 363 | if b.err != nil { 364 | return nil, b.err 365 | } 366 | d := b.buf[b.n : b.n+n] 367 | b.n += n 368 | return d, nil 369 | } 370 | -------------------------------------------------------------------------------- /container/omap/omap.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2011-12 Qtrac Ltd. 2 | // 3 | // This program or package and any associated files are licensed under the 4 | // Apache License, Version 2.0 (the "License"); you may not use these files 5 | // except in compliance with the License. You can get a copy of the License 6 | // at: http://www.apache.org/licenses/LICENSE-2.0. 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | // The data structure is a left-leaning red-black tree based on Robert 15 | // Sedgewick's implementations as described in 16 | // http://www.cs.princeton.edu/~rs/talks/LLRB/LLRB.pdf and 17 | // http://www.cs.princeton.edu/~rs/talks/LLRB/RedBlack.pdf, with some of 18 | // the code based on Lee Stanza's C++ code at 19 | // http://www.teachsolaisgames.com/articles/balanced_left_leaning.html 20 | // Thanks also to Russ Cox for many useful suggestions. 21 | 22 | // Package omap 通过红黑树实现了高效率的有序map。 23 | // 24 | // Keys 与 values 可以是任意类型,但key必须要有支持的less比较 方法。 25 | // 在通过调用New方法的时候,需要提供该方法。 26 | package omap 27 | 28 | import ( 29 | "strings" 30 | ) 31 | 32 | // NewStringKeyed 返回一个初始化完成的空有序Map,其key是大小写敏感的string。 33 | func NewStringKeyed() *Map { 34 | return &Map{less: func(a, b interface{}) bool { 35 | return a.(string) < b.(string) 36 | }} 37 | } 38 | 39 | // NewCaseFoldedKeyed 返回一个初始化完成的空有序Map,其key是大小写不敏感的string。 40 | func NewCaseFoldedKeyed() *Map { 41 | return &Map{less: func(a, b interface{}) bool { 42 | return strings.ToLower(a.(string)) < strings.ToLower(b.(string)) 43 | }} 44 | } 45 | 46 | // NewIntKeyed 返回一个初始化的空有序Map,其key按照int类型识别。 47 | func NewIntKeyed() *Map { 48 | return &Map{less: func(a, b interface{}) bool { 49 | return a.(int) < b.(int) 50 | }} 51 | } 52 | 53 | // NewInt64Keyed 返回一个初始化的空有序Map,其key按照int64类型识别。 54 | func NewInt64Keyed() *Map { 55 | return &Map{less: func(a, b interface{}) bool { 56 | return a.(int64) < b.(int64) 57 | }} 58 | } 59 | 60 | // NewFloat64Keyed 返回一个初始化的空有序Map,其key按照按照float64类型识别。 61 | func NewFloat64Keyed() *Map { 62 | return &Map{less: func(a, b interface{}) bool { 63 | return a.(float64) < b.(float64) 64 | }} 65 | } 66 | 67 | // New 返回一个空的有序Map,其比较函数func(interface{}, interface{}) bool,需要用户自己识别。 68 | // 比如: 69 | // type Point { X, Y int } 70 | // pointMap := omap.New(func(a, b interface{}) bool { 71 | // α, β := a.(Point), b.(Point) 72 | // if α.X != β.X { 73 | // return α.X < β.X 74 | // } 75 | // return α.Y < β.Y 76 | // }) 77 | func New(less func(interface{}, interface{}) bool) *Map { 78 | return &Map{less: less} 79 | } 80 | 81 | // Map 是一个key有序map。 82 | // The zero value is an invalid map! 通过包中提供的构造函数,比如New()等,创建特殊“键-值”类别的map。 83 | type Map struct { 84 | root *node 85 | less func(interface{}, interface{}) bool 86 | length int 87 | } 88 | 89 | type node struct { 90 | key, value interface{} 91 | red bool 92 | left, right *node 93 | } 94 | 95 | // Insert inserts a new key-value into the Map and returns true; or 96 | // replaces an existing key-value pair's value if the keys are equal and 97 | // returns false. For example: 98 | // inserted := myMap.Insert(key, value). 99 | func (m *Map) Insert(key, value interface{}) (inserted bool) { 100 | m.root, inserted = m.insert(m.root, key, value) 101 | m.root.red = false 102 | if inserted { 103 | m.length++ 104 | } 105 | return inserted 106 | } 107 | 108 | // Find returns the value and true if the key is in the Map or nil and 109 | // false otherwise. For example: 110 | // value, found := myMap.Find(key). 111 | func (m *Map) Find(key interface{}) (value interface{}, found bool) { 112 | root := m.root 113 | for root != nil { 114 | if m.less(key, root.key) { 115 | root = root.left 116 | } else if m.less(root.key, key) { 117 | root = root.right 118 | } else { 119 | return root.value, true 120 | } 121 | } 122 | return nil, false 123 | } 124 | 125 | // First 返回有序Map中,最左上角叶子节点的KV组。 126 | func (m *Map) First() (key, value interface{}, found bool) { 127 | if m.root == nil { 128 | return nil, nil, false 129 | } 130 | root := m.root 131 | for root.left != nil { 132 | root = root.left 133 | } 134 | key = root.key 135 | value = root.value 136 | found = true 137 | return 138 | } 139 | 140 | // Latest 返回有序Map中,最右上角叶子节点的KV组。 141 | func (m *Map) Latest() (key, value interface{}, found bool) { 142 | if m.root == nil { 143 | return nil, nil, false 144 | } 145 | root := m.root 146 | for root.right != nil { 147 | root = root.right 148 | } 149 | 150 | key = root.key 151 | value = root.value 152 | found = true 153 | return 154 | } 155 | 156 | // Delete deletes the key-value with the given key from the Map and returns 157 | // true, or does nothing and returns false if there is no key-value with 158 | // the given key. For example: 159 | // deleted := myMap.Delete(key). 160 | func (m *Map) Delete(key interface{}) (deleted bool) { 161 | if m.root != nil { 162 | if m.root, deleted = m.remove(m.root, key); m.root != nil { 163 | m.root.red = false 164 | } 165 | } 166 | if deleted { 167 | m.length-- 168 | } 169 | return deleted 170 | } 171 | 172 | // Do 调用给定的func,有序将key-value作为输入参数执行。 173 | func (m *Map) Do(function func(interface{}, interface{})) { 174 | do(m.root, function) 175 | } 176 | 177 | // Len 返回map中的键值对数量 178 | func (m *Map) Len() int { 179 | return m.length 180 | } 181 | 182 | func (m *Map) insert(root *node, key, value interface{}) (*node, bool) { 183 | inserted := false 184 | if root == nil { // If the key was in the tree it would belong here 185 | return &node{key: key, value: value, red: true}, true 186 | } 187 | if isRed(root.left) && isRed(root.right) { 188 | colorFlip(root) 189 | } 190 | if m.less(key, root.key) { 191 | root.left, inserted = m.insert(root.left, key, value) 192 | } else if m.less(root.key, key) { 193 | root.right, inserted = m.insert(root.right, key, value) 194 | } else { // The key is already in the tree so just replace its value 195 | root.value = value 196 | } 197 | if isRed(root.right) && !isRed(root.left) { 198 | root = rotateLeft(root) 199 | } 200 | if isRed(root.left) && isRed(root.left.left) { 201 | root = rotateRight(root) 202 | } 203 | return root, inserted 204 | } 205 | 206 | func isRed(root *node) bool { return root != nil && root.red } 207 | 208 | func colorFlip(root *node) { 209 | root.red = !root.red 210 | if root.left != nil { 211 | root.left.red = !root.left.red 212 | } 213 | if root.right != nil { 214 | root.right.red = !root.right.red 215 | } 216 | } 217 | 218 | func rotateLeft(root *node) *node { 219 | // 220 | // The illation of left rotation 221 | // 222 | // | | 223 | // root x 224 | // / \ left rotate / \ 225 | // α x -------------> root x 226 | // / \ / \ 227 | // β γ α β 228 | // 229 | // It should be note that during the rotating we do not change 230 | // the Nodes' color. 231 | // 232 | x := root.right 233 | root.right = x.left 234 | x.left = root 235 | x.red = root.red 236 | root.red = true 237 | return x 238 | } 239 | 240 | func rotateRight(root *node) *node { 241 | // 242 | // The illation of right rotation 243 | // 244 | // | | 245 | // root X 246 | // / \ right rotate / \ 247 | // X γ -------------> α root 248 | // / \ / \ 249 | // α β β γ 250 | // 251 | // It should be note that during the rotating we do not change 252 | // the Nodes' color. 253 | // 254 | x := root.left 255 | root.left = x.right 256 | x.right = root 257 | x.red = root.red 258 | root.red = true 259 | return x 260 | } 261 | 262 | func do(root *node, function func(interface{}, interface{})) { 263 | if root != nil { 264 | do(root.left, function) 265 | function(root.key, root.value) 266 | do(root.right, function) 267 | } 268 | } 269 | 270 | // We do not provide an exported First() method because this is an 271 | // implementation detail. 272 | func first(root *node) *node { 273 | for root.left != nil { 274 | root = root.left 275 | } 276 | return root 277 | } 278 | 279 | func (m *Map) remove(root *node, key interface{}) (*node, bool) { 280 | deleted := false 281 | if m.less(key, root.key) { 282 | if root.left != nil { 283 | if !isRed(root.left) && !isRed(root.left.left) { 284 | root = moveRedLeft(root) 285 | } 286 | root.left, deleted = m.remove(root.left, key) 287 | } 288 | } else { 289 | if isRed(root.left) { 290 | root = rotateRight(root) 291 | } 292 | if !m.less(key, root.key) && !m.less(root.key, key) && 293 | root.right == nil { 294 | return nil, true 295 | } 296 | if root.right != nil { 297 | if !isRed(root.right) && !isRed(root.right.left) { 298 | root = moveRedRight(root) 299 | } 300 | if !m.less(key, root.key) && !m.less(root.key, key) { 301 | smallest := first(root.right) 302 | root.key = smallest.key 303 | root.value = smallest.value 304 | root.right = deleteMinimum(root.right) 305 | deleted = true 306 | } else { 307 | root.right, deleted = m.remove(root.right, key) 308 | } 309 | } 310 | } 311 | return fixUp(root), deleted 312 | } 313 | 314 | func moveRedLeft(root *node) *node { 315 | colorFlip(root) 316 | if root.right != nil && isRed(root.right.left) { 317 | root.right = rotateRight(root.right) 318 | root = rotateLeft(root) 319 | colorFlip(root) 320 | } 321 | return root 322 | } 323 | 324 | func moveRedRight(root *node) *node { 325 | colorFlip(root) 326 | if root.left != nil && isRed(root.left.left) { 327 | root = rotateRight(root) 328 | colorFlip(root) 329 | } 330 | return root 331 | } 332 | 333 | func deleteMinimum(root *node) *node { 334 | if root.left == nil { 335 | return nil 336 | } 337 | if !isRed(root.left) && !isRed(root.left.left) { 338 | root = moveRedLeft(root) 339 | } 340 | root.left = deleteMinimum(root.left) 341 | return fixUp(root) 342 | } 343 | 344 | func fixUp(root *node) *node { 345 | if isRed(root.right) { 346 | root = rotateLeft(root) 347 | } 348 | if isRed(root.left) && isRed(root.left.left) { 349 | root = rotateRight(root) 350 | } 351 | if isRed(root.left) && isRed(root.right) { 352 | colorFlip(root) 353 | } 354 | return root 355 | } 356 | -------------------------------------------------------------------------------- /hash/cityhash/cityhash.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Go implementation of Google city hash (MIT license) 3 | * https://code.google.com/p/cityhash/ 4 | * 5 | * MIT License http://www.opensource.org/licenses/mit-license.php 6 | * 7 | * I don't even want to pretend to understand the details of city hash. 8 | * I am only reproducing the logic in Go as faithfully as I can. 9 | * 10 | */ 11 | 12 | package cityhash 13 | 14 | import ( 15 | "encoding/binary" 16 | "unsafe" 17 | ) 18 | 19 | /* 20 | var ( 21 | little bool 22 | ) 23 | func init() { 24 | if IsLittleEndian() { 25 | little = true 26 | } else { 27 | little = false 28 | } 29 | } 30 | */ 31 | 32 | func IsLittleEndian() bool { 33 | var i int32 = 0x01020304 34 | u := unsafe.Pointer(&i) 35 | pb := (*byte)(u) 36 | b := *pb 37 | return (b == 0x04) 38 | } 39 | 40 | func unalignedLoad64(p []byte) (result uint64) { 41 | return binary.LittleEndian.Uint64(p) 42 | /* 43 | if little { 44 | result = binary.LittleEndian.Uint64(p) 45 | } else { 46 | result = binary.BigEndian.Uint64(p) 47 | } 48 | return result 49 | */ 50 | } 51 | 52 | func unalignedLoad32(p []byte) (result uint32) { 53 | return binary.LittleEndian.Uint32(p) 54 | /* 55 | if little { 56 | result = binary.LittleEndian.Uint32(p) 57 | } else { 58 | result = binary.BigEndian.Uint32(p) 59 | } 60 | return result 61 | */ 62 | } 63 | 64 | func bswap64(x uint64) uint64 { 65 | // Copied from netbsd's bswap64.c 66 | return ((x << 56) & 0xff00000000000000) | 67 | ((x << 40) & 0x00ff000000000000) | 68 | ((x << 24) & 0x0000ff0000000000) | 69 | ((x << 8) & 0x000000ff00000000) | 70 | ((x >> 8) & 0x00000000ff000000) | 71 | ((x >> 24) & 0x0000000000ff0000) | 72 | ((x >> 40) & 0x000000000000ff00) | 73 | ((x >> 56) & 0x00000000000000ff) 74 | } 75 | 76 | func bswap32(x uint32) uint32 { 77 | // Copied from netbsd's bswap32.c 78 | return ((x << 24) & 0xff000000) | 79 | ((x << 8) & 0x00ff0000) | 80 | ((x >> 8) & 0x0000ff00) | 81 | ((x >> 24) & 0x000000ff) 82 | } 83 | 84 | func uint32InExpectedOrder(x uint32) uint32 { 85 | /* 86 | if !little { 87 | return bswap32(x) 88 | } 89 | */ 90 | 91 | return x 92 | } 93 | 94 | func uint64InExpectedOrder(x uint64) uint64 { 95 | /* 96 | if !little { 97 | return bswap64(x) 98 | } 99 | */ 100 | 101 | return x 102 | } 103 | 104 | // If I understand the original code correctly, it's expecting to load either 8 or 4 105 | // byes in little endian order. so let's just simplify it a bit since we will do that 106 | // anyway.. 107 | // https://code.google.com/p/cityhash/source/browse/trunk/src/city.cc#112 108 | func fetch64(p []byte) uint64 { 109 | return binary.LittleEndian.Uint64(p) 110 | //return uint64InExpectedOrder(unalignedLoad64(p)) 111 | } 112 | 113 | func fetch32(p []byte) uint32 { 114 | return binary.LittleEndian.Uint32(p) 115 | //return uint32InExpectedOrder(unalignedLoad32(p)) 116 | } 117 | 118 | const ( 119 | k0 uint64 = 0xc3a5c85c97cb3127 120 | k1 uint64 = 0xb492b66fbe98f273 121 | k2 uint64 = 0x9ae16a3b2f90404f 122 | c1 uint32 = 0xcc9e2d51 123 | c2 uint32 = 0x1b873593 124 | ) 125 | 126 | func fmix(h uint32) uint32 { 127 | h ^= h >> 16 128 | h *= 0x85ebca6b 129 | h ^= h >> 13 130 | h *= 0xc2b2ae35 131 | h ^= h >> 16 132 | return h 133 | } 134 | 135 | func rotate64(val uint64, shift uint32) uint64 { 136 | // Avoid shifting by 64: doing so yields an undefined result. 137 | if shift != 0 { 138 | return ((val >> shift) | (val << (64 - shift))) 139 | } 140 | 141 | return val 142 | } 143 | 144 | func rotate32(val uint32, shift uint32) uint32 { 145 | // Avoid shifting by 32: doing so yields an undefined result. 146 | if shift != 0 { 147 | return ((val >> shift) | (val << (32 - shift))) 148 | } 149 | 150 | return val 151 | } 152 | 153 | func swap64(a, b *uint64) { 154 | *a, *b = *b, *a 155 | } 156 | 157 | func swap32(a, b *uint32) { 158 | *a, *b = *b, *a 159 | } 160 | 161 | func permute3(a, b, c *uint32) { 162 | swap32(a, b) 163 | swap32(a, c) 164 | } 165 | 166 | func mur(a, h uint32) uint32 { 167 | a *= c1 168 | a = rotate32(a, 17) 169 | a *= c2 170 | h ^= a 171 | h = rotate32(h, 19) 172 | 173 | //return h * 5 + 0xe6546b64 174 | z := h*5 + 0xe6546b64 175 | return z 176 | } 177 | 178 | func hash32Len13to24(s []byte, length uint32) uint32 { 179 | var a uint32 = fetch32(s[(length>>1)-4:]) 180 | var b uint32 = fetch32(s[4:]) 181 | var c uint32 = fetch32(s[length-8:]) 182 | var d uint32 = fetch32(s[(length >> 1):]) 183 | var e uint32 = fetch32(s) 184 | var f uint32 = fetch32(s[length-4:]) 185 | var h uint32 = length 186 | 187 | return fmix(mur(f, mur(e, mur(d, mur(c, mur(b, mur(a, h))))))) 188 | } 189 | 190 | func hash32Len0to4(s []byte, length uint32) uint32 { 191 | var b, c uint32 = 0, 9 192 | 193 | tmp := s[:length] 194 | for _, v := range tmp { 195 | b = uint32(int64(b)*int64(c1) + int64(int8(v))) 196 | c ^= b 197 | } 198 | 199 | return fmix(mur(b, mur(length, c))) 200 | } 201 | 202 | func hash32Len5to12(s []byte, length uint32) uint32 { 203 | var a, b, c uint32 = length, length * 5, 9 204 | var d uint32 = b 205 | 206 | a += fetch32(s) 207 | b += fetch32(s[length-4:]) 208 | c += fetch32(s[((length >> 1) & 4):]) 209 | 210 | return fmix(mur(c, mur(b, mur(a, d)))) 211 | } 212 | 213 | func CityHash32(s []byte, length uint32) uint32 { 214 | if length <= 4 { 215 | return hash32Len0to4(s, length) 216 | } else if length <= 12 { 217 | return hash32Len5to12(s, length) 218 | } else if length <= 24 { 219 | return hash32Len13to24(s, length) 220 | } 221 | 222 | // length > 24 223 | var h uint32 = length 224 | var g uint32 = c1 * length 225 | var f uint32 = g 226 | var a0 uint32 = rotate32(fetch32(s[length-4:])*c1, 17) * c2 227 | var a1 uint32 = rotate32(fetch32(s[length-8:])*c1, 17) * c2 228 | var a2 uint32 = rotate32(fetch32(s[length-16:])*c1, 17) * c2 229 | var a3 uint32 = rotate32(fetch32(s[length-12:])*c1, 17) * c2 230 | var a4 uint32 = rotate32(fetch32(s[length-20:])*c1, 17) * c2 231 | h ^= a0 232 | h = rotate32(h, 19) 233 | h = h*5 + 0xe6546b64 234 | h ^= a2 235 | h = rotate32(h, 19) 236 | h = h*5 + 0xe6546b64 237 | g ^= a1 238 | g = rotate32(g, 19) 239 | g = g*5 + 0xe6546b64 240 | g ^= a3 241 | g = rotate32(g, 19) 242 | g = g*5 + 0xe6546b64 243 | f += a4 244 | f = rotate32(f, 19) 245 | f = f*5 + 0xe6546b64 246 | 247 | var iters uint32 = (length - 1) / 20 248 | for { 249 | var a0 uint32 = rotate32(fetch32(s)*c1, 17) * c2 250 | var a1 uint32 = fetch32(s[4:]) 251 | var a2 uint32 = rotate32(fetch32(s[8:])*c1, 17) * c2 252 | var a3 uint32 = rotate32(fetch32(s[12:])*c1, 17) * c2 253 | var a4 uint32 = fetch32(s[16:]) 254 | h ^= a0 255 | h = rotate32(h, 18) 256 | h = h*5 + 0xe6546b64 257 | f += a1 258 | f = rotate32(f, 19) 259 | f = f * c1 260 | g += a2 261 | g = rotate32(g, 18) 262 | g = g*5 + 0xe6546b64 263 | h ^= a3 + a1 264 | h = rotate32(h, 19) 265 | h = h*5 + 0xe6546b64 266 | g ^= a4 267 | g = bswap32(g) * 5 268 | h += a4 * 5 269 | h = bswap32(h) 270 | f += a0 271 | permute3(&f, &h, &g) 272 | s = s[20:] 273 | 274 | iters-- 275 | if iters == 0 { 276 | break 277 | } 278 | } 279 | 280 | g = rotate32(g, 11) * c1 281 | g = rotate32(g, 17) * c1 282 | f = rotate32(f, 11) * c1 283 | f = rotate32(f, 17) * c1 284 | h = rotate32(h+g, 19) 285 | h = h*5 + 0xe6546b64 286 | h = rotate32(h, 17) * c1 287 | h = rotate32(h+f, 19) 288 | h = h*5 + 0xe6546b64 289 | h = rotate32(h, 17) * c1 290 | return h 291 | } 292 | 293 | func shiftMix(val uint64) uint64 { 294 | return val ^ (val >> 47) 295 | } 296 | 297 | type Uint128 [2]uint64 298 | 299 | func (this *Uint128) setLower64(l uint64) { 300 | this[0] = l 301 | } 302 | 303 | func (this *Uint128) setHigher64(h uint64) { 304 | this[1] = h 305 | } 306 | 307 | func (this Uint128) Lower64() uint64 { 308 | return this[0] 309 | } 310 | 311 | func (this Uint128) Higher64() uint64 { 312 | return this[1] 313 | } 314 | 315 | func (this Uint128) Bytes() []byte { 316 | b := make([]byte, 16) 317 | binary.LittleEndian.PutUint64(b, this[0]) 318 | binary.LittleEndian.PutUint64(b[8:], this[1]) 319 | return b 320 | } 321 | 322 | func hash128to64(x Uint128) uint64 { 323 | // Murmur-inspired hashing. 324 | const kMul uint64 = 0x9ddfea08eb382d69 325 | var a uint64 = (x.Lower64() ^ x.Higher64()) * kMul 326 | a ^= (a >> 47) 327 | var b uint64 = (x.Higher64() ^ a) * kMul 328 | b ^= (b >> 47) 329 | b *= kMul 330 | return b 331 | } 332 | 333 | func hashLen16(u, v uint64) uint64 { 334 | return hash128to64(Uint128{u, v}) 335 | } 336 | 337 | func hashLen16_3(u, v, mul uint64) uint64 { 338 | // Murmur-inspired hashing. 339 | var a uint64 = (u ^ v) * mul 340 | a ^= (a >> 47) 341 | var b uint64 = (v ^ a) * mul 342 | b ^= (b >> 47) 343 | b *= mul 344 | return b 345 | } 346 | 347 | func hashLen0to16(s []byte, length uint32) uint64 { 348 | if length >= 8 { 349 | var mul uint64 = k2 + uint64(length)*2 350 | var a uint64 = fetch64(s) + k2 351 | var b uint64 = fetch64(s[length-8:]) 352 | var c uint64 = rotate64(b, 37)*mul + a 353 | var d uint64 = (rotate64(a, 25) + b) * mul 354 | return hashLen16_3(c, d, mul) 355 | } 356 | 357 | if length >= 4 { 358 | var mul uint64 = k2 + uint64(length)*2 359 | var a uint64 = uint64(fetch32(s)) 360 | return hashLen16_3(uint64(length)+(a<<3), uint64(fetch32(s[length-4:])), mul) 361 | } 362 | 363 | if length > 0 { 364 | var a uint8 = uint8(s[0]) 365 | var b uint8 = uint8(s[length>>1]) 366 | var c uint8 = uint8(s[length-1]) 367 | var y uint32 = uint32(a) + (uint32(b) << 8) 368 | var z uint32 = length + (uint32(c) << 2) 369 | return shiftMix(uint64(y)*k2^uint64(z)*k0) * k2 370 | } 371 | 372 | return k2 373 | } 374 | 375 | func hashLen17to32(s []byte, length uint32) uint64 { 376 | var mul uint64 = k2 + uint64(length)*2 377 | var a uint64 = fetch64(s) * k1 378 | var b uint64 = fetch64(s[8:]) 379 | var c uint64 = fetch64(s[length-8:]) * mul 380 | var d uint64 = fetch64(s[length-16:]) * k2 381 | return hashLen16_3(rotate64(a+b, 43)+rotate64(c, 30)+d, a+rotate64(b+k2, 18)+c, mul) 382 | } 383 | 384 | func weakHashLen32WithSeeds(w, x, y, z, a, b uint64) Uint128 { 385 | a += w 386 | b = rotate64(b+a+z, 21) 387 | var c uint64 = a 388 | a += x 389 | a += y 390 | b += rotate64(a, 44) 391 | return Uint128{a + z, b + c} 392 | } 393 | 394 | func weakHashLen32WithSeeds_3(s []byte, a, b uint64) Uint128 { 395 | return weakHashLen32WithSeeds(fetch64(s), fetch64(s[8:]), fetch64(s[16:]), fetch64(s[24:]), a, b) 396 | } 397 | 398 | func hashLen33to64(s []byte, length uint32) uint64 { 399 | var mul uint64 = k2 + uint64(length)*2 400 | var a uint64 = fetch64(s) * k2 401 | var b uint64 = fetch64(s[8:]) 402 | var c uint64 = fetch64(s[length-24:]) 403 | var d uint64 = fetch64(s[length-32:]) 404 | var e uint64 = fetch64(s[16:]) * k2 405 | var f uint64 = fetch64(s[24:]) * 9 406 | var g uint64 = fetch64(s[length-8:]) 407 | var h uint64 = fetch64(s[length-16:]) * mul 408 | var u uint64 = rotate64(a+g, 43) + (rotate64(b, 30)+c)*9 409 | var v uint64 = ((a + g) ^ d) + f + 1 410 | var w uint64 = bswap64((u+v)*mul) + h 411 | var x uint64 = rotate64(e+f, 42) + c 412 | var y uint64 = (bswap64((v+w)*mul) + g) * mul 413 | var z uint64 = e + f + c 414 | a = bswap64((x+z)*mul+y) + b 415 | b = shiftMix((z+a)*mul+d+h) * mul 416 | return b + x 417 | } 418 | 419 | func CityHash64(s []byte, length uint32) uint64 { 420 | if length <= 32 { 421 | if length <= 16 { 422 | return hashLen0to16(s, length) 423 | } else { 424 | return hashLen17to32(s, length) 425 | } 426 | } else if length <= 64 { 427 | return hashLen33to64(s, length) 428 | } 429 | 430 | var x uint64 = fetch64(s[length-40:]) 431 | var y uint64 = fetch64(s[length-16:]) + fetch64(s[length-56:]) 432 | var z uint64 = hashLen16(fetch64(s[length-48:])+uint64(length), fetch64(s[length-24:])) 433 | var v Uint128 = weakHashLen32WithSeeds_3(s[length-64:], uint64(length), z) 434 | var w Uint128 = weakHashLen32WithSeeds_3(s[length-32:], y+k1, x) 435 | x = x*k1 + fetch64(s) 436 | 437 | length = (length - 1) & ^uint32(63) 438 | for { 439 | x = rotate64(x+y+v.Lower64()+fetch64(s[8:]), 37) * k1 440 | y = rotate64(y+v.Higher64()+fetch64(s[48:]), 42) * k1 441 | x ^= w.Higher64() 442 | y += v.Lower64() + fetch64(s[40:]) 443 | z = rotate64(z+w.Lower64(), 33) * k1 444 | v = weakHashLen32WithSeeds_3(s, v.Higher64()*k1, x+w.Lower64()) 445 | w = weakHashLen32WithSeeds_3(s[32:], z+w.Higher64(), y+fetch64(s[16:])) 446 | swap64(&z, &x) 447 | s = s[64:] 448 | length -= 64 449 | 450 | if length == 0 { 451 | break 452 | } 453 | } 454 | 455 | return hashLen16(hashLen16(v.Lower64(), w.Lower64())+shiftMix(y)*k1+z, hashLen16(v.Higher64(), w.Higher64())+x) 456 | } 457 | 458 | func CityHash64WithSeed(s []byte, length uint32, seed uint64) uint64 { 459 | return CityHash64WithSeeds(s, length, k2, seed) 460 | } 461 | 462 | func CityHash64WithSeeds(s []byte, length uint32, seed0, seed1 uint64) uint64 { 463 | return hashLen16(CityHash64(s, length)-seed0, seed1) 464 | } 465 | 466 | func cityMurmur(s []byte, length uint32, seed Uint128) Uint128 { 467 | var a uint64 = seed.Lower64() 468 | var b uint64 = seed.Higher64() 469 | var c uint64 = 0 470 | var d uint64 = 0 471 | var l int32 = int32(length) - 16 472 | 473 | if l <= 0 { // len <= 16 474 | a = shiftMix(a*k1) * k1 475 | c = b*k1 + hashLen0to16(s, length) 476 | 477 | if length >= 8 { 478 | d = shiftMix(a + fetch64(s)) 479 | } else { 480 | d = shiftMix(a + c) 481 | } 482 | } else { // len > 16 483 | c = hashLen16(fetch64(s[length-8:])+k1, a) 484 | d = hashLen16(b+uint64(length), c+fetch64(s[length-16:])) 485 | a += d 486 | 487 | for { 488 | a ^= shiftMix(fetch64(s)*k1) * k1 489 | a *= k1 490 | b ^= a 491 | c ^= shiftMix(fetch64(s[8:])*k1) * k1 492 | c *= k1 493 | d ^= c 494 | s = s[16:] 495 | l -= 16 496 | 497 | if l <= 0 { 498 | break 499 | } 500 | } 501 | } 502 | 503 | a = hashLen16(a, c) 504 | b = hashLen16(d, b) 505 | return Uint128{a ^ b, hashLen16(b, a)} 506 | } 507 | 508 | func CityHash128WithSeed(s []byte, length uint32, seed Uint128) Uint128 { 509 | if length < 128 { 510 | return cityMurmur(s, length, seed) 511 | } 512 | 513 | orig_length := length 514 | var t []byte = s 515 | 516 | // We expect length >= 128 to be the common case. Keep 56 bytes of state: 517 | // v, w, x, y, and z. 518 | var v, w Uint128 519 | var x uint64 = seed.Lower64() 520 | var y uint64 = seed.Higher64() 521 | var z uint64 = uint64(length) * k1 522 | 523 | v.setLower64(rotate64(y^k1, 49)*k1 + fetch64(s)) 524 | v.setHigher64(rotate64(v.Lower64(), 42)*k1 + fetch64(s[8:])) 525 | w.setLower64(rotate64(y+z, 35)*k1 + x) 526 | w.setHigher64(rotate64(x+fetch64(s[88:]), 53) * k1) 527 | 528 | // This is the same inner loop as CityHash64(), manually unrolled. 529 | for { 530 | x = rotate64(x+y+v.Lower64()+fetch64(s[8:]), 37) * k1 531 | y = rotate64(y+v.Higher64()+fetch64(s[48:]), 42) * k1 532 | x ^= w.Higher64() 533 | y += v.Lower64() + fetch64(s[40:]) 534 | z = rotate64(z+w.Lower64(), 33) * k1 535 | v = weakHashLen32WithSeeds_3(s, v.Higher64()*k1, x+w.Lower64()) 536 | w = weakHashLen32WithSeeds_3(s[32:], z+w.Higher64(), y+fetch64(s[16:])) 537 | swap64(&z, &x) 538 | s = s[64:] 539 | x = rotate64(x+y+v.Lower64()+fetch64(s[8:]), 37) * k1 540 | y = rotate64(y+v.Higher64()+fetch64(s[48:]), 42) * k1 541 | x ^= w.Higher64() 542 | y += v.Lower64() + fetch64(s[40:]) 543 | z = rotate64(z+w.Lower64(), 33) * k1 544 | v = weakHashLen32WithSeeds_3(s, v.Higher64()*k1, x+w.Lower64()) 545 | w = weakHashLen32WithSeeds_3(s[32:], z+w.Higher64(), y+fetch64(s[16:])) 546 | swap64(&z, &x) 547 | s = s[64:] 548 | length -= 128 549 | 550 | if length < 128 { 551 | break 552 | } 553 | } 554 | 555 | x += rotate64(v.Lower64()+z, 49) * k0 556 | y = y*k0 + rotate64(w.Higher64(), 37) 557 | z = z*k0 + rotate64(w.Lower64(), 27) 558 | w.setLower64(w.Lower64() * 9) 559 | v.setLower64(v.Lower64() * k0) 560 | 561 | // If 0 < length < 128, hash up to 4 chunks of 32 bytes each from the end of s. 562 | var tail_done uint32 563 | for tail_done = 0; tail_done < length; { 564 | tail_done += 32 565 | y = rotate64(x+y, 42)*k0 + v.Higher64() 566 | w.setLower64(w.Lower64() + fetch64(t[orig_length-tail_done+16:])) 567 | x = x*k0 + w.Lower64() 568 | z += w.Higher64() + fetch64(t[orig_length-tail_done:]) 569 | w.setHigher64(w.Higher64() + v.Lower64()) 570 | v = weakHashLen32WithSeeds_3(t[orig_length-tail_done:], v.Lower64()+z, v.Higher64()) 571 | v.setLower64(v.Lower64() * k0) 572 | } 573 | 574 | // At this point our 56 bytes of state should contain more than 575 | // enough information for a strong 128-bit hash. We use two 576 | // different 56-byte-to-8-byte hashes to get a 16-byte final result. 577 | x = hashLen16(x, v.Lower64()) 578 | y = hashLen16(y+z, w.Lower64()) 579 | 580 | return Uint128{hashLen16(x+v.Higher64(), w.Higher64()) + y, 581 | hashLen16(x+w.Higher64(), y+v.Higher64())} 582 | } 583 | 584 | func CityHash128(s []byte, length uint32) (result Uint128) { 585 | if length >= 16 { 586 | result = CityHash128WithSeed(s[16:length], length-16, Uint128{fetch64(s), fetch64(s[8:length]) + k0}) 587 | } else { 588 | result = CityHash128WithSeed(s, length, Uint128{k0, k1}) 589 | } 590 | 591 | return 592 | } 593 | --------------------------------------------------------------------------------