├── .gitignore ├── .idea └── .gitignore ├── README.md ├── algorithms └── snowflake.go ├── dbms ├── README.md ├── gmongo │ ├── mongo.go │ └── options.go ├── gmysql │ ├── mysql.go │ └── options.go └── gredis │ ├── hook.go │ ├── options.go │ ├── redis.go │ ├── res │ └── get_lock.lua │ └── timer.go ├── gcache ├── README.md ├── cache.go ├── cache_test.go ├── func.go ├── func_test.go ├── lru.go ├── policy.go ├── ring.go ├── store.go └── timer.go ├── gcast ├── README.md ├── cast.go ├── cast_test.go ├── caste.go └── timeformattype_string.go ├── gconfig └── gconfig.go ├── gemail ├── config.go └── gemail.go ├── gerr ├── README.md ├── err_code.go ├── err_handle.go ├── err_warn.go └── errtypes_customize.go ├── gexit ├── README.md └── graceful_exit.go ├── ginfos ├── infos.go ├── marshal.go └── version.go ├── gkafka ├── README.md ├── consumer_group.go ├── producer_with_async.go ├── producer_with_sync.go ├── types.go └── with_options.go ├── glogs ├── README.md ├── glogrotate │ └── rotate.go ├── glogrus │ ├── config.go │ ├── daily_log.go │ ├── logrus.go │ └── new_logrus.go ├── gpanic │ ├── alert.go │ ├── panics.go │ └── panics_windows.go └── grecover │ └── recover.go ├── go.mod ├── go.sum ├── gstl ├── README.md ├── compare │ └── less.go ├── fns │ ├── README.md │ ├── fns.go │ └── fns_test.go ├── heap │ ├── decl.go │ ├── heap.go │ └── heap_test.go ├── options │ └── options.go ├── rbtree │ ├── README.md │ ├── node.go │ ├── rbtree.go │ └── rbtree_test.go ├── set │ ├── README.md │ ├── decl.go │ ├── iterator.go │ ├── multi_set.go │ ├── set.go │ └── set_test.go ├── smap │ ├── README.md │ ├── decl.go │ ├── iterator.go │ ├── map.go │ ├── map_test.go │ └── multi_map.go └── vector │ ├── README.md │ ├── decl.go │ ├── vector.go │ ├── vector_iter.go │ └── vector_test.go ├── gtime └── gtime.go ├── gtoken └── gtoken.go ├── gwarn ├── README.md ├── types.go └── warn2lark.go ├── gzip └── gzip.go └── test-example ├── README.md ├── config.yaml ├── dbms_mongo_test.go ├── dbms_mysql_test.go ├── dbms_redis_test.go ├── docker_compose_kafka.yaml ├── docker_compose_kafka_cluster.yaml ├── game_ludo_test.go ├── gcache_test.go ├── gcast_test.go ├── gerr_test.go ├── gexit_test.go ├── gkafka_test.go ├── go.mod ├── go.sum ├── gstl_test.go ├── gtoken_test.go ├── gwarn_test.go ├── hexdeximal_test.go ├── image_test.go └── vedio_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *.xml 3 | *.iml 4 | /test-example/panic.log 5 | /test-example/log/ 6 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # 默认忽略的文件 2 | /shelf/ 3 | /workspace.xml 4 | # 基于编辑器的 HTTP 客户端请求 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-utils 2 | 3 | common components 4 | 5 | ## Todo 6 | - [x] dbms test 7 | - [x] gcache test 8 | - [x] gcast test 9 | - [x] gexit test 10 | - [x] gtime test 11 | - [x] gtoken test 12 | - [x] galert test 13 | - [x] gslice test 14 | - [x] gerr test 15 | - [x] gerr test 16 | - [x] glogs test 17 | - [ ] gkafka test 18 | - [ ] gnet test 19 | - [ ] grpclient test 20 | - [ ] gqpst test 21 | - [ ] xxxxxxxx test 22 | - 23 | - [ ] gcfg test 24 | - [ ] gemail test 25 | - [ ] ginfos test 26 | - [ ] gzip test 27 | 28 | ## Tips 29 | ### go mod 30 | /go-utils/go.mod is needed, or it will be default version go1.16 31 | ### go mod path & go version 32 | ``` 33 | sudo vim /etc/profile 34 | vim .profile 35 | 36 | export PATH=$PATH:/usr/local/go/bin 37 | export GOPROXY="https://goproxy.cn" 38 | 39 | ``` 40 | -------------------------------------------------------------------------------- /algorithms/snowflake.go: -------------------------------------------------------------------------------- 1 | package algorithms 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/bwmarrin/snowflake" 7 | "github.com/sirupsen/logrus" 8 | ) 9 | 10 | var node *snowflake.Node 11 | 12 | type snowflakeMgr struct { 13 | } 14 | 15 | var Snowflake snowflakeMgr 16 | 17 | func init() { 18 | var err error 19 | machineId := rand.Int63() % 1024 20 | node, err = snowflake.NewNode(machineId) 21 | if err != nil { 22 | logrus.Fatalf("init snowflake node error: %v", err) 23 | } 24 | } 25 | 26 | func (snowflakeMgr) NextOptStreamID() int64 { 27 | return node.Generate().Int64() 28 | } 29 | 30 | func (snowflakeMgr) NextRoundID() int64 { 31 | return node.Generate().Int64() 32 | } 33 | 34 | func (snowflakeMgr) NextRpcSeqId() int64 { 35 | return node.Generate().Int64() 36 | } 37 | -------------------------------------------------------------------------------- /dbms/README.md: -------------------------------------------------------------------------------- 1 | # db manage system 2 | 3 | ## mongo 4 | 5 | ## mysql 6 | 7 | ## redis 8 | -------------------------------------------------------------------------------- /dbms/gmongo/mongo.go: -------------------------------------------------------------------------------- 1 | package gmongo 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "go.mongodb.org/mongo-driver/mongo" 9 | "go.mongodb.org/mongo-driver/mongo/options" 10 | 11 | "github.com/aipave/go-utils/gexit" 12 | "github.com/sirupsen/logrus" 13 | ) 14 | 15 | // InitMongo initial mongo connect 16 | // gmongo.MongoConfig or URI String 17 | func InitMongo(cc any) (client *mongo.Client) { 18 | var uri string 19 | switch cc := cc.(type) { 20 | case MongoConfig: 21 | uri = cc.URI() 22 | case string: 23 | uri = cc 24 | default: 25 | panic(fmt.Errorf("unkown type:%v", cc)) 26 | } 27 | 28 | var err error 29 | logrus.Infof("connecting to addr:%v", uri) 30 | 31 | ctx, cancel := context.WithTimeout(context.Background(), time.Second) 32 | defer cancel() 33 | 34 | client, err = mongo.Connect(ctx, options.Client().ApplyURI(uri)) 35 | if err != nil { 36 | panic(fmt.Errorf("connecting to mongo addr:%v err:%v", uri, err)) 37 | } 38 | 39 | gexit.Release(func() { 40 | if err = client.Disconnect(context.Background()); err != nil { 41 | logrus.Errorf("disconnect from mongo addr:%v err:%v", uri, err) 42 | } 43 | 44 | logrus.Infof("mongo addr:%v resource released.", uri) 45 | }) 46 | 47 | logrus.Infof("connected to mongo addr:%v done", uri) 48 | return 49 | } 50 | -------------------------------------------------------------------------------- /dbms/gmongo/options.go: -------------------------------------------------------------------------------- 1 | package gmongo 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "strings" 7 | ) 8 | 9 | type MongoConfig struct { 10 | User string `json:"User" yaml:"User"` 11 | Password string `json:"Password" yaml:"Password"` 12 | Addr []string `json:"Addr" yaml:"Addr"` 13 | Database string `json:"Database" yaml:"Database"` 14 | ConnectTimeout int `json:"ConnectTimeout" yaml:"ConnectTimeout"` 15 | } 16 | 17 | // URI mongo connect with String 18 | func (mc *MongoConfig) URI() string { 19 | return fmt.Sprintf("mongodb://%v:%v@%v/%v", mc.User, url.QueryEscape(mc.Password), strings.Join(mc.Addr, ","), mc.Database) 20 | } 21 | -------------------------------------------------------------------------------- /dbms/gmysql/mysql.go: -------------------------------------------------------------------------------- 1 | package gmysql 2 | 3 | import ( 4 | "database/sql" 5 | "time" 6 | 7 | "github.com/sirupsen/logrus" 8 | "gorm.io/driver/mysql" 9 | "gorm.io/gorm" 10 | "gorm.io/gorm/logger" 11 | ) 12 | 13 | // NewMySQLClient initialclient 14 | func NewMySQLClient(opts ...Option) *gorm.DB { 15 | var c = defaultConfig 16 | for _, fn := range opts { 17 | fn(&c) 18 | } 19 | 20 | db, err := gorm.Open(mysql.Open(c.DSN), &gorm.Config{ 21 | Logger: logger.New(logrus.StandardLogger(), logger.Config{ 22 | SlowThreshold: 100 * time.Millisecond, // 100 ms 23 | Colorful: true, 24 | IgnoreRecordNotFoundError: false, 25 | LogLevel: c.LogLevel, 26 | }), 27 | }) 28 | 29 | if err != nil { 30 | logrus.Fatalf("open mysql fail:%s, dsn:%v", err.Error(), c.DSN) 31 | } 32 | 33 | var sqlDB *sql.DB 34 | sqlDB, err = db.DB() 35 | if err != nil { 36 | logrus.Fatalf("get db fail:%v, dsn: %v", err.Error(), c.DSN) 37 | } 38 | 39 | sqlDB.SetMaxIdleConns(c.MaxIdleCons) 40 | sqlDB.SetMaxOpenConns(c.MaxOpenCons) 41 | sqlDB.SetConnMaxLifetime(c.ConnMaxLifeTime) 42 | 43 | logrus.Infof("initialized mysql: %v", c.DSN) 44 | return db 45 | } 46 | 47 | // CloseRows . 48 | func CloseRows(rows *sql.Rows) { 49 | if rows != nil { 50 | err := rows.Close() 51 | if err != nil { 52 | logrus.Errorf("close rows:%v err:%v", rows, err) 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /dbms/gmysql/options.go: -------------------------------------------------------------------------------- 1 | package gmysql 2 | 3 | import ( 4 | "time" 5 | 6 | "gorm.io/gorm/logger" 7 | ) 8 | 9 | type SQLConfig struct { 10 | DSN string `json:"DSN" yaml:"DSN"` 11 | LogLevel logger.LogLevel `json:"LogLevel" yaml:"LogLevel"` 12 | MaxIdleCons int `json:"MaxIdleCons" yaml:"MaxIdleCons"` 13 | MaxOpenCons int `json:"MaxOpenCons" yaml:"MaxOpenCons"` 14 | ConnMaxLifeTime time.Duration `json:"ConnMaxLifeTime" yaml:"ConnMaxLifeTime"` 15 | } 16 | 17 | var defaultConfig = SQLConfig{ 18 | DSN: "", 19 | LogLevel: logger.Info, 20 | MaxIdleCons: 10, 21 | MaxOpenCons: 20, 22 | ConnMaxLifeTime: 30 * time.Minute, // 主动超时时间，设置30分钟减少TIME_WAIT数量 23 | } 24 | 25 | type Option func(c *SQLConfig) 26 | 27 | // WithLogLevel 28 | func WithLogLevel(level logger.LogLevel) Option { 29 | return func(c *SQLConfig) { 30 | c.LogLevel = level 31 | } 32 | } 33 | 34 | // WithDSN: data source name, dsn) is an identifier used to identify the connection information of the database system, 35 | // such as database type, server address, port number, user name and password, etc. 36 | /* When using this function, we can set up the database connection as follows: 37 | config := NewSQLConfig( 38 | WithDSN("user:password@tcp(localhost:3306)/mydb"), 39 | WithMaxOpenConns(10), 40 | WithMaxIdleConns(5), 41 | ) 42 | */ 43 | func WithDSN(dsn string) Option { 44 | return func(c *SQLConfig) { 45 | c.DSN = dsn 46 | } 47 | } 48 | 49 | // WithMaxIdleCons idle connections count 50 | func WithMaxIdleCons(n int) Option { 51 | return func(c *SQLConfig) { 52 | c.MaxIdleCons = n 53 | } 54 | } 55 | 56 | // WithMaxOpenCons open connections count 57 | func WithMaxOpenCons(n int) Option { 58 | return func(c *SQLConfig) { 59 | c.MaxOpenCons = n 60 | } 61 | } 62 | 63 | // WithConnMaxLifeTime max expiration time of a connection 64 | func WithConnMaxLifeTime(n time.Duration) Option { 65 | return func(c *SQLConfig) { 66 | c.ConnMaxLifeTime = n 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /dbms/gredis/hook.go: -------------------------------------------------------------------------------- 1 | package gredis 2 | 3 | import ( 4 | "context" 5 | "path/filepath" 6 | "runtime" 7 | "strings" 8 | "time" 9 | 10 | "github.com/aipave/go-utils/ginfos" 11 | "github.com/go-redis/redis/v8" 12 | ) 13 | 14 | type startKey struct{} 15 | 16 | type customRedisHook struct { 17 | } 18 | 19 | func (hook *customRedisHook) BeforeProcess(ctx context.Context, cmd redis.Cmder) (context.Context, error) { 20 | return context.WithValue(ctx, startKey{}, time.Now()), nil 21 | } 22 | 23 | func (hook *customRedisHook) AfterProcess(ctx context.Context, cmd redis.Cmder) error { 24 | frame := getCaller() 25 | if frame == nil { 26 | return nil 27 | } 28 | 29 | // todo prometheus 30 | _ = getFunc(filepath.Base(frame.Function)) 31 | 32 | if start, ok := ctx.Value(startKey{}).(time.Time); ok { 33 | // todo prometheus 34 | _ = time.Since(start) 35 | } 36 | 37 | if isActualErr(cmd.Err()) { 38 | // todo prometheus 39 | 40 | } else { 41 | // todo prometheus 42 | } 43 | 44 | return nil 45 | } 46 | 47 | func (hook *customRedisHook) BeforeProcessPipeline(ctx context.Context, cmds []redis.Cmder) (context.Context, error) { 48 | return context.WithValue(ctx, startKey{}, time.Now()), nil 49 | } 50 | 51 | func (hook *customRedisHook) AfterProcessPipeline(ctx context.Context, cmds []redis.Cmder) error { 52 | if err := hook.AfterProcess(ctx, redis.NewCmd(ctx, "pipeline")); err != nil { 53 | return err 54 | } 55 | 56 | frame := getCaller() 57 | if frame == nil { 58 | return nil 59 | } 60 | 61 | // todo prometheus 62 | _ = getFunc(filepath.Base(frame.Function)) 63 | 64 | if start, ok := ctx.Value(startKey{}).(time.Time); ok { 65 | // todo prometheus 66 | _ = time.Since(start) 67 | // todo prometheus 68 | } 69 | 70 | for _, cmd := range cmds { 71 | if isActualErr(cmd.Err()) { 72 | // todo prometheus 73 | } else { 74 | // todo prometheus 75 | } 76 | } 77 | 78 | return nil 79 | } 80 | 81 | func isActualErr(err error) bool { 82 | return err != nil && err != redis.Nil 83 | } 84 | 85 | // getCaller retrieves the name of the first non-logrus calling function 86 | func getCaller() *runtime.Frame { 87 | const maximumCallerDepth = 25 88 | 89 | pcs := make([]uintptr, maximumCallerDepth) 90 | depth := runtime.Callers(3, pcs) 91 | frames := runtime.CallersFrames(pcs[:depth]) 92 | 93 | for f, again := frames.Next(); again; f, again = frames.Next() { 94 | // logrus.Info(f.File, f.Function) 95 | // According to the file name, match the file with the same name under the first a service and return 96 | if strings.Contains(f.Function, ginfos.Runtime.Exec()) { 97 | return &f 98 | } 99 | } 100 | 101 | // if we got here, we failed to find the caller's context 102 | return nil 103 | } 104 | 105 | func getFunc(name string) string { 106 | if fields := strings.Split(name, "."); len(fields) > 0 { 107 | return "redis-" + fields[len(fields)-1] 108 | } 109 | return "redis-" + name 110 | } 111 | -------------------------------------------------------------------------------- /dbms/gredis/options.go: -------------------------------------------------------------------------------- 1 | package gredis 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/aipave/go-utils/gcast" 7 | ) 8 | 9 | type RedisConfig struct { 10 | Addr string `json:"Addr" yaml:"Addr"` 11 | PoolSize int `json:"PoolSize" yaml:"PoolSize"` 12 | ReadTimeout int64 `json:"ReadTimeout" yaml:"ReadTimeout"` 13 | WriteTimeout int64 `json:"WriteTimeout" yaml:"WriteTimeout"` 14 | Password string `json:"Password" yaml:"Password"` 15 | DB int `json:"DB" yaml:"DB"` 16 | } 17 | 18 | var defaultConfig = RedisConfig{ 19 | Addr: "", 20 | PoolSize: 100, 21 | ReadTimeout: gcast.ToInt64(10 * time.Second), 22 | WriteTimeout: gcast.ToInt64(20 * time.Second), 23 | DB: 0, 24 | } 25 | 26 | type Option func(c *RedisConfig) 27 | 28 | func SetDefaultConfig(rc RedisConfig) { 29 | if len(rc.Addr) > 0 { 30 | defaultConfig.Addr = rc.Addr 31 | } 32 | 33 | if rc.PoolSize > 0 { 34 | defaultConfig.PoolSize = rc.PoolSize 35 | } 36 | 37 | if rc.ReadTimeout > 0 { 38 | defaultConfig.ReadTimeout = rc.ReadTimeout 39 | } 40 | 41 | if rc.WriteTimeout > 0 { 42 | defaultConfig.WriteTimeout = rc.WriteTimeout 43 | } 44 | 45 | if len(rc.Password) > 0 { 46 | defaultConfig.Password = rc.Password 47 | } 48 | 49 | if rc.DB > 0 { 50 | defaultConfig.DB = rc.DB 51 | } 52 | } 53 | 54 | func WithAddr(addr string) Option { 55 | return func(c *RedisConfig) { 56 | c.Addr = addr 57 | } 58 | } 59 | 60 | func WithPoolSize(n int) Option { 61 | return func(c *RedisConfig) { 62 | c.PoolSize = n 63 | } 64 | } 65 | 66 | func WithReadTimeOut(n time.Duration) Option { 67 | return func(c *RedisConfig) { 68 | c.ReadTimeout = int64(n) 69 | } 70 | } 71 | 72 | func WithWriteTimeOut(n time.Duration) Option { 73 | return func(c *RedisConfig) { 74 | c.WriteTimeout = int64(n) 75 | } 76 | } 77 | 78 | func WithPassword(p string) Option { 79 | return func(c *RedisConfig) { 80 | c.Password = p 81 | } 82 | } 83 | 84 | func WithDB(db int) Option { 85 | return func(c *RedisConfig) { 86 | c.DB = db 87 | } 88 | } 89 | 90 | func WithDefaultConfig(config RedisConfig) Option { 91 | return func(c *RedisConfig) { 92 | *c = config 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /dbms/gredis/redis.go: -------------------------------------------------------------------------------- 1 | package gredis 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/aipave/go-utils/gexit" 8 | redisprom "github.com/globocom/go-redis-prometheus" 9 | "github.com/go-redis/redis/v8" 10 | "github.com/sirupsen/logrus" 11 | ) 12 | 13 | // NewRedisClient initialredis client， goroutine safety, no need to close 14 | // client := NewRedisClient(Addr("host:port")) 15 | // client.SetNX().Result() 16 | func NewRedisClient(opts ...Option) (redisClient *redis.Client) { 17 | var c = defaultConfig 18 | for _, fn := range opts { 19 | fn(&c) 20 | } 21 | 22 | redisClient = redis.NewClient(&redis.Options{ 23 | Addr: c.Addr, 24 | PoolSize: c.PoolSize, 25 | ReadTimeout: time.Duration(c.ReadTimeout) * time.Millisecond, // time.ParseDuration(str+"ms") 26 | WriteTimeout: time.Duration(c.WriteTimeout) * time.Millisecond, 27 | Password: c.Password, 28 | }) 29 | 30 | if err := redisClient.Ping(context.Background()).Err(); err != nil { 31 | logrus.Fatalf("init redis client err:%v config:%v", err, c) 32 | } 33 | 34 | gexit.Release(func() { 35 | if redisClient == nil { 36 | return 37 | } 38 | 39 | if err := redisClient.Close(); err != nil { 40 | logrus.Errorf("close redis client err:%v", err) 41 | } 42 | 43 | logrus.Infof("redis addr:%v resource released.", c.Addr) 44 | }) 45 | 46 | redisClient.AddHook(&customRedisHook{}) 47 | redisClient.AddHook(redisprom.NewHook()) 48 | 49 | logrus.Infof("init redis client done, config:%+v", c) 50 | return 51 | } 52 | 53 | // ResetNil reset redis.Nil error 54 | func ResetNil(err error) error { 55 | if err == redis.Nil { 56 | return nil 57 | } 58 | return err 59 | } 60 | -------------------------------------------------------------------------------- /dbms/gredis/res/get_lock.lua: -------------------------------------------------------------------------------- 1 | --- 2 | --- Generated by ByteMagic-Lyu 3 | --- DateTime: 2022/10/3 16:48 4 | --- 5 | 6 | --local lock = KEYS[1] 7 | --local ip = ARGV[1] 8 | --local ttl = ARGV[2] + 0 9 | 10 | local ip = redis.call('GET', KEYS[1]) 11 | -- no instance running now 12 | if ip == false then 13 | local result = redis.call('SET', KEYS[1], ARGV[1], 'NX', 'EX', ARGV[2]) -- lock grab 14 | if result ~= false and result.ok == 'OK' then 15 | return 1 -- lock grab success 16 | end 17 | 18 | -- has instance running now 19 | else if ip ~= false and ip == ARGV[1] then 20 | redis.call('EXPIRE', KEYS[1], ARGV[2]) -- lock grab，renewal 21 | return 2 -- renewal success 22 | end 23 | end 24 | 25 | return 0 -- fail to grab lock 26 | -------------------------------------------------------------------------------- /dbms/gredis/timer.go: -------------------------------------------------------------------------------- 1 | package gredis 2 | 3 | import ( 4 | "context" 5 | _ "embed" 6 | "sync" 7 | "time" 8 | 9 | "github.com/aipave/go-utils/gcast" 10 | "github.com/aipave/go-utils/gexit" 11 | "github.com/aipave/go-utils/ginfos" 12 | redis "github.com/go-redis/redis/v8" 13 | //_ "github.com/robfig/cron" //The api of the cron package has changed since version v3, requiring a new import path 14 | "github.com/robfig/cron/v3" 15 | 16 | "github.com/sirupsen/logrus" 17 | ) 18 | 19 | //go:embed res/get_lock.lua 20 | var lockScript string 21 | 22 | // NewCron timer 23 | // cli redis client 24 | // name of timer(redis key)， global unique 25 | // ttl time, validity period of a key is 3*ttl 26 | func NewCron(cli *redis.Client, name string, ttl int64) *Cron { 27 | exec := "gl:" + ginfos.Runtime.Exec() // global lock:progress name:name 28 | if len(name) > 0 { 29 | exec += ":" + name 30 | } 31 | 32 | c := &Cron{cli: cli, name: exec, c: cron.New(cron.WithSeconds()), ttl: ttl} 33 | go c.run() 34 | return c 35 | } 36 | 37 | type Cron struct { 38 | cli *redis.Client 39 | name string 40 | c *cron.Cron 41 | ttl int64 42 | isMaster bool 43 | mux sync.Mutex 44 | } 45 | 46 | func (c *Cron) AddFunc(spec string, cmd func()) error { 47 | _, err := c.c.AddFunc(spec, c.wrapperCmd(cmd)) 48 | return err 49 | } 50 | 51 | func (c *Cron) Start() { 52 | c.c.Start() 53 | } 54 | 55 | func (c *Cron) Stop() { 56 | c.c.Stop() 57 | } 58 | 59 | func (c *Cron) wrapperCmd(cmd func()) func() { 60 | return func() { 61 | if !c.isCurrentMaster() { 62 | logrus.Warningf("machine:%v not master, abort", ginfos.Runtime.IP()) 63 | return 64 | } 65 | 66 | // master machine 67 | cmd() 68 | } 69 | } 70 | 71 | func (c *Cron) run() { 72 | c.grab() 73 | 74 | ticker := time.Tick(time.Duration(c.ttl) * time.Second) 75 | ctx, cancel := context.WithCancel(context.Background()) 76 | gexit.Close(func() { 77 | cancel() 78 | }) 79 | 80 | for { 81 | select { 82 | case <-ticker: 83 | c.grab() 84 | 85 | case <-ctx.Done(): 86 | logrus.Infof("ctx done, cron.run abort") 87 | return 88 | } 89 | } 90 | } 91 | 92 | func (c *Cron) grab() { 93 | result, err := c.cli.Eval(context.Background(), lockScript, []string{c.name}, ginfos.Runtime.IP(), 3*c.ttl).Result() 94 | if err != nil { 95 | c.setSlaver() 96 | logrus.Errorf("grab lock|name=%v|ttl=%v|isMaster=%v|result=%v|err=%v", c.name, c.ttl, c.isMaster, result, err) 97 | return 98 | } 99 | 100 | const ( 101 | resultFail = 0 // fail to grab lock 102 | resultSuccess = 1 // success to grab lock 103 | resultRenewal = 2 // renewal success 104 | ) 105 | 106 | switch gcast.ToInt(result) { 107 | case resultFail: 108 | c.setSlaver() 109 | logrus.Warningf("grab lock fail|name=%v|ttl=%v|isMaster=%v|-", c.name, c.ttl, c.isMaster) 110 | 111 | case resultSuccess: 112 | c.setMaster() 113 | logrus.Infof("grab lock success|name=%v|ttl=%v|isMaster=%v|-", c.name, c.ttl, c.isMaster) 114 | 115 | case resultRenewal: 116 | c.setMaster() 117 | logrus.Infof("renewal success|name=%v|ttl=%v|isMaster=%v|-", c.name, c.ttl, c.isMaster) 118 | } 119 | } 120 | 121 | func (c *Cron) isCurrentMaster() bool { 122 | c.mux.Lock() 123 | defer c.mux.Unlock() 124 | return c.isMaster 125 | } 126 | 127 | func (c *Cron) setMaster() { 128 | c.mux.Lock() 129 | c.isMaster = true 130 | c.mux.Unlock() 131 | } 132 | 133 | func (c *Cron) setSlaver() { 134 | c.mux.Lock() 135 | c.isMaster = false 136 | c.mux.Unlock() 137 | } 138 | -------------------------------------------------------------------------------- /gcache/README.md: -------------------------------------------------------------------------------- 1 | # gcache 2 | 3 | [![BK Pipelines Status](https://api.bkdevops.qq.com/process/api/external/pipelines/projects/pcgtrpcproject/p-c9b0d4b7b7754407ae09b4c8887a0ba6/badge?X-DEVOPS-PROJECT-ID=pcgtrpcproject)](http://devops.oa.com:/ms/process/api-html/user/builds/projects/pcgtrpcproject/pipelines/p-c9b0d4b7b7754407ae09b4c8887a0ba6/latestFinished?X-DEVOPS-PROJECT-ID=pcgtrpcproject)[![Coverage](https://tcoverage.woa.com/api/getCoverage/getTotalImg/?pipeline_id=p-c9b0d4b7b7754407ae09b4c8887a0ba6)](http://macaron.oa.com/api/coverage/getTotalLink/?pipeline_id=p-c9b0d4b7b7754407ae09b4c8887a0ba6)[![GoDoc](https://img.shields.io/badge/API%20Docs-GoDoc-green)](http://godoc.oa.com/git.code.oa.com/trpc-go/trpc-database/cos) 4 | 5 | localcache is a local K-V caching component that runs on a single machine. 6 | It allows concurrent access by multiple goroutines and supports LRU-based and expiration-based eviction policies. 7 | When the localcache capacity reaches its limit, data eviction is performed based on LRU, and expired key-value pairs are 8 | deleted using a time wheel implementation. 9 | 10 | ## how to use 11 | 12 | ```go 13 | package main 14 | 15 | import ( 16 | "xxxxx/gcache" 17 | ) 18 | 19 | func LoadData(ctx context.Context, key string) (interface{}, error) { 20 | return "cat", nil 21 | } 22 | 23 | func main() { 24 | // set 5s tll 25 | gcache.Set("foo", "bar", 5) 26 | 27 | // get value of key 28 | value, found := gcache.Get("foo") 29 | 30 | // get value of key 31 | // set 5s tll 32 | value, err := gcache.GetWithLoad(context.TODO(), "tom", LoadData, 5) 33 | 34 | // del key 35 | gcache.Del("foo") 36 | 37 | // clear 38 | gcache.Clear() 39 | } 40 | ``` 41 | 42 | ## gomod import 43 | 44 | require xxxxx/xxxx/gcache v3.0.0 45 | 46 | ## use config 47 | 48 | New () to generate Cache instance, call the instance function function 49 | 50 | ### Optional parameters 51 | 52 | #### **WithCapacity(capacity int)** 53 | 54 | set cache cap. min = 1，max=1e30 55 | will del tail elem. when to max, default value is 1e30 56 | 57 | #### **WithExpiration(ttl int64)** 58 | 59 | #### **WithLoad(f LoadFunc)** 60 | 61 | ```go 62 | type LoadFunc func(ctx context.Context, key string) (interface{}, error) 63 | ``` 64 | 65 | Set the data load function, the key does not exist in the cache, use this function to load the corresponding value, and 66 | cached in the cache. 67 | 68 | #### Cache interface 69 | 70 | ```go 71 | type Cache interface { 72 | // Get 73 | Get(key string) (interface{}, bool) 74 | 75 | // GetWithLoad 76 | GetWithLoad(ctx context.Context, key string) (interface{}, error) 77 | 78 | // Set 79 | Set(key string, value interface{}) bool 80 | 81 | // SetWithExpire 82 | SetWithExpire(key string, value interface{}, ttl int64) bool 83 | 84 | // Del key 85 | Del(key string) 86 | 87 | // Clear 88 | Clear() 89 | 90 | // Close close cache 91 | Close() 92 | } 93 | ``` 94 | 95 | ## example 96 | 97 | #### set capacity and ttl 98 | 99 | ```go 100 | package main 101 | 102 | import ( 103 | "fmt" 104 | "time" 105 | 106 | "git.code.oa.com/trpc-go/trpc-database/gcache" 107 | ) 108 | 109 | func main() { 110 | var lc gcache.Cache 111 | 112 | // new cap = 100, ttl = 5s 113 | lc = gcache.New(gcache.WithCapacity(100), gcache.WithExpiration(5)) 114 | 115 | // set key-value, ttl = 5s 116 | lc.Set("foo", "bar") 117 | 118 | // re-custom tll for key, 10s 119 | lc.SetWithExpire("tom", "cat", 10) 120 | 121 | // asyn wait 122 | time.Sleep(time.Millisecond) 123 | 124 | // get value 125 | val, found := lc.Get("foo") 126 | 127 | // del key: "foo" 128 | lc.Del("foo") 129 | } 130 | ``` 131 | 132 | #### Custom loading function 133 | 134 | Set a custom data loading function and use GetWithLoad(key) to obtain the value. 135 | 136 | ```go 137 | func main() { 138 | loadFunc := func (ctx context.Context, key string) (interface{}, error) { 139 | return "cat", nil 140 | } 141 | 142 | lc := gcache.New(gcache.WithLoad(loadFunc), gcache.WithExpiration(5)) 143 | 144 | // return err for loadFunc 145 | val, err := lc.GetWithLoad(context.TODO(), "tom") 146 | } 147 | ``` 148 | 149 | ### TODO 150 | 151 | - [ ] Add Metrics data statistics 152 | - [ ] Add control of memory usage 153 | - [ ] Introduce an upgraded version of LRU: W-tinyLRU, which more efficiently controls the write and eviction of keys. 154 | 155 | 156 | -------------------------------------------------------------------------------- /gcache/func.go: -------------------------------------------------------------------------------- 1 | package gcache 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | var defaultLocalCache = New[string, any]() 8 | 9 | // Get 10 | func Get(key string) (interface{}, bool) { 11 | return defaultLocalCache.Get(key) 12 | } 13 | 14 | // GetWithLoad 15 | func GetWithLoad(ctx context.Context, key string, load LoadFunc[string, any]) (interface{}, error) { 16 | value, found := defaultLocalCache.Get(key) 17 | if found { 18 | return value, nil 19 | } 20 | 21 | c, _ := defaultLocalCache.(*cache[string, any]) 22 | 23 | return c.loadData(ctx, key, load) 24 | } 25 | 26 | // Set key, value, ttl(s) 27 | func Set(key string, value interface{}, ttl int64) bool { 28 | return defaultLocalCache.SetWithExpire(key, value, ttl) 29 | } 30 | 31 | // Del key 32 | func Del(key string) { 33 | defaultLocalCache.Del(key) 34 | } 35 | 36 | // Clear 清空所有队列和缓存 37 | func Clear() { 38 | defaultLocalCache.Clear() 39 | } 40 | -------------------------------------------------------------------------------- /gcache/func_test.go: -------------------------------------------------------------------------------- 1 | package gcache 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | // TestFuncGetAndSet 测试Get和Set 11 | func TestFuncGetAndSet(t *testing.T) { 12 | mocks := []struct { 13 | key string 14 | val string 15 | ttl int64 16 | }{ 17 | {"A", "a", 1000}, 18 | {"B", "b", 1000}, 19 | {"", "null", 1000}, 20 | } 21 | 22 | for _, mock := range mocks { 23 | Set(mock.key, mock.val, mock.ttl) 24 | } 25 | 26 | time.Sleep(wait) 27 | 28 | for _, mock := range mocks { 29 | val, found := Get(mock.key) 30 | if !found || val.(string) != mock.val { 31 | t.Fatalf("Unexpected value: %v (%v) to key: %v", val, found, mock.key) 32 | } 33 | } 34 | } 35 | 36 | // TestFuncExpireSet 测试带Expire的Set 37 | func TestFuncExpireSet(t *testing.T) { 38 | Set("Foo", "Bar", 1) 39 | time.Sleep(1) 40 | if val, found := Get("Foo"); found { 41 | t.Fatalf("unexpected expired value: %v to key: %v", val, "Foo") 42 | } 43 | } 44 | 45 | // TestFuncGetWithLoad 测试WithLoad情况下的Get 46 | func TestFuncGetWithLoad(t *testing.T) { 47 | m := map[string]interface{}{ 48 | "A1": "a", 49 | "B": "b", 50 | "": "null", 51 | } 52 | loadFunc := func(ctx context.Context, key string) (interface{}, error) { 53 | if v, exist := m[key]; exist { 54 | return v, nil 55 | } 56 | return nil, errors.New("key not exist") 57 | } 58 | value, err := GetWithLoad(context.TODO(), "A1", loadFunc) 59 | if err != nil || value.(string) != "a" { 60 | t.Fatalf("unexpected GetWithLoad value: %v, want:a, err:%v", value, err) 61 | } 62 | 63 | time.Sleep(wait) 64 | 65 | got2, found := Get("A1") 66 | if !found || got2.(string) != "a" { 67 | t.Fatalf("unexpected Get value: %v, want:a, found:%v", got2, found) 68 | } 69 | } 70 | 71 | //func TestDelAndClear(t *testing.T) { 72 | // Convey("DelAndClear", t, func() { 73 | // Set("Foo", "bar", 10) 74 | // Set("Foo1", "bar", 10) 75 | // time.Sleep(time.Millisecond * 10) 76 | // _, ok := Get("Foo") 77 | // So(ok, ShouldBeTrue) 78 | // Del("Foo") 79 | // time.Sleep(time.Millisecond * 10) 80 | // _, ok = Get("Foo") 81 | // So(ok, ShouldBeFalse) 82 | // Clear() 83 | // time.Sleep(time.Millisecond * 10) 84 | // _, ok = Get("Foo1") 85 | // So(ok, ShouldBeFalse) 86 | // }) 87 | //} 88 | -------------------------------------------------------------------------------- /gcache/lru.go: -------------------------------------------------------------------------------- 1 | package gcache 2 | 3 | import ( 4 | "container/list" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | // entry Storage entity 10 | type entry[K comparable, V any] struct { 11 | mux sync.RWMutex 12 | key K 13 | value V 14 | expireTime time.Time 15 | } 16 | 17 | func getEntry[K comparable, V any](ele *list.Element) *entry[K, V] { 18 | return ele.Value.(*entry[K, V]) 19 | } 20 | 21 | func setEntry[K comparable, V any](ele *list.Element, ent *entry[K, V]) { 22 | ele.Value = ent 23 | } 24 | 25 | // lru The concurrent security lru queues 26 | type lru[K comparable, V any] struct { 27 | ll *list.List 28 | store store[K, V] 29 | capacity int 30 | } 31 | 32 | func newLRU[K comparable, V any](capacity int, store store[K, V]) *lru[K, V] { 33 | return &lru[K, V]{ 34 | ll: list.New(), 35 | store: store, 36 | capacity: capacity, 37 | } 38 | } 39 | 40 | func (l *lru[K, V]) add(ent *entry[K, V]) *entry[K, V] { 41 | val, ok := l.store.get(ent.key) 42 | if ok { 43 | setEntry(val, ent) 44 | l.ll.MoveToFront(val) 45 | return nil 46 | } 47 | if l.capacity <= 0 || l.ll.Len() < l.capacity { 48 | ele := l.ll.PushFront(ent) 49 | l.store.set(ent.key, ele) 50 | return nil 51 | } 52 | // After the expiration of the lru, replace the last element 53 | ele := l.ll.Back() 54 | if ele == nil { 55 | return ent 56 | } 57 | victimEnt := getEntry[K, V](ele) 58 | setEntry(ele, ent) 59 | l.ll.MoveToFront(ele) 60 | 61 | l.store.delete(victimEnt.key) 62 | l.store.set(ent.key, ele) 63 | return victimEnt 64 | } 65 | 66 | func (l *lru[K, V]) hit(ele *list.Element) { 67 | l.ll.MoveToFront(ele) 68 | } 69 | 70 | func (l *lru[K, V]) push(elements []*list.Element) { 71 | for _, ele := range elements { 72 | l.ll.MoveToFront(ele) 73 | } 74 | } 75 | 76 | func (l *lru[K, V]) delete(key K) { 77 | value, ok := l.store.get(key) 78 | if !ok { 79 | return 80 | } 81 | l.ll.Remove(value) 82 | l.store.delete(key) 83 | } 84 | 85 | func (l *lru[K, V]) len() int { 86 | return l.ll.Len() 87 | } 88 | 89 | func (l *lru[K, V]) clear() { 90 | l.ll = list.New() 91 | } 92 | -------------------------------------------------------------------------------- /gcache/policy.go: -------------------------------------------------------------------------------- 1 | package gcache 2 | 3 | import ( 4 | "container/list" 5 | ) 6 | 7 | // Storage policy 8 | type policy[K comparable, V any] interface { 9 | // add an elem. 10 | add(ent *entry[K, V]) *entry[K, V] 11 | // hit processes a hit on an element 12 | hit(elements *list.Element) 13 | // push processes a batch of accessed elements 14 | push(elements []*list.Element) 15 | // delete 16 | delete(key K) 17 | // clear 18 | clear() 19 | } 20 | 21 | func newPolicy[K comparable, V any](capacity int, store store[K, V]) policy[K, V] { 22 | return newLRU[K, V](capacity, store) 23 | } 24 | -------------------------------------------------------------------------------- /gcache/ring.go: -------------------------------------------------------------------------------- 1 | package gcache 2 | 3 | import ( 4 | "container/list" 5 | "sync" 6 | ) 7 | 8 | // ringConsumer accept and consume data 9 | type ringConsumer interface { 10 | push([]*list.Element) bool 11 | } 12 | 13 | // ringStrip ring buffer Metadata caching Get requests to batch update element is located in the LRU 14 | type ringStripe struct { 15 | consumer ringConsumer 16 | data []*list.Element 17 | capacity int 18 | } 19 | 20 | func newRingStripe(consumer ringConsumer, capacity int) *ringStripe { 21 | return &ringStripe{ 22 | consumer: consumer, 23 | data: make([]*list.Element, 0, capacity), 24 | capacity: capacity, 25 | } 26 | } 27 | 28 | // push Recorded by a visit to the element to the ring buffer 29 | func (r *ringStripe) push(ele *list.Element) { 30 | r.data = append(r.data, ele) 31 | if len(r.data) >= r.capacity { 32 | if r.consumer.push(r.data) { 33 | r.data = make([]*list.Element, 0, r.capacity) 34 | } else { 35 | r.data = r.data[:0] 36 | } 37 | } 38 | } 39 | 40 | // Pooling ringStripes of ringBuffer allows multiple goroutines to write elements to 41 | // the buffer without locking, which is more efficient than concurrent writes to the same channel. 42 | // Additionally, objects in the pool will be automatically removed without any notification, 43 | // which reduces cache's LRU operations and minimizes concurrent competition between Set/Expire/Delete operations. 44 | // The cache doesn't need to be strictly LRU, 45 | // actively discarding some access metadata to reduce concurrent competition and 46 | // improve write efficiency is necessary 47 | type ringBuffer struct { 48 | pool *sync.Pool 49 | } 50 | 51 | func newRingBuffer(consumer ringConsumer, capacity int) *ringBuffer { 52 | return &ringBuffer{ 53 | pool: &sync.Pool{ 54 | New: func() interface{} { return newRingStripe(consumer, capacity) }, 55 | }, 56 | } 57 | } 58 | 59 | // push Record a is access to the elements 60 | func (b *ringBuffer) push(ele *list.Element) { 61 | ringStripe := b.pool.Get().(*ringStripe) 62 | ringStripe.push(ele) 63 | b.pool.Put(ringStripe) 64 | } 65 | -------------------------------------------------------------------------------- /gcache/store.go: -------------------------------------------------------------------------------- 1 | package gcache 2 | 3 | import ( 4 | "container/list" 5 | "reflect" 6 | "sync" 7 | 8 | gcast "github.com/aipave/go-utils/gcast" 9 | 10 | "github.com/cespare/xxhash" 11 | ) 12 | 13 | // store 14 | // Is a concurrent safely used to store the key - value data storage. Temporarily use shard map implementation in this file 15 | type store[K comparable, V any] interface { 16 | // get value of key 17 | get(K) (*list.Element, bool) 18 | // set key-value 19 | set(K, *list.Element) 20 | // delete key-value 21 | delete(K) 22 | // clear 23 | clear() 24 | // len Returns the size of the storage 25 | len() int 26 | } 27 | 28 | // newStore 29 | // Returns the default implementation of the storage 30 | func newStore[K comparable, V any]() store[K, V] { 31 | var val K 32 | switch reflect.ValueOf(val).Kind() { 33 | case reflect.Struct, reflect.Array, reflect.Chan, reflect.Interface, reflect.Ptr: 34 | return newLockedMap[K, V]() 35 | } 36 | 37 | return newShardedMap[K, V]() 38 | } 39 | 40 | const numShards uint64 = 256 41 | 42 | // shardedMap Store the shard 43 | type shardedMap[K comparable, V any] struct { 44 | shards []*lockedMap[K, V] 45 | } 46 | 47 | func newShardedMap[K comparable, V any]() *shardedMap[K, V] { 48 | sm := &shardedMap[K, V]{ 49 | shards: make([]*lockedMap[K, V], int(numShards)), 50 | } 51 | for i := range sm.shards { 52 | sm.shards[i] = newLockedMap[K, V]() 53 | } 54 | return sm 55 | } 56 | 57 | func (sm *shardedMap[K, V]) get(key K) (*list.Element, bool) { 58 | 59 | return sm.shards[xxhash.Sum64String(gcast.ToString(key))&(numShards-1)].get(key) 60 | } 61 | 62 | func (sm *shardedMap[K, V]) set(key K, value *list.Element) { 63 | sm.shards[xxhash.Sum64String(gcast.ToString(key))&(numShards-1)].set(key, value) 64 | } 65 | 66 | func (sm *shardedMap[K, V]) delete(key K) { 67 | sm.shards[xxhash.Sum64String(gcast.ToString(key))&(numShards-1)].delete(key) 68 | } 69 | 70 | func (sm *shardedMap[K, V]) clear() { 71 | for i := uint64(0); i < numShards; i++ { 72 | sm.shards[i].clear() 73 | } 74 | } 75 | 76 | func (sm *shardedMap[K, V]) len() int { 77 | length := 0 78 | for i := uint64(0); i < numShards; i++ { 79 | length += sm.shards[i].len() 80 | } 81 | return length 82 | } 83 | 84 | // lockedMap Concurrent security Map 85 | type lockedMap[K comparable, V any] struct { 86 | sync.RWMutex 87 | data map[K]*list.Element 88 | } 89 | 90 | func newLockedMap[K comparable, V any]() *lockedMap[K, V] { 91 | return &lockedMap[K, V]{ 92 | data: make(map[K]*list.Element), 93 | } 94 | } 95 | 96 | func (m *lockedMap[K, V]) get(key K) (*list.Element, bool) { 97 | m.RLock() 98 | val, ok := m.data[key] 99 | m.RUnlock() 100 | return val, ok 101 | } 102 | 103 | func (m *lockedMap[K, V]) set(key K, value *list.Element) { 104 | m.Lock() 105 | m.data[key] = value 106 | m.Unlock() 107 | } 108 | 109 | func (m *lockedMap[K, V]) delete(key K) { 110 | m.Lock() 111 | delete(m.data, key) 112 | m.Unlock() 113 | } 114 | 115 | func (m *lockedMap[K, V]) clear() { 116 | m.Lock() 117 | m.data = make(map[K]*list.Element) 118 | m.Unlock() 119 | } 120 | 121 | func (m *lockedMap[K, V]) len() int { 122 | return len(m.data) 123 | } 124 | -------------------------------------------------------------------------------- /gcache/timer.go: -------------------------------------------------------------------------------- 1 | package gcache 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/RussellLuo/timingwheel" 8 | ) 9 | 10 | // expireQueue Store "key expire automatically deleted after task" 11 | type expireQueue[K comparable, V any] struct { 12 | tick time.Duration 13 | wheelSize int64 14 | // Time round storage time due to delete task 15 | tw *timingwheel.TimingWheel 16 | 17 | mu sync.Mutex 18 | timers map[K]*timingwheel.Timer 19 | } 20 | 21 | // newExpireQueue 22 | ///> Generate expiration queue object, the queue by time round, the elements in the queue according to the expiration time, regularly delete 23 | func newExpireQueue[K comparable, V any](tick time.Duration, wheelSize int64) *expireQueue[K, V] { 24 | queue := &expireQueue[K, V]{ 25 | tick: tick, 26 | wheelSize: wheelSize, 27 | 28 | tw: timingwheel.NewTimingWheel(tick, wheelSize), 29 | timers: make(map[K]*timingwheel.Timer), 30 | } 31 | 32 | // start a goroutine，handle expired entry 33 | go queue.tw.Start() 34 | return queue 35 | } 36 | 37 | // add Add time overdue tasks, each time the task executes, due to independent goroutine is carried out 38 | func (q *expireQueue[K, V]) add(key K, expireTime time.Time, f func()) { 39 | q.mu.Lock() 40 | defer q.mu.Unlock() 41 | 42 | d := expireTime.Sub(currentTime()) 43 | timer := q.tw.AfterFunc(d, q.task(key, f)) 44 | q.timers[key] = timer 45 | 46 | return 47 | } 48 | 49 | // update key's tll 50 | func (q *expireQueue[K, V]) update(key K, expireTime time.Time, f func()) { 51 | q.mu.Lock() 52 | defer q.mu.Unlock() 53 | 54 | if timer, ok := q.timers[key]; ok { 55 | timer.Stop() 56 | } 57 | 58 | d := expireTime.Sub(currentTime()) 59 | timer := q.tw.AfterFunc(d, q.task(key, f)) 60 | q.timers[key] = timer 61 | } 62 | 63 | // remove key 64 | func (q *expireQueue[K, V]) remove(key K) { 65 | q.mu.Lock() 66 | defer q.mu.Unlock() 67 | 68 | if timer, ok := q.timers[key]; ok { 69 | timer.Stop() 70 | delete(q.timers, key) 71 | } 72 | } 73 | 74 | // clear 75 | func (q *expireQueue[K, V]) clear() { 76 | q.tw.Stop() 77 | q.tw = timingwheel.NewTimingWheel(q.tick, q.wheelSize) 78 | q.timers = make(map[K]*timingwheel.Timer) 79 | 80 | // restart goroutine，handle expired entry 81 | go q.tw.Start() 82 | } 83 | 84 | // stop The pause time round run queue 85 | func (q *expireQueue[K, V]) stop() { 86 | q.tw.Stop() 87 | } 88 | 89 | func (q *expireQueue[K, V]) task(key K, f func()) func() { 90 | return func() { 91 | f() 92 | q.mu.Lock() 93 | delete(q.timers, key) 94 | q.mu.Unlock() 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /gcast/README.md: -------------------------------------------------------------------------------- 1 | # gcast 2 | 3 | Mirror from: https://github.com/spf13/cast -------------------------------------------------------------------------------- /gcast/cast.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2014 Steve Francia . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | // Package cast provides easy and safe casting in Go. 7 | package gcast 8 | 9 | import ( 10 | "time" 11 | ) 12 | 13 | // ToBool casts an interface to a bool type. 14 | func ToBool(i interface{}) bool { 15 | v, _ := ToBoolE(i) 16 | return v 17 | } 18 | 19 | // ToTime casts an interface to a time.Time type. 20 | func ToTime(i interface{}) time.Time { 21 | v, _ := ToTimeE(i) 22 | return v 23 | } 24 | 25 | func ToTimeInDefaultLocation(i interface{}, location *time.Location) time.Time { 26 | v, _ := ToTimeInDefaultLocationE(i, location) 27 | return v 28 | } 29 | 30 | // ToDuration casts an interface to a time.Duration type. 31 | func ToDuration(i interface{}) time.Duration { 32 | v, _ := ToDurationE(i) 33 | return v 34 | } 35 | 36 | // ToFloat64 casts an interface to a float64 type. 37 | func ToFloat64(i interface{}) float64 { 38 | v, _ := ToFloat64E(i) 39 | return v 40 | } 41 | 42 | // ToFloat32 casts an interface to a float32 type. 43 | func ToFloat32(i interface{}) float32 { 44 | v, _ := ToFloat32E(i) 45 | return v 46 | } 47 | 48 | // ToInt64 casts an interface to an int64 type. 49 | func ToInt64(i interface{}) int64 { 50 | v, _ := ToInt64E(i) 51 | return v 52 | } 53 | 54 | // ToInt32 casts an interface to an int32 type. 55 | func ToInt32(i interface{}) int32 { 56 | v, _ := ToInt32E(i) 57 | return v 58 | } 59 | 60 | // ToInt16 casts an interface to an int16 type. 61 | func ToInt16(i interface{}) int16 { 62 | v, _ := ToInt16E(i) 63 | return v 64 | } 65 | 66 | // ToInt8 casts an interface to an int8 type. 67 | func ToInt8(i interface{}) int8 { 68 | v, _ := ToInt8E(i) 69 | return v 70 | } 71 | 72 | // ToInt casts an interface to an int type. 73 | func ToInt(i interface{}) int { 74 | v, _ := ToIntE(i) 75 | return v 76 | } 77 | 78 | // ToUint casts an interface to a uint type. 79 | func ToUint(i interface{}) uint { 80 | v, _ := ToUintE(i) 81 | return v 82 | } 83 | 84 | // ToUint64 casts an interface to a uint64 type. 85 | func ToUint64(i interface{}) uint64 { 86 | v, _ := ToUint64E(i) 87 | return v 88 | } 89 | 90 | // ToUint32 casts an interface to a uint32 type. 91 | func ToUint32(i interface{}) uint32 { 92 | v, _ := ToUint32E(i) 93 | return v 94 | } 95 | 96 | // ToUint16 casts an interface to a uint16 type. 97 | func ToUint16(i interface{}) uint16 { 98 | v, _ := ToUint16E(i) 99 | return v 100 | } 101 | 102 | // ToUint8 casts an interface to a uint8 type. 103 | func ToUint8(i interface{}) uint8 { 104 | v, _ := ToUint8E(i) 105 | return v 106 | } 107 | 108 | // ToString casts an interface to a string type. 109 | func ToString(i interface{}) string { 110 | v, _ := ToStringE(i) 111 | return v 112 | } 113 | 114 | // ToStringMapString casts an interface to a map[string]string type. 115 | func ToStringMapString(i interface{}) map[string]string { 116 | v, _ := ToStringMapStringE(i) 117 | return v 118 | } 119 | 120 | // ToStringMapStringSlice casts an interface to a map[string][]string type. 121 | func ToStringMapStringSlice(i interface{}) map[string][]string { 122 | v, _ := ToStringMapStringSliceE(i) 123 | return v 124 | } 125 | 126 | // ToStringMapBool casts an interface to a map[string]bool type. 127 | func ToStringMapBool(i interface{}) map[string]bool { 128 | v, _ := ToStringMapBoolE(i) 129 | return v 130 | } 131 | 132 | // ToStringMapInt casts an interface to a map[string]int type. 133 | func ToStringMapInt(i interface{}) map[string]int { 134 | v, _ := ToStringMapIntE(i) 135 | return v 136 | } 137 | 138 | // ToStringMapInt64 casts an interface to a map[string]int64 type. 139 | func ToStringMapInt64(i interface{}) map[string]int64 { 140 | v, _ := ToStringMapInt64E(i) 141 | return v 142 | } 143 | 144 | // ToStringMap casts an interface to a map[string]interface{} type. 145 | func ToStringMap(i interface{}) map[string]interface{} { 146 | v, _ := ToStringMapE(i) 147 | return v 148 | } 149 | 150 | // ToSlice casts an interface to a []interface{} type. 151 | func ToSlice(i interface{}) []interface{} { 152 | v, _ := ToSliceE(i) 153 | return v 154 | } 155 | 156 | // ToBoolSlice casts an interface to a []bool type. 157 | func ToBoolSlice(i interface{}) []bool { 158 | v, _ := ToBoolSliceE(i) 159 | return v 160 | } 161 | 162 | // ToStringSlice casts an interface to a []string type. 163 | func ToStringSlice(i interface{}) []string { 164 | v, _ := ToStringSliceE(i) 165 | return v 166 | } 167 | 168 | // ToIntSlice casts an interface to a []int type. 169 | func ToIntSlice(i interface{}) []int { 170 | v, _ := ToIntSliceE(i) 171 | return v 172 | } 173 | 174 | // ToDurationSlice casts an interface to a []time.Duration type. 175 | func ToDurationSlice(i interface{}) []time.Duration { 176 | v, _ := ToDurationSliceE(i) 177 | return v 178 | } 179 | 180 | // ToInt32Slice casts an interface to a []int32 type. 181 | func ToInt32Slice(i interface{}) []int32 { 182 | v, _ := ToInt32SliceE(i) 183 | return v 184 | } 185 | 186 | // ToInt64Slice casts an interface to a []int64 type. 187 | func ToInt64Slice(i interface{}) []int64 { 188 | v, _ := ToInt64SliceE(i) 189 | return v 190 | } 191 | -------------------------------------------------------------------------------- /gcast/timeformattype_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type timeFormatType"; DO NOT EDIT. 2 | 3 | package gcast 4 | 5 | import ( 6 | "strconv" 7 | ) 8 | 9 | func _() { 10 | // An "invalid array index" compiler error signifies that the constant values have changed. 11 | // Re-run the stringer command to generate them again. 12 | var x [1]struct{} 13 | _ = x[timeFormatNoTimezone-0] 14 | _ = x[timeFormatNamedTimezone-1] 15 | _ = x[timeFormatNumericTimezone-2] 16 | _ = x[timeFormatNumericAndNamedTimezone-3] 17 | _ = x[timeFormatTimeOnly-4] 18 | } 19 | 20 | const _timeFormatType_name = "timeFormatNoTimezonetimeFormatNamedTimezonetimeFormatNumericTimezonetimeFormatNumericAndNamedTimezonetimeFormatTimeOnly" 21 | 22 | var _timeFormatType_index = [...]uint8{0, 20, 43, 68, 101, 119} 23 | 24 | func (i timeFormatType) String() string { 25 | if i < 0 || i >= timeFormatType(len(_timeFormatType_index)-1) { 26 | return "timeFormatType(" + strconv.FormatInt(int64(i), 10) + ")" 27 | } 28 | return _timeFormatType_name[_timeFormatType_index[i]:_timeFormatType_index[i+1]] 29 | } 30 | -------------------------------------------------------------------------------- /gconfig/gconfig.go: -------------------------------------------------------------------------------- 1 | package gconfig 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "reflect" 7 | "strings" 8 | 9 | "gopkg.in/yaml.v3" 10 | ) 11 | 12 | func LoadYamlCfg(path string, v interface{}) { 13 | data, err := ioutil.ReadFile(path) 14 | if err != nil { 15 | panic(fmt.Errorf("LoadYamlCfg: read file:%v err:%v", path, err)) 16 | } 17 | 18 | var m = make(map[string]interface{}) 19 | err = yaml.Unmarshal(data, &m) 20 | if err != nil { 21 | panic(fmt.Errorf("LoadYamlCfg: unmarshal err:%v", err)) 22 | } 23 | 24 | rv := reflect.TypeOf(v) 25 | if rv.Kind() != reflect.Ptr { 26 | panic("LoadYamlCfg: type v must be Ptr") 27 | } 28 | 29 | checkYaml([]string{rv.Elem().String()}, rv.Elem(), m) 30 | 31 | err = yaml.Unmarshal(data, v) 32 | if err != nil { 33 | panic(fmt.Errorf("LoadYamlCfg: unmarshal err:%v", err)) 34 | } 35 | } 36 | 37 | func checkYaml(root []string, vt reflect.Type, m map[string]interface{}) { 38 | for i := 0; i < vt.NumField(); i++ { 39 | field := vt.Field(i) 40 | fieldName := field.Name 41 | if name := field.Tag.Get("yaml"); len(name) > 0 { 42 | fieldName = name 43 | } 44 | 45 | cpRoot := make([]string, len(root)) 46 | copy(cpRoot, root) 47 | cpRoot = append(cpRoot, fieldName) 48 | if v, ok := m[fieldName]; !ok || v == nil { 49 | panic(fmt.Errorf("LoadYamlCfg: field %v not set", strings.Join(cpRoot, "."))) 50 | } 51 | 52 | if field.Type.Kind() == reflect.Struct { 53 | if mv, ok := m[fieldName].(map[string]interface{}); ok { 54 | checkYaml(cpRoot, field.Type, mv) 55 | 56 | } else { 57 | panic(fmt.Errorf("LoadYamlCfg: field %v is struct", strings.Join(cpRoot, "."))) 58 | } 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /gemail/config.go: -------------------------------------------------------------------------------- 1 | package gemail 2 | 3 | const ( 4 | reportUser = "xxx@xxx.com" 5 | mailPassword = "xxx" 6 | mailHost = "xxx.xxx.com" 7 | mailPostfix = "xxx.com" 8 | ) 9 | -------------------------------------------------------------------------------- /gemail/gemail.go: -------------------------------------------------------------------------------- 1 | package gemail 2 | 3 | import ( 4 | "crypto/tls" 5 | "mime" 6 | "net/smtp" 7 | "net/textproto" 8 | "path/filepath" 9 | 10 | "github.com/jordan-wright/email" 11 | "github.com/sirupsen/logrus" 12 | ) 13 | 14 | // AddAttachments 15 | // attachments (allow initial to nil) 16 | // filename result.json/result.xlsx 17 | // content 18 | func AddAttachments(attachments []*email.Attachment, filename string, content []byte) []*email.Attachment { 19 | return append(attachments, &email.Attachment{ 20 | Filename: filename, 21 | ContentType: mime.TypeByExtension(filepath.Ext(filename)), 22 | Header: textproto.MIMEHeader{}, 23 | Content: content, 24 | }) 25 | } 26 | 27 | // SendMail 28 | // to, cc (send_to@xx.com carbon_copy@xx.com) 29 | // subject theme 30 | // text 31 | // attachments (use AddAttachments) 32 | func SendMail(to, cc []string, subject string, text []byte, attachments []*email.Attachment) (err error) { 33 | client := email.NewEmail() 34 | client.From = reportUser 35 | client.To = to 36 | client.Subject = subject 37 | client.Cc = cc 38 | client.Text = text 39 | client.Attachments = attachments 40 | 41 | err = client.SendWithTLS(mailHost+":465", smtp.PlainAuth("", reportUser, mailPassword, mailHost), &tls.Config{ServerName: mailHost}) 42 | if err != nil { 43 | logrus.Errorf("send email failed|from=%v|to=%v|msg=%v|err=%v", reportUser, to, string(text), err) 44 | } else { 45 | logrus.Infof("send email success|from=%v|to=%v|msg=%v|err=%v", reportUser, to, string(text), err) 46 | } 47 | 48 | return 49 | } 50 | -------------------------------------------------------------------------------- /gerr/README.md: -------------------------------------------------------------------------------- 1 | # Unified error handling 2 | 3 | ## how to use 4 | 5 | ``` 6 | ErrCodeNotImpl = 501 7 | ErrCodeNotImplMsg = "The server does not support the requested feature and cannot complete the request" 8 | err := gerr.NewErrInst(ErrCodeNotImpl, ErrCodeNotImplMsg) 9 | 10 | ``` 11 | 12 | err_code error constant err_expand extended error object err handler logic for error handling err lang error 13 | multilingual description initialization err test Test imitate use: Simulate the use process -------------------------------------------------------------------------------- /gerr/err_code.go: -------------------------------------------------------------------------------- 1 | package gerr 2 | 3 | const ( 4 | Mode_Prod = "pro" 5 | Mode_Test = "dev" 6 | ) 7 | 8 | const ( 9 | // error code constant 10 | ERR_LEVEL_SYS = -1 11 | ERR_LEVEL_NOR = 0 12 | 13 | DefaultErrCode = 100 14 | 15 | // example 16 | SuccessCode = 0 17 | SuccessMsg = "success" 18 | 19 | ErrCodeNotFound = 404 20 | ErrCodeNotFoundMsg = "Not found" 21 | 22 | // example 23 | RetCode_kSuccess = 0 24 | RetCode_kPermissionDenied = 2000 25 | RetCode_kSysInternal = 2001 26 | RetCode_kInvalidReq = 2002 27 | RetCode_kLowVersionLimit = 2003 28 | 29 | RetCode_kSysInternalMsg = "System error, please try again later" 30 | ) 31 | -------------------------------------------------------------------------------- /gerr/err_handle.go: -------------------------------------------------------------------------------- 1 | package gerr 2 | 3 | import "fmt" 4 | 5 | // custom err type 6 | type Err struct { 7 | error 8 | code int32 9 | msg string 10 | } 11 | 12 | type Handle struct { 13 | mode string 14 | } 15 | 16 | func New(code int32, msg string) error { 17 | return &Err{ 18 | code: code, 19 | msg: msg, 20 | } 21 | 22 | } 23 | 24 | func Msg(err error) string { 25 | if err == nil { 26 | return SuccessMsg 27 | } 28 | if e, ok := err.(*Err); ok { 29 | return e.Msg() 30 | } 31 | return err.Error() 32 | } 33 | 34 | // append err msg 35 | func Append(err, appendErr error) error { 36 | if err == nil { 37 | return appendErr 38 | } 39 | if appendErr == nil { 40 | return err 41 | } 42 | if e, ok := err.(*Err); ok { 43 | return e.Append(appendErr) 44 | } 45 | return New(DefaultErrCode, err.Error()+"; "+appendErr.Error()) 46 | } 47 | 48 | func (e *Err) Error() string { 49 | return fmt.Sprintf("code: %v, msg: %v", e.Code(), e.Msg()) 50 | } 51 | 52 | func (e *Err) Code() int32 { 53 | return e.code 54 | } 55 | 56 | func (e *Err) Msg() string { 57 | return e.msg 58 | } 59 | 60 | func (e *Err) Append(err error) error { 61 | if err == nil { 62 | return e 63 | } 64 | e.msg += "; " + err.Error() 65 | return e 66 | } 67 | -------------------------------------------------------------------------------- /gerr/err_warn.go: -------------------------------------------------------------------------------- 1 | package gerr 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | helper "github.com/aipave/go-utils/ginfos" 8 | "github.com/aipave/go-utils/gtime" 9 | yalert "github.com/aipave/go-utils/gwarn" 10 | 11 | "github.com/sirupsen/logrus" 12 | ) 13 | 14 | type Warn struct { 15 | alerter *yalert.Alarm 16 | mode string 17 | } 18 | 19 | var warn *Warn 20 | 21 | // initErrHandler 22 | func initErrHandler(mode, warnUrl, topic string) { 23 | warn = &Warn{ 24 | alerter: yalert.AleterGetter(warnUrl, topic), 25 | mode: mode, 26 | } 27 | } 28 | 29 | // WarnErrInfo 30 | func WarnErrInfo(info, funcN string) { 31 | if warn.mode != Mode_Prod { 32 | // return 33 | } 34 | 35 | msg := fmt.Sprintf("Service:%s \nFunction:%s \nError:%s", warn.mode+":"+helper.Runtime.Exec(), funcN, info) 36 | if warn.alerter.WebHookAddr == "" { 37 | logrus.Warnf("[WarnErrInfo] alerter.WebHookAddr is nil, msg:%v", msg) 38 | return 39 | } 40 | 41 | _ = warn.alerter.Warning(msg) 42 | } 43 | 44 | // NoticeInfo 45 | func NoticeInfo(info, fun string) { 46 | if warn.mode != Mode_Prod { 47 | // return 48 | } 49 | 50 | msg := fmt.Sprintf("Service:%s \nFunction:%s \ninfo:%s", warn.mode+":"+helper.Runtime.Exec(), fun, info) 51 | if warn.alerter.WebHookAddr == "" { 52 | logrus.Warnf("[NoticeInfo] alerter.WebHookAddr is nil, msg:%v", msg) 53 | return 54 | } 55 | 56 | _ = warn.alerter.Notice(msg) 57 | } 58 | 59 | // HandleError Handle errors and report prometheus label interface name lang multilingual start interface call start time 60 | func HandleError(label string, lang string, start time.Time, err error) (int32, string) { 61 | code := Success.Code() 62 | desc := Success.Error() 63 | // isSysErr := false 64 | 65 | if err != nil { 66 | errExpand, ok := err.(*ExpandError) 67 | if !ok { 68 | errExpand = CustomizeSysErr(ErrSys.code, err.Error(), "") 69 | } 70 | 71 | desc = errExpand.LangError(lang) 72 | code = errExpand.Code() 73 | 74 | // System-level error DB error, redis error, rpc error, etc., 75 | // you can specify which system-level errors are counted by yourself, 76 | // report the failure of the prometheus interface, 77 | // and you can also send an alarm to Feishu 78 | if code == ErrSys.Code() || errExpand.Level() == ERR_LEVEL_SYS { 79 | logrus.Errorf("[HandleError] label:%s error:%s", label, desc) 80 | // isSysErr = true 81 | go func() { 82 | WarnErrInfo(errExpand.MoreError(), label) 83 | // todo prometheus 84 | }() 85 | } else { 86 | // General errors are not processed. Parameters are wrong, not in the appropriate range, 87 | // not in opening hours, etc. You can specify which ones are considered general errors. 88 | logrus.Warnf("[HandleError] label:%s warn:%s", label, errExpand.MoreError()) 89 | } 90 | } 91 | 92 | // todo prometheus 93 | //if !isSysErr { 94 | // // go gprometheus.Inc 95 | //} 96 | 97 | cost := time.Since(start) 98 | logrus.Infof("%v: start:%v cost:%v", label, start.Format(gtime.FormatDefault), cost) 99 | // todo prometheus 100 | // go gprometheus.Inc 101 | return code, desc 102 | } 103 | -------------------------------------------------------------------------------- /gerr/errtypes_customize.go: -------------------------------------------------------------------------------- 1 | package gerr 2 | 3 | import ( 4 | "fmt" 5 | "runtime/debug" 6 | ) 7 | 8 | var ( 9 | Success *ExpandError 10 | ErrSys *ExpandError 11 | ErrPermissionDenied *ExpandError 12 | ErrInvalidReq *ExpandError 13 | ErrVersion *ExpandError 14 | ) 15 | 16 | func init() { 17 | Success = &ExpandError{ 18 | code: int32(RetCode_kSuccess), 19 | } 20 | ErrVersion = &ExpandError{ 21 | code: int32(RetCode_kLowVersionLimit), 22 | } 23 | ErrSys = &ExpandError{ 24 | code: int32(RetCode_kSysInternal), 25 | msg: "System error, please try again later", 26 | } 27 | ErrInvalidReq = &ExpandError{ 28 | code: int32(RetCode_kInvalidReq), 29 | } 30 | ErrPermissionDenied = &ExpandError{ 31 | code: int32(RetCode_kPermissionDenied), 32 | } 33 | } 34 | 35 | func Init(mode, warnUrl, topic string) { 36 | initErrHandler(mode, warnUrl, topic) 37 | } 38 | 39 | // ExpandError 40 | type ExpandError struct { 41 | level int32 42 | code int32 43 | msg string 44 | sceneInfos string 45 | stackInfos string 46 | } 47 | 48 | func (e *ExpandError) Error() string { 49 | return e.msg 50 | } 51 | 52 | func (e *ExpandError) Code() int32 { 53 | return e.code 54 | } 55 | 56 | func (e *ExpandError) Level() int32 { 57 | return e.level 58 | } 59 | 60 | // LangError 61 | func (e *ExpandError) LangError(lang string) string { 62 | //todo return with lang 63 | 64 | return e.Error() 65 | } 66 | 67 | // MoreError Error details description, mainly used to display error details when warning 68 | func (e *ExpandError) MoreError() string { 69 | if e.sceneInfos != "" { 70 | if e.stackInfos != "" { 71 | return fmt.Sprintf("msg:%s \nsceneInfos:%s \nstack:%s", e.Error(), e.sceneInfos, e.stackInfos) 72 | } 73 | return fmt.Sprintf("msg:%s \nsceneInfos:%s", e.Error(), e.sceneInfos) 74 | } else { 75 | if e.stackInfos != "" { 76 | return fmt.Sprintf("msg:%s \nstack:%s", e.Error(), e.stackInfos) 77 | } 78 | } 79 | return e.Error() 80 | } 81 | 82 | // CaseErr Generate an error object with some custom information for specific scenarios based on underlying errors 83 | func CaseErr(e *ExpandError, diyInfo string) *ExpandError { 84 | e_ := &ExpandError{ 85 | code: e.code, 86 | msg: e.Error(), 87 | sceneInfos: diyInfo, 88 | } 89 | // If it is a system error that generates stack information, other errors 90 | // are normal business errors and do not generate stack information 91 | if e.code == ErrSys.code || e.level == ERR_LEVEL_SYS { 92 | e_.stackInfos = string(debug.Stack()) 93 | } 94 | return e_ 95 | } 96 | 97 | // CustomizeSysErr Customize the system-level error object, mainly to undertake other service errors, and you can add description information 98 | func CustomizeSysErr(code int32, msg, diyInfo string) *ExpandError { 99 | return &ExpandError{ 100 | code: ErrSys.code, 101 | msg: fmt.Sprintf("%v[%d]", msg, code), 102 | sceneInfos: diyInfo, 103 | } 104 | } 105 | 106 | // CustomizeNorErr 107 | // If you don't want the error object to be initialized at the beginning, you can use this function to generate it at any time 108 | func CustomizeNorErr(code, level int32, msg, langKey, diyInfo string) *ExpandError { 109 | return &ExpandError{ 110 | level: level, 111 | code: code, 112 | msg: msg, 113 | sceneInfos: diyInfo, 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /gexit/README.md: -------------------------------------------------------------------------------- 1 | # gexit 2 | 3 | graceful exit -------------------------------------------------------------------------------- /gexit/graceful_exit.go: -------------------------------------------------------------------------------- 1 | package gexit 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | "time" 9 | 10 | "github.com/sirupsen/logrus" 11 | ) 12 | 13 | var sig = make(chan os.Signal, 1) 14 | var blockChan = make(chan int) 15 | var releaseHandlers []func() 16 | var closeHandlers []func() 17 | 18 | func init() { 19 | signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) 20 | 21 | go wait() 22 | } 23 | 24 | // Close 25 | func Close(handler func()) { 26 | closeHandlers = append(closeHandlers, handler) 27 | } 28 | 29 | // Release 30 | func Release(handler func()) { 31 | releaseHandlers = append(releaseHandlers, handler) 32 | } 33 | 34 | // Wait 35 | func Wait() { 36 | <-blockChan 37 | } 38 | 39 | func wait() { 40 | defer func() { 41 | if err := recover(); err != nil { 42 | fmt.Println(err) 43 | } 44 | }() 45 | 46 | select { 47 | case <-sig: 48 | for _, handler := range closeHandlers { 49 | if handler != nil { 50 | handler() // close all 51 | } 52 | } 53 | } 54 | 55 | // exit 56 | logrus.Infof("received term signal, process will exit after 3 seconds\n") 57 | 58 | time.Sleep(3 * time.Second) 59 | 60 | for _, handler := range releaseHandlers { 61 | if handler != nil { 62 | handler() // release all 63 | } 64 | } 65 | 66 | blockChan <- 0 67 | os.Exit(0) 68 | } 69 | -------------------------------------------------------------------------------- /ginfos/infos.go: -------------------------------------------------------------------------------- 1 | package ginfos 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "os" 7 | "path/filepath" 8 | "runtime" 9 | "strings" 10 | ) 11 | 12 | type _Runtime struct { 13 | } 14 | 15 | // Runtime 16 | var Runtime _Runtime 17 | 18 | // Exec name 19 | func (_Runtime) Exec() string { 20 | pwd, _ := os.Executable() 21 | _, exec := filepath.Split(pwd) 22 | return exec 23 | } 24 | 25 | // Pwd exe path 26 | func (_Runtime) Pwd() string { 27 | pwd, _ := os.Executable() 28 | pwd, _ = filepath.Split(pwd) 29 | return pwd 30 | } 31 | 32 | // Func name 33 | func (_Runtime) Func() string { 34 | const unknown = "Unknown_Func" 35 | pc, _, _, ok := runtime.Caller(1) 36 | if !ok { 37 | return unknown 38 | } 39 | 40 | f := runtime.FuncForPC(pc) 41 | if f == nil { 42 | return unknown 43 | } 44 | 45 | return f.Name() 46 | } 47 | 48 | func FuncName() string { 49 | pc := make([]uintptr, 1) 50 | runtime.Callers(2, pc) 51 | f := runtime.FuncForPC(pc[0]) 52 | funcName := strings.ReplaceAll(f.Name(), "main.", "") 53 | arr := strings.Split(funcName, ".") 54 | if len(arr) >= 1 { 55 | return fmt.Sprintf("%s", arr[len(arr)-1]) 56 | } 57 | return funcName 58 | } 59 | 60 | func CallerFuncName(skip int) string { 61 | pc, _, line, ok := runtime.Caller(skip) 62 | if !ok { 63 | return "" 64 | } 65 | name := runtime.FuncForPC(pc).Name() 66 | arr := strings.Split(name, ".") 67 | if len(arr) >= 1 { 68 | return fmt.Sprintf("[%s-%d]", arr[len(arr)-1], line) 69 | } 70 | return fmt.Sprintf("[%s-%d]", name, line) 71 | } 72 | 73 | // IP 74 | func (_Runtime) IP() string { 75 | ips, _ := net.InterfaceAddrs() 76 | 77 | for _, ip := range ips { 78 | if ipAddr, ok := ip.(*net.IPNet); ok && !ipAddr.IP.IsLoopback() && ipAddr.IP.To4() != nil { 79 | return ipAddr.IP.To4().String() 80 | } 81 | } 82 | 83 | return "" 84 | } 85 | 86 | func (_Runtime) IsPrivateIP(ip string) bool { 87 | addr := net.ParseIP(ip) 88 | return addr.IsPrivate() 89 | } 90 | -------------------------------------------------------------------------------- /ginfos/marshal.go: -------------------------------------------------------------------------------- 1 | package ginfos 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | ) 9 | 10 | // NilStruct if v==nil, return a struct 11 | func NilStruct(v any) any { 12 | if v != nil { 13 | return v 14 | } 15 | 16 | return struct{}{} 17 | } 18 | 19 | // NilSlice if v==nil, return a struct 20 | func NilSlice(v any) any { 21 | if v != nil { 22 | return v 23 | } 24 | 25 | return []interface{}{} 26 | } 27 | 28 | func JsonStr(v any) string { 29 | jsonStr, _ := json.Marshal(v) 30 | return string(jsonStr) 31 | } 32 | 33 | // Md5 md5 sign 34 | func Md5(content string) (md string) { 35 | h := md5.New() 36 | _, _ = io.WriteString(h, content) 37 | md = fmt.Sprintf("%x", h.Sum(nil)) 38 | return 39 | } 40 | -------------------------------------------------------------------------------- /ginfos/version.go: -------------------------------------------------------------------------------- 1 | package ginfos 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | var ( 9 | GoVersion string 10 | GitBranch string 11 | GitCommitID string 12 | GitTag string 13 | GitBuildTime string 14 | ) 15 | 16 | func init() { 17 | if len(os.Args) == 2 && (os.Args[1] == "-v" || os.Args[1] == "version") { 18 | fmt.Println("Go Version: ", GoVersion) 19 | fmt.Println("Git Branch: ", GitBranch) 20 | fmt.Println("Git CommitID: ", GitCommitID) 21 | fmt.Println("Git Tag: ", GitTag) 22 | fmt.Println("Build Time: ", GitBuildTime) 23 | 24 | os.Exit(0) 25 | } 26 | } 27 | 28 | func Version() { 29 | } 30 | -------------------------------------------------------------------------------- /gkafka/README.md: -------------------------------------------------------------------------------- 1 | # Kafka 2 | 3 | ## Partitioning strategy in kafka 4 | 5 | In Kafka, the partitioning strategy determines how messages are distributed across the partitions of a topic. The 6 | partitioning strategy is important because it affects the performance and scalability of the Kafka cluster, and can also 7 | impact the ordering and reliability of messages. 8 | 9 | Kafka provides several built-in partitioning strategies that can be used to distribute messages across the partitions of 10 | a topic: 11 | 12 | Round-robin partitioning: In this strategy, messages are distributed across partitions in a round-robin fashion, so that 13 | each partition receives an equal number of messages. 14 | 15 | Hash-based partitioning: In this strategy, a hash function is applied to the message key (if available) to determine the 16 | partition to which the message is sent. This ensures that messages with the same key are always sent to the same 17 | partition, which can be useful for maintaining message ordering or grouping related messages together. 18 | 19 | Range-based partitioning: In this strategy, partitions are assigned ranges of keys, and messages are sent to the 20 | partition whose range includes the message key. This can be useful for applications that require message keys to be 21 | ordered, as messages with adjacent keys will be sent to adjacent partitions. 22 | 23 | Custom partitioning: Kafka also allows you to define your own custom partitioning strategy by implementing the 24 | org.apache.kafka.clients.producer.Partitioner interface. This can be useful for more advanced use cases where none of 25 | the built-in partitioning strategies are sufficient. 26 | 27 | When choosing a partitioning strategy, it is important to consider factors such as message ordering requirements, key 28 | distribution, and cluster scalability. Different strategies may be more appropriate for different use cases, and it may 29 | be necessary to experiment with different strategies to determine the best one for your application. -------------------------------------------------------------------------------- /gkafka/consumer_group.go: -------------------------------------------------------------------------------- 1 | package gkafka 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "time" 9 | 10 | "github.com/Shopify/sarama" 11 | grace "github.com/aipave/go-utils/gexit" 12 | "github.com/sirupsen/logrus" 13 | ) 14 | 15 | const ( 16 | PartitionStrategyRange = "range" 17 | PartitionStrategyRoundRobin = "roundrobin" 18 | PartitionStrategySticky = "sticky" 19 | ) 20 | 21 | type consumer struct { 22 | session sarama.ConsumerGroupSession 23 | message *sarama.ConsumerMessage 24 | } 25 | 26 | var topicChannel = make(map[string]chan *consumer) 27 | 28 | // InitKafkaConsumerGroup(Host(xxx), Topics(xxx), Handler(xxx)) 29 | func InitKafkaConsumerGroup(opts ...ConfigOptions) { 30 | var option Options 31 | for _, fn := range opts { 32 | fn(&option) 33 | } 34 | 35 | logrus.Infof("start init consumer group addr:%v topics:%v", option.Hosts, option.Topics) 36 | 37 | // kafka config 38 | cfg := sarama.NewConfig() 39 | // defaul sticky strategy 40 | cfg.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategySticky 41 | cfg.Consumer.Group.Session.Timeout = 10 * time.Second 42 | cfg.Consumer.Group.Heartbeat.Interval = 3 * time.Second 43 | cfg.Consumer.IsolationLevel = sarama.ReadCommitted 44 | cfg.Consumer.Offsets.Initial = sarama.OffsetNewest 45 | cfg.Version = sarama.V2_5_0_0 46 | 47 | // set partition 48 | if option.PartitionStrategy == PartitionStrategyRange { 49 | cfg.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRange 50 | } else if option.PartitionStrategy == PartitionStrategyRoundRobin { 51 | cfg.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRoundRobin 52 | } 53 | 54 | // set consumer group 55 | var groupID = option.GroupID 56 | if len(groupID) == 0 { 57 | groupID = defaultPath() // default is process_name 58 | } 59 | consumerGroup, err := sarama.NewConsumerGroup(option.Hosts, groupID, cfg) 60 | if err != nil { 61 | panic(fmt.Sprintf("init kafka consumer err:%v host:%v topics:%v", err, option.Hosts, option.Topics)) 62 | } 63 | 64 | if cfg.Consumer.Return.Errors { 65 | go func() { 66 | for errMsg := range consumerGroup.Errors() { 67 | if option.ConsumerErrorCallback != nil { 68 | option.ConsumerErrorCallback(errMsg) 69 | } 70 | } 71 | }() 72 | } 73 | 74 | // init queue 75 | for _, topic := range option.Topics { 76 | var topicChan = make(chan *consumer, 100) 77 | go topicConsumer(option, topicChan) 78 | topicChannel[topic] = topicChan 79 | } 80 | 81 | ctx, cancel := context.WithCancel(context.Background()) 82 | go func() { 83 | for { 84 | err = consumerGroup.Consume(ctx, option.Topics, &consumerGroupHandler{opts: option, handle: option.Handle}) 85 | if err != nil { 86 | logrus.Errorf("consumer returned with err:%v addr:%v topics:%v", err, option.Hosts, option.Topics) 87 | } 88 | 89 | if ctx.Err() != nil { 90 | logrus.Warningf("ctx done: %v", ctx.Err()) 91 | return 92 | } 93 | } 94 | }() 95 | 96 | grace.Close(func() { 97 | cancel() 98 | err = consumerGroup.Close() 99 | if err != nil { 100 | logrus.Errorf("consumer group close err:%v host:%v topics:%v", err, option.Hosts, option.Topics) 101 | } 102 | 103 | logrus.Infof("kafka addr:%v consumer closed", option.Hosts) 104 | }) 105 | 106 | logrus.Infof("init consumer group success addr:%v topics:%v", option.Hosts, option.Topics) 107 | } 108 | 109 | func defaultPath() string { 110 | pwd, _ := os.Executable() 111 | _, exec := filepath.Split(pwd) 112 | return exec 113 | } 114 | 115 | type consumerGroupHandler struct { 116 | opts Options 117 | handle func(ctx context.Context, message *sarama.ConsumerMessage) error 118 | } 119 | 120 | // Setup hooks before consumeClaim 121 | func (c consumerGroupHandler) Setup(session sarama.ConsumerGroupSession) error { 122 | return nil 123 | } 124 | 125 | // Cleanup hooks after consumeClaim 126 | func (c consumerGroupHandler) Cleanup(session sarama.ConsumerGroupSession) error { 127 | return nil 128 | } 129 | 130 | // ConsumeClaim 131 | func (c consumerGroupHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { 132 | for msg := range claim.Messages() { 133 | if msg == nil { 134 | session.MarkMessage(msg, "") 135 | logrus.Warningf("handle nil msg, continue") 136 | continue 137 | } 138 | 139 | if topicChan, ok := topicChannel[msg.Topic]; ok { 140 | topicChan <- &consumer{session: session, message: msg} 141 | } else { 142 | err := c.opts.Handle(session.Context(), msg) 143 | if err != nil { 144 | logrus.Warningf("default|msg=%v|err=%v", string(msg.Value), err) 145 | } 146 | } 147 | } 148 | 149 | return nil 150 | } 151 | 152 | func topicConsumer(opts Options, consumerChan chan *consumer) { 153 | for consumerMessage := range consumerChan { 154 | session, msg := consumerMessage.session, consumerMessage.message 155 | 156 | // handle msg 157 | err := opts.Handle(session.Context(), msg) 158 | if err != nil { 159 | logrus.Errorf("handle msg:%+v, timestamp:%v, header:%+v, topic:%v, partition:%v, offset:%v, err:%v", string(msg.Value), msg.Timestamp, msg.Headers, msg.Topic, msg.Partition, msg.Offset, err) 160 | } 161 | 162 | // consumer success or drop failed message 163 | if err == nil || !opts.Retry { 164 | session.MarkMessage(msg, "") 165 | } else { 166 | // TODO: handle retry 167 | } 168 | } 169 | 170 | } 171 | -------------------------------------------------------------------------------- /gkafka/producer_with_async.go: -------------------------------------------------------------------------------- 1 | package gkafka 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | 8 | "github.com/Shopify/sarama" 9 | graceful "github.com/aipave/go-utils/gexit" 10 | "github.com/sirupsen/logrus" 11 | ) 12 | 13 | // InitAsyncProducer initializes the asynchronous interface. 14 | // Important: 15 | // If c.Producer.Return.Errors = true, kafka.ErrorCallback must be configured, otherwise there is a risk of panic. 16 | // If c.Producer.Return.Success = true, kafka.SuccessCallback must be configured, otherwise there is a risk of panic. 17 | func InitAsyncProducer(cfg *sarama.Config, opts ...ConfigOptions) (asyncProducer AsyncProducer) { 18 | // config 19 | var option Options 20 | for _, fn := range opts { 21 | fn(&option) 22 | } 23 | 24 | logrus.Infof("begin init async producer options:%+v, config:%+v", option, cfg) 25 | 26 | // init Producer 27 | producer, err := sarama.NewAsyncProducer(option.Hosts, cfg) 28 | if err != nil { 29 | panic(fmt.Sprintf("init kafka async producer err:%v hosts:%v config:%+v", err, option.Hosts, cfg)) 30 | } 31 | 32 | // handle error message 33 | if cfg.Producer.Return.Errors { 34 | go func() { 35 | for errMsg := range producer.Errors() { 36 | go option.ProducerErrorCallback(errMsg) 37 | } 38 | }() 39 | } 40 | 41 | // handle success message 42 | if cfg.Producer.Return.Successes { 43 | go func() { 44 | for successMsg := range producer.Successes() { 45 | go option.ProducerSuccessCallback(successMsg) 46 | } 47 | }() 48 | } 49 | 50 | ctx, cancel := context.WithCancel(context.Background()) 51 | 52 | asyncProducer.Producer = producer 53 | asyncProducer.ctx = ctx 54 | 55 | graceful.Release(func() { 56 | cancel() 57 | err = producer.Close() 58 | if err != nil { 59 | logrus.Errorf("close producer:%+v err:%v", producer, err) 60 | } 61 | 62 | logrus.Infof("kafka producer:%v resource released", option.Hosts) 63 | }) 64 | 65 | logrus.Infof("init async producer success, options:%+v, config:%+v", option, cfg) 66 | return 67 | } 68 | 69 | // PublishAsyncMessage 70 | func PublishAsyncMessage(producer AsyncProducer, topic string, message Message) (err error) { 71 | if len(topic) == 0 { 72 | logrus.Errorf("empty topic:%v is unsupported, message:%+v", topic, message) 73 | return 74 | } 75 | 76 | if !message.Check() { 77 | logrus.Errorf("missing producerSeqID or producerTimestamp, topic:%v, message:%+v", topic, message) 78 | return 79 | } 80 | 81 | value, _ := json.Marshal(message) 82 | 83 | var producerMsg = &sarama.ProducerMessage{ 84 | Topic: topic, 85 | Value: sarama.StringEncoder(value), 86 | } 87 | 88 | select { 89 | case <-producer.ctx.Done(): 90 | logrus.Warningf("context done, drop topic:%v msg:%+v", topic, message) 91 | return 92 | default: 93 | } 94 | 95 | producer.Producer.Input() <- producerMsg 96 | logrus.Infof("produced async msg, topic:%v, message:%+v", topic, message) 97 | return 98 | } 99 | 100 | // PublishAsyncRawMessage 101 | func PublishAsyncRawMessage(producer AsyncProducer, producerMessage *sarama.ProducerMessage) (err error) { 102 | select { 103 | case <-producer.ctx.Done(): 104 | logrus.Warningf("context done, drop message:%+v", producerMessage) 105 | return 106 | default: 107 | } 108 | 109 | producer.Producer.Input() <- producerMessage 110 | logrus.Infof("produced async msg, message:%+v", producerMessage) 111 | return 112 | } 113 | -------------------------------------------------------------------------------- /gkafka/producer_with_sync.go: -------------------------------------------------------------------------------- 1 | package gkafka 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/Shopify/sarama" 8 | grace "github.com/aipave/go-utils/gexit" 9 | ginfos "github.com/aipave/go-utils/ginfos" 10 | "github.com/sirupsen/logrus" 11 | ) 12 | 13 | // InitSyncProducer 14 | func InitSyncProducer(opts ...ConfigOptions) (producer sarama.SyncProducer) { 15 | var option Options 16 | for _, fn := range opts { 17 | fn(&option) 18 | } 19 | 20 | logrus.Infof("begin init sync producer option:%+v", option) 21 | 22 | // init Producer 23 | var err error 24 | var c = sarama.NewConfig() 25 | c.Producer.Return.Successes = true 26 | producer, err = sarama.NewSyncProducer(option.Hosts, c) 27 | if err != nil { 28 | panic(fmt.Sprintf("init kafka sync producer err:%v, hosts:%v, config:%+v", err, option.Hosts, c)) 29 | } 30 | 31 | grace.Release(func() { 32 | err = producer.Close() 33 | if err != nil { 34 | logrus.Errorf("close producer:%+v err:%v", producer, err) 35 | } 36 | 37 | logrus.Infof("producer:%v resource released", option.Hosts) 38 | }) 39 | 40 | logrus.Infof("init sync producer success, option:%+v, config:%v", option, c) 41 | return producer 42 | } 43 | 44 | // PublishSyncMessage 45 | func PublishSyncMessage(producer sarama.SyncProducer, topic string, message Message) (err error) { 46 | if len(topic) == 0 { 47 | logrus.Errorf("empty topic:%v is unsupported, message:%+v", topic, message) 48 | return 49 | } 50 | 51 | if !message.Check() { 52 | logrus.Errorf("missing producerSeqID or producerTimestamp, topic:%v, message:%+v", topic, message) 53 | return 54 | } 55 | 56 | value, _ := json.Marshal(message) 57 | var producerMsg = &sarama.ProducerMessage{ 58 | Topic: topic, 59 | Value: sarama.StringEncoder(value), 60 | } 61 | 62 | var partition, offset = int32(0), int64(0) 63 | partition, offset, err = producer.SendMessage(producerMsg) 64 | if err != nil { 65 | logrus.Errorf("send message failed, topic:%v partition:%v offset:%v err:%v value:%v", topic, partition, offset, err, string(value)) 66 | } else { 67 | logrus.Infof("send message success, topic:%v partition:%v offset:%v value:%v", topic, partition, offset, string(value)) 68 | } 69 | 70 | return 71 | } 72 | 73 | // SyncPublish 74 | func SyncPublish(producer sarama.SyncProducer, topic string, message []byte) (err error) { 75 | if len(topic) == 0 { 76 | logrus.Errorf("empty topic:%v is unsupported, message:%+v", topic, message) 77 | return 78 | } 79 | 80 | var producerMsg = &sarama.ProducerMessage{ 81 | Topic: topic, 82 | Value: sarama.StringEncoder(message), 83 | } 84 | 85 | var partition, offset = int32(0), int64(0) 86 | partition, offset, err = producer.SendMessage(producerMsg) 87 | if err != nil { 88 | logrus.Errorf("send message failed, topic:%v partition:%v offset:%v err:%v value:%v", topic, partition, offset, err, string(message)) 89 | } else { 90 | logrus.Infof("send message success, topic:%v partition:%v offset:%v value:%v", topic, partition, offset, string(message)) 91 | } 92 | 93 | return 94 | } 95 | 96 | // PublishDelayMessage Delayed Queue 97 | // unique -- Unique identifier for the message, ensuring uniqueness within the Topic dimension is sufficient 98 | // deadline -- Deadline for the delay to end, in seconds 99 | // topic -- Delayed queue TOPIC (Message feedback queue) 100 | // msg -- Delayed message content 101 | func PublishDelayMessage(producer sarama.SyncProducer, unique string, deadline int64, topic string, msg []byte) (err error) { 102 | message := ginfos.JsonStr(map[string]interface{}{ 103 | "unique": unique, 104 | "deadline": deadline, 105 | "topic": topic, 106 | "msg": string(msg), 107 | }) 108 | 109 | var producerMsg = &sarama.ProducerMessage{ 110 | Topic: "delay_queue_svr", 111 | Value: sarama.StringEncoder(message), 112 | } 113 | 114 | var partition, offset = int32(0), int64(0) 115 | partition, offset, err = producer.SendMessage(producerMsg) 116 | if err != nil { 117 | logrus.Errorf("send message failed, topic:%v partition:%v offset:%v err:%v value:%v", topic, partition, offset, err, message) 118 | } else { 119 | logrus.Infof("send message success, topic:%v partition:%v offset:%v value:%v", topic, partition, offset, message) 120 | } 121 | 122 | return 123 | } 124 | -------------------------------------------------------------------------------- /gkafka/types.go: -------------------------------------------------------------------------------- 1 | package gkafka 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/Shopify/sarama" 7 | ) 8 | 9 | // Message 10 | type Message interface { 11 | Check() bool 12 | } 13 | 14 | const ( 15 | ContentTypeJSON = "json" 16 | ContentTypeProto = "protobuf" 17 | ContentTypeAppCustom = "app/custom" 18 | ) 19 | 20 | // Header 21 | type Header struct { 22 | // the ID and CorrelationID fields both provide unique identifiers for messages 23 | // the ID is used more generally to identify messages 24 | // the CorrelationID is specifically designed to help correlate related messages within a single request-response flow 25 | SeqID string `json:"id"` // message sequence 26 | ///> import "github.com/google/uuid" with ID uuid.New() to generate a UUID like "f81d4fae-7dec-11d0-a765-00a0c91e6bf6" 27 | ///> the TraceID as follows: "f81d4fae-7dec-11d0-a765-00a0c91e6bf6.auth-service.1" mean "`uuid`.`service name`.`the instance identifier`" 28 | TraceID string `json:"traceID"` // used to track a request as it flows through multiple services, and it may change as the request is processed by different services. Each service that handles the request would typically add its own identifier to the TraceID field, creating a trace that shows the path of the request through the system. 29 | CorrelationID string `json:"correlationID"` // In distributed systems, used to correlate related messages within a single request-response flow, remaining the same throughout the entire request-response flow, regardless of how many services are involved in processing the request. 30 | 31 | Topic string `json:"topic"` // for routing messages to the appropriate topic based on their content or destination 32 | ContentType string `json:"contentType"` // indicating the content type or format of the message payload (e.g. default JSON, Avro, Protobuf, etc.) 33 | 34 | Key KeyInfo `json:"keyInfo"` // This can be useful for ensuring that related messages are sent to the same partition, or for grouping messages based on a shared attribute 35 | 36 | Version string `json:"version"` // Backward compatibility, Graceful migration, Auditability 37 | Source string `json:"source"` // indicating the source of the message, such as the name of the service or application that produced the message, for debugging and tracing msg flows 38 | 39 | Timestamp int64 `json:"timestamp"` // message creation timestamp 40 | } 41 | 42 | const ( 43 | KeyTypeUserID = "key/uid" // all messages related to that the specific user are sent to the same partition, so they can be processed in order. 44 | KeyDeviceID = "key/device" // In an e-commerce platform, use the order ID as the Key for messages related to a specific order 45 | KeyTypeOrderID = "key/order" // use the product ID as the Key for messages related to a specific product 46 | KeyTypeProductID = "key/product" // In a geolocation-based system, ensures that all messages related to that location are sent to the same partition 47 | KeyTypeLocationID = "key/location" // In an IoT platform, ensures that all messages related to that device are sent to the same partition 48 | ) 49 | 50 | type KeyInfo struct { 51 | KeyType string `json:"keyType"` 52 | KeyContent string `json:"keyContent"` 53 | } 54 | 55 | // Check 消息头校验 56 | func (h Header) Check() bool { 57 | if len(h.SeqID) == 0 || h.Timestamp == 0 { 58 | return false 59 | } 60 | 61 | return true 62 | } 63 | 64 | // AsyncProducer 异步生产者 65 | type AsyncProducer struct { 66 | ctx context.Context 67 | Producer sarama.AsyncProducer 68 | } 69 | -------------------------------------------------------------------------------- /gkafka/with_options.go: -------------------------------------------------------------------------------- 1 | package gkafka 2 | 3 | import ( 4 | "context" 5 | "github.com/Shopify/sarama" 6 | ) 7 | 8 | type Options struct { 9 | Hosts []string 10 | Topics []string 11 | GroupID string 12 | Retry bool 13 | Handle func(ctx context.Context, message *sarama.ConsumerMessage) error 14 | ProducerSuccessCallback func(message *sarama.ProducerMessage) 15 | ProducerErrorCallback func(message *sarama.ProducerError) 16 | ConsumerErrorCallback func(message error) 17 | PartitionStrategy string // partitionning strategy 18 | } 19 | 20 | type ConfigOptions func(options *Options) 21 | 22 | // GroupID 23 | func GroupID(id string) ConfigOptions { 24 | return func(options *Options) { 25 | options.GroupID = id 26 | } 27 | } 28 | 29 | // Hosts host name 30 | func Hosts(hosts []string) ConfigOptions { 31 | return func(options *Options) { 32 | options.Hosts = hosts 33 | } 34 | } 35 | 36 | // Topics set Topic 37 | func Topics(topics ...string) ConfigOptions { 38 | return func(options *Options) { 39 | options.Topics = topics 40 | } 41 | } 42 | 43 | // Handler 44 | func Handler(handler func(ctx context.Context, message *sarama.ConsumerMessage) error) ConfigOptions { 45 | return func(options *Options) { 46 | options.Handle = handler 47 | } 48 | } 49 | 50 | // Retry for consumers 51 | func Retry(retry bool) ConfigOptions { 52 | return func(options *Options) { 53 | options.Retry = retry 54 | } 55 | } 56 | 57 | // SuccessCallback for asyn msg send 58 | func SuccessCallback(callback func(message *sarama.ProducerMessage)) ConfigOptions { 59 | return func(options *Options) { 60 | options.ProducerSuccessCallback = callback 61 | } 62 | } 63 | 64 | // ErrorCallback for asyn msg send 65 | func ErrorCallback(callback func(message *sarama.ProducerError)) ConfigOptions { 66 | return func(options *Options) { 67 | options.ProducerErrorCallback = callback 68 | } 69 | } 70 | 71 | // PartitionStrategy 72 | func PartitionStrategy(strategy string) ConfigOptions { 73 | return func(options *Options) { 74 | options.PartitionStrategy = strategy 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /glogs/README.md: -------------------------------------------------------------------------------- 1 | # logs 2 | 3 | ## Description 4 | 1. Define log formatter 5 | 2. Redirect log directory 6 | 3. Set up log rotation. 7 | 8 | ## Usage 9 | ```go 10 | func yourLogic() { 11 | defer grecover.Recover() 12 | // some logic here 13 | } 14 | ``` 15 | 16 | ## Recover 17 | The panic captured by recover will be output to the panic log and then trigger an alert 18 | ```go 19 | func yourLogic() { 20 | defer grecover.Recover() 21 | 22 | // some logic here 23 | } 24 | ``` 25 | 26 | ## Uncaught Panic 27 | After the program is restarted, it will be split into multiple 28 | log blocks according to the following separator, 29 | and the last block will be obtained. 30 | If a panic log is matched, an alert will be triggered. 31 | 32 | ```go 33 | progress started at: ---------%v----------- 34 | ``` -------------------------------------------------------------------------------- /glogs/glogrotate/rotate.go: -------------------------------------------------------------------------------- 1 | package glogrotate 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/robfig/cron" 7 | 8 | "github.com/natefinch/lumberjack" 9 | ) 10 | 11 | var m sync.Mutex 12 | var loggerMap = make(map[string]*lumberjack.Logger) 13 | 14 | // Divide the log every day at 0:00 15 | func init() { 16 | c := cron.New() 17 | _ = c.AddFunc("0 0 0 * * *", rotate) 18 | c.Start() 19 | } 20 | 21 | // NewWriter 22 | func NewWriter(logger *lumberjack.Logger) *lumberjack.Logger { 23 | m.Lock() 24 | defer m.Unlock() 25 | 26 | if logfile, exist := loggerMap[logger.Filename]; exist { 27 | _ = logfile.Close() 28 | } 29 | 30 | loggerMap[logger.Filename] = logger 31 | 32 | return logger 33 | } 34 | 35 | func GetWriter(filename string) *lumberjack.Logger { 36 | m.Lock() 37 | defer m.Unlock() 38 | 39 | // For the same log file, only one copy is initialized 40 | if logger, ok := loggerMap[filename]; ok { 41 | return logger 42 | } 43 | 44 | // init 45 | logger := &lumberjack.Logger{ 46 | Filename: filename, 47 | MaxSize: 256, // max 256M 48 | MaxAge: 30, // max 30day 49 | MaxBackups: 30, // max files 50 | Compress: true, 51 | } 52 | loggerMap[filename] = logger 53 | 54 | return logger 55 | } 56 | 57 | func rotate() { 58 | if len(loggerMap) == 0 { 59 | return // nothing to do 60 | } 61 | 62 | for _, logger := range loggerMap { 63 | _ = logger.Rotate() 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /glogs/glogrus/config.go: -------------------------------------------------------------------------------- 1 | package glogrus 2 | 3 | import ( 4 | "github.com/natefinch/lumberjack" 5 | "github.com/sirupsen/logrus" 6 | ) 7 | 8 | type config struct { 9 | lumLogger *lumberjack.Logger 10 | logLevel *logrus.Level 11 | reportCaller *bool 12 | formatter logrus.Formatter 13 | hooks []logrus.Hook 14 | maxAge int64 15 | filename string 16 | alertUrl string 17 | ignoreIpSet []string 18 | ignoreIpPrefix []string 19 | } 20 | 21 | type LogOption func(*config) 22 | 23 | // WithLumLogger Set Up Log Rotation 24 | func WithLumLogger(lum *lumberjack.Logger) LogOption { 25 | return func(c *config) { 26 | c.lumLogger = lum 27 | } 28 | } 29 | 30 | // WithAlertUrl 31 | func WithAlertUrl(url string) LogOption { 32 | return func(c *config) { 33 | c.alertUrl = url 34 | } 35 | } 36 | 37 | // WithLogLevel 38 | func WithLogLevel(level logrus.Level) LogOption { 39 | return func(c *config) { 40 | c.logLevel = &level 41 | } 42 | } 43 | 44 | func WithIgnoreIpSet(set []string) LogOption { 45 | return func(c *config) { 46 | c.ignoreIpSet = set 47 | } 48 | } 49 | 50 | func WithIgnoreIpPrefix(set []string) LogOption { 51 | return func(c *config) { 52 | c.ignoreIpPrefix = set 53 | } 54 | } 55 | 56 | // WithReportCaller 57 | func WithReportCaller(v bool) LogOption { 58 | return func(c *config) { 59 | c.reportCaller = &v 60 | } 61 | } 62 | 63 | // WithFormatter 64 | func WithFormatter(format logrus.Formatter) LogOption { 65 | return func(c *config) { 66 | c.formatter = format 67 | } 68 | } 69 | 70 | func WithHooks(hook ...logrus.Hook) LogOption { 71 | return func(c *config) { 72 | c.hooks = hook 73 | } 74 | } 75 | 76 | // WithMaxAge function specifies the maximum length of time is the log file storage, 77 | // rather than the biggest number of log files. 78 | func WithMaxAge(days int64) LogOption { 79 | return func(c *config) { 80 | c.maxAge = days 81 | } 82 | } 83 | 84 | func WithFileName(filename string) LogOption { 85 | return func(c *config) { 86 | c.filename = filename 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /glogs/glogrus/daily_log.go: -------------------------------------------------------------------------------- 1 | package glogrus 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "runtime" 8 | "time" 9 | 10 | "github.com/aipave/go-utils/gstl/options" 11 | "github.com/aipave/go-utils/gstl/smap" 12 | "github.com/aipave/go-utils/gtime" 13 | nested "github.com/antonfisher/nested-logrus-formatter" 14 | "github.com/sirupsen/logrus" 15 | ) 16 | 17 | const dailyFormat = gtime.FormatDate 18 | 19 | var fileMap = smap.NewMap[string, int64](options.WithLocker()) 20 | 21 | // NewDailyLog 22 | // yNewDailyLog("svip.log", glogrus.WithMaxAge(30)) 23 | func NewDailyLog(filename string, opts ...LogOption) *logrus.Logger { 24 | var c = config{maxAge: 60} 25 | for _, fn := range opts { 26 | fn(&c) 27 | } 28 | 29 | l := logrus.New() 30 | 31 | l.SetReportCaller(true) 32 | l.SetFormatter(&nested.Formatter{ 33 | TimestampFormat: gtime.FormatDefault, 34 | ShowFullLevel: true, 35 | CallerFirst: true, 36 | CustomCallerFormatter: func(frame *runtime.Frame) string { 37 | return fmt.Sprintf(" [%v:%v %v]", filepath.Base(frame.File), frame.Line, filepath.Base(frame.Function)) 38 | }, 39 | }) 40 | 41 | fileMap.Set(filename, c.maxAge) 42 | 43 | go clean() 44 | 45 | l.SetOutput(&dailyWriter{filename: filename, date: time.Now().Format(dailyFormat)}) 46 | 47 | return l 48 | } 49 | 50 | func clean() { 51 | ticker := time.NewTicker(time.Hour) 52 | 53 | for range ticker.C { 54 | fileMap.Range(func(filename string, maxAge int64) bool { 55 | for { 56 | date := time.Now().Add(-time.Duration(maxAge+1) * 86400 * time.Second).Format(gtime.FormatDate) // 2022-11-11 57 | targetFile := fmt.Sprintf("%v-%v", filename, date) // svip.log-2022-11-11 58 | 59 | _, err := os.Stat(targetFile) 60 | if os.IsNotExist(err) { 61 | break 62 | } 63 | 64 | if err != nil { 65 | fmt.Printf("[ylogs] state logfile:%v err:%v\n", targetFile, maxAge) 66 | } 67 | 68 | err = os.Remove(targetFile) 69 | fmt.Printf("[ylogs] remove logfile:%v, maxage:%v, err:%v\n", targetFile, maxAge, err) 70 | 71 | maxAge++ 72 | } 73 | 74 | return true 75 | }) 76 | } 77 | } 78 | 79 | type dailyWriter struct { 80 | filename string 81 | date string 82 | writer *os.File 83 | } 84 | 85 | func (dw *dailyWriter) openFile() (err error) { 86 | dw.writer, err = os.OpenFile(fmt.Sprintf("%v-%v", dw.filename, dw.date), os.O_CREATE|os.O_RDWR|os.O_APPEND, os.ModePerm) 87 | return 88 | } 89 | 90 | func (dw *dailyWriter) closeFile() (err error) { 91 | if dw.writer == nil { 92 | return 93 | } 94 | 95 | return dw.writer.Close() 96 | } 97 | 98 | func (dw *dailyWriter) Write(b []byte) (n int, err error) { 99 | if dw.writer == nil { 100 | err = dw.openFile() 101 | if err != nil { 102 | return 103 | } 104 | } 105 | 106 | if date := time.Now().Format(dailyFormat); date != dw.date { 107 | err = dw.closeFile() 108 | if err != nil { 109 | return 110 | } 111 | 112 | dw.date = date 113 | err = dw.openFile() 114 | if err != nil { 115 | return 116 | } 117 | } 118 | 119 | return dw.writer.Write(b) 120 | } 121 | -------------------------------------------------------------------------------- /glogs/glogrus/logrus.go: -------------------------------------------------------------------------------- 1 | package glogrus 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "path/filepath" 7 | "runtime" 8 | "sync" 9 | "time" 10 | 11 | "github.com/aipave/go-utils/gtime" 12 | "github.com/natefinch/lumberjack" 13 | 14 | "github.com/aipave/go-utils/ginfos" 15 | "github.com/aipave/go-utils/glogs/glogrotate" 16 | "github.com/aipave/go-utils/glogs/gpanic" 17 | 18 | nested "github.com/antonfisher/nested-logrus-formatter" 19 | 20 | "github.com/sirupsen/logrus" 21 | ) 22 | 23 | type timeCost struct{} 24 | 25 | func fullPathCallerFormatter(frame *runtime.Frame) string { 26 | return fmt.Sprintf(" [%v:%v %v]", frame.File, frame.Line, filepath.Base(frame.Function)) 27 | } 28 | 29 | func shortPathCallerFormatter(frame *runtime.Frame) string { 30 | return fmt.Sprintf(" [%v:%v %v]", filepath.Base(frame.File), frame.Line, filepath.Base(frame.Function)) 31 | } 32 | 33 | var once sync.Once 34 | 35 | var defaultFormatter = &nested.Formatter{ 36 | TimestampFormat: gtime.FormatDefaultMill, 37 | ShowFullLevel: true, 38 | CallerFirst: true, 39 | CustomCallerFormatter: fullPathCallerFormatter, 40 | } 41 | 42 | func init() { 43 | Init() 44 | } 45 | 46 | // Init 初始化log rus 47 | func Init(opts ...LogOption) { 48 | var cfg config 49 | for _, fn := range opts { 50 | fn(&cfg) 51 | } 52 | 53 | once.Do(func() { 54 | gpanic.PAlertMgr.AlertUrl = cfg.alertUrl 55 | gpanic.PAlertMgr.IgnoreIpPrefix = cfg.ignoreIpPrefix 56 | gpanic.PAlertMgr.IgnoreIpSet = cfg.ignoreIpSet 57 | 58 | gpanic.PAlertMgr.Redirect("panic.log") // 重定向panic日志 59 | 60 | logrus.SetOutput(glogrotate.NewWriter(&lumberjack.Logger{ 61 | Filename: "log/" + ginfos.Runtime.Exec() + ".log", 62 | MaxSize: 256, // 256M 63 | MaxAge: 30, // 30d 64 | MaxBackups: 300, // max 300 files 65 | LocalTime: true, 66 | Compress: true, 67 | })) 68 | 69 | logrus.AddHook(NewCtxHook()) 70 | logrus.SetLevel(logrus.DebugLevel) 71 | logrus.SetReportCaller(true) 72 | logrus.SetFormatter(defaultFormatter) 73 | }) 74 | 75 | if len(cfg.filename) > 0 { 76 | logrus.SetOutput(glogrotate.NewWriter(&lumberjack.Logger{ 77 | Filename: cfg.filename, 78 | MaxSize: 256, // 256M 79 | MaxAge: 30, // 30day 80 | MaxBackups: 300, // max 300files 81 | LocalTime: true, 82 | Compress: true, 83 | })) 84 | } 85 | 86 | if cfg.lumLogger != nil { 87 | logrus.SetOutput(glogrotate.NewWriter(cfg.lumLogger)) 88 | } 89 | logrus.Infof("Init log config is %v", cfg) 90 | } 91 | 92 | func WithFields(fields logrus.Fields) *logrus.Entry { 93 | ctx := context.WithValue(context.Background(), timeCost{}, time.Now()) 94 | return logrus.WithContext(ctx).WithFields(fields) 95 | } 96 | 97 | func WithCtx(ctx context.Context) *logrus.Entry { 98 | ctx = context.WithValue(ctx, timeCost{}, time.Now()) 99 | return logrus.WithContext(ctx) 100 | } 101 | 102 | // MustSetLevel set log level 103 | func MustSetLevel(level string) { 104 | logLevel, err := logrus.ParseLevel(level) 105 | if err != nil { 106 | panic(err) 107 | } 108 | logrus.SetLevel(logLevel) 109 | } 110 | 111 | func SetShortLogPath() { 112 | defaultFormatter.CustomCallerFormatter = shortPathCallerFormatter 113 | logrus.SetFormatter(defaultFormatter) 114 | } 115 | 116 | type ctxHook struct{} 117 | 118 | func NewCtxHook() logrus.Hook { 119 | return ctxHook{} 120 | } 121 | 122 | func (c ctxHook) Levels() []logrus.Level { 123 | return []logrus.Level{ 124 | logrus.PanicLevel, 125 | logrus.FatalLevel, 126 | logrus.ErrorLevel, 127 | logrus.WarnLevel, 128 | logrus.InfoLevel, 129 | logrus.DebugLevel, 130 | logrus.TraceLevel, 131 | } 132 | } 133 | 134 | func (c ctxHook) Fire(entry *logrus.Entry) error { 135 | if entry.Context == nil { 136 | return nil 137 | } 138 | 139 | if uid := entry.Context.Value("uid"); uid != nil { 140 | entry.Data["uid"] = uid 141 | } 142 | 143 | if cost := entry.Context.Value(timeCost{}); cost != nil { 144 | begin, ok := cost.(time.Time) 145 | if ok { 146 | entry.Data["cost"] = time.Since(begin) 147 | } 148 | } 149 | 150 | return nil 151 | } 152 | -------------------------------------------------------------------------------- /glogs/glogrus/new_logrus.go: -------------------------------------------------------------------------------- 1 | package glogrus 2 | 3 | import ( 4 | "github.com/aipave/go-utils/glogs/glogrotate" 5 | "github.com/natefinch/lumberjack" 6 | "github.com/sirupsen/logrus" 7 | ) 8 | 9 | // NewLogger 10 | func NewLogger(opts ...LogOption) (l *logrus.Logger) { 11 | var c config 12 | for _, fn := range opts { 13 | fn(&c) 14 | } 15 | 16 | l = logrus.New() 17 | l.AddHook(NewCtxHook()) 18 | l.SetLevel(logrus.DebugLevel) 19 | l.SetReportCaller(true) 20 | l.SetFormatter(defaultFormatter) 21 | l.SetOutput(glogrotate.NewWriter(&lumberjack.Logger{ 22 | Filename: c.filename, 23 | MaxSize: 256, // 256M 24 | MaxAge: 30, // 25 | MaxBackups: 300, // 26 | LocalTime: true, 27 | Compress: true, 28 | })) 29 | 30 | for _, hook := range c.hooks { 31 | l.AddHook(hook) 32 | } 33 | if c.lumLogger != nil { 34 | l.SetOutput(glogrotate.NewWriter(c.lumLogger)) 35 | } 36 | 37 | if c.logLevel != nil { 38 | l.SetLevel(*c.logLevel) 39 | } 40 | 41 | if c.reportCaller != nil { 42 | l.SetReportCaller(*c.reportCaller) 43 | } 44 | 45 | if c.formatter != nil { 46 | l.SetFormatter(c.formatter) 47 | } 48 | 49 | return 50 | } 51 | -------------------------------------------------------------------------------- /glogs/gpanic/alert.go: -------------------------------------------------------------------------------- 1 | package gpanic 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "strings" 10 | 11 | "github.com/aipave/go-utils/ginfos" 12 | "github.com/sirupsen/logrus" 13 | ) 14 | 15 | // https://open.feishu.cn/document/ukTMukTMukTM/uEjNwUjLxYDM14SM2ATN 16 | 17 | type card struct { 18 | MsgType string `json:"msg_type"` 19 | Card struct { 20 | Header struct { 21 | Title struct { 22 | Tag string `json:"tag"` 23 | Content string `json:"content"` 24 | } `json:"title"` 25 | Template string `json:"template"` 26 | } `json:"header"` 27 | Elements []Element `json:"elements"` 28 | } `json:"card"` 29 | } 30 | 31 | type Element struct { 32 | Tag string `json:"tag"` 33 | Content string `json:"content"` 34 | } 35 | 36 | func buildAlert(stack string) (c card) { 37 | c.MsgType = "interactive" // card type 38 | 39 | // header 40 | c.Card.Header.Title.Tag = "plain_text" 41 | c.Card.Header.Title.Content = fmt.Sprintf("%v panic alert", ginfos.Runtime.Exec()) 42 | c.Card.Header.Template = "red" 43 | 44 | // body 45 | c.Card.Elements = append(c.Card.Elements, Element{ 46 | Tag: "markdown", 47 | Content: fmt.Sprintf("IP: %v\n%v", ginfos.Runtime.IP(), stack), 48 | }) 49 | 50 | // dividing line 51 | c.Card.Elements = append(c.Card.Elements, Element{ 52 | Tag: "hr", 53 | }) 54 | 55 | // @all 56 | c.Card.Elements = append(c.Card.Elements, Element{ 57 | Tag: "markdown", 58 | Content: "", 59 | }) 60 | 61 | return 62 | } 63 | 64 | func (a *AlertMgr) triggerAlert(card interface{}) { 65 | var currentIP = ginfos.Runtime.IP() 66 | // todo: Some ip do not alarm 67 | for _, ignorePrefix := range a.IgnoreIpPrefix { 68 | if strings.HasPrefix(currentIP, ignorePrefix) { 69 | return 70 | } 71 | } 72 | for _, ignoreIP := range a.IgnoreIpSet { 73 | if currentIP == ignoreIP { 74 | return 75 | } 76 | 77 | } 78 | 79 | content, _ := json.Marshal(card) 80 | resp, err := http.Post(a.AlertUrl, "application/json", bytes.NewReader(content)) 81 | if err != nil { 82 | logrus.Errorf("alert err:%v\n", err) 83 | return 84 | } 85 | defer resp.Body.Close() 86 | 87 | var data []byte 88 | data, err = io.ReadAll(resp.Body) ///< ioutil.ReadAll deprecated 89 | if err != nil { 90 | logrus.Errorf("read body err:%v\n", err) 91 | return 92 | } 93 | 94 | var codec struct { 95 | Code int64 `json:"code"` 96 | Msg string `json:"msg"` 97 | } 98 | 99 | err = json.Unmarshal(data, &codec) 100 | if err != nil || codec.Code != 0 { 101 | logrus.Errorf("alert failed with err:%v resp:%+v\n", err, string(data)) 102 | } else { 103 | logrus.Infof("alerted success resp:%v", string(data)) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /glogs/gpanic/panics.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | package gpanic 4 | 5 | import ( 6 | "context" 7 | "crypto/md5" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "os" 12 | "regexp" 13 | "strings" 14 | "sync" 15 | "syscall" 16 | "time" 17 | 18 | "github.com/aipave/go-utils/gexit" 19 | "github.com/aipave/go-utils/gtime" 20 | "github.com/fsnotify/fsnotify" 21 | "github.com/sirupsen/logrus" 22 | ) 23 | 24 | var once sync.Once 25 | var offset int64 26 | 27 | var PAlertMgr *AlertMgr = &AlertMgr{} 28 | 29 | type AlertMgr struct { 30 | AlertUrl string 31 | IgnoreIpSet []string 32 | IgnoreIpPrefix []string 33 | } 34 | 35 | // Redirect 36 | func (a *AlertMgr) Redirect(filename string) { 37 | once.Do(func() { 38 | f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) 39 | if err != nil { 40 | panic(err) 41 | } 42 | 43 | _ = syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd())) // Redirect to panic file 44 | 45 | // read last block 46 | var data []byte 47 | data, err = ioutil.ReadAll(f) 48 | if err == nil { 49 | splits := regexp.MustCompile("progress started at: .*-------").Split(string(data), -1) 50 | if len(splits) > 0 { 51 | a.filter([]byte(splits[len(splits)-1])) 52 | } 53 | } 54 | 55 | offset = int64(len(data)) 56 | 57 | // set current block begin 58 | _, _ = fmt.Fprintf(f, "progress started at: ---------%v-----------\n", time.Now().Format(gtime.FormatDefault)) 59 | 60 | go a.watch(filename, f) 61 | }) 62 | } 63 | 64 | func (a *AlertMgr) watch(filename string, f *os.File) { 65 | watcher, err := fsnotify.NewWatcher() 66 | if err != nil { 67 | logrus.Errorf("new wacher err:%v", err) 68 | return 69 | } 70 | 71 | err = watcher.Add(filename) 72 | if err != nil { 73 | logrus.Errorf("watch file:%v err:%v", filename, err) 74 | return 75 | } 76 | 77 | ctx, cancel := context.WithCancel(context.Background()) 78 | gexit.Close(func() { 79 | cancel() 80 | }) 81 | 82 | for { 83 | select { 84 | case event := <-watcher.Events: 85 | if event.Op == fsnotify.Write { 86 | _, _ = f.Seek(offset, io.SeekStart) 87 | data, _ := ioutil.ReadAll(f) 88 | offset += int64(len(data)) 89 | 90 | a.filter(data) 91 | } 92 | 93 | case err = <-watcher.Errors: 94 | logrus.Errorf("watch file:%v err:%v", filename, err) 95 | return 96 | 97 | case <-ctx.Done(): 98 | _ = watcher.Remove(filename) 99 | _ = watcher.Close() 100 | return 101 | } 102 | } 103 | } 104 | 105 | var silentMap = make(map[string]bool) // silence policy 106 | 107 | func (a *AlertMgr) filter(buf []byte) { 108 | var matched bool 109 | var count int 110 | var stack []string 111 | for _, line := range strings.Split(string(buf), "\n") { 112 | if !strings.HasPrefix(line, "panic:") && !matched { 113 | continue 114 | } 115 | 116 | if strings.HasPrefix(line, "panic:") { 117 | matched = true 118 | } 119 | 120 | if count >= 10 { 121 | break 122 | } 123 | 124 | stack = append(stack, line) 125 | } 126 | 127 | if len(stack) > 0 { 128 | alertMsg := strings.Join(stack, "\n") 129 | silentKey := fmt.Sprintf("%v:%v", time.Now().Minute(), localMd5(alertMsg)) // 重复的内容静默一分钟 130 | if silentMap[silentKey] { 131 | return 132 | } 133 | 134 | silentMap[silentKey] = true 135 | a.triggerAlert(buildAlert(alertMsg)) 136 | } 137 | } 138 | 139 | func localMd5(content string) (md string) { 140 | h := md5.New() 141 | _, _ = io.WriteString(h, content) 142 | md = fmt.Sprintf("%x", h.Sum(nil)) 143 | return 144 | } 145 | -------------------------------------------------------------------------------- /glogs/gpanic/panics_windows.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | 3 | package gpanic 4 | 5 | // Redirect 6 | func Redirect(filename string) { 7 | 8 | } 9 | -------------------------------------------------------------------------------- /glogs/grecover/recover.go: -------------------------------------------------------------------------------- 1 | package grecover 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "runtime" 7 | "time" 8 | 9 | "github.com/aipave/go-utils/gtime" 10 | ) 11 | 12 | // Recover catch panic error and trigger alert 13 | func Recover(afterRecover ...func()) { 14 | if err := recover(); err != nil { 15 | _, _ = fmt.Fprintf(os.Stderr, "panic: %v\n\n", err) 16 | buf := make([]byte, 1024) 17 | for { 18 | n := runtime.Stack(buf, false) 19 | if n < len(buf) { 20 | buf = buf[:n] 21 | _, _ = os.Stderr.Write(buf) 22 | break 23 | } 24 | buf = make([]byte, 2*len(buf)) 25 | } 26 | 27 | time.Sleep(500 * time.Millisecond) 28 | _, _ = fmt.Fprintf(os.Stderr, "progress started at: ---------%v-----------\n", time.Now().Format(gtime.FormatDefault)) 29 | 30 | // call after recovered 31 | for _, fn := range afterRecover { 32 | fn() 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/aipave/go-utils 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/RussellLuo/timingwheel v0.0.0-20220218152713-54845bda3108 7 | github.com/Shopify/sarama v1.29.1 8 | github.com/antonfisher/nested-logrus-formatter v1.3.1 9 | github.com/bwmarrin/snowflake v0.3.0 10 | github.com/cespare/xxhash v1.1.0 11 | github.com/frankban/quicktest v1.14.4 12 | github.com/fsnotify/fsnotify v1.6.0 13 | github.com/globocom/go-redis-prometheus v0.4.0 14 | github.com/go-redis/redis/v8 v8.3.3 15 | github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible 16 | github.com/natefinch/lumberjack v2.0.0+incompatible 17 | github.com/robfig/cron v1.2.0 18 | github.com/robfig/cron/v3 v3.0.1 19 | github.com/sirupsen/logrus v1.9.0 20 | go.mongodb.org/mongo-driver v1.11.2 21 | gopkg.in/yaml.v3 v3.0.1 22 | gorm.io/driver/mysql v1.4.7 23 | gorm.io/gorm v1.24.6 24 | ) 25 | 26 | require ( 27 | github.com/BurntSushi/toml v1.2.1 // indirect 28 | github.com/beorn7/perks v1.0.1 // indirect 29 | github.com/cespare/xxhash/v2 v2.1.1 // indirect 30 | github.com/davecgh/go-spew v1.1.1 // indirect 31 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 32 | github.com/eapache/go-resiliency v1.2.0 // indirect 33 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect 34 | github.com/eapache/queue v1.1.0 // indirect 35 | github.com/go-sql-driver/mysql v1.7.0 // indirect 36 | github.com/golang/protobuf v1.4.3 // indirect 37 | github.com/golang/snappy v0.0.3 // indirect 38 | github.com/google/go-cmp v0.5.9 // indirect 39 | github.com/hashicorp/go-uuid v1.0.2 // indirect 40 | github.com/jcmturner/aescts/v2 v2.0.0 // indirect 41 | github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect 42 | github.com/jcmturner/gofork v1.0.0 // indirect 43 | github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect 44 | github.com/jcmturner/rpc/v2 v2.0.3 // indirect 45 | github.com/jinzhu/inflection v1.0.0 // indirect 46 | github.com/jinzhu/now v1.1.5 // indirect 47 | github.com/klauspost/compress v1.13.6 // indirect 48 | github.com/kr/pretty v0.3.1 // indirect 49 | github.com/kr/text v0.2.0 // indirect 50 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 51 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect 52 | github.com/pierrec/lz4 v2.6.0+incompatible // indirect 53 | github.com/pkg/errors v0.9.1 // indirect 54 | github.com/prometheus/client_golang v1.8.0 // indirect 55 | github.com/prometheus/client_model v0.2.0 // indirect 56 | github.com/prometheus/common v0.14.0 // indirect 57 | github.com/prometheus/procfs v0.2.0 // indirect 58 | github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect 59 | github.com/rogpeppe/go-internal v1.9.0 // indirect 60 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 61 | github.com/xdg-go/scram v1.1.1 // indirect 62 | github.com/xdg-go/stringprep v1.0.3 // indirect 63 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect 64 | go.opentelemetry.io/otel v0.13.0 // indirect 65 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect 66 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect 67 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect 68 | golang.org/x/sys v0.0.0-20220908164124-27713097b956 // indirect 69 | golang.org/x/text v0.3.7 // indirect 70 | google.golang.org/protobuf v1.23.0 // indirect 71 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect 72 | gopkg.in/yaml.v2 v2.4.0 // indirect 73 | ) 74 | -------------------------------------------------------------------------------- /gstl/README.md: -------------------------------------------------------------------------------- 1 | # container 2 | 3 | 4 | 5 | ## Getting started 6 | 7 | To make it easy for you to get started with GitLab, here's a list of recommended next steps. 8 | 9 | Already a pro? Just edit this README.md and make it your own. Want to make it easy? [Use the template at the bottom](#editing-this-readme)! 10 | 11 | ## Add your files 12 | 13 | - [ ] [Create](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#create-a-file) or [upload](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#upload-a-file) files 14 | - [ ] [Add files using the command line](https://docs.gitlab.com/ee/gitlab-basics/add-file.html#add-a-file-using-the-command-line) or push an existing Git repository with the following command: 15 | 16 | ``` 17 | cd existing_repo 18 | git branch -M main 19 | git push -uf origin main 20 | ``` 21 | 22 | ## Integrate with your tools 23 | 24 | 25 | ## Collaborate with your team 26 | 27 | - [ ] [Invite team members and collaborators](https://docs.gitlab.com/ee/user/project/members/) 28 | - [ ] [Create a new merge request](https://docs.gitlab.com/ee/user/project/merge_requests/creating_merge_requests.html) 29 | - [ ] [Automatically close issues from merge requests](https://docs.gitlab.com/ee/user/project/issues/managing_issues.html#closing-issues-automatically) 30 | - [ ] [Enable merge request approvals](https://docs.gitlab.com/ee/user/project/merge_requests/approvals/) 31 | - [ ] [Automatically merge when pipeline succeeds](https://docs.gitlab.com/ee/user/project/merge_requests/merge_when_pipeline_succeeds.html) 32 | 33 | ## Test and Deploy 34 | 35 | Use the built-in continuous integration in GitLab. 36 | 37 | - [ ] [Get started with GitLab CI/CD](https://docs.gitlab.com/ee/ci/quick_start/index.html) 38 | - [ ] [Analyze your code for known vulnerabilities with Static Application Security Testing(SAST)](https://docs.gitlab.com/ee/user/application_security/sast/) 39 | - [ ] [Deploy to Kubernetes, Amazon EC2, or Amazon ECS using Auto Deploy](https://docs.gitlab.com/ee/topics/autodevops/requirements.html) 40 | - [ ] [Use pull-based deployments for improved Kubernetes management](https://docs.gitlab.com/ee/user/clusters/agent/) 41 | - [ ] [Set up protected environments](https://docs.gitlab.com/ee/ci/environments/protected_environments.html) 42 | 43 | *** 44 | 45 | # Editing this README 46 | 47 | When you're ready to make this README your own, just edit this file and use the handy template below (or feel free to structure it however you want - this is just a starting point!). Thank you to [makeareadme.com](https://www.makeareadme.com/) for this template. 48 | 49 | ## Suggestions for a good README 50 | Every project is different, so consider which of these sections apply to yours. The sections used in the template are suggestions for most open source projects. Also keep in mind that while a README can be too long and detailed, too long is better than too short. If you think your README is too long, consider utilizing another form of documentation rather than cutting out information. 51 | 52 | ## Name 53 | Choose a self-explaining name for your project. 54 | 55 | ## Description 56 | Let people know what your project can do specifically. Provide context and add a link to any reference visitors might be unfamiliar with. A list of Features or a Background subsection can also be added here. If there are alternatives to your project, this is a good place to list differentiating factors. 57 | 58 | ## Badges 59 | On some READMEs, you may see small images that convey metadata, such as whether or not all the tests are passing for the project. You can use Shields to add some to your README. Many services also have instructions for adding a badge. 60 | 61 | ## Visuals 62 | Depending on what you are making, it can be a good idea to include screenshots or even a video (you'll frequently see GIFs rather than actual videos). Tools like ttygif can help, but check out Asciinema for a more sophisticated method. 63 | 64 | ## Installation 65 | Within a particular ecosystem, there may be a common way of installing things, such as using Yarn, NuGet, or Homebrew. However, consider the possibility that whoever is reading your README is a novice and would like more guidance. Listing specific steps helps remove ambiguity and gets people to using your project as quickly as possible. If it only runs in a specific context like a particular programming language version or operating system or has dependencies that have to be installed manually, also add a Requirements subsection. 66 | 67 | ## Usage 68 | Use examples liberally, and show the expected output if you can. It's helpful to have inline the smallest example of usage that you can demonstrate, while providing links to more sophisticated examples if they are too long to reasonably include in the README. 69 | 70 | ## Support 71 | Tell people where they can go to for help. It can be any combination of an issue tracker, a chat room, an email address, etc. 72 | 73 | ## Roadmap 74 | If you have ideas for releases in the future, it is a good idea to list them in the README. 75 | 76 | ## Contributing 77 | State if you are open to contributions and what your requirements are for accepting them. 78 | 79 | For people who want to make changes to your project, it's helpful to have some documentation on how to get started. Perhaps there is a script that they should run or some environment variables that they need to set. Make these steps explicit. These instructions could also be useful to your future self. 80 | 81 | You can also document commands to lint the code or run tests. These steps help to ensure high code quality and reduce the likelihood that the changes inadvertently break something. Having instructions for running tests is especially helpful if it requires external setup, such as starting a Selenium server for testing in a browser. 82 | 83 | ## Authors and acknowledgment 84 | Show your appreciation to those who have contributed to the project. 85 | 86 | ## License 87 | For open source projects, say how it is licensed. 88 | 89 | ## Project status 90 | If you have run out of energy or time for your project, put a note at the top of the README saying that development has slowed down or stopped completely. Someone may choose to fork your project or volunteer to step in as a maintainer or owner, allowing your project to keep going. You can also make an explicit request for maintainers. 91 | -------------------------------------------------------------------------------- /gstl/compare/less.go: -------------------------------------------------------------------------------- 1 | package compare 2 | 3 | type Ordered interface { 4 | Integer | Float | ~string 5 | } 6 | 7 | type Integer interface { 8 | Signed | Unsigned 9 | } 10 | 11 | type Signed interface { 12 | ~int | ~int8 | ~int16 | ~int32 | ~int64 13 | } 14 | 15 | type Unsigned interface { 16 | ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr 17 | } 18 | 19 | type Float interface { 20 | ~float32 | ~float64 21 | } 22 | 23 | type Complex interface { 24 | ~complex64 | ~complex128 25 | } 26 | 27 | type Less[T any] func(a, b T) int 28 | 29 | func OrderedLess[T Ordered](a, b T) int { 30 | if a < b { 31 | return -1 32 | } 33 | 34 | if a > b { 35 | return 1 36 | } 37 | 38 | return 0 39 | } 40 | 41 | func TypeCheck(x any) bool { 42 | switch x.(type) { 43 | case int, int8, int16, int32, int64: 44 | return true 45 | case uint, uint8, uint16, uint32, uint64: 46 | return true 47 | case float32, float64: 48 | return true 49 | } 50 | 51 | return false 52 | } 53 | -------------------------------------------------------------------------------- /gstl/fns/README.md: -------------------------------------------------------------------------------- 1 | ## fns 2 | 3 | generic template function 4 | 5 | ### Requirements 6 | Go version >= 1.18.1 7 | 8 | ### Usage 9 | 10 | 1. If Ternary operator 11 | ``` 12 | func main() { 13 | var a = fns.If(true, 1, 2) 14 | var b = fns.If(false, 1, 2) 15 | 16 | fmt.Println("a = ", a, ", b = ", b) // a = 1 , b = 2 17 | } 18 | ``` 19 | -------------------------------------------------------------------------------- /gstl/fns/fns.go: -------------------------------------------------------------------------------- 1 | package fns 2 | 3 | // If Ternary operator 4 | func If[T any](isA bool, a T, b T) T { 5 | if isA { 6 | return a 7 | } 8 | 9 | return b 10 | } 11 | 12 | type Pair[K any, V any] struct { 13 | Key K `json:"key"` 14 | Value V `json:"value"` 15 | } 16 | -------------------------------------------------------------------------------- /gstl/fns/fns_test.go: -------------------------------------------------------------------------------- 1 | package fns 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestIf(t *testing.T) { 9 | fmt.Println(If(true, 1, 2)) 10 | fmt.Println(If(false, 1, 2)) 11 | } 12 | -------------------------------------------------------------------------------- /gstl/heap/decl.go: -------------------------------------------------------------------------------- 1 | package heap 2 | 3 | type Heap[T any] interface { 4 | Push(x ...T) 5 | Pop() T 6 | Size() int 7 | } 8 | -------------------------------------------------------------------------------- /gstl/heap/heap.go: -------------------------------------------------------------------------------- 1 | package heap 2 | 3 | import ( 4 | "github.com/aipave/go-utils/gstl/compare" 5 | ) 6 | 7 | type heap[T any] struct { 8 | arr []T 9 | keyCmp compare.Less[T] 10 | } 11 | 12 | func New[T compare.Ordered]() Heap[T] { 13 | return &heap[T]{keyCmp: compare.OrderedLess[T]} 14 | } 15 | 16 | func NewExt[T any](cmp compare.Less[T]) Heap[T] { 17 | return &heap[T]{keyCmp: cmp} 18 | } 19 | 20 | func (h *heap[T]) Push(x ...T) { 21 | h.arr = append(h.arr, x...) 22 | 23 | for i := h.Size() - len(x) - 1; i < h.Size(); i++ { 24 | h.up(i) 25 | } 26 | } 27 | 28 | func (h *heap[T]) Pop() (v T) { 29 | n := h.Size() - 1 30 | v = h.arr[0] 31 | h.swap(0, n) 32 | h.down(0, n) 33 | h.arr = h.arr[:n] 34 | return 35 | } 36 | 37 | func (h *heap[T]) Remove(i int) any { 38 | n := h.Size() - 1 39 | if n != i { 40 | h.swap(i, n) 41 | if !h.down(i, n) { 42 | h.up(i) 43 | } 44 | } 45 | return h.Pop() 46 | } 47 | 48 | func (h *heap[T]) Fix(i int) { 49 | if !h.down(i, h.Size()) { 50 | h.up(i) 51 | } 52 | } 53 | 54 | func (h *heap[T]) up(j int) { 55 | for { 56 | i := (j - 1) / 2 // parent 57 | if i == j || h.keyCmp(h.arr[j], h.arr[i]) >= 0 { 58 | break 59 | } 60 | h.swap(i, j) 61 | j = i 62 | } 63 | } 64 | 65 | func (h *heap[T]) down(i0, n int) bool { 66 | i := i0 67 | for { 68 | j1 := 2*i + 1 69 | if j1 >= n || j1 < 0 { // j1 < 0 after int overflow 70 | break 71 | } 72 | j := j1 // left child 73 | if j2 := j1 + 1; j2 < n && h.keyCmp(h.arr[j2], h.arr[j1]) < 0 { 74 | j = j2 // = 2*i + 2 // right child 75 | } 76 | if h.keyCmp(h.arr[j], h.arr[i]) >= 0 { 77 | break 78 | } 79 | h.swap(i, j) 80 | i = j 81 | } 82 | return i > i0 83 | } 84 | 85 | func (h *heap[T]) swap(i, j int) { 86 | h.arr[i], h.arr[j] = h.arr[j], h.arr[i] 87 | } 88 | 89 | func (h *heap[T]) Size() int { 90 | return len(h.arr) 91 | } 92 | -------------------------------------------------------------------------------- /gstl/heap/heap_test.go: -------------------------------------------------------------------------------- 1 | package heap 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestNew(t *testing.T) { 9 | var hp = New[int]() 10 | var arr = []int{3, 1, 5, 8, 23, 22, 12, 89, 100, 44, 2, 7, 9, 13} 11 | hp.Push(arr...) 12 | 13 | for hp.Size() > 0 { 14 | fmt.Println(hp.Pop()) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /gstl/options/options.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | import "sync" 4 | 5 | type RWLock interface { 6 | RLock() 7 | RUnlock() 8 | sync.Locker 9 | } 10 | 11 | type options struct { 12 | Mux RWLock 13 | } 14 | 15 | func New() options { 16 | return options{Mux: &fakeLock{}} 17 | } 18 | 19 | type Option func(opt *options) 20 | 21 | func WithLocker() Option { 22 | return func(opt *options) { 23 | opt.Mux = &sync.RWMutex{} 24 | } 25 | } 26 | 27 | type fakeLock struct { 28 | } 29 | 30 | func (f fakeLock) RLock() { 31 | 32 | } 33 | 34 | func (f fakeLock) RUnlock() { 35 | } 36 | 37 | func (f fakeLock) Lock() { 38 | } 39 | 40 | func (f fakeLock) Unlock() { 41 | } 42 | -------------------------------------------------------------------------------- /gstl/rbtree/README.md: -------------------------------------------------------------------------------- 1 | ## Vector 2 | 3 | ### Requirements 4 | Go version >= 1.18.1 5 | 6 | ### Usage 7 | 8 | ```go 9 | ``` 10 | 11 | import package 12 | ```go 13 | ``` 14 | 15 | [Reference](https://github.com/liyue201/gostl/tree/master/ds/rbtree) -------------------------------------------------------------------------------- /gstl/rbtree/node.go: -------------------------------------------------------------------------------- 1 | package rbtree 2 | 3 | type Color bool 4 | 5 | const ( 6 | Red = false 7 | Black = true 8 | ) 9 | 10 | type Node[K any, V any] struct { 11 | parent *Node[K, V] 12 | left *Node[K, V] 13 | right *Node[K, V] 14 | color Color 15 | key K 16 | value V 17 | } 18 | 19 | func (n *Node[K, V]) Key() K { 20 | return n.key 21 | } 22 | 23 | func (n *Node[K, V]) Value() V { 24 | return n.value 25 | } 26 | 27 | func (n *Node[K, V]) SetValue(val V) { 28 | n.value = val 29 | } 30 | 31 | func (n *Node[K, V]) Next() *Node[K, V] { 32 | return next(n) 33 | } 34 | 35 | func next[K any, V any](ptr *Node[K, V]) *Node[K, V] { 36 | if ptr.right != nil { 37 | return leftNode(ptr.right) 38 | } 39 | 40 | parent := ptr.parent 41 | for parent != nil && ptr == parent.right { 42 | ptr, parent = parent, parent.parent 43 | } 44 | return parent 45 | } 46 | 47 | func (n *Node[K, V]) Prev() *Node[K, V] { 48 | return prev(n) 49 | } 50 | 51 | func prev[K any, V any](ptr *Node[K, V]) *Node[K, V] { 52 | if ptr.left != nil { 53 | return rightNode(ptr.left) 54 | } 55 | 56 | parent := ptr.parent 57 | for parent != nil && parent.left == ptr { 58 | ptr, parent = parent, parent.parent 59 | } 60 | return parent 61 | } 62 | 63 | // 获取一个节点的最左节点 64 | func leftNode[K any, V any](x *Node[K, V]) *Node[K, V] { 65 | for x.left != nil { 66 | x = x.left 67 | } 68 | return x 69 | } 70 | 71 | // 获取一个节点的最右节点 72 | func rightNode[K any, V any](x *Node[K, V]) *Node[K, V] { 73 | for x.right != nil { 74 | x = x.right 75 | } 76 | return x 77 | } 78 | -------------------------------------------------------------------------------- /gstl/rbtree/rbtree_test.go: -------------------------------------------------------------------------------- 1 | package rbtree 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/aipave/go-utils/gstl/compare" 8 | ) 9 | 10 | type n struct { 11 | K int 12 | V int 13 | } 14 | 15 | func TestNew(t *testing.T) { 16 | var tree = New[int, n](compare.OrderedLess[int]) 17 | for i := 1; i <= 100; i++ { 18 | tree.Insert(i%10, n{K: i % 10, V: i}) 19 | } 20 | 21 | for node := tree.Begin(); node != nil; node = node.Next() { 22 | fmt.Printf("%v ", node.Value()) 23 | } 24 | 25 | fmt.Println() 26 | 27 | for node := tree.RBegin(); node != nil; node = node.Prev() { 28 | fmt.Printf("%v ", node.value) 29 | } 30 | 31 | fmt.Println() 32 | 33 | fmt.Println(tree.FindNode(1).Value(), tree.FindFirstNode(1).Value(), tree.FindLastNode(1).Value()) 34 | fmt.Println(tree.FindNode(2).Value(), tree.FindFirstNode(2).Value(), tree.FindLastNode(2).Value()) 35 | } 36 | -------------------------------------------------------------------------------- /gstl/set/README.md: -------------------------------------------------------------------------------- 1 | ## Set and MultiSet 2 | 3 | ### Requirements 4 | Go version >= 1.18.1 5 | 6 | ### Usage 7 | 8 | ```go 9 | ``` 10 | 11 | import package 12 | ```go 13 | ``` 14 | 15 | 1. Create 16 | > NewSet: only support int,int8,int16,int32,int64,uint,uint8,uint16,uint32,uint64,float32,float64,string类型 17 | > NewExtSet: need Less func 18 | > NewMultiSet: same to NewSet 19 | > NewMultiExtSet: same to NewMultiSet 20 | ```go 21 | type User struct { 22 | UID int64 23 | Age int64 24 | Avatars []string 25 | } 26 | 27 | func less(a, b User) int { 28 | return a.uid - b.uid 29 | } 30 | 31 | func handler() { 32 | // 1. Create 33 | var s = set.NewSet[int]() 34 | 35 | // 2. Create with lock 36 | var s2 = set.NewSet[int](options.WithLock()) 37 | 38 | // 3. Create Ext Set 39 | var s3 = set.NewExtSet[User](less) 40 | 41 | // 4. Create Multi Set 42 | var s4 = set.NewMultiSet[int]() 43 | 44 | // 5. Create Multi Set With Lock 45 | var s5 = set.NewMultiSet[int](options.WithLock()) 46 | } 47 | ``` 48 | 49 | 2. Insert and Erase 50 | ```go 51 | func handler() { 52 | var list = []int{1, 3, 2, 4, 5, 7, 8, 9} 53 | var s = set.NewSet[int]() 54 | 55 | // Insert 56 | s.Insert(list...) // 1, 2, 3, 4, 5, 6, 7, 8, 9 57 | 58 | // Erase 59 | s.Erase(5) // 1, 2, 3, 4, 6, 7, 8, 9 60 | } 61 | ``` 62 | 63 | 3. Slices, Size and Clear 64 | ```go 65 | func handler() { 66 | var list = []int{1, 3, 2, 4, 5, 7, 8, 9} 67 | var s = set.NewSet[int]() 68 | 69 | // Insert 70 | s.Insert(list...) // 1, 2, 3, 4, 5, 6, 7, 8, 9 71 | 72 | fmt.Println("size", s.Size()) 73 | fmt.Println("slices", s.Slices()) // [1,2,3,4,5,6,7,8,9] 74 | s.Clear() 75 | fmt.Println("size", s.Size()) 76 | } 77 | ``` 78 | 79 | 4. Iterator 80 | ```go 81 | func handler() { 82 | var list = []int{1, 3, 2, 4, 5, 7, 8, 9} 83 | var s = set.NewSet[int]() 84 | 85 | // Insert 86 | s.Insert(list...) // 1, 2, 3, 4, 5, 6, 7, 8, 9 87 | 88 | // iterator from left to right 89 | it := s.Begin() 90 | for it.Next() { 91 | fmt.Println("%v ", it.Value()) 92 | } 93 | fmt.Println() 94 | 95 | // iterator from right to left 96 | it := s.RBegin() 97 | for it.Next() { 98 | fmt.Println("%v ", it.Value()) 99 | } 100 | fmt.Println() 101 | 102 | // iterator from find position to right 103 | it := s.BeginFrom(5) 104 | for it.Next() { 105 | fmt.Println("%v ", it.Value()) 106 | } 107 | } 108 | ``` 109 | 110 | 5. Intersect, Union and Diff 111 | ```go 112 | func handler() { 113 | var a, b = NewSet[int](), NewSet[int]() 114 | 115 | for i := 1; i < 20; i++ { 116 | a.Insert(i) 117 | } 118 | 119 | for i := 16; i < 36; i++ { 120 | b.Insert(i) 121 | } 122 | 123 | fmt.Println("intersect", a.Intersect(b).Slices()) 124 | fmt.Println("union", a.Union(b).Slices()) 125 | fmt.Println("diff", a.Diff(b).Slices()) 126 | } 127 | ``` 128 | -------------------------------------------------------------------------------- /gstl/set/decl.go: -------------------------------------------------------------------------------- 1 | package set 2 | 3 | type Iterator[T any] interface { 4 | Next() Iterator[T] 5 | Value() T 6 | } 7 | 8 | type Set[T any] interface { 9 | Size() int 10 | 11 | Slices() []T 12 | 13 | // Clear remove all elements from set 14 | Clear() 15 | 16 | // Insert add elements to set, return false if any node exist 17 | Insert(v ...T) bool 18 | 19 | // Erase delete element from set, return true if node exist 20 | Erase(v T) bool 21 | 22 | Contains(v T) bool 23 | 24 | Range(func(T) bool) 25 | 26 | BeginFrom(v T) Iterator[T] 27 | 28 | Begin() Iterator[T] 29 | 30 | RBeginFrom(v T) Iterator[T] 31 | 32 | RBegin() Iterator[T] 33 | 34 | Intersect(Set[T]) Set[T] 35 | 36 | Union(Set[T]) Set[T] 37 | 38 | Diff(Set[T]) Set[T] 39 | } 40 | 41 | type MultiSet[T any] interface { 42 | Set[T] 43 | } 44 | -------------------------------------------------------------------------------- /gstl/set/iterator.go: -------------------------------------------------------------------------------- 1 | package set 2 | 3 | import ( 4 | "github.com/aipave/go-utils/gstl/rbtree" 5 | ) 6 | 7 | type iterator[T any] struct { 8 | node *rbtree.Node[T, bool] 9 | } 10 | 11 | func newIterator[T any](node *rbtree.Node[T, bool]) Iterator[T] { 12 | if node == nil { 13 | return nil 14 | } 15 | 16 | return iterator[T]{node: node} 17 | } 18 | 19 | func (iter iterator[T]) Next() Iterator[T] { 20 | if iter.node == nil { 21 | return nil 22 | } 23 | 24 | nextNode := iter.node.Next() 25 | if nextNode == nil { 26 | return nil 27 | } 28 | 29 | return iterator[T]{node: nextNode} 30 | } 31 | 32 | func (iter iterator[T]) Value() (v T) { 33 | if iter.node != nil { 34 | return iter.node.Key() 35 | } 36 | 37 | return 38 | } 39 | 40 | type reverseIterator[T any] struct { 41 | node *rbtree.Node[T, bool] 42 | } 43 | 44 | func newReverseIterator[T any](node *rbtree.Node[T, bool]) Iterator[T] { 45 | if node == nil { 46 | return nil 47 | } 48 | 49 | return reverseIterator[T]{node: node} 50 | } 51 | 52 | func (iter reverseIterator[T]) Next() Iterator[T] { 53 | if iter.node == nil { 54 | return nil 55 | } 56 | 57 | nextNode := iter.node.Next() 58 | if nextNode == nil { 59 | return nil 60 | } 61 | 62 | return reverseIterator[T]{node: nextNode} 63 | } 64 | 65 | func (iter reverseIterator[T]) Value() (v T) { 66 | if iter.node != nil { 67 | return iter.node.Key() 68 | } 69 | 70 | return 71 | } 72 | -------------------------------------------------------------------------------- /gstl/set/multi_set.go: -------------------------------------------------------------------------------- 1 | package set 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/aipave/go-utils/gstl/compare" 8 | "github.com/aipave/go-utils/gstl/options" 9 | "github.com/aipave/go-utils/gstl/rbtree" 10 | ) 11 | 12 | type multiSet[T any] struct { 13 | tree *rbtree.RBTree[T, bool] 14 | mux options.RWLock 15 | keyCmp compare.Less[T] 16 | } 17 | 18 | // NewMultiSet 基本数据类型使用默认比较函数 19 | func NewMultiSet[T compare.Ordered](opts ...options.Option) MultiSet[T] { 20 | var option = options.New() 21 | for _, fn := range opts { 22 | fn(&option) 23 | } 24 | 25 | return &multiSet[T]{ 26 | tree: rbtree.New[T, bool](compare.OrderedLess[T]), 27 | mux: option.Mux, 28 | keyCmp: compare.OrderedLess[T], 29 | } 30 | } 31 | 32 | // NewMultiExtSet 非基本数据类型需要提供比较函数 33 | func NewMultiExtSet[T any](keyCmp compare.Less[T], opts ...options.Option) MultiSet[T] { 34 | var option = options.New() 35 | for _, fn := range opts { 36 | fn(&option) 37 | } 38 | 39 | return &multiSet[T]{ 40 | tree: rbtree.New[T, bool](keyCmp), 41 | mux: option.Mux, 42 | keyCmp: keyCmp, 43 | } 44 | } 45 | 46 | func (s *multiSet[T]) Size() int { 47 | return s.tree.Size() 48 | } 49 | 50 | func (s *multiSet[T]) Clear() { 51 | s.mux.Lock() 52 | defer s.mux.Unlock() 53 | 54 | s.tree.Clear() 55 | } 56 | 57 | func (s *multiSet[T]) Range(callback func(v T) bool) { 58 | for it := s.Begin(); it != nil; it = it.Next() { 59 | callback(it.Value()) 60 | } 61 | } 62 | 63 | func (s *multiSet[T]) Insert(list ...T) bool { 64 | s.mux.Lock() 65 | defer s.mux.Unlock() 66 | 67 | for _, v := range list { 68 | s.tree.Insert(v, true) 69 | } 70 | return true 71 | } 72 | 73 | func (s *multiSet[T]) Erase(v T) bool { 74 | s.mux.Lock() 75 | defer s.mux.Unlock() 76 | 77 | for { 78 | node := s.tree.FindNode(v) 79 | if node == nil { 80 | return true 81 | } 82 | s.tree.Delete(node) 83 | } 84 | } 85 | 86 | func (s *multiSet[T]) BeginFrom(v T) Iterator[T] { 87 | s.mux.RLock() 88 | defer s.mux.RUnlock() 89 | 90 | node := s.tree.FindFirstNode(v) 91 | return newIterator(node) 92 | } 93 | 94 | func (s *multiSet[T]) Begin() Iterator[T] { 95 | s.mux.RLock() 96 | defer s.mux.RUnlock() 97 | 98 | return newIterator(s.tree.Begin()) 99 | } 100 | 101 | func (s *multiSet[T]) RBegin() Iterator[T] { 102 | s.mux.RLock() 103 | defer s.mux.RUnlock() 104 | 105 | return newReverseIterator(s.tree.RBegin()) 106 | } 107 | 108 | func (s *multiSet[T]) RBeginFrom(v T) Iterator[T] { 109 | s.mux.RLock() 110 | defer s.mux.RUnlock() 111 | 112 | node := s.tree.FindFirstNode(v) 113 | return newReverseIterator(node) 114 | } 115 | 116 | func (s *multiSet[T]) Slices() (result []T) { 117 | for it := s.Begin(); it != nil; it = it.Next() { 118 | result = append(result, it.Value()) 119 | } 120 | 121 | return 122 | } 123 | 124 | func (s *multiSet[T]) Contains(v T) bool { 125 | s.mux.RLock() 126 | defer s.mux.RUnlock() 127 | 128 | node := s.tree.FindNode(v) 129 | return node != nil 130 | } 131 | 132 | func (s *multiSet[T]) MarshalJSON() ([]byte, error) { 133 | s.mux.RLock() 134 | 135 | var list []T 136 | for it := s.Begin(); it != nil; it = it.Next() { 137 | list = append(list, it.Value()) 138 | } 139 | 140 | s.mux.RUnlock() 141 | 142 | return json.Marshal(list) 143 | } 144 | 145 | func (s *multiSet[T]) UnmarshalJSON(b []byte) (err error) { 146 | var list []T 147 | if !json.Valid(b) { 148 | return fmt.Errorf("input is not a valid json string") 149 | } 150 | 151 | err = json.Unmarshal(b, &list) 152 | if err != nil { 153 | return fmt.Errorf("unmarshal input to array struct err:%v", err) 154 | } 155 | 156 | s.Insert(list...) 157 | 158 | return 159 | } 160 | 161 | // Intersect 交集 162 | func (s *multiSet[T]) Intersect(o Set[T]) Set[T] { 163 | var ans = NewExtSet(s.keyCmp) 164 | 165 | s.mux.RLock() 166 | 167 | for it := s.Begin(); it != nil; it = it.Next() { 168 | if o.Contains(it.Value()) { 169 | ans.Insert(it.Value()) 170 | } 171 | } 172 | 173 | s.mux.RUnlock() 174 | 175 | return ans 176 | } 177 | 178 | func (s *multiSet[T]) Union(o Set[T]) Set[T] { 179 | var ans = NewExtSet(s.keyCmp) 180 | 181 | s.mux.RLock() 182 | 183 | for it := s.Begin(); it != nil; it = it.Next() { 184 | ans.Insert(it.Value()) 185 | } 186 | 187 | s.mux.RUnlock() 188 | 189 | for it := o.Begin(); it != nil; it = it.Next() { 190 | ans.Insert(it.Value()) 191 | } 192 | 193 | return ans 194 | } 195 | 196 | func (s *multiSet[T]) Diff(o Set[T]) Set[T] { 197 | var ans = NewExtSet(s.keyCmp) 198 | 199 | s.mux.RLock() 200 | 201 | for it := s.Begin(); it != nil; it = it.Next() { 202 | if !o.Contains(it.Value()) { 203 | ans.Insert(it.Value()) 204 | } 205 | } 206 | 207 | s.mux.RUnlock() 208 | 209 | for it := o.Begin(); it != nil; it = it.Next() { 210 | if !s.Contains(it.Value()) { 211 | ans.Insert(it.Value()) 212 | } 213 | } 214 | 215 | return ans 216 | } 217 | -------------------------------------------------------------------------------- /gstl/set/set.go: -------------------------------------------------------------------------------- 1 | package set 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/aipave/go-utils/gstl/compare" 8 | "github.com/aipave/go-utils/gstl/options" 9 | "github.com/aipave/go-utils/gstl/rbtree" 10 | ) 11 | 12 | type set[T any] struct { 13 | tree *rbtree.RBTree[T, bool] 14 | mux options.RWLock 15 | keyCmp compare.Less[T] 16 | } 17 | 18 | // NewSet 基本数据类型使用默认比较函数 19 | func NewSet[T compare.Ordered](opts ...options.Option) Set[T] { 20 | var option = options.New() 21 | for _, fn := range opts { 22 | fn(&option) 23 | } 24 | 25 | return &set[T]{ 26 | tree: rbtree.New[T, bool](compare.OrderedLess[T]), 27 | mux: option.Mux, 28 | keyCmp: compare.OrderedLess[T], 29 | } 30 | } 31 | 32 | // NewExtSet 非基本数据类型需要提供比较函数 33 | func NewExtSet[T any](keyCmp compare.Less[T], opts ...options.Option) Set[T] { 34 | var option = options.New() 35 | for _, fn := range opts { 36 | fn(&option) 37 | } 38 | 39 | return &set[T]{ 40 | tree: rbtree.New[T, bool](keyCmp), 41 | mux: option.Mux, 42 | keyCmp: keyCmp, 43 | } 44 | } 45 | 46 | func (s *set[T]) Size() int { 47 | return s.tree.Size() 48 | } 49 | 50 | func (s *set[T]) Clear() { 51 | s.mux.Lock() 52 | defer s.mux.Unlock() 53 | 54 | s.tree.Clear() 55 | } 56 | 57 | func (s *set[T]) Insert(list ...T) (allSuccess bool) { 58 | s.mux.Lock() 59 | defer s.mux.Unlock() 60 | 61 | for _, v := range list { 62 | node := s.tree.FindNode(v) 63 | if node == nil { 64 | s.tree.Insert(v, true) 65 | } else { 66 | allSuccess = false 67 | } 68 | } 69 | 70 | return 71 | } 72 | 73 | func (s *set[T]) Erase(v T) bool { 74 | s.mux.Lock() 75 | defer s.mux.Unlock() 76 | 77 | node := s.tree.FindNode(v) 78 | if node != nil { 79 | s.tree.Delete(node) 80 | } 81 | 82 | return node != nil 83 | } 84 | 85 | func (s *set[T]) BeginFrom(v T) Iterator[T] { 86 | s.mux.RLock() 87 | defer s.mux.RUnlock() 88 | 89 | node := s.tree.FindNode(v) 90 | return newIterator(node) 91 | } 92 | 93 | func (s *set[T]) Begin() Iterator[T] { 94 | s.mux.RLock() 95 | defer s.mux.RUnlock() 96 | 97 | return newIterator(s.tree.Begin()) 98 | } 99 | 100 | func (s *set[T]) RBegin() Iterator[T] { 101 | s.mux.RLock() 102 | defer s.mux.RUnlock() 103 | 104 | return newReverseIterator(s.tree.RBegin()) 105 | } 106 | func (s *set[T]) RBeginFrom(v T) Iterator[T] { 107 | s.mux.RLock() 108 | defer s.mux.RUnlock() 109 | 110 | node := s.tree.FindNode(v) 111 | return newReverseIterator(node) 112 | } 113 | 114 | func (s *set[T]) Slices() (result []T) { 115 | for it := s.Begin(); it != nil; it = it.Next() { 116 | result = append(result, it.Value()) 117 | } 118 | 119 | return 120 | } 121 | 122 | func (s *set[T]) Range(callback func(v T) bool) { 123 | for it := s.Begin(); it != nil; it = it.Next() { 124 | callback(it.Value()) 125 | } 126 | } 127 | 128 | func (s *set[T]) Contains(v T) bool { 129 | s.mux.RLock() 130 | defer s.mux.RUnlock() 131 | 132 | node := s.tree.FindNode(v) 133 | return node != nil 134 | } 135 | 136 | // Intersect 交集 137 | func (s *set[T]) Intersect(o Set[T]) Set[T] { 138 | var ans = NewExtSet(s.keyCmp) 139 | 140 | s.mux.RLock() 141 | 142 | for it := s.Begin(); it != nil; it = it.Next() { 143 | if o.Contains(it.Value()) { 144 | ans.Insert(it.Value()) 145 | } 146 | } 147 | 148 | s.mux.RUnlock() 149 | 150 | return ans 151 | } 152 | 153 | func (s *set[T]) Union(o Set[T]) Set[T] { 154 | var ans = NewExtSet(s.keyCmp) 155 | 156 | s.mux.RLock() 157 | 158 | for it := s.Begin(); it != nil; it = it.Next() { 159 | ans.Insert(it.Value()) 160 | } 161 | 162 | s.mux.RUnlock() 163 | 164 | for it := o.Begin(); it != nil; it = it.Next() { 165 | ans.Insert(it.Value()) 166 | } 167 | 168 | return ans 169 | } 170 | 171 | func (s *set[T]) Diff(o Set[T]) Set[T] { 172 | var ans = NewExtSet(s.keyCmp) 173 | 174 | s.mux.RLock() 175 | 176 | for it := s.Begin(); it != nil; it = it.Next() { 177 | if !o.Contains(it.Value()) { 178 | ans.Insert(it.Value()) 179 | } 180 | } 181 | 182 | s.mux.RUnlock() 183 | 184 | for it := o.Begin(); it != nil; it = it.Next() { 185 | if !s.Contains(it.Value()) { 186 | ans.Insert(it.Value()) 187 | } 188 | } 189 | 190 | return ans 191 | } 192 | 193 | func (s *set[T]) MarshalJSON() ([]byte, error) { 194 | s.mux.RLock() 195 | 196 | var list []T 197 | for it := s.Begin(); it != nil; it = it.Next() { 198 | list = append(list, it.Value()) 199 | } 200 | 201 | s.mux.RUnlock() 202 | 203 | return json.Marshal(list) 204 | } 205 | 206 | func (s *set[T]) UnmarshalJSON(b []byte) (err error) { 207 | var list []T 208 | if !json.Valid(b) { 209 | return fmt.Errorf("input is not a valid json string") 210 | } 211 | 212 | err = json.Unmarshal(b, &list) 213 | if err != nil { 214 | return fmt.Errorf("unmarshal input to array struct err:%v", err) 215 | } 216 | 217 | s.Insert(list...) 218 | 219 | return 220 | } 221 | -------------------------------------------------------------------------------- /gstl/set/set_test.go: -------------------------------------------------------------------------------- 1 | package set 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func BenchmarkSet_Insert(b *testing.B) { 11 | var s = NewSet[int]() 12 | 13 | rand.Seed(time.Now().UnixNano()) 14 | for i := 1; i < b.N; i++ { 15 | s.Insert(rand.Int() % i) 16 | s.Erase(rand.Int() % i) 17 | } 18 | 19 | // fmt.Println(s.Size()) 20 | } 21 | 22 | func TestSet_Begin(t *testing.T) { 23 | var s = NewSet[int]() 24 | 25 | for i := 1; i <= 100; i++ { 26 | s.Insert(i) 27 | } 28 | 29 | for it := s.Begin(); it != nil; it = it.Next() { 30 | fmt.Printf("%v ", it.Value()) 31 | } 32 | 33 | fmt.Println() 34 | 35 | for it := s.RBegin(); it != nil; it = it.Next() { 36 | fmt.Printf("%v ", it.Value()) 37 | } 38 | 39 | fmt.Println() 40 | } 41 | 42 | func TestSet_Contains(t *testing.T) { 43 | var s = NewSet[int]() 44 | 45 | for i := 1; i <= 100; i++ { 46 | s.Insert(i) 47 | } 48 | 49 | fmt.Println(s.Insert(98)) 50 | fmt.Println(s.Erase(98)) 51 | fmt.Println(s.Contains(98), s.Contains(99), s.Contains(101)) 52 | } 53 | 54 | func TestSet_Insert(t *testing.T) { 55 | type User struct { 56 | Name string 57 | Age int 58 | } 59 | 60 | var s = NewExtSet(func(a, b User) int { return a.Age - b.Age }) 61 | s.Insert(User{Name: "a", Age: 28}) 62 | s.Insert(User{Name: "b", Age: 21}) 63 | 64 | for it := s.Begin(); it != nil; it = it.Next() { 65 | fmt.Println(it.Value()) 66 | } 67 | } 68 | 69 | func TestSet_Intersect(t *testing.T) { 70 | var a, b = NewSet[int](), NewSet[int]() 71 | 72 | for i := 1; i < 20; i++ { 73 | a.Insert(i) 74 | } 75 | 76 | for i := 16; i < 36; i++ { 77 | b.Insert(i) 78 | } 79 | 80 | a.Insert(100, 101, 102) 81 | 82 | fmt.Println("intersect", a.Intersect(b).Slices()) 83 | fmt.Println("union", a.Union(b).Slices()) 84 | fmt.Println("diff", a.Diff(b).Slices()) 85 | } 86 | 87 | func TestSet_Slices(t *testing.T) { 88 | var a = NewSet[int64]() 89 | a.Insert(1029) 90 | a.Insert(1029) 91 | a.Insert(1028) 92 | a.Insert(1029) 93 | 94 | fmt.Println(a.Slices()) 95 | for _, id := range a.Slices() { 96 | fmt.Println(id) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /gstl/smap/README.md: -------------------------------------------------------------------------------- 1 | ## Map and MultiMap 2 | 3 | ### Requirements 4 | Go version >= 1.18.1 5 | 6 | ### Usage 7 | 8 | ```go 9 | ``` 10 | 11 | import package 12 | ```go 13 | ``` 14 | 15 | 1. Create a Map or MultiMap 16 | ```go 17 | type User struct { 18 | UID int64 19 | Age int64 20 | Avatars []string 21 | } 22 | 23 | func less(a, b User) int { 24 | return a.UID - b.UID 25 | } 26 | 27 | func handler() { 28 | // 1. Create an int Vector without lock 29 | var Map = smap.New[int, int]() 30 | 31 | var Map2 = smap.New[int, User]() 32 | 33 | // 2. Create an Struct Vector 34 | var structMap = smap.New[int, User](options.WithLocker()) 35 | 36 | var structMap3 = smap.New[int, int](options.WithLocker()) // ok 37 | 38 | // 3. Create Struct key Map 39 | var defMap = smap.NewExt[User, bool](less) 40 | 41 | var defMap2 = smap.NewExt[User, bool](less, options.WithLocker()) 42 | 43 | // 3. Create multi-map 44 | var multiMap = smap.NewMultiMap[int, int]() 45 | 46 | var multiMap2 = smap.NewMultiExtMap[User, bool]() 47 | } 48 | ``` 49 | 50 | 2. Get and Set 51 | ``` 52 | func main() { 53 | var m = smap.New[int, int]() 54 | 55 | fmt.Println("set", m.Set(1, 1)) // return ok if key does't exist 56 | fmt.Println("set", m.Set(1, 2)) // return false means key is exist, then update value to 2 57 | 58 | if v, ok := m.GetOk(1); ok { 59 | fmt.Println("get", v) 60 | } 61 | 62 | if v, ok := m.GetOk(2); ok { 63 | fmt.Println("get", v) 64 | } 65 | 66 | fmt.Println("get", m.Get(2)) // return default value(e.g. 0, "", ...) if key does't exist 67 | 68 | fmt.Println("get set", m.GetSet(1, 3)) // get preview value and set new value 69 | fmt.Println("get set", m.GetSet(1, 4)) 70 | 71 | var mm = smap.NewMultiMap[int, int]() 72 | 73 | mm.Set(1, 2) 74 | mm.Set(1, 1) 75 | mm.Set(1, 3) 76 | 77 | fmt.Println("get", mm.Get(1)) // it will return then first element visited in the tree 78 | 79 | it := mm.GetFirst(1) 80 | for it.Next() { 81 | fmt.Println("get", it.Value()) // from left to right, 2 1 3 82 | } 83 | 84 | it = mm.GetLast(1) 85 | for it.Next() { 86 | fmt.Println("get", it.Value()) // from right to left, 3 1 2 87 | } 88 | } 89 | ``` 90 | 91 | 3. Keys and Values 92 | ``` 93 | func keysAndValues() { 94 | var m = NewMap[int, int]() 95 | 96 | for i := 1; i <= 10; i++ { 97 | m.Set(i, i*2) 98 | } 99 | 100 | fmt.Println("keys", m.Keys()) // 1 2 3 4 5 6 7 8 9 10 101 | fmt.Println("values", m.Values()) // 2 4 6 8 10 12 14 16 18 20 102 | } 103 | 104 | ``` 105 | 106 | 4. Contains 107 | ``` 108 | 109 | func contains() { 110 | var m = NewMap[int, int]() 111 | 112 | for i := 1; i <= 10; i++ { 113 | m.Set(i, i*2) 114 | } 115 | 116 | if m.Contains(3) { 117 | fmt.Println("duplicated key 3") 118 | } 119 | 120 | if m.Erase(3) { 121 | fmt.Println("removed key 3") 122 | } 123 | 124 | if !m.Contains(3) { 125 | fmt.Println("not contains 3") 126 | } 127 | } 128 | ``` 129 | 130 | 5. Iterator 131 | ``` 132 | func iterator() { 133 | var m = NewMap[int, int]() 134 | 135 | for i := 1; i <= 10; i++ { 136 | m.Set(i, i) 137 | } 138 | 139 | it := m.Begin() 140 | for it.Next() { 141 | fmt.Println("iter", it.Value()) 142 | } 143 | 144 | it = m.RBegin() 145 | for it.Next() { 146 | fmt.Println("iter", it.Value()) 147 | } 148 | 149 | it = m.BeginFrom(3) 150 | for it.Next() { 151 | fmt.Println("iter", it.Value()) 152 | } 153 | 154 | it = m.RBeginFrom(3) 155 | for it.Next() { 156 | fmt.Println("iter", it.Value()) 157 | } 158 | } 159 | ``` -------------------------------------------------------------------------------- /gstl/smap/decl.go: -------------------------------------------------------------------------------- 1 | package smap 2 | 3 | type Iterator[K, V any] interface { 4 | Next() Iterator[K, V] 5 | Key() K 6 | Value() V 7 | } 8 | 9 | type Map[K, V any] interface { 10 | // Get return default value of type V if key not exist, otherwise return the value of key 11 | Get(key K) V 12 | 13 | // GetOK return true if key exist, otherwise return false 14 | GetOK(key K) (V, bool) 15 | 16 | // Set return true if key not exist, otherwise return false 17 | Set(key K, value V) bool 18 | 19 | // GetSet return the old value of key and set it to the new value 20 | GetSet(key K, value V) V 21 | 22 | // GetSetOK same as GetSet, but return false if old key does not exist 23 | GetSetOK(key K, value V) (V, bool) 24 | 25 | Contains(key K) bool 26 | 27 | // Erase return true if key exist, otherwise return false 28 | Erase(key K) bool 29 | 30 | First() V 31 | 32 | Last() V 33 | 34 | Begin() Iterator[K, V] 35 | 36 | BeginFrom(key K) Iterator[K, V] 37 | 38 | RBegin() Iterator[K, V] 39 | 40 | RBeginFrom(key K) Iterator[K, V] 41 | 42 | Keys() []K 43 | 44 | Values() []V 45 | 46 | Clear() 47 | 48 | Size() int 49 | 50 | Range(func(K, V) bool) 51 | 52 | Intersect(p Map[K, V]) Map[K, V] 53 | 54 | Union(Map[K, V]) Map[K, V] 55 | 56 | Diff(Map[K, V]) Map[K, V] 57 | } 58 | 59 | type MultiMap[K, V any] interface { 60 | // Get return the node which first visited in the tree 61 | Get(key K) V 62 | 63 | // GetOK return true if key exist, otherwise return false 64 | GetOK(key K) (V, bool) 65 | 66 | // GetFirst return the left node of same key, iterator next will range from left to right 67 | GetFirst(key K) Iterator[K, V] 68 | 69 | // GetLast return the right node of same key, iterator next will range from right to left 70 | GetLast(key K) Iterator[K, V] 71 | 72 | // Set return true if key not exist, otherwise return false 73 | Set(key K, value V) bool 74 | 75 | Contains(key K) bool 76 | 77 | // Erase return true if key exist, otherwise return false 78 | Erase(key K) bool 79 | 80 | First() V 81 | 82 | Last() V 83 | 84 | Begin() Iterator[K, V] 85 | 86 | BeginFrom(key K) Iterator[K, V] 87 | 88 | RBegin() Iterator[K, V] 89 | 90 | RBeginFrom(key K) Iterator[K, V] 91 | 92 | Keys() []K 93 | 94 | Values() []V 95 | 96 | Clear() 97 | 98 | Size() int 99 | } 100 | -------------------------------------------------------------------------------- /gstl/smap/iterator.go: -------------------------------------------------------------------------------- 1 | package smap 2 | 3 | import ( 4 | "github.com/aipave/go-utils/gstl/compare" 5 | "github.com/aipave/go-utils/gstl/rbtree" 6 | ) 7 | 8 | type iterator[K, V any] struct { 9 | current *rbtree.Node[K, V] 10 | sameKey *K 11 | keyCmp compare.Less[K] 12 | } 13 | 14 | func newIterator[K, V any](node *rbtree.Node[K, V]) Iterator[K, V] { 15 | if node == nil { 16 | return nil 17 | } 18 | 19 | return iterator[K, V]{current: node} 20 | } 21 | 22 | func newSameIterator[K, V any](node *rbtree.Node[K, V], sameKey *K, keyCmp compare.Less[K]) Iterator[K, V] { 23 | if node == nil { 24 | return nil 25 | } 26 | 27 | return iterator[K, V]{current: node, sameKey: sameKey, keyCmp: keyCmp} 28 | } 29 | 30 | func (i iterator[K, V]) Next() Iterator[K, V] { 31 | if i.current == nil { 32 | return nil 33 | } 34 | 35 | next := i.current.Next() 36 | if next == nil { 37 | return nil 38 | } 39 | 40 | if i.sameKey != nil && i.keyCmp(next.Key(), *i.sameKey) != 0 { 41 | return nil 42 | } 43 | 44 | return iterator[K, V]{current: next, keyCmp: i.keyCmp} 45 | } 46 | 47 | func (i iterator[K, V]) Key() (k K) { 48 | if i.current != nil { 49 | return i.current.Key() 50 | } 51 | return 52 | } 53 | 54 | func (i iterator[K, V]) Value() (v V) { 55 | if i.current != nil { 56 | return i.current.Value() 57 | } 58 | return 59 | } 60 | 61 | type reverseIterator[K, V any] struct { 62 | current *rbtree.Node[K, V] 63 | sameKey *K 64 | keyCmp compare.Less[K] 65 | } 66 | 67 | func newReverseIterator[K, V any](node *rbtree.Node[K, V]) Iterator[K, V] { 68 | if node == nil { 69 | return nil 70 | } 71 | 72 | return reverseIterator[K, V]{current: node} 73 | } 74 | 75 | func newReverseSameIterator[K, V any](node *rbtree.Node[K, V], sameKey *K, keyCmp compare.Less[K]) Iterator[K, V] { 76 | if node == nil { 77 | return nil 78 | } 79 | 80 | return reverseIterator[K, V]{current: node, sameKey: sameKey, keyCmp: keyCmp} 81 | } 82 | 83 | func (i reverseIterator[K, V]) Next() Iterator[K, V] { 84 | if i.current == nil { 85 | return nil 86 | } 87 | 88 | next := i.current.Next() 89 | if next == nil { 90 | return nil 91 | } 92 | 93 | if i.sameKey != nil && i.keyCmp(next.Key(), *i.sameKey) != 0 { 94 | return nil 95 | } 96 | 97 | return reverseIterator[K, V]{current: next, keyCmp: i.keyCmp} 98 | } 99 | 100 | func (i reverseIterator[K, V]) Key() (k K) { 101 | if i.current != nil { 102 | return i.current.Key() 103 | } 104 | return 105 | } 106 | 107 | func (i reverseIterator[K, V]) Value() (v V) { 108 | if i.current != nil { 109 | return i.current.Value() 110 | } 111 | return 112 | } 113 | -------------------------------------------------------------------------------- /gstl/smap/map.go: -------------------------------------------------------------------------------- 1 | package smap 2 | 3 | import ( 4 | "github.com/aipave/go-utils/gstl/compare" 5 | "github.com/aipave/go-utils/gstl/options" 6 | "github.com/aipave/go-utils/gstl/rbtree" 7 | ) 8 | 9 | type sortedMap[K, V any] struct { 10 | tree *rbtree.RBTree[K, V] 11 | mux options.RWLock 12 | keyCmp compare.Less[K] 13 | } 14 | 15 | // NewMap Primitive data types use default comparison functions 16 | func NewMap[K compare.Ordered, V any](opts ...options.Option) Map[K, V] { 17 | var option = options.New() 18 | for _, fn := range opts { 19 | fn(&option) 20 | } 21 | 22 | return &sortedMap[K, V]{ 23 | tree: rbtree.New[K, V](compare.OrderedLess[K]), 24 | mux: option.Mux, 25 | keyCmp: compare.OrderedLess[K], 26 | } 27 | } 28 | 29 | // NewExtMap Non-basic data types need to provide comparison functions 30 | func NewExtMap[K, V any](keyCmp compare.Less[K], opts ...options.Option) Map[K, V] { 31 | var option = options.New() 32 | for _, fn := range opts { 33 | fn(&option) 34 | } 35 | 36 | return &sortedMap[K, V]{ 37 | tree: rbtree.New[K, V](keyCmp), 38 | mux: option.Mux, 39 | keyCmp: keyCmp, 40 | } 41 | } 42 | 43 | func (s *sortedMap[K, V]) Size() int { 44 | return s.tree.Size() 45 | } 46 | 47 | func (s *sortedMap[K, V]) Clear() { 48 | s.mux.Lock() 49 | defer s.mux.Unlock() 50 | 51 | s.tree.Clear() 52 | } 53 | 54 | func (s *sortedMap[K, V]) Get(key K) (result V) { 55 | s.mux.RLock() 56 | defer s.mux.RUnlock() 57 | 58 | node := s.tree.FindNode(key) 59 | if node != nil { 60 | return node.Value() 61 | } 62 | 63 | return 64 | } 65 | 66 | func (s *sortedMap[K, V]) GetOK(key K) (result V, b bool) { 67 | s.mux.RLock() 68 | defer s.mux.RUnlock() 69 | 70 | node := s.tree.FindNode(key) 71 | if node != nil { 72 | return node.Value(), true 73 | } 74 | 75 | return 76 | } 77 | 78 | func (s *sortedMap[K, V]) Set(key K, value V) (allSuccess bool) { 79 | s.mux.Lock() 80 | defer s.mux.Unlock() 81 | 82 | node := s.tree.FindNode(key) 83 | if node != nil { 84 | node.SetValue(value) 85 | } else { 86 | s.tree.Insert(key, value) 87 | } 88 | 89 | return node == nil 90 | } 91 | 92 | func (s *sortedMap[K, V]) GetSet(key K, value V) (result V) { 93 | s.mux.Lock() 94 | defer s.mux.Unlock() 95 | 96 | node := s.tree.FindNode(key) 97 | if node != nil { 98 | result = node.Value() 99 | node.SetValue(value) 100 | } else { 101 | s.tree.Insert(key, value) 102 | } 103 | 104 | return 105 | } 106 | 107 | func (s *sortedMap[K, V]) GetSetOK(key K, value V) (result V, b bool) { 108 | s.mux.Lock() 109 | defer s.mux.Unlock() 110 | 111 | node := s.tree.FindNode(key) 112 | if node != nil { 113 | result, b = node.Value(), true 114 | node.SetValue(value) 115 | } else { 116 | s.tree.Insert(key, value) 117 | } 118 | 119 | return 120 | } 121 | 122 | func (s *sortedMap[K, V]) Erase(v K) bool { 123 | s.mux.Lock() 124 | defer s.mux.Unlock() 125 | 126 | node := s.tree.FindNode(v) 127 | if node != nil { 128 | s.tree.Delete(node) 129 | } 130 | 131 | return node != nil 132 | } 133 | 134 | func (s *sortedMap[K, V]) First() (v V) { 135 | s.mux.RLock() 136 | defer s.mux.RUnlock() 137 | 138 | node := s.tree.Begin() 139 | if node != nil { 140 | return node.Value() 141 | } 142 | 143 | return 144 | } 145 | 146 | func (s *sortedMap[K, V]) Last() (v V) { 147 | s.mux.RLock() 148 | defer s.mux.RUnlock() 149 | 150 | node := s.tree.RBegin() 151 | if node != nil { 152 | return node.Value() 153 | } 154 | 155 | return 156 | } 157 | 158 | func (s *sortedMap[K, V]) BeginFrom(v K) Iterator[K, V] { 159 | s.mux.RLock() 160 | defer s.mux.RUnlock() 161 | 162 | node := s.tree.FindNode(v) 163 | return newIterator(node) 164 | } 165 | 166 | func (s *sortedMap[K, V]) Begin() Iterator[K, V] { 167 | s.mux.RLock() 168 | defer s.mux.RUnlock() 169 | 170 | return newIterator(s.tree.Begin()) 171 | } 172 | 173 | func (s *sortedMap[K, V]) RBegin() Iterator[K, V] { 174 | s.mux.RLock() 175 | defer s.mux.RUnlock() 176 | 177 | return newReverseIterator(s.tree.RBegin()) 178 | } 179 | func (s *sortedMap[K, V]) RBeginFrom(v K) Iterator[K, V] { 180 | s.mux.RLock() 181 | defer s.mux.RUnlock() 182 | 183 | node := s.tree.FindNode(v) 184 | return newReverseIterator(node) 185 | } 186 | 187 | func (s *sortedMap[K, V]) Keys() (result []K) { 188 | for it := s.Begin(); it != nil; it = it.Next() { 189 | result = append(result, it.Key()) 190 | } 191 | 192 | return 193 | } 194 | 195 | func (s *sortedMap[K, V]) Values() (result []V) { 196 | for it := s.Begin(); it != nil; it = it.Next() { 197 | result = append(result, it.Value()) 198 | } 199 | 200 | return 201 | } 202 | 203 | func (s *sortedMap[K, V]) Contains(v K) bool { 204 | s.mux.RLock() 205 | defer s.mux.RUnlock() 206 | 207 | node := s.tree.FindNode(v) 208 | return node != nil 209 | } 210 | 211 | // Intersect 交集 212 | func (s *sortedMap[K, V]) Intersect(o Map[K, V]) Map[K, V] { 213 | var ans = NewExtMap[K, V](s.keyCmp) 214 | 215 | s.mux.RLock() 216 | 217 | for it := s.Begin(); it != nil; it = it.Next() { 218 | if o.Contains(it.Key()) { 219 | ans.Set(it.Key(), it.Value()) 220 | } 221 | } 222 | 223 | s.mux.RUnlock() 224 | 225 | return ans 226 | } 227 | 228 | func (s *sortedMap[K, V]) Union(o Map[K, V]) Map[K, V] { 229 | var ans = NewExtMap[K, V](s.keyCmp) 230 | 231 | s.mux.RLock() 232 | 233 | for it := s.Begin(); it != nil; it = it.Next() { 234 | ans.Set(it.Key(), it.Value()) 235 | } 236 | 237 | s.mux.RUnlock() 238 | 239 | for it := o.Begin(); it != nil; it = it.Next() { 240 | ans.Set(it.Key(), it.Value()) 241 | } 242 | 243 | return ans 244 | } 245 | 246 | func (s *sortedMap[K, V]) Diff(o Map[K, V]) Map[K, V] { 247 | var ans = NewExtMap[K, V](s.keyCmp) 248 | 249 | s.mux.RLock() 250 | 251 | for it := s.Begin(); it != nil; it = it.Next() { 252 | if !o.Contains(it.Key()) { 253 | ans.Set(it.Key(), it.Value()) 254 | } 255 | } 256 | 257 | s.mux.RUnlock() 258 | 259 | for it := o.Begin(); it != nil; it = it.Next() { 260 | if !s.Contains(it.Key()) { 261 | ans.Set(it.Key(), it.Value()) 262 | } 263 | } 264 | 265 | return ans 266 | } 267 | 268 | func (s *sortedMap[K, V]) Range(callback func(k K, v V) bool) { 269 | for it := s.Begin(); it != nil; it = it.Next() { 270 | callback(it.Key(), it.Value()) 271 | } 272 | } 273 | -------------------------------------------------------------------------------- /gstl/smap/map_test.go: -------------------------------------------------------------------------------- 1 | package smap 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestSortedMap_Get(t *testing.T) { 9 | var m = NewMap[int, int]() 10 | 11 | fmt.Println("set [1, 1]", m.Set(1, 1)) 12 | fmt.Println("set [1, 2]", m.Set(1, 2)) 13 | 14 | fmt.Println("get 1", m.Get(1)) 15 | 16 | if v, ok := m.GetOK(1); ok { 17 | fmt.Println("getOk 1", v) 18 | } 19 | 20 | if v, ok := m.GetOK(2); ok { 21 | fmt.Println("getOk 2", v) 22 | } 23 | 24 | var mm = NewMultiMap[int, int]() 25 | 26 | fmt.Println("set [1, 2]", mm.Set(1, 2)) 27 | fmt.Println("set [1, 1]", mm.Set(1, 1)) 28 | fmt.Println("set [1, 3]", mm.Set(1, 3)) 29 | 30 | fmt.Println("get 1", mm.Get(1)) 31 | 32 | for it := mm.GetFirst(1); it != nil; it = it.Next() { 33 | fmt.Println("get next 1", it.Value()) 34 | } 35 | 36 | for it := mm.GetLast(1); it != nil; it = it.Next() { 37 | fmt.Println("get next 1", it.Value()) 38 | } 39 | 40 | fmt.Println("get set", m.GetSet(1, 3)) 41 | fmt.Println("get set", m.GetSet(1, 4)) 42 | } 43 | 44 | func TestSortedMap_Keys(t *testing.T) { 45 | var m = NewMap[int, int]() 46 | 47 | for i := 1; i <= 10; i++ { 48 | m.Set(i, i*2) 49 | } 50 | 51 | fmt.Println("keys", m.Keys()) 52 | fmt.Println("values", m.Values()) 53 | } 54 | 55 | func TestSortedMap_Contains(t *testing.T) { 56 | var m = NewMap[int, int]() 57 | 58 | for i := 1; i <= 10; i++ { 59 | m.Set(i, i*2) 60 | } 61 | 62 | if m.Contains(3) { 63 | fmt.Println("duplicated key 3") 64 | } 65 | 66 | if m.Erase(3) { 67 | fmt.Println("removed key 3") 68 | } 69 | 70 | if !m.Contains(3) { 71 | fmt.Println("not contains 3") 72 | } 73 | } 74 | 75 | func TestSortedMap_Begin(t *testing.T) { 76 | var m = NewMap[int, int]() 77 | 78 | for i := 1; i <= 10; i++ { 79 | m.Set(i, i) 80 | } 81 | 82 | for it := m.Begin(); it != nil; it = it.Next() { 83 | fmt.Println("iter", it.Value()) 84 | } 85 | 86 | for it := m.RBegin(); it != nil; it = it.Next() { 87 | fmt.Println("iter", it.Value()) 88 | } 89 | 90 | for it := m.BeginFrom(3); it != nil; it = it.Next() { 91 | fmt.Println("iter", it.Value()) 92 | } 93 | 94 | for it := m.RBeginFrom(3); it != nil; it = it.Next() { 95 | fmt.Println("iter", it.Value()) 96 | } 97 | } 98 | 99 | func TestSortedMap_Intersect(t *testing.T) { 100 | var a = NewMap[int, int]() 101 | a.Set(1, 2) 102 | a.Set(2, 4) 103 | a.Set(4, 8) 104 | 105 | var b = NewMap[int, int]() 106 | b.Set(2, 4) 107 | b.Set(4, 8) 108 | b.Set(6, 12) 109 | 110 | var c = a.Intersect(b) 111 | c.Range(func(k, v int) bool { 112 | fmt.Println(k, v) 113 | return true 114 | }) 115 | 116 | fmt.Println(a.Union(b).Keys()) 117 | } 118 | 119 | func TestNewMap(t *testing.T) { 120 | var nm = NewMap[int, int]() 121 | fmt.Println(nm.Keys(), nm.Values()) 122 | } 123 | -------------------------------------------------------------------------------- /gstl/smap/multi_map.go: -------------------------------------------------------------------------------- 1 | package smap 2 | 3 | import ( 4 | "github.com/aipave/go-utils/gstl/compare" 5 | "github.com/aipave/go-utils/gstl/options" 6 | "github.com/aipave/go-utils/gstl/rbtree" 7 | ) 8 | 9 | type sortedMultiMap[K, V any] struct { 10 | tree *rbtree.RBTree[K, V] 11 | mux options.RWLock 12 | keyCmp compare.Less[K] 13 | } 14 | 15 | // NewMultiMap 16 | func NewMultiMap[K compare.Ordered, V any](opts ...options.Option) MultiMap[K, V] { 17 | var option = options.New() 18 | for _, fn := range opts { 19 | fn(&option) 20 | } 21 | 22 | return &sortedMultiMap[K, V]{ 23 | tree: rbtree.New[K, V](compare.OrderedLess[K]), 24 | mux: option.Mux, 25 | keyCmp: compare.OrderedLess[K], 26 | } 27 | } 28 | 29 | // NewMultiExtMap 30 | func NewMultiExtMap[K, V any](keyCmp compare.Less[K], opts ...options.Option) MultiMap[K, V] { 31 | var option = options.New() 32 | for _, fn := range opts { 33 | fn(&option) 34 | } 35 | 36 | return &sortedMultiMap[K, V]{ 37 | tree: rbtree.New[K, V](keyCmp), 38 | mux: option.Mux, 39 | keyCmp: keyCmp, 40 | } 41 | } 42 | 43 | func (s *sortedMultiMap[K, V]) Size() int { 44 | return s.tree.Size() 45 | } 46 | 47 | func (s *sortedMultiMap[K, V]) Clear() { 48 | s.mux.Lock() 49 | defer s.mux.Unlock() 50 | 51 | s.tree.Clear() 52 | } 53 | 54 | func (s *sortedMultiMap[K, V]) Get(key K) (result V) { 55 | s.mux.RLock() 56 | defer s.mux.RUnlock() 57 | 58 | node := s.tree.FindNode(key) 59 | if node != nil { 60 | return node.Value() 61 | } 62 | 63 | return 64 | } 65 | 66 | func (s *sortedMultiMap[K, V]) GetOK(key K) (result V, b bool) { 67 | s.mux.RLock() 68 | defer s.mux.RUnlock() 69 | 70 | node := s.tree.FindNode(key) 71 | if node != nil { 72 | return node.Value(), true 73 | } 74 | 75 | return 76 | } 77 | 78 | func (s *sortedMultiMap[K, V]) GetFirst(key K) Iterator[K, V] { 79 | s.mux.RLock() 80 | defer s.mux.RUnlock() 81 | 82 | node := s.tree.FindFirstNode(key) 83 | if node != nil { 84 | return newSameIterator(node, &key, s.keyCmp) 85 | } 86 | 87 | return nil 88 | } 89 | 90 | func (s *sortedMultiMap[K, V]) GetLast(key K) Iterator[K, V] { 91 | s.mux.RLock() 92 | defer s.mux.RUnlock() 93 | 94 | node := s.tree.FindLastNode(key) 95 | if node != nil { 96 | return newReverseSameIterator(node, &key, s.keyCmp) 97 | } 98 | 99 | return nil 100 | } 101 | 102 | func (s *sortedMultiMap[K, V]) Set(key K, value V) bool { 103 | s.mux.Lock() 104 | defer s.mux.Unlock() 105 | 106 | node := s.tree.FindNode(key) 107 | s.tree.Insert(key, value) 108 | 109 | return node == nil 110 | } 111 | 112 | func (s *sortedMultiMap[K, V]) Erase(v K) (ok bool) { 113 | s.mux.Lock() 114 | defer s.mux.Unlock() 115 | 116 | for { 117 | node := s.tree.FindNode(v) 118 | if node == nil { 119 | break 120 | } 121 | s.tree.Delete(node) 122 | ok = true 123 | } 124 | 125 | return 126 | } 127 | 128 | func (s *sortedMultiMap[K, V]) First() (v V) { 129 | s.mux.RLock() 130 | defer s.mux.RUnlock() 131 | 132 | node := s.tree.Begin() 133 | if node != nil { 134 | return node.Value() 135 | } 136 | 137 | return 138 | } 139 | 140 | func (s *sortedMultiMap[K, V]) Last() (v V) { 141 | s.mux.RLock() 142 | defer s.mux.RUnlock() 143 | 144 | node := s.tree.RBegin() 145 | if node != nil { 146 | return node.Value() 147 | } 148 | 149 | return 150 | } 151 | 152 | func (s *sortedMultiMap[K, V]) BeginFrom(v K) Iterator[K, V] { 153 | s.mux.RLock() 154 | defer s.mux.RUnlock() 155 | 156 | node := s.tree.FindFirstNode(v) 157 | return newIterator(node) 158 | } 159 | 160 | func (s *sortedMultiMap[K, V]) Begin() Iterator[K, V] { 161 | s.mux.RLock() 162 | defer s.mux.RUnlock() 163 | 164 | return newIterator(s.tree.Begin()) 165 | } 166 | 167 | func (s *sortedMultiMap[K, V]) RBegin() Iterator[K, V] { 168 | s.mux.RLock() 169 | defer s.mux.RUnlock() 170 | 171 | return newReverseIterator(s.tree.RBegin()) 172 | } 173 | func (s *sortedMultiMap[K, V]) RBeginFrom(v K) Iterator[K, V] { 174 | s.mux.RLock() 175 | defer s.mux.RUnlock() 176 | 177 | node := s.tree.FindLastNode(v) 178 | return newReverseIterator(node) 179 | } 180 | 181 | func (s *sortedMultiMap[K, V]) Keys() (result []K) { 182 | for it := s.Begin(); it != nil; it = it.Next() { 183 | result = append(result, it.Key()) 184 | } 185 | 186 | return 187 | } 188 | 189 | func (s *sortedMultiMap[K, V]) Values() (result []V) { 190 | for it := s.Begin(); it != nil; it = it.Next() { 191 | result = append(result, it.Value()) 192 | } 193 | 194 | return 195 | } 196 | 197 | func (s *sortedMultiMap[K, V]) Contains(v K) bool { 198 | s.mux.RLock() 199 | defer s.mux.RUnlock() 200 | 201 | node := s.tree.FindNode(v) 202 | return node != nil 203 | } 204 | -------------------------------------------------------------------------------- /gstl/vector/README.md: -------------------------------------------------------------------------------- 1 | ## Vector 2 | 3 | ### Requirements 4 | Go version >= 1.18.1 5 | 6 | ### Usage 7 | 8 | ```go 9 | ``` 10 | 11 | import package 12 | ```go 13 | ``` 14 | 15 | 1. Create a Vector 16 | ```go 17 | type User struct { 18 | UID int64 19 | Age int64 20 | Avatars []string 21 | } 22 | 23 | func handler() { 24 | // 1. Create an int Vector without lock 25 | var vec = vector.New[int](nil) 26 | 27 | var list []int 28 | var vec2 = vector.New[int](list) // ok 29 | 30 | // 2. Create an Struct Vector 31 | var structVec = vector.New[User](nil) 32 | 33 | var ary []User 34 | var structVec3 = vector.New(ary) // ok 35 | } 36 | ``` 37 | 38 | 2. Sort and Reverse 39 | ```go 40 | func sort() { 41 | var vec = vector.New[int]([1, 3, 2, 4, 6, 5, 8, 7, 10, 9]]) // create an unsafeVec 42 | 43 | // sort 44 | vec.Sort(func a, b int) bool { 45 | return a < b // asc 46 | }) 47 | fmt.Println("sorted", vec.Slices()) // result: 1 2 3 4 5 6 7 8 9 10 48 | 49 | // reverse 50 | vec.Reverse() 51 | fmt.Println("reverse", vec.Slices()) // result: 10 9 8 7 6 5 4 3 2 1 52 | } 53 | ``` 54 | 55 | 3. Find and FindBy 56 | ```go 57 | var User struct { 58 | Name string 59 | Age int 60 | } 61 | 62 | func find() { 63 | // find 64 | var vec = vector.New[int]([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) // type int 65 | 66 | pos := vec.Find(4) // -1 means not found 67 | fmt.Println("found pos: ", pos, "pos value: ", vec.Get(pos)) 68 | 69 | var uVec = vector.New[User](nil) 70 | var u = User{Name: "yangmingzi", Age: 28} 71 | uVec.Append(u) 72 | 73 | filter := User{Name: "yangmingzi"} // filter by name is not allowed 74 | pos = uVec.Find(filter) // not found 75 | fmt.Println("found pos: ", pos) 76 | fmt.Println("found pos: ", uVec.Find(u)) // ok 77 | 78 | // findBy 79 | pos = uVec.FindBy(func (x User) bool { 80 | return x.Name == "yangmingzi" // filter by name 81 | }) 82 | fmt.Println("found pos: ", pos, "found value: ", uVec.Get(pos)) // ok 83 | } 84 | ``` 85 | 86 | 4. Erase 87 | ```go 88 | func erase() { 89 | // 1. comparable type 90 | var vec = vector.New([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) // type int 91 | 92 | vec.Erase(4) // ok, remove value 4 93 | vec.EraseAt(4) // ok, remove index 4 94 | } 95 | ``` 96 | 97 | 5. Join 98 | ```go 99 | func join() { 100 | var vec = vector.New([1, 2, 3, 4, 5, 6]) 101 | 102 | fmt.Println(vec.Join("|")) // result: 1|2|3|4|5|6 103 | fmt.Println(vec.Join(",")) // result: 1,2,3,4,5,6 104 | } 105 | ``` 106 | 107 | 6. Iterator and ReverseIterator 108 | ```go 109 | func iterator() { 110 | var vec = vector.New([1, 2, 3, 4, 5, 6]) 111 | 112 | iter := vec.Begin() 113 | for iter.Next() { 114 | fmt.Println("value: ", iter.Value()) 115 | } 116 | 117 | iter = vec.RBegin() 118 | for iter.Next() { 119 | fmt.Println("value: ", iter.Value()) 120 | } 121 | } 122 | ``` 123 | -------------------------------------------------------------------------------- /gstl/vector/decl.go: -------------------------------------------------------------------------------- 1 | package vector 2 | 3 | // Iterator . 4 | type Iterator[T any] interface { 5 | Next() bool 6 | Value() T 7 | } 8 | 9 | // Vector array container 10 | type Vector[T any] interface { 11 | Begin() Iterator[T] 12 | 13 | RBegin() Iterator[T] 14 | 15 | Slice(begin, end int) []T 16 | 17 | Slices() []T 18 | 19 | // Get return value in pos 20 | Get(pos int) T 21 | 22 | Set(pos int, x T) bool 23 | 24 | Append(x ...T) 25 | 26 | Insert(pos int, x ...T) 27 | 28 | // Find return index of x in vector 29 | // -1 => not found 30 | // 0-N => index of x in the array 31 | Find(x T) int 32 | 33 | // FindBy custom defined filter 34 | // -1 => not found 35 | // 0-N => index ofx in the array 36 | FindBy(filter func(x T) bool) int 37 | 38 | Reverse() 39 | 40 | Size() int 41 | 42 | // Clear remove all elements 43 | Clear() 44 | 45 | // EraseAt remove index 'pos' from vector 46 | EraseAt(pos int) bool 47 | 48 | // Erase remove x from vector 49 | Erase(x T) bool 50 | 51 | // Sort by less 52 | Sort(less func(T, T) bool) 53 | 54 | // Front return first element of vector 55 | Front() T 56 | 57 | // Back return last element of vector 58 | Back() T 59 | 60 | // PopFront remove first element of vector 61 | PopFront() bool 62 | 63 | // PopBack remove last element of vector 64 | PopBack() bool 65 | 66 | Join(sep string) string 67 | } 68 | -------------------------------------------------------------------------------- /gstl/vector/vector.go: -------------------------------------------------------------------------------- 1 | package vector 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "reflect" 7 | "sort" 8 | "strings" 9 | "sync" 10 | ) 11 | 12 | // New create a vector[T] of type T 13 | func New[T any](v []T) Vector[T] { 14 | var vec = &vector[T]{data: v} 15 | return vec 16 | } 17 | 18 | type vector[T any] struct { 19 | data []T 20 | mux sync.RWMutex 21 | } 22 | 23 | // Sort by less 24 | func (v *vector[T]) Sort(less func(a, b T) bool) { 25 | sort.Slice(v.data, func(i, j int) bool { 26 | return less(v.data[i], v.data[j]) 27 | }) 28 | } 29 | 30 | func (v *vector[T]) Size() int { 31 | return len(v.data) 32 | } 33 | 34 | func (v *vector[T]) Clear() { 35 | v.data = nil // clear 36 | } 37 | 38 | func (v *vector[T]) Append(value ...T) { 39 | v.mux.Lock() 40 | v.data = append(v.data, value...) // append not goroutine safe 41 | v.mux.Unlock() 42 | } 43 | 44 | // Slice 返回数组切片 45 | // begin 0-n 46 | // end -1 表示数组最后一个数 47 | func (v *vector[T]) Slice(begin, end int) []T { 48 | if begin >= v.Size() { 49 | return nil 50 | } 51 | 52 | if end < 0 || end > v.Size() { 53 | end = v.Size() 54 | } 55 | 56 | return v.data[begin:end] 57 | } 58 | 59 | func (v *vector[T]) Slices() []T { 60 | return v.data 61 | } 62 | 63 | // Reverse 逆序函数 64 | func (v *vector[T]) Reverse() { 65 | for i, j := 0, v.Size()-1; i < j; i, j = i+1, j-1 { 66 | v.data[i], v.data[j] = v.data[j], v.data[i] 67 | } 68 | } 69 | 70 | func (v *vector[T]) Get(pos int) (value T) { 71 | if pos < 0 || pos >= v.Size() { 72 | return 73 | } 74 | 75 | return v.data[pos] 76 | } 77 | 78 | // Set is a helper func to update val at 'pos' if it exists, otherwise nothing modified 79 | func (v *vector[T]) Set(pos int, val T) bool { 80 | if pos >= 0 && pos < v.Size() { 81 | v.data[pos] = val 82 | return true 83 | } 84 | 85 | return false 86 | } 87 | 88 | func (v *vector[T]) Front() (val T) { 89 | if v.Size() == 0 { 90 | return 91 | } 92 | return v.data[0] 93 | } 94 | 95 | func (v *vector[T]) Back() (val T) { 96 | if v.Size() == 0 { 97 | return 98 | } 99 | 100 | index := v.Size() - 1 101 | return v.data[index] 102 | } 103 | 104 | func (v *vector[T]) PopBack() bool { 105 | if v.Size() > 0 { 106 | v.data = v.data[:v.Size()-1] 107 | return true 108 | } 109 | 110 | return false 111 | } 112 | 113 | func (v *vector[T]) PopFront() bool { 114 | if v.Size() > 0 { 115 | v.data = v.data[1:] 116 | return true 117 | } 118 | 119 | return false 120 | } 121 | 122 | func (v *vector[T]) EraseAt(pos int) bool { 123 | if pos < 0 || pos >= v.Size() { 124 | return false 125 | } 126 | 127 | for i := pos; i+1 < v.Size(); i++ { 128 | v.data[i] = v.data[i+1] 129 | } 130 | 131 | if pos < v.Size() { 132 | v.data = v.data[:v.Size()-1] 133 | } 134 | 135 | return true 136 | } 137 | 138 | // RBegin 逆序迭代 139 | func (v *vector[T]) RBegin() Iterator[T] { 140 | return newReverseIterator(v, v.Size()-1) 141 | } 142 | 143 | // Begin 顺序迭代 144 | func (v *vector[T]) Begin() Iterator[T] { 145 | return newIterator(v, 0) 146 | } 147 | 148 | // Erase is func for remote element from vector 149 | // you need to get the iterator by Find before you call this method 150 | func (v *vector[T]) Erase(x T) bool { 151 | return v.EraseAt(v.Find(x)) 152 | } 153 | 154 | // Find parameter x must implements IEqual interface if its un-comparable type 155 | func (v *vector[T]) Find(x T) int { 156 | for i := range v.data { 157 | if reflect.DeepEqual(v.data[i], x) { 158 | return i 159 | } 160 | } 161 | 162 | // find nothing 163 | return -1 164 | } 165 | 166 | func (v *vector[T]) FindBy(filter func(x T) bool) int { 167 | for i := range v.data { 168 | if filter(v.data[i]) { 169 | return i 170 | } 171 | } 172 | 173 | // find nothing 174 | return -1 175 | } 176 | 177 | // Join is a helper func to join vector value to string by 'sep' 178 | func (v *vector[T]) Join(sep string) string { 179 | var elems []string 180 | for i := range v.data { 181 | elems = append(elems, fmt.Sprintf("%v", v.data[i])) 182 | } 183 | return strings.Join(elems, sep) 184 | } 185 | 186 | // Insert is helper func to batch insert elements to vector 187 | func (v *vector[T]) Insert(pos int, val ...T) { 188 | if pos > v.Size() { 189 | pos = v.Size() 190 | } 191 | 192 | v.data = append(v.data, val...) 193 | 194 | for i := v.Size() - 1; i >= pos+len(val); i-- { 195 | v.data[i] = v.data[i-len(val)] 196 | } 197 | 198 | copy(v.data[pos:], val) 199 | } 200 | 201 | func (v *vector[T]) MarshalJSON() (b []byte, err error) { 202 | if v.data == nil { 203 | v.data = make([]T, 0) 204 | } 205 | 206 | return json.Marshal(v.data) 207 | } 208 | 209 | func (v *vector[T]) UnmarshalJSON(b []byte) (err error) { 210 | err = json.Unmarshal(b, &v.data) 211 | return 212 | } 213 | -------------------------------------------------------------------------------- /gstl/vector/vector_iter.go: -------------------------------------------------------------------------------- 1 | package vector 2 | 3 | func newIterator[T any](vec *vector[T], pos int) Iterator[T] { 4 | return &iterator[T]{vec: vec, current: vec.Get(pos), pos: pos} 5 | } 6 | 7 | type iterator[T any] struct { 8 | vec *vector[T] 9 | current T 10 | pos int 11 | } 12 | 13 | func (v *iterator[T]) Next() bool { 14 | if v.pos >= v.vec.Size() { 15 | return false 16 | } 17 | 18 | v.current = v.vec.Get(v.pos) 19 | v.pos++ 20 | return true 21 | } 22 | 23 | func (v *iterator[T]) Value() T { 24 | return v.current 25 | } 26 | 27 | func newReverseIterator[T any](vec *vector[T], pos int) Iterator[T] { 28 | return &reverseIterator[T]{vec: vec, current: vec.Get(pos), pos: pos} 29 | } 30 | 31 | type reverseIterator[T any] struct { 32 | vec *vector[T] 33 | current T 34 | pos int 35 | } 36 | 37 | func (v *reverseIterator[T]) Next() bool { 38 | if v.pos < 0 { 39 | return false 40 | } 41 | 42 | v.current = v.vec.Get(v.pos) 43 | v.pos-- 44 | return true 45 | } 46 | 47 | func (v reverseIterator[T]) Value() T { 48 | return v.current 49 | } 50 | -------------------------------------------------------------------------------- /gtime/gtime.go: -------------------------------------------------------------------------------- 1 | package gtime 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | const ( 8 | RFC3339 = "2019-01-02T15:04:05+08:00" 9 | FormatDefault = "2019-01-02 15:04:05" 10 | FormatDefaultMill = "2019-01-02 15:04:05.000" 11 | FormatYYYYMMDDHHMMSS = "20190102150405" 12 | FormatDate = "2019-01-02" 13 | FormatTime = "15:04:05" 14 | FormatYYYYMM = "201901" 15 | FormatYYMM = "0601" 16 | FormatYYYYMMDD = "20190102" 17 | FormatMMDD = "0102" 18 | ) 19 | 20 | var MaxInt32, _ = time.Parse(FormatDefault, "2019-01-19 03:14:07") 21 | 22 | const ( 23 | MilliSecond = time.Millisecond 24 | Second = time.Second 25 | Minute = time.Minute 26 | Hour = time.Hour 27 | Day = 24 * Hour 28 | Week = 7 * Day 29 | Year = 365 * Day 30 | ) 31 | 32 | const ( 33 | ISecond int64 = 1 34 | IMinute int64 = 60 * ISecond 35 | IHour int64 = 60 * IMinute 36 | IDay int64 = 24 * IHour 37 | IWeek int64 = 7 * IDay 38 | IMonth int64 = 30 * IDay 39 | IYear = 365 * IDay 40 | ) 41 | -------------------------------------------------------------------------------- /gtoken/gtoken.go: -------------------------------------------------------------------------------- 1 | package gtoken 2 | 3 | import ( 4 | "bytes" 5 | "crypto/aes" 6 | "crypto/cipher" 7 | "encoding/hex" 8 | "fmt" 9 | "strconv" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | const ( 15 | deviceType = "1" 16 | rpcAccessToken = "1" 17 | fixedBits = 16 18 | ) 19 | 20 | var ( 21 | key = []byte("\x2e\xe2\x53\xba\x33\x14\x59\x48\xa0\xa4\x4e\x3c\x56\x3c\xa7\xb6") 22 | iv = []byte("\x38\x37\xf5\x98\x84\xf7\x41\x0c\x2f\x05\xa3\x15\x79\x86\xd1\x5a") 23 | ) 24 | 25 | // CreateToken 26 | func CreateToken(uusrid uint64) (string, error) { 27 | expireSeconds := time.Now().Add(time.Hour * 24 * 2).Unix() 28 | plainText := fmt.Sprintf("%v:%s:%s:%s", uusrid, deviceType, strconv.FormatInt(expireSeconds, 10), rpcAccessToken) 29 | return encrypt([]byte(plainText)) 30 | } 31 | 32 | func encrypt(rawData []byte) (string, error) { 33 | data, err := aesCBCEncrypt(rawData) 34 | if err != nil { 35 | return "", err 36 | } 37 | return hex.EncodeToString(data), nil 38 | } 39 | 40 | func aesCBCEncrypt(rawData []byte) ([]byte, error) { 41 | block, err := aes.NewCipher(key) 42 | if err != nil { 43 | return nil, err 44 | } 45 | rawData = pkcs5Padding(rawData, 16) 46 | cipherText := make([]byte, len(rawData)) 47 | mode := cipher.NewCBCEncrypter(block, iv) 48 | mode.CryptBlocks(cipherText, rawData) 49 | return cipherText, nil 50 | } 51 | 52 | func pkcs5Padding(ciphertext []byte, blockSize int) []byte { 53 | padding := blockSize - len(ciphertext)%blockSize 54 | padText := bytes.Repeat([]byte{byte(padding)}, padding) 55 | return append(ciphertext, padText...) 56 | } 57 | 58 | // VerifyToken 59 | func VerifyToken(usrid uint64, ticket string) error { 60 | ticketUid, expiredTime, err := decryptTicket(ticket) 61 | if err != nil { 62 | return err 63 | } 64 | 65 | if usrid != ticketUid { 66 | err := fmt.Errorf("invalid usrid") 67 | return err 68 | } 69 | 70 | if expiredTime < time.Now().Unix() { 71 | err := fmt.Errorf("token expire") 72 | return err 73 | } 74 | 75 | return nil 76 | } 77 | 78 | // DecryptTicket 79 | func decryptTicket(encryptedStr string) (usrid uint64, expire int64, err error) { 80 | plainText, err := decrypt(encryptedStr) 81 | if err != nil { 82 | return 0, -1, err 83 | } 84 | arrays := strings.Split(plainText, ":") 85 | if len(arrays) != 4 { 86 | return 0, -1, fmt.Errorf("invalid plainText(%v)", plainText) 87 | } 88 | 89 | usrid, err = strconv.ParseUint(arrays[0], 10, 64) 90 | if err != nil { 91 | return 0, -1, fmt.Errorf("invalid usrid from plainText(%v)", plainText) 92 | } 93 | 94 | expire, err = strconv.ParseInt(arrays[2], 10, 64) 95 | if err != nil { 96 | return 0, -1, fmt.Errorf("invalid expiredtime from plainText(%v)", plainText) 97 | } 98 | return usrid, expire, nil 99 | } 100 | 101 | func decrypt(encryptedStr string) (string, error) { 102 | data, err := hex.DecodeString(encryptedStr) 103 | if err != nil { 104 | return "", err 105 | } 106 | dnData, err := aesCBCDecrypt(data) 107 | if err != nil { 108 | return "", err 109 | } 110 | return string(dnData), nil 111 | } 112 | 113 | func aesCBCDecrypt(encryptData []byte) ([]byte, error) { 114 | block, err := aes.NewCipher(key) 115 | if err != nil { 116 | return nil, err 117 | } 118 | if len(encryptData)%fixedBits != 0 { 119 | err := fmt.Errorf("ciphertext is not a multiple of the block size") 120 | return nil, err 121 | } 122 | mode := cipher.NewCBCDecrypter(block, iv) 123 | mode.CryptBlocks(encryptData, encryptData) 124 | encryptData = pKCS7UnPadding(encryptData) 125 | return encryptData, nil 126 | } 127 | 128 | func pKCS7UnPadding(origData []byte) []byte { 129 | length := len(origData) 130 | if length <= 0 { 131 | return nil 132 | } 133 | 134 | unPadding := int(origData[length-1]) 135 | if length < unPadding { 136 | return nil 137 | } 138 | 139 | return origData[:(length - unPadding)] 140 | } 141 | -------------------------------------------------------------------------------- /gwarn/README.md: -------------------------------------------------------------------------------- 1 | # warn to lark cli. 2 | 3 | -------------------------------------------------------------------------------- /gwarn/types.go: -------------------------------------------------------------------------------- 1 | package gwarn 2 | 3 | type header struct { 4 | Title element `json:"title"` 5 | Template string `json:"template"` 6 | } 7 | 8 | type element struct { 9 | Tag string `json:"tag"` 10 | Content string `json:"content"` 11 | } 12 | 13 | type cards struct { 14 | Header header `json:"header"` 15 | Elements []element `json:"elements"` 16 | } 17 | 18 | type msgCard struct { 19 | MsgType string `json:"msg_type"` 20 | Card cards `json:"card"` 21 | } 22 | 23 | type FontColor string 24 | 25 | const ( 26 | FontColorGreen FontColor = "green" 27 | FontColorRed FontColor = "red" 28 | FontColorGrey FontColor = "grey" 29 | ) 30 | -------------------------------------------------------------------------------- /gwarn/warn2lark.go: -------------------------------------------------------------------------------- 1 | package gwarn 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "strings" 10 | 11 | ginfos "github.com/aipave/go-utils/ginfos" 12 | "github.com/sirupsen/logrus" 13 | ) 14 | 15 | type Alarm struct { 16 | WebHookAddr string 17 | Title string 18 | ShowIp bool 19 | NoticeAll bool 20 | CardColor string 21 | FontColor string 22 | } 23 | 24 | /* 25 | * 生成卡片 26 | */ 27 | func (m *Alarm) generateCardMsg(content string, level string) (c msgCard) { 28 | c.MsgType = "interactive" // card type 29 | 30 | // header 31 | c.Card.Header.Title.Tag = "plain_text" 32 | c.Card.Header.Title.Content = m.Title 33 | if level == "info" { 34 | c.Card.Header.Template = "green" 35 | } else { 36 | c.Card.Header.Template = "red" 37 | } 38 | if m.CardColor != "default" { 39 | c.Card.Header.Template = m.CardColor 40 | } 41 | 42 | body := fmt.Sprintf("%v", content) 43 | if m.ShowIp { 44 | body = fmt.Sprintf("%v\n(IP:%v)", content, ginfos.Runtime.IP()) 45 | } 46 | 47 | // body 48 | if m.FontColor != "dafault" { 49 | c.Card.Elements = append(c.Card.Elements, element{ 50 | Tag: "markdown", 51 | Content: fmt.Sprintf("%v", m.FontColor, body), 52 | }) 53 | } else { 54 | c.Card.Elements = append(c.Card.Elements, element{ 55 | Tag: "markdown", 56 | Content: body, 57 | }) 58 | } 59 | 60 | // dividing line 61 | c.Card.Elements = append(c.Card.Elements, element{ 62 | Tag: "hr", 63 | }) 64 | 65 | // @all 66 | if m.NoticeAll { 67 | c.Card.Elements = append(c.Card.Elements, element{ 68 | Tag: "markdown", 69 | Content: "", 70 | }) 71 | } 72 | 73 | return 74 | } 75 | 76 | /* 77 | * alarm to lark 78 | */ 79 | func (m *Alarm) alert(contents string, level string) { 80 | card := m.generateCardMsg(contents, level) 81 | content, _ := json.Marshal(card) 82 | logrus.Infof("%v", string(content)) 83 | resp, err := http.Post(m.WebHookAddr, "application/json", bytes.NewReader(content)) 84 | if err != nil { 85 | logrus.Errorf("alert err:%v\n", err) 86 | return 87 | } 88 | defer resp.Body.Close() 89 | 90 | var data []byte 91 | data, err = ioutil.ReadAll(resp.Body) 92 | if err != nil { 93 | logrus.Errorf("read body err:%v\n", err) 94 | return 95 | } 96 | 97 | var codec struct { 98 | Code int64 `json:"code"` 99 | Msg string `json:"msg"` 100 | } 101 | 102 | err = json.Unmarshal(data, &codec) 103 | if err != nil || codec.Code != 0 { 104 | logrus.Errorf("alert failed with err:%v resp:%+v\n", err, string(data)) 105 | } else { 106 | logrus.Infof("alerted success resp:%v", string(data)) 107 | } 108 | } 109 | 110 | /* 111 | * parse content 112 | */ 113 | func (m *Alarm) parse(contents interface{}) (msg string, err error) { 114 | switch contents.(type) { 115 | case []byte: 116 | msg = string(contents.([]byte)) 117 | case string: 118 | msg = contents.(string) 119 | break 120 | case []string: 121 | for _, content := range contents.([]string) { 122 | msg += content + "\n" 123 | } 124 | break 125 | case [][]string: 126 | for _, content := range contents.([][]string) { 127 | msg += strings.Join(content, ",") + "\n" 128 | } 129 | break 130 | default: 131 | err = fmt.Errorf("not support type") 132 | break 133 | } 134 | return msg, err 135 | } 136 | 137 | /* 138 | * get alarm 139 | * WebHookAddr: hook addr 140 | * Title: subject 141 | */ 142 | func AleterGetter(WebHookAddr string, Title string, fns ...func(*Alarm)) *Alarm { 143 | a := &Alarm{ 144 | WebHookAddr: WebHookAddr, 145 | Title: Title, 146 | ShowIp: true, 147 | NoticeAll: true, 148 | CardColor: "default", 149 | FontColor: "default", 150 | } 151 | for _, fn := range fns { 152 | fn(a) 153 | } 154 | return a 155 | } 156 | 157 | // SetShowIp will not valid when use go test -v xxx_test.go, why? 158 | func SetShowIp(showIp bool) func(t *Alarm) { 159 | return func(t *Alarm) { 160 | t.ShowIp = showIp 161 | } 162 | } 163 | 164 | func SetNoticeAll(noticeAll bool) func(t *Alarm) { 165 | return func(t *Alarm) { 166 | t.NoticeAll = noticeAll 167 | } 168 | } 169 | 170 | func SetCardColor(color string) func(t *Alarm) { 171 | return func(t *Alarm) { 172 | t.CardColor = color 173 | } 174 | } 175 | 176 | func SetFontColor(color FontColor) func(t *Alarm) { 177 | return func(t *Alarm) { 178 | t.FontColor = string(color) 179 | } 180 | } 181 | 182 | func (m *Alarm) Warning(contents interface{}) error { 183 | msg, err := m.parse(contents) 184 | if err != nil { 185 | return err 186 | } 187 | m.alert(msg, "alert") 188 | return nil 189 | } 190 | 191 | func (m *Alarm) Notice(contents interface{}) error { 192 | msg, err := m.parse(contents) 193 | if err != nil { 194 | return err 195 | } 196 | m.alert(msg, "info") 197 | return nil 198 | } 199 | -------------------------------------------------------------------------------- /gzip/gzip.go: -------------------------------------------------------------------------------- 1 | package gzip 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "encoding/base64" 7 | "io" 8 | ) 9 | 10 | type _GZip int 11 | 12 | // GZip gzip 13 | var GZip _GZip 14 | 15 | // Compress 16 | func (_GZip) Compress(content string) (string, error) { 17 | var buf bytes.Buffer 18 | zw := gzip.NewWriter(&buf) 19 | 20 | _, err := zw.Write([]byte(content)) 21 | if err != nil { 22 | return content, err 23 | } 24 | 25 | if err = zw.Close(); err != nil { 26 | return content, err 27 | } 28 | 29 | return buf.String(), nil 30 | } 31 | 32 | // UnCompress 33 | func (_GZip) UnCompress(content string) (string, error) { 34 | b := []byte(content) 35 | 36 | var buf = bytes.NewBuffer(b) 37 | zr, err := gzip.NewReader(buf) 38 | if err != nil { 39 | return content, err 40 | } 41 | 42 | var out bytes.Buffer 43 | if _, err = io.Copy(&out, zr); err != nil { 44 | return content, err 45 | } 46 | 47 | if err = zr.Close(); err != nil { 48 | return content, err 49 | } 50 | 51 | return out.String(), nil 52 | } 53 | 54 | // Base64Compress 55 | func (_GZip) Base64Compress(content string) (string, error) { 56 | var buf bytes.Buffer 57 | zw := gzip.NewWriter(&buf) 58 | 59 | _, err := zw.Write([]byte(content)) 60 | if err != nil { 61 | return content, err 62 | } 63 | 64 | if err = zw.Close(); err != nil { 65 | return content, err 66 | } 67 | 68 | return base64.StdEncoding.EncodeToString(buf.Bytes()), nil 69 | } 70 | 71 | // Base64UnCompress 72 | func (_GZip) Base64UnCompress(content string) (string, error) { 73 | b, err := base64.StdEncoding.DecodeString(content) 74 | if err != nil { 75 | return content, err 76 | } 77 | 78 | var buf = bytes.NewBuffer(b) 79 | zr, err := gzip.NewReader(buf) 80 | if err != nil { 81 | return content, err 82 | } 83 | 84 | var out bytes.Buffer 85 | if _, err = io.Copy(&out, zr); err != nil { 86 | return content, err 87 | } 88 | 89 | if err = zr.Close(); err != nil { 90 | return content, err 91 | } 92 | 93 | return out.String(), nil 94 | } 95 | -------------------------------------------------------------------------------- /test-example/config.yaml: -------------------------------------------------------------------------------- 1 | Log: 2 | Level: debug 3 | Path: /Users/ricco/mod/go-utils/test-example 4 | Mode: file 5 | 6 | RRedis: 7 | Addr: localhost:6379 8 | Password: 9 | DB: 0 10 | PoolSize: 100 11 | ReadTimeout: 10000 12 | WriteTimeout: 0 13 | 14 | RWRedis: 15 | Addr: localhost:6379 16 | Password: your_redis_password 17 | DB: 0 18 | PoolSize: 100 19 | ReadTimeout: 10000 20 | WriteTimeout: 20000 21 | 22 | RWMysql: "root:123@tcp(localhost:3306)/mydatabase" 23 | 24 | Mongo: 25 | User: your_mongo_user 26 | Password: your_mongo_password 27 | Addr: 28 | - host1:port1 29 | - host2:port2 30 | Database: your_database_name 31 | PoolSize: your_connection_pool_size 32 | Timeout: your_timeout_duration 33 | 34 | Kafka: localhost:9092,localhost:9093 35 | #Kafka: localhost:9092 36 | -------------------------------------------------------------------------------- /test-example/dbms_mysql_test.go: -------------------------------------------------------------------------------- 1 | package test_example_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aipave/go-utils/dbms/gmysql" 7 | "github.com/aipave/go-utils/ginfos" 8 | "github.com/sirupsen/logrus" 9 | "gorm.io/gorm" 10 | "gorm.io/gorm/logger" 11 | ) 12 | 13 | // 数据库client 14 | var ( 15 | readClient *gorm.DB // readonly 16 | ) 17 | 18 | func InitMysql() { 19 | // user:password@tcp(localhost:3306)/mydb 20 | 21 | var mysqlDSN string = "root:123@tcp(localhost:3306)/mydatabase" 22 | readClient = gmysql.NewMySQLClient(gmysql.WithDSN(mysqlDSN), gmysql.WithLogLevel(logger.Error)) 23 | } 24 | 25 | var MySQL _MySQLMgr 26 | 27 | type _MySQLMgr struct { 28 | } 29 | 30 | // Using orm methods can simplify access and manipulation of databases while improving code readability and maintainability 31 | // If the query condition is a null value when using the orm method, if the is null or is not null condition 32 | // is not used for processing, it will cause uncertainty or randomness in the query results. 33 | 34 | // In gorm, if the query condition is an empty string or zero value, 35 | // the where method will directly ignore the condition and return all the data in the table. 36 | // If you use more than one condition in a query, ignoring the condition may cause non-determinism 37 | // or randomness in the query results. 38 | // 39 | // so the right version is 40 | // if name = "" { 41 | // readClient.Debug.Where("name IS NULL").... 42 | // } else { 43 | // readClient.Debug.Where("name = ?", name).... 44 | // } 45 | 46 | type User struct { 47 | Id int64 `gorm:"column:status"` 48 | Name string `gorm:"column:name"` 49 | } 50 | 51 | func (m *_MySQLMgr) FindIdFromMytable(name string) (user User, err error) { 52 | user.Name = name 53 | rows, err := readClient.Debug().Table("mydatabase.mytable").Select("id"). 54 | Where("name = ?", name).Rows() 55 | if err != nil { 56 | logrus.Errorf("err") 57 | } 58 | for rows.Next() { 59 | err := readClient.ScanRows(rows, &user.Id) 60 | if err != nil { 61 | logrus.Errorf("err") 62 | } 63 | 64 | } 65 | return 66 | } 67 | 68 | // Using orm methods can simplify access and manipulation of databases while improving code readability and maintainability 69 | // If the query condition is a null value when using the orm method, if the is null or is not null condition 70 | // is not used for processing, it will cause uncertainty or randomness in the query results. 71 | func (m *_MySQLMgr) FindNameFromMytable(id int64) (user User, err error) { 72 | rows, err := readClient.Debug().Table("mydatabase.mytable").Select("name"). 73 | Where("id = ?", id).Rows() 74 | if err != nil { 75 | logrus.Errorf("err") 76 | } 77 | for rows.Next() { 78 | err := readClient.ScanRows(rows, &user) 79 | if err != nil { 80 | logrus.Errorf("err") 81 | } 82 | 83 | } 84 | return 85 | } 86 | 87 | // Using orm methods can simplify access and manipulation of databases while improving code readability and maintainability 88 | // If the query condition is a null value when using the orm method, if the is null or is not null condition 89 | // is not used for processing, it will cause uncertainty or randomness in the query results. 90 | func (m *_MySQLMgr) FindNameFromMytable2(id int64) (user User) { 91 | readClient.Debug().Table("mydatabase.mytable").Select("name"). 92 | Where("id = ?", id).Find(&user.Name) 93 | return 94 | } 95 | 96 | // When using native sql statements, 97 | // if the input parameters are not handled correctly, it may cause sql injection problems 98 | func (m *_MySQLMgr) UpdateFromMytable(name string) (err error) { 99 | sqlStr := "update mydatabase.mytable set id = 1234 where name = ?" 100 | var res = &gorm.DB{} 101 | res = readClient.Debug().Exec(sqlStr, name) 102 | if res.Error != nil { 103 | logrus.Errorf("fail: err") 104 | return res.Error 105 | } 106 | logrus.Infof("success %v", res.RowsAffected) 107 | return 108 | } 109 | 110 | func TestInitMysql(t *testing.T) { 111 | InitMysql() 112 | 113 | id, err := MySQL.FindIdFromMytable("lyu") 114 | t.Log(id, err) 115 | logrus.Errorf("ID:%v, %v", id, err) 116 | 117 | t.Logf("---------%v-------", ginfos.FuncName()) 118 | name, err := MySQL.FindNameFromMytable(1234) 119 | t.Log(name, err) 120 | logrus.Errorf("name:%v, %v", name, err) 121 | 122 | t.Logf("---------%v-------", ginfos.FuncName()) 123 | name2 := MySQL.FindNameFromMytable2(0) 124 | t.Log(name2, err) 125 | logrus.Errorf("name:%v, %v", name2, err) 126 | 127 | t.Logf("---------%v-------", ginfos.FuncName()) 128 | err = MySQL.UpdateFromMytable("lyu") 129 | t.Log(err) 130 | } 131 | -------------------------------------------------------------------------------- /test-example/docker_compose_kafka.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | networks: 4 | app-tier: 5 | driver: bridge 6 | external: true 7 | 8 | services: 9 | zookeeper-01: 10 | user: root 11 | image: 'bitnami/zookeeper:latest' 12 | container_name: zookeeper-01 13 | restart: always 14 | hostname: zookeeper-01 15 | ports: 16 | - "2181:2181" 17 | networks: 18 | - app-tier 19 | environment: 20 | - ZOO_MY_ID=1 21 | - ALLOW_ANONYMOUS_LOGIN=yes 22 | - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-01:2181 23 | - ZOOKEEPER_CLIENT_PORT=zookeeper-01:2181 24 | - ZOO_SERVER=server.1=zookeeper-01:2888:3888;2181 server.2=zookeeper-02:2888:3888;2181 server.3=zookeeper-03:2888:3888;2181 25 | zookeeper-02: 26 | user: root 27 | image: 'bitnami/zookeeper:latest' 28 | container_name: zookeeper-02 29 | restart: always 30 | hostname: zookeeper-02 31 | ports: 32 | - "2182:2181" 33 | networks: 34 | - app-tier 35 | environment: 36 | - ZOO_MY_ID=2 37 | - ALLOW_ANONYMOUS_LOGIN=yes 38 | - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-02:2182 39 | - ZOOKEEPER_CLIENT_PORT=zookeeper-02:2182 40 | - ZOO_SERVER=server.1=zookeeper-01:2888:3888;2181 server.2=zookeeper-02:2888:3888;2181 server.3=zookeeper-03:2888:3888;2181 41 | zookeeper-03: 42 | user: root 43 | image: 'bitnami/zookeeper:latest' 44 | container_name: zookeeper-03 45 | restart: always 46 | hostname: zookeeper-03 47 | ports: 48 | - "2183:2181" 49 | networks: 50 | - app-tier 51 | environment: 52 | - ZOO_MY_ID=3 53 | - ALLOW_ANONYMOUS_LOGIN=yes 54 | - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-03:2183 55 | - ZOOKEEPER_CLIENT_PORT=zookeeper-03:2183 56 | - ZOO_SERVER=server.1=zookeeper-01:2888:3888;2181 server.2=zookeeper-02:2888:3888;2181 server.3=zookeeper-03:2888:3888;2181 57 | kafka-01: 58 | user: root 59 | restart: always 60 | container_name: kafka-01 61 | hostname: kafka-01 62 | image: 'bitnami/kafka:latest' 63 | ports: 64 | - '9092:9092' 65 | - '9093:9093' 66 | networks: 67 | - app-tier 68 | environment: 69 | - KAFKA_BROKER_ID=1 70 | - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT 71 | - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT 72 | # If you want to connect to the kafka cluster through the host, you need to add kafka cfg 73 | # listeners and kafka cfg advertised listeners configuration and kafka cfg advertised listeners s configuration address is the corresponding ip of the host, kafka cfg advertised listeners is the listening address exposed to the client 74 | - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://localhost:9092,EXTERNAL://localhost:9093 # an advertised listener named "CLIENT" that clients should use to communicate with the Kafka broker over the localhost interface on port 9093. This listener uses the PLAINTEXT security protocol. 75 | # an advertised listener named "EXTERNAL" that clients should use to communicate with the Kafka broker from outside the Docker network over the localhost interface on port 9093. This listener also uses the PLAINTEXT security protocol. 76 | # The Client port number 9093 is used because it is the port that is mapped to the Kafka broker's internal port 9092 using the ports directive in the Docker Compose file. 77 | - KAFKA_CFG_LISTENERS=CLIENT://:9092,EXTERNAL://:9093 # The CLIENT listener is used for internal communication between brokers; 78 | # The EXTERNAL listener is typically used for communication with clients from outside the Docker network. 79 | - ALLOW_PLAINTEXT_LISTENER=yes 80 | - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-01:2181,zookeeper-02:2182,zookeeper-03:2183/kafka 81 | - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true 82 | 83 | #- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 84 | #- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092 85 | 86 | #> not support to KAFKA_CREATE_TOPICS 87 | #- KAFKA_CREATE_TOPICS="game-center:1:2,sign-server:1:1" # is the name of the topic you want to create. 88 | # is the number of partitions you want to create for the topic. 89 | # is the replication factor you want to use for the topic 90 | depends_on: 91 | - zookeeper-01 92 | - zookeeper-02 93 | - zookeeper-03 94 | #kafka-manager: 95 | # image: "sheepkiller/kafka-manager" 96 | # depends_on: 97 | # - zookeeper-01 98 | # ports: 99 | # - "9190:9000" 100 | # environment: 101 | # - ZK_HOSTS=zookeeper-01:2181 102 | -------------------------------------------------------------------------------- /test-example/docker_compose_kafka_cluster.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | networks: 4 | app-tier: 5 | driver: bridge 6 | external: true 7 | 8 | services: 9 | # When stopping the kafka cluster, be sure to wait until all node processes of kafka are stopped before stopping the zookeeper cluster. Because the kafka cluster related information is recorded in the zookeeper cluster, once the zookeeper cluster is stopped first, 10 | # the kafka cluster has no way to obtain the information of the stopped process, and can only manually kill the kafka process 11 | zookeeper-01: 12 | user: root 13 | image: 'bitnami/zookeeper:latest' 14 | container_name: zookeeper-01 15 | restart: always 16 | hostname: zookeeper-01 17 | ports: 18 | - "2181:2181" 19 | networks: 20 | - app-tier 21 | environment: 22 | - ZOO_MY_ID=1 23 | - ALLOW_ANONYMOUS_LOGIN=yes 24 | - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-01:2181 25 | - ZOOKEEPER_CLIENT_PORT=zookeeper-01:2181 26 | - ZOO_SERVER=server.1=zookeeper-01:2888:3888;2181 server.2=zookeeper-02:2888:3888;2181 server.3=zookeeper-03:2888:3888;2181 27 | zookeeper-02: 28 | user: root 29 | image: 'bitnami/zookeeper:latest' 30 | container_name: zookeeper-02 31 | restart: always 32 | hostname: zookeeper-02 33 | ports: 34 | - "2182:2181" 35 | networks: 36 | - app-tier 37 | environment: 38 | - ZOO_MY_ID=2 39 | - ALLOW_ANONYMOUS_LOGIN=yes 40 | - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-02:2182 41 | - ZOOKEEPER_CLIENT_PORT=zookeeper-02:2182 42 | - ZOO_SERVER=server.1=zookeeper-01:2888:3888;2181 server.2=zookeeper-02:2888:3888;2181 server.3=zookeeper-03:2888:3888;2181 43 | zookeeper-03: 44 | user: root 45 | image: 'bitnami/zookeeper:latest' 46 | container_name: zookeeper-03 47 | restart: always 48 | hostname: zookeeper-03 49 | ports: 50 | - "2183:2181" 51 | networks: 52 | - app-tier 53 | environment: 54 | - ZOO_MY_ID=3 55 | - ALLOW_ANONYMOUS_LOGIN=yes 56 | - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-03:2183 57 | - ZOOKEEPER_CLIENT_PORT=zookeeper-03:2183 58 | - ZOO_SERVER=server.1=zookeeper-01:2888:3888;2181 server.2=zookeeper-02:2888:3888;2181 server.3=zookeeper-03:2888:3888;2181 59 | kafka-01: 60 | user: root 61 | restart: always 62 | container_name: kafka-01 63 | hostname: kafka-01 64 | image: 'bitnami/kafka:latest' 65 | ports: 66 | - '9092:9092' 67 | networks: 68 | - app-tier 69 | environment: 70 | - KAFKA_BROKER_ID=1 71 | - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 72 | - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092 73 | - ALLOW_PLAINTEXT_LISTENER=yes 74 | - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-01:2181,zookeeper-02:2182,zookeeper-03:2183/kafka 75 | - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true 76 | depends_on: 77 | - zookeeper-01 78 | - zookeeper-02 79 | - zookeeper-03 80 | kafka-02: 81 | user: root 82 | restart: always 83 | container_name: kafka-02 84 | hostname: kafka-02 85 | image: 'bitnami/kafka:latest' 86 | ports: 87 | - '9093:9093' 88 | networks: 89 | - app-tier 90 | environment: 91 | - KAFKA_BROKER_ID=2 92 | - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9093 93 | - KAFKA_CFG_LISTENERS=PLAINTEXT://:9093 94 | - ALLOW_PLAINTEXT_LISTENER=yes 95 | - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-01:2181,zookeeper-02:2182,zookeeper-03:2183/kafka 96 | - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true 97 | depends_on: 98 | - zookeeper-01 99 | - zookeeper-02 100 | - zookeeper-03 101 | 102 | #$ docker-compose -f docker_compose_kafka_cluster.yaml up -d 103 | 104 | #$ docker exec -it kafka-01 /bin/bash 105 | 106 | # cd /opt/bitnami/kafka/bin 107 | 108 | -------------------------------------------------------------------------------- /test-example/game_ludo_test.go: -------------------------------------------------------------------------------- 1 | package test_example 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | type stateFn func() stateFn 9 | 10 | type LudoGame struct { 11 | board [15]int 12 | players []*LudoPlayer 13 | currentTurn int 14 | } 15 | 16 | type LudoPlayer struct { 17 | pieces []*LudoPiece /// In the NewLudoGame function, you should initialize each player's pieces slice with the appropriate length. 18 | color string 19 | canMove bool 20 | hasWon bool 21 | startTile int 22 | } 23 | 24 | type LudoPiece struct { 25 | tile int 26 | player *LudoPlayer 27 | } 28 | 29 | func NewLudoGame() *LudoGame { 30 | game := &LudoGame{} 31 | /// In the game of Ludo, players start their pieces from a designated start tile 32 | // on the board. The start tile is usually marked with a colored circle or other 33 | // symbol to indicate which player's pieces start from there. 34 | /// When a player's turn begins, they roll a die or spin a spinner to determine(他们掷骰子或旋转转轮来确定) 35 | // how many spaces to move their piece(s) from the start tile onto the main part of the board(棋盘). 36 | game.players = []*LudoPlayer{ 37 | &LudoPlayer{ 38 | color: "red", 39 | startTile: 0, 40 | pieces: []*LudoPiece{{tile: 0}}, //pieces: make([]*LudoPiece, 4), 41 | }, 42 | &LudoPlayer{ 43 | color: "blue", 44 | startTile: 13, 45 | pieces: []*LudoPiece{{tile: 13}}, //pieces: make([]*LudoPiece, 4), 46 | 47 | }, 48 | &LudoPlayer{ 49 | color: "yellow", 50 | startTile: 26, 51 | pieces: []*LudoPiece{{tile: 26}}, //pieces: make([]*LudoPiece, 4), 52 | }, 53 | &LudoPlayer{ 54 | color: "green", 55 | startTile: 39, 56 | pieces: []*LudoPiece{{tile: 39}}, //pieces: make([]*LudoPiece, 4), 57 | }, 58 | } 59 | game.currentTurn = 0 60 | for i := 0; i < len(game.board); i++ { 61 | game.board[i] = -1 62 | } 63 | return game 64 | } 65 | 66 | func (game *LudoGame) StartState() stateFn { 67 | fmt.Println("Starting game") 68 | game.players[0].canMove = true 69 | return game.NextState 70 | } 71 | 72 | func (game *LudoGame) NextState() stateFn { 73 | player := game.players[game.currentTurn] 74 | if player.hasWon { 75 | return game.EndState 76 | } 77 | if !player.canMove { 78 | game.currentTurn = (game.currentTurn + 1) % len(game.players) 79 | game.players[game.currentTurn].canMove = true 80 | } 81 | fmt.Printf("%s's turn\n", player.color) 82 | return game.WaitForInputState 83 | } 84 | 85 | func (game *LudoGame) WaitForInputState() stateFn { 86 | // wait for user input here 87 | return game.MovePieceState 88 | } 89 | 90 | func (game *LudoGame) MovePieceState() stateFn { 91 | player := game.players[game.currentTurn] 92 | piece := player.pieces[0] 93 | piece.tile = (piece.tile + 1) % len(game.board) 94 | game.board[piece.tile] = game.currentTurn 95 | fmt.Printf("%s moved to tile %d\n", player.color, piece.tile) 96 | if piece.tile == player.startTile { 97 | player.hasWon = true 98 | player.canMove = false 99 | } else { 100 | player.canMove = false 101 | } 102 | return game.NextState 103 | } 104 | 105 | func (game *LudoGame) EndState() stateFn { 106 | fmt.Println("Game over") 107 | return nil 108 | } 109 | 110 | func (game *LudoGame) Run() { 111 | state := game.StartState 112 | for state != nil { 113 | state = state() 114 | } 115 | } 116 | 117 | func TestLudoGame(t *testing.T) { 118 | game := NewLudoGame() 119 | game.Run() 120 | } 121 | -------------------------------------------------------------------------------- /test-example/gcache_test.go: -------------------------------------------------------------------------------- 1 | package test_example 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | gcache "github.com/aipave/go-utils/gcache" 8 | ) 9 | 10 | func TestGcache(t *testing.T) { 11 | gcache.Set("foo", "bar", 5) 12 | v, ok := gcache.Get("foo") 13 | t.Log(v, ok) 14 | 15 | mocks := []struct { 16 | key string 17 | val string 18 | ttl int64 19 | }{ 20 | {"A", "a", 1000}, 21 | {"B", "b", 1000}, 22 | {"", "null", 1000}, 23 | } 24 | 25 | for _, mock := range mocks { 26 | gcache.Set(mock.key, mock.val, mock.ttl) 27 | } 28 | 29 | time.Sleep(1) 30 | 31 | for _, mock := range mocks { 32 | val, found := gcache.Get(mock.key) 33 | if !found || val.(string) != mock.val { 34 | t.Fatalf("Unexpected value: %v (%v) to key: %v", val, found, mock.key) 35 | } 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /test-example/gcast_test.go: -------------------------------------------------------------------------------- 1 | package test_example 2 | 3 | import ( 4 | "testing" 5 | 6 | gcast "github.com/aipave/go-utils/gcast" 7 | ) 8 | 9 | func TestGcast(t *testing.T) { 10 | t.Log(gcast.ToBool("0")) 11 | t.Log(gcast.ToBool(1)) 12 | t.Log(gcast.ToString("11")) 13 | t.Log(gcast.ToInt32("12")) 14 | t.Log(gcast.ToUint64("12.000")) 15 | } 16 | -------------------------------------------------------------------------------- /test-example/gerr_test.go: -------------------------------------------------------------------------------- 1 | package test_example 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | 8 | "github.com/aipave/go-utils/gerr" 9 | "github.com/aipave/go-utils/ginfos" 10 | ) 11 | 12 | func TestGerr(t *testing.T) { 13 | var ErrCodeNotImpl int32 = 501 14 | ErrCodeNotImplMsg := "The server does not support the requested feature and cannot complete the request" 15 | err := gerr.New(ErrCodeNotImpl, ErrCodeNotImplMsg) 16 | t.Log(err.Error()) 17 | t.Log(gerr.Append(err, gerr.New(ErrCodeNotImpl, ErrCodeNotImplMsg))) 18 | 19 | } 20 | 21 | func TestGerrCustom(t *testing.T) { 22 | //mode.SetMode("test") 23 | 24 | gerr.Init("test", "", "topic") 25 | 26 | testErrSys(t) 27 | t.Log("--------------------------------------------------------------------------------------") 28 | testErrInvalidReq(t) 29 | t.Log("--------------------------------------------------------------------------------------") 30 | testCaseErrSys(t) 31 | t.Log("--------------------------------------------------------------------------------------") 32 | testCaseErrGameOn(t) 33 | t.Log("--------------------------------------------------------------------------------------") 34 | testCustomizeSysErr(t) 35 | t.Log("--------------------------------------------------------------------------------------") 36 | testCustomizeNorErr(t) 37 | 38 | gerr.WarnErrInfo("hhhhhhhhhhhhhhhhhhhhhhh", ginfos.FuncName()) ///< ??? only receive msg when have this line 39 | } 40 | 41 | func testErrSys(t *testing.T) { 42 | err := gerr.ErrSys 43 | c, d := gerr.HandleError("test ErrSys1", "zh-tw", time.Now(), err) 44 | fmt.Printf("test ErrSys1: code:{%d} desc:{%s}\n", c, d) 45 | 46 | c, d = gerr.HandleError("test ErrSys2", "", time.Now(), err) 47 | fmt.Printf("test ErrSys2: code:{%d} desc:{%s}\n", c, d) 48 | } 49 | 50 | func testErrInvalidReq(t *testing.T) { 51 | err := gerr.ErrPermissionDenied 52 | c, d := gerr.HandleError("test ErrInvalidReq1", "zh-tw", time.Now(), err) 53 | fmt.Printf("test ErrInvalidReq1: code:{%d} desc:{%s}\n", c, d) 54 | 55 | c, d = gerr.HandleError("test ErrInvalidReq2", "", time.Now(), err) 56 | fmt.Printf("test ErrInvalidReq2: code:{%d} desc:{%s}\n", c, d) 57 | } 58 | 59 | func testCaseErrSys(t *testing.T) { 60 | err := gerr.CaseErr(gerr.ErrSys, "test case err base on sys err") 61 | c, d := gerr.HandleError("test CaseErr ErrSys1", "zh-tw", time.Now(), err) 62 | fmt.Printf("test CaseErr ErrSys1: code:{%d} desc:{%s}\n", c, d) 63 | 64 | c, d = gerr.HandleError("test CaseErr ErrSys2", "", time.Now(), err) 65 | fmt.Printf("test CaseErr ErrSys2: code:{%d} desc:{%s}\n", c, d) 66 | } 67 | 68 | func testCaseErrGameOn(t *testing.T) { 69 | err := gerr.CaseErr(gerr.ErrVersion, "test case err base on game on err") 70 | c, d := gerr.HandleError("test CaseErr", "zh-tw", time.Now(), err) 71 | fmt.Printf("test CaseErr : code:{%d} desc:{%s}\n", c, d) 72 | 73 | c, d = gerr.HandleError("test CaseErr ", "", time.Now(), err) 74 | fmt.Printf("test CaseErr : code:{%d} desc:{%s}\n", c, d) 75 | } 76 | 77 | func testCustomizeSysErr(t *testing.T) { 78 | err := gerr.CustomizeSysErr(gerr.RetCode_kInvalidReq, "CustomizeSysErr", "CustomizeSysErr customizeInfo") 79 | c, d := gerr.HandleError("test CustomizeSysErr", "zh-tw", time.Now(), err) 80 | fmt.Printf("test CustomizeSysErr: code:{%d} desc:{%s}\n", c, d) 81 | 82 | c, d = gerr.HandleError("test CustomizeSysErr2", "", time.Now(), err) 83 | fmt.Printf("test CustomizeSysErr2: code:{%d} desc:{%s}\n", c, d) 84 | } 85 | 86 | func testCustomizeNorErr(t *testing.T) { 87 | err := gerr.CustomizeNorErr(gerr.ErrSys.Code(), gerr.ERR_LEVEL_SYS, "CustomizeNorErr", "", "CustomizeNorErr customizeInfo") 88 | c, d := gerr.HandleError("test CustomizeNorErr1", "zh-tw", time.Now(), err) 89 | fmt.Printf("test CustomizeNorErr1: code:{%d} desc:{%s}\n", c, d) 90 | 91 | c, d = gerr.HandleError("test CustomizeNorErr2", "", time.Now(), err) 92 | fmt.Printf("test CustomizeNorErr2: code:{%d} desc:{%s}\n", c, d) 93 | 94 | err = gerr.CustomizeNorErr(100, 100, "CustomizeNorErr", "", "CustomizeNorErr customizeInfo") 95 | c, d = gerr.HandleError("test CustomizeNorErr3", "zh-tw", time.Now(), err) 96 | fmt.Printf("test CustomizeNorErr3: code:{%d} desc:{%s}\n", c, d) 97 | } 98 | -------------------------------------------------------------------------------- /test-example/gexit_test.go: -------------------------------------------------------------------------------- 1 | package test_example 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/aipave/go-utils/gexit" 9 | ) 10 | 11 | type Test struct { 12 | log *testing.T 13 | } 14 | 15 | func (t *Test) sayHello() { 16 | t.log.Logf("Hello world!") 17 | } 18 | 19 | func (t *Test) handleFoo(ctx context.Context) { 20 | curTimer := time.NewTicker(1 * time.Second) 21 | for { 22 | select { 23 | case <-ctx.Done(): 24 | t.log.Log("timer close") 25 | case <-curTimer.C: 26 | t.sayHello() 27 | } 28 | } 29 | } 30 | 31 | // received term signal, process will exit after 3 seconds 32 | func (t *Test) testGraceExitNoBreakLoop() { 33 | ctx, cancel := context.WithCancel(context.Background()) 34 | 35 | go t.handleFoo(ctx) 36 | 37 | gexit.Close(cancel) 38 | 39 | } 40 | 41 | func sayHello(t *testing.T) { 42 | t.Log("hello world!") 43 | } 44 | 45 | func testGraceExitAddBreakLoop(t *testing.T) { 46 | timeTick := time.Tick(3 * time.Second) 47 | ctx, cancel := context.WithCancel(context.Background()) 48 | gexit.Close(cancel) 49 | 50 | ///> add loop, break loop 51 | Loop: 52 | for { 53 | select { 54 | case <-timeTick: 55 | sayHello(t) 56 | case <-ctx.Done(): 57 | t.Log("progress received kill, abort.") 58 | break Loop 59 | 60 | } 61 | } 62 | 63 | } 64 | 65 | func TestGracefulExit(tt *testing.T) { 66 | t := &Test{ 67 | log: tt, 68 | } 69 | t.testGraceExitNoBreakLoop() 70 | 71 | gexit.Wait() //block 72 | } 73 | 74 | func TestGracefulTimeCrontab(t *testing.T) { 75 | testGraceExitAddBreakLoop(t) 76 | 77 | gexit.Wait() //block 78 | } 79 | 80 | //type Dao struct { 81 | // closeChan chan struct{} 82 | //} 83 | //var dao *Dao 84 | // 85 | //func test-example() { 86 | // dao = &Dao{ 87 | // closeChan: make(chan struct{}), 88 | // } 89 | // 90 | // gexit.Close(func() { 91 | // dao.closeChan <- struct { 92 | // }{} 93 | // }) 94 | // 95 | //} 96 | -------------------------------------------------------------------------------- /test-example/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/aipave/go-utils/example 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/Shopify/sarama v1.29.1 7 | github.com/aipave/go-utils v0.0.75 8 | github.com/go-redis/redis/v8 v8.3.3 9 | github.com/hashicorp/go-uuid v1.0.2 10 | github.com/segmentio/kafka-go v0.4.39 11 | github.com/sirupsen/logrus v1.9.0 12 | go.mongodb.org/mongo-driver v1.11.2 13 | golang.org/x/net v0.6.0 14 | gopkg.in/yaml.v3 v3.0.1 15 | gorm.io/gorm v1.24.6 16 | ) 17 | 18 | require ( 19 | github.com/RussellLuo/timingwheel v0.0.0-20220218152713-54845bda3108 // indirect 20 | github.com/antonfisher/nested-logrus-formatter v1.3.1 // indirect 21 | github.com/beorn7/perks v1.0.1 // indirect 22 | github.com/bwmarrin/snowflake v0.3.0 // indirect 23 | github.com/cespare/xxhash v1.1.0 // indirect 24 | github.com/cespare/xxhash/v2 v2.1.1 // indirect 25 | github.com/davecgh/go-spew v1.1.1 // indirect 26 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 27 | github.com/eapache/go-resiliency v1.2.0 // indirect 28 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect 29 | github.com/eapache/queue v1.1.0 // indirect 30 | github.com/ebitengine/purego v0.3.0 // indirect 31 | github.com/fsnotify/fsnotify v1.6.0 // indirect 32 | github.com/globocom/go-redis-prometheus v0.4.0 // indirect 33 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20221017161538-93cebf72946b // indirect 34 | github.com/go-sql-driver/mysql v1.7.0 // indirect 35 | github.com/golang/protobuf v1.4.3 // indirect 36 | github.com/golang/snappy v0.0.3 // indirect 37 | github.com/hajimehoshi/ebiten/v2 v2.5.0 // indirect 38 | github.com/jcmturner/aescts/v2 v2.0.0 // indirect 39 | github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect 40 | github.com/jcmturner/gofork v1.0.0 // indirect 41 | github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect 42 | github.com/jcmturner/rpc/v2 v2.0.3 // indirect 43 | github.com/jezek/xgb v1.1.0 // indirect 44 | github.com/jinzhu/inflection v1.0.0 // indirect 45 | github.com/jinzhu/now v1.1.5 // indirect 46 | github.com/klauspost/compress v1.15.9 // indirect 47 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 48 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect 49 | github.com/natefinch/lumberjack v2.0.0+incompatible // indirect 50 | github.com/pierrec/lz4 v2.6.0+incompatible // indirect 51 | github.com/pierrec/lz4/v4 v4.1.15 // indirect 52 | github.com/pkg/errors v0.9.1 // indirect 53 | github.com/prometheus/client_golang v1.8.0 // indirect 54 | github.com/prometheus/client_model v0.2.0 // indirect 55 | github.com/prometheus/common v0.14.0 // indirect 56 | github.com/prometheus/procfs v0.2.0 // indirect 57 | github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect 58 | github.com/robfig/cron v1.2.0 // indirect 59 | github.com/robfig/cron/v3 v3.0.1 // indirect 60 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 61 | github.com/xdg-go/scram v1.1.1 // indirect 62 | github.com/xdg-go/stringprep v1.0.3 // indirect 63 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect 64 | go.opentelemetry.io/otel v0.13.0 // indirect 65 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect 66 | golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56 // indirect 67 | golang.org/x/image v0.6.0 // indirect 68 | golang.org/x/mobile v0.0.0-20230301163155-e0f57694e12c // indirect 69 | golang.org/x/sync v0.1.0 // indirect 70 | golang.org/x/sys v0.6.0 // indirect 71 | golang.org/x/text v0.8.0 // indirect 72 | google.golang.org/protobuf v1.23.0 // indirect 73 | gorm.io/driver/mysql v1.4.7 // indirect 74 | ) 75 | -------------------------------------------------------------------------------- /test-example/gstl_test.go: -------------------------------------------------------------------------------- 1 | package test_example 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aipave/go-utils/gstl/vector" 7 | ) 8 | 9 | func TestGSTL(t *testing.T) { 10 | s := []int{1, 2, 3} 11 | idx := vector.New(s).Find(1) 12 | if idx < 0 { 13 | t.Fatal("not exits") 14 | } 15 | t.Logf("exits") 16 | 17 | } 18 | -------------------------------------------------------------------------------- /test-example/gtoken_test.go: -------------------------------------------------------------------------------- 1 | package test_example 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aipave/go-utils/gcast" 7 | "github.com/aipave/go-utils/gtoken" 8 | ) 9 | 10 | func TestGtoken(t *testing.T) { 11 | var usrid uint64 = 123456 12 | token, err := gtoken.CreateToken(usrid) 13 | if err != nil { 14 | t.Fatal(err) 15 | } 16 | t.Logf("token:%v", token) 17 | 18 | err = gtoken.VerifyToken(gcast.ToUint64(123456), token) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | t.Logf("valid token:%v", token) 23 | } 24 | -------------------------------------------------------------------------------- /test-example/gwarn_test.go: -------------------------------------------------------------------------------- 1 | package test_example 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | 8 | "github.com/aipave/go-utils/ginfos" 9 | "github.com/aipave/go-utils/gwarn" 10 | ) 11 | 12 | func TestGwarn(t *testing.T) { 13 | url := "https://open.feishu.cn/open-apis/bot/v2/hook/xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx" 14 | 15 | titleN := "Notice Topic" 16 | // theSetShowIpOnTheRealMachineIsValid 17 | gwarn.AleterGetter(url, titleN, gwarn.SetShowIp(true), gwarn.SetCardColor("grey"), 18 | gwarn.SetFontColor(gwarn.FontColorRed)). 19 | Notice(fmt.Sprintf("area: ID\ntime: %v \nQ: hello world", time.Now().Unix())) 20 | 21 | titleW := "Warning Topic" 22 | gwarn.AleterGetter(url, titleW, gwarn.SetNoticeAll(false), gwarn.SetShowIp(true), gwarn.SetCardColor("blue"), 23 | gwarn.SetFontColor(gwarn.FontColorGrey)). 24 | Warning(fmt.Sprintf("area: ID\ntime: %v \nQ: hello world\nIP:%v", time.Now().Unix(), ginfos.Runtime.IP())) 25 | } 26 | -------------------------------------------------------------------------------- /test-example/hexdeximal_test.go: -------------------------------------------------------------------------------- 1 | package test_example 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "testing" 7 | ) 8 | 9 | type Log struct { 10 | t *testing.T 11 | } 12 | 13 | func (l Log) translateHexadecimal(m int, n int, input string) string { 14 | // to 10 hex 15 | ans := 0 16 | radix := 0 17 | for idx := len(input) - 1; idx >= 0; idx-- { 18 | cur := input[idx] 19 | if '0' <= cur && cur < '9' { 20 | // ⭐️⭐️⭐️ A + 32 = a 21 | ans += int(cur-'0') * int(math.Pow(float64(m), float64(radix))) 22 | } else { 23 | ans += int(cur-'A'+10) * int(math.Pow(float64(m), float64(radix))) 24 | } 25 | radix++ 26 | } 27 | //l.t.Log(ans, m, n) 28 | // to n 29 | radix = 0 30 | result := "" 31 | for ans > 0 { 32 | res := ans % n 33 | // ⭐️ 34 | if res > 9 { 35 | // %v is used to format values using the default format, which may not be appropriate for formatting an integer value as a character. 36 | result = fmt.Sprintf("%c%s", 'A'+res-10, result) // same as += 37 | } else { 38 | result = fmt.Sprintf("%d%s", res, result) // same as += 39 | } 40 | ans /= n 41 | } 42 | 43 | return result 44 | } 45 | 46 | func TestHexDeximal001(t *testing.T) { 47 | var l Log = Log{t: t} 48 | t.Log(l.translateHexadecimal(16, 8, "7B")) 49 | t.Log(l.translateHexadecimal(8, 16, "173")) 50 | } 51 | 52 | func TestTranslateHexadecimal(t *testing.T) { 53 | testCases := []struct { 54 | m int 55 | n int 56 | input string 57 | expected string 58 | }{ 59 | {16, 10, "3E8", "1000"}, 60 | {10, 2, "11", "1011"}, 61 | {15, 35, "ABCDEF", "8GNHV"}, 62 | {35, 2, "1WZ", "110111001"}, 63 | } 64 | 65 | var l Log = Log{t: t} 66 | for _, tc := range testCases { 67 | t.Run(fmt.Sprintf("%d_%d_%s", tc.m, tc.n, tc.input), func(t *testing.T) { 68 | result := l.translateHexadecimal(tc.m, tc.n, tc.input) 69 | if result != tc.expected { 70 | t.Errorf("expected %s, but got %s", tc.expected, result) 71 | } 72 | }) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /test-example/vedio_test.go: -------------------------------------------------------------------------------- 1 | package test_example 2 | 3 | // go list golang.org/x/image/... 4 | 5 | //func TestVideo(t *testing.T) { 6 | // // Open the video file 7 | // f, err := os.Open("video.mp4") 8 | // if err != nil { 9 | // fmt.Println("Error:", err) 10 | // return 11 | // } 12 | // defer f.Close() 13 | // 14 | // // Create a video player 15 | // player, err := ebiten.NewVideoPlayerFromFile(f) 16 | // if err != nil { 17 | // fmt.Println("Error:", err) 18 | // return 19 | // } 20 | // defer player.Close() 21 | // 22 | // // Get the video configuration 23 | // config := player.VideoInfo() 24 | // 25 | // // Print some information about the video 26 | // fmt.Printf("Format: %s\n", config.Format()) 27 | // fmt.Printf("Width: %d\n", config.Width()) 28 | // fmt.Printf("Height: %d\n", config.Height()) 29 | // fmt.Printf("Frame rate: %f\n", config.FPS()) 30 | // 31 | // // Decode and resize the first frame 32 | // frame, err := player.Frame() 33 | // if err != nil { 34 | // fmt.Println("Error:", err) 35 | // return 36 | // } 37 | // dst := image.NewRGBA(image.Rect(0, 0, config.Width()/2, config.Height()/2)) 38 | // graphics.Scale(dst, frame, dst.Bounds(), graphics.Linear) 39 | // 40 | // // Save the resized frame to a file 41 | // out, err := os.Create("frame.png") 42 | // if err != nil { 43 | // fmt.Println("Error:", err) 44 | // return 45 | // } 46 | // defer out.Close() 47 | // png.Encode(out, dst) 48 | //} 49 | --------------------------------------------------------------------------------