├── .gitattributes ├── .github └── workflows │ ├── go.yml │ └── reviewdog.yml ├── .gitignore ├── LICENSE ├── dq ├── config.go ├── connection.go ├── consumer.go ├── consumernode.go ├── producer.go ├── producernode.go ├── vars.go └── wrapper.go ├── example ├── dq │ ├── consumer │ │ └── consumer.go │ └── producer │ │ ├── cluster │ │ └── cluster.go │ │ └── node │ │ └── producer.go ├── kq │ ├── consumer │ │ ├── config.yaml │ │ └── queue.go │ └── producer │ │ └── produce.go ├── natsq │ ├── consumer │ │ └── consumer.go │ └── publisher │ │ └── publisher.go ├── rabbitmq │ ├── admin │ │ └── admin.go │ ├── listener │ │ ├── config │ │ │ └── config.go │ │ ├── listener.yaml │ │ └── main.go │ └── sender │ │ └── main.go └── stan │ ├── comsumer │ └── consumer.go │ └── publisher │ └── producer.go ├── go.mod ├── go.sum ├── kq ├── config.go ├── internal │ ├── message.go │ ├── message_test.go │ ├── trace.go │ └── trace_test.go ├── pusher.go ├── pusher_test.go ├── queue.go └── queue_test.go ├── natsq ├── config.go ├── consumer.go └── producer.go ├── rabbitmq ├── config.go ├── listener.go ├── rabbitmqadmin.go └── sender.go ├── readme.md └── stanq ├── config.go ├── consumer.go └── producer.go /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf 2 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.x 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: ^1.13 20 | id: go 21 | 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v2 24 | 25 | - name: Get dependencies 26 | run: | 27 | go get -v -t -d ./... 28 | 29 | - name: Test 30 | run: go test -race -coverprofile=coverage.txt -covermode=atomic ./... 31 | 32 | - name: Codecov 33 | uses: codecov/codecov-action@v1.0.6 34 | with: 35 | token: ${{secrets.CODECOV_TOKEN}} 36 | -------------------------------------------------------------------------------- /.github/workflows/reviewdog.yml: -------------------------------------------------------------------------------- 1 | name: reviewdog 2 | on: [pull_request] 3 | jobs: 4 | staticcheck: 5 | name: runner / staticcheck 6 | runs-on: ubuntu-latest 7 | steps: 8 | - uses: actions/checkout@v2 9 | - uses: reviewdog/action-staticcheck@v1 10 | with: 11 | github_token: ${{ secrets.github_token }} 12 | # Change reviewdog reporter if you need [github-pr-check,github-check,github-pr-review]. 13 | reporter: github-pr-review 14 | # Report all results. 15 | filter_mode: nofilter 16 | # Exit with 1 when it find at least one finding. 17 | fail_on_error: true 18 | # Set staticcheck flags 19 | staticcheck_flags: -checks=inherit,-SA5008 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore all 2 | * 3 | 4 | # Unignore all with extensions 5 | !*.* 6 | !**/Dockerfile 7 | 8 | # Unignore all dirs 9 | !*/ 10 | !api 11 | 12 | .idea 13 | **/.DS_Store 14 | **/logs 15 | !Makefile 16 | 17 | # gitlab ci 18 | .cache 19 | 20 | # vim auto backup file 21 | *~ 22 | !OWNERS 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 xiaoheiban_server_go 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /dq/config.go: -------------------------------------------------------------------------------- 1 | package dq 2 | 3 | import "github.com/zeromicro/go-zero/core/stores/redis" 4 | 5 | type ( 6 | Beanstalk struct { 7 | Endpoint string 8 | Tube string 9 | } 10 | 11 | DqConf struct { 12 | Beanstalks []Beanstalk 13 | Redis redis.RedisConf 14 | } 15 | ) 16 | -------------------------------------------------------------------------------- /dq/connection.go: -------------------------------------------------------------------------------- 1 | package dq 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/beanstalkd/go-beanstalk" 7 | ) 8 | 9 | type connection struct { 10 | lock sync.RWMutex 11 | endpoint string 12 | tube string 13 | conn *beanstalk.Conn 14 | } 15 | 16 | func newConnection(endpint, tube string) *connection { 17 | return &connection{ 18 | endpoint: endpint, 19 | tube: tube, 20 | } 21 | } 22 | 23 | func (c *connection) Close() error { 24 | c.lock.Lock() 25 | conn := c.conn 26 | c.conn = nil 27 | defer c.lock.Unlock() 28 | 29 | if conn != nil { 30 | return conn.Close() 31 | } 32 | 33 | return nil 34 | } 35 | 36 | func (c *connection) get() (*beanstalk.Conn, error) { 37 | c.lock.RLock() 38 | conn := c.conn 39 | c.lock.RUnlock() 40 | if conn != nil { 41 | return conn, nil 42 | } 43 | 44 | c.lock.Lock() 45 | defer c.lock.Unlock() 46 | 47 | var err error 48 | c.conn, err = beanstalk.Dial("tcp", c.endpoint) 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | c.conn.Tube.Name = c.tube 54 | return c.conn, err 55 | } 56 | 57 | func (c *connection) reset() { 58 | c.lock.Lock() 59 | defer c.lock.Unlock() 60 | 61 | if c.conn != nil { 62 | c.conn.Close() 63 | c.conn = nil 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /dq/consumer.go: -------------------------------------------------------------------------------- 1 | package dq 2 | 3 | import ( 4 | "strconv" 5 | "time" 6 | 7 | "github.com/zeromicro/go-zero/core/hash" 8 | "github.com/zeromicro/go-zero/core/logx" 9 | "github.com/zeromicro/go-zero/core/service" 10 | "github.com/zeromicro/go-zero/core/stores/redis" 11 | ) 12 | 13 | const ( 14 | expiration = 3600 // seconds 15 | tolerance = time.Minute * 30 16 | ) 17 | 18 | var maxCheckBytes = getMaxTimeLen() 19 | 20 | type ( 21 | Consume func(body []byte) 22 | 23 | Consumer interface { 24 | Consume(consume Consume) 25 | } 26 | 27 | consumerCluster struct { 28 | nodes []*consumerNode 29 | red *redis.Redis 30 | } 31 | ) 32 | 33 | func NewConsumer(c DqConf) Consumer { 34 | var nodes []*consumerNode 35 | for _, node := range c.Beanstalks { 36 | nodes = append(nodes, newConsumerNode(node.Endpoint, node.Tube)) 37 | } 38 | return &consumerCluster{ 39 | nodes: nodes, 40 | red: c.Redis.NewRedis(), 41 | } 42 | } 43 | 44 | func (c *consumerCluster) Consume(consume Consume) { 45 | guardedConsume := func(body []byte) { 46 | key := hash.Md5Hex(body) 47 | taskBody, ok := c.unwrap(body) 48 | if !ok { 49 | logx.Errorf("discarded: %q", string(body)) 50 | return 51 | } 52 | 53 | redisLock := redis.NewRedisLock(c.red, key) 54 | redisLock.SetExpire(expiration) 55 | ok, err := redisLock.Acquire() 56 | if err != nil { 57 | logx.Error(err) 58 | } else if ok { 59 | consume(taskBody) 60 | } 61 | } 62 | 63 | group := service.NewServiceGroup() 64 | for _, node := range c.nodes { 65 | group.Add(consumeService{ 66 | c: node, 67 | consume: guardedConsume, 68 | }) 69 | } 70 | group.Start() 71 | } 72 | 73 | func (c *consumerCluster) unwrap(body []byte) ([]byte, bool) { 74 | var pos = -1 75 | for i := 0; i < maxCheckBytes && i < len(body); i++ { 76 | if body[i] == timeSep { 77 | pos = i 78 | break 79 | } 80 | } 81 | if pos < 0 { 82 | return nil, false 83 | } 84 | 85 | val, err := strconv.ParseInt(string(body[:pos]), 10, 64) 86 | if err != nil { 87 | logx.Error(err) 88 | return nil, false 89 | } 90 | 91 | t := time.Unix(0, val) 92 | if t.Add(tolerance).Before(time.Now()) { 93 | return nil, false 94 | } 95 | 96 | return body[pos+1:], true 97 | } 98 | 99 | func getMaxTimeLen() int { 100 | return len(strconv.FormatInt(time.Now().UnixNano(), 10)) + 2 101 | } 102 | -------------------------------------------------------------------------------- /dq/consumernode.go: -------------------------------------------------------------------------------- 1 | package dq 2 | 3 | import ( 4 | "errors" 5 | "time" 6 | 7 | "github.com/beanstalkd/go-beanstalk" 8 | "github.com/zeromicro/go-zero/core/logx" 9 | "github.com/zeromicro/go-zero/core/syncx" 10 | ) 11 | 12 | type ( 13 | consumerNode struct { 14 | conn *connection 15 | tube string 16 | on *syncx.AtomicBool 17 | } 18 | 19 | consumeService struct { 20 | c *consumerNode 21 | consume Consume 22 | } 23 | ) 24 | 25 | func newConsumerNode(endpoint, tube string) *consumerNode { 26 | return &consumerNode{ 27 | conn: newConnection(endpoint, tube), 28 | tube: tube, 29 | on: syncx.ForAtomicBool(true), 30 | } 31 | } 32 | 33 | func (c *consumerNode) dispose() { 34 | c.on.Set(false) 35 | } 36 | 37 | func (c *consumerNode) consumeEvents(consume Consume) { 38 | for c.on.True() { 39 | conn, err := c.conn.get() 40 | if err != nil { 41 | logx.Error(err) 42 | time.Sleep(time.Second) 43 | continue 44 | } 45 | 46 | // because getting conn takes at most one second, reserve tasks at most 5 seconds, 47 | // if don't check on/off here, the conn might not be closed due to 48 | // graceful shutdon waits at most 5.5 seconds. 49 | if !c.on.True() { 50 | break 51 | } 52 | 53 | conn.Tube.Name = c.tube 54 | conn.TubeSet.Name[c.tube] = true 55 | id, body, err := conn.Reserve(reserveTimeout) 56 | if err == nil { 57 | conn.Delete(id) 58 | consume(body) 59 | continue 60 | } 61 | 62 | // the error can only be beanstalk.NameError or beanstalk.ConnError 63 | var cerr beanstalk.ConnError 64 | switch { 65 | case errors.As(err, &cerr): 66 | switch { 67 | case errors.Is(cerr.Err, beanstalk.ErrTimeout): 68 | // timeout error on timeout, just continue the loop 69 | case 70 | errors.Is(cerr.Err, beanstalk.ErrBadChar), 71 | errors.Is(cerr.Err, beanstalk.ErrBadFormat), 72 | errors.Is(cerr.Err, beanstalk.ErrBuried), 73 | errors.Is(cerr.Err, beanstalk.ErrDeadline), 74 | errors.Is(cerr.Err, beanstalk.ErrDraining), 75 | errors.Is(cerr.Err, beanstalk.ErrEmpty), 76 | errors.Is(cerr.Err, beanstalk.ErrInternal), 77 | errors.Is(cerr.Err, beanstalk.ErrJobTooBig), 78 | errors.Is(cerr.Err, beanstalk.ErrNoCRLF), 79 | errors.Is(cerr.Err, beanstalk.ErrNotFound), 80 | errors.Is(cerr.Err, beanstalk.ErrNotIgnored), 81 | errors.Is(cerr.Err, beanstalk.ErrTooLong): 82 | // won't reset 83 | logx.Error(err) 84 | default: 85 | // beanstalk.ErrOOM, beanstalk.ErrUnknown and other errors 86 | logx.Error(err) 87 | c.conn.reset() 88 | time.Sleep(time.Second) 89 | } 90 | default: 91 | logx.Error(err) 92 | } 93 | } 94 | 95 | if err := c.conn.Close(); err != nil { 96 | logx.Error(err) 97 | } 98 | } 99 | 100 | func (cs consumeService) Start() { 101 | cs.c.consumeEvents(cs.consume) 102 | } 103 | 104 | func (cs consumeService) Stop() { 105 | cs.c.dispose() 106 | } 107 | -------------------------------------------------------------------------------- /dq/producer.go: -------------------------------------------------------------------------------- 1 | package dq 2 | 3 | import ( 4 | "log" 5 | "math/rand" 6 | "strings" 7 | "time" 8 | 9 | "github.com/zeromicro/go-zero/core/errorx" 10 | "github.com/zeromicro/go-zero/core/fx" 11 | "github.com/zeromicro/go-zero/core/lang" 12 | "github.com/zeromicro/go-zero/core/logx" 13 | ) 14 | 15 | const ( 16 | replicaNodes = 3 17 | minWrittenNodes = 2 18 | ) 19 | 20 | type ( 21 | Producer interface { 22 | At(body []byte, at time.Time) (string, error) 23 | Close() error 24 | Delay(body []byte, delay time.Duration) (string, error) 25 | Revoke(ids string) error 26 | 27 | at(body []byte, at time.Time) (string, error) 28 | delay(body []byte, delay time.Duration) (string, error) 29 | } 30 | 31 | producerCluster struct { 32 | nodes []Producer 33 | } 34 | ) 35 | 36 | var rng *rand.Rand 37 | 38 | func init() { 39 | source := rand.NewSource(time.Now().UnixNano()) 40 | rng = rand.New(source) 41 | } 42 | 43 | func NewProducer(beanstalks []Beanstalk) Producer { 44 | if len(beanstalks) < minWrittenNodes { 45 | log.Fatalf("nodes must be equal or greater than %d", minWrittenNodes) 46 | } 47 | 48 | var nodes []Producer 49 | producers := make(map[string]lang.PlaceholderType) 50 | for _, node := range beanstalks { 51 | if _, ok := producers[node.Endpoint]; ok { 52 | log.Fatal("all node endpoints must be different") 53 | } 54 | 55 | producers[node.Endpoint] = lang.Placeholder 56 | nodes = append(nodes, NewProducerNode(node.Endpoint, node.Tube)) 57 | } 58 | 59 | return &producerCluster{nodes: nodes} 60 | } 61 | 62 | func (p *producerCluster) At(body []byte, at time.Time) (string, error) { 63 | wrapped := wrap(body, at) 64 | return p.at(wrapped, at) 65 | } 66 | 67 | func (p *producerCluster) Close() error { 68 | var be errorx.BatchError 69 | for _, node := range p.nodes { 70 | if err := node.Close(); err != nil { 71 | be.Add(err) 72 | } 73 | } 74 | return be.Err() 75 | } 76 | 77 | func (p *producerCluster) Delay(body []byte, delay time.Duration) (string, error) { 78 | wrapped := wrap(body, time.Now().Add(delay)) 79 | return p.delay(wrapped, delay) 80 | } 81 | 82 | func (p *producerCluster) Revoke(ids string) error { 83 | var be errorx.BatchError 84 | 85 | fx.From(func(source chan<- interface{}) { 86 | for _, node := range p.nodes { 87 | source <- node 88 | } 89 | }).Map(func(item interface{}) interface{} { 90 | node := item.(Producer) 91 | return node.Revoke(ids) 92 | }).ForEach(func(item interface{}) { 93 | if item != nil { 94 | be.Add(item.(error)) 95 | } 96 | }) 97 | 98 | return be.Err() 99 | } 100 | 101 | func (p *producerCluster) at(body []byte, at time.Time) (string, error) { 102 | return p.insert(func(node Producer) (string, error) { 103 | return node.at(body, at) 104 | }) 105 | } 106 | 107 | func (p *producerCluster) cloneNodes() []Producer { 108 | return append([]Producer(nil), p.nodes...) 109 | } 110 | 111 | func (p *producerCluster) delay(body []byte, delay time.Duration) (string, error) { 112 | return p.insert(func(node Producer) (string, error) { 113 | return node.delay(body, delay) 114 | }) 115 | } 116 | 117 | func (p *producerCluster) getWriteNodes() []Producer { 118 | if len(p.nodes) <= replicaNodes { 119 | return p.nodes 120 | } 121 | 122 | nodes := p.cloneNodes() 123 | rng.Shuffle(len(nodes), func(i, j int) { 124 | nodes[i], nodes[j] = nodes[j], nodes[i] 125 | }) 126 | return nodes[:replicaNodes] 127 | } 128 | 129 | func (p *producerCluster) insert(fn func(node Producer) (string, error)) (string, error) { 130 | type idErr struct { 131 | id string 132 | err error 133 | } 134 | var ret []idErr 135 | fx.From(func(source chan<- interface{}) { 136 | for _, node := range p.getWriteNodes() { 137 | source <- node 138 | } 139 | }).Map(func(item interface{}) interface{} { 140 | node := item.(Producer) 141 | id, err := fn(node) 142 | return idErr{ 143 | id: id, 144 | err: err, 145 | } 146 | }).ForEach(func(item interface{}) { 147 | ret = append(ret, item.(idErr)) 148 | }) 149 | 150 | var ids []string 151 | var be errorx.BatchError 152 | for _, val := range ret { 153 | if val.err != nil { 154 | be.Add(val.err) 155 | } else { 156 | ids = append(ids, val.id) 157 | } 158 | } 159 | 160 | jointId := strings.Join(ids, idSep) 161 | if len(ids) >= minWrittenNodes { 162 | return jointId, nil 163 | } 164 | 165 | if err := p.Revoke(jointId); err != nil { 166 | logx.Error(err) 167 | } 168 | 169 | return "", be.Err() 170 | } 171 | -------------------------------------------------------------------------------- /dq/producernode.go: -------------------------------------------------------------------------------- 1 | package dq 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | "time" 9 | 10 | "github.com/beanstalkd/go-beanstalk" 11 | "github.com/zeromicro/go-zero/core/logx" 12 | ) 13 | 14 | var ErrTimeBeforeNow = errors.New("can't schedule task to past time") 15 | 16 | type producerNode struct { 17 | endpoint string 18 | tube string 19 | conn *connection 20 | } 21 | 22 | func NewProducerNode(endpoint, tube string) Producer { 23 | return &producerNode{ 24 | endpoint: endpoint, 25 | tube: tube, 26 | conn: newConnection(endpoint, tube), 27 | } 28 | } 29 | 30 | func (p *producerNode) At(body []byte, at time.Time) (string, error) { 31 | return p.at(wrap(body, at), at) 32 | } 33 | 34 | func (p *producerNode) Close() error { 35 | return p.conn.Close() 36 | } 37 | 38 | func (p *producerNode) Delay(body []byte, delay time.Duration) (string, error) { 39 | return p.delay(wrap(body, time.Now().Add(delay)), delay) 40 | } 41 | 42 | func (p *producerNode) Revoke(jointId string) error { 43 | ids := strings.Split(jointId, idSep) 44 | for _, id := range ids { 45 | fields := strings.Split(id, "/") 46 | if len(fields) < 3 { 47 | continue 48 | } 49 | if fields[0] != p.endpoint || fields[1] != p.tube { 50 | continue 51 | } 52 | 53 | conn, err := p.conn.get() 54 | if err != nil { 55 | return err 56 | } 57 | 58 | n, err := strconv.ParseUint(fields[2], 10, 64) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | return conn.Delete(n) 64 | } 65 | 66 | // if not in this beanstalk, ignore 67 | return nil 68 | } 69 | 70 | func (p *producerNode) at(body []byte, at time.Time) (string, error) { 71 | now := time.Now() 72 | if at.Before(now) { 73 | return "", ErrTimeBeforeNow 74 | } 75 | 76 | duration := at.Sub(now) 77 | return p.delay(body, duration) 78 | } 79 | 80 | func (p *producerNode) delay(body []byte, delay time.Duration) (string, error) { 81 | conn, err := p.conn.get() 82 | if err != nil { 83 | return "", err 84 | } 85 | 86 | id, err := conn.Put(body, PriNormal, delay, defaultTimeToRun) 87 | if err == nil { 88 | return fmt.Sprintf("%s/%s/%d", p.endpoint, p.tube, id), nil 89 | } 90 | 91 | // the error can only be beanstalk.NameError or beanstalk.ConnError 92 | // just return when the error is beanstalk.NameError, don't reset 93 | var cerr beanstalk.ConnError 94 | switch { 95 | case errors.As(err, &cerr): 96 | switch { 97 | case 98 | errors.Is(cerr.Err, beanstalk.ErrBadChar), 99 | errors.Is(cerr.Err, beanstalk.ErrBadFormat), 100 | errors.Is(cerr.Err, beanstalk.ErrBuried), 101 | errors.Is(cerr.Err, beanstalk.ErrDeadline), 102 | errors.Is(cerr.Err, beanstalk.ErrDraining), 103 | errors.Is(cerr.Err, beanstalk.ErrEmpty), 104 | errors.Is(cerr.Err, beanstalk.ErrInternal), 105 | errors.Is(cerr.Err, beanstalk.ErrJobTooBig), 106 | errors.Is(cerr.Err, beanstalk.ErrNoCRLF), 107 | errors.Is(cerr.Err, beanstalk.ErrNotFound), 108 | errors.Is(cerr.Err, beanstalk.ErrNotIgnored), 109 | errors.Is(cerr.Err, beanstalk.ErrTooLong): 110 | // won't reset 111 | default: 112 | // beanstalk.ErrOOM, beanstalk.ErrTimeout, beanstalk.ErrUnknown and other errors 113 | p.conn.reset() 114 | } 115 | default: 116 | logx.Error(err) 117 | } 118 | 119 | return "", err 120 | } 121 | -------------------------------------------------------------------------------- /dq/vars.go: -------------------------------------------------------------------------------- 1 | package dq 2 | 3 | import "time" 4 | 5 | const ( 6 | PriHigh = 1 7 | PriNormal = 2 8 | PriLow = 3 9 | 10 | defaultTimeToRun = time.Second * 5 11 | reserveTimeout = time.Second * 5 12 | 13 | idSep = "," 14 | timeSep = '/' 15 | ) 16 | -------------------------------------------------------------------------------- /dq/wrapper.go: -------------------------------------------------------------------------------- 1 | package dq 2 | 3 | import ( 4 | "bytes" 5 | "strconv" 6 | "time" 7 | ) 8 | 9 | func wrap(body []byte, at time.Time) []byte { 10 | var builder bytes.Buffer 11 | builder.WriteString(strconv.FormatInt(at.UnixNano(), 10)) 12 | builder.WriteByte(timeSep) 13 | builder.Write(body) 14 | return builder.Bytes() 15 | } 16 | -------------------------------------------------------------------------------- /example/dq/consumer/consumer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/zeromicro/go-queue/dq" 7 | "github.com/zeromicro/go-zero/core/stores/redis" 8 | ) 9 | 10 | func main() { 11 | consumer := dq.NewConsumer(dq.DqConf{ 12 | Beanstalks: []dq.Beanstalk{ 13 | { 14 | Endpoint: "localhost:11300", 15 | Tube: "tube", 16 | }, 17 | { 18 | Endpoint: "localhost:11300", 19 | Tube: "tube", 20 | }, 21 | }, 22 | Redis: redis.RedisConf{ 23 | Host: "localhost:6379", 24 | Type: redis.NodeType, 25 | }, 26 | }) 27 | consumer.Consume(func(body []byte) { 28 | fmt.Println(string(body)) 29 | }) 30 | } 31 | -------------------------------------------------------------------------------- /example/dq/producer/cluster/cluster.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "time" 7 | 8 | "github.com/zeromicro/go-queue/dq" 9 | ) 10 | 11 | func main() { 12 | producer := dq.NewProducer([]dq.Beanstalk{ 13 | { 14 | Endpoint: "localhost:11300", 15 | Tube: "tube", 16 | }, 17 | { 18 | Endpoint: "localhost:11300", 19 | Tube: "tube", 20 | }, 21 | }) 22 | for i := 1000; i < 1005; i++ { 23 | _, err := producer.Delay([]byte(strconv.Itoa(i)), time.Second*5) 24 | if err != nil { 25 | fmt.Println(err) 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /example/dq/producer/node/producer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "time" 7 | 8 | "github.com/zeromicro/go-queue/dq" 9 | ) 10 | 11 | func main() { 12 | producer := dq.NewProducerNode("localhost:11300", "tube") 13 | 14 | for i := 1000; i < 1005; i++ { 15 | _, err := producer.Delay([]byte(strconv.Itoa(i)), time.Second*5) 16 | if err != nil { 17 | fmt.Println(err) 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /example/kq/consumer/config.yaml: -------------------------------------------------------------------------------- 1 | Name: kq 2 | Brokers: 3 | - 127.0.0.1:19092 4 | - 127.0.0.1:19092 5 | - 127.0.0.1:19092 6 | Group: adhoc 7 | Topic: kq 8 | Offset: first 9 | Consumers: 1 10 | -------------------------------------------------------------------------------- /example/kq/consumer/queue.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/zeromicro/go-queue/kq" 8 | "github.com/zeromicro/go-zero/core/conf" 9 | ) 10 | 11 | func main() { 12 | var c kq.KqConf 13 | conf.MustLoad("config.yaml", &c) 14 | 15 | q := kq.MustNewQueue(c, kq.WithHandle(func(ctx context.Context, k, v string) error { 16 | fmt.Printf("=> %s\n", v) 17 | return nil 18 | })) 19 | defer q.Stop() 20 | q.Start() 21 | } 22 | -------------------------------------------------------------------------------- /example/kq/producer/produce.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "math/rand" 9 | "strconv" 10 | "time" 11 | 12 | "github.com/zeromicro/go-queue/kq" 13 | "github.com/zeromicro/go-zero/core/cmdline" 14 | ) 15 | 16 | type message struct { 17 | Key string `json:"key"` 18 | Value string `json:"value"` 19 | Payload string `json:"message"` 20 | } 21 | 22 | func main() { 23 | pusher := kq.NewPusher([]string{ 24 | "127.0.0.1:19092", 25 | "127.0.0.1:19092", 26 | "127.0.0.1:19092", 27 | }, "kq") 28 | 29 | ticker := time.NewTicker(time.Millisecond) 30 | for round := 0; round < 3; round++ { 31 | <-ticker.C 32 | 33 | count := rand.Intn(100) 34 | m := message{ 35 | Key: strconv.FormatInt(time.Now().UnixNano(), 10), 36 | Value: fmt.Sprintf("%d,%d", round, count), 37 | Payload: fmt.Sprintf("%d,%d", round, count), 38 | } 39 | body, err := json.Marshal(m) 40 | if err != nil { 41 | log.Fatal(err) 42 | } 43 | 44 | if err := pusher.Push(context.Background(), string(body)); err != nil { 45 | log.Fatal(err) 46 | } 47 | 48 | if err := pusher.KPush(context.Background(), "test", string(body)); err != nil { 49 | log.Fatal(err) 50 | } 51 | } 52 | 53 | cmdline.EnterToContinue() 54 | } 55 | -------------------------------------------------------------------------------- /example/natsq/consumer/consumer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/zeromicro/go-queue/natsq" 7 | ) 8 | 9 | type MyConsumer struct { 10 | Channel string 11 | } 12 | 13 | func (c *MyConsumer) HandleMessage(m *natsq.Msg) error { 14 | fmt.Printf("%s Received %s's a message: %s\n", c.Channel, m.Subject, string(m.Data)) 15 | return nil 16 | } 17 | 18 | func main() { 19 | 20 | mc1 := &MyConsumer{Channel: "vipUpgrade"} 21 | mc2 := &MyConsumer{Channel: "taskFinish"} 22 | 23 | c := &natsq.NatsConfig{ 24 | ServerUri: "nats://127.0.0.1:4222", 25 | } 26 | 27 | //JetMode 28 | // cq := []*natsq.ConsumerQueue{ 29 | // { 30 | // Consumer: mc1, 31 | // QueueName: "vipUpgrade", 32 | // StreamName: "ccc", 33 | // Subjects: []string{"ddd", "eee"}, 34 | // }, 35 | // { 36 | // Consumer: mc2, 37 | // QueueName: "taskFinish", 38 | // StreamName: "ccc", 39 | // Subjects: []string{"ccc", "eee"}, 40 | // }, 41 | // } 42 | //q := natsq.MustNewConsumerManager(c, cq, natsq.NatJetMode) 43 | 44 | //DefaultMode 45 | cq := []*natsq.ConsumerQueue{ 46 | { 47 | Consumer: mc1, 48 | QueueName: "vipUpgrade", 49 | Subjects: []string{"ddd", "eee"}, 50 | }, 51 | { 52 | Consumer: mc2, 53 | QueueName: "taskFinish", 54 | Subjects: []string{"ccc", "eee"}, 55 | }, 56 | } 57 | q := natsq.MustNewConsumerManager(c, cq, natsq.NatDefaultMode) 58 | q.Start() 59 | defer q.Stop() 60 | } 61 | -------------------------------------------------------------------------------- /example/natsq/publisher/publisher.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "math/rand" 6 | "time" 7 | 8 | "github.com/nats-io/nats.go/jetstream" 9 | "github.com/zeromicro/go-queue/natsq" 10 | ) 11 | 12 | func main() { 13 | 14 | c := natsq.NatsConfig{ 15 | ServerUri: "127.0.0.1:4222", 16 | } 17 | 18 | // Default Mode 19 | p, _ := natsq.NewDefaultProducer(&c) 20 | for i := 0; i < 3; i++ { 21 | payload := randBody() 22 | err := p.Publish(randSub(), payload) 23 | if err != nil { 24 | log.Fatalf("Error publishing message: %v", err) 25 | } else { 26 | log.Printf("Published message: %s", string(payload)) 27 | } 28 | } 29 | p.Close() 30 | 31 | // JetMode 32 | j, _ := natsq.NewJetProducer(&c) 33 | j.CreateOrUpdateStream(jetstream.StreamConfig{ 34 | Name: "ccc", 35 | Subjects: []string{"ccc", "ddd", "eee"}, 36 | Storage: jetstream.FileStorage, 37 | NoAck: false, 38 | }) 39 | for i := 0; i < 3; i++ { 40 | payload := randBody() 41 | err := j.Publish(randSub(), payload) 42 | if err != nil { 43 | log.Fatalf("Error publishing message: %v", err) 44 | } else { 45 | log.Printf("Published message: %s", string(payload)) 46 | } 47 | } 48 | j.Close() 49 | } 50 | 51 | func randSub() string { 52 | source := rand.NewSource(time.Now().UnixNano()) 53 | // 创建一个新的随机数生成器 54 | rng := rand.New(source) 55 | strings := []string{"ccc", "ddd", "eee"} 56 | randomIndex := rng.Intn(len(strings)) 57 | return strings[randomIndex] 58 | } 59 | 60 | func randBody() []byte { 61 | charSet := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" 62 | length := 10 63 | result := make([]byte, length) 64 | for i := range result { 65 | result[i] = charSet[rand.Intn(len(charSet))] 66 | } 67 | return result 68 | } 69 | -------------------------------------------------------------------------------- /example/rabbitmq/admin/admin.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/zeromicro/go-queue/rabbitmq" 7 | ) 8 | 9 | func main() { 10 | conf := rabbitmq.RabbitConf{ 11 | Host: "192.168.253.100", 12 | Port: 5672, 13 | Username: "guest", 14 | Password: "guest", 15 | } 16 | admin := rabbitmq.MustNewAdmin(conf) 17 | exchangeConf := rabbitmq.ExchangeConf{ 18 | ExchangeName: "jiang", 19 | Type: "direct", 20 | Durable: true, 21 | AutoDelete: false, 22 | Internal: false, 23 | NoWait: false, 24 | } 25 | 26 | err := admin.DeclareExchange(exchangeConf, nil) 27 | if err != nil { 28 | log.Fatal(err) 29 | } 30 | 31 | queueConf := rabbitmq.QueueConf{ 32 | Name: "jxj", 33 | Durable: true, 34 | AutoDelete: false, 35 | Exclusive: false, 36 | NoWait: false, 37 | } 38 | err = admin.DeclareQueue(queueConf, nil) 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | 43 | err = admin.Bind("jxj", "jxj", "jiang", false, nil) 44 | if err != nil { 45 | log.Fatal(err) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /example/rabbitmq/listener/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "github.com/zeromicro/go-queue/rabbitmq" 4 | 5 | type Config struct { 6 | ListenerConf rabbitmq.RabbitListenerConf 7 | } 8 | -------------------------------------------------------------------------------- /example/rabbitmq/listener/listener.yaml: -------------------------------------------------------------------------------- 1 | ListenerConf: 2 | Username: guest 3 | Password: guest 4 | Host: 192.168.253.100 5 | Port: 5672 6 | ListenerQueues: 7 | - 8 | Name: gozero 9 | 10 | 11 | -------------------------------------------------------------------------------- /example/rabbitmq/listener/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | 7 | "github.com/zeromicro/go-queue/example/rabbitmq/listener/config" 8 | "github.com/zeromicro/go-queue/rabbitmq" 9 | "github.com/zeromicro/go-zero/core/conf" 10 | "github.com/zeromicro/go-zero/core/service" 11 | ) 12 | 13 | var configFile = flag.String("f", "listener.yaml", "Specify the config file") 14 | 15 | func main() { 16 | flag.Parse() 17 | var c config.Config 18 | conf.MustLoad(*configFile, &c) 19 | 20 | listener := rabbitmq.MustNewListener(c.ListenerConf, Handler{}) 21 | serviceGroup := service.NewServiceGroup() 22 | serviceGroup.Add(listener) 23 | defer serviceGroup.Stop() 24 | serviceGroup.Start() 25 | } 26 | 27 | type Handler struct { 28 | } 29 | 30 | func (h Handler) Consume(message string) error { 31 | fmt.Printf("listener %s\n", message) 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /example/rabbitmq/sender/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | 7 | "github.com/zeromicro/go-queue/rabbitmq" 8 | ) 9 | 10 | func main() { 11 | conf := rabbitmq.RabbitSenderConf{RabbitConf: rabbitmq.RabbitConf{ 12 | Host: "192.168.253.100", 13 | Port: 5672, 14 | Username: "guest", 15 | Password: "guest", 16 | }, ContentType: "application/json"} 17 | sender := rabbitmq.MustNewSender(conf) 18 | data := map[string]interface{}{ 19 | "msg": "json test 111", 20 | } 21 | msg, _ := json.Marshal(data) 22 | err := sender.Send("exchange.direct", "gozero", msg) 23 | if err != nil { 24 | log.Fatal(err) 25 | } 26 | 27 | conf.ContentType = "text/plain" 28 | sender = rabbitmq.MustNewSender(conf) 29 | message := "test message" 30 | err = sender.Send("exchange.direct", "gozero", []byte(message)) 31 | if err != nil { 32 | log.Fatal(err) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /example/stan/comsumer/consumer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/nats-io/stan.go" 6 | "github.com/zeromicro/go-queue/stanq" 7 | ) 8 | 9 | type MyConsumer struct { 10 | Channel string 11 | } 12 | 13 | func (c *MyConsumer) HandleMessage(m *stan.Msg) error { 14 | fmt.Printf("%s Received a message: %s\n", c.Channel, string(m.Data)) 15 | return nil 16 | } 17 | 18 | func main() { 19 | 20 | mc1 := &MyConsumer{Channel: "vip"} 21 | mc2 := &MyConsumer{Channel: "recharge"} 22 | mc3 := &MyConsumer{Channel: "levelUp"} 23 | 24 | c := &stanq.StanqConfig{ 25 | ClusterID: "nats-streaming", 26 | ClientID: "vip-consumer", 27 | Options: []stan.Option{ 28 | stan.NatsURL("nats://127.0.0.1:14222,nats://127.0.0.1:24222,nats://127.0.0.1:34222"), 29 | }, 30 | } 31 | cq := []*stanq.ConsumerQueue{ 32 | { 33 | Consumer: mc1, 34 | GroupName: "vip", 35 | QueueName: "vip1", 36 | Subject: "a", 37 | ManualAckMode: false, 38 | }, 39 | { 40 | Consumer: mc2, 41 | GroupName: "recharge", 42 | QueueName: "recharge1", 43 | Subject: "b", 44 | ManualAckMode: false, 45 | }, 46 | { 47 | Consumer: mc3, 48 | GroupName: "levelUp", 49 | QueueName: "levelUp1", 50 | Subject: "c", 51 | ManualAckMode: false, 52 | }, 53 | } 54 | 55 | q := stanq.MustNewConsumerManager(c, cq) 56 | q.Start() 57 | defer q.Stop() 58 | } 59 | -------------------------------------------------------------------------------- /example/stan/publisher/producer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/nats-io/stan.go" 6 | "github.com/zeromicro/go-queue/stanq" 7 | "math/rand" 8 | "time" 9 | ) 10 | 11 | func main() { 12 | c := &stanq.StanqConfig{ 13 | ClusterID: "nats-streaming", 14 | ClientID: "publish123", 15 | Options: []stan.Option{stan.NatsURL("nats://127.0.0.1:14222,nats://127.0.0.1:24222,nats://127.0.0.1:34222")}, 16 | } 17 | p, err := stanq.NewProducer(c) 18 | if err != nil { 19 | fmt.Println(err.Error()) 20 | } 21 | 22 | publish, err := p.AsyncPublish(randSub(), randBody(), func(guid string, err error) { 23 | if err != nil { 24 | fmt.Printf("failed to publish message with guid %s: %v\n", guid, err) 25 | } else { 26 | fmt.Printf("message with guid %s published\n", guid) 27 | } 28 | }) 29 | if err != nil { 30 | fmt.Println(err.Error()) 31 | } else { 32 | fmt.Println(publish) 33 | } 34 | 35 | for { 36 | time.Sleep(300 * time.Millisecond) 37 | sub := randSub() 38 | err = p.Publish(sub, []byte(fmt.Sprintf("%s-%s", sub, randBody()))) 39 | if err != nil { 40 | fmt.Println(err.Error()) 41 | } 42 | } 43 | 44 | } 45 | 46 | func randSub() string { 47 | source := rand.NewSource(time.Now().UnixNano()) 48 | rng := rand.New(source) 49 | charSet := "abc" 50 | length := 1 51 | result := make([]byte, length) 52 | for i := range result { 53 | result[i] = charSet[rng.Intn(len(charSet))] 54 | } 55 | return string(result) 56 | } 57 | 58 | func randBody() []byte { 59 | charSet := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" 60 | length := 10 61 | result := make([]byte, length) 62 | for i := range result { 63 | result[i] = charSet[rand.Intn(len(charSet))] 64 | } 65 | return result 66 | } 67 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/zeromicro/go-queue 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/beanstalkd/go-beanstalk v0.2.0 7 | github.com/nats-io/nats.go v1.34.1 8 | github.com/nats-io/stan.go v0.10.4 9 | github.com/rabbitmq/amqp091-go v1.10.0 10 | github.com/segmentio/kafka-go v0.4.47 11 | github.com/stretchr/testify v1.9.0 12 | github.com/zeromicro/go-zero v1.6.6 13 | go.opentelemetry.io/otel v1.19.0 14 | ) 15 | 16 | require ( 17 | github.com/beorn7/perks v1.0.1 // indirect 18 | github.com/cenkalti/backoff/v4 v4.2.1 // indirect 19 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 20 | github.com/davecgh/go-spew v1.1.1 // indirect 21 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 22 | github.com/fatih/color v1.17.0 // indirect 23 | github.com/go-logr/logr v1.3.0 // indirect 24 | github.com/go-logr/stdr v1.2.2 // indirect 25 | github.com/gogo/protobuf v1.3.2 // indirect 26 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect 27 | github.com/klauspost/compress v1.17.9 // indirect 28 | github.com/mattn/go-colorable v0.1.13 // indirect 29 | github.com/mattn/go-isatty v0.0.20 // indirect 30 | github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect 31 | github.com/nats-io/nats-server/v2 v2.9.15 // indirect 32 | github.com/nats-io/nats-streaming-server v0.25.3 // indirect 33 | github.com/nats-io/nkeys v0.4.7 // indirect 34 | github.com/nats-io/nuid v1.0.1 // indirect 35 | github.com/openzipkin/zipkin-go v0.4.2 // indirect 36 | github.com/pelletier/go-toml/v2 v2.2.2 // indirect 37 | github.com/pierrec/lz4/v4 v4.1.21 // indirect 38 | github.com/pmezard/go-difflib v1.0.0 // indirect 39 | github.com/prometheus/client_golang v1.18.0 // indirect 40 | github.com/prometheus/client_model v0.5.0 // indirect 41 | github.com/prometheus/common v0.45.0 // indirect 42 | github.com/prometheus/procfs v0.12.0 // indirect 43 | github.com/redis/go-redis/v9 v9.5.3 // indirect 44 | github.com/spaolacci/murmur3 v1.1.0 // indirect 45 | github.com/stretchr/objx v0.5.2 // indirect 46 | go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect 47 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect 48 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect 49 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 // indirect 50 | go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.19.0 // indirect 51 | go.opentelemetry.io/otel/exporters/zipkin v1.19.0 // indirect 52 | go.opentelemetry.io/otel/metric v1.19.0 // indirect 53 | go.opentelemetry.io/otel/sdk v1.19.0 // indirect 54 | go.opentelemetry.io/otel/trace v1.19.0 // indirect 55 | go.opentelemetry.io/proto/otlp v1.0.0 // indirect 56 | go.uber.org/automaxprocs v1.5.3 // indirect 57 | golang.org/x/crypto v0.24.0 // indirect 58 | golang.org/x/net v0.26.0 // indirect 59 | golang.org/x/sys v0.21.0 // indirect 60 | golang.org/x/text v0.16.0 // indirect 61 | google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect 62 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect 63 | google.golang.org/grpc v1.64.0 // indirect 64 | google.golang.org/protobuf v1.34.2 // indirect 65 | gopkg.in/yaml.v2 v2.4.0 // indirect 66 | gopkg.in/yaml.v3 v3.0.1 // indirect 67 | ) 68 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= 2 | github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE= 3 | github.com/alicebob/miniredis/v2 v2.33.0 h1:uvTF0EDeu9RLnUEG27Db5I68ESoIxTiXbNUiji6lZrA= 4 | github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= 5 | github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= 6 | github.com/beanstalkd/go-beanstalk v0.2.0 h1:6UOJugnu47uNB2jJO/lxyDgeD1Yds7owYi1USELqexA= 7 | github.com/beanstalkd/go-beanstalk v0.2.0/go.mod h1:/G8YTyChOtpOArwLTQPY1CHB+i212+av35bkPXXj56Y= 8 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 9 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 10 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 11 | github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= 12 | github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= 13 | github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= 14 | github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= 15 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 16 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 17 | github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= 18 | github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= 19 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 20 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 21 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 22 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 23 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 24 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 25 | github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= 26 | github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= 27 | github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 28 | github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= 29 | github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 30 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 31 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 32 | github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= 33 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 34 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 35 | github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= 36 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 37 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 38 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 39 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 40 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 41 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 42 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 43 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 44 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 45 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 46 | github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 47 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 48 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 49 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 h1:RtRsiaGvWxcwd8y3BiRZxsylPT8hLWZ5SPcfI+3IDNk= 50 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0/go.mod h1:TzP6duP4Py2pHLVPPQp42aoYI92+PCrVotyR5e8Vqlk= 51 | github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= 52 | github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 53 | github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= 54 | github.com/hashicorp/go-hclog v1.1.0 h1:QsGcniKx5/LuX2eYoeL+Np3UKYPNaN7YKpTh29h8rbw= 55 | github.com/hashicorp/go-hclog v1.1.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= 56 | github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= 57 | github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 58 | github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= 59 | github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs= 60 | github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4= 61 | github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= 62 | github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 63 | github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= 64 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 65 | github.com/hashicorp/raft v1.3.11 h1:p3v6gf6l3S797NnK5av3HcczOC1T5CLoaRvg0g9ys4A= 66 | github.com/hashicorp/raft v1.3.11/go.mod h1:J8naEwc6XaaCfts7+28whSeRvCqTd6e20BlCU3LtEO4= 67 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 68 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 69 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 70 | github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= 71 | github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= 72 | github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= 73 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 74 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 75 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 76 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 77 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 78 | github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 79 | github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 80 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 81 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 82 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 83 | github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= 84 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 85 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 86 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 87 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 88 | github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= 89 | github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= 90 | github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= 91 | github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= 92 | github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= 93 | github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= 94 | github.com/nats-io/nats-server/v2 v2.9.11/go.mod h1:b0oVuxSlkvS3ZjMkncFeACGyZohbO4XhSqW1Lt7iRRY= 95 | github.com/nats-io/nats-server/v2 v2.9.15 h1:MuwEJheIwpvFgqvbs20W8Ish2azcygjf4Z0liVu2I4c= 96 | github.com/nats-io/nats-server/v2 v2.9.15/go.mod h1:QlCTy115fqpx4KSOPFIxSV7DdI6OxtZsGOL1JLdeRlE= 97 | github.com/nats-io/nats-streaming-server v0.25.3 h1:A9dmf4fMIxFPBGqgwePljWsBePMEjl3ugsIwK6F2wow= 98 | github.com/nats-io/nats-streaming-server v0.25.3/go.mod h1:0Cyht7y75el3if3Qdq31OPc/TNjVwzglMPz04Hn77kk= 99 | github.com/nats-io/nats.go v1.19.0/go.mod h1:tLqubohF7t4z3du1QDPYJIQQyhb4wl6DhjxEajSI7UA= 100 | github.com/nats-io/nats.go v1.22.1/go.mod h1:tLqubohF7t4z3du1QDPYJIQQyhb4wl6DhjxEajSI7UA= 101 | github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4= 102 | github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= 103 | github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= 104 | github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= 105 | github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= 106 | github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= 107 | github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= 108 | github.com/nats-io/stan.go v0.10.4 h1:19GS/eD1SeQJaVkeM9EkvEYattnvnWrZ3wkSWSw4uXw= 109 | github.com/nats-io/stan.go v0.10.4/go.mod h1:3XJXH8GagrGqajoO/9+HgPyKV5MWsv7S5ccdda+pc6k= 110 | github.com/openzipkin/zipkin-go v0.4.2 h1:zjqfqHjUpPmB3c1GlCvvgsM1G4LkvqQbBDueDOCg/jA= 111 | github.com/openzipkin/zipkin-go v0.4.2/go.mod h1:ZeVkFjuuBiSy13y8vpSDCjMi9GoI3hPpCJSBx/EYFhY= 112 | github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 113 | github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= 114 | github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= 115 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 116 | github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= 117 | github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 118 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 119 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 120 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 121 | github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= 122 | github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= 123 | github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= 124 | github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= 125 | github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= 126 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 127 | github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= 128 | github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= 129 | github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 130 | github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= 131 | github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= 132 | github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 133 | github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= 134 | github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= 135 | github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= 136 | github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= 137 | github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= 138 | github.com/redis/go-redis/v9 v9.5.3 h1:fOAp1/uJG+ZtcITgZOfYFmTKPE7n4Vclj1wZFgRciUU= 139 | github.com/redis/go-redis/v9 v9.5.3/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= 140 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 141 | github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0= 142 | github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= 143 | github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= 144 | github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 145 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 146 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 147 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 148 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 149 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 150 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 151 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 152 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 153 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 154 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 155 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 156 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 157 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 158 | github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= 159 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= 160 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 161 | github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= 162 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 163 | github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= 164 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 165 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 166 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 167 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 168 | github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= 169 | github.com/zeromicro/go-zero v1.6.6 h1:nZTVYObklHiBdYJ/nPoAZ8kGVAplWSDjT7DGE7ur0uk= 170 | github.com/zeromicro/go-zero v1.6.6/go.mod h1:olKf1/hELbSmuIgLgJeoeNVp3tCbLqj6UmO7ATSta4A= 171 | go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= 172 | go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= 173 | go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= 174 | go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= 175 | go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4= 176 | go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI= 177 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= 178 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= 179 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= 180 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= 181 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= 182 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= 183 | go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.19.0 h1:Nw7Dv4lwvGrI68+wULbcq7su9K2cebeCUrDjVrUJHxM= 184 | go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.19.0/go.mod h1:1MsF6Y7gTqosgoZvHlzcaaM8DIMNZgJh87ykokoNH7Y= 185 | go.opentelemetry.io/otel/exporters/zipkin v1.19.0 h1:EGY0h5mGliP9o/nIkVuLI0vRiQqmsYOcbwCuotksO1o= 186 | go.opentelemetry.io/otel/exporters/zipkin v1.19.0/go.mod h1:JQgTGJP11yi3o4GHzIWYodhPisxANdqxF1eHwDSnJrI= 187 | go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= 188 | go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= 189 | go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= 190 | go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= 191 | go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= 192 | go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= 193 | go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= 194 | go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= 195 | go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= 196 | go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= 197 | go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= 198 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 199 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 200 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 201 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 202 | golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= 203 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 204 | golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= 205 | golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= 206 | golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= 207 | golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= 208 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 209 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 210 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 211 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 212 | golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 213 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 214 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 215 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 216 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 217 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 218 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 219 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 220 | golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= 221 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 222 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 223 | golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= 224 | golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= 225 | golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= 226 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 227 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 228 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 229 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 230 | golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 231 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 232 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 233 | golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 234 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 235 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 236 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 237 | golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 238 | golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 239 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 240 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 241 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 242 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 243 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 244 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 245 | golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 246 | golang.org/x/sys v0.4.1-0.20230105183443-b8be2fde2a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 247 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 248 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 249 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 250 | golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 251 | golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= 252 | golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 253 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 254 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 255 | golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= 256 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 257 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= 258 | golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= 259 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 260 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 261 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 262 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 263 | golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 264 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 265 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 266 | golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= 267 | golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= 268 | golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= 269 | golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 270 | golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= 271 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 272 | golang.org/x/tools v0.0.0-20190424220101-1e8e1cfdf96b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 273 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 274 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 275 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 276 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 277 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 278 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 279 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 280 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 281 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 282 | google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= 283 | google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= 284 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= 285 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= 286 | google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= 287 | google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= 288 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 289 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 290 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 291 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 292 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 293 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 294 | google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= 295 | google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= 296 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 297 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 298 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 299 | gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= 300 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 301 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 302 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 303 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 304 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 305 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 306 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= 307 | -------------------------------------------------------------------------------- /kq/config.go: -------------------------------------------------------------------------------- 1 | package kq 2 | 3 | import "github.com/zeromicro/go-zero/core/service" 4 | 5 | const ( 6 | firstOffset = "first" 7 | lastOffset = "last" 8 | ) 9 | 10 | type KqConf struct { 11 | service.ServiceConf 12 | Brokers []string 13 | Group string 14 | Topic string 15 | CaFile string `json:",optional"` 16 | Offset string `json:",options=first|last,default=last"` 17 | Conns int `json:",default=1"` 18 | Consumers int `json:",default=8"` 19 | Processors int `json:",default=8"` 20 | MinBytes int `json:",default=10240"` // 10K 21 | MaxBytes int `json:",default=10485760"` // 10M 22 | Username string `json:",optional"` 23 | Password string `json:",optional"` 24 | ForceCommit bool `json:",default=true"` 25 | CommitInOrder bool `json:",default=false"` 26 | } 27 | -------------------------------------------------------------------------------- /kq/internal/message.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import "github.com/segmentio/kafka-go" 4 | 5 | type Message struct { 6 | *kafka.Message 7 | } 8 | 9 | func NewMessage(msg *kafka.Message) *Message { 10 | return &Message{Message: msg} 11 | } 12 | 13 | func (m *Message) GetHeader(key string) string { 14 | for _, h := range m.Headers { 15 | if h.Key == key { 16 | return string(h.Value) 17 | } 18 | } 19 | return "" 20 | } 21 | 22 | func (m *Message) SetHeader(key, val string) { 23 | // Ensure uniqueness of keys 24 | for i := 0; i < len(m.Headers); i++ { 25 | if m.Headers[i].Key == key { 26 | m.Headers = append(m.Headers[:i], m.Headers[i+1:]...) 27 | i-- 28 | } 29 | } 30 | m.Headers = append(m.Headers, kafka.Header{ 31 | Key: key, 32 | Value: []byte(val), 33 | }) 34 | } 35 | -------------------------------------------------------------------------------- /kq/internal/message_test.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/segmentio/kafka-go" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestMessageGetHeader(t *testing.T) { 11 | testCases := []struct { 12 | name string 13 | msg *Message 14 | key string 15 | expected string 16 | }{ 17 | { 18 | name: "exists", 19 | msg: &Message{ 20 | Message: &kafka.Message{Headers: []kafka.Header{ 21 | {Key: "foo", Value: []byte("bar")}, 22 | }}}, 23 | key: "foo", 24 | expected: "bar", 25 | }, 26 | { 27 | name: "not exists", 28 | msg: &Message{Message: &kafka.Message{Headers: []kafka.Header{}}}, 29 | key: "foo", 30 | expected: "", 31 | }, 32 | } 33 | 34 | for _, tc := range testCases { 35 | t.Run(tc.name, func(t *testing.T) { 36 | result := tc.msg.GetHeader(tc.key) 37 | assert.Equal(t, tc.expected, result) 38 | }) 39 | } 40 | } 41 | 42 | func TestMessageSetHeader(t *testing.T) { 43 | msg := &Message{Message: &kafka.Message{Headers: []kafka.Header{ 44 | {Key: "foo", Value: []byte("bar")}}, 45 | }} 46 | 47 | msg.SetHeader("foo", "bar2") 48 | msg.SetHeader("foo2", "bar2") 49 | msg.SetHeader("foo2", "bar3") 50 | msg.SetHeader("foo3", "bar4") 51 | 52 | assert.ElementsMatch(t, msg.Headers, []kafka.Header{ 53 | {Key: "foo", Value: []byte("bar2")}, 54 | {Key: "foo2", Value: []byte("bar3")}, 55 | {Key: "foo3", Value: []byte("bar4")}, 56 | }) 57 | } 58 | -------------------------------------------------------------------------------- /kq/internal/trace.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import "go.opentelemetry.io/otel/propagation" 4 | 5 | var _ propagation.TextMapCarrier = (*MessageCarrier)(nil) 6 | 7 | // MessageCarrier injects and extracts traces from a types.Message. 8 | type MessageCarrier struct { 9 | msg *Message 10 | } 11 | 12 | // NewMessageCarrier returns a new MessageCarrier. 13 | func NewMessageCarrier(msg *Message) MessageCarrier { 14 | return MessageCarrier{msg: msg} 15 | } 16 | 17 | // Get returns the value associated with the passed key. 18 | func (m MessageCarrier) Get(key string) string { 19 | return m.msg.GetHeader(key) 20 | } 21 | 22 | // Set stores the key-value pair. 23 | func (m MessageCarrier) Set(key string, value string) { 24 | m.msg.SetHeader(key, value) 25 | } 26 | 27 | // Keys lists the keys stored in this carrier. 28 | func (m MessageCarrier) Keys() []string { 29 | out := make([]string, len(m.msg.Headers)) 30 | for i, h := range m.msg.Headers { 31 | out[i] = h.Key 32 | } 33 | 34 | return out 35 | } 36 | -------------------------------------------------------------------------------- /kq/internal/trace_test.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/segmentio/kafka-go" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestMessageCarrierGet(t *testing.T) { 11 | testCases := []struct { 12 | name string 13 | carrier MessageCarrier 14 | key string 15 | expected string 16 | }{ 17 | { 18 | name: "exists", 19 | carrier: NewMessageCarrier(&Message{&kafka.Message{Headers: []kafka.Header{ 20 | {Key: "foo", Value: []byte("bar")}, 21 | }}}), 22 | key: "foo", 23 | expected: "bar", 24 | }, 25 | { 26 | name: "not exists", 27 | carrier: NewMessageCarrier(&Message{&kafka.Message{Headers: []kafka.Header{}}}), 28 | key: "foo", 29 | expected: "", 30 | }, 31 | } 32 | 33 | for _, tc := range testCases { 34 | t.Run(tc.name, func(t *testing.T) { 35 | result := tc.carrier.Get(tc.key) 36 | assert.Equal(t, tc.expected, result) 37 | }) 38 | } 39 | } 40 | 41 | func TestMessageCarrierSet(t *testing.T) { 42 | msg := Message{&kafka.Message{Headers: []kafka.Header{ 43 | {Key: "foo", Value: []byte("bar")}, 44 | }}} 45 | carrier := MessageCarrier{msg: &msg} 46 | 47 | carrier.Set("foo", "bar2") 48 | carrier.Set("foo2", "bar2") 49 | carrier.Set("foo2", "bar3") 50 | carrier.Set("foo3", "bar4") 51 | 52 | assert.ElementsMatch(t, carrier.msg.Headers, []kafka.Header{ 53 | {Key: "foo", Value: []byte("bar2")}, 54 | {Key: "foo2", Value: []byte("bar3")}, 55 | {Key: "foo3", Value: []byte("bar4")}, 56 | }) 57 | } 58 | 59 | func TestMessageCarrierKeys(t *testing.T) { 60 | testCases := []struct { 61 | name string 62 | carrier MessageCarrier 63 | expected []string 64 | }{ 65 | { 66 | name: "one", 67 | carrier: MessageCarrier{msg: &Message{&kafka.Message{Headers: []kafka.Header{ 68 | {Key: "foo", Value: []byte("bar")}, 69 | }}}}, 70 | expected: []string{"foo"}, 71 | }, 72 | { 73 | name: "none", 74 | carrier: MessageCarrier{msg: &Message{&kafka.Message{Headers: []kafka.Header{}}}}, 75 | expected: []string{}, 76 | }, 77 | { 78 | name: "many", 79 | carrier: MessageCarrier{msg: &Message{&kafka.Message{Headers: []kafka.Header{ 80 | {Key: "foo", Value: []byte("bar")}, 81 | {Key: "baz", Value: []byte("quux")}, 82 | }}}}, 83 | expected: []string{"foo", "baz"}, 84 | }, 85 | } 86 | 87 | for _, tc := range testCases { 88 | t.Run(tc.name, func(t *testing.T) { 89 | result := tc.carrier.Keys() 90 | assert.Equal(t, tc.expected, result) 91 | }) 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /kq/pusher.go: -------------------------------------------------------------------------------- 1 | package kq 2 | 3 | import ( 4 | "context" 5 | "strconv" 6 | "time" 7 | 8 | "github.com/segmentio/kafka-go" 9 | "github.com/zeromicro/go-queue/kq/internal" 10 | "github.com/zeromicro/go-zero/core/executors" 11 | "github.com/zeromicro/go-zero/core/logx" 12 | "go.opentelemetry.io/otel" 13 | ) 14 | 15 | type ( 16 | PushOption func(options *pushOptions) 17 | 18 | Pusher struct { 19 | topic string 20 | producer kafkaWriter 21 | executor *executors.ChunkExecutor 22 | } 23 | 24 | kafkaWriter interface { 25 | Close() error 26 | WriteMessages(ctx context.Context, msgs ...kafka.Message) error 27 | } 28 | 29 | pushOptions struct { 30 | // kafka.Writer options 31 | allowAutoTopicCreation bool 32 | balancer kafka.Balancer 33 | 34 | // executors.ChunkExecutor options 35 | chunkSize int 36 | flushInterval time.Duration 37 | 38 | // syncPush is used to enable sync push 39 | syncPush bool 40 | } 41 | ) 42 | 43 | // NewPusher returns a Pusher with the given Kafka addresses and topic. 44 | func NewPusher(addrs []string, topic string, opts ...PushOption) *Pusher { 45 | producer := &kafka.Writer{ 46 | Addr: kafka.TCP(addrs...), 47 | Topic: topic, 48 | Balancer: &kafka.LeastBytes{}, 49 | Compression: kafka.Snappy, 50 | } 51 | 52 | var options pushOptions 53 | for _, opt := range opts { 54 | opt(&options) 55 | } 56 | 57 | // apply kafka.Writer options 58 | producer.AllowAutoTopicCreation = options.allowAutoTopicCreation 59 | if options.balancer != nil { 60 | producer.Balancer = options.balancer 61 | } 62 | 63 | pusher := &Pusher{ 64 | producer: producer, 65 | topic: topic, 66 | } 67 | 68 | // if syncPush is true, return the pusher directly 69 | if options.syncPush { 70 | producer.BatchSize = 1 71 | return pusher 72 | } 73 | 74 | // apply ChunkExecutor options 75 | var chunkOpts []executors.ChunkOption 76 | if options.chunkSize > 0 { 77 | chunkOpts = append(chunkOpts, executors.WithChunkBytes(options.chunkSize)) 78 | } 79 | if options.flushInterval > 0 { 80 | chunkOpts = append(chunkOpts, executors.WithFlushInterval(options.flushInterval)) 81 | } 82 | 83 | pusher.executor = executors.NewChunkExecutor(func(tasks []interface{}) { 84 | chunk := make([]kafka.Message, len(tasks)) 85 | for i := range tasks { 86 | chunk[i] = tasks[i].(kafka.Message) 87 | } 88 | if err := pusher.producer.WriteMessages(context.Background(), chunk...); err != nil { 89 | logx.Error(err) 90 | } 91 | }, chunkOpts...) 92 | 93 | return pusher 94 | } 95 | 96 | // Close closes the Pusher and releases any resources used by it. 97 | func (p *Pusher) Close() error { 98 | if p.executor != nil { 99 | p.executor.Flush() 100 | } 101 | 102 | return p.producer.Close() 103 | } 104 | 105 | // Name returns the name of the Kafka topic that the Pusher is sending messages to. 106 | func (p *Pusher) Name() string { 107 | return p.topic 108 | } 109 | 110 | // KPush sends a message to the Kafka topic. 111 | func (p *Pusher) KPush(ctx context.Context, k, v string) error { 112 | msg := kafka.Message{ 113 | Key: []byte(k), // current timestamp 114 | Value: []byte(v), 115 | } 116 | if p.executor != nil { 117 | return p.executor.Add(msg, len(v)) 118 | } else { 119 | return p.producer.WriteMessages(ctx, msg) 120 | } 121 | } 122 | 123 | // Push sends a message to the Kafka topic. 124 | func (p *Pusher) Push(ctx context.Context, v string) error { 125 | return p.PushWithKey(ctx, strconv.FormatInt(time.Now().UnixNano(), 10), v) 126 | } 127 | 128 | // PushWithKey sends a message with the given key to the Kafka topic. 129 | func (p *Pusher) PushWithKey(ctx context.Context, key, v string) error { 130 | msg := kafka.Message{ 131 | Key: []byte(key), 132 | Value: []byte(v), 133 | } 134 | 135 | // wrap message into message carrier 136 | mc := internal.NewMessageCarrier(internal.NewMessage(&msg)) 137 | // inject trace context into message 138 | otel.GetTextMapPropagator().Inject(ctx, mc) 139 | 140 | if p.executor != nil { 141 | return p.executor.Add(msg, len(v)) 142 | } else { 143 | return p.producer.WriteMessages(ctx, msg) 144 | } 145 | } 146 | 147 | // WithAllowAutoTopicCreation allows the Pusher to create the given topic if it does not exist. 148 | func WithAllowAutoTopicCreation() PushOption { 149 | return func(options *pushOptions) { 150 | options.allowAutoTopicCreation = true 151 | } 152 | } 153 | 154 | // WithBalancer customizes the Pusher with the given balancer. 155 | func WithBalancer(balancer kafka.Balancer) PushOption { 156 | return func(options *pushOptions) { 157 | options.balancer = balancer 158 | } 159 | } 160 | 161 | // WithChunkSize customizes the Pusher with the given chunk size. 162 | func WithChunkSize(chunkSize int) PushOption { 163 | return func(options *pushOptions) { 164 | options.chunkSize = chunkSize 165 | } 166 | } 167 | 168 | // WithFlushInterval customizes the Pusher with the given flush interval. 169 | func WithFlushInterval(interval time.Duration) PushOption { 170 | return func(options *pushOptions) { 171 | options.flushInterval = interval 172 | } 173 | } 174 | 175 | // WithSyncPush enables the Pusher to push messages synchronously. 176 | func WithSyncPush() PushOption { 177 | return func(options *pushOptions) { 178 | options.syncPush = true 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /kq/pusher_test.go: -------------------------------------------------------------------------------- 1 | package kq 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | "time" 8 | 9 | "github.com/segmentio/kafka-go" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/mock" 12 | ) 13 | 14 | // mockKafkaWriter is a mock for kafka.Writer 15 | type mockKafkaWriter struct { 16 | mock.Mock 17 | } 18 | 19 | func (m *mockKafkaWriter) WriteMessages(ctx context.Context, msgs ...kafka.Message) error { 20 | args := m.Called(ctx, msgs) 21 | return args.Error(0) 22 | } 23 | 24 | func (m *mockKafkaWriter) Close() error { 25 | args := m.Called() 26 | return args.Error(0) 27 | } 28 | 29 | func TestNewPusher(t *testing.T) { 30 | addrs := []string{"localhost:9092"} 31 | topic := "test-topic" 32 | 33 | t.Run("DefaultOptions", func(t *testing.T) { 34 | pusher := NewPusher(addrs, topic) 35 | assert.NotNil(t, pusher) 36 | assert.NotNil(t, pusher.producer) 37 | assert.Equal(t, topic, pusher.topic) 38 | assert.NotNil(t, pusher.executor) 39 | }) 40 | 41 | t.Run("WithSyncPush", func(t *testing.T) { 42 | pusher := NewPusher(addrs, topic, WithSyncPush()) 43 | assert.NotNil(t, pusher) 44 | assert.NotNil(t, pusher.producer) 45 | assert.Equal(t, topic, pusher.topic) 46 | assert.Nil(t, pusher.executor) 47 | }) 48 | 49 | t.Run("WithChunkSize", func(t *testing.T) { 50 | pusher := NewPusher(addrs, topic, WithChunkSize(100)) 51 | assert.NotNil(t, pusher) 52 | assert.NotNil(t, pusher.executor) 53 | }) 54 | 55 | t.Run("WithFlushInterval", func(t *testing.T) { 56 | pusher := NewPusher(addrs, topic, WithFlushInterval(time.Second)) 57 | assert.NotNil(t, pusher) 58 | assert.NotNil(t, pusher.executor) 59 | }) 60 | 61 | t.Run("WithAllowAutoTopicCreation", func(t *testing.T) { 62 | pusher := NewPusher(addrs, topic, WithAllowAutoTopicCreation()) 63 | assert.NotNil(t, pusher) 64 | assert.True(t, pusher.producer.(*kafka.Writer).AllowAutoTopicCreation) 65 | }) 66 | } 67 | 68 | func TestPusher_Close(t *testing.T) { 69 | mockWriter := new(mockKafkaWriter) 70 | pusher := &Pusher{ 71 | producer: mockWriter, 72 | } 73 | 74 | mockWriter.On("Close").Return(nil) 75 | 76 | err := pusher.Close() 77 | assert.NoError(t, err) 78 | mockWriter.AssertExpectations(t) 79 | } 80 | 81 | func TestPusher_Name(t *testing.T) { 82 | topic := "test-topic" 83 | pusher := &Pusher{topic: topic} 84 | 85 | assert.Equal(t, topic, pusher.Name()) 86 | } 87 | 88 | func TestPusher_Push(t *testing.T) { 89 | mockWriter := new(mockKafkaWriter) 90 | pusher := &Pusher{ 91 | producer: mockWriter, 92 | topic: "test-topic", 93 | } 94 | 95 | ctx := context.Background() 96 | value := "test-value" 97 | 98 | mockWriter.On("WriteMessages", mock.Anything, mock.AnythingOfType("[]kafka.Message")).Return(nil) 99 | 100 | err := pusher.Push(ctx, value) 101 | assert.NoError(t, err) 102 | mockWriter.AssertExpectations(t) 103 | } 104 | 105 | func TestPusher_PushWithKey(t *testing.T) { 106 | mockWriter := new(mockKafkaWriter) 107 | pusher := &Pusher{ 108 | producer: mockWriter, 109 | topic: "test-topic", 110 | } 111 | 112 | ctx := context.Background() 113 | key := "test-key" 114 | value := "test-value" 115 | 116 | mockWriter.On("WriteMessages", mock.Anything, mock.AnythingOfType("[]kafka.Message")).Return(nil) 117 | 118 | err := pusher.PushWithKey(ctx, key, value) 119 | assert.NoError(t, err) 120 | mockWriter.AssertExpectations(t) 121 | } 122 | 123 | func TestPusher_PushWithKey_Error(t *testing.T) { 124 | mockWriter := new(mockKafkaWriter) 125 | pusher := &Pusher{ 126 | producer: mockWriter, 127 | topic: "test-topic", 128 | } 129 | 130 | ctx := context.Background() 131 | key := "test-key" 132 | value := "test-value" 133 | 134 | expectedError := errors.New("write error") 135 | mockWriter.On("WriteMessages", mock.Anything, mock.AnythingOfType("[]kafka.Message")).Return(expectedError) 136 | 137 | err := pusher.PushWithKey(ctx, key, value) 138 | assert.Error(t, err) 139 | assert.Equal(t, expectedError, err) 140 | mockWriter.AssertExpectations(t) 141 | } 142 | -------------------------------------------------------------------------------- /kq/queue.go: -------------------------------------------------------------------------------- 1 | package kq 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "crypto/x509" 7 | "errors" 8 | "io" 9 | "log" 10 | "os" 11 | "time" 12 | 13 | "github.com/segmentio/kafka-go" 14 | _ "github.com/segmentio/kafka-go/gzip" 15 | _ "github.com/segmentio/kafka-go/lz4" 16 | "github.com/segmentio/kafka-go/sasl/plain" 17 | _ "github.com/segmentio/kafka-go/snappy" 18 | "github.com/zeromicro/go-queue/kq/internal" 19 | "github.com/zeromicro/go-zero/core/contextx" 20 | "github.com/zeromicro/go-zero/core/logc" 21 | "github.com/zeromicro/go-zero/core/logx" 22 | "github.com/zeromicro/go-zero/core/queue" 23 | "github.com/zeromicro/go-zero/core/service" 24 | "github.com/zeromicro/go-zero/core/stat" 25 | "github.com/zeromicro/go-zero/core/threading" 26 | "github.com/zeromicro/go-zero/core/timex" 27 | "go.opentelemetry.io/otel" 28 | ) 29 | 30 | const ( 31 | defaultCommitInterval = time.Second 32 | defaultMaxWait = time.Second 33 | defaultQueueCapacity = 1000 34 | ) 35 | 36 | type ( 37 | ConsumeHandle func(ctx context.Context, key, value string) error 38 | 39 | ConsumeErrorHandler func(ctx context.Context, msg kafka.Message, err error) 40 | 41 | ConsumeHandler interface { 42 | Consume(ctx context.Context, key, value string) error 43 | } 44 | 45 | kafkaReader interface { 46 | FetchMessage(ctx context.Context) (kafka.Message, error) 47 | CommitMessages(ctx context.Context, msgs ...kafka.Message) error 48 | Close() error 49 | } 50 | 51 | queueOptions struct { 52 | commitInterval time.Duration 53 | queueCapacity int 54 | maxWait time.Duration 55 | metrics *stat.Metrics 56 | errorHandler ConsumeErrorHandler 57 | } 58 | 59 | QueueOption func(*queueOptions) 60 | 61 | kafkaQueue struct { 62 | c KqConf 63 | consumer kafkaReader 64 | handler ConsumeHandler 65 | channel chan kafka.Message 66 | producerRoutines *threading.RoutineGroup 67 | consumerRoutines *threading.RoutineGroup 68 | commitRunner *threading.StableRunner[kafka.Message, kafka.Message] 69 | metrics *stat.Metrics 70 | errorHandler ConsumeErrorHandler 71 | } 72 | 73 | kafkaQueues struct { 74 | queues []queue.MessageQueue 75 | group *service.ServiceGroup 76 | } 77 | ) 78 | 79 | func MustNewQueue(c KqConf, handler ConsumeHandler, opts ...QueueOption) queue.MessageQueue { 80 | q, err := NewQueue(c, handler, opts...) 81 | if err != nil { 82 | log.Fatal(err) 83 | } 84 | 85 | return q 86 | } 87 | 88 | func NewQueue(c KqConf, handler ConsumeHandler, opts ...QueueOption) (queue.MessageQueue, error) { 89 | if err := c.SetUp(); err != nil { 90 | return nil, err 91 | } 92 | 93 | var options queueOptions 94 | for _, opt := range opts { 95 | opt(&options) 96 | } 97 | ensureQueueOptions(c, &options) 98 | 99 | if c.Conns < 1 { 100 | c.Conns = 1 101 | } 102 | q := kafkaQueues{ 103 | group: service.NewServiceGroup(), 104 | } 105 | for i := 0; i < c.Conns; i++ { 106 | q.queues = append(q.queues, newKafkaQueue(c, handler, options)) 107 | } 108 | 109 | return q, nil 110 | } 111 | 112 | func newKafkaQueue(c KqConf, handler ConsumeHandler, options queueOptions) queue.MessageQueue { 113 | var offset int64 114 | if c.Offset == firstOffset { 115 | offset = kafka.FirstOffset 116 | } else { 117 | offset = kafka.LastOffset 118 | } 119 | 120 | readerConfig := kafka.ReaderConfig{ 121 | Brokers: c.Brokers, 122 | GroupID: c.Group, 123 | Topic: c.Topic, 124 | StartOffset: offset, 125 | MinBytes: c.MinBytes, // 10KB 126 | MaxBytes: c.MaxBytes, // 10MB 127 | MaxWait: options.maxWait, 128 | CommitInterval: options.commitInterval, 129 | QueueCapacity: options.queueCapacity, 130 | } 131 | if len(c.Username) > 0 && len(c.Password) > 0 { 132 | readerConfig.Dialer = &kafka.Dialer{ 133 | SASLMechanism: plain.Mechanism{ 134 | Username: c.Username, 135 | Password: c.Password, 136 | }, 137 | } 138 | } 139 | if len(c.CaFile) > 0 { 140 | caCert, err := os.ReadFile(c.CaFile) 141 | if err != nil { 142 | log.Fatal(err) 143 | } 144 | 145 | caCertPool := x509.NewCertPool() 146 | ok := caCertPool.AppendCertsFromPEM(caCert) 147 | if !ok { 148 | log.Fatal(err) 149 | } 150 | 151 | readerConfig.Dialer.TLS = &tls.Config{ 152 | RootCAs: caCertPool, 153 | InsecureSkipVerify: true, 154 | } 155 | } 156 | consumer := kafka.NewReader(readerConfig) 157 | 158 | q := &kafkaQueue{ 159 | c: c, 160 | consumer: consumer, 161 | handler: handler, 162 | channel: make(chan kafka.Message), 163 | producerRoutines: threading.NewRoutineGroup(), 164 | consumerRoutines: threading.NewRoutineGroup(), 165 | metrics: options.metrics, 166 | errorHandler: options.errorHandler, 167 | } 168 | if c.CommitInOrder { 169 | q.commitRunner = threading.NewStableRunner(func(msg kafka.Message) kafka.Message { 170 | if err := q.consumeOne(context.Background(), string(msg.Key), string(msg.Value)); err != nil { 171 | if q.errorHandler != nil { 172 | q.errorHandler(context.Background(), msg, err) 173 | } 174 | } 175 | 176 | return msg 177 | }) 178 | } 179 | 180 | return q 181 | } 182 | 183 | func (q *kafkaQueue) Start() { 184 | if q.c.CommitInOrder { 185 | go q.commitInOrder() 186 | 187 | if err := q.consume(func(msg kafka.Message) { 188 | if e := q.commitRunner.Push(msg); e != nil { 189 | logx.Error(e) 190 | } 191 | }); err != nil { 192 | logx.Error(err) 193 | } 194 | } else { 195 | q.startConsumers() 196 | q.startProducers() 197 | q.producerRoutines.Wait() 198 | close(q.channel) 199 | q.consumerRoutines.Wait() 200 | 201 | logx.Infof("Consumer %s is closed", q.c.Name) 202 | } 203 | } 204 | 205 | func (q *kafkaQueue) Stop() { 206 | q.consumer.Close() 207 | logx.Close() 208 | } 209 | 210 | func (q *kafkaQueue) consumeOne(ctx context.Context, key, val string) error { 211 | startTime := timex.Now() 212 | err := q.handler.Consume(ctx, key, val) 213 | q.metrics.Add(stat.Task{ 214 | Duration: timex.Since(startTime), 215 | }) 216 | return err 217 | } 218 | 219 | func (q *kafkaQueue) startConsumers() { 220 | for i := 0; i < q.c.Processors; i++ { 221 | q.consumerRoutines.Run(func() { 222 | for msg := range q.channel { 223 | // wrap message into message carrier 224 | mc := internal.NewMessageCarrier(internal.NewMessage(&msg)) 225 | // extract trace context from message 226 | ctx := otel.GetTextMapPropagator().Extract(context.Background(), mc) 227 | // remove deadline and error control 228 | ctx = contextx.ValueOnlyFrom(ctx) 229 | 230 | if err := q.consumeOne(ctx, string(msg.Key), string(msg.Value)); err != nil { 231 | if q.errorHandler != nil { 232 | q.errorHandler(ctx, msg, err) 233 | } 234 | 235 | if !q.c.ForceCommit { 236 | continue 237 | } 238 | } 239 | 240 | if err := q.consumer.CommitMessages(ctx, msg); err != nil { 241 | logc.Errorf(ctx, "commit failed, error: %v", err) 242 | } 243 | } 244 | }) 245 | } 246 | } 247 | 248 | func (q *kafkaQueue) startProducers() { 249 | for i := 0; i < q.c.Consumers; i++ { 250 | i := i 251 | q.producerRoutines.Run(func() { 252 | if err := q.consume(func(msg kafka.Message) { 253 | q.channel <- msg 254 | }); err != nil { 255 | logx.Infof("Consumer %s-%d is closed, error: %q", q.c.Name, i, err.Error()) 256 | return 257 | } 258 | }) 259 | } 260 | } 261 | 262 | func (q *kafkaQueue) consume(handle func(msg kafka.Message)) error { 263 | for { 264 | msg, err := q.consumer.FetchMessage(context.Background()) 265 | // io.EOF means consumer closed 266 | // io.ErrClosedPipe means committing messages on the consumer, 267 | // kafka will refire the messages on uncommitted messages, ignore 268 | if err == io.EOF || errors.Is(err, io.ErrClosedPipe) { 269 | return err 270 | } 271 | if err != nil { 272 | logx.Errorf("Error on reading message, %q", err.Error()) 273 | continue 274 | } 275 | 276 | handle(msg) 277 | } 278 | } 279 | 280 | func (q *kafkaQueue) commitInOrder() { 281 | for { 282 | msg, err := q.commitRunner.Get() 283 | if err != nil { 284 | logx.Error(err) 285 | return 286 | } 287 | 288 | if err := q.consumer.CommitMessages(context.Background(), msg); err != nil { 289 | logx.Errorf("commit failed, error: %v", err) 290 | } 291 | } 292 | } 293 | 294 | func (q kafkaQueues) Start() { 295 | for _, each := range q.queues { 296 | q.group.Add(each) 297 | } 298 | q.group.Start() 299 | } 300 | 301 | func (q kafkaQueues) Stop() { 302 | q.group.Stop() 303 | } 304 | 305 | func WithCommitInterval(interval time.Duration) QueueOption { 306 | return func(options *queueOptions) { 307 | options.commitInterval = interval 308 | } 309 | } 310 | 311 | func WithQueueCapacity(queueCapacity int) QueueOption { 312 | return func(options *queueOptions) { 313 | options.queueCapacity = queueCapacity 314 | } 315 | } 316 | 317 | func WithHandle(handle ConsumeHandle) ConsumeHandler { 318 | return innerConsumeHandler{ 319 | handle: handle, 320 | } 321 | } 322 | 323 | func WithMaxWait(wait time.Duration) QueueOption { 324 | return func(options *queueOptions) { 325 | options.maxWait = wait 326 | } 327 | } 328 | 329 | func WithMetrics(metrics *stat.Metrics) QueueOption { 330 | return func(options *queueOptions) { 331 | options.metrics = metrics 332 | } 333 | } 334 | 335 | func WithErrorHandler(errorHandler ConsumeErrorHandler) QueueOption { 336 | return func(options *queueOptions) { 337 | options.errorHandler = errorHandler 338 | } 339 | } 340 | 341 | type innerConsumeHandler struct { 342 | handle ConsumeHandle 343 | } 344 | 345 | func (ch innerConsumeHandler) Consume(ctx context.Context, k, v string) error { 346 | return ch.handle(ctx, k, v) 347 | } 348 | 349 | func ensureQueueOptions(c KqConf, options *queueOptions) { 350 | if options.commitInterval == 0 { 351 | options.commitInterval = defaultCommitInterval 352 | } 353 | if options.queueCapacity == 0 { 354 | options.queueCapacity = defaultQueueCapacity 355 | } 356 | if options.maxWait == 0 { 357 | options.maxWait = defaultMaxWait 358 | } 359 | if options.metrics == nil { 360 | options.metrics = stat.NewMetrics(c.Name) 361 | } 362 | if options.errorHandler == nil { 363 | options.errorHandler = func(ctx context.Context, msg kafka.Message, err error) { 364 | logc.Errorf(ctx, "consume: %s, error: %v", string(msg.Value), err) 365 | } 366 | } 367 | } 368 | -------------------------------------------------------------------------------- /kq/queue_test.go: -------------------------------------------------------------------------------- 1 | package kq 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "testing" 7 | "time" 8 | 9 | "github.com/segmentio/kafka-go" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/mock" 12 | "github.com/zeromicro/go-zero/core/service" 13 | "github.com/zeromicro/go-zero/core/stat" 14 | "github.com/zeromicro/go-zero/core/threading" 15 | ) 16 | 17 | // mockKafkaReader is a mock for kafka.Reader 18 | type mockKafkaReader struct { 19 | mock.Mock 20 | } 21 | 22 | func (m *mockKafkaReader) FetchMessage(ctx context.Context) (kafka.Message, error) { 23 | args := m.Called(ctx) 24 | return args.Get(0).(kafka.Message), args.Error(1) 25 | } 26 | 27 | func (m *mockKafkaReader) CommitMessages(ctx context.Context, msgs ...kafka.Message) error { 28 | args := m.Called(ctx, msgs) 29 | return args.Error(0) 30 | } 31 | 32 | func (m *mockKafkaReader) Close() error { 33 | args := m.Called() 34 | return args.Error(0) 35 | } 36 | 37 | // mockConsumeHandler is a mock for ConsumeHandler 38 | type mockConsumeHandler struct { 39 | mock.Mock 40 | } 41 | 42 | func (m *mockConsumeHandler) Consume(ctx context.Context, key, value string) error { 43 | args := m.Called(ctx, key, value) 44 | return args.Error(0) 45 | } 46 | 47 | func TestNewQueue(t *testing.T) { 48 | c := KqConf{ 49 | ServiceConf: service.ServiceConf{ 50 | Name: "test-queue", 51 | }, 52 | Brokers: []string{"localhost:9092"}, 53 | Group: "test-group", 54 | Topic: "test-topic", 55 | Offset: "first", 56 | Conns: 1, 57 | } 58 | handler := &mockConsumeHandler{} 59 | 60 | q, err := NewQueue(c, handler) 61 | assert.NoError(t, err) 62 | assert.NotNil(t, q) 63 | } 64 | 65 | func TestKafkaQueue_consumeOne(t *testing.T) { 66 | handler := &mockConsumeHandler{} 67 | q := &kafkaQueue{ 68 | handler: handler, 69 | metrics: stat.NewMetrics("test"), 70 | } 71 | 72 | ctx := context.Background() 73 | key := "test-key" 74 | value := "test-value" 75 | 76 | handler.On("Consume", ctx, key, value).Return(nil) 77 | 78 | err := q.consumeOne(ctx, key, value) 79 | assert.NoError(t, err) 80 | handler.AssertExpectations(t) 81 | } 82 | 83 | func TestKafkaQueue_consume(t *testing.T) { 84 | mockReader := &mockKafkaReader{} 85 | q := &kafkaQueue{ 86 | consumer: mockReader, 87 | } 88 | 89 | msg := kafka.Message{ 90 | Key: []byte("test-key"), 91 | Value: []byte("test-value"), 92 | } 93 | 94 | mockReader.On("FetchMessage", mock.Anything).Return(msg, nil).Once() 95 | mockReader.On("FetchMessage", mock.Anything).Return(kafka.Message{}, io.EOF).Once() 96 | 97 | called := false 98 | err := q.consume(func(msg kafka.Message) { 99 | called = true 100 | assert.Equal(t, "test-key", string(msg.Key)) 101 | assert.Equal(t, "test-value", string(msg.Value)) 102 | }) 103 | 104 | assert.Error(t, err) 105 | assert.True(t, called) 106 | mockReader.AssertExpectations(t) 107 | } 108 | 109 | func TestKafkaQueue_Start(t *testing.T) { 110 | mockReader := &mockKafkaReader{} 111 | handler := &mockConsumeHandler{} 112 | q := &kafkaQueue{ 113 | c: KqConf{ 114 | ServiceConf: service.ServiceConf{ 115 | Name: "test-queue", 116 | }, 117 | Processors: 1, 118 | Consumers: 1, 119 | }, 120 | consumer: mockReader, 121 | handler: handler, 122 | consumerRoutines: threading.NewRoutineGroup(), 123 | producerRoutines: threading.NewRoutineGroup(), 124 | channel: make(chan kafka.Message, 1), 125 | metrics: stat.NewMetrics("test"), 126 | } 127 | 128 | msg := kafka.Message{ 129 | Key: []byte("test-key"), 130 | Value: []byte("test-value"), 131 | } 132 | 133 | mockReader.On("FetchMessage", mock.Anything).Return(msg, nil).Once() 134 | mockReader.On("FetchMessage", mock.Anything).Return(kafka.Message{}, io.EOF).Once() 135 | handler.On("Consume", mock.Anything, "test-key", "test-value").Return(nil) 136 | mockReader.On("CommitMessages", mock.Anything, []kafka.Message{msg}).Return(nil) 137 | mockReader.On("Close").Return(nil) 138 | 139 | group := threading.NewRoutineGroup() 140 | group.Run(func() { 141 | time.Sleep(100 * time.Millisecond) 142 | q.Stop() 143 | }) 144 | 145 | q.Start() 146 | group.Wait() 147 | 148 | mockReader.AssertExpectations(t) 149 | handler.AssertExpectations(t) 150 | } 151 | 152 | func TestKafkaQueue_Stop(t *testing.T) { 153 | mockReader := &mockKafkaReader{} 154 | q := &kafkaQueue{ 155 | consumer: mockReader, 156 | } 157 | 158 | mockReader.On("Close").Return(nil) 159 | 160 | q.Stop() 161 | 162 | mockReader.AssertExpectations(t) 163 | } 164 | 165 | func TestWithCommitInterval(t *testing.T) { 166 | options := &queueOptions{} 167 | interval := time.Second * 5 168 | WithCommitInterval(interval)(options) 169 | assert.Equal(t, interval, options.commitInterval) 170 | } 171 | 172 | func TestWithQueueCapacity(t *testing.T) { 173 | options := &queueOptions{} 174 | capacity := 100 175 | WithQueueCapacity(capacity)(options) 176 | assert.Equal(t, capacity, options.queueCapacity) 177 | } 178 | 179 | func TestWithMaxWait(t *testing.T) { 180 | options := &queueOptions{} 181 | wait := time.Second * 2 182 | WithMaxWait(wait)(options) 183 | assert.Equal(t, wait, options.maxWait) 184 | } 185 | 186 | func TestWithMetrics(t *testing.T) { 187 | options := &queueOptions{} 188 | metrics := stat.NewMetrics("test") 189 | WithMetrics(metrics)(options) 190 | assert.Equal(t, metrics, options.metrics) 191 | } 192 | 193 | func TestWithErrorHandler(t *testing.T) { 194 | options := &queueOptions{} 195 | handler := func(ctx context.Context, msg kafka.Message, err error) {} 196 | WithErrorHandler(handler)(options) 197 | assert.NotNil(t, options.errorHandler) 198 | } 199 | -------------------------------------------------------------------------------- /natsq/config.go: -------------------------------------------------------------------------------- 1 | package natsq 2 | 3 | import ( 4 | "github.com/nats-io/nats.go" 5 | ) 6 | 7 | type NatsConfig struct { 8 | ServerUri string 9 | ClientName string 10 | Options []nats.Option 11 | } 12 | -------------------------------------------------------------------------------- /natsq/consumer.go: -------------------------------------------------------------------------------- 1 | package natsq 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "log" 7 | "sync" 8 | 9 | "github.com/nats-io/nats.go" 10 | "github.com/nats-io/nats.go/jetstream" 11 | "github.com/zeromicro/go-zero/core/logx" 12 | "github.com/zeromicro/go-zero/core/queue" 13 | ) 14 | 15 | const ( 16 | NatDefaultMode = iota 17 | NatJetMode 18 | ) 19 | 20 | type ( 21 | Msg struct { 22 | Subject string 23 | Data []byte 24 | } 25 | 26 | ConsumeHandle func(m *Msg) error 27 | 28 | // ConsumeHandler Consumer interface, used to define the methods required by the consumer 29 | ConsumeHandler interface { 30 | HandleMessage(m *Msg) error 31 | } 32 | 33 | // ConsumerQueue Consumer queue, used to maintain the relationship between a consumer queue 34 | ConsumerQueue struct { 35 | StreamName string // stream name 36 | QueueName string // queue name 37 | Subjects []string // Subscribe subject 38 | Consumer ConsumeHandler // consumer object 39 | JetOption []jetstream.PullConsumeOpt // Jetstream configuration 40 | } 41 | 42 | // ConsumerManager Consumer manager for managing multiple consumer queues 43 | ConsumerManager struct { 44 | mutex sync.RWMutex // read-write lock 45 | conn *nats.Conn // nats connect 46 | mode uint // nats mode 47 | queues []ConsumerQueue // consumer queue list 48 | options []nats.Option // Connection configuration items 49 | doneChan chan struct{} // close channel 50 | } 51 | ) 52 | 53 | // MustNewConsumerManager creates a new ConsumerManager instance. 54 | // It connects to NATS server, registers the provided consumer queues, and returns the ConsumerManager. 55 | // If any error occurs during the process, it logs the error and continues. 56 | func MustNewConsumerManager(cfg *NatsConfig, cq []*ConsumerQueue, mode uint) queue.MessageQueue { 57 | sc, err := nats.Connect(cfg.ServerUri, cfg.Options...) 58 | if err != nil { 59 | logx.Errorf("failed to connect nats, error: %v", err) 60 | } 61 | cm := &ConsumerManager{ 62 | conn: sc, 63 | options: cfg.Options, 64 | mode: mode, 65 | doneChan: make(chan struct{}), 66 | } 67 | if len(cq) == 0 { 68 | logx.Errorf("failed consumerQueue register to nats, error: cq len is 0") 69 | } 70 | for _, item := range cq { 71 | err = cm.registerQueue(item) 72 | if err != nil { 73 | logx.Errorf("failed to register nats, error: %v", err) 74 | } 75 | } 76 | 77 | return cm 78 | } 79 | 80 | // Start starts consuming messages from all the registered consumer queues. 81 | // It launches a goroutine for each consumer queue to subscribe and process messages. 82 | // The method blocks until the doneChan is closed. 83 | func (cm *ConsumerManager) Start() { 84 | cm.mutex.RLock() 85 | defer cm.mutex.RUnlock() 86 | 87 | if len(cm.queues) == 0 { 88 | logx.Errorf("no consumer queues found") 89 | } 90 | for _, consumerQueue := range cm.queues { 91 | go cm.subscribe(consumerQueue) 92 | } 93 | <-cm.doneChan 94 | } 95 | 96 | // Stop closes the NATS connection and stops the ConsumerManager. 97 | func (cm *ConsumerManager) Stop() { 98 | if cm.conn != nil { 99 | cm.conn.Close() 100 | } 101 | } 102 | 103 | // registerQueue registers a new consumer queue with the ConsumerManager. 104 | // It validates the required fields of the ConsumerQueue and adds it to the list of queues. 105 | // If any required field is missing, it returns an error. 106 | func (cm *ConsumerManager) registerQueue(queue *ConsumerQueue) error { 107 | cm.mutex.Lock() 108 | defer cm.mutex.Unlock() 109 | 110 | if cm.mode == NatJetMode && queue.StreamName == "" { 111 | return errors.New("stream name is required") 112 | } 113 | 114 | if queue.QueueName == "" { 115 | return errors.New("queue name is required") 116 | } 117 | if len(queue.Subjects) == 0 { 118 | return errors.New("subject is required") 119 | } 120 | if queue.Consumer == nil { 121 | return errors.New("consumer is required") 122 | } 123 | 124 | cm.queues = append(cm.queues, *queue) 125 | return nil 126 | } 127 | 128 | // subscribe subscribes to the specified consumer queue and starts processing messages. 129 | // If the NATS mode is NatJetMode, it creates a JetStream consumer and consumes messages using the provided options. 130 | // If the NATS mode is NatDefaultMode, it subscribes to the specified subjects using the queue name. 131 | // The method blocks until the doneChan is closed. 132 | func (cm *ConsumerManager) subscribe(queue ConsumerQueue) { 133 | ctx := context.Background() 134 | if cm.mode == NatJetMode { 135 | js, _ := jetstream.New(cm.conn) 136 | stream, err := js.Stream(ctx, "ccc") 137 | if err != nil { 138 | log.Fatalf("Error creating stream: %v", err) 139 | return 140 | } 141 | consumer, _ := stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ 142 | Name: queue.QueueName, 143 | AckPolicy: jetstream.AckExplicitPolicy, 144 | FilterSubjects: queue.Subjects, 145 | }) 146 | consContext, subErr := consumer.Consume(func(msg jetstream.Msg) { 147 | err := queue.Consumer.HandleMessage(&Msg{Subject: msg.Subject(), Data: msg.Data()}) 148 | if err != nil { 149 | logx.Errorf("error handling message: %v", err.Error()) 150 | } else { 151 | msg.Ack() 152 | } 153 | }, queue.JetOption...) 154 | if subErr != nil { 155 | logx.Errorf("error subscribing to queue %s: %v", queue.QueueName, subErr.Error()) 156 | return 157 | } 158 | defer consContext.Stop() 159 | } 160 | if cm.mode == NatDefaultMode { 161 | for _, subject := range queue.Subjects { 162 | cm.conn.QueueSubscribe(subject, queue.QueueName, func(m *nats.Msg) { 163 | err := queue.Consumer.HandleMessage(&Msg{Subject: m.Subject, Data: m.Data}) 164 | if err != nil { 165 | logx.Errorf("error handling message: %v", err.Error()) 166 | } else { 167 | m.Ack() 168 | } 169 | }) 170 | } 171 | } 172 | 173 | <-cm.doneChan 174 | } 175 | -------------------------------------------------------------------------------- /natsq/producer.go: -------------------------------------------------------------------------------- 1 | package natsq 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/nats-io/nats.go" 7 | "github.com/nats-io/nats.go/jetstream" 8 | ) 9 | 10 | type DefaultProducer struct { 11 | conn *nats.Conn 12 | } 13 | 14 | // NewDefaultProducer creates a new default NATS producer. 15 | // It takes a NatsConfig as input and returns a pointer to a DefaultProducer and an error. 16 | // It connects to the NATS server using the provided configuration. 17 | func NewDefaultProducer(c *NatsConfig) (*DefaultProducer, error) { 18 | sc, err := nats.Connect(c.ServerUri, c.Options...) 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | return &DefaultProducer{ 24 | conn: sc, 25 | }, nil 26 | } 27 | 28 | // Publish publishes a message with the specified subject and data using the default NATS producer. 29 | // It takes a subject string and data byte slice as input and returns an error if the publish fails. 30 | func (p *DefaultProducer) Publish(subject string, data []byte) error { 31 | return p.conn.Publish(subject, data) 32 | } 33 | 34 | // Close closes the NATS connection of the default producer. 35 | func (p *DefaultProducer) Close() { 36 | if p.conn != nil { 37 | p.conn.Close() 38 | } 39 | } 40 | 41 | type JetProducer struct { 42 | conn *nats.Conn 43 | js jetstream.JetStream 44 | ctx context.Context 45 | } 46 | 47 | // NewJetProducer creates a new JetStream producer. 48 | // It takes a NatsConfig as input and returns a pointer to a JetProducer and an error. 49 | // It connects to the NATS server using the provided configuration and creates a new JetStream context. 50 | func NewJetProducer(c *NatsConfig) (*JetProducer, error) { 51 | sc, err := nats.Connect(c.ServerUri, c.Options...) 52 | if err != nil { 53 | return nil, err 54 | } 55 | js, err := jetstream.New(sc) 56 | if err != nil { 57 | return nil, err 58 | } 59 | return &JetProducer{ 60 | conn: sc, 61 | js: js, 62 | }, nil 63 | } 64 | 65 | // CreateOrUpdateStream creates or updates a JetStream stream with the specified configuration. 66 | // It takes a jetstream.StreamConfig as input and returns an error if the operation fails. 67 | func (j *JetProducer) CreateOrUpdateStream(config jetstream.StreamConfig) error { 68 | _, err := j.js.CreateOrUpdateStream(j.ctx, config) 69 | if err != nil { 70 | return err 71 | } 72 | return nil 73 | } 74 | 75 | // Publish publishes a message with the specified subject and data using the JetStream producer. 76 | // It takes a subject string and data byte slice as input and returns an error if the publish fails. 77 | func (j *JetProducer) Publish(subject string, data []byte) error { 78 | _, err := j.js.Publish(j.ctx, subject, data) 79 | if err != nil { 80 | return err 81 | } 82 | return nil 83 | } 84 | 85 | // Close closes the NATS connection of the JetStream producer. 86 | func (j *JetProducer) Close() { 87 | j.conn.Close() 88 | } 89 | -------------------------------------------------------------------------------- /rabbitmq/config.go: -------------------------------------------------------------------------------- 1 | package rabbitmq 2 | 3 | import "fmt" 4 | 5 | type RabbitConf struct { 6 | Username string 7 | Password string 8 | Host string 9 | Port int 10 | VHost string `json:",optional"` 11 | } 12 | 13 | type RabbitListenerConf struct { 14 | RabbitConf 15 | ListenerQueues []ConsumerConf 16 | } 17 | 18 | type ConsumerConf struct { 19 | Name string 20 | AutoAck bool `json:",default=true"` 21 | Exclusive bool `json:",default=false"` 22 | // Set to true, which means that messages sent by producers in the same connection 23 | // cannot be delivered to consumers in this connection. 24 | NoLocal bool `json:",default=false"` 25 | // Whether to block processing 26 | NoWait bool `json:",default=false"` 27 | } 28 | 29 | type RabbitSenderConf struct { 30 | RabbitConf 31 | ContentType string `json:",default=text/plain"` // MIME content type 32 | } 33 | 34 | type QueueConf struct { 35 | Name string 36 | Durable bool `json:",default=true"` 37 | AutoDelete bool `json:",default=false"` 38 | Exclusive bool `json:",default=false"` 39 | NoWait bool `json:",default=false"` 40 | } 41 | 42 | type ExchangeConf struct { 43 | ExchangeName string 44 | Type string `json:",options=direct|fanout|topic|headers"` // exchange type 45 | Durable bool `json:",default=true"` 46 | AutoDelete bool `json:",default=false"` 47 | Internal bool `json:",default=false"` 48 | NoWait bool `json:",default=false"` 49 | Queues []QueueConf 50 | } 51 | 52 | func getRabbitURL(rabbitConf RabbitConf) string { 53 | return fmt.Sprintf("amqp://%s:%s@%s:%d/%s", rabbitConf.Username, rabbitConf.Password, 54 | rabbitConf.Host, rabbitConf.Port, rabbitConf.VHost) 55 | } 56 | -------------------------------------------------------------------------------- /rabbitmq/listener.go: -------------------------------------------------------------------------------- 1 | package rabbitmq 2 | 3 | import ( 4 | "log" 5 | 6 | amqp "github.com/rabbitmq/amqp091-go" 7 | "github.com/zeromicro/go-zero/core/logx" 8 | "github.com/zeromicro/go-zero/core/queue" 9 | ) 10 | 11 | type ( 12 | ConsumeHandle func(message string) error 13 | 14 | ConsumeHandler interface { 15 | Consume(message string) error 16 | } 17 | 18 | RabbitListener struct { 19 | conn *amqp.Connection 20 | channel *amqp.Channel 21 | forever chan bool 22 | handler ConsumeHandler 23 | queues RabbitListenerConf 24 | } 25 | ) 26 | 27 | func MustNewListener(listenerConf RabbitListenerConf, handler ConsumeHandler) queue.MessageQueue { 28 | listener := RabbitListener{queues: listenerConf, handler: handler, forever: make(chan bool)} 29 | conn, err := amqp.Dial(getRabbitURL(listenerConf.RabbitConf)) 30 | if err != nil { 31 | log.Fatalf("failed to connect rabbitmq, error: %v", err) 32 | } 33 | 34 | listener.conn = conn 35 | channel, err := listener.conn.Channel() 36 | if err != nil { 37 | log.Fatalf("failed to open a channel: %v", err) 38 | } 39 | 40 | listener.channel = channel 41 | return listener 42 | } 43 | 44 | func (q RabbitListener) Start() { 45 | for _, que := range q.queues.ListenerQueues { 46 | msg, err := q.channel.Consume( 47 | que.Name, 48 | "", 49 | que.AutoAck, 50 | que.Exclusive, 51 | que.NoLocal, 52 | que.NoWait, 53 | nil, 54 | ) 55 | if err != nil { 56 | log.Fatalf("failed to listener, error: %v", err) 57 | } 58 | 59 | go func() { 60 | for d := range msg { 61 | if err := q.handler.Consume(string(d.Body)); err != nil { 62 | logx.Errorf("Error on consuming: %s, error: %v", string(d.Body), err) 63 | } 64 | } 65 | }() 66 | } 67 | 68 | <-q.forever 69 | } 70 | 71 | func (q RabbitListener) Stop() { 72 | q.channel.Close() 73 | q.conn.Close() 74 | close(q.forever) 75 | } 76 | -------------------------------------------------------------------------------- /rabbitmq/rabbitmqadmin.go: -------------------------------------------------------------------------------- 1 | package rabbitmq 2 | 3 | import ( 4 | "log" 5 | 6 | amqp "github.com/rabbitmq/amqp091-go" 7 | ) 8 | 9 | type Admin struct { 10 | conn *amqp.Connection 11 | channel *amqp.Channel 12 | } 13 | 14 | func MustNewAdmin(rabbitMqConf RabbitConf) *Admin { 15 | var admin Admin 16 | conn, err := amqp.Dial(getRabbitURL(rabbitMqConf)) 17 | if err != nil { 18 | log.Fatalf("failed to connect rabbitmq, error: %v", err) 19 | } 20 | 21 | admin.conn = conn 22 | channel, err := admin.conn.Channel() 23 | if err != nil { 24 | log.Fatalf("failed to open a channel, error: %v", err) 25 | } 26 | 27 | admin.channel = channel 28 | return &admin 29 | } 30 | 31 | func (q *Admin) DeclareExchange(conf ExchangeConf, args amqp.Table) error { 32 | return q.channel.ExchangeDeclare( 33 | conf.ExchangeName, 34 | conf.Type, 35 | conf.Durable, 36 | conf.AutoDelete, 37 | conf.Internal, 38 | conf.NoWait, 39 | args, 40 | ) 41 | } 42 | 43 | func (q *Admin) DeclareQueue(conf QueueConf, args amqp.Table) error { 44 | _, err := q.channel.QueueDeclare( 45 | conf.Name, 46 | conf.Durable, 47 | conf.AutoDelete, 48 | conf.Exclusive, 49 | conf.NoWait, 50 | args, 51 | ) 52 | 53 | return err 54 | } 55 | 56 | func (q *Admin) Bind(queueName string, routekey string, exchange string, notWait bool, args amqp.Table) error { 57 | return q.channel.QueueBind( 58 | queueName, 59 | routekey, 60 | exchange, 61 | notWait, 62 | args, 63 | ) 64 | } 65 | -------------------------------------------------------------------------------- /rabbitmq/sender.go: -------------------------------------------------------------------------------- 1 | package rabbitmq 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | amqp "github.com/rabbitmq/amqp091-go" 8 | ) 9 | 10 | type ( 11 | Sender interface { 12 | Send(exchange string, routeKey string, msg []byte) error 13 | } 14 | 15 | RabbitMqSender struct { 16 | conn *amqp.Connection 17 | channel *amqp.Channel 18 | ContentType string 19 | } 20 | ) 21 | 22 | func MustNewSender(rabbitMqConf RabbitSenderConf) Sender { 23 | sender := &RabbitMqSender{ContentType: rabbitMqConf.ContentType} 24 | conn, err := amqp.Dial(getRabbitURL(rabbitMqConf.RabbitConf)) 25 | if err != nil { 26 | log.Fatalf("failed to connect rabbitmq, error: %v", err) 27 | } 28 | 29 | sender.conn = conn 30 | channel, err := sender.conn.Channel() 31 | if err != nil { 32 | log.Fatalf("failed to open a channel, error: %v", err) 33 | } 34 | 35 | sender.channel = channel 36 | return sender 37 | } 38 | 39 | func (q *RabbitMqSender) Send(exchange string, routeKey string, msg []byte) error { 40 | return q.channel.PublishWithContext( 41 | context.Background(), 42 | exchange, 43 | routeKey, 44 | false, 45 | false, 46 | amqp.Publishing{ 47 | ContentType: q.ContentType, 48 | Body: msg, 49 | }, 50 | ) 51 | } 52 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # go-queue 2 | 3 | ## dq 4 | 5 | High available beanstalkd. 6 | 7 | ### consumer example 8 | ```go 9 | consumer := dq.NewConsumer(dq.DqConf{ 10 | Beanstalks: []dq.Beanstalk{ 11 | { 12 | Endpoint: "localhost:11300", 13 | Tube: "tube", 14 | }, 15 | { 16 | Endpoint: "localhost:11300", 17 | Tube: "tube", 18 | }, 19 | }, 20 | Redis: redis.RedisConf{ 21 | Host: "localhost:6379", 22 | Type: redis.NodeType, 23 | }, 24 | }) 25 | consumer.Consume(func(body []byte) { 26 | fmt.Println(string(body)) 27 | }) 28 | ``` 29 | ### producer example 30 | ```go 31 | producer := dq.NewProducer([]dq.Beanstalk{ 32 | { 33 | Endpoint: "localhost:11300", 34 | Tube: "tube", 35 | }, 36 | { 37 | Endpoint: "localhost:11300", 38 | Tube: "tube", 39 | }, 40 | }) 41 | 42 | for i := 1000; i < 1005; i++ { 43 | _, err := producer.Delay([]byte(strconv.Itoa(i)), time.Second*5) 44 | if err != nil { 45 | fmt.Println(err) 46 | } 47 | } 48 | ``` 49 | 50 | ## kq 51 | 52 | Kafka Pub/Sub framework 53 | 54 | ### consumer example 55 | 56 | config.yaml 57 | ```yaml 58 | Name: kq 59 | Brokers: 60 | - 127.0.0.1:19092 61 | - 127.0.0.1:19092 62 | - 127.0.0.1:19092 63 | Group: adhoc 64 | Topic: kq 65 | Offset: first 66 | Consumers: 1 67 | ``` 68 | 69 | example code 70 | ```go 71 | var c kq.KqConf 72 | conf.MustLoad("config.json", &c) 73 | 74 | q := kq.MustNewQueue(c, kq.WithHandle(func(k, v string) error { 75 | fmt.Printf("=> %s\n", v) 76 | return nil 77 | })) 78 | defer q.Stop() 79 | q.Start() 80 | ``` 81 | 82 | ### producer example 83 | 84 | ```go 85 | type message struct { 86 | Key string `json:"key"` 87 | Value string `json:"value"` 88 | Payload string `json:"message"` 89 | } 90 | 91 | 92 | pusher := kq.NewPusher([]string{ 93 | "127.0.0.1:19092", 94 | "127.0.0.1:19092", 95 | "127.0.0.1:19092", 96 | }, "kq") 97 | 98 | ticker := time.NewTicker(time.Millisecond) 99 | for round := 0; round < 3; round++ { 100 | select { 101 | case <-ticker.C: 102 | count := rand.Intn(100) 103 | m := message{ 104 | Key: strconv.FormatInt(time.Now().UnixNano(), 10), 105 | Value: fmt.Sprintf("%d,%d", round, count), 106 | Payload: fmt.Sprintf("%d,%d", round, count), 107 | } 108 | body, err := json.Marshal(m) 109 | if err != nil { 110 | log.Fatal(err) 111 | } 112 | 113 | fmt.Println(string(body)) 114 | if err := pusher.Push(string(body)); err != nil { 115 | log.Fatal(err) 116 | } 117 | } 118 | } 119 | cmdline.EnterToContinue() 120 | ``` 121 | -------------------------------------------------------------------------------- /stanq/config.go: -------------------------------------------------------------------------------- 1 | package stanq 2 | 3 | import ( 4 | stan "github.com/nats-io/stan.go" 5 | ) 6 | 7 | type StanqConfig struct { 8 | ClusterID string `json:"cluster_id"` 9 | ClientID string `json:"client_id"` 10 | Options []stan.Option `json:"options"` 11 | } 12 | -------------------------------------------------------------------------------- /stanq/consumer.go: -------------------------------------------------------------------------------- 1 | package stanq 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | stan "github.com/nats-io/stan.go" 7 | "github.com/zeromicro/go-zero/core/logx" 8 | "github.com/zeromicro/go-zero/core/queue" 9 | "reflect" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | type ( 15 | ConsumeHandle func(m *stan.Msg) error 16 | 17 | // ConsumeHandler Consumer interface, used to define the methods required by the consumer 18 | ConsumeHandler interface { 19 | HandleMessage(m *stan.Msg) error 20 | } 21 | 22 | // ConsumerQueue Consumer queue, used to maintain the relationship between a consumer group and queue 23 | ConsumerQueue struct { 24 | GroupName string // consumer group name 25 | QueueName string // queue name 26 | Subject string // Subscribe subject 27 | Consumer ConsumeHandler // consumer object 28 | AckWaitTime int // Waiting time for Ack 29 | MaxInflight int // Maximum number of unacked messages 30 | ManualAckMode bool //Whether to manually ack 31 | Options []stan.SubscriptionOption // Subscription configuration item 32 | } 33 | 34 | // ConsumerManager Consumer manager for managing multiple consumer queues 35 | ConsumerManager struct { 36 | mutex sync.RWMutex // read-write lock 37 | conn stan.Conn // nats-streaming connect 38 | queues []ConsumerQueue // consumer queue list 39 | options []stan.Option // Connection configuration items 40 | doneChan chan struct{} // close channel 41 | } 42 | ) 43 | 44 | // MustNewConsumerManager 45 | func MustNewConsumerManager(cfg *StanqConfig, cq []*ConsumerQueue) queue.MessageQueue { 46 | sc, err := stan.Connect(cfg.ClusterID, cfg.ClientID, cfg.Options...) 47 | if err != nil { 48 | logx.Errorf("failed to connect stan, error: %v", err) 49 | } 50 | cm := &ConsumerManager{ 51 | conn: sc, 52 | options: cfg.Options, 53 | doneChan: make(chan struct{}), 54 | } 55 | if len(cq) == 0 { 56 | logx.Errorf("failed consumerQueue register to stan, error: cq len is 0") 57 | } 58 | for _, item := range cq { 59 | err = cm.registerQueue(item) 60 | if err != nil { 61 | logx.Errorf("failed to register stan, error: %v", err) 62 | } 63 | } 64 | 65 | return cm 66 | } 67 | 68 | // Start consuming messages in the queue 69 | func (cm *ConsumerManager) Start() { 70 | cm.mutex.RLock() 71 | defer cm.mutex.RUnlock() 72 | 73 | if len(cm.queues) == 0 { 74 | logx.Errorf("no consumer queues found") 75 | } 76 | for _, consumerQueue := range cm.queues { 77 | go cm.subscribe(consumerQueue) 78 | } 79 | <-cm.doneChan 80 | } 81 | 82 | // Stop close connect 83 | func (cm *ConsumerManager) Stop() { 84 | if cm.conn != nil { 85 | _ = cm.conn.Close() 86 | } 87 | } 88 | 89 | // RegisterQueue Register a consumer queue 90 | func (cm *ConsumerManager) registerQueue(queue *ConsumerQueue) error { 91 | cm.mutex.Lock() 92 | defer cm.mutex.Unlock() 93 | 94 | if queue.GroupName == "" { 95 | return errors.New("group name is required") 96 | } 97 | if queue.QueueName == "" { 98 | return errors.New("queue name is required") 99 | } 100 | if queue.Subject == "" { 101 | return errors.New("subject is required") 102 | } 103 | if queue.Consumer == nil { 104 | return errors.New("consumer is required") 105 | } 106 | 107 | cm.queues = append(cm.queues, *queue) 108 | return nil 109 | } 110 | 111 | // subscribe news 112 | func (cm *ConsumerManager) subscribe(queue ConsumerQueue) { 113 | var opts []stan.SubscriptionOption 114 | if queue.AckWaitTime > 0 { 115 | opts = append(opts, stan.AckWait(time.Duration(queue.AckWaitTime)*time.Second)) 116 | } 117 | if queue.MaxInflight > 0 { 118 | opts = append(opts, stan.MaxInflight(queue.MaxInflight)) 119 | } 120 | if len(queue.Options) > 0 { 121 | opts = append(opts, queue.Options...) 122 | } 123 | durableName := fmt.Sprintf("%s-%s", queue.GroupName, queue.QueueName) 124 | opts = append(opts, stan.DurableName(durableName)) 125 | if queue.ManualAckMode { 126 | opts = append(opts, stan.SetManualAckMode()) 127 | } 128 | 129 | sub, err := cm.conn.QueueSubscribe(queue.Subject, queue.QueueName, func(m *stan.Msg) { 130 | err := queue.Consumer.HandleMessage(m) 131 | if err != nil { 132 | logx.Errorf("error handling message: %v", err) 133 | } else { 134 | if queue.ManualAckMode { 135 | err := m.Ack() 136 | if err != nil { 137 | logx.Errorf("error acking message: %v", err) 138 | } 139 | } 140 | } 141 | }, opts...) 142 | if err != nil { 143 | logx.Errorf("error subscribing to queue %s: %v", queue.QueueName, err) 144 | return 145 | } 146 | <-cm.doneChan 147 | 148 | err = sub.Unsubscribe() 149 | if err != nil { 150 | logx.Errorf("error unsubscribing from queue %s: %v", queue.QueueName, err) 151 | } 152 | 153 | // delete consumer queue 154 | cm.mutex.Lock() 155 | defer cm.mutex.Unlock() 156 | for i, q := range cm.queues { 157 | if reflect.DeepEqual(q, queue) { 158 | cm.queues = append(cm.queues[:i], cm.queues[i+1:]...) 159 | break 160 | } 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /stanq/producer.go: -------------------------------------------------------------------------------- 1 | package stanq 2 | 3 | import "github.com/nats-io/stan.go" 4 | 5 | type Producer struct { 6 | conn stan.Conn 7 | } 8 | 9 | func NewProducer(c *StanqConfig) (*Producer, error) { 10 | sc, err := stan.Connect(c.ClusterID, c.ClientID, c.Options...) 11 | if err != nil { 12 | return nil, err 13 | } 14 | 15 | return &Producer{ 16 | conn: sc, 17 | }, nil 18 | } 19 | 20 | func (p *Producer) Publish(subject string, data []byte) error { 21 | return p.conn.Publish(subject, data) 22 | } 23 | 24 | func (p *Producer) AsyncPublish(subject string, data []byte, ackHandler func(guid string, err error)) (string, error) { 25 | return p.conn.PublishAsync(subject, data, ackHandler) 26 | } 27 | 28 | func (p *Producer) Close() { 29 | if p.conn != nil { 30 | _ = p.conn.Close() 31 | } 32 | } 33 | --------------------------------------------------------------------------------