├── .github ├── dependabot.yml └── workflows │ └── quality-gate.yml ├── .gitignore ├── .travis.yml ├── LICENSE.txt ├── Makefile ├── README.md ├── client.go ├── example ├── Makefile ├── docker-compose.yml ├── handler.go ├── main.go ├── readme.md └── service.go ├── go-kafka.go ├── go.mod ├── go.sum ├── instrumenting.go ├── instrumenting_test.go ├── listener.go ├── listener_test.go ├── mocks ├── consumer_group.go ├── consumer_group_claim.go ├── consumer_group_handler.go ├── consumer_group_session.go ├── producer.go ├── std_logger.go └── sync_producer.go ├── murmur.go ├── murmur_test.go ├── options.go ├── producer.go ├── producer_instrumenting.go └── producer_test.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directories: 5 | - "*" 6 | schedule: 7 | interval: "monthly" 8 | groups: 9 | critical-dependencies: 10 | patterns: 11 | - "github.com/IBM/sarama" 12 | - "github.com/opentracing/opentracing-go" 13 | - "github.com/prometheus/client_golang" 14 | - "github.com/stretchr/testify" 15 | update-types: 16 | - "minor" 17 | - "patch" 18 | other-dependencies: 19 | patterns: 20 | - "*" 21 | update-types: 22 | - "minor" 23 | - "patch" -------------------------------------------------------------------------------- /.github/workflows/quality-gate.yml: -------------------------------------------------------------------------------- 1 | name: Go Kafka Build and Test 2 | run-name: Go Kafka Build and Test by ${{ github.actor }} on ${{ github.head_ref }} 3 | 4 | on: 5 | push: 6 | branches: 7 | - master 8 | 9 | pull_request: 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.ref }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | build: 17 | runs-on: blacksmith-2vcpu-ubuntu-2204 18 | steps: 19 | - uses: actions/checkout@v4 20 | - name: Setup Go 21 | uses: actions/setup-go@v5 22 | with: 23 | go-version-file: go.mod 24 | cache-dependency-path: go.sum 25 | - name: Download dependencies 26 | run: go install ./... 27 | shell: sh 28 | - name: Run tests 29 | run: go test -count=1 -race -v ./... 30 | shell: sh 31 | 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | 14 | # ignore IDE 15 | .idea 16 | 17 | vendor -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - "1.19.x" 5 | 6 | notifications: 7 | email: false 8 | 9 | before_install: 10 | - go get github.com/mattn/goveralls 11 | 12 | script: 13 | - make install 14 | - make build 15 | - make test 16 | - $GOPATH/bin/goveralls -service=travis-ci -ignore "mocks/*" 17 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 ricardo.ch 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build 2 | build: 3 | CGO_ENABLED=0 go build -a -ldflags '-s' -installsuffix cgo 4 | 5 | .PHONY: test 6 | test: 7 | go test -race -v ./... 8 | 9 | 10 | .MOCKERY_PATH := $(shell [ -z "$${GOBIN}" ] && echo $${GOPATH}/bin/mockery || echo $${GOBIN}/mockery; ) 11 | 12 | .PHONY: mocks 13 | mocks: 14 | ifeq (,$(shell which mockery)) 15 | $(error "No mockery in PATH, consider doing brew install mockery") 16 | else 17 | go mod vendor 18 | mockery --case "underscore" --dir vendor/github.com/IBM/sarama --output ./mocks --case "underscore" --name="(ConsumerGroupHandler)|(SyncProducer)|(ConsumerGroup)|(ConsumerGroupClaim)|(ConsumerGroupSession)" 19 | mockery --case "underscore" --dir ./ --output ./mocks --name=StdLogger 20 | endif 21 | 22 | .PHONY: install 23 | install: 24 | go get ./... -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GO-KAFKA 2 | 3 | ![Build Status](https://github.com/ricardo-ch/go-kafka/actions/workflows/quality-gate.yml/badge.svg) 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/ricardo-ch/go-kafka)](https://goreportcard.com/report/github.com/ricardo-ch/go-kafka) 5 | 6 | Go-kafka provides an easy way to use kafka listeners and producers with only a few lines of code. 7 | The listener is able to consume from multiple topics, and will execute a separate handler for each topic. 8 | 9 | > 📘 Important note for v3 upgrade: 10 | > - The library now relies on the IBM/sarama library instead of Shopify/sarama, which is no longer maintained. 11 | > - The `kafka.Handler` type has been changed to a struct containing both the function to execute and the handler's optional configuration. 12 | > - The global variable `PushConsumerErrorsToTopic` has been replaced by the `PushConsumerErrorsToRetryTopic` and `PushConsumerErrorsToDeadletterTopic` properties on the handler. 13 | > 14 | > These two changes should be the only breaking changes in the v3 release. The rest of the library should be compatible with the previous version. 15 | 16 | ## Quick start 17 | 18 | Simple consumer 19 | ```golang 20 | // topic-specific handlers 21 | var handler1 kafka.Handler 22 | var handler2 kafka.Handler 23 | 24 | // map your topics to their handlers 25 | handlers := map[string]kafka.Handler{ 26 | "topic-1": handler1, 27 | "topic-2": handler2, 28 | } 29 | 30 | // define your listener 31 | kafka.Brokers = []string{"localhost:9092"} 32 | listener, _ := kafka.NewListener("my-consumer-group", handlers) 33 | defer listener.Close() 34 | 35 | // listen and enjoy 36 | errc <- listener.Listen(ctx) 37 | ``` 38 | 39 | Simple producer 40 | ```golang 41 | // define your producer 42 | kafka.Brokers = []string{"localhost:9092"} 43 | producer, _ := kafka.NewProducer() 44 | 45 | // send your message 46 | message := &sarama.ProducerMessage{ 47 | Topic: "my-topic", 48 | Value: sarama.StringEncoder("my-message"), 49 | } 50 | _ = producer.Produce(message) 51 | ``` 52 | 53 | ## Features 54 | 55 | * Create a listener on multiple topics 56 | * Retry policy on message handling 57 | * Create a producer 58 | * Prometheus instrumenting 59 | 60 | ## Consumer error handling 61 | 62 | You can customize the error handling of the consumer, using various patterns: 63 | * Blocking retries of the same event (Max number, and delay are configurable by handler) 64 | * Forward to retry topic for automatic retry without blocking the consumer 65 | * Forward to deadletter topic for manual investigation 66 | 67 | Here is the overall logic applied to handle errors: 68 | ```mermaid 69 | stateDiagram-v2 70 | 71 | init: Error processing an event 72 | state is_omitable_err <> 73 | skipWithoutCounting: Skip the event without impacting counters 74 | state is_retriable_err <> 75 | state is_deadletter_configured <> 76 | skip: Skip the event 77 | forwardDL: Forward to deadletter topic 78 | state should_retry <> 79 | blocking_retry : Blocking Retry of this event 80 | state is_retry_topic_configured <> 81 | state is_deadletter_configured2 <> 82 | forwardRQ: Forward to Retry topic 83 | skip2: Skip the event 84 | defaultDL: Forward to Deadletter topic 85 | 86 | init --> is_omitable_err 87 | is_omitable_err --> skipWithoutCounting: Error is of type ErrEventOmitted 88 | is_omitable_err --> is_retriable_err: Error is not an ErrEventOmitted 89 | is_retriable_err --> is_deadletter_configured: Error is of type ErrEventUnretriable 90 | is_retriable_err --> should_retry: Error is retriable 91 | should_retry --> blocking_retry: There are some retries left 92 | should_retry --> is_retry_topic_configured : No more blocking retry 93 | is_deadletter_configured --> skip: No Deadletter topic configured 94 | is_deadletter_configured --> forwardDL: Deadletter topic configured 95 | is_retry_topic_configured --> forwardRQ: Retry Topic Configured 96 | is_retry_topic_configured --> is_deadletter_configured2: No Retry Topic Configured 97 | is_deadletter_configured2 --> skip2: No Deadletter topic configured 98 | is_deadletter_configured2 --> defaultDL: Deadletter topic configured 99 | 100 | ``` 101 | ### Error types 102 | Two types of errors are introduced, so that application code can return them whenever relevant 103 | * `kafka.ErrEventUnretriable` - Errors that should not be retried 104 | * `kafka.ErrEventOmitted` - Errors that should lead to the event being omitted without impacting metrics 105 | 106 | All the other errors will be considered as "retryable" errors. 107 | 108 | Depending on the Retry topic/Deadletter topic/Max retries configuration, the event will be retried, forwarded to a retry topic, or forwarded to a deadletter topic. 109 | 110 | ### Blocking Retries 111 | 112 | By default, failed events consumptions will be retried 3 times (each attempt is separated by 2 seconds) with no exponential backoff. 113 | This can be globally configured through the following properties: 114 | * `ConsumerMaxRetries` (int) 115 | * `DurationBeforeRetry` (duration) 116 | 117 | These properties can also be configured on a per-topic basis by setting the `ConsumerMaxRetries`, `DurationBeforeRetry` and `ExponentialBackoff` properties on the handler. 118 | 119 | If you want to achieve a blocking retry pattern (ie. continuously retrying until the event is successfully consumed), you can set `ConsumerMaxRetries` to `InfiniteRetries` (-1). 120 | 121 | If you want to **not** retry specific errors, you can wrap them in a `kafka.ErrEventUnretriable` error before returning them, or return a `kafka.ErrNonRetriable` directly. 122 | ```go 123 | // This error will not be retried 124 | err := errors.New("my error") 125 | return errors.Wrap(kafka.ErrNonRetriable, err.Error()) 126 | 127 | // This error will also not be retried 128 | return kafka.ErrNonRetriable 129 | ``` 130 | 131 | #### exponential backoff 132 | You can activate it by setting `ExponentialBackoff` config variable as true. You can set this properties as global, you have to use the configuration per-topic. This configuration is useful in case of infinite retry configuration. 133 | The exponential backoff algorithm is defined like this. 134 | 135 | $`retryDuration = durationBeforeRetry * 2^{retries}`$ 136 | 137 | ### Deadletter And Retry topics 138 | 139 | By default, events that have exceeded the maximum number of blocking retries will be pushed to a retry topic or dead letter topic. 140 | This behaviour can be disabled through the `PushConsumerErrorsToRetryTopic` and `PushConsumerErrorsToDeadletterTopic` properties. 141 | ```go 142 | PushConsumerErrorsToRetryTopic = false 143 | PushConsumerErrorsToDeadletterTopic = false 144 | ``` 145 | If these switches are ON, the names of the deadletter and retry topics are dynamically generated based on the original topic name and the consumer group. 146 | For example, if the original topic is `my-topic` and the consumer group is `my-consumer-group`, the deadletter topic will be `my-consumer-group-my-topic-deadletter`. 147 | This pattern can be overridden through the `ErrorTopicPattern` property. 148 | Also, the retry and deadletter topics name can be overridden through the `RetryTopic` and `DeadLetterTopic` properties on the handler. 149 | 150 | Note that, if global `PushConsumerErrorsToRetryTopic` or `PushConsumerErrorsToDeadletterTopic` property are false, but you configure `RetryTopic` or `DeadLetterTopic` properties on a handler, then the events in error will be forwarder to the error topics only for this handler. 151 | 152 | ### Omitting specific errors 153 | 154 | In certain scenarios, you might want to omit some errors. For example, you might want to discard outdated events that are not relevant anymore. 155 | Such events would increase a separate, dedicated metric instead of the error one, and would not be retried. 156 | To do so, wrap the errors that should lead to omitted events in a ErrEventOmitted, or return a kafka.ErrEventOmitted directly. 157 | ```go 158 | // This error will be omitted 159 | err := errors.New("my error") 160 | return errors.Wrap(kafka.ErrEventOmitted, err.Error()) 161 | 162 | // This error will also be omitted 163 | return kafka.ErrEventOmitted 164 | ``` 165 | 166 | ## Instrumenting 167 | 168 | Metrics for the listener and the producer can be exported to Prometheus. 169 | The following metrics are available: 170 | | Metric name | Labels | Description | 171 | |-------------|--------|-------------| 172 | | kafka_consumer_record_consumed_total | kafka_topic, consumer_group | Number of messages consumed | 173 | | kafka_consumer_record_latency_seconds | kafka_topic, consumer_group | Latency of consuming a message | 174 | | kafka_consumer_record_omitted_total | kafka_topic, consumer_group | Number of messages omitted | 175 | | kafka_consumer_record_error_total | kafka_topic, consumer_group | Number of errors when consuming a message | 176 | | kafka_consumergroup_current_message_timestamp| kafka_topic, consumer_group, partition, type | Timestamp of the current message being processed. Type can be either of `LogAppendTime` or `CreateTime`. | 177 | | kafka_producer_record_send_total | kafka_topic | Number of messages sent | 178 | | kafka_producer_dead_letter_created_total | kafka_topic | Number of messages sent to a dead letter topic | 179 | | kafka_producer_record_error_total | kafka_topic | Number of errors when sending a message | 180 | 181 | To activate the tracing on go-Kafka: 182 | 183 | ```golang 184 | // define your listener 185 | listener, _ := kafka.NewListener(brokers, "my-consumer-group", handlers, kafka.WithInstrumenting()) 186 | defer listener.Close() 187 | 188 | // Instances a new HTTP server for metrics using prometheus 189 | go func() { 190 | httpAddr := ":8080" 191 | mux.Handle("/metrics", promhttp.Handler()) 192 | errc <- http.ListenAndServe(httpAddr, mux) 193 | }() 194 | 195 | ``` 196 | 197 | ## Default configuration 198 | 199 | Configuration of consumer/producer is opinionated. It aim to resolve simply problems that have taken us by surprise in the past. 200 | For this reason: 201 | - the default partioner is based on murmur2 instead of the one sarama use by default 202 | - offset retention is set to 30 days 203 | - initial offset is oldest 204 | 205 | ## License 206 | 207 | go-kafka is licensed under the MIT license. (http://opensource.org/licenses/MIT) 208 | 209 | ## Contributing 210 | 211 | Pull requests are the way to help us here. We will be really grateful. 212 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | 7 | "github.com/IBM/sarama" 8 | ) 9 | 10 | var ( 11 | client *sarama.Client 12 | clientMutex = &sync.Mutex{} 13 | ) 14 | 15 | func getClient() (*sarama.Client, error) { 16 | if client != nil { 17 | return client, nil 18 | } 19 | 20 | clientMutex.Lock() 21 | defer clientMutex.Unlock() 22 | 23 | if client == nil { 24 | var c sarama.Client 25 | var err error 26 | 27 | if len(Brokers) == 0 { 28 | return nil, errors.New("cannot create new client, Brokers must be specified") 29 | } 30 | c, err = sarama.NewClient(Brokers, Config) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | client = &c 36 | } 37 | 38 | return client, nil 39 | } 40 | -------------------------------------------------------------------------------- /example/Makefile: -------------------------------------------------------------------------------- 1 | run_example: 2 | docker compose up -d 3 | @make send_msg > /dev/null # this part blocks until the broker is up 4 | @echo "----" 5 | @echo "kafka is running" 6 | @echo "you can execute 'go run .' to see the application consuming messages" 7 | @echo "----" 8 | @while true; do make send_msg; sleep 5; done; 9 | 10 | send_msg: 11 | docker exec -it kafka bash -c 'echo "{\"Content\":\"hello\"}" | kafka-console-producer --bootstrap-server localhost:9092 --topic test-users' 12 | -------------------------------------------------------------------------------- /example/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.9' 2 | services: 3 | kafka: 4 | image: landoop/fast-data-dev:2.6.2 5 | container_name: kafka 6 | ports: 7 | - 2181:2181 # zookeeper 8 | - 3030:3030 # ui 9 | - 9092:9092 # broker 10 | - 8081:8081 # schema registry 11 | - 8082:8082 # rest proxy 12 | - 8083:8083 # kafka connect 13 | environment: 14 | - ADV_HOST=localhost 15 | - SAMPLEDATA=0 16 | - RUNNING_SAMPLEDATA=0 17 | - RUNTESTS=0 18 | - FORWARDLOGS=0 19 | - DISABLE_JMX=1 20 | - DEBUG=1 21 | - SUPERVISORWEB=0 -------------------------------------------------------------------------------- /example/handler.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "time" 7 | 8 | "github.com/IBM/sarama" 9 | "github.com/ricardo-ch/go-kafka/v3" 10 | ) 11 | 12 | func makeUserHandler(s Service) kafka.Handler { 13 | return kafka.Handler{ 14 | Processor: func(ctx context.Context, msg *sarama.ConsumerMessage) error { 15 | parsedMsg, err := decodeUserEvent(msg.Value) 16 | if err != nil { 17 | return err 18 | } 19 | 20 | return s.OnUserEvent(parsedMsg) 21 | }, 22 | Config: kafka.HandlerConfig{ 23 | ConsumerMaxRetries: kafka.Ptr(2), 24 | DurationBeforeRetry: kafka.Ptr(5 * time.Second), 25 | ExponentialBackoff: true, 26 | }, 27 | } 28 | } 29 | 30 | func decodeUserEvent(data []byte) (UserEvent, error) { 31 | parsedMsg := UserEvent{} 32 | err := json.Unmarshal(data, &parsedMsg) 33 | return parsedMsg, err 34 | } 35 | -------------------------------------------------------------------------------- /example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "github.com/ricardo-ch/go-kafka/v3" 8 | ) 9 | 10 | var ( 11 | brokers = []string{"localhost:9092"} 12 | appName = "example-kafka" 13 | ) 14 | 15 | func main() { 16 | handlers := kafka.Handlers{} 17 | handlers["test-users"] = makeUserHandler(NewService()) 18 | kafka.Brokers = brokers 19 | 20 | listener, err := kafka.NewListener(appName, handlers) 21 | if err != nil { 22 | log.Fatalln("could not initialise listener:", err) 23 | } 24 | 25 | err = listener.Listen(context.Background()) 26 | if err != nil { 27 | log.Fatalln("listener closed with error:", err) 28 | } 29 | log.Println("listener stopped") 30 | } 31 | -------------------------------------------------------------------------------- /example/readme.md: -------------------------------------------------------------------------------- 1 | ## How to run this example 2 | 3 | run `make run_example` 4 | 5 | then run `go run .` in another terminal -------------------------------------------------------------------------------- /example/service.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | type UserEvent struct { 6 | Content string 7 | } 8 | 9 | type Service interface { 10 | OnUserEvent(UserEvent) error 11 | } 12 | 13 | func NewService() Service { 14 | return service{} 15 | } 16 | 17 | type service struct{} 18 | 19 | func (s service) OnUserEvent(msg UserEvent) error { 20 | fmt.Println("received this message", msg) 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /go-kafka.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "os" 7 | "time" 8 | 9 | "github.com/IBM/sarama" 10 | ) 11 | 12 | // Brokers is the list of Kafka brokers to connect to. 13 | var Brokers []string 14 | 15 | // StdLogger is used to log messages. 16 | 17 | // StdLogger is the interface used to log messages. 18 | // Print and println provides this type of log. 19 | // print(ctx, err, "key", "value") 20 | // print(err, "key", "value") 21 | // print(ctx, "key", "value") 22 | // print(ctx, err) 23 | type StdLogger interface { 24 | Print(v ...interface{}) 25 | Printf(format string, v ...interface{}) 26 | Println(v ...interface{}) 27 | } 28 | 29 | // Logger is the instance of a StdLogger interface. 30 | // By default it is set to discard all log messages via ioutil.Discard, 31 | // but you can set it to redirect wherever you want. 32 | var Logger StdLogger = log.New(io.Discard, "[Go-Kafka] ", log.LstdFlags) 33 | 34 | // ErrorLogger is the instance of a StdLogger interface. 35 | // By default it is set to output on stderr all log messages, 36 | // but you can set it to redirect wherever you want. 37 | var ErrorLogger StdLogger = log.New(os.Stderr, "[Go-Kafka] ", log.LstdFlags) 38 | 39 | // ConsumerMaxRetries is the maximum number of time we want to retry 40 | // to process an event before throwing the error. 41 | // By default 3 times. 42 | var ConsumerMaxRetries = 3 43 | 44 | // InfiniteRetries is a constant to define infinite retries. 45 | // It is used to set the ConsumerMaxRetries to a blocking retry process. 46 | const InfiniteRetries = -1 47 | 48 | // DurationBeforeRetry is the duration we wait between process retries. 49 | // By default 2 seconds. 50 | var DurationBeforeRetry = 2 * time.Second 51 | 52 | // PushConsumerErrorsToRetryTopic is a boolean to define if messages in error have to be pushed to a retry topic. 53 | var PushConsumerErrorsToRetryTopic = true 54 | 55 | // PushConsumerErrorsToDeadletterTopic is a boolean to define if messages in error have to be pushed to a deadletter topic. 56 | var PushConsumerErrorsToDeadletterTopic = true 57 | 58 | // RetryTopicPattern is the retry topic name pattern. 59 | // By default "consumergroup-topicname-retry" 60 | // Use $$CG$$ as consumer group placeholder 61 | // Use $$T$$ as original topic name placeholder 62 | var RetryTopicPattern = "$$CG$$-$$T$$-retry" 63 | 64 | // DeadletterTopicPattern is the deadletter topic name pattern. 65 | // By default "consumergroup-topicname-deadletter" 66 | // Use $$CG$$ as consumer group placeholder 67 | // Use $$T$$ as original topic name placeholder 68 | var DeadletterTopicPattern = "$$CG$$-$$T$$-deadletter" 69 | 70 | // Config is the sarama (cluster) config used for the consumer and producer. 71 | var Config = sarama.NewConfig() 72 | 73 | func init() { 74 | // Init config with default values 75 | Config.Consumer.Return.Errors = true 76 | Config.Consumer.Offsets.Initial = sarama.OffsetOldest 77 | Config.Consumer.Offsets.Retention = 30 * 24 * time.Hour // 30 days, because we tend to increase the retention of a topic to a few weeks for practical purpose 78 | Config.Producer.Timeout = 5 * time.Second 79 | Config.Producer.Retry.Max = 3 80 | Config.Producer.Return.Successes = true 81 | Config.Producer.RequiredAcks = sarama.WaitForAll 82 | Config.Producer.Partitioner = NewJVMCompatiblePartitioner 83 | Config.Version = sarama.V1_1_1_0 84 | } 85 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ricardo-ch/go-kafka/v3 2 | 3 | go 1.23.0 4 | 5 | require ( 6 | github.com/IBM/sarama v1.45.2 7 | github.com/opentracing/opentracing-go v1.2.0 8 | github.com/prometheus/client_golang v1.22.0 9 | github.com/ricardo-ch/go-tracing v0.5.1 10 | github.com/stretchr/testify v1.10.0 11 | ) 12 | 13 | require ( 14 | github.com/beorn7/perks v1.0.1 // indirect 15 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 16 | github.com/davecgh/go-spew v1.1.1 // indirect 17 | github.com/eapache/go-resiliency v1.7.0 // indirect 18 | github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect 19 | github.com/eapache/queue v1.1.0 // indirect 20 | github.com/go-kit/kit v0.13.0 // indirect 21 | github.com/golang/snappy v1.0.0 // indirect 22 | github.com/hashicorp/errwrap v1.1.0 // indirect 23 | github.com/hashicorp/go-multierror v1.1.1 // indirect 24 | github.com/hashicorp/go-uuid v1.0.3 // indirect 25 | github.com/jcmturner/aescts/v2 v2.0.0 // indirect 26 | github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect 27 | github.com/jcmturner/gofork v1.7.6 // indirect 28 | github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect 29 | github.com/jcmturner/rpc/v2 v2.0.3 // indirect 30 | github.com/klauspost/compress v1.18.0 // indirect 31 | github.com/kr/text v0.2.0 // indirect 32 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 33 | github.com/pierrec/lz4/v4 v4.1.22 // indirect 34 | github.com/pkg/errors v0.9.1 // indirect 35 | github.com/pmezard/go-difflib v1.0.0 // indirect 36 | github.com/prometheus/client_model v0.6.1 // indirect 37 | github.com/prometheus/common v0.62.0 // indirect 38 | github.com/prometheus/procfs v0.15.1 // indirect 39 | github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect 40 | github.com/rogpeppe/go-internal v1.12.0 // indirect 41 | github.com/stretchr/objx v0.5.2 // indirect 42 | github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect 43 | github.com/uber/jaeger-lib v2.4.1+incompatible // indirect 44 | go.uber.org/atomic v1.11.0 // indirect 45 | golang.org/x/crypto v0.39.0 // indirect 46 | golang.org/x/net v0.41.0 // indirect 47 | golang.org/x/sys v0.33.0 // indirect 48 | google.golang.org/protobuf v1.36.5 // indirect 49 | gopkg.in/yaml.v3 v3.0.1 // indirect 50 | ) 51 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= 2 | github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= 3 | github.com/IBM/sarama v1.45.2 h1:8m8LcMCu3REcwpa7fCP6v2fuPuzVwXDAM2DOv3CBrKw= 4 | github.com/IBM/sarama v1.45.2/go.mod h1:ppaoTcVdGv186/z6MEKsMm70A5fwJfRTpstI37kVn3Y= 5 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 6 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 7 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 8 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 9 | github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= 10 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 11 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= 15 | github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= 16 | github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= 17 | github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= 18 | github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= 19 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 20 | github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= 21 | github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= 22 | github.com/go-kit/kit v0.7.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 23 | github.com/go-kit/kit v0.13.0 h1:OoneCcHKHQ03LfBpoQCUfCluwd2Vt3ohz+kvbJneZAU= 24 | github.com/go-kit/kit v0.13.0/go.mod h1:phqEHMMUbyrCFCTgH48JueqrM3md2HcAZ8N3XE4FKDg= 25 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 26 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 27 | github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= 28 | github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 29 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 30 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 31 | github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= 32 | github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= 33 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 34 | github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= 35 | github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 36 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 37 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 38 | github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 39 | github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= 40 | github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 41 | github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= 42 | github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= 43 | github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= 44 | github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= 45 | github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= 46 | github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= 47 | github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= 48 | github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= 49 | github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= 50 | github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= 51 | github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= 52 | github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= 53 | github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 54 | github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 55 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 56 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 57 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 58 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 59 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 60 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 61 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 62 | github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 63 | github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= 64 | github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= 65 | github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= 66 | github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 67 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 68 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 69 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 70 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 71 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 72 | github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= 73 | github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= 74 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 75 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 76 | github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= 77 | github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= 78 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 79 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 80 | github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg= 81 | github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 82 | github.com/ricardo-ch/go-tracing v0.5.1 h1:uUQERGJ/00wlOyOCF/RbGinRQFkEr+0gHrcPESqb5sk= 83 | github.com/ricardo-ch/go-tracing v0.5.1/go.mod h1:lTRigtf3AQ3s4UhkJ2hTUBHxzKG6aS1iAW0MdIS7Q34= 84 | github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= 85 | github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= 86 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 87 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 88 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 89 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 90 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 91 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 92 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 93 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 94 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 95 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 96 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 97 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 98 | github.com/uber/jaeger-client-go v2.23.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= 99 | github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= 100 | github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= 101 | github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= 102 | github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= 103 | github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= 104 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 105 | go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= 106 | go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= 107 | go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= 108 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 109 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 110 | golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= 111 | golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= 112 | golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= 113 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 114 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 115 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 116 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 117 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 118 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 119 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 120 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 121 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 122 | golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= 123 | golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= 124 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 125 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 126 | golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= 127 | golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 128 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 129 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 130 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 131 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 132 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 133 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 134 | golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= 135 | golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 136 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 137 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 138 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 139 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 140 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 141 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 142 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 143 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 144 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 145 | golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 146 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 147 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 148 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 149 | google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= 150 | google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 151 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 152 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 153 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 154 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 155 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 156 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 157 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 158 | -------------------------------------------------------------------------------- /instrumenting.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | 8 | "github.com/IBM/sarama" 9 | "github.com/prometheus/client_golang/prometheus" 10 | ) 11 | 12 | const ( 13 | TimestampTypeLogAppendTime = "LogAppendTime" 14 | TimestampTypeCreateTime = "CreateTime" 15 | ) 16 | 17 | var ( 18 | consumerRecordConsumedCounter *prometheus.CounterVec 19 | consumerRecordConsumedLatency *prometheus.HistogramVec 20 | consumerRecordErrorCounter *prometheus.CounterVec 21 | consumerRecordOmittedCounter *prometheus.CounterVec 22 | 23 | consumergroupCurrentMessageTimestamp *prometheus.GaugeVec 24 | 25 | consumerMetricsMutex = &sync.Mutex{} 26 | consumerMetricLabels = []string{"kafka_topic", "consumer_group"} 27 | ) 28 | 29 | // ConsumerMetricsService object represents consumer metrics 30 | type ConsumerMetricsService struct { 31 | groupID string 32 | 33 | recordConsumedCounter *prometheus.CounterVec 34 | recordConsumedLatency *prometheus.HistogramVec 35 | recordErrorCounter *prometheus.CounterVec 36 | recordOmittedCounter *prometheus.CounterVec 37 | 38 | currentMessageTimestamp *prometheus.GaugeVec 39 | } 40 | 41 | func getPrometheusRecordConsumedInstrumentation() *prometheus.CounterVec { 42 | if consumerRecordConsumedCounter != nil { 43 | return consumerRecordConsumedCounter 44 | } 45 | 46 | consumerMetricsMutex.Lock() 47 | defer consumerMetricsMutex.Unlock() 48 | if consumerRecordConsumedCounter == nil { 49 | consumerRecordConsumedCounter = prometheus.NewCounterVec( 50 | prometheus.CounterOpts{ 51 | Namespace: "kafka", 52 | Subsystem: "consumer", 53 | Name: "record_consumed_total", 54 | Help: "Number of records consumed", 55 | }, consumerMetricLabels) 56 | prometheus.MustRegister(consumerRecordConsumedCounter) 57 | } 58 | 59 | return consumerRecordConsumedCounter 60 | } 61 | 62 | func getPrometheusRecordConsumedLatencyInstrumentation() *prometheus.HistogramVec { 63 | if consumerRecordConsumedLatency != nil { 64 | return consumerRecordConsumedLatency 65 | } 66 | 67 | consumerMetricsMutex.Lock() 68 | defer consumerMetricsMutex.Unlock() 69 | if consumerRecordConsumedLatency == nil { 70 | consumerRecordConsumedLatency = prometheus.NewHistogramVec( 71 | prometheus.HistogramOpts{ 72 | Namespace: "kafka", 73 | Subsystem: "consumer", 74 | Name: "record_latency_seconds", 75 | Help: "Total duration in milliseconds", 76 | }, consumerMetricLabels) 77 | prometheus.MustRegister(consumerRecordConsumedLatency) 78 | } 79 | 80 | return consumerRecordConsumedLatency 81 | } 82 | 83 | func getPrometheusRecordConsumedErrorInstrumentation() *prometheus.CounterVec { 84 | if consumerRecordErrorCounter != nil { 85 | return consumerRecordErrorCounter 86 | } 87 | 88 | consumerMetricsMutex.Lock() 89 | defer consumerMetricsMutex.Unlock() 90 | if consumerRecordErrorCounter == nil { 91 | consumerRecordErrorCounter = prometheus.NewCounterVec( 92 | prometheus.CounterOpts{ 93 | Namespace: "kafka", 94 | Subsystem: "consumer", 95 | Name: "record_error_total", 96 | Help: "Number of requests dropped", 97 | }, consumerMetricLabels) 98 | prometheus.MustRegister(consumerRecordErrorCounter) 99 | } 100 | 101 | return consumerRecordErrorCounter 102 | } 103 | 104 | func getPrometheusRecordOmittedInstrumentation() *prometheus.CounterVec { 105 | if consumerRecordOmittedCounter != nil { 106 | return consumerRecordOmittedCounter 107 | } 108 | 109 | consumerMetricsMutex.Lock() 110 | defer consumerMetricsMutex.Unlock() 111 | if consumerRecordOmittedCounter == nil { 112 | consumerRecordOmittedCounter = prometheus.NewCounterVec( 113 | prometheus.CounterOpts{ 114 | Namespace: "kafka", 115 | Subsystem: "consumer", 116 | Name: "record_omitted_total", 117 | Help: "Number of requests dropped", 118 | }, consumerMetricLabels) 119 | prometheus.MustRegister(consumerRecordOmittedCounter) 120 | } 121 | 122 | return consumerRecordOmittedCounter 123 | } 124 | 125 | func getPrometheusCurrentMessageTimestampInstrumentation() *prometheus.GaugeVec { 126 | if consumergroupCurrentMessageTimestamp != nil { 127 | return consumergroupCurrentMessageTimestamp 128 | } 129 | 130 | consumerMetricsMutex.Lock() 131 | defer consumerMetricsMutex.Unlock() 132 | if consumergroupCurrentMessageTimestamp == nil { 133 | consumergroupCurrentMessageTimestamp = prometheus.NewGaugeVec( 134 | prometheus.GaugeOpts{ 135 | Namespace: "kafka", 136 | Subsystem: "consumergroup", 137 | Name: "current_message_timestamp", 138 | Help: "Current message timestamp", 139 | }, []string{"kafka_topic", "consumer_group", "partition", "type"}) 140 | prometheus.MustRegister(consumergroupCurrentMessageTimestamp) 141 | } 142 | 143 | return consumergroupCurrentMessageTimestamp 144 | } 145 | 146 | // NewConsumerMetricsService creates a layer of service that add metrics capability 147 | func NewConsumerMetricsService(groupID string) *ConsumerMetricsService { 148 | return &ConsumerMetricsService{ 149 | groupID: groupID, 150 | recordConsumedCounter: getPrometheusRecordConsumedInstrumentation(), 151 | recordConsumedLatency: getPrometheusRecordConsumedLatencyInstrumentation(), 152 | recordErrorCounter: getPrometheusRecordConsumedErrorInstrumentation(), 153 | recordOmittedCounter: getPrometheusRecordOmittedInstrumentation(), 154 | currentMessageTimestamp: getPrometheusCurrentMessageTimestampInstrumentation(), 155 | } 156 | } 157 | 158 | // Instrumentation middleware used to add metrics 159 | func (c *ConsumerMetricsService) Instrumentation(next Handler) Handler { 160 | return Handler{ 161 | Processor: func(ctx context.Context, msg *sarama.ConsumerMessage) (err error) { 162 | defer func(begin time.Time) { 163 | c.recordConsumedLatency.WithLabelValues(msg.Topic, c.groupID).Observe(time.Since(begin).Seconds()) 164 | }(time.Now()) 165 | 166 | // If sarama sets the timestamp to the block timestamp, it means that the message was 167 | // produced with the LogAppendTime timestamp type. Otherwise, it was produced with the 168 | // CreateTime timestamp type. 169 | // Since sarama anyways sets msg.BlockTimestamp to the block timestamp, 170 | // we can compare it with msg.Timestamp to know if the message was produced with the 171 | // LogAppendTime timestamp type or not. 172 | timestampType := TimestampTypeLogAppendTime 173 | if msg.Timestamp != msg.BlockTimestamp { 174 | timestampType = TimestampTypeCreateTime 175 | } 176 | c.currentMessageTimestamp.WithLabelValues(msg.Topic, c.groupID, string(msg.Partition), timestampType).Set(float64(msg.Timestamp.Unix())) 177 | 178 | err = next.Processor(ctx, msg) 179 | if err == nil { 180 | c.recordConsumedCounter.WithLabelValues(msg.Topic, c.groupID).Inc() 181 | } 182 | return 183 | }, 184 | Config: next.Config, 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /instrumenting_test.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/IBM/sarama" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | var testEncodedMessage = []byte{10, 3, 49, 50, 51} 12 | 13 | func Test_NewConsumerMetricsService_Should_Return_Success_When_Success(t *testing.T) { 14 | // Arrange 15 | s := NewConsumerMetricsService("test_ok") 16 | hp := func(context.Context, *sarama.ConsumerMessage) error { return nil } 17 | h := Handler{ 18 | Processor: hp, 19 | } 20 | // Act 21 | handler := s.Instrumentation(h) 22 | 23 | err := handler.Processor(context.Background(), &sarama.ConsumerMessage{Value: testEncodedMessage, Topic: "test-topic"}) 24 | 25 | // Assert 26 | assert.Nil(t, err) 27 | } 28 | 29 | func Test_NewConsumerMetricsService_Should_Allow_Multiple_Instance(t *testing.T) { 30 | // Arrange 31 | group1 := "test_ok" 32 | group2 := "test_ok_other" 33 | s1 := NewConsumerMetricsService(group1) 34 | s2 := NewConsumerMetricsService(group2) 35 | 36 | assert.Equal(t, group1, s1.groupID) 37 | assert.Equal(t, group2, s2.groupID) 38 | } 39 | -------------------------------------------------------------------------------- /listener.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "math" 8 | "strings" 9 | "time" 10 | 11 | "github.com/IBM/sarama" 12 | "github.com/opentracing/opentracing-go" 13 | ) 14 | 15 | var ( 16 | ErrEventUnretriable = errors.New("the event will not be retried") 17 | ErrEventOmitted = errors.New("the event will be omitted") 18 | ) 19 | 20 | type HandlerConfig struct { 21 | ConsumerMaxRetries *int 22 | DurationBeforeRetry *time.Duration 23 | ExponentialBackoff bool 24 | RetryTopic string 25 | DeadletterTopic string 26 | } 27 | 28 | // Handler Processor that handle received kafka messages 29 | // Handler Config can be used to override global configuration for a specific handler 30 | type Handler struct { 31 | Processor func(ctx context.Context, msg *sarama.ConsumerMessage) error 32 | Config HandlerConfig 33 | } 34 | 35 | // Handlers defines a handler for a given topic 36 | type Handlers map[string]Handler 37 | 38 | // listener object represents kafka consumer 39 | // Listener implement both `Listener` interface and `ConsumerGroupHandler` from sarama 40 | type listener struct { 41 | consumerGroup sarama.ConsumerGroup 42 | deadletterProducer Producer 43 | topics []string 44 | handlers Handlers 45 | groupID string 46 | instrumenting *ConsumerMetricsService 47 | tracer TracingFunc 48 | } 49 | 50 | // listenerContextKey defines the key to provide in context 51 | // needs to be define to avoid collision. 52 | // Explanation https://golang.org/pkg/context/#WithValue 53 | type listenerContextKey string 54 | 55 | const ( 56 | contextTopicKey = listenerContextKey("topic") 57 | contextkeyKey = listenerContextKey("key") 58 | contextOffsetKey = listenerContextKey("offset") 59 | contextTimestampKey = listenerContextKey("timestamp") 60 | ) 61 | 62 | // Listener is able to listen multiple topics with one handler by topic 63 | type Listener interface { 64 | Listen(ctx context.Context) error 65 | Close() 66 | GroupID() string 67 | } 68 | 69 | // NewListener creates a new instance of Listener 70 | func NewListener(groupID string, handlers Handlers, options ...ListenerOption) (Listener, error) { 71 | if groupID == "" { 72 | return nil, errors.New("cannot create new listener, group_id cannot be empty") 73 | } 74 | if len(handlers) == 0 { 75 | return nil, errors.New("cannot create new listener, handlers cannot be empty") 76 | } 77 | 78 | // Init consumer, consume errors & messages 79 | var topics []string 80 | for k := range handlers { 81 | topics = append(topics, k) 82 | } 83 | client, err := getClient() 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | producer, err := NewProducer(WithDeadletterProducerInstrumenting()) 89 | if err != nil { 90 | return nil, err 91 | } 92 | 93 | consumerGroup, err := sarama.NewConsumerGroupFromClient(groupID, *client) 94 | if err != nil { 95 | return nil, err 96 | } 97 | 98 | go func() { 99 | errConsumer := <-consumerGroup.Errors() 100 | if errConsumer != nil { 101 | ErrorLogger.Println(err, "error", "sarama error") 102 | } 103 | }() 104 | 105 | // Fill handler config unset elements with global default values. 106 | fillHandlerConfigWithDefault(handlers) 107 | 108 | // Sanity check for error topics, to avoid infinite loop 109 | err = checkErrorTopicToAvoidInfiniteLoop(handlers) 110 | if err != nil { 111 | return nil, err 112 | } 113 | 114 | l := &listener{ 115 | groupID: groupID, 116 | deadletterProducer: producer, 117 | handlers: handlers, 118 | consumerGroup: consumerGroup, 119 | topics: topics, 120 | } 121 | 122 | // execute all method passed as option 123 | for _, o := range options { 124 | o(l) 125 | } 126 | 127 | return l, nil 128 | } 129 | 130 | // GroupID return the groupID of the listener 131 | func (l *listener) GroupID() string { 132 | return l.groupID 133 | } 134 | 135 | func checkErrorTopicToAvoidInfiniteLoop(handlers Handlers) error { 136 | for topic, handler := range handlers { 137 | if handler.Config.RetryTopic == topic { 138 | return fmt.Errorf("retry topic cannot be the same as the original topic: %s", topic) 139 | } 140 | if handler.Config.DeadletterTopic == topic { 141 | return fmt.Errorf("deadletter topic cannot be the same as the original topic: %s", topic) 142 | } 143 | } 144 | return nil 145 | } 146 | 147 | func fillHandlerConfigWithDefault(handlers Handlers) { 148 | for k, h := range handlers { 149 | if h.Config.ConsumerMaxRetries == nil { 150 | h.Config.ConsumerMaxRetries = &ConsumerMaxRetries 151 | } 152 | if h.Config.DurationBeforeRetry == nil { 153 | h.Config.DurationBeforeRetry = &DurationBeforeRetry 154 | } 155 | handlers[k] = h 156 | } 157 | } 158 | 159 | func Ptr[T any](v T) *T { 160 | return &v 161 | } 162 | 163 | // ListenerOption add listener option 164 | type ListenerOption func(l *listener) 165 | 166 | // Listen process incoming kafka messages with handlers configured by the listener 167 | func (l *listener) Listen(consumerContext context.Context) error { 168 | if l.consumerGroup == nil { 169 | return errors.New("consumerGroup is nil, cannot listen") 170 | } 171 | 172 | // When a session is over, make consumer join a new session, as long as the context is not cancelled 173 | for { 174 | // Consume make this consumer join the next session 175 | // This block until the `session` is over. (basically until next rebalance) 176 | err := l.consumerGroup.Consume(consumerContext, l.topics, l) 177 | if err != nil { 178 | return err 179 | } 180 | if err := consumerContext.Err(); err != nil { 181 | // Check if context is cancelled 182 | return err 183 | } 184 | } 185 | } 186 | 187 | // Close the listener and dependencies 188 | func (l *listener) Close() { 189 | if l.consumerGroup != nil { 190 | err := l.consumerGroup.Close() 191 | if err != nil { 192 | ErrorLogger.Println(err, "error", "unable to close sarama consumerGroup") 193 | } 194 | } 195 | } 196 | 197 | // The `Setup`, `Cleanup` and `ConsumeClaim` are actually implementation of ConsumerGroupHandler from sarama 198 | // Copied from From the sarama lib: 199 | // 200 | // ConsumerGroupHandler instances are used to handle individual topic/partition claims. 201 | // It also provides hooks for your consumer group session life-cycle and allow you to 202 | // trigger logic before or after the consume loop(s). 203 | // 204 | // PLEASE NOTE that handlers are likely be called from several goroutines concurrently, 205 | // ensure that all state is safely protected against race conditions. 206 | 207 | // Setup is run at the beginning of a new session, before ConsumeClaim 208 | func (l *listener) Setup(sarama.ConsumerGroupSession) error { 209 | // Mark the consumer as ready 210 | return nil 211 | } 212 | 213 | // Cleanup is run at the end of a session, once all ConsumeClaim goroutines have exited 214 | func (l *listener) Cleanup(sarama.ConsumerGroupSession) error { 215 | return nil 216 | } 217 | 218 | // ConsumeClaim must start a consumer loop of ConsumerGroupClaim's Messages(). 219 | func (l *listener) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { 220 | for msg := range claim.Messages() { 221 | l.onNewMessage(msg, session) 222 | } 223 | return nil 224 | } 225 | 226 | func (l *listener) onNewMessage(msg *sarama.ConsumerMessage, session sarama.ConsumerGroupSession) { 227 | messageContext := context.WithValue(session.Context(), contextTopicKey, msg.Topic) 228 | messageContext = context.WithValue(messageContext, contextkeyKey, msg.Key) 229 | messageContext = context.WithValue(messageContext, contextOffsetKey, msg.Offset) 230 | messageContext = context.WithValue(messageContext, contextTimestampKey, msg.Timestamp) 231 | for _, h := range msg.Headers { 232 | messageContext = context.WithValue(messageContext, listenerContextKey(h.Key), h.Value) 233 | } 234 | 235 | var span opentracing.Span 236 | if l.tracer != nil { 237 | span, messageContext = l.tracer(messageContext, msg) 238 | if span != nil { 239 | defer span.Finish() 240 | } 241 | } 242 | 243 | handler := l.handlers[msg.Topic] 244 | if l.instrumenting != nil { 245 | handler = l.instrumenting.Instrumentation(handler) 246 | } 247 | 248 | err := l.handleMessageWithRetry(messageContext, handler, msg, *handler.Config.ConsumerMaxRetries, 0, handler.Config.ExponentialBackoff) 249 | if err != nil { 250 | err = fmt.Errorf("processing failed: %w", err) 251 | l.handleErrorMessage(err, handler, msg) 252 | } 253 | 254 | if !errors.Is(err, context.Canceled) { 255 | session.MarkMessage(msg, "") 256 | } 257 | } 258 | 259 | func (l *listener) handleErrorMessage(initialError error, handler Handler, msg *sarama.ConsumerMessage) { 260 | if errors.Is(initialError, ErrEventOmitted) { 261 | l.handleOmittedMessage(initialError, msg) 262 | return 263 | } 264 | 265 | // Log 266 | ErrorLogger.Println(initialError, "error", "unable to process message, we apply retry topic policy") 267 | 268 | if isRetriableError(initialError) { 269 | // First, check if handler's config defines retry topic 270 | if handler.Config.RetryTopic != "" { 271 | Logger.Printf("sending message to retry topic: %s", handler.Config.RetryTopic) 272 | err := forwardToTopic(l, msg, handler.Config.RetryTopic) 273 | if err != nil { 274 | errLog := []interface{}{err, "error", "cannot send message to handler's retry topic", "retry_topic", handler.Config.RetryTopic} 275 | errLog = append(errLog, extractMessageInfoForLog(msg)...) 276 | ErrorLogger.Println(errLog...) 277 | } 278 | return 279 | } 280 | 281 | // If not, check if global retry topic pattern is defined 282 | if PushConsumerErrorsToRetryTopic { 283 | topicName := l.deduceTopicNameFromPattern(msg.Topic, RetryTopicPattern) 284 | Logger.Printf("sending message to retry topic: %s", topicName) 285 | err := forwardToTopic(l, msg, topicName) 286 | if err != nil { 287 | errLog := []interface{}{err, "error", "cannot send message to handler's retry topic defined with global pattern", "topic", topicName} 288 | errLog = append(errLog, extractMessageInfoForLog(msg)...) 289 | ErrorLogger.Println(errLog...) 290 | } 291 | return 292 | } 293 | } 294 | 295 | // If the error is not retriable, or if there is no retry topic defined at all, then try to send to dead letter topic 296 | // First, check if handler's config defines deadletter topic 297 | if handler.Config.DeadletterTopic != "" { 298 | Logger.Printf("sending message to handler's deadletter topic: %s", handler.Config.DeadletterTopic) 299 | err := forwardToTopic(l, msg, handler.Config.DeadletterTopic) 300 | if err != nil { 301 | errLog := []interface{}{err, "error", "cannot send message to handler's deadletter topic", "deadletter_topic", handler.Config.DeadletterTopic} 302 | errLog = append(errLog, extractMessageInfoForLog(msg)...) 303 | ErrorLogger.Println(errLog...) 304 | } 305 | return 306 | } 307 | 308 | // If not, check if global deadletter topic pattern is defined 309 | if PushConsumerErrorsToDeadletterTopic { 310 | topicName := l.deduceTopicNameFromPattern(msg.Topic, DeadletterTopicPattern) 311 | Logger.Printf("sending message to deadletter topic: %s", topicName) 312 | err := forwardToTopic(l, msg, topicName) 313 | if err != nil { 314 | errorLog := []interface{}{err, "error", "cannot send message to handler's deadletter topic defined with global pattern", "topic", topicName} 315 | errorLog = append(errorLog, extractMessageInfoForLog(msg)...) 316 | ErrorLogger.Println(errorLog...) 317 | } 318 | return 319 | } 320 | // If we do nothing the message is implicitly omitted 321 | if l.instrumenting != nil && l.instrumenting.recordOmittedCounter != nil { 322 | l.instrumenting.recordOmittedCounter.With(map[string]string{"kafka_topic": msg.Topic, "consumer_group": l.groupID}).Inc() 323 | } 324 | } 325 | 326 | func (l *listener) deduceTopicNameFromPattern(topic string, pattern string) string { 327 | topicName := pattern 328 | topicName = strings.Replace(topicName, "$$CG$$", l.groupID, 1) 329 | topicName = strings.Replace(topicName, "$$T$$", topic, 1) 330 | return topicName 331 | } 332 | 333 | func forwardToTopic(l *listener, msg *sarama.ConsumerMessage, topicName string) error { 334 | err := l.deadletterProducer.Produce(&sarama.ProducerMessage{ 335 | Key: sarama.ByteEncoder(msg.Key), 336 | Value: sarama.ByteEncoder(msg.Value), 337 | Topic: topicName, 338 | }) 339 | return err 340 | } 341 | 342 | func isRetriableError(initialError error) bool { 343 | return !errors.Is(initialError, ErrEventUnretriable) && !errors.Is(initialError, ErrEventOmitted) 344 | } 345 | 346 | func (l *listener) handleOmittedMessage(initialError error, msg *sarama.ConsumerMessage) { 347 | ErrorLogger.Println(initialError, "error", "omitted message") 348 | 349 | // Inc dropped messages metrics 350 | if l.instrumenting != nil && l.instrumenting.recordOmittedCounter != nil { 351 | l.instrumenting.recordOmittedCounter.With(map[string]string{"kafka_topic": msg.Topic, "consumer_group": l.groupID}).Inc() 352 | } 353 | } 354 | 355 | // handleMessageWithRetry call the handler function and retry if it fails 356 | func (l *listener) handleMessageWithRetry(ctx context.Context, handler Handler, msg *sarama.ConsumerMessage, retries, retryNumber int, exponentialBackoff bool) (err error) { 357 | defer func() { 358 | if r := recover(); r != nil { 359 | err = fmt.Errorf("panic happened during handle of message: %v", r) 360 | } 361 | }() 362 | 363 | // Check if context is still valid 364 | if ctx.Err() != nil { 365 | return ctx.Err() 366 | } 367 | 368 | err = handler.Processor(ctx, msg) 369 | if err != nil { 370 | // Inc dropped messages metrics 371 | if l.instrumenting != nil && l.instrumenting.recordErrorCounter != nil { 372 | l.instrumenting.recordErrorCounter.With(map[string]string{"kafka_topic": msg.Topic, "consumer_group": l.groupID}).Inc() 373 | } 374 | if shouldRetry(retries, err) { 375 | if exponentialBackoff { 376 | backoffDuration := calculateExponentialBackoffDuration(retryNumber, handler.Config.DurationBeforeRetry) 377 | Logger.Printf("exponential backoff enabled: we will retry in %s", backoffDuration) 378 | time.Sleep(backoffDuration) 379 | } else { 380 | time.Sleep(*handler.Config.DurationBeforeRetry) 381 | } 382 | if retries != InfiniteRetries { 383 | retries-- 384 | } else { 385 | errLog := []interface{}{ctx, err, "error", "unable to process message we retry indefinitely", "retry_number", retryNumber} 386 | errLog = append(errLog, extractMessageInfoForLog(msg)...) 387 | ErrorLogger.Println(errLog...) 388 | retryNumber++ 389 | } 390 | return l.handleMessageWithRetry(ctx, handler, msg, retries, retryNumber, exponentialBackoff) 391 | } 392 | } 393 | return err 394 | } 395 | 396 | func shouldRetry(retries int, err error) bool { 397 | if retries == 0 { 398 | return false 399 | } 400 | 401 | if errors.Is(err, ErrEventUnretriable) || errors.Is(err, ErrEventOmitted) { 402 | return false 403 | } 404 | 405 | return true 406 | } 407 | 408 | func extractMessageInfoForLog(msg *sarama.ConsumerMessage) []interface{} { 409 | if msg == nil { 410 | return []interface{}{"message", "nil"} 411 | } 412 | return []interface{}{"message_topic", msg.Topic, "topic_partition", msg.Partition, "message_offset", msg.Offset, "message_key", string(msg.Key)} 413 | } 414 | 415 | func calculateExponentialBackoffDuration(retries int, baseDuration *time.Duration) time.Duration { 416 | var duration time.Duration 417 | if baseDuration == nil { 418 | duration = 0 419 | } else { 420 | duration = *baseDuration 421 | } 422 | return duration * time.Duration(math.Pow(2, float64(retries))) 423 | } 424 | -------------------------------------------------------------------------------- /listener_test.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | "errors" 10 | 11 | "github.com/IBM/sarama" 12 | "github.com/ricardo-ch/go-kafka/v3/mocks" 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/mock" 15 | ) 16 | 17 | var ( 18 | testHandler = Handler{ 19 | Processor: func(ctx context.Context, msg *sarama.ConsumerMessage) error { return nil }, 20 | } 21 | testHandlerConfig = HandlerConfig{ 22 | ConsumerMaxRetries: Ptr(10), 23 | DurationBeforeRetry: Ptr(1 * time.Millisecond), 24 | RetryTopic: "retry-topic", 25 | DeadletterTopic: "deadletter-topic", 26 | } 27 | testHandlerWithConfig = Handler{ 28 | Processor: func(ctx context.Context, msg *sarama.ConsumerMessage) error { return nil }, 29 | Config: testHandlerConfig, 30 | } 31 | ) 32 | 33 | func Test_NewListener_Should_Return_Error_When_No_Broker_Provided(t *testing.T) { 34 | // Arrange 35 | handlers := map[string]Handler{"topic": testHandler} 36 | groupID := "groupID" 37 | Brokers = []string{} 38 | 39 | // Act 40 | l, err := NewListener(groupID, handlers) 41 | 42 | // Assert 43 | assert.Error(t, err) 44 | assert.Nil(t, l) 45 | } 46 | 47 | func Test_NewListener_Should_Return_Error_When_No_GroupID_Provided(t *testing.T) { 48 | // Arrange 49 | handlers := map[string]Handler{"topic": testHandler} 50 | groupID := "" 51 | Brokers = []string{"localhost:9092"} 52 | 53 | // Act 54 | l, err := NewListener(groupID, handlers) 55 | 56 | // Assert 57 | assert.Error(t, err) 58 | assert.Nil(t, l) 59 | } 60 | 61 | func Test_NewListener_Should_Return_Error_When_No_Handlers_Provided(t *testing.T) { 62 | // Arrange 63 | handlers := map[string]Handler{} 64 | groupID := "groupID" 65 | Brokers = []string{"localhost:9092"} 66 | 67 | // Act 68 | l, err := NewListener(groupID, handlers) 69 | 70 | // Assert 71 | assert.Error(t, err) 72 | assert.Nil(t, l) 73 | } 74 | 75 | func Test_NewListener_Should_Return_Error_When_Initial_Topic_Equals_Retry_Topic(t *testing.T) { 76 | // Arrange 77 | leaderBroker := sarama.NewMockBroker(t, 1) 78 | 79 | metadataResponse := &sarama.MetadataResponse{ 80 | Version: 5, 81 | } 82 | metadataResponse.AddBroker(leaderBroker.Addr(), leaderBroker.BrokerID()) 83 | metadataResponse.AddTopicPartition("retry-topic", 0, leaderBroker.BrokerID(), nil, nil, nil, sarama.ErrNoError) 84 | leaderBroker.Returns(metadataResponse) 85 | 86 | consumerMetadataResponse := sarama.ConsumerMetadataResponse{ 87 | CoordinatorID: leaderBroker.BrokerID(), 88 | CoordinatorHost: leaderBroker.Addr(), 89 | CoordinatorPort: leaderBroker.Port(), 90 | Err: sarama.ErrNoError, 91 | } 92 | leaderBroker.Returns(&consumerMetadataResponse) 93 | 94 | Brokers = []string{leaderBroker.Addr()} 95 | 96 | handlers := map[string]Handler{"retry-topic": testHandlerWithConfig} 97 | 98 | // Act 99 | l, err := NewListener("groupID", handlers) 100 | 101 | // Assert 102 | assert.Error(t, err) 103 | assert.Nil(t, l) 104 | } 105 | 106 | func Test_NewListener_Should_Return_Error_When_Initial_Topic_Equals_Deadletter_Topic(t *testing.T) { 107 | // Arrange 108 | leaderBroker := sarama.NewMockBroker(t, 1) 109 | 110 | metadataResponse := &sarama.MetadataResponse{ 111 | Version: 5, 112 | } 113 | metadataResponse.AddBroker(leaderBroker.Addr(), leaderBroker.BrokerID()) 114 | metadataResponse.AddTopicPartition("deadletter-topic", 0, leaderBroker.BrokerID(), nil, nil, nil, sarama.ErrNoError) 115 | leaderBroker.Returns(metadataResponse) 116 | 117 | consumerMetadataResponse := sarama.ConsumerMetadataResponse{ 118 | CoordinatorID: leaderBroker.BrokerID(), 119 | CoordinatorHost: leaderBroker.Addr(), 120 | CoordinatorPort: leaderBroker.Port(), 121 | Err: sarama.ErrNoError, 122 | } 123 | leaderBroker.Returns(&consumerMetadataResponse) 124 | 125 | Brokers = []string{leaderBroker.Addr()} 126 | 127 | handlers := map[string]Handler{"deadletter-topic": testHandlerWithConfig} 128 | 129 | // Act 130 | l, err := NewListener("groupID", handlers) 131 | 132 | // Assert 133 | assert.Error(t, err) 134 | assert.Nil(t, l) 135 | } 136 | 137 | func Test_NewListener_Happy_Path(t *testing.T) { 138 | leaderBroker := sarama.NewMockBroker(t, 1) 139 | 140 | metadataResponse := &sarama.MetadataResponse{ 141 | Version: 5, 142 | } 143 | metadataResponse.AddBroker(leaderBroker.Addr(), leaderBroker.BrokerID()) 144 | metadataResponse.AddTopicPartition("topic-test", 0, leaderBroker.BrokerID(), nil, nil, nil, sarama.ErrNoError) 145 | leaderBroker.Returns(metadataResponse) 146 | 147 | consumerMetadataResponse := sarama.ConsumerMetadataResponse{ 148 | CoordinatorID: leaderBroker.BrokerID(), 149 | CoordinatorHost: leaderBroker.Addr(), 150 | CoordinatorPort: leaderBroker.Port(), 151 | Err: sarama.ErrNoError, 152 | } 153 | leaderBroker.Returns(&consumerMetadataResponse) 154 | 155 | Brokers = []string{leaderBroker.Addr()} 156 | 157 | handlers := map[string]Handler{"topic-test": testHandler} 158 | listener, err := NewListener("groupID", handlers) 159 | assert.NotNil(t, listener) 160 | assert.Nil(t, err) 161 | } 162 | 163 | func Test_ConsumeClaim_Happy_Path(t *testing.T) { 164 | msgChanel := make(chan *sarama.ConsumerMessage, 1) 165 | msgChanel <- &sarama.ConsumerMessage{ 166 | Topic: "topic-test", 167 | Headers: []*sarama.RecordHeader{{Key: []byte("user-id"), Value: []byte("123456")}}, 168 | } 169 | close(msgChanel) 170 | 171 | consumerGroupClaim := &mocks.ConsumerGroupClaim{} 172 | consumerGroupClaim.On("Messages").Return((<-chan *sarama.ConsumerMessage)(msgChanel)) 173 | 174 | consumerGroupSession := &mocks.ConsumerGroupSession{} 175 | consumerGroupSession.On("Context").Return(context.Background()) 176 | consumerGroupSession.On("MarkMessage", mock.Anything, mock.Anything).Return() 177 | 178 | handlerCalled := false 179 | var headerVal interface{} 180 | handlerProcessor := func(ctx context.Context, msg *sarama.ConsumerMessage) error { 181 | headerVal = ctx.Value(listenerContextKey("user-id")) 182 | handlerCalled = true 183 | return nil 184 | } 185 | handler := Handler{ 186 | Processor: handlerProcessor, 187 | Config: testHandlerConfig, 188 | } 189 | 190 | tested := listener{ 191 | handlers: map[string]Handler{"topic-test": handler}, 192 | } 193 | 194 | err := tested.ConsumeClaim(consumerGroupSession, consumerGroupClaim) 195 | 196 | assert.NoError(t, err) 197 | assert.True(t, handlerCalled) 198 | assert.Equal(t, string(headerVal.([]byte)), "123456") 199 | consumerGroupClaim.AssertExpectations(t) 200 | consumerGroupSession.AssertExpectations(t) 201 | } 202 | 203 | func Test_ConsumeClaim_Message_Error_WithErrorTopic(t *testing.T) { 204 | // Reduce the retry interval to speed up the test 205 | DurationBeforeRetry = 1 * time.Millisecond 206 | 207 | PushConsumerErrorsToDeadletterTopic = true 208 | 209 | msgChanel := make(chan *sarama.ConsumerMessage, 1) 210 | msgChanel <- &sarama.ConsumerMessage{ 211 | Topic: "topic-test", 212 | } 213 | close(msgChanel) 214 | 215 | consumerGroupClaim := &mocks.ConsumerGroupClaim{} 216 | consumerGroupClaim.On("Messages").Return((<-chan *sarama.ConsumerMessage)(msgChanel)) 217 | 218 | consumerGroupSession := &mocks.ConsumerGroupSession{} 219 | consumerGroupSession.On("Context").Return(context.Background()) 220 | consumerGroupSession.On("MarkMessage", mock.Anything, mock.Anything).Return() 221 | 222 | producer := &mocks.MockProducer{} 223 | producer.On("Produce", mock.Anything).Return(nil) 224 | 225 | handlerCalled := false 226 | handlerProcessor := func(ctx context.Context, msg *sarama.ConsumerMessage) error { 227 | handlerCalled = true 228 | return fmt.Errorf("I want an error to be logged") 229 | } 230 | handler := Handler{ 231 | Processor: handlerProcessor, 232 | Config: testHandlerConfig, 233 | } 234 | 235 | defaultLogger := ErrorLogger 236 | defer func() { ErrorLogger = defaultLogger }() 237 | 238 | errorLogged := false 239 | mockLogger := &mocks.StdLogger{} 240 | mockLogger.On("Println", mock.Anything, mock.Anything, mock.Anything).Return().Run(func(mock.Arguments) { 241 | errorLogged = true 242 | }) 243 | ErrorLogger = mockLogger 244 | 245 | tested := listener{ 246 | handlers: map[string]Handler{"topic-test": handler}, 247 | deadletterProducer: producer, 248 | } 249 | 250 | err := tested.ConsumeClaim(consumerGroupSession, consumerGroupClaim) 251 | 252 | assert.NoError(t, err) 253 | assert.True(t, handlerCalled) 254 | assert.True(t, errorLogged) 255 | consumerGroupClaim.AssertExpectations(t) 256 | consumerGroupSession.AssertExpectations(t) 257 | producer.AssertExpectations(t) 258 | } 259 | 260 | func Test_ConsumeClaim_Message_Error_WithPanicTopic(t *testing.T) { 261 | PushConsumerErrorsToDeadletterTopic = true 262 | 263 | msgChanel := make(chan *sarama.ConsumerMessage, 1) 264 | msgChanel <- &sarama.ConsumerMessage{ 265 | Topic: "topic-test", 266 | } 267 | close(msgChanel) 268 | 269 | consumerGroupClaim := &mocks.ConsumerGroupClaim{} 270 | consumerGroupClaim.On("Messages").Return((<-chan *sarama.ConsumerMessage)(msgChanel)) 271 | 272 | consumerGroupSession := &mocks.ConsumerGroupSession{} 273 | consumerGroupSession.On("Context").Return(context.Background()) 274 | consumerGroupSession.On("MarkMessage", mock.Anything, mock.Anything).Return() 275 | 276 | producer := &mocks.MockProducer{} 277 | producer.On("Produce", mock.Anything).Return(nil) 278 | 279 | handlerCalled := false 280 | handlerProcessor := func(ctx context.Context, msg *sarama.ConsumerMessage) error { 281 | handlerCalled = true 282 | panic("I want an error to be logged") 283 | } 284 | handler := Handler{ 285 | Processor: handlerProcessor, 286 | Config: testHandlerConfig, 287 | } 288 | 289 | defaultLogger := ErrorLogger 290 | defer func() { ErrorLogger = defaultLogger }() 291 | 292 | errorLogged := false 293 | mockLogger := &mocks.StdLogger{} 294 | mockLogger.On("Println", mock.Anything, mock.Anything, mock.Anything).Return().Run(func(mock.Arguments) { 295 | errorLogged = true 296 | }) 297 | ErrorLogger = mockLogger 298 | 299 | tested := listener{ 300 | handlers: map[string]Handler{"topic-test": handler}, 301 | deadletterProducer: producer, 302 | } 303 | 304 | err := tested.ConsumeClaim(consumerGroupSession, consumerGroupClaim) 305 | 306 | assert.NoError(t, err) 307 | assert.True(t, handlerCalled) 308 | assert.True(t, errorLogged) 309 | consumerGroupClaim.AssertExpectations(t) 310 | consumerGroupSession.AssertExpectations(t) 311 | producer.AssertExpectations(t) 312 | } 313 | 314 | func Test_ConsumeClaim_Message_Error_WithHandlerSpecificRetryTopic(t *testing.T) { 315 | PushConsumerErrorsToRetryTopic = false // global value that is overwritten for the handler in this test 316 | 317 | // Arrange 318 | msgChanel := make(chan *sarama.ConsumerMessage, 1) 319 | msgChanel <- &sarama.ConsumerMessage{ 320 | Topic: "topic-test", 321 | } 322 | close(msgChanel) 323 | 324 | consumerGroupClaim := &mocks.ConsumerGroupClaim{} 325 | consumerGroupClaim.On("Messages").Return((<-chan *sarama.ConsumerMessage)(msgChanel)) 326 | 327 | consumerGroupSession := &mocks.ConsumerGroupSession{} 328 | consumerGroupSession.On("Context").Return(context.Background()) 329 | consumerGroupSession.On("MarkMessage", mock.Anything, mock.Anything).Return() 330 | 331 | producer := &mocks.MockProducer{} 332 | producer.On("Produce", mock.Anything).Return(nil) 333 | 334 | handlerCalled := false 335 | handlerProcessor := func(ctx context.Context, msg *sarama.ConsumerMessage) error { 336 | handlerCalled = true 337 | panic("I want an error to be logged") 338 | } 339 | handler := Handler{ 340 | Processor: handlerProcessor, 341 | Config: HandlerConfig{ 342 | ConsumerMaxRetries: Ptr(3), 343 | DurationBeforeRetry: Ptr(1 * time.Millisecond), 344 | RetryTopic: "retry-topic", // Here is the important part 345 | }, 346 | } 347 | 348 | defaultLogger := ErrorLogger 349 | defer func() { ErrorLogger = defaultLogger }() 350 | 351 | errorLogged := false 352 | mockLogger := &mocks.StdLogger{} 353 | mockLogger.On("Println", mock.Anything, mock.Anything, mock.Anything).Return().Run(func(mock.Arguments) { 354 | errorLogged = true 355 | }) 356 | ErrorLogger = mockLogger 357 | 358 | tested := listener{ 359 | handlers: map[string]Handler{"topic-test": handler}, 360 | deadletterProducer: producer, 361 | } 362 | 363 | // Act 364 | err := tested.ConsumeClaim(consumerGroupSession, consumerGroupClaim) 365 | 366 | // Assert 367 | assert.NoError(t, err) 368 | assert.True(t, handlerCalled) 369 | assert.True(t, errorLogged) 370 | consumerGroupClaim.AssertExpectations(t) 371 | consumerGroupSession.AssertExpectations(t) 372 | producer.AssertExpectations(t) 373 | } 374 | 375 | func Test_ConsumeClaim_Message_Error_Context_Cancelled_Does_Not_Commit_Offset(t *testing.T) { 376 | PushConsumerErrorsToRetryTopic = false 377 | PushConsumerErrorsToDeadletterTopic = false 378 | 379 | // Arrange 380 | msgChanel := make(chan *sarama.ConsumerMessage, 1) 381 | msgChanel <- &sarama.ConsumerMessage{ 382 | Topic: "topic-test", 383 | } 384 | close(msgChanel) 385 | 386 | consumerGroupClaim := &mocks.ConsumerGroupClaim{} 387 | consumerGroupClaim.On("Messages").Return((<-chan *sarama.ConsumerMessage)(msgChanel)) 388 | 389 | consumerGroupSession := &mocks.ConsumerGroupSession{} 390 | consumerGroupSession.On("Context").Return(context.Background()) 391 | 392 | producer := &mocks.MockProducer{} 393 | 394 | handlerCalled := false 395 | handlerProcessor := func(ctx context.Context, msg *sarama.ConsumerMessage) error { 396 | handlerCalled = true 397 | return context.Canceled 398 | } 399 | handler := Handler{ 400 | Processor: handlerProcessor, 401 | Config: HandlerConfig{ 402 | ConsumerMaxRetries: Ptr(3), 403 | DurationBeforeRetry: Ptr(1 * time.Millisecond), 404 | }, 405 | } 406 | 407 | tested := listener{ 408 | handlers: map[string]Handler{"topic-test": handler}, 409 | deadletterProducer: producer, 410 | } 411 | 412 | // Act 413 | err := tested.ConsumeClaim(consumerGroupSession, consumerGroupClaim) 414 | 415 | // Assert 416 | assert.NoError(t, err) 417 | assert.True(t, handlerCalled) 418 | consumerGroupClaim.AssertExpectations(t) 419 | consumerGroupSession.AssertExpectations(t) 420 | producer.AssertExpectations(t) 421 | } 422 | 423 | func Test_handleErrorMessage_OmittedError(t *testing.T) { 424 | 425 | omittedError := errors.New("This error should be omitted") 426 | 427 | l := listener{} 428 | 429 | defaultLogger := ErrorLogger 430 | defer func() { ErrorLogger = defaultLogger }() 431 | 432 | errorLogged := false 433 | mockLogger := &mocks.StdLogger{} 434 | mockLogger.On("Println", mock.Anything, "error", "omitted message").Return().Run(func(mock.Arguments) { 435 | errorLogged = true 436 | }).Once() 437 | ErrorLogger = mockLogger 438 | 439 | l.handleErrorMessage(fmt.Errorf("%w: %w", omittedError, ErrEventOmitted), Handler{}, nil) 440 | 441 | assert.True(t, errorLogged) 442 | } 443 | 444 | func Test_handleMessageWithRetry(t *testing.T) { 445 | // Reduce the retry interval to speed up the test 446 | DurationBeforeRetry = 1 * time.Millisecond 447 | 448 | err := errors.New("This error should be retried") 449 | handlerCalled := 0 450 | handlerProcessor := func(ctx context.Context, msg *sarama.ConsumerMessage) error { 451 | handlerCalled++ 452 | return err 453 | } 454 | handler := Handler{ 455 | Processor: handlerProcessor, 456 | Config: testHandlerConfig, 457 | } 458 | 459 | l := listener{} 460 | l.handleMessageWithRetry(context.Background(), handler, nil, 3, 0, false) 461 | 462 | assert.Equal(t, 4, handlerCalled) 463 | } 464 | 465 | func Test_handleMessageWithRetryWithBackoff(t *testing.T) { 466 | // Reduce the retry interval to speed up the test 467 | DurationBeforeRetry = 1 * time.Millisecond 468 | 469 | err := errors.New("This error should be retried") 470 | handlerCalled := 0 471 | handlerProcessor := func(ctx context.Context, msg *sarama.ConsumerMessage) error { 472 | handlerCalled++ 473 | return err 474 | } 475 | handler := Handler{ 476 | Processor: handlerProcessor, 477 | Config: testHandlerConfig, 478 | } 479 | 480 | l := listener{} 481 | l.handleMessageWithRetry(context.Background(), handler, nil, 3, 0, true) 482 | 483 | assert.Equal(t, 4, handlerCalled) 484 | } 485 | 486 | func Test_handleMessageWithRetry_UnretriableError(t *testing.T) { 487 | err := errors.New("This error should not be retried") 488 | handlerCalled := 0 489 | handlerProcessor := func(ctx context.Context, msg *sarama.ConsumerMessage) error { 490 | handlerCalled++ 491 | return fmt.Errorf("%w: %w", err, ErrEventUnretriable) 492 | } 493 | handler := Handler{ 494 | Processor: handlerProcessor, 495 | Config: testHandlerConfig, 496 | } 497 | 498 | l := listener{} 499 | l.handleMessageWithRetry(context.Background(), handler, nil, 3, 0, false) 500 | 501 | assert.Equal(t, 1, handlerCalled) 502 | } 503 | 504 | func Test_handleMessageWithRetry_UnretriableErrorWithBackoff(t *testing.T) { 505 | err := errors.New("This error should not be retried") 506 | handlerCalled := 0 507 | handlerProcessor := func(ctx context.Context, msg *sarama.ConsumerMessage) error { 508 | handlerCalled++ 509 | return fmt.Errorf("%w: %w", err, ErrEventUnretriable) 510 | } 511 | handler := Handler{ 512 | Processor: handlerProcessor, 513 | Config: testHandlerConfig, 514 | } 515 | 516 | l := listener{} 517 | l.handleMessageWithRetry(context.Background(), handler, nil, 3, 0, true) 518 | 519 | assert.Equal(t, 1, handlerCalled) 520 | } 521 | 522 | func Test_handleMessageWithRetry_InfiniteRetries(t *testing.T) { 523 | // Reduce the retry interval to speed up the test 524 | DurationBeforeRetry = 1 * time.Millisecond 525 | 526 | err := errors.New("This error should be retried") 527 | handlerCalled := 0 528 | handlerProcessor := func(ctx context.Context, msg *sarama.ConsumerMessage) error { 529 | handlerCalled++ 530 | 531 | // We simulate an infinite retry by failing 5 times, and then succeeding, 532 | // which is above the 3 retries normally expected 533 | if handlerCalled < 5 { 534 | return err 535 | } 536 | return nil 537 | } 538 | 539 | handler := Handler{ 540 | Processor: handlerProcessor, 541 | Config: testHandlerConfig, 542 | } 543 | 544 | l := listener{} 545 | l.handleMessageWithRetry(context.Background(), handler, nil, InfiniteRetries, 0, false) 546 | 547 | assert.Equal(t, 5, handlerCalled) 548 | 549 | } 550 | func Test_handleMessageWithRetry_InfiniteRetriesWithBackoff(t *testing.T) { 551 | // Reduce the retry interval to speed up the test 552 | DurationBeforeRetry = 1 * time.Millisecond 553 | 554 | err := errors.New("This error should be retried") 555 | handlerCalled := 0 556 | handlerProcessor := func(ctx context.Context, msg *sarama.ConsumerMessage) error { 557 | handlerCalled++ 558 | 559 | // We simulate an infinite retry by failing 5 times, and then succeeding, 560 | // which is above the 3 retries normally expected 561 | if handlerCalled < 5 { 562 | return err 563 | } 564 | return nil 565 | } 566 | 567 | handler := Handler{ 568 | Processor: handlerProcessor, 569 | Config: testHandlerConfig, 570 | } 571 | 572 | l := listener{} 573 | l.handleMessageWithRetry(context.Background(), handler, nil, InfiniteRetries, 0, true) 574 | 575 | assert.Equal(t, 5, handlerCalled) 576 | 577 | } 578 | 579 | func Test_handleMessageWithRetry_InfiniteRetriesWithContextCancel(t *testing.T) { 580 | // Reduce the retry interval to speed up the test 581 | DurationBeforeRetry = 1 * time.Millisecond 582 | err := errors.New("This error should be retried") 583 | 584 | handlerCalled := 0 585 | ctx := context.Background() 586 | ctx, cancel := context.WithCancel(ctx) 587 | handlerProcessor := func(ctx context.Context, msg *sarama.ConsumerMessage) error { 588 | handlerCalled++ 589 | 590 | // We simulate an infinite retry by failing 5 times, and then a context is canceled, 591 | // which is above the 3 retries normally expected 592 | if handlerCalled > 4 { 593 | cancel() 594 | } 595 | return err 596 | } 597 | 598 | handler := Handler{ 599 | Processor: handlerProcessor, 600 | Config: testHandlerConfig, 601 | } 602 | 603 | l := listener{} 604 | l.handleMessageWithRetry(ctx, handler, nil, InfiniteRetries, 0, false) 605 | 606 | assert.Equal(t, 5, handlerCalled) 607 | 608 | } 609 | 610 | // Basically a copy paste of the happy path but with tracing 611 | // This test only checks that the tracing is not preventing the consumption 612 | func Test_ConsumerClaim_HappyPath_WithTracing(t *testing.T) { 613 | msgChanel := make(chan *sarama.ConsumerMessage, 1) 614 | msgChanel <- &sarama.ConsumerMessage{ 615 | Topic: "topic-test", 616 | } 617 | close(msgChanel) 618 | 619 | consumerGroupClaim := &mocks.ConsumerGroupClaim{} 620 | consumerGroupClaim.On("Messages").Return((<-chan *sarama.ConsumerMessage)(msgChanel)) 621 | 622 | consumerGroupSession := &mocks.ConsumerGroupSession{} 623 | consumerGroupSession.On("Context").Return(context.Background()) 624 | consumerGroupSession.On("MarkMessage", mock.Anything, mock.Anything).Return() 625 | 626 | handlerCalled := false 627 | handlerProcessor := func(ctx context.Context, msg *sarama.ConsumerMessage) error { 628 | handlerCalled = true 629 | return nil 630 | } 631 | handler := Handler{ 632 | Processor: handlerProcessor, 633 | Config: testHandlerConfig, 634 | } 635 | 636 | tested := listener{ 637 | handlers: map[string]Handler{"topic-test": handler}, 638 | tracer: DefaultTracing, // this is the important part 639 | } 640 | 641 | err := tested.ConsumeClaim(consumerGroupSession, consumerGroupClaim) 642 | 643 | assert.NoError(t, err) 644 | assert.True(t, handlerCalled) 645 | consumerGroupClaim.AssertExpectations(t) 646 | consumerGroupSession.AssertExpectations(t) 647 | } 648 | 649 | // Test that as long as context is not canceled and not error is returned, `Consume` is called again 650 | // (when rebalance is called, the consumer will be part of next session) 651 | func Test_Listen_Happy_Path(t *testing.T) { 652 | calledCounter := 0 653 | consumeCalled := make(chan interface{}) 654 | consumerGroup := &mocks.ConsumerGroup{} 655 | 656 | // Mimic the end of a consumerGroup session by just not blocking 657 | consumerGroup.On("Consume", mock.Anything, mock.Anything, mock.Anything). 658 | Run(func(args mock.Arguments) { 659 | calledCounter++ 660 | consumeCalled <- true 661 | if calledCounter >= 2 { 662 | time.Sleep(1000 * time.Second) // just wait 663 | } 664 | }). 665 | Return(nil).Twice() 666 | 667 | tested := listener{consumerGroup: consumerGroup} 668 | 669 | // Listen() is blocking as long as there is no error or context is not canceled 670 | go func() { 671 | tested.Listen(context.Background()) 672 | assert.Fail(t, `We should have blocked on "listen", even if a consumer group session has ended`) 673 | }() 674 | 675 | // Assert that consume is called twice (2 consumer group sessions are expected) 676 | <-consumeCalled 677 | <-consumeCalled 678 | 679 | consumerGroup.AssertExpectations(t) 680 | } 681 | 682 | // Test that when the context is canceled, as soon as the consumerGroup's session ends, `Listen` returns 683 | func Test_Listen_ContextCanceled(t *testing.T) { 684 | consumerGroup := &mocks.ConsumerGroup{} 685 | 686 | consumerGroup.On("Consume", mock.Anything, mock.Anything, mock.Anything). 687 | Run(func(args mock.Arguments) { 688 | ctx := args.Get(0).(context.Context) 689 | <-ctx.Done() 690 | }). 691 | Return(nil) 692 | 693 | tested := listener{consumerGroup: consumerGroup} 694 | 695 | ctx, cancel := context.WithCancel(context.Background()) 696 | cancel() 697 | 698 | err := tested.Listen(ctx) 699 | 700 | assert.Equal(t, context.Canceled, err) 701 | consumerGroup.AssertExpectations(t) 702 | } 703 | 704 | func Test_calculateExponentialBackoffDuration(t *testing.T) { 705 | tests := []struct { 706 | name string 707 | retries int 708 | baseDuration *time.Duration 709 | expectedDelay time.Duration 710 | }{ 711 | { 712 | name: "nil base duration", 713 | retries: 3, 714 | baseDuration: nil, 715 | expectedDelay: 0, 716 | }, 717 | { 718 | name: "zero retries", 719 | retries: 0, 720 | baseDuration: Ptr(1 * time.Second), 721 | expectedDelay: 1 * time.Second, 722 | }, 723 | { 724 | name: "one retry", 725 | retries: 1, 726 | baseDuration: Ptr(1 * time.Second), 727 | expectedDelay: 2 * time.Second, 728 | }, 729 | { 730 | name: "two retries", 731 | retries: 2, 732 | baseDuration: Ptr(1 * time.Second), 733 | expectedDelay: 4 * time.Second, 734 | }, 735 | { 736 | name: "three retries", 737 | retries: 3, 738 | baseDuration: Ptr(1 * time.Second), 739 | expectedDelay: 8 * time.Second, 740 | }, 741 | { 742 | name: "three retries with different base duration", 743 | retries: 3, 744 | baseDuration: Ptr(500 * time.Millisecond), 745 | expectedDelay: 4 * time.Second, 746 | }, 747 | } 748 | 749 | for _, tt := range tests { 750 | t.Run(tt.name, func(t *testing.T) { 751 | delay := calculateExponentialBackoffDuration(tt.retries, tt.baseDuration) 752 | assert.Equal(t, tt.expectedDelay, delay) 753 | }) 754 | } 755 | } 756 | -------------------------------------------------------------------------------- /mocks/consumer_group.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.37.1. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | context "context" 7 | 8 | sarama "github.com/IBM/sarama" 9 | mock "github.com/stretchr/testify/mock" 10 | ) 11 | 12 | // ConsumerGroup is an autogenerated mock type for the ConsumerGroup type 13 | type ConsumerGroup struct { 14 | mock.Mock 15 | } 16 | 17 | // Close provides a mock function with given fields: 18 | func (_m *ConsumerGroup) Close() error { 19 | ret := _m.Called() 20 | 21 | var r0 error 22 | if rf, ok := ret.Get(0).(func() error); ok { 23 | r0 = rf() 24 | } else { 25 | r0 = ret.Error(0) 26 | } 27 | 28 | return r0 29 | } 30 | 31 | // Consume provides a mock function with given fields: ctx, topics, handler 32 | func (_m *ConsumerGroup) Consume(ctx context.Context, topics []string, handler sarama.ConsumerGroupHandler) error { 33 | ret := _m.Called(ctx, topics, handler) 34 | 35 | var r0 error 36 | if rf, ok := ret.Get(0).(func(context.Context, []string, sarama.ConsumerGroupHandler) error); ok { 37 | r0 = rf(ctx, topics, handler) 38 | } else { 39 | r0 = ret.Error(0) 40 | } 41 | 42 | return r0 43 | } 44 | 45 | // Errors provides a mock function with given fields: 46 | func (_m *ConsumerGroup) Errors() <-chan error { 47 | ret := _m.Called() 48 | 49 | var r0 <-chan error 50 | if rf, ok := ret.Get(0).(func() <-chan error); ok { 51 | r0 = rf() 52 | } else { 53 | if ret.Get(0) != nil { 54 | r0 = ret.Get(0).(<-chan error) 55 | } 56 | } 57 | 58 | return r0 59 | } 60 | 61 | // Pause provides a mock function with given fields: partitions 62 | func (_m *ConsumerGroup) Pause(partitions map[string][]int32) { 63 | _m.Called(partitions) 64 | } 65 | 66 | // PauseAll provides a mock function with given fields: 67 | func (_m *ConsumerGroup) PauseAll() { 68 | _m.Called() 69 | } 70 | 71 | // Resume provides a mock function with given fields: partitions 72 | func (_m *ConsumerGroup) Resume(partitions map[string][]int32) { 73 | _m.Called(partitions) 74 | } 75 | 76 | // ResumeAll provides a mock function with given fields: 77 | func (_m *ConsumerGroup) ResumeAll() { 78 | _m.Called() 79 | } 80 | 81 | // NewConsumerGroup creates a new instance of ConsumerGroup. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 82 | // The first argument is typically a *testing.T value. 83 | func NewConsumerGroup(t interface { 84 | mock.TestingT 85 | Cleanup(func()) 86 | }) *ConsumerGroup { 87 | mock := &ConsumerGroup{} 88 | mock.Mock.Test(t) 89 | 90 | t.Cleanup(func() { mock.AssertExpectations(t) }) 91 | 92 | return mock 93 | } 94 | -------------------------------------------------------------------------------- /mocks/consumer_group_claim.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.37.1. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | sarama "github.com/IBM/sarama" 7 | mock "github.com/stretchr/testify/mock" 8 | ) 9 | 10 | // ConsumerGroupClaim is an autogenerated mock type for the ConsumerGroupClaim type 11 | type ConsumerGroupClaim struct { 12 | mock.Mock 13 | } 14 | 15 | // HighWaterMarkOffset provides a mock function with given fields: 16 | func (_m *ConsumerGroupClaim) HighWaterMarkOffset() int64 { 17 | ret := _m.Called() 18 | 19 | var r0 int64 20 | if rf, ok := ret.Get(0).(func() int64); ok { 21 | r0 = rf() 22 | } else { 23 | r0 = ret.Get(0).(int64) 24 | } 25 | 26 | return r0 27 | } 28 | 29 | // InitialOffset provides a mock function with given fields: 30 | func (_m *ConsumerGroupClaim) InitialOffset() int64 { 31 | ret := _m.Called() 32 | 33 | var r0 int64 34 | if rf, ok := ret.Get(0).(func() int64); ok { 35 | r0 = rf() 36 | } else { 37 | r0 = ret.Get(0).(int64) 38 | } 39 | 40 | return r0 41 | } 42 | 43 | // Messages provides a mock function with given fields: 44 | func (_m *ConsumerGroupClaim) Messages() <-chan *sarama.ConsumerMessage { 45 | ret := _m.Called() 46 | 47 | var r0 <-chan *sarama.ConsumerMessage 48 | if rf, ok := ret.Get(0).(func() <-chan *sarama.ConsumerMessage); ok { 49 | r0 = rf() 50 | } else { 51 | if ret.Get(0) != nil { 52 | r0 = ret.Get(0).(<-chan *sarama.ConsumerMessage) 53 | } 54 | } 55 | 56 | return r0 57 | } 58 | 59 | // Partition provides a mock function with given fields: 60 | func (_m *ConsumerGroupClaim) Partition() int32 { 61 | ret := _m.Called() 62 | 63 | var r0 int32 64 | if rf, ok := ret.Get(0).(func() int32); ok { 65 | r0 = rf() 66 | } else { 67 | r0 = ret.Get(0).(int32) 68 | } 69 | 70 | return r0 71 | } 72 | 73 | // Topic provides a mock function with given fields: 74 | func (_m *ConsumerGroupClaim) Topic() string { 75 | ret := _m.Called() 76 | 77 | var r0 string 78 | if rf, ok := ret.Get(0).(func() string); ok { 79 | r0 = rf() 80 | } else { 81 | r0 = ret.Get(0).(string) 82 | } 83 | 84 | return r0 85 | } 86 | 87 | // NewConsumerGroupClaim creates a new instance of ConsumerGroupClaim. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 88 | // The first argument is typically a *testing.T value. 89 | func NewConsumerGroupClaim(t interface { 90 | mock.TestingT 91 | Cleanup(func()) 92 | }) *ConsumerGroupClaim { 93 | mock := &ConsumerGroupClaim{} 94 | mock.Mock.Test(t) 95 | 96 | t.Cleanup(func() { mock.AssertExpectations(t) }) 97 | 98 | return mock 99 | } 100 | -------------------------------------------------------------------------------- /mocks/consumer_group_handler.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.37.1. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | sarama "github.com/IBM/sarama" 7 | mock "github.com/stretchr/testify/mock" 8 | ) 9 | 10 | // ConsumerGroupHandler is an autogenerated mock type for the ConsumerGroupHandler type 11 | type ConsumerGroupHandler struct { 12 | mock.Mock 13 | } 14 | 15 | // Cleanup provides a mock function with given fields: _a0 16 | func (_m *ConsumerGroupHandler) Cleanup(_a0 sarama.ConsumerGroupSession) error { 17 | ret := _m.Called(_a0) 18 | 19 | var r0 error 20 | if rf, ok := ret.Get(0).(func(sarama.ConsumerGroupSession) error); ok { 21 | r0 = rf(_a0) 22 | } else { 23 | r0 = ret.Error(0) 24 | } 25 | 26 | return r0 27 | } 28 | 29 | // ConsumeClaim provides a mock function with given fields: _a0, _a1 30 | func (_m *ConsumerGroupHandler) ConsumeClaim(_a0 sarama.ConsumerGroupSession, _a1 sarama.ConsumerGroupClaim) error { 31 | ret := _m.Called(_a0, _a1) 32 | 33 | var r0 error 34 | if rf, ok := ret.Get(0).(func(sarama.ConsumerGroupSession, sarama.ConsumerGroupClaim) error); ok { 35 | r0 = rf(_a0, _a1) 36 | } else { 37 | r0 = ret.Error(0) 38 | } 39 | 40 | return r0 41 | } 42 | 43 | // Setup provides a mock function with given fields: _a0 44 | func (_m *ConsumerGroupHandler) Setup(_a0 sarama.ConsumerGroupSession) error { 45 | ret := _m.Called(_a0) 46 | 47 | var r0 error 48 | if rf, ok := ret.Get(0).(func(sarama.ConsumerGroupSession) error); ok { 49 | r0 = rf(_a0) 50 | } else { 51 | r0 = ret.Error(0) 52 | } 53 | 54 | return r0 55 | } 56 | 57 | // NewConsumerGroupHandler creates a new instance of ConsumerGroupHandler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 58 | // The first argument is typically a *testing.T value. 59 | func NewConsumerGroupHandler(t interface { 60 | mock.TestingT 61 | Cleanup(func()) 62 | }) *ConsumerGroupHandler { 63 | mock := &ConsumerGroupHandler{} 64 | mock.Mock.Test(t) 65 | 66 | t.Cleanup(func() { mock.AssertExpectations(t) }) 67 | 68 | return mock 69 | } 70 | -------------------------------------------------------------------------------- /mocks/consumer_group_session.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.37.1. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | context "context" 7 | 8 | sarama "github.com/IBM/sarama" 9 | mock "github.com/stretchr/testify/mock" 10 | ) 11 | 12 | // ConsumerGroupSession is an autogenerated mock type for the ConsumerGroupSession type 13 | type ConsumerGroupSession struct { 14 | mock.Mock 15 | } 16 | 17 | // Claims provides a mock function with given fields: 18 | func (_m *ConsumerGroupSession) Claims() map[string][]int32 { 19 | ret := _m.Called() 20 | 21 | var r0 map[string][]int32 22 | if rf, ok := ret.Get(0).(func() map[string][]int32); ok { 23 | r0 = rf() 24 | } else { 25 | if ret.Get(0) != nil { 26 | r0 = ret.Get(0).(map[string][]int32) 27 | } 28 | } 29 | 30 | return r0 31 | } 32 | 33 | // Commit provides a mock function with given fields: 34 | func (_m *ConsumerGroupSession) Commit() { 35 | _m.Called() 36 | } 37 | 38 | // Context provides a mock function with given fields: 39 | func (_m *ConsumerGroupSession) Context() context.Context { 40 | ret := _m.Called() 41 | 42 | var r0 context.Context 43 | if rf, ok := ret.Get(0).(func() context.Context); ok { 44 | r0 = rf() 45 | } else { 46 | if ret.Get(0) != nil { 47 | r0 = ret.Get(0).(context.Context) 48 | } 49 | } 50 | 51 | return r0 52 | } 53 | 54 | // GenerationID provides a mock function with given fields: 55 | func (_m *ConsumerGroupSession) GenerationID() int32 { 56 | ret := _m.Called() 57 | 58 | var r0 int32 59 | if rf, ok := ret.Get(0).(func() int32); ok { 60 | r0 = rf() 61 | } else { 62 | r0 = ret.Get(0).(int32) 63 | } 64 | 65 | return r0 66 | } 67 | 68 | // MarkMessage provides a mock function with given fields: msg, metadata 69 | func (_m *ConsumerGroupSession) MarkMessage(msg *sarama.ConsumerMessage, metadata string) { 70 | _m.Called(msg, metadata) 71 | } 72 | 73 | // MarkOffset provides a mock function with given fields: topic, partition, offset, metadata 74 | func (_m *ConsumerGroupSession) MarkOffset(topic string, partition int32, offset int64, metadata string) { 75 | _m.Called(topic, partition, offset, metadata) 76 | } 77 | 78 | // MemberID provides a mock function with given fields: 79 | func (_m *ConsumerGroupSession) MemberID() string { 80 | ret := _m.Called() 81 | 82 | var r0 string 83 | if rf, ok := ret.Get(0).(func() string); ok { 84 | r0 = rf() 85 | } else { 86 | r0 = ret.Get(0).(string) 87 | } 88 | 89 | return r0 90 | } 91 | 92 | // ResetOffset provides a mock function with given fields: topic, partition, offset, metadata 93 | func (_m *ConsumerGroupSession) ResetOffset(topic string, partition int32, offset int64, metadata string) { 94 | _m.Called(topic, partition, offset, metadata) 95 | } 96 | 97 | // NewConsumerGroupSession creates a new instance of ConsumerGroupSession. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 98 | // The first argument is typically a *testing.T value. 99 | func NewConsumerGroupSession(t interface { 100 | mock.TestingT 101 | Cleanup(func()) 102 | }) *ConsumerGroupSession { 103 | mock := &ConsumerGroupSession{} 104 | mock.Mock.Test(t) 105 | 106 | t.Cleanup(func() { mock.AssertExpectations(t) }) 107 | 108 | return mock 109 | } 110 | -------------------------------------------------------------------------------- /mocks/producer.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.24.0. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | sarama "github.com/IBM/sarama" 7 | mock "github.com/stretchr/testify/mock" 8 | ) 9 | 10 | // MockProducer is an autogenerated mock type for the Producer type 11 | type MockProducer struct { 12 | mock.Mock 13 | } 14 | 15 | type MockProducer_Expecter struct { 16 | mock *mock.Mock 17 | } 18 | 19 | func (_m *MockProducer) EXPECT() *MockProducer_Expecter { 20 | return &MockProducer_Expecter{mock: &_m.Mock} 21 | } 22 | 23 | // Produce provides a mock function with given fields: msg 24 | func (_m *MockProducer) Produce(msg *sarama.ProducerMessage) error { 25 | ret := _m.Called(msg) 26 | 27 | var r0 error 28 | if rf, ok := ret.Get(0).(func(*sarama.ProducerMessage) error); ok { 29 | r0 = rf(msg) 30 | } else { 31 | r0 = ret.Error(0) 32 | } 33 | 34 | return r0 35 | } 36 | 37 | // MockProducer_Produce_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Produce' 38 | type MockProducer_Produce_Call struct { 39 | *mock.Call 40 | } 41 | 42 | // Produce is a helper method to define mock.On call 43 | // - msg *sarama.ProducerMessage 44 | func (_e *MockProducer_Expecter) Produce(msg interface{}) *MockProducer_Produce_Call { 45 | return &MockProducer_Produce_Call{Call: _e.mock.On("Produce", msg)} 46 | } 47 | 48 | func (_c *MockProducer_Produce_Call) Run(run func(msg *sarama.ProducerMessage)) *MockProducer_Produce_Call { 49 | _c.Call.Run(func(args mock.Arguments) { 50 | run(args[0].(*sarama.ProducerMessage)) 51 | }) 52 | return _c 53 | } 54 | 55 | func (_c *MockProducer_Produce_Call) Return(_a0 error) *MockProducer_Produce_Call { 56 | _c.Call.Return(_a0) 57 | return _c 58 | } 59 | 60 | func (_c *MockProducer_Produce_Call) RunAndReturn(run func(*sarama.ProducerMessage) error) *MockProducer_Produce_Call { 61 | _c.Call.Return(run) 62 | return _c 63 | } 64 | 65 | type mockConstructorTestingTNewMockProducer interface { 66 | mock.TestingT 67 | Cleanup(func()) 68 | } 69 | 70 | // NewMockProducer creates a new instance of MockProducer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 71 | func NewMockProducer(t mockConstructorTestingTNewMockProducer) *MockProducer { 72 | mock := &MockProducer{} 73 | mock.Mock.Test(t) 74 | 75 | t.Cleanup(func() { mock.AssertExpectations(t) }) 76 | 77 | return mock 78 | } 79 | -------------------------------------------------------------------------------- /mocks/std_logger.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.37.1. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import mock "github.com/stretchr/testify/mock" 6 | 7 | // StdLogger is an autogenerated mock type for the StdLogger type 8 | type StdLogger struct { 9 | mock.Mock 10 | } 11 | 12 | // Print provides a mock function with given fields: v 13 | func (_m *StdLogger) Print(v ...interface{}) { 14 | var _ca []interface{} 15 | _ca = append(_ca, v...) 16 | _m.Called(_ca...) 17 | } 18 | 19 | // Printf provides a mock function with given fields: format, v 20 | func (_m *StdLogger) Printf(format string, v ...interface{}) { 21 | var _ca []interface{} 22 | _ca = append(_ca, format) 23 | _ca = append(_ca, v...) 24 | _m.Called(_ca...) 25 | } 26 | 27 | // Println provides a mock function with given fields: v 28 | func (_m *StdLogger) Println(v ...interface{}) { 29 | var _ca []interface{} 30 | _ca = append(_ca, v...) 31 | _m.Called(_ca...) 32 | } 33 | 34 | // NewStdLogger creates a new instance of StdLogger. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 35 | // The first argument is typically a *testing.T value. 36 | func NewStdLogger(t interface { 37 | mock.TestingT 38 | Cleanup(func()) 39 | }) *StdLogger { 40 | mock := &StdLogger{} 41 | mock.Mock.Test(t) 42 | 43 | t.Cleanup(func() { mock.AssertExpectations(t) }) 44 | 45 | return mock 46 | } 47 | -------------------------------------------------------------------------------- /mocks/sync_producer.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.37.1. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | sarama "github.com/IBM/sarama" 7 | mock "github.com/stretchr/testify/mock" 8 | ) 9 | 10 | // SyncProducer is an autogenerated mock type for the SyncProducer type 11 | type SyncProducer struct { 12 | mock.Mock 13 | } 14 | 15 | // AbortTxn provides a mock function with given fields: 16 | func (_m *SyncProducer) AbortTxn() error { 17 | ret := _m.Called() 18 | 19 | var r0 error 20 | if rf, ok := ret.Get(0).(func() error); ok { 21 | r0 = rf() 22 | } else { 23 | r0 = ret.Error(0) 24 | } 25 | 26 | return r0 27 | } 28 | 29 | // AddMessageToTxn provides a mock function with given fields: msg, groupId, metadata 30 | func (_m *SyncProducer) AddMessageToTxn(msg *sarama.ConsumerMessage, groupId string, metadata *string) error { 31 | ret := _m.Called(msg, groupId, metadata) 32 | 33 | var r0 error 34 | if rf, ok := ret.Get(0).(func(*sarama.ConsumerMessage, string, *string) error); ok { 35 | r0 = rf(msg, groupId, metadata) 36 | } else { 37 | r0 = ret.Error(0) 38 | } 39 | 40 | return r0 41 | } 42 | 43 | // AddOffsetsToTxn provides a mock function with given fields: offsets, groupId 44 | func (_m *SyncProducer) AddOffsetsToTxn(offsets map[string][]*sarama.PartitionOffsetMetadata, groupId string) error { 45 | ret := _m.Called(offsets, groupId) 46 | 47 | var r0 error 48 | if rf, ok := ret.Get(0).(func(map[string][]*sarama.PartitionOffsetMetadata, string) error); ok { 49 | r0 = rf(offsets, groupId) 50 | } else { 51 | r0 = ret.Error(0) 52 | } 53 | 54 | return r0 55 | } 56 | 57 | // BeginTxn provides a mock function with given fields: 58 | func (_m *SyncProducer) BeginTxn() error { 59 | ret := _m.Called() 60 | 61 | var r0 error 62 | if rf, ok := ret.Get(0).(func() error); ok { 63 | r0 = rf() 64 | } else { 65 | r0 = ret.Error(0) 66 | } 67 | 68 | return r0 69 | } 70 | 71 | // Close provides a mock function with given fields: 72 | func (_m *SyncProducer) Close() error { 73 | ret := _m.Called() 74 | 75 | var r0 error 76 | if rf, ok := ret.Get(0).(func() error); ok { 77 | r0 = rf() 78 | } else { 79 | r0 = ret.Error(0) 80 | } 81 | 82 | return r0 83 | } 84 | 85 | // CommitTxn provides a mock function with given fields: 86 | func (_m *SyncProducer) CommitTxn() error { 87 | ret := _m.Called() 88 | 89 | var r0 error 90 | if rf, ok := ret.Get(0).(func() error); ok { 91 | r0 = rf() 92 | } else { 93 | r0 = ret.Error(0) 94 | } 95 | 96 | return r0 97 | } 98 | 99 | // IsTransactional provides a mock function with given fields: 100 | func (_m *SyncProducer) IsTransactional() bool { 101 | ret := _m.Called() 102 | 103 | var r0 bool 104 | if rf, ok := ret.Get(0).(func() bool); ok { 105 | r0 = rf() 106 | } else { 107 | r0 = ret.Get(0).(bool) 108 | } 109 | 110 | return r0 111 | } 112 | 113 | // SendMessage provides a mock function with given fields: msg 114 | func (_m *SyncProducer) SendMessage(msg *sarama.ProducerMessage) (int32, int64, error) { 115 | ret := _m.Called(msg) 116 | 117 | var r0 int32 118 | var r1 int64 119 | var r2 error 120 | if rf, ok := ret.Get(0).(func(*sarama.ProducerMessage) (int32, int64, error)); ok { 121 | return rf(msg) 122 | } 123 | if rf, ok := ret.Get(0).(func(*sarama.ProducerMessage) int32); ok { 124 | r0 = rf(msg) 125 | } else { 126 | r0 = ret.Get(0).(int32) 127 | } 128 | 129 | if rf, ok := ret.Get(1).(func(*sarama.ProducerMessage) int64); ok { 130 | r1 = rf(msg) 131 | } else { 132 | r1 = ret.Get(1).(int64) 133 | } 134 | 135 | if rf, ok := ret.Get(2).(func(*sarama.ProducerMessage) error); ok { 136 | r2 = rf(msg) 137 | } else { 138 | r2 = ret.Error(2) 139 | } 140 | 141 | return r0, r1, r2 142 | } 143 | 144 | // SendMessages provides a mock function with given fields: msgs 145 | func (_m *SyncProducer) SendMessages(msgs []*sarama.ProducerMessage) error { 146 | ret := _m.Called(msgs) 147 | 148 | var r0 error 149 | if rf, ok := ret.Get(0).(func([]*sarama.ProducerMessage) error); ok { 150 | r0 = rf(msgs) 151 | } else { 152 | r0 = ret.Error(0) 153 | } 154 | 155 | return r0 156 | } 157 | 158 | // TxnStatus provides a mock function with given fields: 159 | func (_m *SyncProducer) TxnStatus() sarama.ProducerTxnStatusFlag { 160 | ret := _m.Called() 161 | 162 | var r0 sarama.ProducerTxnStatusFlag 163 | if rf, ok := ret.Get(0).(func() sarama.ProducerTxnStatusFlag); ok { 164 | r0 = rf() 165 | } else { 166 | r0 = ret.Get(0).(sarama.ProducerTxnStatusFlag) 167 | } 168 | 169 | return r0 170 | } 171 | 172 | // NewSyncProducer creates a new instance of SyncProducer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 173 | // The first argument is typically a *testing.T value. 174 | func NewSyncProducer(t interface { 175 | mock.TestingT 176 | Cleanup(func()) 177 | }) *SyncProducer { 178 | mock := &SyncProducer{} 179 | mock.Mock.Test(t) 180 | 181 | t.Cleanup(func() { mock.AssertExpectations(t) }) 182 | 183 | return mock 184 | } 185 | -------------------------------------------------------------------------------- /murmur.go: -------------------------------------------------------------------------------- 1 | /*MIT License 2 | 3 | Copyright (c) 2019 Alexandr Burdiyan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | // Package kafka copied from https://github.com/burdiyan/kafkautil/blob/master/partitioner.go 25 | // copied here to ensure this stay. 26 | package kafka 27 | 28 | import ( 29 | "hash" 30 | 31 | "github.com/IBM/sarama" 32 | ) 33 | 34 | // NewJVMCompatiblePartitioner creates a Sarama partitioner that uses 35 | // the same hashing algorithm as JVM Kafka clients. 36 | func NewJVMCompatiblePartitioner(topic string) sarama.Partitioner { 37 | return sarama.NewCustomHashPartitioner(MurmurHasher)(topic) 38 | } 39 | 40 | // murmurHash implements hash.Hash32 interface, 41 | // solely to conform to required hasher for Sarama. 42 | // it does not support streaming since it is not required for Sarama. 43 | type murmurHash struct { 44 | v int32 45 | } 46 | 47 | // MurmurHasher creates murmur2 hasher implementing hash.Hash32 interface. 48 | // The implementation is not full and does not support streaming. 49 | // It only implements the interface to comply with sarama.NewCustomHashPartitioner signature. 50 | // But Sarama only uses Write method once, when writing keys and values of the message, 51 | // so streaming support is not necessary. 52 | func MurmurHasher() hash.Hash32 { 53 | return new(murmurHash) 54 | } 55 | 56 | func (m *murmurHash) Write(d []byte) (n int, err error) { 57 | n = len(d) 58 | m.v = murmur2(d) 59 | return 60 | } 61 | 62 | func (m *murmurHash) Reset() { 63 | m.v = 0 64 | } 65 | 66 | func (m *murmurHash) Size() int { return 32 } 67 | 68 | func (m *murmurHash) BlockSize() int { return 4 } 69 | 70 | // Sum is noop. 71 | func (m *murmurHash) Sum(in []byte) []byte { 72 | return in 73 | } 74 | 75 | func (m *murmurHash) Sum32() uint32 { 76 | return uint32(toPositive(m.v)) 77 | } 78 | 79 | // murmur2 implements hashing algorithm used by JVM clients for Kafka. 80 | // See the original implementation: https://github.com/apache/kafka/blob/1.0.0/clients/src/main/java/org/apache/kafka/common/utils/Utils.java#L353 81 | func murmur2(data []byte) int32 { 82 | length := int32(len(data)) 83 | seed := uint32(0x9747b28c) 84 | m := int32(0x5bd1e995) 85 | r := uint32(24) 86 | 87 | h := int32(seed ^ uint32(length)) 88 | length4 := length / 4 89 | 90 | for i := int32(0); i < length4; i++ { 91 | i4 := i * 4 92 | k := int32(data[i4+0]&0xff) + (int32(data[i4+1]&0xff) << 8) + (int32(data[i4+2]&0xff) << 16) + (int32(data[i4+3]&0xff) << 24) 93 | k *= m 94 | k ^= int32(uint32(k) >> r) 95 | k *= m 96 | h *= m 97 | h ^= k 98 | } 99 | 100 | switch length % 4 { 101 | case 3: 102 | h ^= int32(data[(length & ^3)+2]&0xff) << 16 103 | fallthrough 104 | case 2: 105 | h ^= int32(data[(length & ^3)+1]&0xff) << 8 106 | fallthrough 107 | case 1: 108 | h ^= int32(data[length & ^3] & 0xff) 109 | h *= m 110 | } 111 | 112 | h ^= int32(uint32(h) >> 13) 113 | h *= m 114 | h ^= int32(uint32(h) >> 15) 115 | 116 | return h 117 | } 118 | 119 | // toPositive converts i to positive number as per the original implementation in the JVM clients for Kafka. 120 | // See the original implementation: https://github.com/apache/kafka/blob/1.0.0/clients/src/main/java/org/apache/kafka/common/utils/Utils.java#L741 121 | func toPositive(i int32) int32 { 122 | return i & 0x7fffffff 123 | } 124 | -------------------------------------------------------------------------------- /murmur_test.go: -------------------------------------------------------------------------------- 1 | /*MIT License 2 | 3 | Copyright (c) 2019 Alexandr Burdiyan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | // from https://github.com/burdiyan/kafkautil/blob/master/partitioner.go 25 | // copied here to ensure this stay. 26 | 27 | package kafka 28 | 29 | import ( 30 | "testing" 31 | ) 32 | 33 | func TestHashInterface(t *testing.T) { 34 | var _ = MurmurHasher() 35 | } 36 | 37 | func TestMurmur2(t *testing.T) { 38 | // Test cases are generated offline using JVM Kafka client for version 1.0.0. 39 | cases := []struct { 40 | Input []byte 41 | Expected int32 42 | ExpectedPositive uint32 43 | }{ 44 | {[]byte("21"), -973932308, 1173551340}, 45 | {[]byte("foobar"), -790332482, 1357151166}, 46 | {[]byte{12, 42, 56, 24, 109, 111}, 274204207, 274204207}, 47 | {[]byte("a-little-bit-long-string"), -985981536, 1161502112}, 48 | {[]byte("a-little-bit-longer-string"), -1486304829, 661178819}, 49 | {[]byte("lkjh234lh9fiuh90y23oiuhsafujhadof229phr9h19h89h8"), -58897971, 2088585677}, 50 | {[]byte{'a', 'b', 'c'}, 479470107, 479470107}, 51 | } 52 | 53 | hasher := MurmurHasher() 54 | 55 | for _, c := range cases { 56 | if res := murmur2(c.Input); res != c.Expected { 57 | t.Errorf("for %q expected: %d, got: %d", c.Input, c.Expected, res) 58 | } 59 | 60 | hasher.Reset() 61 | hasher.Write(c.Input) 62 | 63 | if res2 := hasher.Sum32(); res2 != uint32(c.ExpectedPositive) { 64 | t.Errorf("hasher: for %q expected: %d, got: %d", c.Input, c.ExpectedPositive, res2) 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /options.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | 8 | "github.com/IBM/sarama" 9 | "github.com/opentracing/opentracing-go" 10 | "github.com/ricardo-ch/go-tracing" 11 | ) 12 | 13 | // WithInstrumenting adds the instrumenting layer on a listener. 14 | func WithInstrumenting() ListenerOption { 15 | return func(l *listener) { 16 | l.instrumenting = NewConsumerMetricsService(l.groupID) 17 | } 18 | } 19 | 20 | // ProducerOption is a function that is passed to the producer constructor to configure it. 21 | type ProducerOption func(p *producer) 22 | 23 | // WithProducerInstrumenting adds the instrumenting layer on a producer. 24 | func WithProducerInstrumenting() ProducerOption { 25 | return func(p *producer) { 26 | p.instrumenting = NewProducerMetricsService() 27 | p.handler = p.instrumenting.Instrumentation(p.handler) 28 | } 29 | } 30 | 31 | // WithDeadletterProducerInstrumenting adds the instrumenting layer on a deadletter producer. 32 | func WithDeadletterProducerInstrumenting() ProducerOption { 33 | return func(p *producer) { 34 | p.instrumenting = NewDeadletterProducerMetricsService() 35 | p.handler = p.instrumenting.DeadletterInstrumentation(p.handler) 36 | } 37 | } 38 | 39 | // TracingFunc is used to create tracing and/or propagate the tracing context from each messages to the go context. 40 | type TracingFunc func(ctx context.Context, msg *sarama.ConsumerMessage) (opentracing.Span, context.Context) 41 | 42 | // WithTracing accepts a TracingFunc to execute before each message 43 | func WithTracing(tracer TracingFunc) ListenerOption { 44 | return func(l *listener) { 45 | l.tracer = tracer 46 | } 47 | } 48 | 49 | // DefaultTracing implements TracingFunc 50 | // It fetches opentracing headers from the kafka message headers, then creates a span using the opentracing.GlobalTracer() 51 | // usage: `listener, err = kafka.NewListener(brokers, appName, handlers, kafka.WithTracing(kafka.DefaultTracing))` 52 | func DefaultTracing(ctx context.Context, msg *sarama.ConsumerMessage) (opentracing.Span, context.Context) { 53 | if ctx == nil { 54 | ctx = context.Background() 55 | } 56 | carrier := make(map[string]string, len(msg.Headers)) 57 | for _, h := range msg.Headers { 58 | carrier[string(h.Key)] = string(h.Value) 59 | } 60 | return tracing.ExtractFromCarrier(ctx, carrier, fmt.Sprintf("message from %s", msg.Topic), 61 | &map[string]interface{}{"offset": msg.Offset, "partition": msg.Partition, "key": string(msg.Key)}, 62 | ) 63 | } 64 | 65 | // GetKafkaHeadersFromContext fetch tracing metadata from context and returns them in format []RecordHeader 66 | func GetKafkaHeadersFromContext(ctx context.Context) []sarama.RecordHeader { 67 | carrier := tracing.InjectIntoCarrier(ctx) 68 | 69 | recordHeaders := make([]sarama.RecordHeader, 0, len(carrier)) 70 | for headerKey, headerValue := range carrier { 71 | recordHeaders = append(recordHeaders, sarama.RecordHeader{Key: []byte(headerKey), Value: []byte(headerValue)}) 72 | } 73 | return recordHeaders 74 | } 75 | 76 | // GetContextFromKafkaMessage fetches tracing headers from the kafka message 77 | func GetContextFromKafkaMessage(ctx context.Context, msg *sarama.ConsumerMessage) (opentracing.Span, context.Context) { 78 | if ctx == nil { 79 | ctx = context.Background() 80 | } 81 | carrier := make(map[string]string, len(msg.Headers)) 82 | for _, h := range msg.Headers { 83 | carrier[string(h.Key)] = string(h.Value) 84 | } 85 | return tracing.ExtractFromCarrier(ctx, carrier, fmt.Sprintf("message from %s", msg.Topic), nil) 86 | } 87 | 88 | // SerializeKafkaHeadersFromContext fetches tracing metadata from context and serialize it into a json map[string]string 89 | func SerializeKafkaHeadersFromContext(ctx context.Context) (string, error) { 90 | kafkaHeaders := tracing.InjectIntoCarrier(ctx) 91 | kafkaHeadersJSON, err := json.Marshal(kafkaHeaders) 92 | 93 | return string(kafkaHeadersJSON), err 94 | } 95 | 96 | // DeserializeContextFromKafkaHeaders fetches tracing headers from json encoded carrier and returns the context 97 | func DeserializeContextFromKafkaHeaders(ctx context.Context, kafkaheaders string) (context.Context, error) { 98 | if ctx == nil { 99 | ctx = context.Background() 100 | } 101 | 102 | var rawHeaders map[string]string 103 | if err := json.Unmarshal([]byte(kafkaheaders), &rawHeaders); err != nil { 104 | return nil, err 105 | } 106 | 107 | _, ctx = tracing.ExtractFromCarrier(ctx, rawHeaders, "", nil) 108 | 109 | return ctx, nil 110 | } 111 | -------------------------------------------------------------------------------- /producer.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "github.com/IBM/sarama" 5 | ) 6 | 7 | type Producer interface { 8 | Produce(msg *sarama.ProducerMessage) error 9 | } 10 | 11 | // ProducerHandler is a function that handles the production of a message. It is exposed to allow for easy middleware building. 12 | type ProducerHandler func(p *producer, msg *sarama.ProducerMessage) error 13 | 14 | type producer struct { 15 | handler ProducerHandler 16 | producer sarama.SyncProducer 17 | instrumenting *ProducerMetricsService 18 | } 19 | 20 | // NewProducer creates a new producer that uses the default sarama client. 21 | func NewProducer(options ...ProducerOption) (Producer, error) { 22 | client, err := getClient() 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | p, err := sarama.NewSyncProducerFromClient(*client) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | producer := &producer{ 33 | producer: p, 34 | handler: produce, 35 | } 36 | 37 | for _, option := range options { 38 | option(producer) 39 | } 40 | 41 | return producer, nil 42 | } 43 | 44 | // Produce sends a message to the kafka cluster. 45 | func (p *producer) Produce(msg *sarama.ProducerMessage) error { 46 | return p.handler(p, msg) 47 | } 48 | 49 | func produce(p *producer, msg *sarama.ProducerMessage) error { 50 | _, _, err := p.producer.SendMessage(msg) 51 | return err 52 | } 53 | -------------------------------------------------------------------------------- /producer_instrumenting.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/IBM/sarama" 8 | "github.com/prometheus/client_golang/prometheus" 9 | ) 10 | 11 | var ( 12 | producerRecordSendCounter *prometheus.CounterVec 13 | producerDeadletterSendCounter *prometheus.CounterVec 14 | producerRecordSendLatency *prometheus.HistogramVec 15 | producerRecordErrorCounter *prometheus.CounterVec 16 | 17 | producerMetricsMutex = &sync.Mutex{} 18 | producerMetricsLabel = []string{"kafka_topic"} 19 | ) 20 | 21 | // ProducerMetricsService is a service that provides metrics for the producer. 22 | type ProducerMetricsService struct { 23 | recordSendCounter *prometheus.CounterVec 24 | deadletterRecordSendCounter *prometheus.CounterVec 25 | recordSendLatency *prometheus.HistogramVec 26 | errorCounter *prometheus.CounterVec 27 | } 28 | 29 | func getPrometheusRecordSendInstrumentation() *prometheus.CounterVec { 30 | if producerRecordSendCounter != nil { 31 | return producerRecordSendCounter 32 | } 33 | 34 | producerMetricsMutex.Lock() 35 | defer producerMetricsMutex.Unlock() 36 | 37 | if producerRecordSendCounter == nil { 38 | producerRecordSendCounter = prometheus.NewCounterVec( 39 | prometheus.CounterOpts{ 40 | Namespace: "kafka", 41 | Subsystem: "producer", 42 | Name: "record_send_total", 43 | Help: "Number of records sent", 44 | }, producerMetricsLabel) 45 | prometheus.MustRegister(producerRecordSendCounter) 46 | } 47 | 48 | return producerRecordSendCounter 49 | } 50 | 51 | func getPrometheusDeadletterRecordSendInstrumentation() *prometheus.CounterVec { 52 | if producerDeadletterSendCounter != nil { 53 | return producerDeadletterSendCounter 54 | } 55 | 56 | producerMetricsMutex.Lock() 57 | defer producerMetricsMutex.Unlock() 58 | 59 | if producerDeadletterSendCounter == nil { 60 | producerDeadletterSendCounter = prometheus.NewCounterVec( 61 | prometheus.CounterOpts{ 62 | Namespace: "kafka", 63 | Subsystem: "producer", 64 | Name: "dead_letter_created_total", 65 | Help: "Number of dead letter created", 66 | }, producerMetricsLabel) 67 | prometheus.MustRegister(producerDeadletterSendCounter) 68 | } 69 | 70 | return producerDeadletterSendCounter 71 | } 72 | 73 | func getPrometheusRecordSendLatencyInstrumentation() *prometheus.HistogramVec { 74 | if producerRecordSendLatency != nil { 75 | return producerRecordSendLatency 76 | } 77 | 78 | producerMetricsMutex.Lock() 79 | defer producerMetricsMutex.Unlock() 80 | 81 | if producerRecordSendLatency == nil { 82 | producerRecordSendLatency = prometheus.NewHistogramVec( 83 | prometheus.HistogramOpts{ 84 | Namespace: "kafka", 85 | Subsystem: "producer", 86 | Name: "record_send_latency_seconds", 87 | Help: "Latency of records sent", 88 | }, producerMetricsLabel) 89 | prometheus.MustRegister(producerRecordSendLatency) 90 | } 91 | 92 | return producerRecordSendLatency 93 | } 94 | 95 | func getPrometheusSendErrorInstrumentation() *prometheus.CounterVec { 96 | if producerRecordErrorCounter != nil { 97 | return producerRecordErrorCounter 98 | } 99 | 100 | producerMetricsMutex.Lock() 101 | defer producerMetricsMutex.Unlock() 102 | 103 | if producerRecordErrorCounter == nil { 104 | producerRecordErrorCounter = prometheus.NewCounterVec( 105 | prometheus.CounterOpts{ 106 | Namespace: "kafka", 107 | Subsystem: "producer", 108 | Name: "record_error_total", 109 | Help: "Number of records send error", 110 | }, producerMetricsLabel) 111 | prometheus.MustRegister(producerRecordErrorCounter) 112 | } 113 | 114 | return producerRecordErrorCounter 115 | } 116 | 117 | func NewProducerMetricsService() *ProducerMetricsService { 118 | return &ProducerMetricsService{ 119 | recordSendCounter: getPrometheusRecordSendInstrumentation(), 120 | recordSendLatency: getPrometheusRecordSendLatencyInstrumentation(), 121 | errorCounter: getPrometheusSendErrorInstrumentation(), 122 | } 123 | } 124 | 125 | func NewDeadletterProducerMetricsService() *ProducerMetricsService { 126 | return &ProducerMetricsService{ 127 | deadletterRecordSendCounter: getPrometheusDeadletterRecordSendInstrumentation(), 128 | recordSendLatency: getPrometheusRecordSendLatencyInstrumentation(), 129 | errorCounter: getPrometheusSendErrorInstrumentation(), 130 | } 131 | } 132 | 133 | // Instrumentation is a middleware that provides metrics for the producer. 134 | func (p *ProducerMetricsService) Instrumentation(next ProducerHandler) ProducerHandler { 135 | return func(producer *producer, msg *sarama.ProducerMessage) (err error) { 136 | defer func(begin time.Time) { 137 | p.recordSendLatency.WithLabelValues(msg.Topic).Observe(time.Since(begin).Seconds()) 138 | }(time.Now()) 139 | 140 | err = next(producer, msg) 141 | 142 | if err != nil { 143 | p.errorCounter.WithLabelValues(msg.Topic).Inc() 144 | } else { 145 | p.recordSendCounter.WithLabelValues(msg.Topic).Inc() 146 | } 147 | return 148 | } 149 | } 150 | 151 | // DeadletterInstrumentation is a middleware that provides metrics for a deadletter producer. 152 | func (p *ProducerMetricsService) DeadletterInstrumentation(next ProducerHandler) ProducerHandler { 153 | return func(producer *producer, msg *sarama.ProducerMessage) (err error) { 154 | defer func(begin time.Time) { 155 | p.recordSendLatency.WithLabelValues(msg.Topic).Observe(time.Since(begin).Seconds()) 156 | }(time.Now()) 157 | 158 | err = next(producer, msg) 159 | 160 | if err != nil { 161 | p.errorCounter.WithLabelValues(msg.Topic).Inc() 162 | } else { 163 | p.deadletterRecordSendCounter.WithLabelValues(msg.Topic).Inc() 164 | } 165 | return 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /producer_test.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/IBM/sarama" 8 | "github.com/ricardo-ch/go-kafka/v3/mocks" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/mock" 11 | ) 12 | 13 | func Test_Producer_SyncProducer_Error(t *testing.T) { 14 | mockProducer := &mocks.SyncProducer{} 15 | mockProducer.On("SendMessage", mock.Anything).Return(int32(0), int64(0), errors.New("error")) 16 | 17 | p := &producer{ 18 | producer: mockProducer, 19 | handler: produce, 20 | } 21 | 22 | err := p.Produce(&sarama.ProducerMessage{}) 23 | assert.NotNil(t, err) 24 | mockProducer.AssertExpectations(t) 25 | } 26 | 27 | func Test_Producer_SyncProducer_OK(t *testing.T) { 28 | mockProducer := &mocks.SyncProducer{} 29 | mockProducer.On("SendMessage", mock.Anything).Return(int32(0), int64(0), nil) 30 | 31 | p := &producer{ 32 | producer: mockProducer, 33 | handler: produce, 34 | } 35 | 36 | err := p.Produce(&sarama.ProducerMessage{}) 37 | assert.Nil(t, err) 38 | mockProducer.AssertExpectations(t) 39 | } 40 | --------------------------------------------------------------------------------