├── LICENSE ├── README.md ├── consumer ├── consume_batch_handler.go ├── consume_multi_async_handler.go ├── consume_multi_batch_handler.go ├── consume_sync_handler.go ├── consumer.go └── consumer_test.go ├── go.mod ├── go.sum ├── main.go └── producer └── producer.go /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 sceneryback 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kafka-best-practices 2 | 3 | This repo discusses some techniques of consuming kafka (Sync, Batch, MultiAsync and MultiBatch) and try to demonstrate some best practices which I think would be generally useful to consume data efficiently. 4 | 5 | * Sync 6 | 7 | consume messages one by one 8 | 9 | * Batch 10 | 11 | consume messages batch by batch 12 | 13 | * MultiAsync 14 | 15 | the "Fan In / Fan Out" pattern 16 | 17 | * MultiBatch 18 | 19 | the "Fan In / Fan Out" pattern batch by batch 20 | 21 | -------------------------------------------------------------------------------- /consumer/consume_batch_handler.go: -------------------------------------------------------------------------------- 1 | package consumer 2 | 3 | import ( 4 | "github.com/Shopify/sarama" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | // ----- batch handler 10 | 11 | type BatchConsumerConfig struct { 12 | BufferCapacity int // msg capacity 13 | MaxBufSize int // max message size 14 | TickerIntervalSeconds int 15 | Callback func([]*ConsumerSessionMessage) error 16 | } 17 | 18 | type batchConsumerGroupHandler struct { 19 | cfg *BatchConsumerConfig 20 | 21 | ready chan bool 22 | 23 | // buffer 24 | ticker *time.Ticker 25 | msgBuf []*ConsumerSessionMessage 26 | 27 | // lock to protect buffer operation 28 | mu sync.RWMutex 29 | 30 | // callback 31 | cb func([]*ConsumerSessionMessage) error 32 | } 33 | 34 | func NewBatchConsumerGroupHandler(cfg *BatchConsumerConfig) ConsumerGroupHandler { 35 | handler := batchConsumerGroupHandler{ 36 | ready: make(chan bool, 0), 37 | cb: cfg.Callback, 38 | } 39 | 40 | if cfg.BufferCapacity == 0 { 41 | cfg.BufferCapacity = 10000 42 | } 43 | handler.msgBuf = make([]*ConsumerSessionMessage, 0, cfg.BufferCapacity) 44 | if cfg.MaxBufSize == 0 { 45 | cfg.MaxBufSize = 8000 46 | } 47 | 48 | if cfg.TickerIntervalSeconds == 0 { 49 | cfg.TickerIntervalSeconds = 60 50 | } 51 | handler.cfg = cfg 52 | 53 | handler.ticker = time.NewTicker(time.Duration(cfg.TickerIntervalSeconds) * time.Second) 54 | 55 | return &handler 56 | } 57 | 58 | // Setup is run at the beginning of a new session, before ConsumeClaim 59 | func (h *batchConsumerGroupHandler) Setup(sarama.ConsumerGroupSession) error { 60 | // Mark the consumer as ready 61 | close(h.ready) 62 | return nil 63 | } 64 | 65 | // Cleanup is run at the end of a session, once all ConsumeClaim goroutines have exited 66 | func (h *batchConsumerGroupHandler) Cleanup(sarama.ConsumerGroupSession) error { 67 | return nil 68 | } 69 | 70 | func (h *batchConsumerGroupHandler) WaitReady() { 71 | <-h.ready 72 | return 73 | } 74 | 75 | func (h *batchConsumerGroupHandler) Reset() { 76 | h.ready = make(chan bool, 0) 77 | return 78 | } 79 | 80 | func (h *batchConsumerGroupHandler) flushBuffer() { 81 | if len(h.msgBuf) > 0 { 82 | if err := h.cb(h.msgBuf); err == nil { 83 | h.msgBuf = make([]*ConsumerSessionMessage, 0, h.cfg.BufferCapacity) 84 | } 85 | } 86 | } 87 | 88 | func (h *batchConsumerGroupHandler) insertMessage(msg *ConsumerSessionMessage) { 89 | h.mu.Lock() 90 | defer h.mu.Unlock() 91 | h.msgBuf = append(h.msgBuf, msg) 92 | if len(h.msgBuf) >= h.cfg.MaxBufSize { 93 | h.flushBuffer() 94 | } 95 | } 96 | 97 | func (h *batchConsumerGroupHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { 98 | 99 | // NOTE: 100 | // Do not move the code below to a goroutine. 101 | // The `ConsumeClaim` itself is called within a goroutine, see: 102 | // https://github.com/Shopify/sarama/blob/master/consumer_group.go#L27-L29 103 | claimMsgChan := claim.Messages() 104 | 105 | for { 106 | select { 107 | case message, ok := <-claimMsgChan: 108 | if ok { 109 | h.insertMessage(&ConsumerSessionMessage{ 110 | Message: message, 111 | Session: session, 112 | }) 113 | } else { 114 | return nil 115 | } 116 | case <-h.ticker.C: 117 | h.mu.Lock() 118 | h.flushBuffer() 119 | h.mu.Unlock() 120 | } 121 | } 122 | 123 | return nil 124 | } 125 | 126 | -------------------------------------------------------------------------------- /consumer/consume_multi_async_handler.go: -------------------------------------------------------------------------------- 1 | package consumer 2 | 3 | import ( 4 | "github.com/Shopify/sarama" 5 | ) 6 | 7 | // ----- batch handler 8 | 9 | type MultiAsyncConsumerConfig struct { 10 | BufChan chan *ConsumerSessionMessage 11 | } 12 | 13 | type multiAsyncConsumerGroupHandler struct { 14 | cfg *MultiAsyncConsumerConfig 15 | 16 | ready chan bool 17 | } 18 | 19 | func NewMultiAsyncConsumerGroupHandler(cfg *MultiAsyncConsumerConfig) ConsumerGroupHandler { 20 | handler := multiAsyncConsumerGroupHandler{ 21 | ready: make(chan bool, 0), 22 | } 23 | 24 | handler.cfg = cfg 25 | 26 | return &handler 27 | } 28 | 29 | // Setup is run at the beginning of a new session, before ConsumeClaim 30 | func (h *multiAsyncConsumerGroupHandler) Setup(sarama.ConsumerGroupSession) error { 31 | // Mark the consumer as ready 32 | close(h.ready) 33 | return nil 34 | } 35 | 36 | // Cleanup is run at the end of a session, once all ConsumeClaim goroutines have exited 37 | func (h *multiAsyncConsumerGroupHandler) Cleanup(sarama.ConsumerGroupSession) error { 38 | return nil 39 | } 40 | 41 | func (h *multiAsyncConsumerGroupHandler) WaitReady() { 42 | <-h.ready 43 | return 44 | } 45 | 46 | func (h *multiAsyncConsumerGroupHandler) Reset() { 47 | h.ready = make(chan bool, 0) 48 | return 49 | } 50 | 51 | func (h *multiAsyncConsumerGroupHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { 52 | 53 | // NOTE: 54 | // Do not move the code below to a goroutine. 55 | // The `ConsumeClaim` itself is called within a goroutine, see: 56 | // https://github.com/Shopify/sarama/blob/master/consumer_group.go#L27-L29 57 | claimMsgChan := claim.Messages() 58 | 59 | for message := range claimMsgChan { 60 | h.cfg.BufChan <- &ConsumerSessionMessage{ 61 | Session: session, 62 | Message: message, 63 | } 64 | } 65 | 66 | return nil 67 | } 68 | 69 | -------------------------------------------------------------------------------- /consumer/consume_multi_batch_handler.go: -------------------------------------------------------------------------------- 1 | package consumer 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/Shopify/sarama" 8 | ) 9 | 10 | // ----- batch handler 11 | 12 | type MultiBatchConsumerConfig struct { 13 | BufferCapacity int // msg capacity 14 | MaxBufSize int // max message size 15 | TickerIntervalSeconds int 16 | 17 | BufChan chan batchMessages 18 | } 19 | 20 | type batchMessages []*ConsumerSessionMessage 21 | 22 | type multiBatchConsumerGroupHandler struct { 23 | cfg *MultiBatchConsumerConfig 24 | 25 | ready chan bool 26 | 27 | // buffer 28 | ticker *time.Ticker 29 | msgBuf batchMessages 30 | 31 | // lock to protect buffer operation 32 | mu sync.RWMutex 33 | } 34 | 35 | func NewMultiBatchConsumerGroupHandler(cfg *MultiBatchConsumerConfig) ConsumerGroupHandler { 36 | handler := multiBatchConsumerGroupHandler{ 37 | ready: make(chan bool, 0), 38 | } 39 | 40 | if cfg.BufferCapacity == 0 { 41 | cfg.BufferCapacity = 10000 42 | } 43 | handler.msgBuf = make([]*ConsumerSessionMessage, 0, cfg.BufferCapacity) 44 | if cfg.MaxBufSize == 0 { 45 | cfg.MaxBufSize = 8000 46 | } 47 | 48 | if cfg.TickerIntervalSeconds == 0 { 49 | cfg.TickerIntervalSeconds = 60 50 | } 51 | handler.cfg = cfg 52 | 53 | handler.ticker = time.NewTicker(time.Duration(cfg.TickerIntervalSeconds) * time.Second) 54 | 55 | return &handler 56 | } 57 | 58 | // Setup is run at the beginning of a new session, before ConsumeClaim 59 | func (h *multiBatchConsumerGroupHandler) Setup(sarama.ConsumerGroupSession) error { 60 | // Mark the consumer as ready 61 | close(h.ready) 62 | return nil 63 | } 64 | 65 | // Cleanup is run at the end of a session, once all ConsumeClaim goroutines have exited 66 | func (h *multiBatchConsumerGroupHandler) Cleanup(sarama.ConsumerGroupSession) error { 67 | return nil 68 | } 69 | 70 | func (h *multiBatchConsumerGroupHandler) WaitReady() { 71 | <-h.ready 72 | return 73 | } 74 | 75 | func (h *multiBatchConsumerGroupHandler) Reset() { 76 | h.ready = make(chan bool, 0) 77 | return 78 | } 79 | 80 | func (h *multiBatchConsumerGroupHandler) flushBuffer() { 81 | if len(h.msgBuf) > 0 { 82 | h.cfg.BufChan <- h.msgBuf 83 | h.msgBuf = make([]*ConsumerSessionMessage, 0, h.cfg.BufferCapacity) 84 | } 85 | } 86 | 87 | func (h *multiBatchConsumerGroupHandler) insertMessage(msg *ConsumerSessionMessage) { 88 | h.mu.Lock() 89 | defer h.mu.Unlock() 90 | h.msgBuf = append(h.msgBuf, msg) 91 | if len(h.msgBuf) >= h.cfg.MaxBufSize { 92 | h.flushBuffer() 93 | } 94 | } 95 | 96 | func (h *multiBatchConsumerGroupHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { 97 | 98 | // NOTE: 99 | // Do not move the code below to a goroutine. 100 | // The `ConsumeClaim` itself is called within a goroutine, see: 101 | // https://github.com/Shopify/sarama/blob/master/consumer_group.go#L27-L29 102 | claimMsgChan := claim.Messages() 103 | 104 | for { 105 | select { 106 | case message, ok := <-claimMsgChan: 107 | if ok { 108 | h.insertMessage(&ConsumerSessionMessage{ 109 | Message: message, 110 | Session: session, 111 | }) 112 | } else { 113 | return nil 114 | } 115 | case <-h.ticker.C: 116 | h.mu.Lock() 117 | h.flushBuffer() 118 | h.mu.Unlock() 119 | } 120 | } 121 | 122 | return nil 123 | } 124 | 125 | -------------------------------------------------------------------------------- /consumer/consume_sync_handler.go: -------------------------------------------------------------------------------- 1 | package consumer 2 | 3 | import ( 4 | "github.com/Shopify/sarama" 5 | ) 6 | 7 | type syncConsumerGroupHandler struct { 8 | ready chan bool 9 | 10 | cb func([]byte) error 11 | } 12 | 13 | func NewSyncConsumerGroupHandler(cb func([]byte) error) ConsumerGroupHandler { 14 | handler := syncConsumerGroupHandler{ 15 | ready: make(chan bool, 0), 16 | cb: cb, 17 | } 18 | return &handler 19 | } 20 | 21 | // Setup is run at the beginning of a new session, before ConsumeClaim 22 | func (h *syncConsumerGroupHandler) Setup(sarama.ConsumerGroupSession) error { 23 | // Mark the consumer as ready 24 | close(h.ready) 25 | return nil 26 | } 27 | 28 | // Cleanup is run at the end of a session, once all ConsumeClaim goroutines have exited 29 | func (h *syncConsumerGroupHandler) Cleanup(sarama.ConsumerGroupSession) error { 30 | return nil 31 | } 32 | 33 | func (h *syncConsumerGroupHandler) WaitReady() { 34 | <-h.ready 35 | return 36 | } 37 | 38 | func (h *syncConsumerGroupHandler) Reset() { 39 | h.ready = make(chan bool, 0) 40 | return 41 | } 42 | 43 | func (h *syncConsumerGroupHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { 44 | 45 | // NOTE: 46 | // Do not move the code below to a goroutine. 47 | // The `ConsumeClaim` itself is called within a goroutine, see: 48 | // https://github.com/Shopify/sarama/blob/master/consumer_group.go#L27-L29 49 | claimMsgChan := claim.Messages() 50 | 51 | for message := range claimMsgChan { 52 | if h.cb(message.Value) == nil { 53 | session.MarkMessage(message, "") 54 | } 55 | } 56 | 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /consumer/consumer.go: -------------------------------------------------------------------------------- 1 | package consumer 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "sync/atomic" 8 | "time" 9 | 10 | "github.com/Shopify/sarama" 11 | 12 | "github.com/sceneryback/kafka-best-practices/producer" 13 | ) 14 | 15 | type ConsumerGroupHandler interface { 16 | sarama.ConsumerGroupHandler 17 | WaitReady() 18 | Reset() 19 | } 20 | 21 | type ConsumerGroup struct { 22 | cg sarama.ConsumerGroup 23 | } 24 | 25 | func NewConsumerGroup(broker string, topics []string, group string, handler ConsumerGroupHandler) (*ConsumerGroup, error) { 26 | ctx := context.Background() 27 | cfg := sarama.NewConfig() 28 | cfg.Version = sarama.V0_10_2_0 29 | cfg.Consumer.Offsets.Initial = sarama.OffsetOldest 30 | client, err := sarama.NewConsumerGroup([]string{broker}, group, cfg) 31 | if err != nil { 32 | panic(err) 33 | } 34 | 35 | go func() { 36 | for { 37 | err := client.Consume(ctx, topics, handler) 38 | if err != nil { 39 | if err == sarama.ErrClosedConsumerGroup { 40 | break 41 | } else { 42 | panic(err) 43 | } 44 | } 45 | if ctx.Err() != nil { 46 | return 47 | } 48 | handler.Reset() 49 | } 50 | }() 51 | 52 | handler.WaitReady() // Await till the consumer has been set up 53 | 54 | return &ConsumerGroup{ 55 | cg: client, 56 | }, nil 57 | } 58 | 59 | func (c *ConsumerGroup) Close() error { 60 | return c.cg.Close() 61 | } 62 | 63 | type ConsumerSessionMessage struct { 64 | Session sarama.ConsumerGroupSession 65 | Message *sarama.ConsumerMessage 66 | } 67 | 68 | func decodeMessage(data []byte) error { 69 | var msg producer.Message 70 | err := json.Unmarshal(data, &msg) 71 | if err != nil { 72 | return err 73 | } 74 | return nil 75 | } 76 | 77 | func StartSyncConsumer(broker, topic string) (*ConsumerGroup, error) { 78 | var count int64 79 | var start = time.Now() 80 | handler := NewSyncConsumerGroupHandler(func(data []byte) error { 81 | if err := decodeMessage(data); err != nil { 82 | return err 83 | } 84 | count++ 85 | if count % 5000 == 0 { 86 | fmt.Printf("sync consumer consumed %d messages at speed %.2f/s\n", count, float64(count) / time.Since(start).Seconds()) 87 | } 88 | return nil 89 | }) 90 | consumer, err := NewConsumerGroup(broker, []string{topic}, "sync-consumer-" + fmt.Sprintf("%d", time.Now().Unix()), handler) 91 | if err != nil { 92 | return nil, err 93 | } 94 | return consumer, nil 95 | } 96 | 97 | func StartBatchConsumer(broker, topic string) (*ConsumerGroup, error) { 98 | var count int64 99 | var start = time.Now() 100 | handler := NewBatchConsumerGroupHandler(&BatchConsumerConfig{ 101 | MaxBufSize: 1000, 102 | Callback: func(messages []*ConsumerSessionMessage) error { 103 | for i := range messages { 104 | if err := decodeMessage(messages[i].Message.Value); err == nil { 105 | messages[i].Session.MarkMessage(messages[i].Message, "") 106 | } 107 | } 108 | count += int64(len(messages)) 109 | if count % 5000 == 0 { 110 | fmt.Printf("batch consumer consumed %d messages at speed %.2f/s\n", count, float64(count) / time.Since(start).Seconds()) 111 | } 112 | return nil 113 | }, 114 | }) 115 | consumer, err := NewConsumerGroup(broker, []string{topic}, "batch-consumer-" + fmt.Sprintf("%d", time.Now().Unix()), handler) 116 | if err != nil { 117 | return nil, err 118 | } 119 | return consumer, nil 120 | } 121 | 122 | func StartMultiAsyncConsumer(broker, topic string) (*ConsumerGroup, error) { 123 | var count int64 124 | var start = time.Now() 125 | var bufChan = make(chan *ConsumerSessionMessage, 1000) 126 | for i := 0; i < 8; i++ { 127 | go func() { 128 | for message := range bufChan { 129 | if err := decodeMessage(message.Message.Value); err == nil { 130 | message.Session.MarkMessage(message.Message, "") 131 | } 132 | cur := atomic.AddInt64(&count, 1) 133 | if cur % 5000 == 0 { 134 | fmt.Printf("multi async consumer consumed %d messages at speed %.2f/s\n", cur, float64(cur) / time.Since(start).Seconds()) 135 | } 136 | } 137 | }() 138 | } 139 | handler := NewMultiAsyncConsumerGroupHandler(&MultiAsyncConsumerConfig{ 140 | BufChan: bufChan, 141 | }) 142 | consumer, err := NewConsumerGroup(broker, []string{topic}, "multi-async-consumer-" + fmt.Sprintf("%d", time.Now().Unix()), handler) 143 | if err != nil { 144 | return nil, err 145 | } 146 | return consumer, nil 147 | } 148 | 149 | func StartMultiBatchConsumer(broker, topic string) (*ConsumerGroup, error) { 150 | var count int64 151 | var start = time.Now() 152 | var bufChan = make(chan batchMessages, 1000) 153 | for i := 0; i < 8; i++ { 154 | go func() { 155 | for messages := range bufChan { 156 | for j := range messages { 157 | if err := decodeMessage(messages[j].Message.Value); err == nil { 158 | messages[j].Session.MarkMessage(messages[j].Message, "") 159 | } 160 | } 161 | cur := atomic.AddInt64(&count, int64(len(messages))) 162 | if cur % 1000 == 0 { 163 | fmt.Printf("multi batch consumer consumed %d messages at speed %.2f/s\n", cur, float64(cur) / time.Since(start).Seconds()) 164 | } 165 | } 166 | }() 167 | } 168 | handler := NewMultiBatchConsumerGroupHandler(&MultiBatchConsumerConfig{ 169 | MaxBufSize: 1000, 170 | BufChan: bufChan, 171 | }) 172 | consumer, err := NewConsumerGroup(broker, []string{topic}, "multi-batch-consumer-" + fmt.Sprintf("%d", time.Now().Unix()), handler) 173 | if err != nil { 174 | return nil, err 175 | } 176 | return consumer, nil 177 | } 178 | -------------------------------------------------------------------------------- /consumer/consumer_test.go: -------------------------------------------------------------------------------- 1 | package consumer 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | "github.com/Shopify/sarama" 10 | "github.com/stretchr/testify/assert" 11 | 12 | "github.com/sceneryback/kafka-best-practices/producer" 13 | ) 14 | 15 | var ( 16 | testTopicPrefix = "test-topic" 17 | testBroker = "127.0.0.1:9092" 18 | ) 19 | 20 | func testProduce(topic string, limit int) <-chan struct{} { 21 | var produceDone = make(chan struct{}) 22 | 23 | p, err := sarama.NewAsyncProducer([]string{"127.0.0.1:9092"}, sarama.NewConfig()) 24 | if err != nil { 25 | return nil 26 | } 27 | 28 | go func() { 29 | defer close(produceDone) 30 | 31 | for i := 0; i < limit; i++ { 32 | msg := producer.Message{i} 33 | msgBytes, err := json.Marshal(msg) 34 | if err != nil { 35 | continue 36 | } 37 | select { 38 | case p.Input() <- &sarama.ProducerMessage{ 39 | Topic: topic, 40 | Value: sarama.ByteEncoder(msgBytes), 41 | }: 42 | case err := <-p.Errors(): 43 | fmt.Printf("Failed to send message to kafka, err: %s, msg: %s\n", err, msgBytes) 44 | } 45 | } 46 | }() 47 | 48 | return produceDone 49 | } 50 | 51 | func TestSyncConsumer(t *testing.T) { 52 | limit := 500000 53 | 54 | topic := testTopicPrefix + fmt.Sprintf("%d", time.Now().Unix()) 55 | 56 | produceDone := testProduce(topic, limit) 57 | 58 | var consumeMsgMap = make(map[int]struct{}) 59 | var resChan = make(chan int) 60 | go func() { 61 | for r := range resChan { 62 | consumeMsgMap[r] = struct{}{} 63 | } 64 | }() 65 | 66 | handler := NewSyncConsumerGroupHandler(func(data []byte) error { 67 | var msg producer.Message 68 | err := json.Unmarshal(data, &msg) 69 | if err != nil { 70 | return err 71 | } 72 | resChan <- msg.Id 73 | return nil 74 | }) 75 | consumer, err := NewConsumerGroup(testBroker, []string{topic}, "sync-consumer-" + fmt.Sprintf("%d", time.Now().Unix()), handler) 76 | if err != nil { 77 | return 78 | } 79 | defer consumer.Close() 80 | 81 | <-produceDone 82 | 83 | time.Sleep(1*time.Second) 84 | 85 | assert.Equal(t, limit, len(consumeMsgMap)) 86 | } 87 | 88 | func TestBatchConsumer(t *testing.T) { 89 | limit := 500000 90 | 91 | topic := testTopicPrefix + fmt.Sprintf("%d", time.Now().Unix()) 92 | 93 | produceDone := testProduce(topic, limit) 94 | 95 | var consumeMsgMap = make(map[int]struct{}) 96 | var resChan = make(chan int) 97 | go func() { 98 | for r := range resChan { 99 | consumeMsgMap[r] = struct{}{} 100 | } 101 | }() 102 | 103 | handler := NewBatchConsumerGroupHandler(&BatchConsumerConfig{ 104 | MaxBufSize: 1000, 105 | Callback: func(messages []*ConsumerSessionMessage) error { 106 | for i := range messages { 107 | var msg producer.Message 108 | err := json.Unmarshal(messages[i].Message.Value, &msg) 109 | if err != nil { 110 | return err 111 | } 112 | resChan <- msg.Id 113 | } 114 | return nil 115 | }, 116 | }) 117 | consumer, err := NewConsumerGroup(testBroker, []string{topic}, "batch-consumer-" + fmt.Sprintf("%d", time.Now().Unix()), handler) 118 | if err != nil { 119 | return 120 | } 121 | defer consumer.Close() 122 | 123 | <-produceDone 124 | 125 | time.Sleep(1*time.Second) 126 | 127 | assert.Equal(t, limit, len(consumeMsgMap)) 128 | } 129 | 130 | func TestMultiAsyncConsumer(t *testing.T) { 131 | limit := 500000 132 | 133 | topic := testTopicPrefix + fmt.Sprintf("%d", time.Now().Unix()) 134 | 135 | produceDone := testProduce(topic, limit) 136 | 137 | var consumeMsgMap = make(map[int]struct{}) 138 | var resChan = make(chan int) 139 | go func() { 140 | for r := range resChan { 141 | consumeMsgMap[r] = struct{}{} 142 | } 143 | }() 144 | 145 | var bufChan = make(chan *ConsumerSessionMessage, 1000) 146 | for i := 0; i < 8; i++ { 147 | go func() { 148 | for message := range bufChan { 149 | var msg producer.Message 150 | err := json.Unmarshal(message.Message.Value, &msg) 151 | if err != nil { 152 | continue 153 | } 154 | resChan <- msg.Id 155 | } 156 | }() 157 | } 158 | handler := NewMultiAsyncConsumerGroupHandler(&MultiAsyncConsumerConfig{ 159 | BufChan: bufChan, 160 | }) 161 | consumer, err := NewConsumerGroup(testBroker, []string{topic}, "multi-async-consumer-" + fmt.Sprintf("%d", time.Now().Unix()), handler) 162 | if err != nil { 163 | return 164 | } 165 | defer consumer.Close() 166 | 167 | <-produceDone 168 | 169 | time.Sleep(1*time.Second) 170 | 171 | assert.Equal(t, limit, len(consumeMsgMap)) 172 | } 173 | 174 | func TestMultiBatchConsumer(t *testing.T) { 175 | limit := 500000 176 | 177 | topic := testTopicPrefix + fmt.Sprintf("%d", time.Now().Unix()) 178 | 179 | produceDone := testProduce(topic, limit) 180 | 181 | var consumeMsgMap = make(map[int]struct{}) 182 | var resChan = make(chan int) 183 | go func() { 184 | for r := range resChan { 185 | consumeMsgMap[r] = struct{}{} 186 | } 187 | }() 188 | 189 | var bufChan = make(chan batchMessages, 1000) 190 | for i := 0; i < 8; i++ { 191 | go func() { 192 | for messages := range bufChan { 193 | for j := range messages { 194 | var msg producer.Message 195 | err := json.Unmarshal(messages[j].Message.Value, &msg) 196 | if err != nil { 197 | continue 198 | } 199 | resChan <- msg.Id 200 | } 201 | } 202 | }() 203 | } 204 | handler := NewMultiBatchConsumerGroupHandler(&MultiBatchConsumerConfig{ 205 | MaxBufSize: 1000, 206 | BufChan: bufChan, 207 | TickerIntervalSeconds: 1, 208 | }) 209 | consumer, err := NewConsumerGroup(testBroker, []string{topic}, "multi-batch-consumer-" + fmt.Sprintf("%d", time.Now().Unix()), handler) 210 | if err != nil { 211 | return 212 | } 213 | defer consumer.Close() 214 | 215 | <-produceDone 216 | 217 | time.Sleep(1*time.Second) 218 | 219 | assert.Equal(t, limit, len(consumeMsgMap)) 220 | } 221 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/sceneryback/kafka-best-practices 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/Shopify/sarama v1.26.4 7 | github.com/stretchr/testify v1.4.0 8 | golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375 // indirect 9 | ) 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Shopify/sarama v1.26.4 h1:+17TxUq/PJEAfZAll0T7XJjSgQWCpaQSoki/x5yN8o8= 2 | github.com/Shopify/sarama v1.26.4/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= 3 | github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= 4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 6 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= 8 | github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= 9 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= 10 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 11 | github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= 12 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 13 | github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= 14 | github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= 15 | github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= 16 | github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= 17 | github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= 18 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 19 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 20 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 21 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 22 | github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= 23 | github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 24 | github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= 25 | github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= 26 | github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA= 27 | github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 28 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 29 | github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= 30 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 31 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 32 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 33 | github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= 34 | github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 35 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 36 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 37 | github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= 38 | github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 39 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 40 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 41 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 42 | github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= 43 | github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= 44 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 45 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 46 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 47 | golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 h1:+ELyKg6m8UBf0nPFSqD0mi7zUfwPyXo23HNjMnXPz7w= 48 | golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 49 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 50 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 51 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 52 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= 53 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 54 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= 55 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 56 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 57 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 58 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 59 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 60 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 61 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 62 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e h1:FDhOuMEY4JVRztM/gsbk+IKUQ8kj74bxZrgw87eMMVc= 63 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 64 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 65 | golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375 h1:SjQ2+AKWgZLc1xej6WSzL+Dfs5Uyd5xcZH1mGC411IA= 66 | golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 67 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 68 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 69 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 70 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 71 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 72 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 73 | gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= 74 | gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= 75 | gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= 76 | gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= 77 | gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= 78 | gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg= 79 | gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= 80 | gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= 81 | gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= 82 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 83 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 84 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 85 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | 10 | "github.com/sceneryback/kafka-best-practices/consumer" 11 | "github.com/sceneryback/kafka-best-practices/producer" 12 | ) 13 | 14 | const ( 15 | ProduceMode = "produce" 16 | SyncMode = "sync" 17 | BatchMode = "batch" 18 | MultiAsyncMode = "multiAsync" 19 | MultiBatchMode = "multiBatch" 20 | ) 21 | 22 | var mode string 23 | var broker string 24 | 25 | func init() { 26 | flag.StringVar(&mode, "m", "", "cmd mode, 'produce', 'sync', 'batch' or 'multiBatch'") 27 | flag.StringVar(&broker, "h", "127.0.0.1:9092", "kafka broker host:port") 28 | } 29 | 30 | func main() { 31 | flag.Parse() 32 | flag.Usage = func() { 33 | fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) 34 | flag.PrintDefaults() 35 | } 36 | if mode == "" { 37 | flag.Usage() 38 | return 39 | } 40 | 41 | var topic = "test-practice-topic" 42 | 43 | var done = make(chan struct{}) 44 | defer close(done) 45 | 46 | switch mode { 47 | case ProduceMode: 48 | producer, err := producer.NewProducer(broker) 49 | if err != nil { 50 | panic(err) 51 | } 52 | defer producer.Close() 53 | go producer.StartProduce(done, topic) 54 | case SyncMode: 55 | // 1. sync consumer 56 | consumer, err := consumer.StartSyncConsumer(broker, topic) 57 | if err != nil { 58 | panic(err) 59 | } 60 | defer consumer.Close() 61 | case BatchMode: 62 | // 2. batch consumer 63 | consumer, err := consumer.StartBatchConsumer(broker, topic) 64 | if err != nil { 65 | panic(err) 66 | } 67 | defer consumer.Close() 68 | case MultiAsyncMode: 69 | // 3. multi async consumer 70 | consumer, err := consumer.StartMultiAsyncConsumer(broker, topic) 71 | if err != nil { 72 | panic(err) 73 | } 74 | defer consumer.Close() 75 | case MultiBatchMode: 76 | // 4. multi batch consumer 77 | consumer, err := consumer.StartMultiBatchConsumer(broker, topic) 78 | if err != nil { 79 | panic(err) 80 | } 81 | defer consumer.Close() 82 | default: 83 | flag.Usage() 84 | return 85 | } 86 | 87 | c := make(chan os.Signal, 1) 88 | signal.Notify(c, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) 89 | fmt.Println("received signal", <-c) 90 | } 91 | 92 | 93 | -------------------------------------------------------------------------------- /producer/producer.go: -------------------------------------------------------------------------------- 1 | package producer 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/Shopify/sarama" 9 | ) 10 | 11 | type Producer struct { 12 | p sarama.AsyncProducer 13 | } 14 | 15 | func NewProducer(broker string) (*Producer, error) { 16 | producer, err := sarama.NewAsyncProducer([]string{broker}, sarama.NewConfig()) 17 | if err != nil { 18 | return nil, err 19 | } 20 | return &Producer{ 21 | p: producer, 22 | }, nil 23 | } 24 | 25 | type Message struct { 26 | Id int `json:"id"` 27 | } 28 | 29 | func (p *Producer) StartProduce(done chan struct{}, topic string) { 30 | start := time.Now() 31 | for i := 0; ; i++ { 32 | msg := Message{i} 33 | msgBytes, err := json.Marshal(msg) 34 | if err != nil { 35 | continue 36 | } 37 | select { 38 | case <-done: 39 | return 40 | case p.p.Input() <- &sarama.ProducerMessage{ 41 | Topic: topic, 42 | Value: sarama.ByteEncoder(msgBytes), 43 | }: 44 | if i % 5000 == 0 { 45 | fmt.Printf("produced %d messages with speed %.2f/s\n", i, float64(i) / time.Since(start).Seconds()) 46 | } 47 | case err := <-p.p.Errors(): 48 | fmt.Printf("Failed to send message to kafka, err: %s, msg: %s\n", err, msgBytes) 49 | } 50 | } 51 | } 52 | 53 | func (p *Producer) Close() error { 54 | if p != nil { 55 | return p.p.Close() 56 | } 57 | return nil 58 | } 59 | --------------------------------------------------------------------------------