├── README.MD ├── cmd ├── consumer │ └── main.go └── publiser │ └── main.go ├── consumer ├── consumer.go └── consumer_test.go ├── docker-compose.yml ├── go.mod ├── go.sum └── producer ├── producer.go └── producer_test.go /README.MD: -------------------------------------------------------------------------------- 1 | # GO Kafka Example 2 | 3 | This is repository how to use kafka in golang, with shopify sarama 4 | 5 | ## Tutorial 6 | 7 | You can see tutorial to make this repository in [here](https://medium.com/easyread/implementasi-kafka-menggunakan-golang-testing-db183e0b3c29) 8 | 9 | ## How to run 10 | 11 | ```bash 12 | # clone this repo 13 | git clone git@github.com:mufti1/kafka-example.git 14 | # or 15 | git clone https://github.com/mufti1/kafka-example.git 16 | 17 | # Add kafka to your hosts 18 | sudo nano /etc/hosts 19 | 127.0.0.1 kafka 20 | 21 | cd kafka-example 22 | 23 | # running docker compose 24 | docker-compose up 25 | 26 | # run consumer with command 27 | go run cmd/consumer/main.go 28 | 29 | # run publisher with command 30 | go run cmd/publiser/main.go 31 | ``` 32 | -------------------------------------------------------------------------------- /cmd/consumer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "time" 6 | 7 | "github.com/mufti1/kafka-example/consumer" 8 | 9 | "github.com/Shopify/sarama" 10 | "github.com/sirupsen/logrus" 11 | ) 12 | 13 | func main() { 14 | // Setup Logging 15 | customFormatter := new(logrus.TextFormatter) 16 | customFormatter.TimestampFormat = "2006-01-02 15:04:05" 17 | customFormatter.FullTimestamp = true 18 | logrus.SetFormatter(customFormatter) 19 | 20 | kafkaConfig := getKafkaConfig("", "") 21 | 22 | consumers, err := sarama.NewConsumer([]string{"kafka:9092"}, kafkaConfig) 23 | if err != nil { 24 | logrus.Errorf("Error create kakfa consumer got error %v", err) 25 | } 26 | defer func() { 27 | if err := consumers.Close(); err != nil { 28 | logrus.Fatal(err) 29 | return 30 | } 31 | }() 32 | 33 | kafkaConsumer := &consumer.KafkaConsumer{ 34 | Consumer: consumers, 35 | } 36 | 37 | signals := make(chan os.Signal, 1) 38 | kafkaConsumer.Consume([]string{"test_topic"}, signals) 39 | } 40 | 41 | func getKafkaConfig(username, password string) *sarama.Config { 42 | kafkaConfig := sarama.NewConfig() 43 | kafkaConfig.Producer.Return.Successes = true 44 | kafkaConfig.Net.WriteTimeout = 5 * time.Second 45 | kafkaConfig.Producer.Retry.Max = 0 46 | 47 | if username != "" { 48 | kafkaConfig.Net.SASL.Enable = true 49 | kafkaConfig.Net.SASL.User = username 50 | kafkaConfig.Net.SASL.Password = password 51 | } 52 | return kafkaConfig 53 | } 54 | -------------------------------------------------------------------------------- /cmd/publiser/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/Shopify/sarama" 8 | "github.com/mufti1/kafka-example/producer" 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | func main() { 13 | // Setup Logging 14 | customFormatter := new(logrus.TextFormatter) 15 | customFormatter.TimestampFormat = "2006-01-02 15:04:05" 16 | customFormatter.FullTimestamp = true 17 | logrus.SetFormatter(customFormatter) 18 | 19 | kafkaConfig := getKafkaConfig("", "") 20 | producers, err := sarama.NewSyncProducer([]string{"kafka:9092"}, kafkaConfig) 21 | if err != nil { 22 | logrus.Errorf("Unable to create kafka producer got error %v", err) 23 | return 24 | } 25 | defer func() { 26 | if err := producers.Close(); err != nil { 27 | logrus.Errorf("Unable to stop kafka producer: %v", err) 28 | return 29 | } 30 | }() 31 | 32 | logrus.Infof("Success create kafka sync-producer") 33 | 34 | kafka := &producer.KafkaProducer{ 35 | Producer: producers, 36 | } 37 | 38 | for i := 1; i <= 10; i++ { 39 | msg := fmt.Sprintf("message number %v", i) 40 | err := kafka.SendMessage("test_topic", msg) 41 | if err != nil { 42 | panic(err) 43 | } 44 | } 45 | } 46 | 47 | func getKafkaConfig(username, password string) *sarama.Config { 48 | kafkaConfig := sarama.NewConfig() 49 | kafkaConfig.Producer.Return.Successes = true 50 | kafkaConfig.Net.WriteTimeout = 5 * time.Second 51 | kafkaConfig.Producer.Retry.Max = 0 52 | 53 | if username != "" { 54 | kafkaConfig.Net.SASL.Enable = true 55 | kafkaConfig.Net.SASL.User = username 56 | kafkaConfig.Net.SASL.Password = password 57 | } 58 | return kafkaConfig 59 | } 60 | -------------------------------------------------------------------------------- /consumer/consumer.go: -------------------------------------------------------------------------------- 1 | package consumer 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/Shopify/sarama" 7 | "github.com/sirupsen/logrus" 8 | ) 9 | 10 | // KafkaConsumer hold sarama consumer 11 | type KafkaConsumer struct { 12 | Consumer sarama.Consumer 13 | } 14 | 15 | // Consume function to consume message from apache kafka 16 | func (c *KafkaConsumer) Consume(topics []string, signals chan os.Signal) { 17 | chanMessage := make(chan *sarama.ConsumerMessage, 256) 18 | 19 | for _, topic := range topics { 20 | partitionList, err := c.Consumer.Partitions(topic) 21 | if err != nil { 22 | logrus.Errorf("Unable to get partition got error %v", err) 23 | continue 24 | } 25 | for _, partition := range partitionList { 26 | go consumeMessage(c.Consumer, topic, partition, chanMessage) 27 | } 28 | } 29 | logrus.Infof("Kafka is consuming....") 30 | 31 | ConsumerLoop: 32 | for { 33 | select { 34 | case msg := <-chanMessage: 35 | logrus.Infof("New Message from kafka, message: %v", string(msg.Value)) 36 | case sig := <-signals: 37 | if sig == os.Interrupt { 38 | break ConsumerLoop 39 | } 40 | } 41 | } 42 | } 43 | 44 | func consumeMessage(consumer sarama.Consumer, topic string, partition int32, c chan *sarama.ConsumerMessage) { 45 | msg, err := consumer.ConsumePartition(topic, partition, sarama.OffsetNewest) 46 | if err != nil { 47 | logrus.Errorf("Unable to consume partition %v got error %v", partition, err) 48 | return 49 | } 50 | 51 | defer func() { 52 | if err := msg.Close(); err != nil { 53 | logrus.Errorf("Unable to close partition %v: %v", partition, err) 54 | } 55 | }() 56 | 57 | for { 58 | msg := <-msg.Messages() 59 | c <- msg 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /consumer/consumer_test.go: -------------------------------------------------------------------------------- 1 | package consumer_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | "time" 7 | 8 | "github.com/Shopify/sarama" 9 | "github.com/Shopify/sarama/mocks" 10 | "github.com/mufti1/kafka-example/consumer" 11 | ) 12 | 13 | func TestConsume(t *testing.T) { 14 | consumers := mocks.NewConsumer(t, nil) 15 | defer func() { 16 | if err := consumers.Close(); err != nil { 17 | t.Error(err) 18 | } 19 | }() 20 | 21 | consumers.SetTopicMetadata(map[string][]int32{ 22 | "test_topic": {0}, 23 | }) 24 | 25 | kafka := &consumer.KafkaConsumer{ 26 | Consumer: consumers, 27 | } 28 | 29 | consumers.ExpectConsumePartition("test_topic", 0, sarama.OffsetNewest).YieldMessage(&sarama.ConsumerMessage{Value: []byte("hello world")}) 30 | 31 | signals := make(chan os.Signal, 1) 32 | go kafka.Consume([]string{"test_topic"}, signals) 33 | timeout := time.After(2 * time.Second) 34 | for { 35 | select { 36 | case <-timeout: 37 | signals <- os.Interrupt 38 | return 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | services: 4 | kafka: 5 | image: confluentinc/cp-kafka:4.1.0 6 | container_name: kafka 7 | environment: 8 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 9 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 10 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 11 | ports: 12 | - 9092:9092 13 | networks: 14 | - kafka 15 | 16 | zookeeper: 17 | image: confluentinc/cp-zookeeper:4.1.0 18 | container_name: zookeeper 19 | environment: 20 | ZOOKEEPER_CLIENT_PORT: 2181 21 | networks: 22 | - kafka 23 | 24 | networks: 25 | kafka: -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mufti1/kafka-example 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/Shopify/sarama v1.21.0 7 | github.com/sirupsen/logrus v1.3.0 8 | ) 9 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/DataDog/zstd v1.3.5 h1:DtpNbljikUepEPD16hD4LvIcmhnhdLTiW/5pHgbmp14= 2 | github.com/DataDog/zstd v1.3.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= 3 | github.com/Shopify/sarama v1.21.0 h1:0GKs+e8mn1RRUzfg9oUXv3v7ZieQLmOZF/bfnmmGhM8= 4 | github.com/Shopify/sarama v1.21.0/go.mod h1:yuqtN/pe8cXRWG5zPaO7hCfNJp5MwmkoJEoLjkm5tCQ= 5 | github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU= 9 | github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= 10 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= 11 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 12 | github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= 13 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 14 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= 15 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 16 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 17 | github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= 18 | github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 19 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 20 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= 21 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 22 | github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= 23 | github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 24 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 25 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 26 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= 27 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 28 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= 29 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 30 | -------------------------------------------------------------------------------- /producer/producer.go: -------------------------------------------------------------------------------- 1 | package producer 2 | 3 | import ( 4 | "github.com/Shopify/sarama" 5 | "github.com/sirupsen/logrus" 6 | ) 7 | 8 | // KafkaProducer hold kafka producer session 9 | type KafkaProducer struct { 10 | Producer sarama.SyncProducer 11 | } 12 | 13 | // SendMessage function to send message into kafka 14 | func (p *KafkaProducer) SendMessage(topic, msg string) error { 15 | 16 | kafkaMsg := &sarama.ProducerMessage{ 17 | Topic: topic, 18 | Value: sarama.StringEncoder(msg), 19 | } 20 | 21 | partition, offset, err := p.Producer.SendMessage(kafkaMsg) 22 | if err != nil { 23 | logrus.Errorf("Send message error: %v", err) 24 | return err 25 | } 26 | 27 | logrus.Infof("Send message success, Topic %v, Partition %v, Offset %d", topic, partition, offset) 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /producer/producer_test.go: -------------------------------------------------------------------------------- 1 | package producer_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/Shopify/sarama/mocks" 8 | "github.com/mufti1/kafka-example/producer" 9 | ) 10 | 11 | func TestSendMessage(t *testing.T) { 12 | t.Run("Send message OK", func(t *testing.T) { 13 | mockedProducer := mocks.NewSyncProducer(t, nil) 14 | mockedProducer.ExpectSendMessageAndSucceed() 15 | kafka := &producer.KafkaProducer{ 16 | Producer: mockedProducer, 17 | } 18 | 19 | msg := "Message 1" 20 | 21 | err := kafka.SendMessage("test_topic", msg) 22 | if err != nil { 23 | t.Errorf("Send message should not be error but have: %v", err) 24 | } 25 | }) 26 | 27 | t.Run("Send message NOK", func(t *testing.T) { 28 | mockedProducer := mocks.NewSyncProducer(t, nil) 29 | mockedProducer.ExpectSendMessageAndFail(fmt.Errorf("Error")) 30 | kafka := &producer.KafkaProducer{ 31 | Producer: mockedProducer, 32 | } 33 | 34 | msg := "Message 1" 35 | 36 | err := kafka.SendMessage("test_topic", msg) 37 | if err == nil { 38 | t.Error("this should be error") 39 | } 40 | }) 41 | } 42 | --------------------------------------------------------------------------------