├── README.md ├── go.mod ├── go.sum └── main.go /README.md: -------------------------------------------------------------------------------- 1 | ## Kafka Golang Example 2 | 3 | Example code for my [blog post](https://sohamkamani.com/golang/working-with-kafka/). 4 | 5 | Steps to run: 6 | 7 | 1. Clone this repository 8 | 2. Install dependencies : `go mod tidy && go mod vendor` 9 | 3. Make sure [Kafka is running](https://www.sohamkamani.com/blog/2017/11/22/how-to-install-and-run-kafka/), and change the value of `brokerAddress` to the address of you Kafka instance 10 | 4. Run the code : `go run main.go` 11 | 12 | > You can see how to install and run a Kafka cluster in my [other tutorial](https://www.sohamkamani.com/blog/2017/11/22/how-to-install-and-run-kafka/) 13 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/sohamkamani/golang-kafka-example 2 | 3 | go 1.14 4 | 5 | require github.com/segmentio/kafka-go v0.4.2 // indirect 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 2 | github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= 3 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 4 | github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA= 5 | github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 6 | github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= 7 | github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 8 | github.com/segmentio/kafka-go v0.4.2 h1:QXZ6q9Bu1JkAJQ/CQBb2Av8pFRG8LQ0kWCrLXgQyL8c= 9 | github.com/segmentio/kafka-go v0.4.2/go.mod h1:Inh7PqOsxmfgasV8InZYKVXWsdjcCq2d9tFV75GLbuM= 10 | github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= 11 | github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= 12 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 13 | golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 14 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 15 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 16 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 17 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 18 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "os" 8 | "strconv" 9 | "time" 10 | 11 | "github.com/segmentio/kafka-go" 12 | ) 13 | 14 | const ( 15 | topic = "message-log" 16 | brokerAddress = "localhost:9092" 17 | ) 18 | 19 | func main() { 20 | // create a new context 21 | ctx := context.Background() 22 | // produce messages in a new go routine, since 23 | // both the produce and consume functions are 24 | // blocking 25 | go produce(ctx) 26 | consume(ctx) 27 | } 28 | 29 | func produce(ctx context.Context) { 30 | // initialize a counter 31 | i := 0 32 | 33 | l := log.New(os.Stdout, "kafka writer: ", 0) 34 | // intialize the writer with the broker addresses, and the topic 35 | w := kafka.NewWriter(kafka.WriterConfig{ 36 | Brokers: []string{brokerAddress}, 37 | Topic: topic, 38 | // assign the logger to the writer 39 | Logger: l, 40 | }) 41 | 42 | for { 43 | // each kafka message has a key and value. The key is used 44 | // to decide which partition (and consequently, which broker) 45 | // the message gets published on 46 | err := w.WriteMessages(ctx, kafka.Message{ 47 | Key: []byte(strconv.Itoa(i)), 48 | // create an arbitrary message payload for the value 49 | Value: []byte("this is message" + strconv.Itoa(i)), 50 | }) 51 | if err != nil { 52 | panic("could not write message " + err.Error()) 53 | } 54 | 55 | // log a confirmation once the message is written 56 | fmt.Println("writes:", i) 57 | i++ 58 | // sleep for a second 59 | time.Sleep(time.Second) 60 | } 61 | } 62 | 63 | func consume(ctx context.Context) { 64 | // create a new logger that outputs to stdout 65 | // and has the `kafka reader` prefix 66 | l := log.New(os.Stdout, "kafka reader: ", 0) 67 | // initialize a new reader with the brokers and topic 68 | // the groupID identifies the consumer and prevents 69 | // it from receiving duplicate messages 70 | r := kafka.NewReader(kafka.ReaderConfig{ 71 | Brokers: []string{brokerAddress}, 72 | Topic: topic, 73 | GroupID: "my-group", 74 | // assign the logger to the reader 75 | Logger: l, 76 | }) 77 | for { 78 | // the `ReadMessage` method blocks until we receive the next event 79 | msg, err := r.ReadMessage(ctx) 80 | if err != nil { 81 | panic("could not read message " + err.Error()) 82 | } 83 | // after receiving the message, log its value 84 | fmt.Println("received: ", string(msg.Value)) 85 | } 86 | } 87 | --------------------------------------------------------------------------------