├── .github
    └── workflows
    │   └── ci.yml
├── LICENSE
├── Makefile
├── README.md
├── consumer.go
├── consumer_test.go
├── containers
    └── dc.dev.yml
├── doc
    └── seo.do.png
├── go.mod
├── go.sum
├── producer.go
└── producer_test.go


/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
 1 | name: Go
 2 | 
 3 | on:
 4 |   push:
 5 |     branches: [main]
 6 |   pull_request:
 7 |     branches: [main]
 8 | 
 9 | jobs:
10 |   build:
11 |     name: Build
12 |     runs-on: ubuntu-latest
13 |     steps:
14 |       - name: Set up Go 1.x
15 |         uses: actions/setup-go@v2
16 |         with:
17 |           go-version: ^1.16
18 | 
19 |       - name: Check out code into the Go module directory
20 |         uses: actions/checkout@v2
21 | 
22 |       - name: Set up Docker Buildx
23 |         id: buildx
24 |         uses: docker/setup-buildx-action@v1
25 | 
26 |       - name: Start redpanda at localhost:9092.
27 |         run: make run-redpanda
28 | 
29 |       - name: Wait for Redpanda.
30 |         run: |
31 |           wget https://github.com/ufoscout/docker-compose-wait/releases/download/2.7.3/wait &&
32 |           chmod u+x wait &&
33 |           export WAIT_HOSTS=localhost:9092 && 
34 |           export WAIT_HOSTS_TIMEOUT=120 && 
35 |           ./wait
36 | 
37 |       - name: Create a Topic
38 |         run: docker exec redpanda-1 rpk topic create kafka_do_test
39 | 
40 |       - name: Test
41 |         run: go test ./... -v -race -count=1 -timeout 6m -cover
42 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2021 SEO DO <hi@seo.do>
 2 | 
 3 | Permission is hereby granted, free of charge, to any person obtaining
 4 | a copy of this software and associated documentation files (the
 5 | "Software"), to deal in the Software without restriction, including
 6 | without limitation the rights to use, copy, modify, merge, publish,
 7 | distribute, sublicense, and/or sell copies of the Software, and to
 8 | permit persons to whom the Software is furnished to do so, subject to
 9 | the following conditions:
10 | 
11 | The above copyright notice and this permission notice shall be
12 | included in all copies or substantial portions of the Software.
13 | 
14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | # REDPANDA
 2 | 
 3 | clean-redpanda:
 4 | 	docker-compose -f containers/dc.dev.yml down --volume
 5 | 
 6 | run-redpanda:
 7 | 	docker-compose -f containers/dc.dev.yml up -d
 8 | 
 9 | logs-redpanda:
10 | 	docker-compose -f containers/dc.dev.yml logs -f
11 | 
12 | # TEST
13 | 
14 | test: 
15 | 	go test ./... -v -race -count=1 -cover -timeout 6m
16 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # kafka-do
 2 | 
 3 | <div align="center">
 4 | 	<div align="right">
 5 | 		<strong><code>v0.3.4</code></strong>
 6 | 	</div>
 7 | 	<img height="100px" src="doc/seo.do.png"><br>
 8 | 	<strong>kafka-do</strong>
 9 | </div>
10 | 
11 | [![Go Reference](https://pkg.go.dev/badge/github.com/wopehq/kafka-do.svg)](https://pkg.go.dev/github.com/wopehq/kafka-do)
12 | 
13 | ## What
14 | 
15 | Higher level abstraction for franz-go. 
16 | 
17 | ## Why
18 | 
19 | We want to be able to write our kafka applications without making the same things over and over.
20 | 
21 | **Batch Consume**  
22 | Consume messages as much as you defined.
23 | 
24 | **Batch Produce**  
25 | Produce messages as a batch to a topic.
26 | 
27 | 
28 | ## Example
29 | 
30 | For e2e example, check [**here**](https://github.com/wopehq/kafka-do-example).
31 | 
32 | ```go
33 | 	producer, err := kafka.NewProducer("127.0.0.1:9092")
34 | 	if err != nil {
35 | 		log.Fatal(err)
36 | 	}
37 | 	defer producer.Close()
38 | 
39 | 	producer.Produce(context.Background(), []kafka.Message{
40 | 		kafka.Message("message 1"),
41 | 		kafka.Message("message 2"),
42 | 		kafka.Message("message 3"),
43 | 		kafka.Message("message 4"),
44 | 	}, "messages")
45 | 
46 | 	consumer, err := kafka.NewConsumer("kafka_do", []string{"messages"}, []string{"127.0.0.1:9092"})
47 | 	if err != nil {
48 | 		log.Fatal(err)
49 | 	}
50 | 	defer consumer.Close()
51 | 
52 | 	messages, errs := consumer.ConsumeBatch(context.Background(), 2)
53 | 	for _, message := range messages {
54 | 		log.Println(message)
55 | 	}
56 | 
57 | 	for _, err := range errs {
58 | 		log.Println(err)
59 | 	}
60 | ```
61 | 
62 | ## Development
63 | 
64 | To run tests, start a kafka that runs on ":9092".  
65 | ```sh
66 | go test ./... -v -cover -count=1 -race
67 | ```
68 | 


--------------------------------------------------------------------------------
/consumer.go:
--------------------------------------------------------------------------------
 1 | package kafka
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"os"
 6 | 	"time"
 7 | 
 8 | 	"github.com/twmb/franz-go/pkg/kgo"
 9 | )
10 | 
11 | type Message []byte
12 | 
13 | type Consumer struct {
14 | 	client *kgo.Client
15 | }
16 | 
17 | func NewConsumer(groupName string, topics []string, brokers []string, logger bool) (*Consumer, error) {
18 | 	opts := []kgo.Opt{
19 | 		kgo.SeedBrokers(brokers...),
20 | 		kgo.ConsumerGroup(groupName),
21 | 		kgo.ConsumeTopics(topics...),
22 | 		kgo.DisableAutoCommit(),
23 | 		kgo.GroupProtocol("roundrobin"),
24 | 		kgo.Balancers(kgo.RoundRobinBalancer()),
25 | 	}
26 | 
27 | 	if logger {
28 | 		opts = append(opts, kgo.WithLogger(kgo.BasicLogger(os.Stderr, kgo.LogLevelInfo, nil)))
29 | 	}
30 | 
31 | 	cl, err := kgo.NewClient(opts...)
32 | 	if err != nil {
33 | 		return nil, err
34 | 	}
35 | 
36 | 	return &Consumer{
37 | 		client: cl,
38 | 	}, nil
39 | }
40 | 
41 | func (c *Consumer) ConsumeBatch(ctx context.Context, batchSize int) []Message {
42 | 	var messages []Message
43 | 
44 | 	for batchSize > 0 {
45 | 		timeout, cancel := context.WithTimeout(ctx, time.Minute*1)
46 | 		defer cancel()
47 | 
48 | 		fetches := c.client.PollRecords(timeout, batchSize)
49 | 
50 | 		iter := fetches.RecordIter()
51 | 		for !iter.Done() {
52 | 			record := iter.Next()
53 | 			messages = append(messages, record.Value)
54 | 		}
55 | 
56 | 		batchSize = batchSize - len(messages)
57 | 
58 | 		if ctx.Err() != nil {
59 | 			break
60 | 		}
61 | 	}
62 | 	c.client.CommitUncommittedOffsets(ctx)
63 | 
64 | 	return messages
65 | }
66 | 
67 | func (c *Consumer) Close() {
68 | 	c.client.Close()
69 | }
70 | 


--------------------------------------------------------------------------------
/consumer_test.go:
--------------------------------------------------------------------------------
 1 | package kafka
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"testing"
 6 | )
 7 | 
 8 | func TestConsumeBatch(t *testing.T) {
 9 | 	tests := []struct {
10 | 		name     string
11 | 		messages []Message
12 | 	}{
13 | 		{
14 | 			name: "should-got-message",
15 | 			messages: []Message{
16 | 				Message("message 1"),
17 | 			},
18 | 		},
19 | 		{
20 | 			name: "should-got-all-messages",
21 | 			messages: []Message{
22 | 				Message("message 1"), Message("message 2"), Message("message 3"), Message("message 4"), Message("message 5"),
23 | 			},
24 | 		},
25 | 	}
26 | 
27 | 	for _, test := range tests {
28 | 		t.Run(test.name, func(t *testing.T) {
29 | 			producer, err := NewProducer(2<<20, "127.0.0.1:9092")
30 | 			if err != nil {
31 | 				t.Fatal(err)
32 | 			}
33 | 			defer producer.Close()
34 | 			producer.Produce(context.Background(), test.messages, "kafka_do_test")
35 | 
36 | 			consumer, err := NewConsumer("kafka_do", []string{"kafka_do_test"}, []string{"127.0.0.1:9092"}, false)
37 | 			if err != nil {
38 | 				t.Fatal(err)
39 | 			}
40 | 			defer consumer.Close()
41 | 
42 | 			messages := consumer.ConsumeBatch(context.Background(), len(test.messages))
43 | 
44 | 			if len(messages) != len(test.messages) {
45 | 				t.Errorf("ConsumeBatch got %d, want %d", len(messages), len(test.messages))
46 | 			}
47 | 		})
48 | 	}
49 | }
50 | 


--------------------------------------------------------------------------------
/containers/dc.dev.yml:
--------------------------------------------------------------------------------
 1 | version: "3.7"
 2 | services:
 3 |   redpanda:
 4 |     command:
 5 |       - redpanda
 6 |       - start
 7 |       - --smp
 8 |       - "1"
 9 |       - --reserve-memory
10 |       - 0M
11 |       - --overprovisioned
12 |       - --node-id
13 |       - "0"
14 |       - --kafka-addr
15 |       - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
16 |       - --advertise-kafka-addr
17 |       - PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092
18 |     # NOTE: Please use the latest version here!
19 |     image: docker.vectorized.io/vectorized/redpanda:v21.7.6
20 |     container_name: redpanda-1
21 |     ports:
22 |       - 9092:9092
23 |       - 29092:29092
24 | 


--------------------------------------------------------------------------------
/doc/seo.do.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/wopehq/kafka-do/c974f94f4f9798fa5408e0cf408b8e9873120702/doc/seo.do.png


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/wopehq/kafka-do
2 | 
3 | go 1.16
4 | 
5 | require (
6 | 	github.com/google/go-cmp v0.5.5 // indirect
7 | 	github.com/twmb/franz-go v1.1.1
8 | )
9 | 


--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 2 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 3 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 4 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 5 | github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 6 | github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 7 | github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 8 | github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
 9 | github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
10 | github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
11 | github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
12 | github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc=
13 | github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
14 | github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4=
15 | github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
16 | github.com/pierrec/lz4/v4 v4.1.8 h1:ieHkV+i2BRzngO4Wd/3HGowuZStgq6QkPsD1eolNAO4=
17 | github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
18 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
19 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
20 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
21 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
22 | github.com/twmb/franz-go v1.1.1 h1:XR3JGvsRlvvTPakMcsW8vkubaHdtR4Vwy1RTiiiFshs=
23 | github.com/twmb/franz-go v1.1.1/go.mod h1:KerrVhzNpasYrWJLr2Yj6Cui43f1BxH4U9SJEDVOjqQ=
24 | github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210914042331-106aef61b693 h1:5O4u9Lc69/GIOnSIWieuwwpr0hZr7vDOhCp0hXJAqXw=
25 | github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210914042331-106aef61b693/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY=
26 | github.com/twmb/go-rbtree v1.0.0 h1:KxN7dXJ8XaZ4cvmHV1qqXTshxX3EBvX/toG5+UR49Mg=
27 | github.com/twmb/go-rbtree v1.0.0/go.mod h1:UlIAI8gu3KRPkXSobZnmJfVwCJgEhD/liWzT5ppzIyc=
28 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
29 | golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
30 | golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
31 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
32 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
33 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
34 | golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
35 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
36 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
37 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
38 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
39 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
40 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
41 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
42 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
43 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
44 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
45 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
46 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
47 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
48 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
49 | 


--------------------------------------------------------------------------------
/producer.go:
--------------------------------------------------------------------------------
 1 | package kafka
 2 | 
 3 | import (
 4 | 	"context"
 5 | 
 6 | 	"github.com/twmb/franz-go/pkg/kgo"
 7 | )
 8 | 
 9 | type Producer struct {
10 | 	client *kgo.Client
11 | }
12 | 
13 | func NewProducer(maxBytes int32, brokers ...string) (*Producer, error) {
14 | 	cl, err := kgo.NewClient(
15 | 		kgo.SeedBrokers(brokers...),
16 | 		kgo.ProducerBatchMaxBytes(maxBytes),
17 | 	)
18 | 	if err != nil {
19 | 		return nil, err
20 | 	}
21 | 
22 | 	return &Producer{
23 | 		client: cl,
24 | 	}, nil
25 | }
26 | 
27 | func (p *Producer) Produce(ctx context.Context, messages []Message, topic string) kgo.ProduceResults {
28 | 	var records []*kgo.Record
29 | 
30 | 	for _, message := range messages {
31 | 		records = append(records, &kgo.Record{Topic: topic, Value: message})
32 | 	}
33 | 
34 | 	return p.client.ProduceSync(ctx, records...)
35 | }
36 | 
37 | func (p *Producer) Close() {
38 | 	p.client.Close()
39 | }
40 | 


--------------------------------------------------------------------------------
/producer_test.go:
--------------------------------------------------------------------------------
 1 | package kafka
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"testing"
 6 | )
 7 | 
 8 | func TestProduce(t *testing.T) {
 9 | 	tests := []struct {
10 | 		name     string
11 | 		messages []Message
12 | 	}{
13 | 		{
14 | 			name: "should-got-message",
15 | 			messages: []Message{
16 | 				Message("message 1"),
17 | 			},
18 | 		},
19 | 		{
20 | 			name: "should-got-all-messages",
21 | 			messages: []Message{
22 | 				Message("message 1"), Message("message 2"), Message("message 3"), Message("message 4"), Message("message 5"),
23 | 			},
24 | 		},
25 | 	}
26 | 
27 | 	for _, test := range tests {
28 | 		t.Run(test.name, func(t *testing.T) {
29 | 			producer, err := NewProducer(2<<20, "127.0.0.1:9092")
30 | 			if err != nil {
31 | 				t.Fatal(err)
32 | 			}
33 | 			defer producer.Close()
34 | 
35 | 			producer.Produce(context.Background(), test.messages, "kafka_do_test")
36 | 
37 | 			consumer, err := NewConsumer("kafka_do", []string{"kafka_do_test"}, []string{"127.0.0.1:9092"}, false)
38 | 			if err != nil {
39 | 				t.Fatal(err)
40 | 			}
41 | 			defer consumer.Close()
42 | 
43 | 			messages := consumer.ConsumeBatch(context.Background(), len(test.messages))
44 | 
45 | 			if len(messages) != len(test.messages) {
46 | 				t.Errorf("Produce got %d, want %d", len(messages), len(test.messages))
47 | 			}
48 | 		})
49 | 	}
50 | }
51 | 


--------------------------------------------------------------------------------