├── .github └── workflows │ ├── assets │ └── img │ │ ├── hash.png │ │ └── leastbytes.png │ ├── release.yml │ └── production.yml ├── Dockerfile ├── pkg ├── stringgenerator │ └── main.go ├── kafkadialer │ └── main.go ├── clients │ └── main.go └── fakejson │ └── main.go ├── go.mod ├── .goreleaser.yml ├── LICENSE ├── docs └── DEVELOPMENT.md ├── docker-compose.yml ├── main.go ├── README.md └── go.sum /.github/workflows/assets/img/hash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msfidelis/kafka-stress/HEAD/.github/workflows/assets/img/hash.png -------------------------------------------------------------------------------- /.github/workflows/assets/img/leastbytes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msfidelis/kafka-stress/HEAD/.github/workflows/assets/img/leastbytes.png -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.22 AS builder 2 | 3 | WORKDIR $GOPATH/src/kafka-stress 4 | 5 | COPY . ./ 6 | 7 | RUN go get -u 8 | 9 | RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o kafka-stress . 10 | 11 | 12 | FROM cgr.dev/chainguard/wolfi-base:latest 13 | 14 | COPY --from=builder /go/src/kafka-stress/kafka-stress ./ 15 | 16 | 17 | ENTRYPOINT ["./kafka-stress"] -------------------------------------------------------------------------------- /pkg/stringgenerator/main.go: -------------------------------------------------------------------------------- 1 | package stringgenerator 2 | 3 | import ( 4 | "math/rand" 5 | ) 6 | 7 | const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" 8 | 9 | // RandStringBytes Gerenerate random string bytes size 10 | func RandStringBytes(n int) string { 11 | b := make([]byte, n) 12 | for i := range b { 13 | b[i] = letterBytes[rand.Intn(len(letterBytes))] 14 | } 15 | return string(b) 16 | } 17 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module kafka-stress 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/bxcodec/faker/v3 v3.8.1 7 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect 8 | github.com/frankban/quicktest v1.11.3 // indirect 9 | github.com/golang/snappy v0.0.4 // indirect 10 | github.com/google/uuid v1.3.0 11 | github.com/klauspost/compress v1.15.14 // indirect 12 | github.com/pierrec/lz4 v2.6.1+incompatible // indirect 13 | github.com/pierrec/lz4/v4 v4.1.17 // indirect 14 | github.com/segmentio/kafka-go v0.4.38 15 | golang.org/x/tools v0.1.9 // indirect 16 | ) 17 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | release: 2 | prerelease: false 3 | 4 | builds: 5 | - binary: kafka-stress 6 | env: 7 | - TOP=0 8 | goos: 9 | - windows 10 | - darwin 11 | - linux 12 | - freebsd 13 | goarch: 14 | - amd64 15 | - arm64 16 | 17 | # brews: 18 | # - github: 19 | # owner: msfidelis 20 | # name: homebrew-kafka-stress 21 | # homepage: "https://github.com/msfidelis/kafka-stress/" 22 | # description: "Kafka Producer / Consumer Stress Test Tool" 23 | 24 | archives: 25 | - format: binary 26 | format_overrides: 27 | - goos: windows 28 | format: zip -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: 'release packages' 2 | on: 3 | push: 4 | tags: 5 | - '*' 6 | jobs: 7 | goreleaser: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - 11 | name: Checkout 12 | uses: actions/checkout@v2 13 | with: 14 | fetch-depth: 0 15 | - 16 | name: Set up Go 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: 1.22 20 | - 21 | name: Run GoReleaser 22 | uses: goreleaser/goreleaser-action@v2 23 | with: 24 | version: latest 25 | args: release --rm-dist 26 | env: 27 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /pkg/kafkadialer/main.go: -------------------------------------------------------------------------------- 1 | package kafkadialer 2 | 3 | import ( 4 | "crypto/tls" 5 | "os" 6 | "time" 7 | 8 | "github.com/segmentio/kafka-go" 9 | ) 10 | 11 | // GetDialer generate a kafka.Dialer instance configured 12 | func GetDialer(ssl bool) kafka.Dialer { 13 | var dialer kafka.Dialer 14 | name, err := os.Hostname() 15 | 16 | if err != nil { 17 | panic(err) 18 | } 19 | 20 | if ssl { 21 | dialer = kafka.Dialer{ 22 | Timeout: 20 * time.Second, 23 | DualStack: true, 24 | ClientID: name, 25 | TLS: &tls.Config{}, 26 | } 27 | } else { 28 | dialer = kafka.Dialer{ 29 | Timeout: 20 * time.Second, 30 | DualStack: true, 31 | ClientID: name, 32 | } 33 | } 34 | 35 | return dialer 36 | } 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Matheus Fidelis 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /pkg/clients/main.go: -------------------------------------------------------------------------------- 1 | package clients 2 | 3 | import ( 4 | "kafka-stress/pkg/kafkadialer" 5 | "strings" 6 | "time" 7 | 8 | "github.com/segmentio/kafka-go" 9 | ) 10 | 11 | // GetConsumer return a Kafka Consumer Client 12 | func GetConsumer(bootstrapServers, topic, consumerGroup string, consumer int, ssl bool) *kafka.Reader { 13 | dialer := kafkadialer.GetDialer(ssl) 14 | 15 | return kafka.NewReader(kafka.ReaderConfig{ 16 | Brokers: strings.Split(bootstrapServers, ","), 17 | Topic: topic, 18 | GroupID: consumerGroup, 19 | Dialer: &dialer, 20 | MinBytes: 10e3, // 10KB 21 | MaxBytes: 10e6, // 10MB 22 | CommitInterval: time.Second, 23 | }) 24 | } 25 | 26 | // GetProducer return a Kafka Producer Client 27 | func GetProducer(bootstrapServers string, topic string, batchSize int, acks int, ssl bool, balancer string) *kafka.Writer { 28 | 29 | dialer := kafkadialer.GetDialer(ssl) 30 | 31 | var config kafka.WriterConfig 32 | 33 | config = kafka.WriterConfig{ 34 | Brokers: strings.Split(bootstrapServers, ","), 35 | Topic: topic, 36 | Balancer: &kafka.LeastBytes{}, 37 | BatchSize: batchSize, 38 | BatchTimeout: 2 * time.Second, 39 | RequiredAcks: acks, 40 | Dialer: &dialer, 41 | WriteTimeout: 10 * time.Second, 42 | ReadTimeout: 10 * time.Second, 43 | } 44 | 45 | switch balancer { 46 | case "hash": 47 | config.Balancer = &kafka.Hash{} 48 | case "murmur2": 49 | config.Balancer = &kafka.Murmur2Balancer{} 50 | case "crc32": 51 | config.Balancer = &kafka.CRC32Balancer{} 52 | default: 53 | config.Balancer = &kafka.Hash{} 54 | } 55 | 56 | return kafka.NewWriter(config) 57 | 58 | } 59 | -------------------------------------------------------------------------------- /docs/DEVELOPMENT.md: -------------------------------------------------------------------------------- 1 | # Setup local environment 2 | 3 | ### Setup Development Dependencies with Docker 4 | 5 | ```bash 6 | docker-compose up --force-recreate 7 | ``` 8 | 9 | ### Create a test topic 10 | 11 | 12 | ```bash 13 | docker-compose exec broker kafka-topics --create --topic kafka-stress --partitions 3 --replication-factor 1 --if-not-exists --zookeeper zookeeper:2181 14 | ``` 15 | 16 | 17 | # Schema Registry 18 | 19 | ## Utils 20 | 21 | [Json to AVRO Converter](https://toolslick.com/generation/metadata/avro-schema-from-json) 22 | 23 | ## List Schemas 24 | 25 | ```bash 26 | curl -X GET http://0.0.0.0:8081/subjects 27 | ``` 28 | 29 | ## AVRO 30 | 31 | Create and Schema in AVRO format 32 | 33 | ``` 34 | curl http://0.0.0.0:8081/subjects/example/versions -X POST \ 35 | -H "Content-Type: application/vnd.schemaregistry.v1+json" \ 36 | -d ' 37 | { 38 | "schema":"{\n \"name\":\"Example\",\n \"type\":\"record\",\n \"namespace\":\"com.acme.avro\",\n \"fields\":[\n {\n \"name\":\"name\",\n \"type\":\"string\"\n },\n {\n \"name\":\"age\",\n \"type\":\"int\"\n }\n ]\n }" 39 | } 40 | ' 41 | ``` 42 | 43 | 44 | 45 | # Consumer 46 | 47 | ## Algoritms 48 | 49 | Producing 15000 events in 3 partitions topic 50 | 51 | ### Hash 52 | 53 | ```bash 54 | ❯ go run main.go --bootstrap-servers 0.0.0.0:9092 --events 15000 --topic brabo --test-mode producer --consumers 3 55 | Sent 15000 messages to topic brabo with 0 errors 56 | Tests finished in 1.406677516s. Producer mean time 10663.42/s 57 | ``` 58 | 59 | ### LeastBytes 60 | 61 | ```bash 62 | ❯ go run main.go --bootstrap-servers 0.0.0.0:9092 --events 15000 --topic brabo --test-mode producer --consumers 3 63 | Sent 15000 messages to topic brabo with 0 errors 64 | Tests finished in 885.728755ms. Producer mean time 16935.21/s 65 | ``` 66 | 67 | [!LeastBytes]() -------------------------------------------------------------------------------- /pkg/fakejson/main.go: -------------------------------------------------------------------------------- 1 | package fakejson 2 | 3 | import ( 4 | "fmt" 5 | "github.com/bxcodec/faker/v3" 6 | "encoding/json" 7 | ) 8 | 9 | type fake struct { 10 | UserName string `faker:"username" json:"username"` 11 | PhoneNumber string `faker:"phone_number" json:"phone_number"` 12 | IPV4 string `faker:"ipv4" json:"ipv4"` 13 | IPV6 string `faker:"ipv6" json:"ipv6"` 14 | MacAddress string `faker:"mac_address" json:"mac_address"` 15 | URL string `faker:"url" json: "url"` 16 | DayOfWeek string `faker:"day_of_week" json: "day_of_week"` 17 | DayOfMonth string `faker:"day_of_month" json: "day_of_month"` 18 | Timestamp string `faker:"timestamp" json: "timestamp"` 19 | Century string `faker:"century" json: "century"` 20 | TimeZone string `faker:"timezone", json:"timezone"` 21 | TimePeriod string `faker:"time_period" json:"time_period"` 22 | Word string `faker:"word" json:"word"` 23 | Sentence string `faker:"sentence" json:"sentence"` 24 | Paragraph string `faker:"paragraph" json:"paragraph"` 25 | Currency string `faker:"currency" json:"currency"` 26 | Amount float64 `faker:"amount" json:"amount" ` 27 | AmountWithCurrency string `faker:"amount_with_currency" json:"amount_with_currency"` 28 | UUIDHypenated string `faker:"uuid_hyphenated" json:"uuid_hyphenated"` 29 | UUID string `faker:"uuid_digit" json:"uuid_digit"` 30 | PaymentMethod string `faker:"oneof: cc, paypal, check, money order"` 31 | } 32 | 33 | // RandJSONPayload Gerenerate random json with fake data 34 | func RandJSONPayload() string { 35 | 36 | a := fake{} 37 | err := faker.FakeData(&a) 38 | if err != nil { 39 | fmt.Println(err) 40 | return "{}" 41 | } 42 | 43 | b, err := json.Marshal(a) 44 | if err != nil { 45 | fmt.Println(err) 46 | return "{}" 47 | } 48 | return string(b) 49 | 50 | } -------------------------------------------------------------------------------- /.github/workflows/production.yml: -------------------------------------------------------------------------------- 1 | name: 'kafka-stress ci' 2 | on: 3 | push: 4 | pull_request: 5 | types: [ opened, reopened ] 6 | jobs: 7 | unit-test: 8 | strategy: 9 | matrix: 10 | go-version: [1.22.x] 11 | platform: [ubuntu-latest, macos-latest] 12 | runs-on: ${{ matrix.platform }} 13 | steps: 14 | 15 | - uses: actions/setup-go@v1 16 | with: 17 | go-version: ${{ matrix.go-version }} 18 | 19 | - name: setup GOPATH into PATH 20 | run: | 21 | echo "::set-env name=GOPATH::$(go env GOPATH)" 22 | echo "::add-path::$(go env GOPATH)/bin" 23 | shell: bash 24 | env: 25 | ACTIONS_ALLOW_UNSECURE_COMMANDS: true 26 | 27 | - uses: actions/checkout@v2 28 | 29 | - name: Install dependencies 30 | run: go get -u 31 | 32 | - name: Test 33 | run: go test -v 34 | 35 | build-docker-artifacts: 36 | needs: [ unit-test ] 37 | runs-on: ubuntu-latest 38 | if: contains(github.ref, 'main') 39 | steps: 40 | - uses: actions/setup-go@v1 41 | with: 42 | go-version: '1.22.x' 43 | 44 | - uses: actions/checkout@v1 45 | 46 | - name: Docker Build 47 | run: docker build -t kafka-stress:latest . 48 | 49 | - name: Docker Tag Latest 50 | run: docker tag kafka-stress:latest fidelissauro/kafka-stress:latest 51 | 52 | - name: Docker Tag Latest Release 53 | run: | 54 | TAG=$(git describe --tags --abbrev=0) 55 | docker tag kafka-stress:latest fidelissauro/kafka-stress:$TAG 56 | - name: Login to DockerHub 57 | uses: docker/login-action@v1 58 | with: 59 | username: ${{ secrets.DOCKER_USERNAME }} 60 | password: ${{ secrets.DOCKER_PASSWORD}} 61 | 62 | - name: Docker Push Latest 63 | run: docker push fidelissauro/kafka-stress:latest 64 | 65 | - name: Docker Push Release Tag 66 | run: | 67 | TAG=$(git describe --tags --abbrev=0) 68 | docker push fidelissauro/kafka-stress:$TAG -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | version: "3" 3 | 4 | services: 5 | zookeeper: 6 | image: confluentinc/cp-zookeeper:5.2.5 7 | hostname: zookeeper 8 | container_name: zookeeper 9 | ports: 10 | - "2181:2181" 11 | environment: 12 | ZOOKEEPER_CLIENT_PORT: 2181 13 | ZOOKEEPER_TICK_TIME: 2000 14 | 15 | broker: 16 | image: confluentinc/cp-server:5.4.0 17 | hostname: broker 18 | container_name: broker 19 | depends_on: 20 | - zookeeper 21 | ports: 22 | - "9092:9092" 23 | environment: 24 | KAFKA_BROKER_ID: 1 25 | KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" 26 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 27 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 28 | KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter 29 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 30 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 31 | KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 32 | CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092 33 | CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181 34 | CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 35 | CONFLUENT_METRICS_ENABLE: "true" 36 | CONFLUENT_SUPPORT_CUSTOMER_ID: "anonymous" 37 | 38 | schema-registry: 39 | image: confluentinc/cp-schema-registry:6.1.2 40 | hostname: schema-registry 41 | container_name: schema-registry 42 | depends_on: 43 | - zookeeper 44 | - broker 45 | ports: 46 | - "8081:8081" 47 | environment: 48 | SCHEMA_REGISTRY_HOST_NAME: schema-registry 49 | SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181" 50 | 51 | control-center: 52 | image: confluentinc/cp-enterprise-control-center:5.4.0 53 | hostname: control-center 54 | container_name: control-center 55 | depends_on: 56 | - zookeeper 57 | - broker 58 | - schema-registry 59 | ports: 60 | - "9021:9021" 61 | environment: 62 | CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092' 63 | CONTROL_CENTER_ZOOKEEPER_CONNECT: 'zookeeper:2181' 64 | CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" 65 | CONTROL_CENTER_REPLICATION_FACTOR: 1 66 | CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1 67 | CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1 68 | CONFLUENT_METRICS_TOPIC_REPLICATION: 1 69 | PORT: 9021 -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "strings" 8 | "sync" 9 | "sync/atomic" 10 | "time" 11 | 12 | "kafka-stress/pkg/clients" 13 | "kafka-stress/pkg/stringgenerator" 14 | "kafka-stress/pkg/fakejson" 15 | 16 | guuid "github.com/google/uuid" 17 | kafka "github.com/segmentio/kafka-go" 18 | ) 19 | 20 | func main() { 21 | topic := flag.String("topic", "kafka-stress", "Kafka Stress Topics") 22 | createTopic := flag.Bool("create-topic", false, "Auto Create Topic?") 23 | ssl := flag.Bool("ssl-enabled", false, "SSL Mode") 24 | testMode := flag.String("test-mode", "producer", "Test Type; Ex producer;consumer. Default: producer") 25 | bootstrapServers := flag.String("bootstrap-servers", "0.0.0.0:9092", "Kafka Bootstrap Servers Broker Lists") 26 | zookeeperServers := flag.String("zookeeper-servers", "0.0.0.0:2181", "Zookeeper Connection String") 27 | schemaRegistryURL := flag.String("schema-registry", "0.0.0.0:8081", "Schema Registry URL") 28 | size := flag.Int("size", 62, "Message size in bytes") 29 | acks := flag.Int("ack", 1, "Required ACKs to produce messages") 30 | batchSize := flag.Int("batch-size", 0, "Batch size for producer mode") 31 | schema := flag.String("schema", "", "Schema") 32 | events := flag.Int("events", 10000, "Numer of events will be created in topic") 33 | consumers := flag.Int("consumers", 1, "Number of consumers will be used in topic") 34 | consumerGroup := flag.String("consumer-group", "kafka-stress", "Consumer group name") 35 | format := flag.String("format", "string", "Events Format; ex string,json,avro") 36 | verbose := flag.Bool("verbose", false, "Verbose Mode; It Prints Events consumed") 37 | balancer := flag.String("balancer", "hash", "Balance algorithm for producer mode; Ex: hash,murmur2,crc32") 38 | 39 | 40 | flag.Parse() 41 | 42 | if *createTopic { 43 | createTopicBeforeTest(*topic, *zookeeperServers) 44 | } 45 | 46 | switch strings.ToLower(*testMode) { 47 | case "producer": 48 | produce(*bootstrapServers, *topic, *events, *size, *batchSize, *acks, *schemaRegistryURL, *schema, *ssl, *format, *balancer) 49 | break 50 | case "consumer": 51 | consume(*bootstrapServers, *topic, *consumerGroup, *consumers, *ssl, *verbose) 52 | default: 53 | return 54 | } 55 | } 56 | 57 | func produce(bootstrapServers string, topic string, events int, size int, batchSize int, acks int, schemaRegistryURL string, schema string, ssl bool, format string, balancer string) { 58 | 59 | var wg sync.WaitGroup 60 | var executions uint64 61 | var errors uint64 62 | var message string 63 | 64 | producer := clients.GetProducer(bootstrapServers, topic, batchSize, acks, ssl, balancer) 65 | defer producer.Close() 66 | 67 | start := time.Now() 68 | 69 | for i := 0; i < events; i++ { 70 | wg.Add(1) 71 | 72 | switch format { 73 | case "string": 74 | message = stringgenerator.RandStringBytes(size) 75 | break; 76 | case "json": 77 | message = fakejson.RandJSONPayload() 78 | break; 79 | default: 80 | message = stringgenerator.RandStringBytes(size) 81 | } 82 | 83 | go func() { 84 | msg := kafka.Message{ 85 | Key: []byte(guuid.New().String()), 86 | Value: []byte(message), 87 | } 88 | 89 | err := producer.WriteMessages(context.Background(), msg) 90 | if err != nil { 91 | atomic.AddUint64(&errors, 1) 92 | } else { 93 | atomic.AddUint64(&executions, 1) 94 | } 95 | 96 | var multiple = executions % 1000 97 | if multiple == 0 && executions != 0 { 98 | fmt.Printf("Sent %v messages to topic %s with %v errors \n", executions, topic, errors) 99 | } 100 | 101 | wg.Done() 102 | }() 103 | } 104 | 105 | wg.Wait() 106 | elapsed := time.Since(start) 107 | meanEventsSent := float64(executions) / elapsed.Seconds() 108 | 109 | fmt.Printf("Tests finished in %v. Produce %v messages with mean time %.2f/s using %s balance algorithm \n", elapsed, executions, meanEventsSent, balancer) 110 | } 111 | 112 | func consume(bootstrapServers, topic, consumerGroup string, consumers int, ssl bool, verbose bool) { 113 | 114 | var wg sync.WaitGroup 115 | var counter uint64 116 | 117 | for i := 0; i < consumers; i++ { 118 | wg.Add(1) 119 | var consumerID = i + 1 120 | consumer := clients.GetConsumer(bootstrapServers, topic, consumerGroup, consumerID, ssl) 121 | consumerName := fmt.Sprintf("%v-%v", consumerGroup, consumerID) 122 | 123 | fmt.Printf("[Consumer] Starting consumer %v\n", consumerName) 124 | 125 | go func() { 126 | for { 127 | m, err := consumer.ReadMessage(context.Background()) 128 | if err != nil { 129 | wg.Done() 130 | break 131 | } 132 | 133 | atomic.AddUint64(&counter, 1) 134 | 135 | if verbose == true { 136 | fmt.Printf("[Key] %s | [Value] %s\n\n\n", m.Key, m.Value) 137 | } 138 | 139 | var multiple = counter % 100 140 | if multiple == 0 && counter != 0 { 141 | fmt.Printf("[Consumer] %v Messages retrived from topic %v by consumer group %s \n", counter, m.Topic, consumerGroup) 142 | } 143 | } 144 | wg.Done() 145 | }() 146 | 147 | } 148 | wg.Wait() 149 | } 150 | 151 | func createTopicBeforeTest(topic string, zookeeper string) { 152 | fmt.Printf("Creating topic %s\n", topic) 153 | } 154 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kafka Stress - Stress Test Tool for Kafka Clusters, Producers and Consumers Tunning 2 | 3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |