├── kafka ├── zk1 │ ├── data │ │ └── .gitkeep │ └── log │ │ └── .gitkeep ├── zk2 │ ├── data │ │ └── .gitkeep │ └── log │ │ └── .gitkeep ├── zk3 │ ├── data │ │ └── .gitkeep │ └── log │ │ └── .gitkeep ├── kfk1 │ └── data │ │ └── .gitkeep ├── kfk2 │ └── data │ │ └── .gitkeep ├── kfk3 │ └── data │ │ └── .gitkeep └── docker-compose.yml ├── .env ├── gopher.png ├── test └── ab_post_test.json ├── .gitignore ├── src ├── docker-compose.yml ├── consume │ ├── go.mod │ ├── vendor │ │ └── modules.txt │ ├── dockerfile │ ├── main.go │ └── go.sum └── produce │ ├── go.mod │ ├── dockerfile │ ├── vendor │ └── modules.txt │ ├── go.sum │ └── main.go ├── LICENSE ├── Makefile └── README.md /kafka/zk1/data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kafka/zk1/log/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kafka/zk2/data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kafka/zk2/log/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kafka/zk3/data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kafka/zk3/log/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kafka/kfk1/data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kafka/kfk2/data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kafka/kfk3/data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | GOPROXY=https://goproxy.io 2 | KFKADDR=kfk1:19092 -------------------------------------------------------------------------------- /gopher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ErikJiang/kafka_cluster_example/HEAD/gopher.png -------------------------------------------------------------------------------- /test/ab_post_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "text": "Hi guys! This is kafka cluster example!" 3 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Ignore all files in kafka/ & src/ 15 | kafka/kfk1/data/* 16 | kafka/kfk2/data/* 17 | kafka/kfk3/data/* 18 | kafka/zk1/data/* 19 | kafka/zk1/log/* 20 | kafka/zk2/data/* 21 | kafka/zk2/log/* 22 | kafka/zk3/data/* 23 | kafka/zk3/log/* 24 | src/produce/vendor/* 25 | src/consume/vendor/* 26 | 27 | # Except for .gitkeep & modules.txt 28 | !.gitkeep 29 | !modules.txt -------------------------------------------------------------------------------- /src/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | produce: 4 | build: ./produce 5 | container_name: produce 6 | ports: 7 | - "9000:9000" 8 | environment: 9 | LISTEN_ADDRESS: '0.0.0.0:9000' 10 | KAFKA_BROKERS: 'kfk1:19092,kfk2:29092,kfk3:39092' 11 | KAFKA_TOPIC: 'foo' 12 | 13 | consume1: 14 | build: ./consume 15 | container_name: consume1 16 | environment: 17 | KAFKA_BROKERS: 'kfk1:19092,kfk2:29092,kfk3:39092' 18 | KAFKA_CONSUMER_GROUP_ID: 'consumer-group' 19 | KAFKA_TOPIC: 'foo' 20 | 21 | consume2: 22 | build: ./consume 23 | container_name: consume2 24 | environment: 25 | KAFKA_BROKERS: 'kfk1:19092,kfk2:29092,kfk3:39092' 26 | KAFKA_CONSUMER_GROUP_ID: 'consumer-group' 27 | KAFKA_TOPIC: 'foo' 28 | 29 | networks: 30 | default: 31 | external: 32 | name: kafka_default -------------------------------------------------------------------------------- /src/consume/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ErikJiang/kafka_tutorial/src/consume 2 | 3 | require ( 4 | github.com/DataDog/zstd v1.3.5 // indirect 5 | github.com/Shopify/sarama v1.20.1 6 | github.com/Shopify/toxiproxy v2.1.4+incompatible // indirect 7 | github.com/bsm/sarama-cluster v2.1.15+incompatible 8 | github.com/davecgh/go-spew v1.1.1 // indirect 9 | github.com/eapache/go-resiliency v1.1.0 // indirect 10 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect 11 | github.com/eapache/queue v1.1.0 // indirect 12 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect 13 | github.com/onsi/ginkgo v1.7.0 // indirect 14 | github.com/onsi/gomega v1.4.3 // indirect 15 | github.com/pierrec/lz4 v2.0.5+incompatible // indirect 16 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect 17 | github.com/rs/zerolog v1.11.0 18 | github.com/urfave/cli v1.20.0 19 | gopkg.in/yaml.v2 v2.2.2 // indirect 20 | ) 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 ERIK 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/consume/vendor/modules.txt: -------------------------------------------------------------------------------- 1 | # github.com/DataDog/zstd v1.3.5 2 | github.com/DataDog/zstd 3 | # github.com/Shopify/sarama v1.20.1 4 | github.com/Shopify/sarama 5 | # github.com/bsm/sarama-cluster v2.1.15+incompatible 6 | github.com/bsm/sarama-cluster 7 | # github.com/davecgh/go-spew v1.1.1 8 | github.com/davecgh/go-spew/spew 9 | # github.com/eapache/go-resiliency v1.1.0 10 | github.com/eapache/go-resiliency/breaker 11 | # github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 12 | github.com/eapache/go-xerial-snappy 13 | # github.com/eapache/queue v1.1.0 14 | github.com/eapache/queue 15 | # github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db 16 | github.com/golang/snappy 17 | # github.com/pierrec/lz4 v2.0.5+incompatible 18 | github.com/pierrec/lz4 19 | github.com/pierrec/lz4/internal/xxh32 20 | # github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a 21 | github.com/rcrowley/go-metrics 22 | # github.com/rs/zerolog v1.11.0 23 | github.com/rs/zerolog/log 24 | github.com/rs/zerolog 25 | github.com/rs/zerolog/internal/cbor 26 | github.com/rs/zerolog/internal/json 27 | # github.com/urfave/cli v1.20.0 28 | github.com/urfave/cli 29 | -------------------------------------------------------------------------------- /src/produce/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ErikJiang/kafka_tutorial/src/produce 2 | 3 | require ( 4 | github.com/DataDog/zstd v1.3.5 // indirect 5 | github.com/Shopify/sarama v1.20.1 6 | github.com/Shopify/toxiproxy v2.1.4+incompatible // indirect 7 | github.com/eapache/go-resiliency v1.1.0 // indirect 8 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect 9 | github.com/eapache/queue v1.1.0 // indirect 10 | github.com/gin-contrib/sse v0.0.0-20190125020943-a7658810eb74 // indirect 11 | github.com/gin-gonic/gin v1.3.0 12 | github.com/golang/protobuf v1.2.0 // indirect 13 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect 14 | github.com/json-iterator/go v1.1.5 // indirect 15 | github.com/mattn/go-isatty v0.0.4 // indirect 16 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 17 | github.com/modern-go/reflect2 v1.0.1 // indirect 18 | github.com/pierrec/lz4 v2.0.7+incompatible // indirect 19 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect 20 | github.com/rs/zerolog v1.11.0 21 | github.com/stretchr/testify v1.3.0 // indirect 22 | github.com/ugorji/go/codec v0.0.0-20190128213124-ee1426cffec0 // indirect 23 | github.com/urfave/cli v1.20.0 24 | golang.org/x/net v0.0.0-20190110200230-915654e7eabc // indirect 25 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect 26 | golang.org/x/sys v0.0.0-20190114130336-2be517255631 // indirect 27 | gopkg.in/go-playground/assert.v1 v1.2.1 // indirect 28 | gopkg.in/go-playground/validator.v8 v8.18.2 // indirect 29 | gopkg.in/yaml.v2 v2.2.2 // indirect 30 | ) 31 | -------------------------------------------------------------------------------- /src/consume/dockerfile: -------------------------------------------------------------------------------- 1 | ############################ 2 | # STEP 1 构建可执行文件 3 | ############################ 4 | 5 | # 指定 GO 版本号 6 | ARG GO_VERSION=1.11.1 7 | 8 | # 指定构建环境 9 | FROM golang:${GO_VERSION}-alpine3.7 AS builder 10 | 11 | # china aliyun mirrors 12 | RUN echo "http://mirrors.aliyun.com/alpine/v3.7/main/" > /etc/apk/repositories 13 | 14 | # ca-certificates is required to call HTTPS endpoints. 15 | # tzdata is required to time zone info. 16 | RUN apk update && apk upgrade && apk add --no-cache ca-certificates tzdata && update-ca-certificates 17 | 18 | # 创建用户 appuser 19 | RUN adduser -D -g '' appuser 20 | 21 | # 复制源码并指定工作目录 22 | RUN mkdir -p /src/app 23 | COPY . /src/app 24 | WORKDIR /src/app 25 | 26 | # 为 go build 设置环境变量: 27 | # * CGO_ENABLED=0 表示构建一个静态链接的可执行程序 28 | # * GOOS=linux GOARCH=amd64 表示指定linux 64位的运行环境 29 | # * GOPROXY=https://goproxy.io 指定代理地址 30 | # * GOFLAGS=-mod=vendor 在执行 `go build` 强制查看 `/vendor` 目录 31 | ENV CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor 32 | 33 | # 构建可执行文件 34 | RUN go build -a -installsuffix cgo -ldflags="-w -s" -o /src/app/consume 35 | 36 | ############################ 37 | # STEP 2 构建镜像 38 | ############################ 39 | 40 | # 指定最小镜像源 41 | FROM scratch AS final 42 | 43 | # 设置系统语言 44 | ENV LANG en_US.UTF-8 45 | 46 | # 从 builder 中导入时区信息 47 | COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo 48 | 49 | # 从 builder 中导入证书 50 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 51 | 52 | # 从 builder 中导入用户及组相关文件 53 | COPY --from=builder /etc/passwd /etc/passwd 54 | 55 | # 将构建的可执行文件复制到新镜像中 56 | COPY --from=builder /src/app/consume /consume 57 | 58 | # 运行 59 | ENTRYPOINT [ "/consume" ] 60 | -------------------------------------------------------------------------------- /src/produce/dockerfile: -------------------------------------------------------------------------------- 1 | ############################ 2 | # STEP 1 构建可执行文件 3 | ############################ 4 | 5 | # 指定 GO 版本号 6 | ARG GO_VERSION=1.11.1 7 | 8 | # 指定构建环境 9 | FROM golang:${GO_VERSION}-alpine3.7 AS builder 10 | 11 | # china aliyun mirrors 12 | RUN echo "http://mirrors.aliyun.com/alpine/v3.7/main/" > /etc/apk/repositories 13 | 14 | # ca-certificates is required to call HTTPS endpoints. 15 | # tzdata is required to time zone info. 16 | RUN apk update && apk upgrade && apk add --no-cache ca-certificates tzdata && update-ca-certificates 17 | 18 | # 创建用户 appuser 19 | RUN adduser -D -g '' appuser 20 | 21 | # 复制源码并指定工作目录 22 | RUN mkdir -p /src/app 23 | COPY . /src/app 24 | WORKDIR /src/app 25 | 26 | # 为 go build 设置环境变量: 27 | # * CGO_ENABLED=0 表示构建一个静态链接的可执行程序 28 | # * GOOS=linux GOARCH=amd64 表示指定linux 64位的运行环境 29 | # * GOPROXY=https://goproxy.io 指定代理地址 30 | # * GOFLAGS=-mod=vendor 在执行 `go build` 强制查看 `/vendor` 目录 31 | ENV CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor 32 | 33 | # 构建可执行文件 34 | RUN go build -a -installsuffix cgo -ldflags="-w -s" -o /src/app/produce 35 | 36 | ############################ 37 | # STEP 2 构建镜像 38 | ############################ 39 | 40 | # 指定最小镜像源 41 | FROM scratch AS final 42 | 43 | # 设置系统语言 44 | ENV LANG en_US.UTF-8 45 | 46 | # 从 builder 中导入时区信息 47 | COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo 48 | 49 | # 从 builder 中导入证书 50 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 51 | 52 | # 从 builder 中导入用户及组相关文件 53 | COPY --from=builder /etc/passwd /etc/passwd 54 | 55 | # 将构建的可执行文件复制到新镜像中 56 | COPY --from=builder /src/app/produce /produce 57 | 58 | # 端口申明 59 | EXPOSE 9000 60 | 61 | # 运行 62 | ENTRYPOINT [ "/produce" ] 63 | -------------------------------------------------------------------------------- /src/produce/vendor/modules.txt: -------------------------------------------------------------------------------- 1 | # github.com/DataDog/zstd v1.3.5 2 | github.com/DataDog/zstd 3 | # github.com/Shopify/sarama v1.20.1 4 | github.com/Shopify/sarama 5 | # github.com/davecgh/go-spew v1.1.0 6 | github.com/davecgh/go-spew/spew 7 | # github.com/eapache/go-resiliency v1.1.0 8 | github.com/eapache/go-resiliency/breaker 9 | # github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 10 | github.com/eapache/go-xerial-snappy 11 | # github.com/eapache/queue v1.1.0 12 | github.com/eapache/queue 13 | # github.com/gin-contrib/sse v0.0.0-20190125020943-a7658810eb74 14 | github.com/gin-contrib/sse 15 | # github.com/gin-gonic/gin v1.3.0 16 | github.com/gin-gonic/gin 17 | github.com/gin-gonic/gin/binding 18 | github.com/gin-gonic/gin/json 19 | github.com/gin-gonic/gin/render 20 | # github.com/golang/protobuf v1.2.0 21 | github.com/golang/protobuf/proto 22 | # github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db 23 | github.com/golang/snappy 24 | # github.com/json-iterator/go v1.1.5 25 | github.com/json-iterator/go 26 | # github.com/mattn/go-isatty v0.0.4 27 | github.com/mattn/go-isatty 28 | # github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd 29 | github.com/modern-go/concurrent 30 | # github.com/modern-go/reflect2 v1.0.1 31 | github.com/modern-go/reflect2 32 | # github.com/pierrec/lz4 v2.0.7+incompatible 33 | github.com/pierrec/lz4 34 | github.com/pierrec/lz4/internal/xxh32 35 | # github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a 36 | github.com/rcrowley/go-metrics 37 | # github.com/rs/zerolog v1.11.0 38 | github.com/rs/zerolog/log 39 | github.com/rs/zerolog 40 | github.com/rs/zerolog/internal/cbor 41 | github.com/rs/zerolog/internal/json 42 | # github.com/ugorji/go/codec v0.0.0-20190128213124-ee1426cffec0 43 | github.com/ugorji/go/codec 44 | # github.com/urfave/cli v1.20.0 45 | github.com/urfave/cli 46 | # golang.org/x/sys v0.0.0-20190114130336-2be517255631 47 | golang.org/x/sys/unix 48 | # gopkg.in/go-playground/validator.v8 v8.18.2 49 | gopkg.in/go-playground/validator.v8 50 | # gopkg.in/yaml.v2 v2.2.2 51 | gopkg.in/yaml.v2 52 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include .env 2 | 3 | PROJECTNAME=$(shell basename "$(PWD)") 4 | 5 | all: help 6 | 7 | ## vendor Auto generate go vendor dir. 8 | .PHONY: vendor 9 | vendor: 10 | @echo "auto generate vendor dir ..."; 11 | export GOPROXY=$(GOPROXY); cd src/produce/; go mod vendor; 12 | export GOPROXY=$(GOPROXY); cd src/consume/; go mod vendor; 13 | 14 | ## up Docker compose up for src. 15 | .PHONY: up 16 | up: 17 | @echo "docker compose up ..."; 18 | docker-compose -f src/docker-compose.yml up -d; 19 | 20 | ## down Docker compose down for src. 21 | .PHONY: down 22 | down: 23 | @echo "docker compose down ..."; 24 | docker-compose -f src/docker-compose.yml down; 25 | 26 | ## ps Docker compose ps for src. 27 | .PHONY: ps 28 | ps: 29 | @echo "docker compose ps ..."; 30 | docker-compose -f src/docker-compose.yml ps; 31 | 32 | ## logs Docker compose logs for src. 33 | .PHONY: logs 34 | logs: 35 | @echo "docker compose logs ..."; 36 | docker-compose -f src/docker-compose.yml logs -f; 37 | 38 | ## clean Clean up docker images for src. 39 | .PHONY: clean 40 | clean: 41 | @echo "docker image clean ..."; 42 | docker image prune -f; 43 | docker rmi src_produce src_consume1 src_consume2 -f; 44 | 45 | ## test Apache benchmark test for src. 46 | .PHONY: test 47 | test: 48 | @echo "apache benchmark test ..."; 49 | ab -n100 -c10 -T application/json -p test/ab_post_test.json http://127.0.0.1:9000/api/v1/data; 50 | 51 | ## kafka-up Docker compose up for kafka services. 52 | .PHONY: kafka-up 53 | kafka-up: 54 | @echo "docker compose up for kafka ..."; 55 | docker-compose -f kafka/docker-compose.yml up -d; 56 | 57 | ## kafka-down Docker compose down for kafka services. 58 | .PHONY: kafka-down 59 | kafka-down: 60 | @echo "docker compose down for kafka ..."; 61 | docker-compose -f kafka/docker-compose.yml down; 62 | 63 | ## kafka-clean Clean up log and data files for kafka services. 64 | .PHONY: kafka-clean 65 | kafka-clean: 66 | @echo "kafka dir clean ..."; 67 | cd kafka/kfk1/data/; ls|grep -v .gitkeep|xargs rm -rf; 68 | cd kafka/kfk2/data/; ls|grep -v .gitkeep|xargs rm -rf; 69 | cd kafka/kfk3/data/; ls|grep -v .gitkeep|xargs rm -rf; 70 | cd kafka/zk1/data/; ls|grep -v .gitkeep|xargs rm -rf; 71 | cd kafka/zk1/log/; ls|grep -v .gitkeep|xargs rm -rf; 72 | cd kafka/zk2/data/; ls|grep -v .gitkeep|xargs rm -rf; 73 | cd kafka/zk2/log/; ls|grep -v .gitkeep|xargs rm -rf; 74 | cd kafka/zk3/data/; ls|grep -v .gitkeep|xargs rm -rf; 75 | cd kafka/zk3/log/; ls|grep -v .gitkeep|xargs rm -rf; 76 | 77 | ## kafka-test Check running state of the kafka service. 78 | .PHONY: kafka-test 79 | kafka-test: 80 | @echo "check kafka run status ..."; 81 | kafkacat -L -b $(KFKADDR); 82 | 83 | ## help print this help message and exit. 84 | .PHONY: help 85 | help: Makefile 86 | @echo "" 87 | @echo "Choose a command run in "$(PROJECTNAME)":" 88 | @echo "" 89 | @echo "Usage: make [target]" 90 | @echo "" 91 | @echo "Valid target values are:" 92 | @echo "" 93 | @sed -n 's/^## //p' $< -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # :beetle: kafka_cluster_example 2 | 3 |

4 | 5 |

6 |

7 | 8 | 9 |

10 | 11 | --- 12 | 13 | ### 项目获取： 14 | ``` sh 15 | $ git clone --depth=1 https://github.com/ErikJiang/kafka_cluster_example.git 16 | ``` 17 | 18 | ### 项目依赖工具： 19 | * [Make](https://www.gnu.org/software/make/) 20 | * [Kafkacat](https://github.com/edenhill/kafkacat) 21 | * [ApacheBench](https://httpd.apache.org/docs/2.4/programs/ab.html) 22 | 23 | ### 支持 make 构建： 24 | ``` 25 | $ make 26 | 27 | Choose a command run in kafka_cluster_example: 28 | 29 | Usage: make [target] 30 | 31 | Valid target values are: 32 | 33 | vendor Auto generate go vendor dir. 34 | up Docker compose up for src. 35 | down Docker compose down for src. 36 | ps Docker compose ps for src. 37 | logs Docker compose logs for src. 38 | clean Clean up docker images for src. 39 | test Apache benchmark test for src. 40 | kafka-up Docker compose up for kafka services. 41 | kafka-down Docker compose down for kafka services. 42 | kafka-clean Clean up log and data files for kafka services. 43 | kafka-test Check running state of the kafka service. 44 | help print this help message and exit. 45 | ``` 46 | 47 | --- 48 | 49 | ### 1. 配置 hosts 域名 50 | 使用 `ifconfig -a` 查看本地IP地址； 51 | 52 | 配置 /etc/hosts 文件，将域名 kfk1、kfk2、kfk3 映射到当前本地 IP 地址，例如； 53 | 54 | ``` sh 55 | # 假设本地IP为: 192.168.0.166 56 | 192.168.0.166 kfk1 kfk2 kfk3 57 | ``` 58 | 59 | ### 2. 构建 Kafka 集群 60 | 61 | ``` sh 62 | # docker compose 构建方式： 63 | $ docker-compose -f kafka/docker-compose.yml up -d 64 | 65 | # 或使用 make 构建方式： 66 | $ make kafka-up 67 | ``` 68 | 69 | 如若在构建下载过程中，出现等待连接超时，可尝试在 docker 的 `daemon.json` 中添加注册镜像： 70 | ``` json 71 | { 72 | "registry-mirrors":["https://docker.mirrors.ustc.edu.cn"] 73 | } 74 | ``` 75 | 76 | 若构建完成，可使用 kafkacat 检测服务是否正常运行： 77 | ``` sh 78 | # 直接进行检测验证： 79 | $ kafkacat -L -b kfk1:19092 80 | 81 | # 或使用 make 方式验证： 82 | $ make kafka-test 83 | ``` 84 | 85 | ### 3. 构建 Produce & Consume 服务 86 | 87 | 为 produce 和 consume 生成 vendor 依赖： 88 | ``` sh 89 | $ make vendor 90 | ``` 91 | 92 | 构建 Produce & Consume Docker 服务 93 | ``` sh 94 | # 使用 docker compose 直接构建方式： 95 | $ docker-compose -f src/docker-compose.yml up -d 96 | 97 | # 或者使用 make 构建方式： 98 | $ make up 99 | ``` 100 | 101 | ### 4. 最终测试 102 | 103 | 使用 ApacheBench 进行并发测试（并发数为10，总计100个请求）： 104 | ``` sh 105 | # 直接使用 ab 命令进行测试： 106 | $ ab -n100 -c10 -T application/json -p test/ab_post_test.json http://127.0.0.1:9000/api/v1/data 107 | 108 | # 或者使用 make 方式测试： 109 | $ make test 110 | ``` 111 | 112 | --- 113 | 114 | > 文档详见：[Wiki](https://github.com/ErikJiang/kafka_cluster_example/wiki) -------------------------------------------------------------------------------- /kafka/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | zk1: 4 | image: confluentinc/cp-zookeeper:5.1.0 5 | container_name: zk1 6 | ports: 7 | - "12181:12181" 8 | environment: 9 | ZOOKEEPER_SERVER_ID: 1 10 | ZOOKEEPER_CLIENT_PORT: 12181 11 | ZOOKEEPER_TICK_TIME: 2000 12 | ZOOKEEPER_INIT_LIMIT: 5 13 | ZOOKEEPER_SYNC_LIMIT: 2 14 | ZOOKEEPER_SERVERS: zk1:12888:13888;zk2:22888:23888;zk3:32888:33888 15 | volumes: 16 | - ./zk1/data:/var/lib/zookeeper/data 17 | - ./zk1/log:/var/lib/zookeeper/log 18 | 19 | zk2: 20 | image: confluentinc/cp-zookeeper:5.1.0 21 | container_name: zk2 22 | ports: 23 | - "22181:22181" 24 | environment: 25 | ZOOKEEPER_SERVER_ID: 2 26 | ZOOKEEPER_CLIENT_PORT: 22181 27 | ZOOKEEPER_TICK_TIME: 2000 28 | ZOOKEEPER_INIT_LIMIT: 5 29 | ZOOKEEPER_SYNC_LIMIT: 2 30 | ZOOKEEPER_SERVERS: zk1:12888:13888;zk2:22888:23888;zk3:32888:33888 31 | volumes: 32 | - ./zk2/data:/var/lib/zookeeper/data 33 | - ./zk2/log:/var/lib/zookeeper/log 34 | 35 | zk3: 36 | image: confluentinc/cp-zookeeper:5.1.0 37 | container_name: zk3 38 | ports: 39 | - "32181:32181" 40 | environment: 41 | ZOOKEEPER_SERVER_ID: 3 42 | ZOOKEEPER_CLIENT_PORT: 32181 43 | ZOOKEEPER_TICK_TIME: 2000 44 | ZOOKEEPER_INIT_LIMIT: 5 45 | ZOOKEEPER_SYNC_LIMIT: 2 46 | ZOOKEEPER_SERVERS: zk1:12888:13888;zk2:22888:23888;zk3:32888:33888 47 | volumes: 48 | - ./zk3/data:/var/lib/zookeeper/data 49 | - ./zk3/log:/var/lib/zookeeper/log 50 | 51 | kfk1: 52 | image: confluentinc/cp-kafka:5.1.0 53 | container_name: kfk1 54 | ports: 55 | - "19092:19092" 56 | expose: 57 | - "19092" 58 | depends_on: 59 | - zk1 60 | - zk2 61 | - zk3 62 | environment: 63 | KAFKA_BROKER_ID: 1 64 | KAFKA_ZOOKEEPER_CONNECT: zk1:12181,zk2:22181,zk3:32181 65 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kfk1:19092 66 | volumes: 67 | - ./kfk1/data:/var/lib/kafka/data 68 | 69 | kfk2: 70 | image: confluentinc/cp-kafka:5.1.0 71 | container_name: kfk2 72 | ports: 73 | - "29092:29092" 74 | expose: 75 | - "29092" 76 | depends_on: 77 | - zk1 78 | - zk2 79 | - zk3 80 | environment: 81 | KAFKA_BROKER_ID: 2 82 | KAFKA_ZOOKEEPER_CONNECT: zk1:12181,zk2:22181,zk3:32181 83 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kfk2:29092 84 | volumes: 85 | - ./kfk2/data:/var/lib/kafka/data 86 | 87 | kfk3: 88 | image: confluentinc/cp-kafka:5.1.0 89 | container_name: kfk3 90 | ports: 91 | - "39092:39092" 92 | expose: 93 | - "39092" 94 | depends_on: 95 | - zk1 96 | - zk2 97 | - zk3 98 | environment: 99 | KAFKA_BROKER_ID: 3 100 | KAFKA_ZOOKEEPER_CONNECT: zk1:12181,zk2:22181,zk3:32181 101 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kfk3:39092 102 | volumes: 103 | - ./kfk3/data:/var/lib/kafka/data 104 | -------------------------------------------------------------------------------- /src/consume/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "os/signal" 7 | "sort" 8 | "strings" 9 | "sync" 10 | "syscall" 11 | 12 | "github.com/Shopify/sarama" 13 | "github.com/bsm/sarama-cluster" 14 | "github.com/rs/zerolog/log" 15 | "github.com/urfave/cli" 16 | ) 17 | 18 | func main() { 19 | app := cli.NewApp() 20 | app.Name = "kafka Tutorial Consume Commandline" 21 | app.Usage = "Run Consume" 22 | app.Version = "1.0.0" 23 | app.Flags = args() 24 | sort.Sort(cli.FlagsByName(app.Flags)) 25 | app.Action = action 26 | err := app.Run(os.Args) 27 | if err != nil { 28 | log.Error().Msgf("error: %v", err) 29 | } 30 | log.Debug().Msgf("args: %v", os.Args) 31 | } 32 | 33 | // args 命令行参数定义 34 | func args() []cli.Flag { 35 | return []cli.Flag{ 36 | cli.StringFlag{ 37 | Name: "kafka-brokers, kb", 38 | Value: "kfk1:19092,kfk2:29092,kfk3:39092", 39 | Usage: "Kafka brokers in comma separated value", 40 | EnvVar: "KAFKA_BROKERS", 41 | }, 42 | cli.StringFlag{ 43 | Name: "kafka-consumer-group, kcg", 44 | Value: "consumer-group", 45 | Usage: "Kafka consumer group", 46 | EnvVar: "KAFKA_CONSUMER_GROUP_ID", 47 | }, 48 | cli.StringFlag{ 49 | Name: "kafka-topic, kt", 50 | Value: "hello", 51 | Usage: "Kafka topic to push", 52 | EnvVar: "KAFKA_TOPIC", 53 | }, 54 | } 55 | } 56 | 57 | // action 创建 Kafka 生产者并启动路由服务 58 | func action(c *cli.Context) error { 59 | log.Info().Msg("kafka tutorial consume.") 60 | log.Info().Msg("(c) Erik 2019") 61 | 62 | brokerUrls := c.String("kafka-brokers") 63 | topic := c.String("kafka-topic") 64 | consumerGroup := c.String("kafka-consumer-group") 65 | 66 | log.Info().Msgf("kafka-brokers: %s", brokerUrls) 67 | log.Info().Msgf("kafka-topic: %s", topic) 68 | log.Info().Msgf("kafka-consumer-group: %s", consumerGroup) 69 | 70 | wg := &sync.WaitGroup{} 71 | wg.Add(1) 72 | go clusterConsumer(wg, strings.Split(brokerUrls, ","), []string{topic}, consumerGroup) 73 | wg.Wait() 74 | 75 | return nil 76 | } 77 | 78 | // 支持brokers cluster的消费者 79 | func clusterConsumer(wg *sync.WaitGroup, brokers, topics []string, groupID string) { 80 | defer wg.Done() 81 | config := cluster.NewConfig() 82 | config.Consumer.Return.Errors = true 83 | config.Group.Return.Notifications = true 84 | config.Version = sarama.V2_1_0_0 85 | config.Consumer.Offsets.Initial = sarama.OffsetNewest 86 | 87 | // 初始化消费者 88 | consumer, err := cluster.NewConsumer(brokers, groupID, topics, config) 89 | if err != nil { 90 | log.Debug().Msgf("%s: sarama.NewSyncProducer err, message=%s \n", groupID, err) 91 | return 92 | } 93 | defer consumer.Close() 94 | 95 | // 捕获终止中断信号触发程序退出 96 | signals := make(chan os.Signal, 1) 97 | signal.Notify(signals, os.Interrupt, syscall.SIGTERM) 98 | 99 | // 消费错误信息 100 | go func() { 101 | for err := range consumer.Errors() { 102 | log.Debug().Msgf("%s:Error: %s\n", groupID, err.Error()) 103 | } 104 | }() 105 | 106 | // 消费通知信息 107 | go func() { 108 | for ntf := range consumer.Notifications() { 109 | log.Debug().Msgf("%s:Rebalanced: %v \n", groupID, ntf) 110 | } 111 | }() 112 | 113 | // 消费信息及监听信号 114 | var successes int 115 | Loop: 116 | for { 117 | select { 118 | case msg, ok := <-consumer.Messages(): 119 | if ok { 120 | value := struct { 121 | Text string `form:"text" json:"text"` 122 | }{} 123 | err := json.Unmarshal(msg.Value, &value) 124 | if err != nil { 125 | log.Error().Msgf("consume message json format error, %v", err) 126 | break Loop 127 | } 128 | log.Debug().Msgf("GroupID: %s, Topic: %s, Partition: %d, Offset: %d, Key: %s, Value: %s", 129 | groupID, msg.Topic, msg.Partition, msg.Offset, msg.Key, value.Text) 130 | consumer.MarkOffset(msg, "") // 标记信息为已处理 131 | successes++ 132 | } 133 | case <-signals: 134 | break Loop 135 | } 136 | } 137 | log.Debug().Msgf("%s consume %d messages", groupID, successes) 138 | } 139 | -------------------------------------------------------------------------------- /src/consume/go.sum: -------------------------------------------------------------------------------- 1 | github.com/DataDog/zstd v1.3.5 h1:DtpNbljikUepEPD16hD4LvIcmhnhdLTiW/5pHgbmp14= 2 | github.com/DataDog/zstd v1.3.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= 3 | github.com/Shopify/sarama v1.20.1 h1:Bb0h3I++r4eX333Y0uZV2vwUXepJbt6ig05TUU1qt9I= 4 | github.com/Shopify/sarama v1.20.1/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= 5 | github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= 6 | github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= 7 | github.com/bsm/sarama-cluster v2.1.15+incompatible h1:RkV6WiNRnqEEbp81druK8zYhmnIgdOjqSVi0+9Cnl2A= 8 | github.com/bsm/sarama-cluster v2.1.15+incompatible/go.mod h1:r7ao+4tTNXvWm+VRpRJchr2kQhqxgmAp2iEX5W96gMM= 9 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 10 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU= 12 | github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= 13 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= 14 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 15 | github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= 16 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 17 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 18 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 19 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 20 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 21 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= 22 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 23 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 24 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 25 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 26 | github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= 27 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 28 | github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= 29 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 30 | github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= 31 | github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 32 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= 33 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 34 | github.com/rs/zerolog v1.11.0 h1:DRuq/S+4k52uJzBQciUcofXx45GrMC6yrEbb/CoK6+M= 35 | github.com/rs/zerolog v1.11.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= 36 | github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= 37 | github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= 38 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= 39 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 40 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= 41 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 42 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= 43 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 44 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 45 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 46 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 47 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 48 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 49 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 50 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 51 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 52 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 53 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 54 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 55 | -------------------------------------------------------------------------------- /src/produce/go.sum: -------------------------------------------------------------------------------- 1 | github.com/DataDog/zstd v1.3.5 h1:DtpNbljikUepEPD16hD4LvIcmhnhdLTiW/5pHgbmp14= 2 | github.com/DataDog/zstd v1.3.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= 3 | github.com/Shopify/sarama v1.20.1 h1:Bb0h3I++r4eX333Y0uZV2vwUXepJbt6ig05TUU1qt9I= 4 | github.com/Shopify/sarama v1.20.1/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= 5 | github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= 6 | github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= 7 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU= 10 | github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= 11 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= 12 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 13 | github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= 14 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 15 | github.com/gin-contrib/sse v0.0.0-20190125020943-a7658810eb74 h1:FaI7wNyesdMBSkIRVUuEEYEvmzufs7EqQvRAxfEXGbQ= 16 | github.com/gin-contrib/sse v0.0.0-20190125020943-a7658810eb74/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= 17 | github.com/gin-gonic/gin v1.3.0 h1:kCmZyPklC0gVdL728E6Aj20uYBJV93nj/TkwBTKhFbs= 18 | github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= 19 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 20 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 21 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= 22 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 23 | github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE= 24 | github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 25 | github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= 26 | github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 27 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 28 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 29 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 30 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 31 | github.com/pierrec/lz4 v2.0.7+incompatible h1:CbATlhU1ACG9h9JIvjWcdRjNOyxpSe/zZww7SPj8ruk= 32 | github.com/pierrec/lz4 v2.0.7+incompatible/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= 33 | github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= 34 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 35 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 36 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= 37 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 38 | github.com/rs/zerolog v1.11.0 h1:DRuq/S+4k52uJzBQciUcofXx45GrMC6yrEbb/CoK6+M= 39 | github.com/rs/zerolog v1.11.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= 40 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 41 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 42 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 43 | github.com/ugorji/go v1.1.2 h1:JON3E2/GPW2iDNGoSAusl1KDf5TRQ8k8q7Tp097pZGs= 44 | github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= 45 | github.com/ugorji/go/codec v0.0.0-20190128213124-ee1426cffec0 h1:Q3Bh5Dwzek5LreV9l86IftyLaexgU1mag9WNntbAW9c= 46 | github.com/ugorji/go/codec v0.0.0-20190128213124-ee1426cffec0/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= 47 | github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= 48 | github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= 49 | golang.org/x/net v0.0.0-20190110200230-915654e7eabc h1:Yx9JGxI1SBhVLFjpAkWMaO1TF+xyqtHLjZpvQboJGiM= 50 | golang.org/x/net v0.0.0-20190110200230-915654e7eabc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 51 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= 52 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 53 | golang.org/x/sys v0.0.0-20190114130336-2be517255631 h1:g/5trXm6f9Tm+ochb21RlFNnF63lt+elB9hVBqtPu5Y= 54 | golang.org/x/sys v0.0.0-20190114130336-2be517255631/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 55 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 56 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 57 | gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= 58 | gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= 59 | gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ= 60 | gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= 61 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 62 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 63 | -------------------------------------------------------------------------------- /src/produce/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "crypto/sha1" 6 | "encoding/hex" 7 | "encoding/json" 8 | "fmt" 9 | "net/http" 10 | "os" 11 | "os/signal" 12 | "sort" 13 | "strings" 14 | "syscall" 15 | "time" 16 | 17 | "github.com/Shopify/sarama" 18 | "github.com/gin-gonic/gin" 19 | "github.com/rs/zerolog/log" 20 | "github.com/urfave/cli" 21 | ) 22 | 23 | func main() { 24 | app := cli.NewApp() 25 | app.Name = "kafka Tutorial Produce Commandline" 26 | app.Usage = "Run Produce" 27 | app.Version = "1.0.0" 28 | app.Flags = args() 29 | sort.Sort(cli.FlagsByName(app.Flags)) 30 | app.Action = action 31 | app.Run(os.Args) 32 | } 33 | 34 | // args 命令行参数定义 35 | func args() []cli.Flag { 36 | return []cli.Flag{ 37 | cli.StringFlag{ 38 | Name: "listen-address, la", 39 | Value: "0.0.0.0:9000", 40 | Usage: "Listen address for api", 41 | EnvVar: "LISTEN_ADDRESS", 42 | }, 43 | cli.StringFlag{ 44 | Name: "kafka-brokers, kb", 45 | Value: "kfk1:19092,kfk2:29092,kfk3:39092", 46 | Usage: "Kafka brokers in comma separated value", 47 | EnvVar: "KAFKA_BROKERS", 48 | }, 49 | cli.StringFlag{ 50 | Name: "kafka-topic, kt", 51 | Value: "hello", 52 | Usage: "Kafka topic to push", 53 | EnvVar: "KAFKA_TOPIC", 54 | }, 55 | } 56 | } 57 | 58 | // action 创建 Kafka 生产者并启动路由服务 59 | func action(c *cli.Context) error { 60 | log.Info().Msg("kafka tutorial produce.") 61 | log.Info().Msg("(c) Erik 2019") 62 | 63 | listenAddr := c.String("listen-address") 64 | kafkaBrokers := c.String("kafka-brokers") 65 | topic := c.String("kafka-topic") 66 | 67 | log.Info().Msgf("listen-address: %s", listenAddr) 68 | log.Info().Msgf("kafka-brokers: %s", kafkaBrokers) 69 | log.Info().Msgf("kafka-topic: %s", topic) 70 | 71 | brokerUrls := strings.Split(kafkaBrokers, ",") 72 | 73 | config := sarama.NewConfig() 74 | 75 | config.Producer.RequiredAcks = sarama.WaitForAll 76 | 77 | config.Producer.Partitioner = sarama.NewRandomPartitioner 78 | 79 | config.Producer.Return.Successes = true 80 | 81 | config.Producer.Return.Errors = true 82 | 83 | config.Version = sarama.V2_1_0_0 84 | 85 | log.Info().Msg("start make topic") 86 | err := createTopic(config, brokerUrls[0], topic) 87 | if err != nil { 88 | log.Error().Msgf("%v", err) 89 | return err 90 | } 91 | 92 | log.Info().Msg("start make producer") 93 | producer, err := sarama.NewAsyncProducer(brokerUrls, config) 94 | if err != nil { 95 | log.Error().Msgf("%v", err) 96 | return err 97 | } 98 | defer producer.AsyncClose() 99 | 100 | log.Info().Msgf("starting server at %s", listenAddr) 101 | errChan := make(chan error, 1) 102 | go func(p sarama.AsyncProducer) { 103 | errChan <- httpServer(p, topic, listenAddr) 104 | }(producer) 105 | 106 | var signalChan = make(chan os.Signal, 1) 107 | signal.Notify(signalChan, os.Interrupt, syscall.SIGTERM) 108 | 109 | Loop: 110 | for { 111 | select { 112 | case succ := <-producer.Successes(): 113 | log.Info().Msgf("success: offset: %d, timestamp: %s, partitions: %d", 114 | succ.Offset, succ.Timestamp.String(), succ.Partition) 115 | case fail := <-producer.Errors(): 116 | log.Error().Err(fail).Msg("fail while produce message, exiting...") 117 | break Loop 118 | case <-signalChan: 119 | log.Info().Msg("got an interrupt, exiting...") 120 | break Loop 121 | case err := <-errChan: 122 | log.Error().Err(err).Msg("error while runing api, exiting...") 123 | break Loop 124 | } 125 | } 126 | return nil 127 | } 128 | 129 | // createTopic 创建 Topic 130 | func createTopic(config *sarama.Config, brokerURL, topicName string) error { 131 | broker := sarama.NewBroker(brokerURL) 132 | broker.Open(config) 133 | yes, err := broker.Connected() 134 | if err != nil { 135 | log.Error().Msgf("broker connect fail, %v", err) 136 | return err 137 | } 138 | log.Debug().Msgf("broker connect status: %v", yes) 139 | 140 | topicDetail := &sarama.TopicDetail{ 141 | NumPartitions: 2, 142 | ReplicationFactor: 1, 143 | ConfigEntries: make(map[string]*string), 144 | } 145 | 146 | topicDetails := make(map[string]*sarama.TopicDetail) 147 | topicDetails[topicName] = topicDetail 148 | 149 | request := sarama.CreateTopicsRequest{ 150 | Timeout: time.Second * 15, 151 | TopicDetails: topicDetails, 152 | } 153 | 154 | response, err := broker.CreateTopics(&request) 155 | if err != nil { 156 | log.Error().Msgf("create topics fail, %v", err) 157 | return err 158 | } 159 | log.Debug().Msgf("response length: %d", len(response.TopicErrors)) 160 | for key, val := range response.TopicErrors { 161 | log.Debug().Msgf("Key is %s", key) 162 | log.Debug().Msgf("Val is %#v", val.Err.Error()) 163 | log.Debug().Msgf("ValMsg is %#v", val.ErrMsg) 164 | } 165 | log.Info().Msgf("create topics response: %v", response) 166 | broker.Close() 167 | return nil 168 | } 169 | 170 | // httpServer 启动 HTTP 服务 171 | func httpServer(producer sarama.AsyncProducer, topicName, listenAddr string) error { 172 | gin.SetMode(gin.ReleaseMode) 173 | 174 | router := gin.New() 175 | router.POST("/api/v1/data", func(ctx *gin.Context) { 176 | parent := context.Background() 177 | defer parent.Done() 178 | 179 | form := &struct { 180 | Text string `form:"text" json:"text"` 181 | }{} 182 | 183 | err := ctx.ShouldBindJSON(form) 184 | if err != nil { 185 | ctx.JSON(http.StatusBadRequest, map[string]interface{}{ 186 | "error": map[string]interface{}{ 187 | "message": fmt.Sprintf("error while bind request param: %s", err.Error()), 188 | }, 189 | }) 190 | ctx.Abort() 191 | return 192 | } 193 | formInBytes, err := json.Marshal(form) 194 | if err != nil { 195 | ctx.JSON(http.StatusUnprocessableEntity, map[string]interface{}{ 196 | "error": map[string]interface{}{ 197 | "message": fmt.Sprintf("error while marshalling json: %s", err.Error()), 198 | }, 199 | }) 200 | ctx.Abort() 201 | return 202 | } 203 | 204 | // send message to kafka 205 | msg := &sarama.ProducerMessage{ 206 | Topic: topicName, 207 | Key: sarama.StringEncoder(MakeSha1(form.Text)), 208 | Value: sarama.ByteEncoder(formInBytes), 209 | Timestamp: time.Now(), 210 | } 211 | producer.Input() <- msg 212 | 213 | ctx.JSON(http.StatusOK, map[string]interface{}{ 214 | "success": true, 215 | "message": "success push data into kafka", 216 | "data": form, 217 | }) 218 | }) 219 | return router.Run(listenAddr) 220 | } 221 | 222 | // MakeSha1 计算字符串的 sha1 hash 值 223 | func MakeSha1(source string) string { 224 | sha1Hash := sha1.New() 225 | sha1Hash.Write([]byte(source)) 226 | return hex.EncodeToString(sha1Hash.Sum(nil)) 227 | } 228 | --------------------------------------------------------------------------------