├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── consumer ├── handler │ ├── comment.go │ ├── handler.go │ └── like.go └── main.go ├── docker-compose.yml ├── go.mod ├── go.sum ├── media └── flow.png ├── packages ├── event │ ├── comment.go │ ├── event.go │ └── like.go └── utils │ └── utils.go └── producer └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.13-alpine as builder 2 | ENV GOPATH="$HOME/go" 3 | RUN apk --no-cache add git 4 | WORKDIR $GOPATH/src 5 | 6 | COPY . $GOPATH/src 7 | 8 | RUN go get -d -v golang.org/x/net/html 9 | RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o app consumer/*.go 10 | 11 | 12 | FROM alpine:latest 13 | RUN apk --no-cache add ca-certificates 14 | WORKDIR /root/ 15 | 16 | COPY --from=builder $HOME/go/src/app . 17 | CMD ["./app"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Filipe Alves 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | docker: 2 | @docker-compose down 3 | @docker-compose build 4 | @docker-compose up -d 5 | 6 | dockerdown: 7 | @docker-compose down 8 | 9 | build: 10 | @echo "---- Building Application ----" 11 | @go build -o consumer consumer/*.go 12 | @go build -o producer producer/*.go 13 | 14 | consume: 15 | @echo "---- Running Consumer ----" 16 | @export REDIS_HOST=localhost 17 | @export STREAM=events 18 | @export GROUP=GroupOne 19 | @go run consumer/*.go 20 | 21 | run: 22 | @echo "---- Running Producer ----" 23 | @export REDIS_HOST=localhost 24 | @export STREAM=events 25 | @go run producer/*.go -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-redis-streams 2 | Demo using messaging with Redis Streams in Golang 3 | 4 | **Utilizando mensageria na pratica com Redis streams e Golang** 5 | 6 | [![YouTube Video Explanation](http://img.youtube.com/vi/Kc-tcrP0c10/0.jpg)](http://www.youtube.com/watch?v=Kc-tcrP0c10 "Utilizando mensageria na pratica com Redis streams e Golang") 7 | 8 | # Redis 9 | 10 | **O que e?** 11 | 12 | Redis e um Banco de dados não relacional OpenSource, que tem dentro de sua estrutura o armazenamento chave-valor. 13 | O Redis tem estratégias para guardar os dados em memória e em disco, garantindo resposta rápida e persistência de dados. Os principais casos de uso do Redis incluem cache, gerenciamento de sessões, PUB/SUB. 14 | 15 | # Redis Streams para Mensageria (ou Messaging) 16 | 17 | ![Design of flow](/media/flow.png) 18 | 19 | **Pontos Positivos** 20 | 21 | - Suporta Topicos e Filas 22 | - Persistencia em disco (através dos arquivos RDB) 23 | - Alta disponibilidade (com Clusterizacao) 24 | - Alto Throughput 25 | - Permite Reprocessamento 26 | - Possui Consumer Groups 27 | - Latencia minima 28 | - Nao necessita de zookeper 29 | - Ocupa muito menos recursos em relacao ao (Kafka/RabbitMQ) 30 | 31 | **Pontos Negativos** 32 | 33 | - Nao garante ordem de entrega (ainda) 34 | - Msgs processadas com error nao retorna para redistribuicao 35 | 36 | 37 | # Links 38 | 39 | https://www.youtube.com/watch?v=JpeHIbzmGP4 40 | 41 | https://redis.io/topics/streams-intro 42 | 43 | https://redislabs.com/blog/use-redis-streams-apps/ 44 | 45 | https://redislabs.com/blog/getting-started-with-redis-streams-and-java/ 46 | 47 | -------------------------------------------------------------------------------- /consumer/handler/comment.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | evt "github.com/felipeagger/go-redis-streams/packages/event" 8 | ) 9 | 10 | type commentHandler struct { 11 | } 12 | 13 | //NewCommentHandler ... 14 | func NewCommentHandler() Handler { 15 | return &commentHandler{} 16 | } 17 | 18 | func (h *commentHandler) Handle(e evt.Event, retry bool) error { 19 | event, ok := e.(*evt.CommentEvent) 20 | 21 | if !ok { 22 | return fmt.Errorf("incorrect event type") 23 | } 24 | 25 | if event.UserID == 5 && !retry { 26 | return errors.New("Falhou") 27 | } 28 | 29 | fmt.Printf("processed event %+v UserID: %v Comment:%v \n", event, event.UserID, event.Comment) 30 | 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /consumer/handler/handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/felipeagger/go-redis-streams/packages/event" 7 | evt "github.com/felipeagger/go-redis-streams/packages/event" 8 | ) 9 | 10 | //HandlerFactory ... 11 | func HandlerFactory() func(t event.Type) Handler { 12 | 13 | return func(t event.Type) Handler { 14 | switch t { 15 | case event.LikeType: 16 | return NewLikeHandler() 17 | case event.CommentType: 18 | return NewCommentHandler() 19 | default: 20 | return NewDefaultHandler() 21 | } 22 | } 23 | } 24 | 25 | type Handler interface { 26 | Handle(e event.Event, retry bool) error 27 | } 28 | 29 | type defaultHandler struct { 30 | } 31 | 32 | //NewViewHandler ... 33 | func NewDefaultHandler() Handler { 34 | return &defaultHandler{} 35 | } 36 | 37 | func (h *defaultHandler) Handle(e evt.Event, retry bool) error { 38 | fmt.Printf("undefined event %+v\n", e) 39 | return nil 40 | } 41 | -------------------------------------------------------------------------------- /consumer/handler/like.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | evt "github.com/felipeagger/go-redis-streams/packages/event" 8 | ) 9 | 10 | type likeHandler struct { 11 | } 12 | 13 | //NewLikeHandler ... 14 | func NewLikeHandler() Handler { 15 | return &likeHandler{} 16 | } 17 | 18 | func (h *likeHandler) Handle(e evt.Event, retry bool) error { 19 | event, ok := e.(*evt.LikeEvent) 20 | 21 | if !ok { 22 | return fmt.Errorf("incorrect event type") 23 | } 24 | 25 | if event.UserID == 5 && !retry { 26 | return errors.New("Falhou") 27 | } 28 | 29 | fmt.Printf("completed like %+v UserID: %v\n", event, event.UserID) 30 | 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /consumer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "os/signal" 8 | "strings" 9 | "sync" 10 | "syscall" 11 | "time" 12 | 13 | "github.com/felipeagger/go-redis-streams/consumer/handler" 14 | "github.com/felipeagger/go-redis-streams/packages/event" 15 | "github.com/felipeagger/go-redis-streams/packages/utils" 16 | "github.com/go-redis/redis/v7" 17 | uuid "github.com/satori/go.uuid" 18 | ) 19 | 20 | var ( 21 | waitGrp sync.WaitGroup 22 | client *redis.Client 23 | start string = ">" 24 | streamName string = os.Getenv("STREAM") 25 | consumerGroup string = os.Getenv("GROUP") 26 | consumerName string = uuid.NewV4().String() 27 | ) 28 | 29 | func init() { 30 | var err error 31 | client, err = utils.NewRedisClient() 32 | if err != nil { 33 | panic(err) 34 | } 35 | 36 | createConsumerGroup() 37 | } 38 | 39 | func main() { 40 | fmt.Printf("Initializing Consumer:%v\nConsumerGroup: %v \nStream: %v\n", 41 | consumerName, consumerGroup, streamName) 42 | 43 | go consumeEvents() 44 | go consumePendingEvents() 45 | 46 | //Gracefully disconection 47 | chanOS := make(chan os.Signal) 48 | signal.Notify(chanOS, syscall.SIGINT, syscall.SIGTERM) 49 | <-chanOS 50 | 51 | waitGrp.Wait() 52 | client.Close() 53 | } 54 | 55 | func createConsumerGroup() { 56 | 57 | if _, err := client.XGroupCreateMkStream(streamName, consumerGroup, "0").Result(); err != nil { 58 | 59 | if !strings.Contains(fmt.Sprint(err), "BUSYGROUP") { 60 | fmt.Printf("Error on create Consumer Group: %v ...\n", consumerGroup) 61 | panic(err) 62 | } 63 | 64 | } 65 | } 66 | 67 | // start consume events 68 | func consumeEvents() { 69 | 70 | for { 71 | func() { 72 | fmt.Println("new round ", time.Now().Format(time.RFC3339)) 73 | 74 | streams, err := client.XReadGroup(&redis.XReadGroupArgs{ 75 | Streams: []string{streamName, start}, 76 | Group: consumerGroup, 77 | Consumer: consumerName, 78 | Count: 10, 79 | Block: 0, 80 | }).Result() 81 | 82 | if err != nil { 83 | log.Printf("err on consume events: %+v\n", err) 84 | return 85 | } 86 | 87 | for _, stream := range streams[0].Messages { 88 | waitGrp.Add(1) 89 | go processStream(stream, false, handler.HandlerFactory()) 90 | } 91 | waitGrp.Wait() 92 | }() 93 | } 94 | 95 | } 96 | 97 | func consumePendingEvents() { 98 | 99 | ticker := time.Tick(time.Second * 30) 100 | for { 101 | select { 102 | case <-ticker: 103 | 104 | func() { 105 | 106 | var streamsRetry []string 107 | pendingStreams, err := client.XPendingExt(&redis.XPendingExtArgs{ 108 | Stream: streamName, 109 | Group: consumerGroup, 110 | Start: "0", 111 | End: "+", 112 | Count: 10, 113 | //Consumer string 114 | }).Result() 115 | 116 | if err != nil { 117 | panic(err) 118 | } 119 | 120 | for _, stream := range pendingStreams { 121 | streamsRetry = append(streamsRetry, stream.ID) 122 | } 123 | 124 | if len(streamsRetry) > 0 { 125 | 126 | streams, err := client.XClaim(&redis.XClaimArgs{ 127 | Stream: streamName, 128 | Group: consumerGroup, 129 | Consumer: consumerName, 130 | Messages: streamsRetry, 131 | MinIdle: 30 * time.Second, 132 | }).Result() 133 | 134 | if err != nil { 135 | log.Printf("err on process pending: %+v\n", err) 136 | return 137 | } 138 | 139 | for _, stream := range streams { 140 | waitGrp.Add(1) 141 | go processStream(stream, true, handler.HandlerFactory()) 142 | } 143 | waitGrp.Wait() 144 | } 145 | 146 | fmt.Println("process pending streams at ", time.Now().Format(time.RFC3339)) 147 | 148 | }() 149 | 150 | } 151 | 152 | } 153 | 154 | } 155 | 156 | func processStream(stream redis.XMessage, retry bool, handlerFactory func(t event.Type) handler.Handler) { 157 | defer waitGrp.Done() 158 | 159 | typeEvent := stream.Values["type"].(string) 160 | newEvent, _ := event.New(event.Type(typeEvent)) 161 | 162 | err := newEvent.UnmarshalBinary([]byte(stream.Values["data"].(string))) 163 | if err != nil { 164 | fmt.Printf("error on unmarshal stream:%v\n", stream.ID) 165 | return 166 | } 167 | 168 | newEvent.SetID(stream.ID) 169 | 170 | h := handlerFactory(newEvent.GetType()) 171 | err = h.Handle(newEvent, retry) 172 | if err != nil { 173 | fmt.Printf("error on process event:%v\n", newEvent) 174 | fmt.Println(err) 175 | return 176 | } 177 | 178 | //client.XDel(streamName, stream.ID) 179 | client.XAck(streamName, consumerGroup, stream.ID) 180 | 181 | //time.Sleep(2 * time.Second) 182 | } 183 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | redis: 5 | image: redis:alpine 6 | container_name: redis-srv 7 | ports: 8 | - 6379:6379 9 | 10 | redis-commander: 11 | container_name: redis-commander-web 12 | hostname: redis-commander 13 | image: rediscommander/redis-commander:latest 14 | restart: always 15 | environment: 16 | - REDIS_HOSTS=local:redis:6379 17 | ports: 18 | - "8081:8081" 19 | 20 | consumer_one: 21 | container_name: go-consumer-one 22 | build: '.' 23 | environment: 24 | REDIS_HOST: redis 25 | STREAM: events 26 | GROUP: GroupOne 27 | depends_on: 28 | - redis 29 | restart: always 30 | 31 | consumer_two: 32 | container_name: go-consumer-two 33 | build: '.' 34 | environment: 35 | REDIS_HOST: redis 36 | STREAM: events 37 | GROUP: GroupTwo 38 | depends_on: 39 | - redis 40 | restart: always 41 | 42 | consumer_three: 43 | container_name: go-consumer-three 44 | build: '.' 45 | environment: 46 | REDIS_HOST: redis 47 | STREAM: events 48 | GROUP: GroupTwo 49 | depends_on: 50 | - redis 51 | restart: always -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felipeagger/go-redis-streams 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/go-redis/redis/v7 v7.0.0-beta.4 7 | github.com/satori/go.uuid v1.2.0 8 | github.com/vmihailenco/msgpack/v4 v4.2.1 9 | ) 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 4 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 5 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 6 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 7 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 8 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 9 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 10 | github.com/go-redis/redis/v7 v7.0.0-beta.4 h1:p6z7Pde69EGRWvlC++y8aFcaWegyrKHzOBGo0zUACTQ= 11 | github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s= 12 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 13 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 14 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 15 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 16 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 17 | github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= 18 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 19 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 20 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 21 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 22 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 23 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 24 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 25 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 26 | github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 27 | github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 28 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 29 | github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= 30 | github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= 31 | github.com/vmihailenco/msgpack/v4 v4.2.1 h1:9GCtYfnRH3FhtqfQTc1wYXmIB2UJd/K890yNae8kpOw= 32 | github.com/vmihailenco/msgpack/v4 v4.2.1/go.mod h1:Mu3B7ZwLd5nNOLVOKt9DecVl7IVg0xkDiEjk6CwMrww= 33 | github.com/vmihailenco/tagparser v0.1.0 h1:u6yzKTY6gW/KxL/K2NTEQUOSXZipyGiIRarGjJKmQzU= 34 | github.com/vmihailenco/tagparser v0.1.0/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= 35 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 36 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 37 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 38 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 39 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 40 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 41 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 42 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 43 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 44 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 45 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 46 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 47 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 48 | golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 49 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 50 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 51 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 52 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 53 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 54 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 55 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 56 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 57 | golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 58 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 59 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 60 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 61 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 62 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 63 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 64 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 65 | golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 66 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 67 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 68 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 69 | google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= 70 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 71 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 72 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 73 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 74 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 75 | google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE= 76 | google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 77 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 78 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 79 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 80 | gopkg.in/redis.v5 v5.2.9 h1:MNZYOLPomQzZMfpN3ZtD1uyJ2IDonTTlxYiV/pEApiw= 81 | gopkg.in/redis.v5 v5.2.9/go.mod h1:6gtv0/+A4iM08kdRfocWYB3bLX2tebpNtfKlFT6H4mY= 82 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 83 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 84 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 85 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 86 | -------------------------------------------------------------------------------- /media/flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felipeagger/go-redis-streams/7afda5e362d3fc10e742fd3b6caccf9e99683114/media/flow.png -------------------------------------------------------------------------------- /packages/event/comment.go: -------------------------------------------------------------------------------- 1 | package event 2 | 3 | import "github.com/vmihailenco/msgpack/v4" 4 | 5 | type CommentEvent struct { 6 | *Base 7 | UserID uint64 8 | Comment string 9 | } 10 | 11 | func (o *CommentEvent) MarshalBinary() (data []byte, err error) { 12 | return msgpack.Marshal(o) 13 | } 14 | 15 | func (o *CommentEvent) UnmarshalBinary(data []byte) error { 16 | return msgpack.Unmarshal(data, o) 17 | } 18 | -------------------------------------------------------------------------------- /packages/event/event.go: -------------------------------------------------------------------------------- 1 | package event 2 | 3 | import ( 4 | "encoding" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | type Type string 10 | 11 | const ( 12 | LikeType Type = "LikeType" 13 | CommentType Type = "CommentType" 14 | ) 15 | 16 | type Base struct { 17 | ID string 18 | Type Type 19 | DateTime time.Time 20 | Retry bool 21 | } 22 | 23 | // Event ... 24 | type Event interface { 25 | GetID() string 26 | GetType() Type 27 | GetDateTime() time.Time 28 | SetID(id string) 29 | encoding.BinaryMarshaler 30 | encoding.BinaryUnmarshaler 31 | } 32 | 33 | func New(t Type) (Event, error) { 34 | b := &Base{ 35 | Type: t, 36 | } 37 | 38 | switch t { 39 | 40 | case LikeType: 41 | return &LikeEvent{ 42 | Base: b, 43 | }, nil 44 | 45 | case CommentType: 46 | return &CommentEvent{ 47 | Base: b, 48 | }, nil 49 | 50 | } 51 | 52 | return nil, fmt.Errorf("type %v not supported", t) 53 | } 54 | 55 | func (o *Base) GetID() string { 56 | return o.ID 57 | } 58 | 59 | func (o *Base) SetID(id string) { 60 | o.ID = id 61 | } 62 | 63 | func (o *Base) GetType() Type { 64 | return o.Type 65 | } 66 | 67 | func (o *Base) GetDateTime() time.Time { 68 | return o.DateTime 69 | } 70 | 71 | func (o *Base) String() string { 72 | 73 | return fmt.Sprintf("id:%s type:%s", o.ID, o.Type) 74 | } 75 | -------------------------------------------------------------------------------- /packages/event/like.go: -------------------------------------------------------------------------------- 1 | package event 2 | 3 | import "github.com/vmihailenco/msgpack/v4" 4 | 5 | type LikeEvent struct { 6 | *Base 7 | UserID uint64 8 | } 9 | 10 | func (o *LikeEvent) MarshalBinary() (data []byte, err error) { 11 | return msgpack.Marshal(o) 12 | } 13 | 14 | func (o *LikeEvent) UnmarshalBinary(data []byte) error { 15 | return msgpack.Unmarshal(data, o) 16 | } 17 | -------------------------------------------------------------------------------- /packages/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/go-redis/redis/v7" 8 | ) 9 | 10 | //NewRedisClient create a new instace of client redis 11 | func NewRedisClient() (*redis.Client, error) { 12 | client := redis.NewClient(&redis.Options{ 13 | Addr: fmt.Sprintf("%s:6379", os.Getenv("REDIS_HOST")), 14 | Password: "", 15 | DB: 0, // use default DB 16 | }) 17 | 18 | _, err := client.Ping().Result() 19 | return client, err 20 | 21 | } 22 | -------------------------------------------------------------------------------- /producer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os" 7 | "time" 8 | 9 | evt "github.com/felipeagger/go-redis-streams/packages/event" 10 | "github.com/felipeagger/go-redis-streams/packages/utils" 11 | "github.com/go-redis/redis/v7" 12 | ) 13 | 14 | var ( 15 | streamName string = os.Getenv("STREAM") 16 | client *redis.Client 17 | ) 18 | 19 | func init() { 20 | var err error 21 | client, err = utils.NewRedisClient() 22 | if err != nil { 23 | panic(err) 24 | } 25 | } 26 | 27 | func main() { 28 | generateEvent() 29 | } 30 | 31 | func generateEvent() { 32 | var userID uint64 = 0 33 | for i := 0; i < 10; i++ { 34 | 35 | userID++ //uint64(rand.Intn(1000)) 36 | 37 | eventType := []evt.Type{evt.LikeType, evt.CommentType}[rand.Intn(2)] 38 | 39 | if eventType == evt.LikeType { 40 | 41 | newID, err := produceMsg(map[string]interface{}{ 42 | "type": string(eventType), 43 | "data": &evt.LikeEvent{ 44 | Base: &evt.Base{ 45 | Type: eventType, 46 | DateTime: time.Now(), 47 | }, 48 | UserID: userID, 49 | }, 50 | }) 51 | 52 | checkError(err, newID, string(eventType), userID) 53 | 54 | } else { 55 | 56 | comment := []string{"Go e Top!", "Go e demais!", "Go e vida!"}[rand.Intn(3)] 57 | 58 | newID, err := produceMsg(map[string]interface{}{ 59 | "type": string(eventType), 60 | "data": &evt.CommentEvent{ 61 | Base: &evt.Base{ 62 | Type: eventType, 63 | DateTime: time.Now(), 64 | }, 65 | UserID: userID, 66 | Comment: comment, 67 | }, 68 | }) 69 | 70 | checkError(err, newID, string(eventType), userID, comment) 71 | } 72 | 73 | } 74 | } 75 | 76 | func produceMsg(event map[string]interface{}) (string, error) { 77 | 78 | return client.XAdd(&redis.XAddArgs{ 79 | Stream: streamName, 80 | Values: event, 81 | }).Result() 82 | } 83 | 84 | func checkError(err error, newID, eventType string, userID uint64, comment ...string) { 85 | if err != nil { 86 | fmt.Printf("produce event error:%v\n", err) 87 | } else { 88 | 89 | if len(comment) > 0 { 90 | fmt.Printf("produce event success Type:%v UserID:%v Comment:%v offset:%v\n", 91 | string(eventType), userID, comment, newID) 92 | } else { 93 | fmt.Printf("produce event success Type:%v UserID:%v offset:%v\n", 94 | string(eventType), userID, newID) 95 | } 96 | 97 | } 98 | } 99 | --------------------------------------------------------------------------------