├── .idea └── .gitignore ├── Dockerfile ├── Dockerfile.envoy ├── Makefile ├── README.md ├── client ├── grpc_config.json └── main.go ├── docker-compose.yml ├── envoy.yaml ├── go.mod ├── go.sum ├── nginx.conf ├── proto ├── broadcast.pb.go ├── broadcast.proto └── broadcast_grpc.pb.go ├── publisher └── main.go └── server └── main.go /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build stage 2 | FROM golang:1.22-alpine AS builder 3 | 4 | WORKDIR /app 5 | 6 | COPY . . 7 | 8 | RUN go build -o main ./server/main.go 9 | 10 | # Run stage 11 | FROM alpine:latest 12 | WORKDIR /app 13 | 14 | COPY --from=builder /app/main . 15 | 16 | RUN chmod +x /app/main 17 | 18 | EXPOSE 50051 19 | 20 | CMD ["sh", "-c","/app/main"] -------------------------------------------------------------------------------- /Dockerfile.envoy: -------------------------------------------------------------------------------- 1 | FROM envoyproxy/envoy:v1.22-latest 2 | 3 | COPY envoy.yaml /etc/envoy/envoy.yaml 4 | 5 | CMD ["/usr/local/bin/envoy", "-c", "/etc/envoy/envoy.yaml", "-l", "info", "--log-format", "[%Y-%m-%d %T.%e][%t][%l][%n] %v"] -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | run: 2 | docker-compose down && docker-compose up --build -d 3 | stop: 4 | docker-compose down 5 | cc: 6 | go run client/main.go 7 | 8 | main: 9 | go run server/*.go 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # redis-go-grpc-distributed 2 | -------------------------------------------------------------------------------- /client/grpc_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "loadBalancingConfig": [ 3 | { "round_robin": {} } 4 | ], 5 | "methodConfig": [{ 6 | "name": [{"service": "users.PostDataService"}], 7 | "retryPolicy": { 8 | "maxAttempts": 4, 9 | "initialBackoff": "0.1s", 10 | "maxBackoff": "1s", 11 | "backoffMultiplier": 2, 12 | "retryableStatusCodes": ["UNAVAILABLE"] 13 | } 14 | }] 15 | } 16 | -------------------------------------------------------------------------------- /client/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | broadcasts "distributed-redis/proto" 6 | "google.golang.org/grpc" 7 | "google.golang.org/grpc/credentials/insecure" 8 | "io" 9 | "log" 10 | "sync" 11 | "sync/atomic" 12 | "time" 13 | ) 14 | 15 | const ( 16 | totalClients = 30000 17 | connectionsPerSecond = 1000 18 | ) 19 | 20 | func main() { 21 | var wg sync.WaitGroup 22 | var connectedClients int32 23 | done := make(chan struct{}) 24 | 25 | // Rate limiter for connection 26 | rateLimiter := time.NewTicker(time.Second / time.Duration(connectionsPerSecond)) 27 | //rateLimiter := time.NewTicker(time.Millisecond * 100) 28 | for i := 1; i <= totalClients; i++ { 29 | <-rateLimiter.C // Wait for the rate limiter 30 | wg.Add(1) 31 | go runClient(i, &wg, done, &connectedClients) 32 | } 33 | 34 | log.Printf("All clients finished. Total connected: %d", atomic.LoadInt32(&connectedClients)) 35 | wg.Wait() 36 | } 37 | 38 | func runClient(id int, wg *sync.WaitGroup, done chan struct{}, connectedClients *int32) { 39 | defer wg.Done() 40 | 41 | backoff := time.Second 42 | maxBackoff := 2 * time.Minute 43 | 44 | for { 45 | select { 46 | case <-done: 47 | log.Printf("Client %d shutting down", id) 48 | return 49 | default: 50 | if err := connectAndListen(id, &backoff, done); err != nil { 51 | log.Printf("Client %d: connection attempt failed, retrying...", id) 52 | time.Sleep(backoff) 53 | backoff *= 2 54 | if backoff > maxBackoff { 55 | backoff = maxBackoff 56 | } 57 | } else { 58 | atomic.AddInt32(connectedClients, 1) 59 | // If connectAndListen returns nil, it means we've shut down gracefully 60 | return 61 | } 62 | } 63 | } 64 | } 65 | 66 | func connectAndListen(id int, backoff *time.Duration, done chan struct{}) error { 67 | log.Printf("Client %d attempting to connect", id) 68 | conn, err := grpc.Dial("localhost:80", grpc.WithTransportCredentials(insecure.NewCredentials()), 69 | grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`)) 70 | if err != nil { 71 | log.Printf("Client %d failed to connect: %v", id, err) 72 | return err 73 | } 74 | defer conn.Close() 75 | log.Printf("Client %d connected successfully", id) 76 | 77 | client := broadcasts.NewBroadcasterClient(conn) 78 | stream, err := client.Subscribe(context.Background(), &broadcasts.SubscribeRequest{}) 79 | if err != nil { 80 | log.Printf("Client %d failed to subscribe: %v", id, err) 81 | return err 82 | } 83 | log.Printf("Client %d subscribed successfully", id) 84 | 85 | // Reset backoff on successful connection 86 | *backoff = time.Second 87 | 88 | for { 89 | select { 90 | case <-done: 91 | log.Printf("Client %d shutting down", id) 92 | return nil 93 | default: 94 | msg, err := stream.Recv() 95 | if err != nil { 96 | if err == io.EOF { 97 | log.Printf("Client %d: Stream ended", id) 98 | return err 99 | } 100 | log.Printf("Client %d failed to receive message: %v", id, err) 101 | time.Sleep(time.Second) 102 | return err 103 | } 104 | log.Printf("Client %d received message: %s", id, msg.Content) 105 | time.Sleep(time.Second) 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | distributed-chat-service: 5 | build: . 6 | ports: 7 | - "50051" 8 | deploy: 9 | replicas: 3 10 | restart_policy: 11 | condition: on-failure 12 | resources: 13 | limits: 14 | cpus: '2' # Limit to 2 CPU cores 15 | memory: '2g' # Limit to 4GB of memory 16 | environment: 17 | - GOMAXPROCS=14 18 | ulimits: 19 | nofile: 20 | soft: 65536 21 | hard: 65536 22 | 23 | # nginx: 24 | # image: nginx:latest 25 | # ports: 26 | # - "80:80" 27 | # volumes: 28 | # - ./nginx.conf:/etc/nginx/nginx.conf 29 | # depends_on: 30 | # - distributed-chat-service 31 | 32 | envoy: 33 | build: 34 | context: . 35 | dockerfile: Dockerfile.envoy 36 | ports: 37 | - "80:80" 38 | volumes: 39 | - ./envoy.yaml:/etc/envoy/envoy.yaml 40 | depends_on: 41 | - distributed-chat-service 42 | 43 | redis: 44 | image: 'redis:6.2-alpine' 45 | ports: 46 | - "6379:6379" 47 | deploy: 48 | mode: replicated 49 | replicas: 1 50 | volumes: 51 | - ./db-data/redis/:/data/db 52 | command: [ "redis-server", "--appendonly", "yes" ] 53 | -------------------------------------------------------------------------------- /envoy.yaml: -------------------------------------------------------------------------------- 1 | static_resources: 2 | listeners: 3 | - name: listener_0 4 | address: 5 | socket_address: { address: 0.0.0.0, port_value: 80 } 6 | filter_chains: 7 | - filters: 8 | - name: envoy.filters.network.http_connection_manager 9 | typed_config: 10 | "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager 11 | stat_prefix: ingress_http 12 | codec_type: AUTO 13 | route_config: 14 | name: local_route 15 | virtual_hosts: 16 | - name: local_service 17 | domains: ["*"] 18 | routes: 19 | - match: { prefix: "/" } 20 | route: 21 | cluster: grpc_service 22 | timeout: 300s 23 | http_filters: 24 | - name: envoy.filters.http.router 25 | typed_config: 26 | "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router 27 | 28 | clusters: 29 | - name: grpc_service 30 | connect_timeout: 300s 31 | type: STRICT_DNS 32 | lb_policy: ROUND_ROBIN 33 | http2_protocol_options: {} 34 | load_assignment: 35 | cluster_name: grpc_service 36 | endpoints: 37 | - lb_endpoints: 38 | - endpoint: 39 | address: 40 | socket_address: 41 | address: distributed-chat-service 42 | port_value: 50051 43 | health_checks: 44 | - timeout: 1s 45 | interval: 10s 46 | unhealthy_threshold: 3 47 | healthy_threshold: 2 48 | grpc_health_check: 49 | service_name: "grpc.health.v1.Health" 50 | max_requests_per_connection: 10000 51 | circuit_breakers: 52 | thresholds: 53 | - priority: DEFAULT 54 | max_connections: 50000 55 | max_pending_requests: 10000 56 | max_requests: 50000 57 | max_retries: 3 58 | outlier_detection: 59 | consecutive_5xx: 5 60 | base_ejection_time: 30s 61 | max_ejection_percent: 50 62 | 63 | admin: 64 | access_log_path: /tmp/admin_access.log 65 | address: 66 | socket_address: { address: 0.0.0.0, port_value: 9901 } -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module distributed-redis 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/redis/go-redis/v9 v9.6.0 7 | golang.org/x/sync v0.7.0 8 | google.golang.org/grpc v1.65.0 9 | google.golang.org/protobuf v1.34.1 10 | ) 11 | 12 | require ( 13 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 14 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 15 | golang.org/x/net v0.25.0 // indirect 16 | golang.org/x/sys v0.20.0 // indirect 17 | golang.org/x/text v0.15.0 // indirect 18 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= 2 | github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= 3 | github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= 4 | github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= 5 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 6 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 7 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 8 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 9 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 10 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 11 | github.com/redis/go-redis/v9 v9.6.0 h1:NLck+Rab3AOTHw21CGRpvQpgTrAU4sgdCswqGtlhGRA= 12 | github.com/redis/go-redis/v9 v9.6.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= 13 | golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= 14 | golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= 15 | golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= 16 | golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 17 | golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= 18 | golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 19 | golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= 20 | golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 21 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= 22 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= 23 | google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= 24 | google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= 25 | google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= 26 | google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 27 | -------------------------------------------------------------------------------- /nginx.conf: -------------------------------------------------------------------------------- 1 | worker_processes auto; 2 | 3 | events { 4 | worker_connections 65535; 5 | multi_accept on; 6 | use epoll; 7 | } 8 | 9 | http { 10 | upstream grpc_backend { 11 | server distributed-chat-service:50051 max_fails=3 fail_timeout=30s; 12 | server distributed-chat-service:50052 max_fails=3 fail_timeout=30s; 13 | server distributed-chat-service:50053 max_fails=3 fail_timeout=30s; 14 | server distributed-chat-service:50054 max_fails=3 fail_timeout=30s; 15 | server distributed-chat-service:50055 max_fails=3 fail_timeout=30s; 16 | } 17 | 18 | server { 19 | listen 80 http2; 20 | 21 | location / { 22 | grpc_pass grpc://grpc_backend; 23 | error_page 502 = /error502grpc; 24 | 25 | # Headers for gRPC 26 | grpc_set_header Host $host; 27 | grpc_set_header X-Real-IP $remote_addr; 28 | grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 29 | grpc_set_header X-Forwarded-Proto $scheme; 30 | 31 | # Timeout settings 32 | grpc_connect_timeout 300s; 33 | grpc_send_timeout 300s; 34 | grpc_read_timeout 300s; 35 | send_timeout 300s; 36 | 37 | proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; 38 | proxy_next_upstream_timeout 300s; 39 | proxy_next_upstream_tries 3; 40 | } 41 | 42 | location = /error502grpc { 43 | internal; 44 | default_type application/grpc; 45 | add_header grpc-status 14; 46 | add_header content-length 0; 47 | return 204; 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /proto/broadcast.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.27.1 4 | // protoc v5.26.1 5 | // source: broadcast.proto 6 | 7 | package broadcasts 8 | 9 | import ( 10 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 11 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 12 | reflect "reflect" 13 | sync "sync" 14 | ) 15 | 16 | const ( 17 | // Verify that this generated code is sufficiently up-to-date. 18 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 19 | // Verify that runtime/protoimpl is sufficiently up-to-date. 20 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 21 | ) 22 | 23 | type SubscribeRequest struct { 24 | state protoimpl.MessageState 25 | sizeCache protoimpl.SizeCache 26 | unknownFields protoimpl.UnknownFields 27 | } 28 | 29 | func (x *SubscribeRequest) Reset() { 30 | *x = SubscribeRequest{} 31 | if protoimpl.UnsafeEnabled { 32 | mi := &file_broadcast_proto_msgTypes[0] 33 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 34 | ms.StoreMessageInfo(mi) 35 | } 36 | } 37 | 38 | func (x *SubscribeRequest) String() string { 39 | return protoimpl.X.MessageStringOf(x) 40 | } 41 | 42 | func (*SubscribeRequest) ProtoMessage() {} 43 | 44 | func (x *SubscribeRequest) ProtoReflect() protoreflect.Message { 45 | mi := &file_broadcast_proto_msgTypes[0] 46 | if protoimpl.UnsafeEnabled && x != nil { 47 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 48 | if ms.LoadMessageInfo() == nil { 49 | ms.StoreMessageInfo(mi) 50 | } 51 | return ms 52 | } 53 | return mi.MessageOf(x) 54 | } 55 | 56 | // Deprecated: Use SubscribeRequest.ProtoReflect.Descriptor instead. 57 | func (*SubscribeRequest) Descriptor() ([]byte, []int) { 58 | return file_broadcast_proto_rawDescGZIP(), []int{0} 59 | } 60 | 61 | type BroadcastMessage struct { 62 | state protoimpl.MessageState 63 | sizeCache protoimpl.SizeCache 64 | unknownFields protoimpl.UnknownFields 65 | 66 | Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` 67 | Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` 68 | } 69 | 70 | func (x *BroadcastMessage) Reset() { 71 | *x = BroadcastMessage{} 72 | if protoimpl.UnsafeEnabled { 73 | mi := &file_broadcast_proto_msgTypes[1] 74 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 75 | ms.StoreMessageInfo(mi) 76 | } 77 | } 78 | 79 | func (x *BroadcastMessage) String() string { 80 | return protoimpl.X.MessageStringOf(x) 81 | } 82 | 83 | func (*BroadcastMessage) ProtoMessage() {} 84 | 85 | func (x *BroadcastMessage) ProtoReflect() protoreflect.Message { 86 | mi := &file_broadcast_proto_msgTypes[1] 87 | if protoimpl.UnsafeEnabled && x != nil { 88 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 89 | if ms.LoadMessageInfo() == nil { 90 | ms.StoreMessageInfo(mi) 91 | } 92 | return ms 93 | } 94 | return mi.MessageOf(x) 95 | } 96 | 97 | // Deprecated: Use BroadcastMessage.ProtoReflect.Descriptor instead. 98 | func (*BroadcastMessage) Descriptor() ([]byte, []int) { 99 | return file_broadcast_proto_rawDescGZIP(), []int{1} 100 | } 101 | 102 | func (x *BroadcastMessage) GetId() int64 { 103 | if x != nil { 104 | return x.Id 105 | } 106 | return 0 107 | } 108 | 109 | func (x *BroadcastMessage) GetContent() string { 110 | if x != nil { 111 | return x.Content 112 | } 113 | return "" 114 | } 115 | 116 | var File_broadcast_proto protoreflect.FileDescriptor 117 | 118 | var file_broadcast_proto_rawDesc = []byte{ 119 | 0x0a, 0x0f, 0x62, 0x72, 0x6f, 0x61, 0x64, 0x63, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 120 | 0x6f, 0x12, 0x09, 0x62, 0x72, 0x6f, 0x61, 0x64, 0x63, 0x61, 0x73, 0x74, 0x22, 0x12, 0x0a, 0x10, 121 | 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 122 | 0x22, 0x3c, 0x0a, 0x10, 0x42, 0x72, 0x6f, 0x61, 0x64, 0x63, 0x61, 0x73, 0x74, 0x4d, 0x65, 0x73, 123 | 0x73, 0x61, 0x67, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 124 | 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 125 | 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x32, 0x58, 126 | 0x0a, 0x0b, 0x42, 0x72, 0x6f, 0x61, 0x64, 0x63, 0x61, 0x73, 0x74, 0x65, 0x72, 0x12, 0x49, 0x0a, 127 | 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x1b, 0x2e, 0x62, 0x72, 0x6f, 128 | 0x61, 0x64, 0x63, 0x61, 0x73, 0x74, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 129 | 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x62, 0x72, 0x6f, 0x61, 0x64, 0x63, 130 | 0x61, 0x73, 0x74, 0x2e, 0x42, 0x72, 0x6f, 0x61, 0x64, 0x63, 0x61, 0x73, 0x74, 0x4d, 0x65, 0x73, 131 | 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x42, 0x0d, 0x5a, 0x0b, 0x2f, 0x62, 0x72, 0x6f, 132 | 0x61, 0x64, 0x63, 0x61, 0x73, 0x74, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 133 | } 134 | 135 | var ( 136 | file_broadcast_proto_rawDescOnce sync.Once 137 | file_broadcast_proto_rawDescData = file_broadcast_proto_rawDesc 138 | ) 139 | 140 | func file_broadcast_proto_rawDescGZIP() []byte { 141 | file_broadcast_proto_rawDescOnce.Do(func() { 142 | file_broadcast_proto_rawDescData = protoimpl.X.CompressGZIP(file_broadcast_proto_rawDescData) 143 | }) 144 | return file_broadcast_proto_rawDescData 145 | } 146 | 147 | var file_broadcast_proto_msgTypes = make([]protoimpl.MessageInfo, 2) 148 | var file_broadcast_proto_goTypes = []interface{}{ 149 | (*SubscribeRequest)(nil), // 0: broadcast.SubscribeRequest 150 | (*BroadcastMessage)(nil), // 1: broadcast.BroadcastMessage 151 | } 152 | var file_broadcast_proto_depIdxs = []int32{ 153 | 0, // 0: broadcast.Broadcaster.Subscribe:input_type -> broadcast.SubscribeRequest 154 | 1, // 1: broadcast.Broadcaster.Subscribe:output_type -> broadcast.BroadcastMessage 155 | 1, // [1:2] is the sub-list for method output_type 156 | 0, // [0:1] is the sub-list for method input_type 157 | 0, // [0:0] is the sub-list for extension type_name 158 | 0, // [0:0] is the sub-list for extension extendee 159 | 0, // [0:0] is the sub-list for field type_name 160 | } 161 | 162 | func init() { file_broadcast_proto_init() } 163 | func file_broadcast_proto_init() { 164 | if File_broadcast_proto != nil { 165 | return 166 | } 167 | if !protoimpl.UnsafeEnabled { 168 | file_broadcast_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 169 | switch v := v.(*SubscribeRequest); i { 170 | case 0: 171 | return &v.state 172 | case 1: 173 | return &v.sizeCache 174 | case 2: 175 | return &v.unknownFields 176 | default: 177 | return nil 178 | } 179 | } 180 | file_broadcast_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { 181 | switch v := v.(*BroadcastMessage); i { 182 | case 0: 183 | return &v.state 184 | case 1: 185 | return &v.sizeCache 186 | case 2: 187 | return &v.unknownFields 188 | default: 189 | return nil 190 | } 191 | } 192 | } 193 | type x struct{} 194 | out := protoimpl.TypeBuilder{ 195 | File: protoimpl.DescBuilder{ 196 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 197 | RawDescriptor: file_broadcast_proto_rawDesc, 198 | NumEnums: 0, 199 | NumMessages: 2, 200 | NumExtensions: 0, 201 | NumServices: 1, 202 | }, 203 | GoTypes: file_broadcast_proto_goTypes, 204 | DependencyIndexes: file_broadcast_proto_depIdxs, 205 | MessageInfos: file_broadcast_proto_msgTypes, 206 | }.Build() 207 | File_broadcast_proto = out.File 208 | file_broadcast_proto_rawDesc = nil 209 | file_broadcast_proto_goTypes = nil 210 | file_broadcast_proto_depIdxs = nil 211 | } 212 | -------------------------------------------------------------------------------- /proto/broadcast.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package broadcast; 4 | 5 | option go_package = "/broadcasts"; 6 | 7 | service Broadcaster { 8 | rpc Subscribe(SubscribeRequest) returns (stream BroadcastMessage) {} 9 | } 10 | 11 | message SubscribeRequest{} 12 | 13 | message BroadcastMessage { 14 | int64 id = 1; 15 | string content = 2; 16 | } -------------------------------------------------------------------------------- /proto/broadcast_grpc.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go-grpc. DO NOT EDIT. 2 | // versions: 3 | // - protoc-gen-go-grpc v1.2.0 4 | // - protoc v5.26.1 5 | // source: broadcast.proto 6 | 7 | package broadcasts 8 | 9 | import ( 10 | context "context" 11 | grpc "google.golang.org/grpc" 12 | codes "google.golang.org/grpc/codes" 13 | status "google.golang.org/grpc/status" 14 | ) 15 | 16 | // This is a compile-time assertion to ensure that this generated file 17 | // is compatible with the grpc package it is being compiled against. 18 | // Requires gRPC-Go v1.32.0 or later. 19 | const _ = grpc.SupportPackageIsVersion7 20 | 21 | // BroadcasterClient is the client API for Broadcaster service. 22 | // 23 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 24 | type BroadcasterClient interface { 25 | Subscribe(ctx context.Context, in *SubscribeRequest, opts ...grpc.CallOption) (Broadcaster_SubscribeClient, error) 26 | } 27 | 28 | type broadcasterClient struct { 29 | cc grpc.ClientConnInterface 30 | } 31 | 32 | func NewBroadcasterClient(cc grpc.ClientConnInterface) BroadcasterClient { 33 | return &broadcasterClient{cc} 34 | } 35 | 36 | func (c *broadcasterClient) Subscribe(ctx context.Context, in *SubscribeRequest, opts ...grpc.CallOption) (Broadcaster_SubscribeClient, error) { 37 | stream, err := c.cc.NewStream(ctx, &Broadcaster_ServiceDesc.Streams[0], "/broadcast.Broadcaster/Subscribe", opts...) 38 | if err != nil { 39 | return nil, err 40 | } 41 | x := &broadcasterSubscribeClient{stream} 42 | if err := x.ClientStream.SendMsg(in); err != nil { 43 | return nil, err 44 | } 45 | if err := x.ClientStream.CloseSend(); err != nil { 46 | return nil, err 47 | } 48 | return x, nil 49 | } 50 | 51 | type Broadcaster_SubscribeClient interface { 52 | Recv() (*BroadcastMessage, error) 53 | grpc.ClientStream 54 | } 55 | 56 | type broadcasterSubscribeClient struct { 57 | grpc.ClientStream 58 | } 59 | 60 | func (x *broadcasterSubscribeClient) Recv() (*BroadcastMessage, error) { 61 | m := new(BroadcastMessage) 62 | if err := x.ClientStream.RecvMsg(m); err != nil { 63 | return nil, err 64 | } 65 | return m, nil 66 | } 67 | 68 | // BroadcasterServer is the server API for Broadcaster service. 69 | // All implementations must embed UnimplementedBroadcasterServer 70 | // for forward compatibility 71 | type BroadcasterServer interface { 72 | Subscribe(*SubscribeRequest, Broadcaster_SubscribeServer) error 73 | mustEmbedUnimplementedBroadcasterServer() 74 | } 75 | 76 | // UnimplementedBroadcasterServer must be embedded to have forward compatible implementations. 77 | type UnimplementedBroadcasterServer struct { 78 | } 79 | 80 | func (UnimplementedBroadcasterServer) Subscribe(*SubscribeRequest, Broadcaster_SubscribeServer) error { 81 | return status.Errorf(codes.Unimplemented, "method Subscribe not implemented") 82 | } 83 | func (UnimplementedBroadcasterServer) mustEmbedUnimplementedBroadcasterServer() {} 84 | 85 | // UnsafeBroadcasterServer may be embedded to opt out of forward compatibility for this service. 86 | // Use of this interface is not recommended, as added methods to BroadcasterServer will 87 | // result in compilation errors. 88 | type UnsafeBroadcasterServer interface { 89 | mustEmbedUnimplementedBroadcasterServer() 90 | } 91 | 92 | func RegisterBroadcasterServer(s grpc.ServiceRegistrar, srv BroadcasterServer) { 93 | s.RegisterService(&Broadcaster_ServiceDesc, srv) 94 | } 95 | 96 | func _Broadcaster_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error { 97 | m := new(SubscribeRequest) 98 | if err := stream.RecvMsg(m); err != nil { 99 | return err 100 | } 101 | return srv.(BroadcasterServer).Subscribe(m, &broadcasterSubscribeServer{stream}) 102 | } 103 | 104 | type Broadcaster_SubscribeServer interface { 105 | Send(*BroadcastMessage) error 106 | grpc.ServerStream 107 | } 108 | 109 | type broadcasterSubscribeServer struct { 110 | grpc.ServerStream 111 | } 112 | 113 | func (x *broadcasterSubscribeServer) Send(m *BroadcastMessage) error { 114 | return x.ServerStream.SendMsg(m) 115 | } 116 | 117 | // Broadcaster_ServiceDesc is the grpc.ServiceDesc for Broadcaster service. 118 | // It's only intended for direct use with grpc.RegisterService, 119 | // and not to be introspected or modified (even as a copy) 120 | var Broadcaster_ServiceDesc = grpc.ServiceDesc{ 121 | ServiceName: "broadcast.Broadcaster", 122 | HandlerType: (*BroadcasterServer)(nil), 123 | Methods: []grpc.MethodDesc{}, 124 | Streams: []grpc.StreamDesc{ 125 | { 126 | StreamName: "Subscribe", 127 | Handler: _Broadcaster_Subscribe_Handler, 128 | ServerStreams: true, 129 | }, 130 | }, 131 | Metadata: "broadcast.proto", 132 | } 133 | -------------------------------------------------------------------------------- /publisher/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/redis/go-redis/v9" 7 | "log" 8 | "time" 9 | ) 10 | 11 | func main() { 12 | ctx := context.Background() 13 | rdb := redis.NewClient(&redis.Options{ 14 | Addr: "localhost:6379", 15 | }) 16 | // Test connection 17 | _, err := rdb.Ping(ctx).Result() 18 | if err != nil { 19 | log.Fatalf("Failed to connect to Redis: %v", err) 20 | } 21 | log.Println("Successfully connected to Redis") 22 | 23 | time.Sleep(1 * time.Second) 24 | 25 | for i := 0; ; i++ { 26 | // broadcast 27 | message := fmt.Sprintf("This is secret #%d", i) 28 | log.Printf("Broadcasting message: %s", message) 29 | err := rdb.Publish(ctx, "broadcast", message).Err() 30 | if err != nil { 31 | log.Printf("Error publishing message: %v", err) 32 | } else { 33 | log.Printf("Published message: %s", message) 34 | } 35 | time.Sleep(3 * time.Second) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | broadcasts "distributed-redis/proto" 6 | "fmt" 7 | "github.com/redis/go-redis/v9" 8 | "google.golang.org/grpc" 9 | "google.golang.org/grpc/codes" 10 | "google.golang.org/grpc/keepalive" 11 | "google.golang.org/grpc/status" 12 | "log" 13 | "net" 14 | "runtime" 15 | "sync" 16 | "sync/atomic" 17 | "syscall" 18 | "time" 19 | ) 20 | 21 | var ( 22 | batchSize = 100 23 | batchTimeout = 100 * time.Millisecond 24 | connectionsPerSecond = 5000 25 | currentCount int64 = 0 26 | ) 27 | 28 | type Server struct { 29 | broadcasts.UnimplementedBroadcasterServer 30 | redisClient *redis.Client 31 | mu sync.RWMutex 32 | clients map[string]chan *broadcasts.BroadcastMessage 33 | done struct{} 34 | newConnections chan broadcasts.Broadcaster_SubscribeServer 35 | connectionBatches chan []broadcasts.Broadcaster_SubscribeServer 36 | broadcastChannel chan *broadcasts.BroadcastMessage 37 | messageCounter int64 38 | messageStats map[int64]*MessageStats 39 | statsMu sync.RWMutex 40 | } 41 | 42 | type MessageStats struct { 43 | TotalClients int 44 | SentCount int64 45 | } 46 | 47 | func NewServer(redisAddr string) *Server { 48 | return &Server{ 49 | redisClient: redis.NewClient(&redis.Options{ 50 | Addr: redisAddr, 51 | Password: "", 52 | DB: 0, 53 | }), 54 | clients: make(map[string]chan *broadcasts.BroadcastMessage), 55 | newConnections: make(chan broadcasts.Broadcaster_SubscribeServer, 100000), 56 | connectionBatches: make(chan []broadcasts.Broadcaster_SubscribeServer, 10000), 57 | broadcastChannel: make(chan *broadcasts.BroadcastMessage, 100000), 58 | messageStats: make(map[int64]*MessageStats), 59 | } 60 | } 61 | 62 | func main() { 63 | // Increase file descriptor limit 64 | var rLimit syscall.Rlimit 65 | if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil { 66 | log.Println("Error getting Rlimit:", err) 67 | } 68 | rLimit.Cur = rLimit.Max 69 | if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil { 70 | log.Println("Error setting Rlimit:", err) 71 | } 72 | // Increase the number of operating system threads 73 | runtime.GOMAXPROCS(runtime.NumCPU()) 74 | 75 | lis, err := net.Listen("tcp", ":50051") 76 | if err != nil { 77 | log.Fatalf("failed to listen: %v", err) 78 | } 79 | 80 | // Configure keepalive and other options 81 | opts := []grpc.ServerOption{ 82 | grpc.KeepaliveParams(keepalive.ServerParameters{ 83 | MaxConnectionIdle: 20 * time.Minute, 84 | Time: 5 * time.Second, 85 | Timeout: 1 * time.Second, 86 | }), 87 | grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ 88 | MinTime: 5 * time.Second, 89 | PermitWithoutStream: true, 90 | }), 91 | grpc.MaxConcurrentStreams(100000), 92 | grpc.InitialWindowSize(1 << 26), 93 | grpc.InitialConnWindowSize(1 << 26), 94 | grpc.WriteBufferSize(1 << 20), 95 | grpc.ReadBufferSize(1 << 20), 96 | } 97 | 98 | s := grpc.NewServer(opts...) 99 | server := NewServer("redis:6379") 100 | ctx := context.Background() 101 | _, err = server.redisClient.Ping(ctx).Result() 102 | if err != nil { 103 | log.Fatalf("failed to start server: %v", err) 104 | } else { 105 | log.Printf("redis server is running") 106 | } 107 | 108 | broadcasts.RegisterBroadcasterServer(s, server) 109 | 110 | context, cancel := context.WithCancel(context.Background()) 111 | defer cancel() 112 | 113 | go server.StartRedisSubscriber(context) 114 | go server.batchConnections() 115 | go server.handleConnections(runtime.NumCPU() * 2) 116 | go server.handleBroadcasts(runtime.NumCPU() / 2) 117 | go server.monitorMessageStats() 118 | 119 | log.Println("started grpc on port: 50051") 120 | if err := s.Serve(lis); err != nil { 121 | log.Fatalf("failed to serve: %v", err) 122 | } 123 | 124 | } 125 | 126 | func (s *Server) Subscribe(_ *broadcasts.SubscribeRequest, stream broadcasts.Broadcaster_SubscribeServer) error { 127 | select { 128 | case s.newConnections <- stream: 129 | // Connection accepted 130 | case <-time.After(5 * time.Second): 131 | return status.Error(codes.ResourceExhausted, "server is at capacity, please try again later") 132 | } 133 | <-stream.Context().Done() 134 | return nil 135 | } 136 | 137 | func (s *Server) batchConnections() { 138 | var batch []broadcasts.Broadcaster_SubscribeServer 139 | timer := time.NewTimer(batchTimeout) 140 | rateLimiter := time.NewTicker(time.Second / time.Duration(connectionsPerSecond)) 141 | 142 | for { 143 | select { 144 | case <-rateLimiter.C: 145 | select { 146 | case conn := <-s.newConnections: 147 | batch = append(batch, conn) 148 | if len(batch) >= batchSize { 149 | s.connectionBatches <- batch 150 | batch = nil 151 | timer.Reset(batchTimeout) 152 | } 153 | default: 154 | 155 | } 156 | case <-timer.C: 157 | if len(batch) > 0 { 158 | s.connectionBatches <- batch 159 | batch = nil 160 | } 161 | timer.Reset(batchTimeout) 162 | } 163 | } 164 | } 165 | 166 | func (s *Server) handleConnections(workers int) { 167 | for i := 0; i < workers; i++ { 168 | go func() { 169 | for batch := range s.connectionBatches { 170 | for _, stream := range batch { 171 | clientID := generateUniqueID() 172 | clientChan := make(chan *broadcasts.BroadcastMessage, 100) 173 | 174 | s.mu.Lock() 175 | s.clients[clientID] = clientChan 176 | atomic.AddInt64(¤tCount, 1) 177 | s.mu.Unlock() 178 | 179 | log.Printf("New client subscribed: %s, currentCount: %d", clientID, currentCount) 180 | 181 | go s.handleClientStream(clientID, clientChan, stream) 182 | } 183 | } 184 | }() 185 | } 186 | } 187 | 188 | func (s *Server) handleClientStream(clientID string, clientChan chan *broadcasts.BroadcastMessage, stream broadcasts.Broadcaster_SubscribeServer) { 189 | defer func() { 190 | s.mu.Lock() 191 | delete(s.clients, clientID) 192 | s.mu.Unlock() 193 | close(clientChan) 194 | log.Printf("Client unsubscribed: %s", clientID) 195 | }() 196 | 197 | for msg := range clientChan { 198 | if err := stream.Send(msg); err != nil { 199 | log.Printf("Error sending message to client %s: %v", clientID, err) 200 | return 201 | } 202 | } 203 | } 204 | 205 | func (s *Server) StartRedisSubscriber(ctx context.Context) { 206 | pubsub := s.redisClient.Subscribe(ctx, "broadcast") 207 | defer pubsub.Close() 208 | 209 | log.Println("Started Redis subscriber") 210 | 211 | ch := pubsub.Channel() 212 | for msg := range ch { 213 | messageID := atomic.AddInt64(&s.messageCounter, 1) 214 | log.Printf("Received message from Redis: %s (ID: %d)", msg.Payload, messageID) 215 | 216 | s.statsMu.Lock() 217 | s.messageStats[messageID] = &MessageStats{TotalClients: len(s.clients)} 218 | s.statsMu.Unlock() 219 | 220 | broadcastMsg := &broadcasts.BroadcastMessage{ 221 | Content: msg.Payload, 222 | Id: messageID, 223 | } 224 | s.broadcastChannel <- broadcastMsg 225 | } 226 | } 227 | 228 | func (s *Server) handleBroadcasts(workers int) { 229 | for i := 0; i < workers; i++ { 230 | go func() { 231 | for msg := range s.broadcastChannel { 232 | s.mu.RLock() 233 | clientCount := len(s.clients) 234 | for _, clientChan := range s.clients { 235 | select { 236 | case clientChan <- msg: 237 | atomic.AddInt64(&s.messageStats[msg.Id].SentCount, 1) 238 | default: 239 | // Channel full, skip this client 240 | log.Printf("Warning: Client channel full, skipping message %d", msg.Id) 241 | } 242 | } 243 | s.mu.RUnlock() 244 | 245 | // Check if all clients received the message 246 | if atomic.LoadInt64(&s.messageStats[msg.Id].SentCount) == int64(clientCount) { 247 | log.Printf("Message %d successfully sent to all %d clients", msg.Id, clientCount) 248 | s.statsMu.Lock() 249 | delete(s.messageStats, msg.Id) 250 | s.statsMu.Unlock() 251 | } 252 | } 253 | }() 254 | } 255 | } 256 | 257 | func (s *Server) monitorMessageStats() { 258 | ticker := time.NewTicker(5 * time.Second) 259 | for range ticker.C { 260 | s.statsMu.RLock() 261 | for msgID, stats := range s.messageStats { 262 | sentCount := atomic.LoadInt64(&stats.SentCount) 263 | if sentCount < int64(stats.TotalClients) { 264 | log.Printf("Warning: Message %d sent to %d/%d clients", msgID, sentCount, stats.TotalClients) 265 | } 266 | } 267 | s.statsMu.RUnlock() 268 | } 269 | } 270 | 271 | func generateUniqueID() string { 272 | return fmt.Sprintf("client-%d", time.Now().UnixNano()) 273 | } 274 | --------------------------------------------------------------------------------