├── main.go ├── .gitignore ├── README.md ├── pkg └── models.go ├── docker-compose.yml ├── read.txt ├── go.mod └── cmd ├── producer └── producer.go └── consumer └── consumer.go /main.go: -------------------------------------------------------------------------------- 1 | package main -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | 3 | *.sum -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/high-horse/go-kafka-notification-server/HEAD/README.md -------------------------------------------------------------------------------- /pkg/models.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type User struct { 4 | ID int `json:"id"` 5 | Name string `json:"name"` 6 | } 7 | 8 | type Notification struct { 9 | From User `json:"from"` 10 | To User `json:"to"` 11 | Message string `json:"message"` 12 | } -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Copyright Broadcom, Inc. All Rights Reserved. 2 | # SPDX-License-Identifier: APACHE-2.0 3 | 4 | version: "2" 5 | 6 | services: 7 | kafka: 8 | image: docker.io/bitnami/kafka:3.7 9 | ports: 10 | - "9092:9092" 11 | volumes: 12 | - "kafka_data:/bitnami" 13 | environment: 14 | # KRaft settings 15 | - KAFKA_CFG_NODE_ID=0 16 | - KAFKA_CFG_PROCESS_ROLES=controller,broker 17 | - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093 18 | # Listeners 19 | - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 20 | # - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092 21 | - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 22 | - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT 23 | - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER 24 | - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT 25 | volumes: 26 | kafka_data: 27 | driver: local 28 | -------------------------------------------------------------------------------- /read.txt: -------------------------------------------------------------------------------- 1 | Get docker image -linux 2 | curl -sSL https://raw.githubusercontent.com/bitnami/containers/main/bitnami/kafka/docker-compose.yml -o docker-compose.yml 3 | 4 | Get docker image -windows 5 | Invoke-WebRequest -Uri "https://raw.githubusercontent.com/bitnami/containers/main/bitnami/kafka/docker-compose.yml" -OutFile "docker-compose.yml" 6 | 7 | Run the following command to start the Kafka and Zookeeper services: 8 | docker-compose up -d 9 | 10 | Check the running Docker containers: 11 | docker ps 12 | 13 | 14 | source 15 | https://www.freecodecamp.org/news/build-a-real-time-notification-system-with-go-and-kafka/ 16 | 17 | dockerhub kafka 18 | https://hub.docker.com/r/bitnami/kafka/ 19 | 20 | 21 | run the servers 22 | go run cmd/producer/producer.go 23 | go run cmd/consumer/consumer.go 24 | 25 | send message 26 | curl -X POST http://localhost:8080/send -d "fromID=2&toID=1&message=Bruno started following you." 27 | Invoke-RestMethod -Uri http://localhost:8080/send -Method Post -Body @{fromID='2'; toID='1'; message='Bruno started following you.'} -ContentType "application/x-www-form-urlencoded" 28 | 29 | 30 | curl -X POST http://localhost:8080/send -d "fromID=4&toID=1&message=Lena liked your post: 'My weekend getaway!'" 31 | Invoke-RestMethod -Uri http://localhost:8080/send -Method Post -Body @{fromID='4'; toID='1'; message="Lena liked your post: 'My weekend getaway!'"} -ContentType "application/x-www-form-urlencoded" 32 | 33 | Retrieving notifications for User 1 34 | curl http://localhost:8081/notifications/1 -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module kafka-notify 2 | 3 | go 1.22.0 4 | 5 | require ( 6 | github.com/IBM/sarama v1.43.2 // indirect 7 | github.com/bytedance/sonic v1.11.6 // indirect 8 | github.com/bytedance/sonic/loader v0.1.1 // indirect 9 | github.com/cloudwego/base64x v0.1.4 // indirect 10 | github.com/cloudwego/iasm v0.2.0 // indirect 11 | github.com/davecgh/go-spew v1.1.1 // indirect 12 | github.com/eapache/go-resiliency v1.6.0 // indirect 13 | github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect 14 | github.com/eapache/queue v1.1.0 // indirect 15 | github.com/gabriel-vasile/mimetype v1.4.3 // indirect 16 | github.com/gin-contrib/sse v0.1.0 // indirect 17 | github.com/gin-gonic/gin v1.10.0 // indirect 18 | github.com/go-playground/locales v0.14.1 // indirect 19 | github.com/go-playground/universal-translator v0.18.1 // indirect 20 | github.com/go-playground/validator/v10 v10.20.0 // indirect 21 | github.com/goccy/go-json v0.10.2 // indirect 22 | github.com/golang/snappy v0.0.4 // indirect 23 | github.com/hashicorp/errwrap v1.0.0 // indirect 24 | github.com/hashicorp/go-multierror v1.1.1 // indirect 25 | github.com/hashicorp/go-uuid v1.0.3 // indirect 26 | github.com/jcmturner/aescts/v2 v2.0.0 // indirect 27 | github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect 28 | github.com/jcmturner/gofork v1.7.6 // indirect 29 | github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect 30 | github.com/jcmturner/rpc/v2 v2.0.3 // indirect 31 | github.com/json-iterator/go v1.1.12 // indirect 32 | github.com/klauspost/compress v1.17.8 // indirect 33 | github.com/klauspost/cpuid/v2 v2.2.7 // indirect 34 | github.com/leodido/go-urn v1.4.0 // indirect 35 | github.com/mattn/go-isatty v0.0.20 // indirect 36 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 37 | github.com/modern-go/reflect2 v1.0.2 // indirect 38 | github.com/pelletier/go-toml/v2 v2.2.2 // indirect 39 | github.com/pierrec/lz4/v4 v4.1.21 // indirect 40 | github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect 41 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 42 | github.com/ugorji/go/codec v1.2.12 // indirect 43 | golang.org/x/arch v0.8.0 // indirect 44 | golang.org/x/crypto v0.23.0 // indirect 45 | golang.org/x/net v0.25.0 // indirect 46 | golang.org/x/sys v0.20.0 // indirect 47 | golang.org/x/text v0.15.0 // indirect 48 | google.golang.org/protobuf v1.34.1 // indirect 49 | gopkg.in/yaml.v3 v3.0.1 // indirect 50 | ) 51 | -------------------------------------------------------------------------------- /cmd/producer/producer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | models "kafka-notify/pkg" 8 | "log" 9 | "net/http" 10 | "strconv" 11 | 12 | "github.com/IBM/sarama" 13 | "github.com/gin-gonic/gin" 14 | ) 15 | 16 | const ( 17 | ProducerPort = ":8080" 18 | KafkaServerAddress = "localhost:9092" 19 | KafkaTopic = "notifications" 20 | ) 21 | 22 | // HELPER FUNCTION 23 | var ErrUserNotFoundInProducer = errors.New("user not found") 24 | 25 | func findUserById(id int, users []models.User) (models.User, error) { 26 | for _, user := range users { 27 | if user.ID == id { 28 | return user, nil 29 | } 30 | } 31 | return models.User{}, ErrUserNotFoundInProducer 32 | } 33 | 34 | func getIDFromRequest(fromValue string, ctx *gin.Context) (int, error) { 35 | id, err := strconv.Atoi(ctx.PostForm(fromValue)) 36 | if err != nil { 37 | return 0, fmt.Errorf( 38 | "failed to parse ID from form value %s: %w", fromValue, err, 39 | ) 40 | } 41 | return id, nil 42 | } 43 | 44 | 45 | // KAFKA functions 46 | func sendKafkaMesage( 47 | producer sarama.SyncProducer, 48 | users []models.User, 49 | ctx *gin.Context, 50 | fromID , toID int, 51 | ) error { 52 | message := ctx.PostForm("message") 53 | 54 | fromUser, err := findUserById(fromID, users) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | toUser, err := findUserById(toID, users) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | notification := models.Notification { 65 | From: fromUser, 66 | To: toUser, 67 | Message: message, 68 | } 69 | 70 | notificationJSON, err := json.Marshal(notification) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | msg := &sarama.ProducerMessage { 76 | Topic: KafkaTopic, 77 | Key: sarama.StringEncoder(strconv.Itoa(toUser.ID)), 78 | Value: sarama.StringEncoder(notificationJSON), 79 | } 80 | 81 | _, _, err = producer.SendMessage(msg) 82 | return err 83 | } 84 | 85 | func sendMessageHandler( 86 | producer sarama.SyncProducer, 87 | users []models.User, 88 | ) gin.HandlerFunc { 89 | return func(ctx *gin.Context) { 90 | fromID, err := getIDFromRequest("fromID", ctx) 91 | if err != nil { 92 | ctx.JSON(http.StatusBadRequest, gin.H{"message": err.Error()}) 93 | return 94 | } 95 | 96 | toID, err := getIDFromRequest("toID", ctx) 97 | if err != nil { 98 | ctx.JSON(http.StatusBadRequest, gin.H{"message": err.Error()}) 99 | return 100 | } 101 | 102 | err = sendKafkaMesage(producer, users, ctx, fromID, toID) 103 | if errors.Is(err, ErrUserNotFoundInProducer) { 104 | ctx.JSON(http.StatusBadRequest, gin.H{"message": "user not found"}) 105 | return 106 | } 107 | 108 | if err != nil { 109 | ctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) 110 | return 111 | } 112 | 113 | ctx.JSON(http.StatusOK, gin.H{"message": "notification sent successfully"}) 114 | } 115 | } 116 | 117 | func setupProducer() (sarama.SyncProducer, error) { 118 | config := sarama.NewConfig() 119 | config.Producer.Return.Successes = true 120 | producer, err := sarama.NewSyncProducer([]string{KafkaServerAddress}, config) 121 | if err != nil { 122 | return nil, err 123 | } 124 | return producer, nil 125 | } 126 | 127 | 128 | 129 | func main() { 130 | users := []models.User { 131 | {ID: 1, Name: "John"}, 132 | {ID: 2, Name: "Jane"}, 133 | {ID: 3, Name: "Jill"}, 134 | {ID: 4, Name: "Jack"}, 135 | {ID: 5, Name: "Judy"}, 136 | } 137 | 138 | producer, err := setupProducer() 139 | if err != nil { 140 | log.Fatalf("failed to initialize producer: %v", err) 141 | } 142 | 143 | defer producer.Close() 144 | 145 | gin.SetMode(gin.ReleaseMode) 146 | router := gin.Default() 147 | router.POST("/send", sendMessageHandler(producer, users)) 148 | 149 | fmt.Printf("Kafka PRODUCER 📨 started at http://localhost%s\n", ProducerPort) 150 | 151 | if err := router.Run(ProducerPort); err != nil { 152 | log.Printf("failed to run the server: %v", err) 153 | } 154 | } -------------------------------------------------------------------------------- /cmd/consumer/consumer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | "sync" 11 | 12 | models "kafka-notify/pkg" 13 | 14 | "github.com/IBM/sarama" 15 | "github.com/gin-gonic/gin" 16 | ) 17 | 18 | const ( 19 | ConsumerGroup = "notifications-group" 20 | ConsumerTopic = "notifications" 21 | ConsumerPort = ":8081" 22 | KafkaServerAddress = "localhost:9092" 23 | ) 24 | 25 | // ============== HELPER FUNCTIONS ============== 26 | var ErrNoMessagesFound = errors.New("no messages found") 27 | 28 | func getUserIDFromRequest(ctx *gin.Context) (string, error) { 29 | userID := ctx.Param("userID") 30 | if userID == "" { 31 | return "", ErrNoMessagesFound 32 | } 33 | return userID, nil 34 | } 35 | 36 | // ====== NOTIFICATION STORAGE ====== 37 | type UserNotifications map[string][]models.Notification 38 | 39 | type NotificationStore struct { 40 | data UserNotifications 41 | mu sync.RWMutex 42 | } 43 | 44 | func (ns *NotificationStore) Add(userID string, 45 | notification models.Notification) { 46 | ns.mu.Lock() 47 | defer ns.mu.Unlock() 48 | ns.data[userID] = append(ns.data[userID], notification) 49 | } 50 | 51 | func (ns *NotificationStore) Get(userID string) []models.Notification { 52 | ns.mu.RLock() 53 | defer ns.mu.RUnlock() 54 | return ns.data[userID] 55 | } 56 | 57 | // ============== KAFKA RELATED FUNCTIONS ============== 58 | type Consumer struct { 59 | store *NotificationStore 60 | } 61 | 62 | func (*Consumer) Setup(sarama.ConsumerGroupSession) error { return nil } 63 | func (*Consumer) Cleanup(sarama.ConsumerGroupSession) error { return nil } 64 | 65 | func (consumer *Consumer) ConsumeClaim( 66 | sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { 67 | for msg := range claim.Messages() { 68 | userID := string(msg.Key) 69 | var notification models.Notification 70 | err := json.Unmarshal(msg.Value, ¬ification) 71 | if err != nil { 72 | log.Printf("failed to unmarshal notification: %v", err) 73 | continue 74 | } 75 | consumer.store.Add(userID, notification) 76 | sess.MarkMessage(msg, "") 77 | } 78 | return nil 79 | } 80 | 81 | func initializeConsumerGroup() (sarama.ConsumerGroup, error) { 82 | config := sarama.NewConfig() 83 | 84 | consumerGroup, err := sarama.NewConsumerGroup( 85 | []string{KafkaServerAddress}, ConsumerGroup, config) 86 | if err != nil { 87 | return nil, fmt.Errorf("failed to initialize consumer group: %w", err) 88 | } 89 | 90 | return consumerGroup, nil 91 | } 92 | 93 | func setupConsumerGroup(ctx context.Context, store *NotificationStore) { 94 | consumerGroup, err := initializeConsumerGroup() 95 | if err != nil { 96 | log.Printf("initialization error: %v", err) 97 | } 98 | defer consumerGroup.Close() 99 | 100 | consumer := &Consumer{ 101 | store: store, 102 | } 103 | 104 | for { 105 | err = consumerGroup.Consume(ctx, []string{ConsumerTopic}, consumer) 106 | if err != nil { 107 | log.Printf("error from consumer: %v", err) 108 | } 109 | if ctx.Err() != nil { 110 | return 111 | } 112 | } 113 | } 114 | 115 | func handleNotifications(ctx *gin.Context, store *NotificationStore) { 116 | userID, err := getUserIDFromRequest(ctx) 117 | if err != nil { 118 | ctx.JSON(http.StatusNotFound, gin.H{"message": err.Error()}) 119 | return 120 | } 121 | 122 | notes := store.Get(userID) 123 | if len(notes) == 0 { 124 | ctx.JSON(http.StatusOK, 125 | gin.H{ 126 | "message": "No notifications found for user", 127 | "notifications": []models.Notification{}, 128 | }) 129 | return 130 | } 131 | 132 | ctx.JSON(http.StatusOK, gin.H{"notifications": notes}) 133 | } 134 | 135 | func main() { 136 | store := &NotificationStore{ 137 | data: make(UserNotifications), 138 | } 139 | 140 | ctx, cancel := context.WithCancel(context.Background()) 141 | go setupConsumerGroup(ctx, store) 142 | defer cancel() 143 | 144 | gin.SetMode(gin.ReleaseMode) 145 | router := gin.Default() 146 | router.GET("/notifications/:userID", func(ctx *gin.Context) { 147 | handleNotifications(ctx, store) 148 | }) 149 | 150 | fmt.Printf("Kafka CONSUMER (Group: %s) 👥📥 "+ 151 | "started at http://localhost%s\n", ConsumerGroup, ConsumerPort) 152 | 153 | if err := router.Run(ConsumerPort); err != nil { 154 | log.Printf("failed to run the server: %v", err) 155 | } 156 | } --------------------------------------------------------------------------------