├── .env.example ├── .gitignore ├── README.md ├── api ├── Dockerfile ├── app.py └── requirements.txt ├── docker-compose.yaml └── webhook ├── Dockerfile ├── go.mod ├── go.sum ├── main.go ├── queue └── worker.go ├── redis └── redis.go ├── sender └── webhook.go └── webhook /.env.example: -------------------------------------------------------------------------------- 1 | REDIS_ADDRESS=redis:6379 2 | WEBHOOK_ADDRESS= -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python 2 | __pycache__/ 3 | *.py[cod] 4 | *.egg-info/ 5 | *.egg 6 | *.so 7 | *.swp 8 | .Python 9 | build/ 10 | develop-eggs/ 11 | dist/ 12 | downloads/ 13 | eggs/ 14 | .eggs/ 15 | lib/ 16 | lib64/ 17 | parts/ 18 | sdist/ 19 | var/ 20 | wheels/ 21 | *.sqlite3 22 | .pipenv 23 | .env 24 | *.venv 25 | venv/ 26 | env/ 27 | 28 | # Go 29 | *.exe 30 | *.exe~ 31 | *.test 32 | *.out 33 | *.o 34 | *.a 35 | *.so 36 | *~ 37 | # Compiled Object files 38 | *.S 39 | *.rdb 40 | 41 | # Log files 42 | *.log 43 | 44 | # Dependency directories 45 | vendor/ 46 | Godeps/ 47 | 48 | # IDE and editor files 49 | .vscode/ 50 | .idea/ 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Webhook Service in Golang 2 | 3 | This project demonstrates a webhook service implemented in Golang. It uses Redis for message queuing and supports exponential backoff and concurrent processing using goroutines. 4 | 5 | ## Features 6 | 7 | - **Concurrent Processing**: Processes multiple webhooks simultaneously using goroutines. 8 | - **Exponential Backoff**: Implements a retry mechanism with exponential backoff for failed webhook deliveries. 9 | - **Redis Integration**: Utilizes Redis for message queuing and pub/sub messaging. 10 | - **Dockerized Application**: Easily deployable using Docker. 11 | 12 | ## Prerequisites 13 | 14 | - Go 1.21 or higher 15 | - Redis server 16 | - Docker (optional for containerized deployment) 17 | 18 | ## Getting Started 19 | 20 | ### Clone the Repository 21 | 22 | ```bash 23 | git https://github.com/koladev32/golang-wehook.git 24 | cd golang-wehook 25 | ``` 26 | 27 | ### Set Up Environment Variables 28 | 29 | Create a `.env` file at the root of the project and add the following content: 30 | 31 | ```txt 32 | REDIS_ADDRESS=redis:6379 33 | WEBHOOK_ADDRESS= 34 | ``` 35 | 36 | Replace `` with your webhook URL. 37 | 38 | ### Running with Docker 39 | 40 | Build and start the container: 41 | 42 | ```bash 43 | docker compose up -d --build 44 | ``` 45 | 46 | Track the logs: 47 | 48 | ```bash 49 | docker compose logs -f 50 | ``` 51 | 52 | Hit `http://127.0.0.1:8000/payment` to start sending data to the webhook service through Redis. 53 | 54 | ## Testing 55 | 56 | You can use a service like [webhook.site](https://webhook.site) to obtain a free webhook URL for testing. 57 | -------------------------------------------------------------------------------- /api/Dockerfile: -------------------------------------------------------------------------------- 1 | # Use an official Python runtime as a parent image 2 | FROM python:3.11-alpine 3 | 4 | # Set the working directory inside the container 5 | WORKDIR /usr/src/app 6 | 7 | # Install necessary packages and Flask 8 | COPY requirements.txt ./ 9 | RUN pip install --no-cache-dir -r requirements.txt 10 | 11 | # Copy the current directory contents into the container at /usr/src/app 12 | COPY . . 13 | 14 | # Make port 8000 available to the world outside this container 15 | EXPOSE 8000 16 | 17 | # Run app.py when the container launches 18 | CMD ["python", "app.py"] 19 | -------------------------------------------------------------------------------- /api/app.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import json 3 | import os 4 | import random 5 | import uuid 6 | from flask import Flask 7 | import redis 8 | 9 | 10 | def get_payment(): 11 | return { 12 | 'url': os.getenv("WEBHOOK_ADDRESS", ""), 13 | 'webhookId': uuid.uuid4().hex, 14 | 'data': { 15 | 'id': uuid.uuid4().hex, 16 | 'payment': f"PY-{''.join((random.choice('abcdxyzpqr').capitalize() for i in range(5)))}", 17 | 'event': random.choice(["accepted", "completed", "canceled"]), 18 | 'created': datetime.now().strftime("%d/%m/%Y, %H:%M:%S"), 19 | } 20 | } 21 | 22 | redis_address = os.getenv("REDIS_ADDRESS", "") 23 | host, port = redis_address.split(":") 24 | port = int(port) 25 | # Create a connection to the Redis server 26 | redis_connection = redis.StrictRedis(host=host, port=port) 27 | 28 | 29 | app = Flask(__name__) 30 | 31 | @app.route('/payment') 32 | def payment(): 33 | webhook_payload_json = json.dumps(get_payment()) 34 | 35 | # Publish the JSON string to the "payments" channel in Redis 36 | redis_connection.publish('payments', webhook_payload_json) 37 | 38 | return webhook_payload_json 39 | 40 | if __name__ == '__main__': 41 | app.run(host='0.0.0.0', port=8000) 42 | -------------------------------------------------------------------------------- /api/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==2.3.2 2 | redis==4.6.0 3 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.9' 2 | 3 | services: 4 | 5 | redis: 6 | image: redis:alpine 7 | 8 | api: 9 | build: ./api/ 10 | container_name: api 11 | env_file: 12 | - .env 13 | restart: always 14 | ports: 15 | - "8000:8000" 16 | depends_on: 17 | - webhook 18 | 19 | webhook: 20 | build: ./webhook/ 21 | env_file: 22 | - .env 23 | depends_on: 24 | - redis -------------------------------------------------------------------------------- /webhook/Dockerfile: -------------------------------------------------------------------------------- 1 | # Start from a Debian-based Golang official image 2 | FROM golang:1.21-alpine as builder 3 | 4 | # Set the working directory inside the container 5 | WORKDIR /app 6 | 7 | # Copy the go mod and sum files 8 | COPY go.mod go.sum ./ 9 | 10 | # Download all dependencies 11 | RUN go mod download 12 | 13 | # Copy the source code from your host to your image filesystem. 14 | COPY . . 15 | 16 | # Build the Go app 17 | RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o main . 18 | 19 | # Use a minimal alpine image for the final stage 20 | FROM alpine:latest 21 | 22 | # Set the working directory inside the container 23 | WORKDIR /root/ 24 | 25 | # Copy the binary from the builder stage 26 | COPY --from=builder /app/main . 27 | 28 | # Run the binary 29 | CMD ["./main"] 30 | -------------------------------------------------------------------------------- /webhook/go.mod: -------------------------------------------------------------------------------- 1 | module webhook 2 | 3 | go 1.20 4 | 5 | require github.com/go-redis/redis/v8 v8.11.5 6 | 7 | require ( 8 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 9 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 10 | ) 11 | -------------------------------------------------------------------------------- /webhook/go.sum: -------------------------------------------------------------------------------- 1 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 2 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 3 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 4 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 5 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 6 | github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= 7 | github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= 8 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 9 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= 10 | github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= 11 | golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= 12 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= 13 | golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= 14 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 15 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 16 | -------------------------------------------------------------------------------- /webhook/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "os" 7 | 8 | redisClient "webhook/redis" 9 | 10 | "webhook/queue" 11 | 12 | "github.com/go-redis/redis/v8" // Make sure to use the correct version 13 | ) 14 | 15 | func main() { 16 | // Create a context 17 | ctx, cancel := context.WithCancel(context.Background()) 18 | defer cancel() 19 | 20 | // Initialize the Redis client 21 | client := redis.NewClient(&redis.Options{ 22 | Addr: os.Getenv("REDIS_ADDRESS"), // Use an environment variable to set the address 23 | Password: "", // No password 24 | DB: 0, // Default DB 25 | }) 26 | 27 | // Create a channel to act as the queue 28 | webhookQueue := make(chan redisClient.WebhookPayload, 100) // Buffer size 100 29 | 30 | go queue.ProcessWebhooks(ctx, webhookQueue) 31 | 32 | // Subscribe to the "transactions" channel 33 | err := redisClient.Subscribe(ctx, client, webhookQueue) 34 | 35 | if err != nil { 36 | log.Println("Error:", err) 37 | } 38 | 39 | select {} 40 | 41 | } 42 | -------------------------------------------------------------------------------- /webhook/queue/worker.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "time" 7 | "webhook/sender" 8 | 9 | redisClient "webhook/redis" 10 | ) 11 | 12 | func ProcessWebhooks(ctx context.Context, webhookQueue chan redisClient.WebhookPayload) { 13 | for payload := range webhookQueue { 14 | go func(p redisClient.WebhookPayload) { 15 | backoffTime := time.Second // starting backoff time 16 | maxBackoffTime := time.Hour // maximum backoff time 17 | retries := 0 18 | maxRetries := 5 19 | 20 | for { 21 | err := sender.SendWebhook(p.Data, p.Url, p.WebhookId) 22 | if err == nil { 23 | break 24 | } 25 | log.Println("Error sending webhook:", err) 26 | 27 | retries++ 28 | if retries >= maxRetries { 29 | log.Println("Max retries reached. Giving up on webhook:", p.WebhookId) 30 | break 31 | } 32 | 33 | time.Sleep(backoffTime) 34 | 35 | // Double the backoff time for the next iteration, capped at the max 36 | backoffTime *= 2 37 | log.Println(backoffTime) 38 | if backoffTime > maxBackoffTime { 39 | backoffTime = maxBackoffTime 40 | } 41 | } 42 | }(payload) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /webhook/redis/redis.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "log" 7 | 8 | "github.com/go-redis/redis/v8" 9 | ) 10 | 11 | // WebhookPayload defines the structure of the data expected 12 | // to be received from Redis, including URL, Webhook ID, and relevant data. 13 | type WebhookPayload struct { 14 | Url string `json:"url"` 15 | WebhookId string `json:"webhookId"` 16 | Data struct { 17 | Id string `json:"id"` 18 | Payment string `json:"payment"` 19 | Event string `json:"event"` 20 | Date string `json:"created"` 21 | } `json:"data"` 22 | } 23 | 24 | // Subscribe subscribes to the "webhooks" channel in Redis, listens for messages, 25 | // unmarshals them into the WebhookPayload type, and sends them to the specified URL. 26 | func Subscribe(ctx context.Context, client *redis.Client, webhookQueue chan WebhookPayload) error { 27 | // Subscribe to the "webhooks" channel in Redis 28 | pubSub := client.Subscribe(ctx, "payments") 29 | 30 | // Ensure that the PubSub connection is closed when the function exits 31 | defer func(pubSub *redis.PubSub) { 32 | if err := pubSub.Close(); err != nil { 33 | log.Println("Error closing PubSub:", err) 34 | } 35 | }(pubSub) 36 | 37 | var payload WebhookPayload 38 | 39 | // Infinite loop to continuously receive messages from the "webhooks" channel 40 | for { 41 | // Receive a message from the channel 42 | msg, err := pubSub.ReceiveMessage(ctx) 43 | if err != nil { 44 | return err // Return the error if there's an issue receiving the message 45 | } 46 | 47 | // Unmarshal the JSON payload into the WebhookPayload structure 48 | err = json.Unmarshal([]byte(msg.Payload), &payload) 49 | if err != nil { 50 | log.Println("Error unmarshalling payload:", err) 51 | continue // Continue with the next message if there's an error unmarshalling 52 | } 53 | 54 | webhookQueue <- payload // Sending the payload to the channel 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /webhook/sender/webhook.go: -------------------------------------------------------------------------------- 1 | package sender 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "io" 8 | "log" 9 | "net/http" 10 | ) 11 | 12 | // Payload represents the structure of the data expected to be sent as a webhook 13 | type Payload struct { 14 | Event string 15 | Date string 16 | Id string 17 | Payment string 18 | } 19 | 20 | // SendWebhook sends a JSON POST request to the specified URL and updates the event status in the database 21 | func SendWebhook(data interface{}, url string, webhookId string) error { 22 | // Marshal the data into JSON 23 | jsonBytes, err := json.Marshal(data) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | // Prepare the webhook request 29 | req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonBytes)) 30 | if err != nil { 31 | return err 32 | } 33 | req.Header.Set("Content-Type", "application/json") 34 | 35 | // Send the webhook request 36 | client := &http.Client{} 37 | resp, err := client.Do(req) 38 | if err != nil { 39 | return err 40 | } 41 | defer func(Body io.ReadCloser) { 42 | if err := Body.Close(); err != nil { 43 | log.Println("Error closing response body:", err) 44 | } 45 | }(resp.Body) 46 | 47 | // Determine the status based on the response code 48 | status := "failed" 49 | if resp.StatusCode == http.StatusOK { 50 | status = "delivered" 51 | } 52 | 53 | log.Println(status) 54 | 55 | if status == "failed" { 56 | return errors.New(status) 57 | } 58 | 59 | return nil 60 | } 61 | -------------------------------------------------------------------------------- /webhook/webhook: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/koladev32/golang-wehook/b9348278cc3569465fea44ac13559b1f0a10de9a/webhook/webhook --------------------------------------------------------------------------------