├── .github └── workflows │ ├── go.yml │ └── reviewdog.yml ├── .gitignore ├── .golangci.yml ├── LICENSE ├── Makefile ├── README.md ├── _examples ├── consumer │ └── main.go ├── delay-message │ └── main.go ├── producer │ └── main.go └── rabbids.yaml ├── config.go ├── config_test.go ├── consumer.go ├── declarations.go ├── declarations_test.go ├── delayed_messages.go ├── delayed_messages_test.go ├── go.mod ├── go.sum ├── integration_consumer_test.go ├── integration_helper_test.go ├── integration_producer_test.go ├── logger.go ├── message.go ├── options.go ├── producer.go ├── producer_test.go ├── rabbids.go ├── serialization └── json.go ├── supervisor.go └── testdata ├── valid_queue_and_exchange_config.yml └── valid_two_connections.yml /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Pull Request CI Check 2 | on: [push, pull_request] 3 | jobs: 4 | build: 5 | name: Run CI 6 | runs-on: ubuntu-latest 7 | steps: 8 | - name: Set up Go 1.15 9 | uses: actions/setup-go@v1 10 | with: 11 | go-version: 1.15 12 | id: go 13 | 14 | - name: Go env 15 | run: go env 16 | 17 | - name: Check out code into the Go module directory 18 | uses: actions/checkout@v1 19 | 20 | - name: Setup 21 | run: make setup 22 | 23 | - name: Tests 24 | run: make integration-ci 25 | 26 | - name: Send coverage 27 | uses: codecov/codecov-action@v1 28 | with: 29 | file: ./coverage.txt 30 | flags: unittests 31 | name: codecov-umbrella 32 | fail_ci_if_error: false 33 | -------------------------------------------------------------------------------- /.github/workflows/reviewdog.yml: -------------------------------------------------------------------------------- 1 | name: reviewdog 2 | on: [pull_request] 3 | jobs: 4 | golangci-lint: 5 | name: runner / golangci-lint 6 | runs-on: ubuntu-latest 7 | steps: 8 | - name: Check out code into the Go module directory 9 | uses: actions/checkout@v1 10 | - name: golangci-lint 11 | uses: docker://reviewdog/action-golangci-lint:v1.6 # Pre-built image 12 | with: 13 | github_token: ${{ secrets.github_token }} 14 | golangci_lint_flags: "--config=.golangci.yml -D golint -D errcheck" 15 | reporter: github-pr-review 16 | 17 | # Use golint via golangci-lint binary with "warning" level. 18 | golint: 19 | name: runner / golint 20 | runs-on: ubuntu-latest 21 | steps: 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v1 24 | - name: golint 25 | uses: reviewdog/action-golangci-lint@v1 26 | with: 27 | github_token: ${{ secrets.github_token }} 28 | golangci_lint_flags: "--disable-all -E golint" 29 | tool_name: golint # Change reporter name. 30 | level: error 31 | reporter: github-pr-review 32 | 33 | errcheck: 34 | name: runner / errcheck 35 | runs-on: ubuntu-latest 36 | steps: 37 | - name: Check out code into the Go module directory 38 | uses: actions/checkout@v1 39 | - name: errcheck 40 | uses: reviewdog/action-golangci-lint@v1 41 | with: 42 | github_token: ${{ secrets.github_token }} 43 | golangci_lint_flags: "--disable-all -E errcheck" 44 | tool_name: errcheck 45 | level: warning 46 | reporter: github-pr-review 47 | 48 | staticcheck: 49 | name: runner / staticcheck 50 | runs-on: ubuntu-latest 51 | steps: 52 | - uses: actions/checkout@v2 53 | - uses: reviewdog/action-staticcheck@v1 54 | with: 55 | github_token: ${{ secrets.github_token }} 56 | reporter: github-pr-review 57 | filter_mode: nofilter 58 | fail_on_error: true 59 | 60 | misspell: 61 | name: runner / misspell 62 | runs-on: ubuntu-latest 63 | steps: 64 | - uses: actions/checkout@v2 65 | - uses: reviewdog/action-misspell@v1 66 | with: 67 | github_token: ${{ secrets.github_token }} 68 | locale: "US" 69 | reporter: github-pr-review 70 | 71 | languagetool: 72 | name: runner / languagetool 73 | runs-on: ubuntu-latest 74 | steps: 75 | - uses: actions/checkout@v2 76 | - uses: reviewdog/action-languagetool@v1 77 | with: 78 | github_token: ${{ secrets.github_token }} 79 | reporter: github-pr-review 80 | level: info 81 | patterns: | 82 | **/*.md 83 | !**/testdata/** 84 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | coverage.* 14 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | # This file contains all available configuration options 2 | # with their default values. 3 | 4 | # options for analysis running 5 | run: 6 | # timeout for analysis, e.g. 30s, 5m, default is 1m 7 | deadline: 10m 8 | 9 | # output configuration options 10 | output: 11 | # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number" 12 | format: colored-line-number 13 | 14 | # print lines of code with issue, default is true 15 | print-issued-lnes: true 16 | 17 | # print linter name in the end of issue text, default is true 18 | print-linter-name: true 19 | 20 | # all available settings of specific linters 21 | linters-settings: 22 | golint: 23 | # minimal confidence for issues, default is 0.8 24 | min-confidence: 0.8 25 | gocyclo: 26 | # minimal code complexity to report, 30 by default (but we recommend 10-20) 27 | min-complexity: 20 28 | maligned: 29 | # print struct with more effective memory layout or not, false by default 30 | suggest-new: true 31 | goimports: 32 | local-prefixes: github.com/leveeml/rabbids 33 | lll: 34 | # max line length, lines longer will be reported. Default is 120. 35 | # '\t' is counted as 1 character by default, and can be changed with the tab-width option 36 | line-length: 130 37 | # tab width in spaces. Default to 1. 38 | tab-width: 4 39 | unused: 40 | # treat code as a program (not a library) and report unused exported identifiers; default is false. 41 | # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: 42 | # if it's called for subdir of a project it can't find funcs usages. All text editor integrations 43 | # with golangci-lint call it on a directory with the changed file. 44 | check-exported: false 45 | unparam: 46 | # Inspect exported functions, default is false. Set to true if no external program/library imports your code. 47 | # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: 48 | # if it's called for subdir of a project it can't find external interfaces. All text editor integrations 49 | # with golangci-lint call it on a directory with the changed file. 50 | check-exported: false 51 | wsl: 52 | # If true append is only allowed to be cuddled if appending value is 53 | # matching variables, fields or types on line above. Default is true. 54 | strict-append: true 55 | # Allow calls and assignments to be cuddled as long as the lines have any 56 | # matching variables, fields or types. Default is true. 57 | allow-assign-and-call: true 58 | # Allow multiline assignments to be cuddled. Default is true. 59 | allow-multiline-assign: true 60 | # Allow declarations (var) to be cuddled. 61 | allow-cuddle-declarations: true 62 | # Allow trailing comments in ending of blocks 63 | allow-trailing-comment: false 64 | # Force newlines in end of case at this limit (0 = never). 65 | force-case-trailing-whitespace: 0 66 | 67 | linters: 68 | enable: 69 | - goconst 70 | - godot 71 | - gofmt 72 | - wsl 73 | - unparam 74 | - gocyclo 75 | - maligned 76 | - misspell 77 | 78 | issues: 79 | # List of regexps of issue texts to exclude, empty list by default. 80 | # But independently from this option we use default exclude patterns, 81 | # it can be disabled by `exclude-use-default: false`. To list all 82 | # excluded by default patterns execute `golangci-lint run --help` 83 | # exclude: 84 | # - newHTTP - result 1 (error) is always nil 85 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 [Levee] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SOURCE_FILES?=$$(go list ./... | grep -v /vendor/) 2 | TEST_PATTERN?=./... 3 | TEST_OPTIONS?=-race 4 | 5 | setup: ## Install all the build and lint dependencies 6 | sudo curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin v1.27.0 7 | GO111MODULE=off go get github.com/mfridman/tparse 8 | GO111MODULE=off go get golang.org/x/tools/cmd/cover 9 | go get -v -t ./... 10 | 11 | test: ## Run all the tests 12 | go test $(TEST_OPTIONS) -covermode=atomic -coverprofile=coverage.txt -timeout=1m -cover -json $(SOURCE_FILES) | $$(go env GOPATH)/bin/tparse -all 13 | 14 | integration: ## Run all the integration tests 15 | go test $(TEST_OPTIONS) -covermode=atomic -coverprofile=coverage.txt -integration -timeout=5m -cover -json $(SOURCE_FILES) | $$(go env GOPATH)/bin/tparse -top -all -dump 16 | 17 | integration-ci: ## Run all the integration tests without any log and test dump 18 | go test $(TEST_OPTIONS) -covermode=atomic -coverprofile=coverage.txt -short -integration -timeout=5m -cover -json $(SOURCE_FILES) | $$(go env GOPATH)/bin/tparse -top -smallscreen -all 19 | 20 | bench: ## Run the benchmark tests 21 | go test -bench=. $(TEST_PATTERN) 22 | 23 | cover: integration ## Run all the tests and opens the coverage report 24 | go tool cover -html=coverage.txt 25 | 26 | fmt: ## gofmt and goimports all go files 27 | find . -name '*.go' -not -wholename './vendor/*' | while read -r file; do gofmt -w -s "$$file"; goimports -w "$$file"; done 28 | 29 | lint: ## Run all the linters 30 | golangci-lint run 31 | 32 | # Absolutely awesome: http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html 33 | help: 34 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 35 | 36 | .DEFAULT_GOAL := help 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rabbids 2 | 3 | A library to create AMQP consumers and producers nicely. 4 | 5 | [![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square)](LICENSE.md) 6 | [![Go](https://img.shields.io/github/workflow/status/leveeml/rabbids/Go/main?style=flat-square)](https://github.com/leveeml/rabbids/actions?query=workflow%3AGo) 7 | [![Coverage Status](https://img.shields.io/codecov/c/github/leveeml/rabbids/main.svg?style=flat-square)](https://codecov.io/gh/leveeml/rabbids) 8 | [![Go Doc](https://img.shields.io/badge/godoc-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/leveeml/rabbids) 9 | [![Go Report Card](https://goreportcard.com/badge/github.com/leveeml/rabbids?style=flat-square)](https://goreportcard.com/report/github.com/leveeml/rabbids) 10 | 11 | - A wrapper over [amqp](https://github.com/streadway/amqp) to make possible declare all the blocks (exchanges, queues, dead-letters, bindings) from a YAML or struct. 12 | - Handle connection problems 13 | - reconnect when a connection is lost or closed. 14 | - retry with exponential backoff for sending messages 15 | - Go channel API for the producer (we are fans of github.com/rafaeljesus/rabbus API). 16 | - Support for multiple connections. 17 | - Delayed messages - send messages to arrive in the queue only after the time duration is passed. 18 | - The consumer uses a handler approach, so it's possible to add middlewares wrapping the handler 19 | 20 | ## Installation 21 | 22 | ```bash 23 | go get -u github.com/leveeml/rabbids 24 | ``` 25 | 26 | ## Usage 27 | 28 | We create some examples inside the `_example` directory. 29 | To run the examples first you need a rabbitMQ server running. 30 | If you didn't have a server already running you can run one with docker: 31 | 32 | ```sh 33 | docker run -d -p 15672:15672 -p 5672:5672 rabbitmq:3-management 34 | ``` 35 | 36 | The examples expect an ENV var `RABBITMQ_ADDRESS` with the amqp address. 37 | 38 | In one console tab run the consumer: 39 | 40 | ```sh 41 | cd _examples 42 | export RABBITMQ_ADDRESS=amqp://0.0.0.0:5672 43 | go run consumer/main.go 44 | ``` 45 | 46 | In another tab run the producer: 47 | 48 | ```sh 49 | cd _examples 50 | export RABBITMQ_ADDRESS=amqp://0.0.0.0:5672 51 | go run producer/main.go 52 | ``` 53 | 54 | Or send some delayed messages: 55 | 56 | ```sh 57 | cd _examples 58 | export RABBITMQ_ADDRESS=amqp://0.0.0.0:5672 59 | go run delay-message/main.go 60 | ``` 61 | 62 | ## Delayed Messages 63 | 64 | The delayed message implementation is based on the implementation created by the NServiceBus project. 65 | For more information go to the docs [here](https://docs.particular.net/transports/rabbitmq/delayed-delivery). 66 | 67 | ## MessageHandler 68 | 69 | MessageHandler is an interface expected by a consumer to process the messages from rabbitMQ. 70 | See the godocs for more details. If you don't need the close something you can use the `rabbids.MessageHandlerFunc` to pass a function as a MessageHandler. 71 | 72 | ## Concurency 73 | 74 | Every consumer runs on a separated goroutine and by default process every message (call the MessageHandler) synchronously but it's possible to change that and process the messages with a pool of goroutines. 75 | To make this you need to set the `worker` attribute inside the ConsumerConfig with the number of concurrent workers you need. [example](https://github.com/leveeml/rabbids/blob/master/_examples/rabbids.yaml#L29). 76 | -------------------------------------------------------------------------------- /_examples/consumer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | "time" 9 | 10 | "github.com/leveeml/rabbids" 11 | ) 12 | 13 | func main() { 14 | // Create the initial config from a yaml file 15 | config, err := rabbids.ConfigFromFilename("rabbids.yaml") 16 | if err != nil { 17 | log.Fatalf("failed getting the rabbids config from file: %s", err) 18 | } 19 | 20 | // register all the handlers for all the consumers 21 | // The name MUST be equal to the name used inside the yaml or in the map index 22 | config.RegisterHandler("consumer-example-1", rabbids.MessageHandlerFunc(processExample1)) 23 | 24 | rab, err := rabbids.New(config, logRabbids) 25 | if err != nil { 26 | log.Fatalf("failed to create the rabbids client: %s", err) 27 | } 28 | 29 | // start running the consumers 30 | stop, err := rabbids.StartSupervisor(rab, time.Second) 31 | if err != nil { 32 | log.Fatalf("failed to run the consumer supervisor: %s", err) 33 | } 34 | 35 | sig := make(chan os.Signal) 36 | signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) 37 | <-sig 38 | log.Print("received an interrupt signal, shutting down the consumers") 39 | // shutdown the consumers 40 | stop() 41 | } 42 | 43 | func processExample1(m rabbids.Message) { 44 | log.Printf("[consumer-example-1] ID: %s, key: %s, body: %s", m.MessageId, m.RoutingKey, string(m.Body)) 45 | 46 | m.Ack(false) 47 | } 48 | 49 | func logRabbids(message string, fields rabbids.Fields) { 50 | format := "[rabbids] " + message + " fields: " 51 | values := []interface{}{} 52 | 53 | for k, v := range fields { 54 | format += "%s=%v " 55 | 56 | values = append(values, k, v) 57 | } 58 | 59 | log.Printf(format, values...) 60 | } 61 | -------------------------------------------------------------------------------- /_examples/delay-message/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "time" 7 | 8 | "github.com/leveeml/rabbids" 9 | ) 10 | 11 | type delayEvent struct { 12 | ID int `json:"id,omitempty"` 13 | CreatedAt time.Time `json:"created_at,omitempty"` 14 | ExpectedDeliveryAt time.Time `json:"expected_process_at,omitempty"` 15 | } 16 | 17 | func main() { 18 | // showing another way to create a new producer directly without the config and rabbids 19 | producer, err := rabbids.NewProducer( 20 | os.Getenv("RABBITMQ_ADDRESS"), 21 | rabbids.WithLogger(logRabbids), 22 | ) 23 | if err != nil { 24 | log.Fatalf("failed to create the producer: %s", err) 25 | } 26 | 27 | for i := 1; i <= 10; i++ { 28 | duration := time.Minute * time.Duration(i) 29 | event := delayEvent{ 30 | ID: i, 31 | CreatedAt: time.Now(), 32 | ExpectedDeliveryAt: time.Now().Add(duration), 33 | } 34 | queue := "queue-consumer-example-1" 35 | err := producer.Send(rabbids.NewDelayedPublishing(queue, duration, event)) 36 | if err != nil { 37 | log.Printf("failed to publish a message: %s", err) 38 | continue 39 | } 40 | log.Printf("send message with delay: ID: %d, deliveryAt: %s", event.ID, event.ExpectedDeliveryAt) 41 | } 42 | 43 | err = producer.Close() 44 | if err != nil { 45 | log.Printf("faield to close the producer: %s", err) 46 | } 47 | } 48 | 49 | func logRabbids(message string, fields rabbids.Fields) { 50 | format := "[rabbids] " + message + " fields: " 51 | values := []interface{}{} 52 | 53 | for k, v := range fields { 54 | format += "%s=%v " 55 | 56 | values = append(values, k, v) 57 | } 58 | 59 | log.Printf(format, values...) 60 | } 61 | -------------------------------------------------------------------------------- /_examples/producer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "os/signal" 7 | "sync" 8 | "syscall" 9 | "time" 10 | 11 | "github.com/leveeml/rabbids" 12 | ) 13 | 14 | type event struct { 15 | ID int `json:"id"` 16 | CreatedAt time.Time `json:"created_at"` 17 | } 18 | 19 | func main() { 20 | var wg sync.WaitGroup 21 | 22 | // Create the initial config from a yaml file 23 | config, err := rabbids.ConfigFromFilename("rabbids.yaml") 24 | if err != nil { 25 | log.Fatalf("failed getting the rabbids config from file: %s", err) 26 | } 27 | 28 | rab, err := rabbids.New(config, logRabbids) 29 | if err != nil { 30 | log.Fatalf("failed to create the rabbids client: %s", err) 31 | } 32 | 33 | producer, err := rab.CreateProducer("default") 34 | if err != nil { 35 | log.Fatalf("failed to create the producer: %s", err) 36 | } 37 | 38 | // goroutine to publish some messages 39 | tick := time.NewTicker(3 * time.Second) 40 | wg.Add(1) 41 | go func() { 42 | defer wg.Done() 43 | currentID := 0 44 | for t := range tick.C { 45 | currentID++ 46 | key := "example.user.updated" 47 | if currentID%2 == 0 { 48 | key = "example.company.updated" 49 | } 50 | err := producer.Send(rabbids.NewPublishing( 51 | "events", 52 | key, 53 | event{ID: currentID, CreatedAt: t}, 54 | )) 55 | if err != nil { 56 | log.Printf("failed to publish a message: %s", err) 57 | } 58 | } 59 | }() 60 | 61 | sig := make(chan os.Signal) 62 | signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) 63 | <-sig 64 | log.Print("received an interrupt signal, shutting down") 65 | tick.Stop() 66 | wg.Wait() 67 | err = producer.Close() 68 | if err != nil { 69 | log.Printf("faield to close the producer: %s", err) 70 | } 71 | } 72 | 73 | func logRabbids(message string, fields rabbids.Fields) { 74 | format := "[rabbids] " + message + " fields: " 75 | values := []interface{}{} 76 | 77 | for k, v := range fields { 78 | format += "%s=%v " 79 | 80 | values = append(values, k, v) 81 | } 82 | 83 | log.Printf(format, values...) 84 | } 85 | -------------------------------------------------------------------------------- /_examples/rabbids.yaml: -------------------------------------------------------------------------------- 1 | connections: 2 | default: 3 | dsn: "${RABBITMQ_ADDRESS:=amqp://0.0.0.0:5672}" 4 | timeout: 1s 5 | sleep: 500ms 6 | exchanges: 7 | events: 8 | type: topic 9 | options: 10 | no_wait: false 11 | fallback: 12 | type: topic 13 | dead_letters: 14 | fallback: 15 | queue: 16 | name: "fallback" 17 | options: 18 | durable: true 19 | args: 20 | "x-dead-letter-exchange": "" 21 | "x-message-ttl": 300000 22 | bindings: 23 | - routing_keys: ["#"] 24 | exchange: fallback 25 | consumers: 26 | consumer-example-1: 27 | connection: default 28 | dead_letter: fallback 29 | workers: 3 30 | prefetch_count: 10 31 | queue: 32 | name: "queue-consumer-example-1" 33 | options: 34 | durable: true 35 | args: 36 | "x-dead-letter-exchange": "fallback" 37 | "x-dead-letter-routing-key": "queue-consumer-example-1" 38 | bindings: 39 | - routing_keys: 40 | - "*.user.*" 41 | - "*.company.*" 42 | exchange: events 43 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package rabbids 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "time" 10 | 11 | "github.com/a8m/envsubst" 12 | "github.com/mitchellh/mapstructure" 13 | "github.com/streadway/amqp" 14 | yaml "gopkg.in/yaml.v3" 15 | ) 16 | 17 | const ( 18 | Version = "0.0.1" 19 | DefaultTimeout = 2 * time.Second 20 | DefaultSleep = 500 * time.Millisecond 21 | DefaultRetries = 5 22 | ) 23 | 24 | // File represents the file operations needed to works with our config loader. 25 | type File interface { 26 | io.Reader 27 | Stat() (os.FileInfo, error) 28 | } 29 | 30 | // Config describes all available options to declare all the components used by 31 | // rabbids Consumers and Producers. 32 | type Config struct { 33 | // Connections describe the connections used by consumers. 34 | Connections map[string]Connection `mapstructure:"connections"` 35 | // Exchanges have all the exchanges used by consumers. 36 | // This exchanges are declared on startup of the rabbids client. 37 | Exchanges map[string]ExchangeConfig `mapstructure:"exchanges"` 38 | // DeadLetters have all the deadletters queues used internally by other queues 39 | // This will be declared at startup of the rabbids client. 40 | DeadLetters map[string]DeadLetter `mapstructure:"dead_letters"` 41 | // Consumers describes configuration list for consumers. 42 | Consumers map[string]ConsumerConfig `mapstructure:"consumers"` 43 | // Registered Message handlers used by consumers 44 | Handlers map[string]MessageHandler 45 | } 46 | 47 | // Connection describe a config for one connection. 48 | type Connection struct { 49 | DSN string `mapstructure:"dsn"` 50 | Timeout time.Duration `mapstructure:"timeout"` 51 | Sleep time.Duration `mapstructure:"sleep"` 52 | Retries int `mapstructure:"retries"` 53 | } 54 | 55 | // ConsumerConfig describes consumer's configuration. 56 | type ConsumerConfig struct { 57 | Connection string `mapstructure:"connection"` 58 | Workers int `mapstructure:"workers"` 59 | PrefetchCount int `mapstructure:"prefetch_count"` 60 | DeadLetter string `mapstructure:"dead_letter"` 61 | Queue QueueConfig `mapstructure:"queue"` 62 | Options Options `mapstructure:"options"` 63 | } 64 | 65 | // ExchangeConfig describes exchange's configuration. 66 | type ExchangeConfig struct { 67 | Type string `mapstructure:"type"` 68 | Options Options `mapstructure:"options"` 69 | } 70 | 71 | // DeadLetter describe all the dead letters queues to be declared before declare other queues. 72 | type DeadLetter struct { 73 | Queue QueueConfig `mapstructure:"queue"` 74 | } 75 | 76 | // QueueConfig describes queue's configuration. 77 | type QueueConfig struct { 78 | Name string `mapstructure:"name"` 79 | Bindings []Binding `mapstructure:"bindings"` 80 | Options Options `mapstructure:"options"` 81 | } 82 | 83 | // Binding describe how a queue connects to a exchange. 84 | type Binding struct { 85 | Exchange string `mapstructure:"exchange"` 86 | RoutingKeys []string `mapstructure:"routing_keys"` 87 | Options Options `mapstructure:"options"` 88 | } 89 | 90 | // Options describes optionals configuration 91 | // for consumer, queue, bindings and exchanges declaration. 92 | type Options struct { 93 | Durable bool `mapstructure:"durable"` 94 | Internal bool `mapstructure:"internal"` 95 | AutoDelete bool `mapstructure:"auto_delete"` 96 | Exclusive bool `mapstructure:"exclusive"` 97 | NoWait bool `mapstructure:"no_wait"` 98 | NoLocal bool `mapstructure:"no_local"` 99 | AutoAck bool `mapstructure:"auto_ack"` 100 | Args amqp.Table `mapstructure:"args"` 101 | } 102 | 103 | func setConfigDefaults(config *Config) { 104 | for k := range config.Connections { 105 | cfg := config.Connections[k] 106 | if cfg.Retries == 0 { 107 | cfg.Retries = DefaultRetries 108 | } 109 | 110 | if cfg.Sleep == 0 { 111 | cfg.Sleep = DefaultSleep 112 | } 113 | 114 | if cfg.Timeout == 0 { 115 | cfg.Timeout = DefaultTimeout 116 | } 117 | 118 | config.Connections[k] = cfg 119 | } 120 | 121 | for k := range config.Consumers { 122 | cfg := config.Consumers[k] 123 | if cfg.Workers <= 0 { 124 | cfg.Workers = 1 125 | } 126 | 127 | if cfg.PrefetchCount <= 0 { 128 | // we need at least 2 more messages than our worker to be able to see workers blocked 129 | cfg.PrefetchCount = cfg.Workers + 2 130 | } 131 | 132 | config.Consumers[k] = cfg 133 | } 134 | } 135 | 136 | // RegisterHandler is used to set the MessageHandler used by one Consumer. 137 | // The consumerName MUST be equal as the name used by the Consumer 138 | // (the key inside the map of consumers). 139 | func (c *Config) RegisterHandler(consumerName string, h MessageHandler) { 140 | if c.Handlers == nil { 141 | c.Handlers = map[string]MessageHandler{} 142 | } 143 | 144 | c.Handlers[consumerName] = h 145 | } 146 | 147 | // ConfigFromFilename is a wrapper to open the file and pass to ConfigFromFile. 148 | func ConfigFromFilename(filename string) (*Config, error) { 149 | file, err := os.Open(filename) 150 | if err != nil { 151 | return nil, fmt.Errorf("failed to open %s: %w", filename, err) 152 | } 153 | 154 | defer file.Close() 155 | 156 | return ConfigFromFile(file) 157 | } 158 | 159 | // ConfigFromFilename read a YAML file and convert it into a Config struct 160 | // with all the configuration to build the Consumers and producers. 161 | // Also, it Is possible to use environment variables values inside the YAML file. 162 | // The syntax is like the syntax used inside the docker-compose file. 163 | // To use a required variable just use like this: ${ENV_NAME} 164 | // and to put an default value you can use: ${ENV_NAME:=some-value} inside any value. 165 | // If a required variable didn't exist, an error will be returned. 166 | func ConfigFromFile(file File) (*Config, error) { 167 | input := map[string]interface{}{} 168 | output := &Config{} 169 | 170 | body, err := ioutil.ReadAll(file) 171 | if err != nil { 172 | return nil, fmt.Errorf("failed to read the file: %w", err) 173 | } 174 | 175 | in, err := envsubst.BytesRestricted(body, true, false) 176 | if err != nil { 177 | return nil, fmt.Errorf("failed to parse some environment variables: %w", err) 178 | } 179 | 180 | stat, err := file.Stat() 181 | if err != nil { 182 | return nil, fmt.Errorf("failed to get the file stats: %w", err) 183 | } 184 | 185 | switch getConfigType(stat.Name()) { 186 | case "yaml", "yml": 187 | err = yaml.Unmarshal(in, &input) 188 | if err != nil { 189 | return nil, fmt.Errorf("failed to decode the yaml configuration. %w", err) 190 | } 191 | default: 192 | return nil, fmt.Errorf("file extension %s not supported", getConfigType(stat.Name())) 193 | } 194 | 195 | decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ 196 | Metadata: nil, 197 | Result: output, 198 | WeaklyTypedInput: true, 199 | DecodeHook: mapstructure.ComposeDecodeHookFunc( 200 | mapstructure.StringToTimeDurationHookFunc(), 201 | mapstructure.StringToSliceHookFunc(","), 202 | ), 203 | }) 204 | if err != nil { 205 | return nil, err 206 | } 207 | 208 | err = decoder.Decode(input) 209 | 210 | return output, err 211 | } 212 | 213 | func getConfigType(file string) string { 214 | ext := filepath.Ext(file) 215 | 216 | if len(ext) > 1 { 217 | return ext[1:] 218 | } 219 | 220 | return "" 221 | } 222 | -------------------------------------------------------------------------------- /config_test.go: -------------------------------------------------------------------------------- 1 | package rabbids 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func Test_withDefaults(t *testing.T) { 11 | config := Config{ 12 | Connections: map[string]Connection{ 13 | "de": {DSN: "amqp://localhost:5672"}, 14 | }, 15 | Consumers: map[string]ConsumerConfig{ 16 | "consumer1": { 17 | Connection: "server1", 18 | Queue: QueueConfig{Name: "fooo"}, 19 | }, 20 | }, 21 | } 22 | 23 | setConfigDefaults(&config) 24 | require.Equal(t, 5, config.Connections["de"].Retries) 25 | require.Equal(t, 2*time.Second, config.Connections["de"].Timeout) 26 | require.Equal(t, 500*time.Millisecond, config.Connections["de"].Sleep) 27 | require.Equal(t, 1, config.Consumers["consumer1"].Workers) 28 | require.Equal(t, 3, config.Consumers["consumer1"].PrefetchCount) 29 | } 30 | -------------------------------------------------------------------------------- /consumer.go: -------------------------------------------------------------------------------- 1 | package rabbids 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | "gopkg.in/tomb.v2" 8 | 9 | "github.com/ivpusic/grpool" 10 | "github.com/streadway/amqp" 11 | ) 12 | 13 | // Consumer is a high level rabbitMQ consumer. 14 | type Consumer struct { 15 | handler MessageHandler 16 | number int64 17 | name string 18 | queue string 19 | workerPool *grpool.Pool 20 | opts Options 21 | channel *amqp.Channel 22 | t tomb.Tomb 23 | log LoggerFN 24 | } 25 | 26 | // Run start a goroutine to consume messages from a queue and pass to one runner. 27 | func (c *Consumer) Run() { 28 | c.t.Go(func() error { 29 | defer func() { 30 | if c.channel == nil { 31 | return 32 | } 33 | err := c.channel.Close() 34 | if err != nil { 35 | c.log("Error closing the consumer channel", Fields{"error": err, "name": c.name}) 36 | } 37 | }() 38 | d, err := c.channel.Consume(c.queue, fmt.Sprintf("rabbitmq-%s-%d", c.name, c.number), 39 | c.opts.AutoAck, 40 | c.opts.Exclusive, 41 | c.opts.NoLocal, 42 | c.opts.NoWait, 43 | c.opts.Args) 44 | if err != nil { 45 | c.log("Failed to start consume", Fields{"error": err, "name": c.name}) 46 | return err 47 | } 48 | dying := c.t.Dying() 49 | closed := c.channel.NotifyClose(make(chan *amqp.Error)) 50 | for { 51 | select { 52 | case <-dying: 53 | // When dying we wait for any remaining worker to finish and close the handler 54 | c.workerPool.WaitAll() 55 | c.handler.Close() 56 | return nil 57 | case err := <-closed: 58 | return err 59 | case msg, ok := <-d: 60 | if !ok { 61 | return errors.New("internal channel closed") 62 | } 63 | c.workerPool.WaitCount(1) 64 | fn := func(msg amqp.Delivery) func() { 65 | return func() { 66 | c.handler.Handle(Message{msg}) 67 | c.workerPool.JobDone() 68 | } 69 | }(msg) 70 | // When Workers goroutines are in flight, Send a Job blocks until one of the 71 | // workers finishes. 72 | c.workerPool.JobQueue <- fn 73 | } 74 | } 75 | }) 76 | } 77 | 78 | // Kill will try to stop the internal work. 79 | func (c *Consumer) Kill() { 80 | c.t.Kill(nil) 81 | <-c.t.Dead() 82 | } 83 | 84 | // Alive returns true if the tomb is not in a dying or dead state. 85 | func (c *Consumer) Alive() bool { 86 | return c.t.Alive() 87 | } 88 | 89 | // Name return the consumer name. 90 | func (c *Consumer) Name() string { 91 | return c.name 92 | } 93 | -------------------------------------------------------------------------------- /declarations.go: -------------------------------------------------------------------------------- 1 | package rabbids 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/pkg/errors" 7 | "github.com/streadway/amqp" 8 | ) 9 | 10 | // declarations is the block responsible for create consumers and restart the rabbitMQ connections. 11 | type declarations struct { 12 | config *Config 13 | log LoggerFN 14 | } 15 | 16 | func (f *declarations) declareExchange(ch *amqp.Channel, name string) error { 17 | if len(name) == 0 { 18 | return fmt.Errorf("receive a blank exchange. Wrong config?") 19 | } 20 | 21 | ex, ok := f.config.Exchanges[name] 22 | if !ok { 23 | f.log("exchange config didn't exist, we will try to continue", Fields{"name": name}) 24 | return nil 25 | } 26 | 27 | f.log("declaring exchange", Fields{ 28 | "ex": name, 29 | "type": ex.Type, 30 | "options": ex.Options, 31 | }) 32 | 33 | err := ch.ExchangeDeclare( 34 | name, 35 | ex.Type, 36 | ex.Options.Durable, 37 | ex.Options.AutoDelete, 38 | ex.Options.Internal, 39 | ex.Options.NoWait, 40 | assertRightTableTypes(ex.Options.Args)) 41 | if err != nil { 42 | return fmt.Errorf("failed to declare the exchange %s, err: %w", name, err) 43 | } 44 | 45 | return nil 46 | } 47 | 48 | func (f *declarations) declareQueue(ch *amqp.Channel, queue QueueConfig) error { 49 | f.log("declaring queue", Fields{ 50 | "queue": queue.Name, 51 | "options": queue.Options, 52 | }) 53 | 54 | q, err := ch.QueueDeclare( 55 | queue.Name, 56 | queue.Options.Durable, 57 | queue.Options.AutoDelete, 58 | queue.Options.Exclusive, 59 | queue.Options.NoWait, 60 | assertRightTableTypes(queue.Options.Args)) 61 | if err != nil { 62 | return fmt.Errorf("failed to declare the queue \"%s\"", queue.Name) 63 | } 64 | 65 | for _, b := range queue.Bindings { 66 | f.log("declaring queue bind", Fields{ 67 | "queue": queue.Name, 68 | "exchange": b.Exchange, 69 | }) 70 | 71 | err = f.declareExchange(ch, b.Exchange) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | for _, k := range b.RoutingKeys { 77 | err = ch.QueueBind(q.Name, k, b.Exchange, 78 | b.Options.NoWait, assertRightTableTypes(b.Options.Args)) 79 | if err != nil { 80 | return errors.Wrapf(err, "failed to bind the queue \"%s\" to exchange: \"%s\"", q.Name, b.Exchange) 81 | } 82 | } 83 | } 84 | 85 | return nil 86 | } 87 | 88 | func (f *declarations) declareDeadLetters(ch *amqp.Channel, name string) error { 89 | f.log("declaring deadletter", Fields{"dlx": name}) 90 | 91 | dead, ok := f.config.DeadLetters[name] 92 | if !ok { 93 | f.log("deadletter config didn't exist, we will try to continue", Fields{"dlx": name}) 94 | return nil 95 | } 96 | 97 | err := f.declareQueue(ch, dead.Queue) 98 | 99 | return errors.Wrapf(err, "failed to declare the queue for deadletter %s", name) 100 | } 101 | 102 | func assertRightTableTypes(args amqp.Table) amqp.Table { 103 | nArgs := amqp.Table{} 104 | 105 | for k, v := range args { 106 | switch v := v.(type) { 107 | case int: 108 | nArgs[k] = int64(v) 109 | default: 110 | nArgs[k] = v 111 | } 112 | } 113 | 114 | return nArgs 115 | } 116 | -------------------------------------------------------------------------------- /declarations_test.go: -------------------------------------------------------------------------------- 1 | package rabbids 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/streadway/amqp" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func Test_assertRightArgsTypes(t *testing.T) { 11 | type args struct { 12 | args amqp.Table 13 | } 14 | 15 | tests := []struct { 16 | name string 17 | args args 18 | want amqp.Table 19 | }{ 20 | { 21 | "int to int64", 22 | args{amqp.Table{"x-dead-letter-exchange": "", "x-message-ttl": int(3600000)}}, 23 | amqp.Table{"x-dead-letter-exchange": "", "x-message-ttl": int64(3600000)}, 24 | }, 25 | } 26 | for _, tt := range tests { 27 | ctt := tt 28 | t.Run(tt.name, func(t *testing.T) { 29 | got := assertRightTableTypes(ctt.args.args) 30 | require.Exactly(t, ctt.want, got) 31 | }) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /delayed_messages.go: -------------------------------------------------------------------------------- 1 | package rabbids 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "math" 7 | "sync" 8 | "time" 9 | 10 | "github.com/streadway/amqp" 11 | ) 12 | 13 | const ( 14 | maxNumberOfBitsToUse int = 28 15 | maxLevel int = maxNumberOfBitsToUse - 1 16 | 17 | MaxDelay time.Duration = ((1 << maxNumberOfBitsToUse) - 1) * time.Second 18 | DelayDeliveryExchange string = "rabbids.delay-delivery" 19 | ) 20 | 21 | // delayDelivery is based on the setup of delay messages created by the NServiceBus project. 22 | // For more information go to the docs on https://docs.particular.net/transports/rabbitmq/delayed-delivery. 23 | type delayDelivery struct { 24 | delayDeclaredOnce sync.Once 25 | } 26 | 27 | // Declare create all the layers of exchanges and queues on rabbitMQ 28 | // and declare the bind between the last rabbids.delay-delivery ex and the queue. 29 | func (d *delayDelivery) Declare(ch *amqp.Channel, key string) error { 30 | var declaredErr error 31 | 32 | queue := getQueueFromRoutingKey(key) 33 | 34 | d.delayDeclaredOnce.Do(func() { 35 | declaredErr = d.build(ch) 36 | }) 37 | 38 | if declaredErr != nil { 39 | return declaredErr 40 | } 41 | 42 | return ch.QueueBind(queue, fmt.Sprintf("#.%s", queue), DelayDeliveryExchange, false, amqp.Table{}) 43 | } 44 | 45 | //nolint:funlen 46 | func (d *delayDelivery) build(ch *amqp.Channel) error { 47 | bindingKey := "1.#" 48 | 49 | for level := maxLevel; level >= 0; level-- { 50 | currentLevel := delayedLevelName(level) 51 | nextLevel := delayedLevelName(level - 1) 52 | 53 | if level == 0 { 54 | nextLevel = DelayDeliveryExchange 55 | } 56 | 57 | err := ch.ExchangeDeclare("fooo", amqp.ExchangeTopic, true, false, false, false, amqp.Table{}) 58 | if err != nil { 59 | return fmt.Errorf("failed to declare exchange \"%s\": %v", "foo", err) 60 | } 61 | 62 | err = ch.ExchangeDeclare(currentLevel, amqp.ExchangeTopic, true, false, false, false, amqp.Table{}) 63 | if err != nil { 64 | return fmt.Errorf("failed to declare exchange \"%s\": %v", currentLevel, err) 65 | } 66 | 67 | _, err = ch.QueueDeclare(currentLevel, true, false, false, false, amqp.Table{ 68 | "x-queue-mode": "lazy", 69 | "x-message-ttl": int64(math.Pow(2, float64(level)) * 1000), 70 | "x-dead-letter-exchange": nextLevel, 71 | }) 72 | if err != nil { 73 | return fmt.Errorf("failed to declare queue \"%s\": %v", currentLevel, err) 74 | } 75 | 76 | err = ch.QueueBind(currentLevel, bindingKey, currentLevel, false, amqp.Table{}) 77 | if err != nil { 78 | return fmt.Errorf("failed to bind queue \"%s\" to exchange \"%s\": %v", currentLevel, currentLevel, err) 79 | } 80 | 81 | bindingKey = "*." + bindingKey 82 | } 83 | 84 | bindingKey = "0.#" 85 | 86 | for level := maxLevel; level >= 0; level-- { 87 | currentLevel := delayedLevelName(level) 88 | nextLevel := delayedLevelName(level - 1) 89 | 90 | if level == 0 { 91 | break 92 | } 93 | 94 | err := ch.ExchangeBind(nextLevel, bindingKey, currentLevel, false, amqp.Table{}) 95 | if err != nil { 96 | return fmt.Errorf("failed to exchange the bind %s->%s: %v", currentLevel, nextLevel, err) 97 | } 98 | 99 | bindingKey = "*." + bindingKey 100 | } 101 | 102 | err := ch.ExchangeDeclare(DelayDeliveryExchange, amqp.ExchangeTopic, true, false, false, false, amqp.Table{}) 103 | if err != nil { 104 | return fmt.Errorf("failed to declare exchange %s: %v", DelayDeliveryExchange, err) 105 | } 106 | 107 | err = ch.ExchangeBind(DelayDeliveryExchange, bindingKey, delayedLevelName(0), false, amqp.Table{}) 108 | 109 | return err 110 | } 111 | 112 | // calculateRoutingKey return the routingkey and the first applicable exchange 113 | // to avoid unnecessary traversal through the delay infrastructure. 114 | func calculateRoutingKey(delay time.Duration, queue string) (string, string) { 115 | if delay > MaxDelay { 116 | delay = MaxDelay 117 | } 118 | 119 | var buf bytes.Buffer 120 | 121 | sec := uint(delay.Seconds()) 122 | firstLevel := 0 123 | 124 | for level := maxLevel; level >= 0; level-- { 125 | if firstLevel == 0 && sec&(1< Setup 37 | dockerPool, err := dockertest.NewPool("") 38 | require.NoError(t, err, "Coud not connect to docker") 39 | resource, err := dockerPool.Run("rabbitmq", "3.6.12-management", []string{}) 40 | require.NoError(t, err, "Could not start resource") 41 | 42 | // -> TearDown 43 | t.Cleanup(func() { 44 | if err := dockerPool.Purge(resource); err != nil { 45 | t.Errorf("Could not purge resource: %s", err) 46 | } 47 | }) 48 | 49 | // -> Run! 50 | for _, test := range tests { 51 | tt := test 52 | t.Run(test.scenario, func(st *testing.T) { 53 | tt.method(st, resource) 54 | }) 55 | } 56 | } 57 | 58 | func testRabbidsShouldReturnConnectionErrors(t *testing.T, _ *dockertest.Resource) { 59 | t.Parallel() 60 | 61 | c := getConfigHelper(t, "valid_queue_and_exchange_config.yml") 62 | 63 | t.Run("when we pass an invalid port", func(t *testing.T) { 64 | conn := c.Connections["default"] 65 | conn.DSN = "amqp://guest:guest@localhost:80/" 66 | conn.Sleep = 10 * time.Microsecond 67 | conn.Retries = 0 68 | c.Connections["default"] = conn 69 | 70 | _, err := rabbids.New(c, logFNHelper(t)) 71 | require.NotNil(t, err) 72 | assert.Contains(t, err.Error(), "error opening the connection \"default\": ") 73 | }) 74 | 75 | t.Run("when we pass an invalid host", func(t *testing.T) { 76 | conn := c.Connections["default"] 77 | conn.DSN = "amqp://guest:guest@10.255.255.1:5672/" 78 | conn.Sleep = 10 * time.Microsecond 79 | conn.Retries = 0 80 | c.Connections["default"] = conn 81 | 82 | _, err := rabbids.New(c, logFNHelper(t)) 83 | assert.EqualError(t, err, "error opening the connection \"default\": dial tcp 10.255.255.1:5672: i/o timeout") 84 | }) 85 | } 86 | 87 | func testConsumerProcess(t *testing.T, resource *dockertest.Resource) { 88 | t.Parallel() 89 | 90 | handler := &mockHandler{count: 0, ack: true, tb: t} 91 | config := getConfigHelper(t, "valid_queue_and_exchange_config.yml") 92 | 93 | config.Connections["default"] = setDSN(resource, config.Connections["default"]) 94 | config.RegisterHandler("messaging_consumer", handler) 95 | 96 | rab, err := rabbids.New(config, logFNHelper(t)) 97 | require.NoError(t, err, "Failed to creating rabbids") 98 | 99 | stop, err := rabbids.StartSupervisor(rab, 10*time.Millisecond) 100 | require.NoError(t, err, "Failed to create the Supervisor") 101 | 102 | defer stop() 103 | 104 | ch := getChannelHelper(t, resource) 105 | 106 | for i := 0; i < 5; i++ { 107 | err = ch.Publish("event_bus", "service.whatssapp.send", false, false, amqp.Publishing{ 108 | Body: []byte(`{"fooo": "bazzz"}`), 109 | }) 110 | require.NoError(t, err, "error publishing to rabbitMQ") 111 | } 112 | 113 | <-time.After(400 * time.Millisecond) 114 | require.EqualValues(t, 5, handler.messagesProcessed()) 115 | 116 | for _, cfg := range config.Consumers { 117 | _, err := ch.QueueDelete(cfg.Queue.Name, false, false, false) 118 | require.NoError(t, err) 119 | } 120 | 121 | for name := range config.Exchanges { 122 | err := ch.ExchangeDelete(name, false, false) 123 | require.NoError(t, err) 124 | } 125 | } 126 | 127 | func testConsumerReconnect(t *testing.T, resource *dockertest.Resource) { 128 | t.Parallel() 129 | 130 | config := getConfigHelper(t, "valid_two_connections.yml") 131 | config.Connections["default"] = setDSN(resource, config.Connections["default"]) 132 | config.Connections["test1"] = setDSN(resource, config.Connections["test1"]) 133 | received := make(chan string, 10) 134 | handler := rabbids.MessageHandlerFunc(func(m rabbids.Message) { 135 | received <- string(m.Body) 136 | err := m.Ack(false) 137 | require.NoError(t, err, "failed to ack the message") 138 | }) 139 | 140 | config.RegisterHandler("send_consumer", handler) 141 | config.RegisterHandler("response_consumer", handler) 142 | 143 | rab, err := rabbids.New(config, logFNHelper(t)) 144 | require.NoError(t, err, "failed to initialize the rabbids client") 145 | 146 | stop, err := rabbids.StartSupervisor(rab, 10*time.Millisecond) 147 | require.NoError(t, err, "Failed to create the Supervisor") 148 | 149 | defer stop() 150 | 151 | sendMessages(t, resource, "event_bus", "service.whatssapp.send", 0, 2) 152 | sendMessages(t, resource, "event_bus", "service.whatssapp.response", 3, 4) 153 | time.Sleep(1 * time.Second) 154 | require.Len(t, received, 5, "consumer should be processed 5 messages before close connections") 155 | 156 | // get the http client and force to close all the connections 157 | closeRabbitMQConnections(t, getRabbitClient(t, resource), "rabbids.test1") 158 | 159 | // send new messages 160 | sendMessages(t, resource, "event_bus", "service.whatssapp.send", 5, 6) 161 | sendMessages(t, resource, "event_bus", "service.whatssapp.response", 7, 8) 162 | time.Sleep(1 * time.Second) 163 | 164 | require.Len(t, received, 9, "consumer should be processed 9 messages") 165 | } 166 | 167 | type mockHandler struct { 168 | count int64 169 | ack bool 170 | tb testing.TB 171 | } 172 | 173 | func (m *mockHandler) Handle(msg rabbids.Message) { 174 | atomic.AddInt64(&m.count, 1) 175 | 176 | if m.ack { 177 | if err := msg.Ack(false); err != nil { 178 | m.tb.Errorf("Failed to ack the message. err: %v, tag: %d", err, msg.DeliveryTag) 179 | } 180 | 181 | return 182 | } 183 | 184 | if err := msg.Nack(false, false); err != nil { 185 | m.tb.Errorf("Failed to nack the message. err: %v, tag: %d", err, msg.DeliveryTag) 186 | } 187 | } 188 | 189 | func (m *mockHandler) Close() {} 190 | 191 | func (m *mockHandler) messagesProcessed() int64 { 192 | return atomic.LoadInt64(&m.count) 193 | } 194 | -------------------------------------------------------------------------------- /integration_helper_test.go: -------------------------------------------------------------------------------- 1 | package rabbids_test 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "path/filepath" 7 | "testing" 8 | "time" 9 | 10 | "github.com/leveeml/rabbids" 11 | rabbithole "github.com/michaelklishin/rabbit-hole" 12 | "github.com/streadway/amqp" 13 | "github.com/stretchr/testify/require" 14 | "gopkg.in/ory-am/dockertest.v3" 15 | ) 16 | 17 | var integration = flag.Bool("integration", false, "run integration tests") 18 | 19 | func integrationTest(tb testing.TB) { 20 | if !*integration { 21 | tb.SkipNow() 22 | } 23 | } 24 | 25 | func getDSN(resource *dockertest.Resource) string { 26 | return fmt.Sprintf("amqp://localhost:%s", resource.GetPort("5672/tcp")) 27 | } 28 | 29 | func getRabbitClient(t *testing.T, resource *dockertest.Resource) *rabbithole.Client { 30 | t.Helper() 31 | 32 | client, err := rabbithole.NewClient( 33 | fmt.Sprintf("http://localhost:%s", resource.GetPort("15672/tcp")), 34 | "guest", "guest") 35 | require.NoError(t, err, "Fail to create the rabbithole client") 36 | 37 | return client 38 | } 39 | 40 | // close all open connections to the rabbitmq via the management api. 41 | func closeRabbitMQConnections(t *testing.T, client *rabbithole.Client, names ...string) { 42 | t.Helper() 43 | 44 | timeout := time.After(10 * time.Second) 45 | namesIdx := make(map[string]struct{}) 46 | 47 | for _, name := range names { 48 | namesIdx[name] = struct{}{} 49 | } 50 | 51 | for { 52 | select { 53 | default: 54 | connections, err := client.ListConnections() 55 | require.NoError(t, err, "failed to get the connections") 56 | 57 | if len(connections) >= 1 { 58 | for _, c := range connections { 59 | connectionName, _ := c.ClientProperties["connection_name"].(string) 60 | if _, exists := namesIdx[connectionName]; exists { 61 | t.Logf("killing connection: (%s) name: %s", c.Name, connectionName) 62 | 63 | _, err := client.CloseConnection(c.Name) 64 | require.NoError(t, err, "impossible to kill connection", c.Name) 65 | } else { 66 | t.Logf("skipping closing the connection with name: %v", c.ClientProperties["connection_name"]) 67 | } 68 | } 69 | 70 | return 71 | } 72 | 73 | time.Sleep(time.Second) 74 | case <-timeout: 75 | t.Log("timeout for killing connection reached") 76 | 77 | return 78 | } 79 | } 80 | } 81 | 82 | //nolint:unparam 83 | func sendMessages(t *testing.T, resource *dockertest.Resource, ex, key string, start, count int) { 84 | t.Helper() 85 | 86 | conn, err := amqp.Dial(fmt.Sprintf("amqp://localhost:%s", resource.GetPort("5672/tcp"))) 87 | require.NoError(t, err, "failed to open a new connection for tests") 88 | 89 | ch, err := conn.Channel() 90 | require.NoError(t, err, "failed to open a channel for tests") 91 | 92 | for i := start; i <= count; i++ { 93 | err := ch.Publish(ex, key, false, false, amqp.Publishing{ 94 | Body: []byte(fmt.Sprintf("%d ", i)), 95 | }) 96 | require.NoError(t, err, "error publishing to rabbitMQ") 97 | } 98 | } 99 | 100 | func getConfigHelper(t *testing.T, configFile string) *rabbids.Config { 101 | t.Helper() 102 | 103 | config, err := rabbids.ConfigFromFilename(filepath.Join("testdata", configFile)) 104 | require.NoError(t, err) 105 | 106 | return config 107 | } 108 | 109 | func setDSN(resource *dockertest.Resource, conn rabbids.Connection) rabbids.Connection { 110 | conn.DSN = fmt.Sprintf("amqp://localhost:%s", resource.GetPort("5672/tcp")) 111 | return conn 112 | } 113 | 114 | func logFNHelper(tb testing.TB) rabbids.LoggerFN { 115 | if testing.Short() { 116 | return rabbids.NoOPLoggerFN 117 | } 118 | 119 | return func(message string, fields rabbids.Fields) { 120 | pattern := message + " fields: " 121 | values := []interface{}{} 122 | 123 | for k, v := range fields { 124 | pattern += "%s=%v " 125 | 126 | values = append(values, k, v) 127 | } 128 | 129 | tb.Helper() 130 | tb.Logf(pattern, values...) 131 | } 132 | } 133 | 134 | func getChannelHelper(tb testing.TB, resource *dockertest.Resource) *amqp.Channel { 135 | tb.Helper() 136 | 137 | conn, err := amqp.Dial(fmt.Sprintf("amqp://localhost:%s", resource.GetPort("5672/tcp"))) 138 | if err != nil { 139 | tb.Fatal("Failed to connect with rabbitMQ: ", err) 140 | } 141 | 142 | ch, err := conn.Channel() 143 | if err != nil { 144 | tb.Fatal("Failed to create a new channel: ", err) 145 | } 146 | 147 | return ch 148 | } 149 | 150 | func getQueueLength(t *testing.T, client *rabbithole.Client, queuename string, duration time.Duration) int { 151 | t.Helper() 152 | 153 | timeout := time.After(duration) 154 | equalCounts := 0 155 | lastCount := 0 156 | 157 | for { 158 | info, err := client.GetQueue("/", queuename) 159 | require.NoError(t, err, "error getting the queue info") 160 | 161 | if info.Messages == lastCount { 162 | equalCounts++ 163 | } else { 164 | equalCounts = 0 165 | } 166 | 167 | lastCount = info.Messages 168 | 169 | if equalCounts >= 3 { 170 | return info.Messages 171 | } 172 | 173 | select { 174 | case <-timeout: 175 | return info.Messages 176 | default: 177 | time.Sleep(time.Second) 178 | } 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /integration_producer_test.go: -------------------------------------------------------------------------------- 1 | package rabbids_test 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | "testing" 7 | "time" 8 | 9 | "github.com/leveeml/rabbids" 10 | "github.com/streadway/amqp" 11 | "github.com/stretchr/testify/require" 12 | "gopkg.in/ory-am/dockertest.v3" 13 | ) 14 | 15 | func TestBasicIntegrationProducer(t *testing.T) { 16 | integrationTest(t) 17 | 18 | tests := []struct { 19 | scenario string 20 | method func(*testing.T, *dockertest.Resource) 21 | }{ 22 | { 23 | scenario: "test producer with connection problems", 24 | method: testProducerWithReconnect, 25 | }, 26 | { 27 | scenario: "test send delay messages", 28 | method: testPublishWithDelay, 29 | }, 30 | } 31 | // -> Setup 32 | dockerPool, err := dockertest.NewPool("") 33 | require.NoError(t, err, "Coud not connect to docker") 34 | resource, err := dockerPool.Run("rabbitmq", "3.6.12-management", []string{}) 35 | require.NoError(t, err, "Could not start resource") 36 | 37 | // -> TearDown 38 | t.Cleanup(func() { 39 | if err := dockerPool.Purge(resource); err != nil { 40 | t.Errorf("Could not purge resource: %s", err) 41 | } 42 | }) 43 | 44 | // -> Run! 45 | for _, test := range tests { 46 | tt := test 47 | t.Run(test.scenario, func(st *testing.T) { 48 | tt.method(st, resource) 49 | }) 50 | } 51 | } 52 | 53 | func testProducerWithReconnect(t *testing.T, resource *dockertest.Resource) { 54 | t.Parallel() 55 | 56 | var wg sync.WaitGroup 57 | 58 | adminClient := getRabbitClient(t, resource) 59 | producer, err := rabbids.NewProducer(getDSN(resource), rabbids.WithCustomName("test-reconnect")) 60 | require.NoError(t, err, "could not connect to: ", getDSN(resource)) 61 | 62 | ch := producer.GetAMQPChannel() 63 | 64 | _, err = ch.QueueDeclare("testProducerWithReconnect", true, false, false, false, amqp.Table{}) 65 | require.NoError(t, err) 66 | 67 | var emitWithErrors int64 68 | 69 | wg.Add(1) 70 | 71 | go func() { 72 | defer wg.Done() 73 | 74 | for pErr := range producer.EmitErr() { 75 | t.Logf("received a emitErr: %v", pErr) 76 | atomic.AddInt64(&emitWithErrors, 1) 77 | } 78 | }() 79 | 80 | for i := 1; i <= 1000; i++ { 81 | if i%100 == 0 { 82 | closeRabbitMQConnections(t, adminClient, "test-reconnect") 83 | } 84 | producer.Emit() <- rabbids.NewPublishing("", "testProducerWithReconnect", 85 | map[string]int{"test": i}, 86 | ) 87 | time.Sleep(time.Millisecond) 88 | } 89 | 90 | err = producer.Close() 91 | require.NoError(t, err, "error closing the connection") 92 | wg.Wait() 93 | 94 | count := getQueueLength(t, adminClient, "testProducerWithReconnect", 40*time.Second) 95 | t.Logf("Finished published with %d messages inside the queue and %d messages with error", count, emitWithErrors) 96 | } 97 | 98 | func testPublishWithDelay(t *testing.T, resource *dockertest.Resource) { 99 | t.Parallel() 100 | 101 | adminClient := getRabbitClient(t, resource) 102 | producer, err := rabbids.NewProducer(getDSN(resource)) 103 | require.NoError(t, err, "could not connect to: ", getDSN(resource)) 104 | 105 | ch := producer.GetAMQPChannel() 106 | 107 | _, err = ch.QueueDeclare("testPublishWithDelay", true, false, false, false, amqp.Table{}) 108 | require.NoError(t, err) 109 | 110 | err = producer.Send(rabbids.NewDelayedPublishing( 111 | "testPublishWithDelay", 112 | 10*time.Second, 113 | map[string]string{"test": "fooo"}, 114 | )) 115 | require.NoError(t, err, "error on rab.Send") 116 | time.Sleep(15 * time.Second) 117 | 118 | err = producer.Close() 119 | require.NoError(t, err, "error closing the connection") 120 | 121 | count := getQueueLength(t, adminClient, "testPublishWithDelay", 10*time.Second) 122 | require.Equal(t, 1, count, "expecting the message inside the queue") 123 | } 124 | -------------------------------------------------------------------------------- /logger.go: -------------------------------------------------------------------------------- 1 | package rabbids 2 | 3 | type Fields map[string]interface{} 4 | 5 | type LoggerFN func(message string, fields Fields) 6 | 7 | func NoOPLoggerFN(message string, fields Fields) {} 8 | -------------------------------------------------------------------------------- /message.go: -------------------------------------------------------------------------------- 1 | package rabbids 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/google/uuid" 7 | "github.com/streadway/amqp" 8 | ) 9 | 10 | // Serializer is the base interface for all message serializers. 11 | type Serializer interface { 12 | Marshal(interface{}) ([]byte, error) 13 | // Name return the name used on the content type of the messsage 14 | Name() string 15 | } 16 | 17 | // Publishing have the fields for sending a message to rabbitMQ. 18 | type Publishing struct { 19 | // Exchange name 20 | Exchange string 21 | // The routing key 22 | Key string 23 | // Data to be encoded inside the message 24 | Data interface{} 25 | // Delay is the duration to wait until the message is delivered to the queue. 26 | // The max delay period is 268,435,455 seconds, or about 8.5 years. 27 | Delay time.Duration 28 | 29 | options []PublishingOption 30 | amqp.Publishing 31 | } 32 | 33 | // PublishingError is returned by the async error reporting. 34 | // When an async publishing message is sent and an error happens 35 | // the Publishing and the error will be sent to the EmitErr channel. 36 | // To get this channel, call the EmitErr method inside the producer. 37 | type PublishingError struct { 38 | Publishing 39 | Err error 40 | } 41 | 42 | // NewPublishing create a message to be sent by some consumer. 43 | func NewPublishing(exchange, key string, data interface{}, options ...PublishingOption) Publishing { 44 | id, err := uuid.NewRandom() 45 | if err != nil { 46 | id = uuid.Must(uuid.NewUUID()) 47 | } 48 | 49 | return Publishing{ 50 | Exchange: exchange, 51 | Key: key, 52 | Data: data, 53 | Publishing: amqp.Publishing{ 54 | MessageId: id.String(), 55 | Priority: 0, 56 | Headers: amqp.Table{}, 57 | }, 58 | options: options, 59 | } 60 | } 61 | 62 | // SendWithDelay send a message to arrive the queue only after the time is passed. 63 | // The minimum delay is one second, if the delay is less than the minimum, the minimum will be used. 64 | // The max delay period is 268,435,455 seconds, or about 8.5 years. 65 | func NewDelayedPublishing(queue string, delay time.Duration, data interface{}, options ...PublishingOption) Publishing { 66 | if delay < time.Second { 67 | delay = time.Second 68 | } 69 | 70 | id, err := uuid.NewRandom() 71 | if err != nil { 72 | id = uuid.Must(uuid.NewUUID()) 73 | } 74 | 75 | key, ex := calculateRoutingKey(delay, queue) 76 | 77 | return Publishing{ 78 | Exchange: ex, 79 | Key: key, 80 | Data: data, 81 | Delay: delay, 82 | Publishing: amqp.Publishing{ 83 | Priority: 0, 84 | MessageId: id.String(), 85 | Headers: amqp.Table{}, 86 | }, 87 | options: options, 88 | } 89 | } 90 | 91 | // Message is an ampq.Delivery with some helper methods used by our systems. 92 | type Message struct { 93 | amqp.Delivery 94 | } 95 | 96 | // MessageHandler is the base interface used to consumer AMPQ messages. 97 | type MessageHandler interface { 98 | // Handle a single message, this method MUST be safe for concurrent use 99 | Handle(m Message) 100 | // Close the handler, this method is called when the consumer is closing 101 | Close() 102 | } 103 | 104 | // MessageHandlerFunc implements the MessageHandler interface. 105 | type MessageHandlerFunc func(m Message) 106 | 107 | func (h MessageHandlerFunc) Handle(m Message) { 108 | h(m) 109 | } 110 | 111 | func (h MessageHandlerFunc) Close() {} 112 | -------------------------------------------------------------------------------- /options.go: -------------------------------------------------------------------------------- 1 | package rabbids 2 | 3 | // PublishingOption represents an option you can pass to setup some data inside the Publishing. 4 | type PublishingOption func(*Publishing) 5 | 6 | // ProducerOption represents an option function to add some functionality or change the producer 7 | // state on creation time. 8 | type ProducerOption func(*Producer) error 9 | 10 | // WithPriority change the priority of the Publishing message. 11 | func WithPriority(v int) PublishingOption { 12 | return func(p *Publishing) { 13 | if v < 0 { 14 | v = 0 15 | } 16 | 17 | if v > 9 { 18 | v = 9 19 | } 20 | 21 | p.Priority = uint8(v) 22 | } 23 | } 24 | 25 | func WithCustomName(name string) ProducerOption { 26 | return func(p *Producer) error { 27 | p.name = name 28 | 29 | return nil 30 | } 31 | } 32 | 33 | // WithLogger will override the default logger (no Operation Log). 34 | func WithLogger(log LoggerFN) ProducerOption { 35 | return func(p *Producer) error { 36 | p.log = log 37 | 38 | return nil 39 | } 40 | } 41 | 42 | // withDeclarations will add the AMQP declarations and be able to declare the exchanges used. 43 | func withDeclarations(d *declarations) ProducerOption { 44 | return func(p *Producer) error { 45 | p.declarations = d 46 | 47 | return nil 48 | } 49 | } 50 | 51 | // withConnection add the connection config to set up the Connection instead the default values. 52 | func withConnection(conf Connection) ProducerOption { 53 | return func(p *Producer) error { 54 | p.conf = conf 55 | 56 | return nil 57 | } 58 | } 59 | 60 | func WithSerializer(s Serializer) ProducerOption { 61 | return func(p *Producer) error { 62 | p.serializer = s 63 | 64 | return nil 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /producer.go: -------------------------------------------------------------------------------- 1 | package rabbids 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/leveeml/rabbids/serialization" 9 | retry "github.com/rafaeljesus/retry-go" 10 | "github.com/streadway/amqp" 11 | ) 12 | 13 | // Producer is an high level rabbitMQ producer instance. 14 | type Producer struct { 15 | mutex sync.RWMutex 16 | conf Connection 17 | conn *amqp.Connection 18 | ch *amqp.Channel 19 | closed chan struct{} 20 | emit chan Publishing 21 | emitErr chan PublishingError 22 | notifyClose chan *amqp.Error 23 | log LoggerFN 24 | serializer Serializer 25 | declarations *declarations 26 | exDeclared map[string]struct{} 27 | delayDelivery *delayDelivery 28 | name string 29 | } 30 | 31 | // NewProcucer create a new high level rabbitMQ producer instance 32 | // 33 | // dsn is a string in the AMQP URI format 34 | // the ProducerOptions can be: 35 | // rabbids.WithLogger - to set a logger instance 36 | // rabbids.WithFactory - to use one instance of a factory. 37 | // when added the factory is used to declare the topics 38 | // in the first time the topic is used. 39 | // rabbids.WithSerializer - used to set a specific serializer 40 | // the default is the a JSON serializer. 41 | func NewProducer(dsn string, opts ...ProducerOption) (*Producer, error) { 42 | p := &Producer{ 43 | conf: Connection{ 44 | DSN: dsn, 45 | Retries: DefaultRetries, 46 | Sleep: DefaultSleep, 47 | Timeout: DefaultTimeout, 48 | }, 49 | emit: make(chan Publishing, 250), 50 | emitErr: make(chan PublishingError, 250), 51 | closed: make(chan struct{}), 52 | log: NoOPLoggerFN, 53 | serializer: &serialization.JSON{}, 54 | exDeclared: make(map[string]struct{}), 55 | delayDelivery: &delayDelivery{}, 56 | name: fmt.Sprintf("rabbids.producer.%d", time.Now().Unix()), 57 | } 58 | 59 | for _, opt := range opts { 60 | if err := opt(p); err != nil { 61 | return nil, err 62 | } 63 | } 64 | 65 | err := p.startConnection() 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | go p.loop() 71 | 72 | return p, nil 73 | } 74 | 75 | // the internal loop to handle signals from rabbitMQ and the async api. 76 | func (p *Producer) loop() { 77 | for { 78 | select { 79 | case err := <-p.notifyClose: 80 | if err == nil { 81 | return // graceful shutdown? 82 | } 83 | 84 | p.handleAMPQClose(err) 85 | case pub, ok := <-p.emit: 86 | if !ok { 87 | p.closed <- struct{}{} 88 | return // graceful shutdown 89 | } 90 | 91 | err := p.Send(pub) 92 | if err != nil { 93 | p.tryToEmitErr(pub, err) 94 | } 95 | } 96 | } 97 | } 98 | 99 | // Emit emits a message to rabbitMQ but does not wait for the response from the broker. 100 | // Errors with the Publishing (encoding, validation) or with the broker will be sent to the EmitErr channel. 101 | // It's your responsibility to handle these errors somehow. 102 | func (p *Producer) Emit() chan<- Publishing { return p.emit } 103 | 104 | // EmitErr returns a channel used to receive all the errors from Emit channel. 105 | // The error handle is not required but and the send inside this channel is buffered. 106 | // WARNING: If the channel gets full, new errors will be dropped to avoid stop the producer internal loop. 107 | func (p *Producer) EmitErr() <-chan PublishingError { return p.emitErr } 108 | 109 | // Send a message to rabbitMQ. 110 | // In case of connection errors, the send will block and retry until the reconnection is done. 111 | // It returns an error if the Serializer returned an error OR the connection error persisted after the retries. 112 | func (p *Producer) Send(m Publishing) error { 113 | for _, op := range m.options { 114 | op(&m) 115 | } 116 | 117 | b, err := p.serializer.Marshal(m.Data) 118 | if err != nil { 119 | return fmt.Errorf("failed to marshal: %w", err) 120 | } 121 | 122 | m.Body = b 123 | m.ContentType = p.serializer.Name() 124 | 125 | if m.Delay > 0 { 126 | err := p.delayDelivery.Declare(p.ch, m.Key) 127 | if err != nil { 128 | return err 129 | } 130 | } 131 | 132 | return retry.Do(func() error { 133 | p.mutex.RLock() 134 | p.tryToDeclareTopic(m.Exchange) 135 | 136 | err := p.ch.Publish(m.Exchange, m.Key, false, false, m.Publishing) 137 | p.mutex.RUnlock() 138 | 139 | return err 140 | }, 10, 10*time.Millisecond) 141 | } 142 | 143 | // Close will close all the underline channels and close the connection with rabbitMQ. 144 | // Any Emit call after calling the Close method will panic. 145 | func (p *Producer) Close() error { 146 | close(p.emit) 147 | <-p.closed 148 | 149 | p.mutex.Lock() 150 | defer p.mutex.Unlock() 151 | 152 | if p.ch != nil && p.conn != nil && !p.conn.IsClosed() { 153 | if err := p.ch.Close(); err != nil { 154 | return fmt.Errorf("error closing the channel: %w", err) 155 | } 156 | 157 | if err := p.conn.Close(); err != nil { 158 | return fmt.Errorf("error closing the connection: %w", err) 159 | } 160 | } 161 | 162 | close(p.emitErr) 163 | 164 | return nil 165 | } 166 | 167 | // GetAMQPChannel returns the current connection channel. 168 | func (p *Producer) GetAMQPChannel() *amqp.Channel { 169 | return p.ch 170 | } 171 | 172 | // GetAGetAMQPConnection returns the current amqp connetion. 173 | func (p *Producer) GetAMQPConnection() *amqp.Connection { 174 | return p.conn 175 | } 176 | 177 | func (p *Producer) handleAMPQClose(err error) { 178 | p.log("ampq connection closed", Fields{"error": err}) 179 | 180 | for { 181 | connErr := p.startConnection() 182 | if connErr == nil { 183 | return 184 | } 185 | 186 | p.log("ampq reconnection failed", Fields{"error": connErr}) 187 | time.Sleep(time.Second) 188 | } 189 | } 190 | 191 | func (p *Producer) startConnection() error { 192 | p.log("opening a new rabbitmq connection", Fields{}) 193 | 194 | conn, err := openConnection(p.conf, p.name) 195 | if err != nil { 196 | return err 197 | } 198 | 199 | p.mutex.Lock() 200 | 201 | p.conn = conn 202 | p.ch, err = p.conn.Channel() 203 | p.notifyClose = p.conn.NotifyClose(make(chan *amqp.Error)) 204 | 205 | p.mutex.Unlock() 206 | 207 | return err 208 | } 209 | 210 | func (p *Producer) tryToEmitErr(m Publishing, err error) { 211 | data := PublishingError{Publishing: m, Err: err} 212 | select { 213 | case p.emitErr <- data: 214 | default: 215 | } 216 | } 217 | 218 | func (p *Producer) tryToDeclareTopic(ex string) { 219 | if p.declarations == nil || ex == "" { 220 | return 221 | } 222 | 223 | if _, ok := p.exDeclared[ex]; !ok { 224 | err := p.declarations.declareExchange(p.ch, ex) 225 | if err != nil { 226 | p.log("failed declaring a exchange", Fields{"err": err, "ex": ex}) 227 | return 228 | } 229 | 230 | p.exDeclared[ex] = struct{}{} 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /producer_test.go: -------------------------------------------------------------------------------- 1 | package rabbids_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/leveeml/rabbids" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestConnectionErrors(t *testing.T) { 11 | t.Parallel() 12 | t.Run("passing an invalid port", func(t *testing.T) { 13 | t.Parallel() 14 | 15 | _, err := rabbids.NewProducer("amqp://guest:guest@localhost:80/") 16 | require.Error(t, err, "expect an error") 17 | require.Contains(t, err.Error(), "connect: connection refuse") 18 | }) 19 | t.Run("passing an invalid host", func(t *testing.T) { 20 | t.Parallel() 21 | 22 | _, err := rabbids.NewProducer("amqp://guest:guest@10.255.255.1:5672/") 23 | 24 | require.Error(t, err, "expect to return an error") 25 | require.EqualError(t, err, "dial tcp 10.255.255.1:5672: i/o timeout") 26 | }) 27 | } 28 | -------------------------------------------------------------------------------- /rabbids.go: -------------------------------------------------------------------------------- 1 | package rabbids 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "strings" 7 | "sync/atomic" 8 | 9 | "github.com/google/uuid" 10 | "github.com/ivpusic/grpool" 11 | retry "github.com/rafaeljesus/retry-go" 12 | "github.com/streadway/amqp" 13 | "gopkg.in/tomb.v2" 14 | ) 15 | 16 | // Rabbids is the main block used to create and run rabbitMQ consumers and producers. 17 | type Rabbids struct { 18 | conns map[string]*amqp.Connection 19 | config *Config 20 | declarations *declarations 21 | log LoggerFN 22 | number int64 23 | } 24 | 25 | func New(config *Config, log LoggerFN) (*Rabbids, error) { 26 | setConfigDefaults(config) 27 | 28 | conns := make(map[string]*amqp.Connection) 29 | 30 | for name, cfgConn := range config.Connections { 31 | log("opening connection with rabbitMQ", Fields{ 32 | "sleep": cfgConn.Sleep, 33 | "timeout": cfgConn.Timeout, 34 | "connection": name, 35 | }) 36 | 37 | conn, err := openConnection(cfgConn, fmt.Sprintf("rabbids.%s", name)) 38 | if err != nil { 39 | return nil, fmt.Errorf("error opening the connection \"%s\": %w", name, err) 40 | } 41 | 42 | conns[name] = conn 43 | } 44 | 45 | r := &Rabbids{ 46 | conns: conns, 47 | config: config, 48 | declarations: &declarations{ 49 | config: config, 50 | log: log, 51 | }, 52 | log: log, 53 | number: 0, 54 | } 55 | 56 | return r, nil 57 | } 58 | 59 | // CreateConsumers will iterate over config and create all the consumers. 60 | func (r *Rabbids) CreateConsumers() ([]*Consumer, error) { 61 | var consumers []*Consumer 62 | 63 | for name, cfg := range r.config.Consumers { 64 | consumer, err := r.newConsumer(name, cfg) 65 | if err != nil { 66 | return consumers, err 67 | } 68 | 69 | consumers = append(consumers, consumer) 70 | } 71 | 72 | return consumers, nil 73 | } 74 | 75 | // CreateConsumer create a new consumer for a specific name using the config provided. 76 | func (r *Rabbids) CreateConsumer(name string) (*Consumer, error) { 77 | cfg, ok := r.config.Consumers[name] 78 | if !ok { 79 | return nil, fmt.Errorf("consumer \"%s\" did not exist", name) 80 | } 81 | 82 | return r.newConsumer(name, cfg) 83 | } 84 | 85 | func (r *Rabbids) newConsumer(name string, cfg ConsumerConfig) (*Consumer, error) { 86 | ch, err := r.getChannel(cfg.Connection) 87 | if err != nil { 88 | return nil, fmt.Errorf("failed to open the rabbitMQ channel for consumer %s: %w", name, err) 89 | } 90 | 91 | if len(cfg.DeadLetter) > 0 { 92 | err = r.declarations.declareDeadLetters(ch, cfg.DeadLetter) 93 | if err != nil { 94 | return nil, err 95 | } 96 | } 97 | 98 | err = r.declarations.declareQueue(ch, cfg.Queue) 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | if err = ch.Qos(cfg.PrefetchCount, 0, false); err != nil { 104 | return nil, fmt.Errorf("failed to set QoS: %w", err) 105 | } 106 | 107 | handler, ok := r.config.Handlers[name] 108 | if !ok { 109 | return nil, fmt.Errorf("failed to create the \"%s\" consumer, Handler not registered", name) 110 | } 111 | 112 | r.log("consumer created", 113 | Fields{ 114 | "max-workers": cfg.Workers, 115 | "consumer": name, 116 | }) 117 | 118 | return &Consumer{ 119 | queue: cfg.Queue.Name, 120 | name: name, 121 | number: atomic.AddInt64(&r.number, 1), 122 | opts: cfg.Options, 123 | channel: ch, 124 | t: tomb.Tomb{}, 125 | handler: handler, 126 | workerPool: grpool.NewPool(cfg.Workers, 0), 127 | log: r.log, 128 | }, nil 129 | } 130 | 131 | // CreateConsumer create a new consumer using the connection inside the config. 132 | func (r *Rabbids) CreateProducer(connectionName string, customOpts ...ProducerOption) (*Producer, error) { 133 | conn, exists := r.config.Connections[connectionName] 134 | if !exists { 135 | return nil, fmt.Errorf("connection \"%s\" did not exist", connectionName) 136 | } 137 | 138 | opts := []ProducerOption{ 139 | withConnection(conn), 140 | WithLogger(r.log), 141 | withDeclarations(r.declarations), 142 | } 143 | 144 | return NewProducer("", append(opts, customOpts...)...) 145 | } 146 | 147 | func (r *Rabbids) getChannel(connectionName string) (*amqp.Channel, error) { 148 | _, ok := r.conns[connectionName] 149 | if !ok { 150 | available := []string{} 151 | for name := range r.conns { 152 | available = append(available, name) 153 | } 154 | 155 | return nil, fmt.Errorf( 156 | "connection (%s) did not exist, connections names available: %s", 157 | connectionName, 158 | strings.Join(available, ", ")) 159 | } 160 | 161 | var ch *amqp.Channel 162 | var errCH error 163 | 164 | conn := r.conns[connectionName] 165 | ch, errCH = conn.Channel() 166 | // Reconnect the connection when receive an connection closed error 167 | if errCH != nil && errCH.Error() == amqp.ErrClosed.Error() { 168 | cfgConn := r.config.Connections[connectionName] 169 | r.log("reopening one connection closed", 170 | Fields{ 171 | "sleep": cfgConn.Sleep, 172 | "timeout": cfgConn.Timeout, 173 | "connection": connectionName, 174 | }, 175 | ) 176 | 177 | conn, err := openConnection(cfgConn, fmt.Sprintf("rabbids.%s", connectionName)) 178 | if err != nil { 179 | return nil, fmt.Errorf("error reopening the connection \"%s\": %w", connectionName, err) 180 | } 181 | 182 | r.conns[connectionName] = conn 183 | ch, errCH = conn.Channel() 184 | } 185 | 186 | return ch, errCH 187 | } 188 | 189 | func openConnection(config Connection, name string) (*amqp.Connection, error) { 190 | var conn *amqp.Connection 191 | 192 | id, err := uuid.NewRandom() 193 | if err != nil { 194 | id = uuid.Must(uuid.NewUUID()) 195 | } 196 | 197 | err = retry.Do(func() error { 198 | var err error 199 | conn, err = amqp.DialConfig(config.DSN, amqp.Config{ 200 | Dial: func(network, addr string) (net.Conn, error) { 201 | return net.DialTimeout(network, addr, config.Timeout) 202 | }, 203 | Properties: amqp.Table{ 204 | "information": "https://github.com/EmpregoLigado/rabbids", 205 | "product": "Rabbids", 206 | "version": Version, 207 | "id": id.String(), 208 | "connection_name": name, 209 | }, 210 | }) 211 | return err 212 | }, 5, config.Sleep) 213 | 214 | return conn, err 215 | } 216 | -------------------------------------------------------------------------------- /serialization/json.go: -------------------------------------------------------------------------------- 1 | package serialization 2 | 3 | import "encoding/json" 4 | 5 | // JSON implements the rabbids.Serializer interface. 6 | type JSON struct{} 7 | 8 | // Marshal returns the data in json format or an error. 9 | func (j *JSON) Marshal(v interface{}) ([]byte, error) { 10 | return json.Marshal(v) 11 | } 12 | 13 | // Name returns the name of the serialization used. 14 | // This value is used as the ContentType value on the message. 15 | func (j *JSON) Name() string { 16 | return "application/json" 17 | } 18 | -------------------------------------------------------------------------------- /supervisor.go: -------------------------------------------------------------------------------- 1 | package rabbids 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // supervisor start all the consumers from Rabbids and 8 | // keep track of the consumers status, restating them when needed. 9 | type supervisor struct { 10 | checkAliveness time.Duration 11 | rabbids *Rabbids 12 | consumers map[string]*Consumer 13 | close chan struct{} 14 | } 15 | 16 | // StartSupervisor init a new supervisor that will start all the consumers from Rabbids 17 | // and check if the consumers are alive, if not alive it will be restarted. 18 | // It returns the stop function to gracefully shutdown the consumers and 19 | // an error if fail to create the consumers the first time. 20 | func StartSupervisor(rabbids *Rabbids, intervalChecks time.Duration) (stop func(), err error) { 21 | s := &supervisor{ 22 | checkAliveness: intervalChecks, 23 | rabbids: rabbids, 24 | consumers: map[string]*Consumer{}, 25 | close: make(chan struct{}), 26 | } 27 | 28 | cs, err := s.rabbids.CreateConsumers() 29 | if err != nil { 30 | return s.Stop, err 31 | } 32 | 33 | for _, c := range cs { 34 | c.Run() 35 | s.consumers[c.Name()] = c 36 | } 37 | 38 | go s.loop() 39 | 40 | return s.Stop, nil 41 | } 42 | 43 | func (s *supervisor) loop() { 44 | ticker := time.NewTicker(s.checkAliveness) 45 | 46 | for { 47 | select { 48 | case <-s.close: 49 | for name, c := range s.consumers { 50 | c.Kill() 51 | delete(s.consumers, name) 52 | } 53 | s.close <- struct{}{} 54 | 55 | return 56 | case <-ticker.C: 57 | s.restartDeadConsumers() 58 | } 59 | } 60 | } 61 | 62 | // Stop all the running consumers. 63 | func (s *supervisor) Stop() { 64 | s.close <- struct{}{} 65 | <-s.close 66 | } 67 | 68 | func (s *supervisor) restartDeadConsumers() { 69 | for name, c := range s.consumers { 70 | if !c.Alive() { 71 | s.rabbids.log("recreating one consumer", Fields{ 72 | "consumer-name": name, 73 | }) 74 | 75 | nc, err := s.rabbids.CreateConsumer(name) 76 | if err != nil { 77 | s.rabbids.log("error recreating one consumer", Fields{ 78 | "consumer-name": name, 79 | "error": err, 80 | }) 81 | 82 | continue 83 | } 84 | 85 | delete(s.consumers, name) 86 | s.consumers[name] = nc 87 | nc.Run() 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /testdata/valid_queue_and_exchange_config.yml: -------------------------------------------------------------------------------- 1 | connections: 2 | default: 3 | dsn: "amqp://localhost:5672" 4 | timeout: 1s 5 | sleep: 500ms 6 | exchanges: 7 | event_bus: 8 | type: topic 9 | options: 10 | no_wait: false 11 | fallback: 12 | type: topic 13 | dead_letters: 14 | fallback: 15 | queue: 16 | name: "fallback" 17 | options: 18 | durable: true 19 | args: 20 | "x-dead-letter-exchange": "" 21 | "x-message-ttl": 300000 22 | bindings: 23 | - routing_keys: ["#"] 24 | exchange: fallback 25 | consumers: 26 | messaging_consumer: 27 | connection: default 28 | dead_letter: fallback 29 | queue: 30 | name: "messaging_send" 31 | options: 32 | durable: true 33 | args: 34 | "x-dead-letter-exchange": "fallback" 35 | "x-dead-letter-routing-key": "messaging_send" 36 | bindings: 37 | - routing_keys: 38 | - "service.whatssapp.send" 39 | - "service.sms.send" 40 | exchange: event_bus 41 | -------------------------------------------------------------------------------- /testdata/valid_two_connections.yml: -------------------------------------------------------------------------------- 1 | connections: 2 | default: 3 | dsn: "amqp://localhost:5672" 4 | timeout: 1s 5 | sleep: 500ms 6 | retries: 10 7 | test1: 8 | dsn: "amqp://localhost:5672" 9 | timeout: 1s 10 | sleep: 1s 11 | retries: 5 12 | exchanges: 13 | event_bus: 14 | type: topic 15 | options: 16 | no_wait: false 17 | fallback: 18 | type: topic 19 | dead_letters: 20 | fallback: 21 | queue: 22 | name: "fallback" 23 | options: 24 | durable: true 25 | args: 26 | "x-dead-letter-exchange": "" 27 | "x-message-ttl": 300000 28 | bindings: 29 | - routing_keys: ["#"] 30 | exchange: fallback 31 | consumers: 32 | send_consumer: 33 | connection: default 34 | dead_letter: fallback 35 | queue: 36 | name: "messaging_send" 37 | options: 38 | durable: true 39 | args: 40 | "x-dead-letter-exchange": "fallback" 41 | "x-dead-letter-routing-key": "messaging_send" 42 | bindings: 43 | - routing_keys: 44 | - "service.whatssapp.send" 45 | - "service.sms.send" 46 | exchange: event_bus 47 | response_consumer: 48 | connection: test1 49 | dead_letter: fallback 50 | queue: 51 | name: "messaging_responses" 52 | options: 53 | durable: true 54 | args: 55 | "x-dead-letter-exchange": "fallback" 56 | "x-dead-letter-routing-key": "messaging_responses" 57 | bindings: 58 | - routing_keys: 59 | - "service.whatssapp.response" 60 | - "service.sms.response" 61 | exchange: event_bus 62 | --------------------------------------------------------------------------------