├── .go-version ├── tools.go ├── .editorconfig ├── .travis.yml ├── message.go ├── scripts └── coverage.sh ├── signals_test.go ├── signals.go ├── .chglog ├── config.yml └── CHANGELOG.tpl.md ├── .golangci.yml ├── redis_test.go ├── go.mod ├── LICENSE.txt ├── .gitignore ├── CHANGELOG.md ├── Makefile ├── producer_test.go ├── redis.go ├── doc.go ├── producer.go ├── README.md ├── go.sum ├── consumer_test.go └── consumer.go /.go-version: -------------------------------------------------------------------------------- 1 | 1.14.2 2 | -------------------------------------------------------------------------------- /tools.go: -------------------------------------------------------------------------------- 1 | // +build tools 2 | 3 | package tools 4 | 5 | import ( 6 | _ "github.com/git-chglog/git-chglog/cmd/git-chglog" 7 | _ "github.com/mattn/goveralls" 8 | ) 9 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | charset = utf-8 3 | end_of_line = lf 4 | insert_final_newline = true 5 | trim_trailing_whitespace = true 6 | 7 | [Makefile] 8 | indent_style = tab 9 | 10 | [*.go] 11 | indent_style = tab 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | sudo: false 3 | language: go 4 | addons: 5 | apt: 6 | packages: 7 | - redis-server 8 | go: "1.14.2" 9 | env: 10 | - GO111MODULE=on 11 | install: 12 | - make setup 13 | - make install 14 | script: 15 | - make lint 16 | - make test 17 | - make enforce 18 | - make coveralls 19 | -------------------------------------------------------------------------------- /message.go: -------------------------------------------------------------------------------- 1 | package redisqueue 2 | 3 | // Message constitutes a message that will be enqueued and dequeued from Redis. 4 | // When enqueuing, it's recommended to leave ID empty and let Redis generate it, 5 | // unless you know what you're doing. 6 | type Message struct { 7 | ID string 8 | Stream string 9 | Values map[string]interface{} 10 | } 11 | -------------------------------------------------------------------------------- /scripts/coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | THRESHOLD=80 4 | COVERAGE_PROFILE=$1 5 | 6 | if [ -z "$COVERAGE_PROFILE" ]; then 7 | COVERAGE_PROFILE=./coverage.out 8 | fi 9 | 10 | PERCENT=$(go tool cover -func $COVERAGE_PROFILE | grep total: | sed 's/ / /g' | tr -s ' ' | cut -d ' ' -f 3 | sed 's/%//' | awk -F. '{print $1}') 11 | 12 | if (( $PERCENT < $THRESHOLD )); then 13 | echo "Error: coverage $PERCENT% doesn't meet the threshold of $THRESHOLD%" 14 | exit 1 15 | else 16 | echo "Success: coverage $PERCENT% meets the threshold of $THRESHOLD%" 17 | fi 18 | -------------------------------------------------------------------------------- /signals_test.go: -------------------------------------------------------------------------------- 1 | package redisqueue 2 | 3 | import ( 4 | "syscall" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestNewSignalHandler(t *testing.T) { 12 | t.Run("closes the returned channel on SIGINT", func(tt *testing.T) { 13 | ch := newSignalHandler() 14 | 15 | err := syscall.Kill(syscall.Getpid(), syscall.SIGINT) 16 | require.NoError(tt, err) 17 | 18 | select { 19 | case <-time.After(2 * time.Second): 20 | t.Error("timed out waiting for signal") 21 | case <-ch: 22 | } 23 | }) 24 | } 25 | -------------------------------------------------------------------------------- /signals.go: -------------------------------------------------------------------------------- 1 | package redisqueue 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | "syscall" 7 | ) 8 | 9 | // newSignalHandler registered for SIGTERM and SIGINT. A stop channel is 10 | // returned which is closed on one of these signals. If a second signal is 11 | // caught, the program is terminated with exit code 1. 12 | func newSignalHandler() <-chan struct{} { 13 | stop := make(chan struct{}) 14 | c := make(chan os.Signal, 2) 15 | signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) 16 | go func() { 17 | <-c 18 | close(stop) 19 | <-c 20 | os.Exit(1) 21 | }() 22 | 23 | return stop 24 | } 25 | -------------------------------------------------------------------------------- /.chglog/config.yml: -------------------------------------------------------------------------------- 1 | style: github 2 | template: CHANGELOG.tpl.md 3 | info: 4 | title: CHANGELOG 5 | repository_url: https://github.com/robinjoseph08/go-pg-migrations 6 | options: 7 | commit_groups: 8 | title_maps: 9 | docs: Documentation 10 | feat: Features 11 | fix: Bug Fixes 12 | perf: Performance Improvements 13 | refactor: Code Refactoring 14 | header: 15 | pattern: "^(\\w*)(?:\\(([\\w\\$\\.\\-\\*\\s]*)\\))?\\:\\s(.*)$" 16 | pattern_maps: 17 | - Type 18 | - Scope 19 | - Subject 20 | notes: 21 | keywords: 22 | - BREAKING CHANGE 23 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | output: 2 | format: tab 3 | linters: 4 | disable-all: true 5 | enable: 6 | - deadcode 7 | - depguard 8 | - dupl 9 | - goconst 10 | - gocritic 11 | - gocyclo 12 | - gofmt 13 | - goimports 14 | - golint 15 | - gosec 16 | - govet 17 | - ineffassign 18 | - maligned 19 | - misspell 20 | - prealloc 21 | - scopelint 22 | - structcheck 23 | - typecheck 24 | - unconvert 25 | - varcheck 26 | issues: 27 | exclude-use-default: false 28 | max-per-linter: 0 29 | max-same-issues: 0 30 | exclude-rules: 31 | - path: _test\.go 32 | linters: 33 | - dupl 34 | - scopelint 35 | -------------------------------------------------------------------------------- /.chglog/CHANGELOG.tpl.md: -------------------------------------------------------------------------------- 1 | {{ range .Versions }} 2 | 3 | ## {{ if .Tag.Previous }}[{{ .Tag.Name }}]({{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}){{ else }}{{ .Tag.Name }}{{ end }} ({{ datetime "2006-01-02" .Tag.Date }}) 4 | 5 | {{ range .CommitGroups -}} 6 | ### {{ .Title }} 7 | 8 | {{ range .Commits -}} 9 | * {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} 10 | {{ end }} 11 | {{ end -}} 12 | 13 | {{- if .RevertCommits -}} 14 | ### Reverts 15 | 16 | {{ range .RevertCommits -}} 17 | * {{ .Revert.Header }} 18 | {{ end }} 19 | {{ end -}} 20 | 21 | {{- if .NoteGroups -}} 22 | {{ range .NoteGroups -}} 23 | ### {{ .Title }} 24 | 25 | {{ range .Notes }} 26 | {{ .Body }} 27 | {{ end }} 28 | {{ end -}} 29 | {{ end -}} 30 | {{ end -}} -------------------------------------------------------------------------------- /redis_test.go: -------------------------------------------------------------------------------- 1 | package redisqueue 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestNewRedisClient(t *testing.T) { 11 | t.Run("returns a new redis client", func(tt *testing.T) { 12 | options := &RedisOptions{} 13 | r := newRedisClient(options) 14 | 15 | err := r.Ping().Err() 16 | assert.NoError(tt, err) 17 | }) 18 | 19 | t.Run("defaults options if it's nil", func(tt *testing.T) { 20 | r := newRedisClient(nil) 21 | 22 | err := r.Ping().Err() 23 | assert.NoError(tt, err) 24 | }) 25 | } 26 | 27 | func TestRedisPreflightChecks(t *testing.T) { 28 | t.Run("bubbles up errors", func(tt *testing.T) { 29 | options := &RedisOptions{Addr: "localhost:0"} 30 | r := newRedisClient(options) 31 | 32 | err := redisPreflightChecks(r) 33 | require.Error(tt, err) 34 | 35 | assert.Contains(tt, err.Error(), "dial tcp") 36 | }) 37 | } 38 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/robinjoseph08/redisqueue/v2 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/davecgh/go-spew v1.1.1 // indirect 7 | github.com/fatih/color v1.7.0 // indirect 8 | github.com/git-chglog/git-chglog v0.0.0-20190611050339-63a4e637021f 9 | github.com/go-redis/redis/v7 v7.3.0 10 | github.com/golang/protobuf v1.3.3 // indirect 11 | github.com/imdario/mergo v0.3.7 // indirect 12 | github.com/mattn/go-colorable v0.1.2 // indirect 13 | github.com/mattn/goveralls v0.0.2 14 | github.com/pborman/uuid v1.2.0 // indirect 15 | github.com/pkg/errors v0.9.1 16 | github.com/stretchr/testify v1.5.1 17 | github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df // indirect 18 | github.com/urfave/cli v1.20.0 // indirect 19 | golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db // indirect 20 | gopkg.in/AlecAivazis/survey.v1 v1.8.5 // indirect 21 | gopkg.in/kyokomi/emoji.v1 v1.5.1 // indirect 22 | gopkg.in/yaml.v2 v2.3.0 // indirect 23 | ) 24 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Robin Joseph 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io/api/go,vim,macos 2 | 3 | ### Go ### 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.dll 7 | *.so 8 | *.dylib 9 | 10 | # Test binary, build with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | coverage.out 15 | coverage.html 16 | 17 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 18 | .glide/ 19 | 20 | # Vendored dependencies 21 | vendor/* 22 | _vendor*/* 23 | 24 | # Binary executables 25 | bin/ 26 | 27 | ### macOS ### 28 | *.DS_Store 29 | .AppleDouble 30 | .LSOverride 31 | 32 | # Icon must end with two \r 33 | Icon 34 | 35 | # Thumbnails 36 | ._* 37 | 38 | # Files that might appear in the root of a volume 39 | .DocumentRevisions-V100 40 | .fseventsd 41 | .Spotlight-V100 42 | .TemporaryItems 43 | .Trashes 44 | .VolumeIcon.icns 45 | .com.apple.timemachine.donotpresent 46 | 47 | # Directories potentially created on remote AFP share 48 | .AppleDB 49 | .AppleDesktop 50 | Network Trash Folder 51 | Temporary Items 52 | .apdisk 53 | 54 | ### Vim ### 55 | # swap 56 | .sw[a-p] 57 | .*.sw[a-p] 58 | # session 59 | Session.vim 60 | # temporary 61 | .netrwhist 62 | *~ 63 | # auto-generated tag files 64 | tags 65 | 66 | 67 | # End of https://www.gitignore.io/api/go,vim,macos 68 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## [v2.1.0](https://github.com/robinjoseph08/go-pg-migrations/compare/v2.0.0...v2.1.0) (2020-10-15) 4 | 5 | ### Features 6 | 7 | * **redis:** allow passing in redis.UniversalClient 8 | 9 | 10 | 11 | ## [v2.0.0](https://github.com/robinjoseph08/go-pg-migrations/compare/v1.1.0...v2.0.0) (2020-05-26) 12 | 13 | ### Features 14 | 15 | * **redis:** update to go-redis/v7 and switch to redisqueue/v2 ([#11](https://github.com/robinjoseph08/go-pg-migrations/issues/11)) 16 | 17 | 18 | 19 | ## [v1.1.0](https://github.com/robinjoseph08/go-pg-migrations/compare/v1.0.1...v1.1.0) (2020-05-26) 20 | 21 | ### Chore 22 | 23 | * **makefile:** update Makefile ([#10](https://github.com/robinjoseph08/go-pg-migrations/issues/10)) 24 | 25 | 26 | 27 | ## [v1.0.1](https://github.com/robinjoseph08/go-pg-migrations/compare/v1.0.0...v1.0.1) (2019-08-03) 28 | 29 | ### Bug Fixes 30 | 31 | * **reclaim:** increment ID when looping ([#4](https://github.com/robinjoseph08/go-pg-migrations/issues/4)) 32 | 33 | 34 | 35 | ## v1.0.0 (2019-07-13) 36 | 37 | ### Documentation 38 | 39 | * **README:** add version badge ([#3](https://github.com/robinjoseph08/go-pg-migrations/issues/3)) 40 | 41 | ### Features 42 | 43 | * **base:** create first version of producer and consumer ([#2](https://github.com/robinjoseph08/go-pg-migrations/issues/2)) 44 | 45 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BIN_DIR ?= ./bin 2 | GO_TOOLS := \ 3 | github.com/git-chglog/git-chglog/cmd/git-chglog \ 4 | github.com/mattn/goveralls \ 5 | 6 | TFLAGS ?= 7 | 8 | COVERAGE_PROFILE ?= coverage.out 9 | HTML_OUTPUT ?= coverage.html 10 | 11 | PSQL := $(shell command -v psql 2> /dev/null) 12 | 13 | TEST_DATABASE_USER ?= go_pg_migrations_user 14 | TEST_DATABASE_NAME ?= go_pg_migrations 15 | 16 | default: install 17 | 18 | .PHONY: clean 19 | clean: 20 | @echo "---> Cleaning" 21 | go clean 22 | 23 | coveralls: 24 | @echo "---> Sending coverage info to Coveralls" 25 | $(BIN_DIR)/goveralls -coverprofile=$(COVERAGE_PROFILE) -service=travis-ci 26 | 27 | .PHONY: enforce 28 | enforce: 29 | @echo "---> Enforcing coverage" 30 | ./scripts/coverage.sh $(COVERAGE_PROFILE) 31 | 32 | .PHONY: html 33 | html: 34 | @echo "---> Generating HTML coverage report" 35 | go tool cover -html $(COVERAGE_PROFILE) -o $(HTML_OUTPUT) 36 | open $(HTML_OUTPUT) 37 | 38 | .PHONY: install 39 | install: 40 | @echo "---> Installing dependencies" 41 | go mod download 42 | 43 | .PHONY: lint 44 | lint: $(BIN_DIR)/golangci-lint 45 | @echo "---> Linting" 46 | $(BIN_DIR)/golangci-lint run 47 | 48 | .PHONY: release 49 | release: 50 | @echo "---> Creating new release" 51 | ifndef tag 52 | $(error tag must be specified) 53 | endif 54 | $(BIN_DIR)/git-chglog --output CHANGELOG.md --next-tag $(tag) 55 | sed -i "" "s/version-.*-green/version-$(tag)-green/" README.md 56 | git add CHANGELOG.md README.md 57 | git commit -m $(tag) 58 | git tag $(tag) 59 | git push origin master --tags 60 | 61 | .PHONY: setup 62 | setup: $(BIN_DIR)/golangci-lint 63 | @echo "--> Setting up" 64 | GOBIN=$(PWD)/$(subst ./,,$(BIN_DIR)) go install $(GO_TOOLS) 65 | 66 | $(BIN_DIR)/golangci-lint: 67 | @echo "--> Installing linter" 68 | curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(BIN_DIR) v1.27.0 69 | 70 | .PHONY: test 71 | test: 72 | @echo "---> Testing" 73 | go test ./... -coverprofile $(COVERAGE_PROFILE) $(TFLAGS) 74 | -------------------------------------------------------------------------------- /producer_test.go: -------------------------------------------------------------------------------- 1 | package redisqueue 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestNewProducer(t *testing.T) { 11 | t.Run("creates a new producer", func(tt *testing.T) { 12 | p, err := NewProducer() 13 | require.NoError(tt, err) 14 | 15 | assert.NotNil(tt, p) 16 | }) 17 | } 18 | 19 | func TestNewProducerWithOptions(t *testing.T) { 20 | t.Run("creates a new producer", func(tt *testing.T) { 21 | p, err := NewProducerWithOptions(&ProducerOptions{}) 22 | require.NoError(tt, err) 23 | 24 | assert.NotNil(tt, p) 25 | }) 26 | 27 | t.Run("allows custom *redis.Client", func(tt *testing.T) { 28 | rc := newRedisClient(nil) 29 | 30 | p, err := NewProducerWithOptions(&ProducerOptions{ 31 | RedisClient: rc, 32 | }) 33 | require.NoError(tt, err) 34 | 35 | assert.NotNil(tt, p) 36 | assert.Equal(tt, rc, p.redis) 37 | }) 38 | 39 | t.Run("bubbles up errors", func(tt *testing.T) { 40 | _, err := NewProducerWithOptions(&ProducerOptions{ 41 | RedisOptions: &RedisOptions{Addr: "localhost:0"}, 42 | }) 43 | require.Error(tt, err) 44 | 45 | assert.Contains(tt, err.Error(), "dial tcp") 46 | }) 47 | } 48 | 49 | func TestEnqueue(t *testing.T) { 50 | t.Run("puts the message in the stream", func(tt *testing.T) { 51 | p, err := NewProducerWithOptions(&ProducerOptions{}) 52 | require.NoError(t, err) 53 | 54 | msg := &Message{ 55 | Stream: tt.Name(), 56 | Values: map[string]interface{}{"test": "value"}, 57 | } 58 | err = p.Enqueue(msg) 59 | require.NoError(tt, err) 60 | 61 | res, err := p.redis.XRange(msg.Stream, msg.ID, msg.ID).Result() 62 | require.NoError(tt, err) 63 | assert.Equal(tt, "value", res[0].Values["test"]) 64 | }) 65 | 66 | t.Run("bubbles up errors", func(tt *testing.T) { 67 | p, err := NewProducerWithOptions(&ProducerOptions{ApproximateMaxLength: true}) 68 | require.NoError(t, err) 69 | 70 | msg := &Message{ 71 | Stream: tt.Name(), 72 | } 73 | err = p.Enqueue(msg) 74 | require.Error(tt, err) 75 | 76 | assert.Contains(tt, err.Error(), "wrong number of arguments") 77 | }) 78 | } 79 | -------------------------------------------------------------------------------- /redis.go: -------------------------------------------------------------------------------- 1 | package redisqueue 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strconv" 7 | "strings" 8 | 9 | "github.com/go-redis/redis/v7" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | var redisVersionRE = regexp.MustCompile(`redis_version:(.+)`) 14 | 15 | // RedisOptions is an alias to redis.Options so that users can this instead of 16 | // having to import go-redis directly. 17 | type RedisOptions = redis.Options 18 | 19 | // newRedisClient creates a new Redis client with the given options. If options 20 | // is nil, it will use default options. 21 | func newRedisClient(options *RedisOptions) *redis.Client { 22 | if options == nil { 23 | options = &RedisOptions{} 24 | } 25 | return redis.NewClient(options) 26 | } 27 | 28 | // redisPreflightChecks makes sure the Redis instance backing the *redis.Client 29 | // offers the functionality we need. Specifically, it also that it can connect 30 | // to the actual instance and that the instance supports Redis streams (i.e. 31 | // it's at least v5). 32 | func redisPreflightChecks(client redis.UniversalClient) error { 33 | info, err := client.Info("server").Result() 34 | if err != nil { 35 | return err 36 | } 37 | 38 | match := redisVersionRE.FindAllStringSubmatch(info, -1) 39 | if len(match) < 1 { 40 | return fmt.Errorf("could not extract redis version") 41 | } 42 | version := strings.TrimSpace(match[0][1]) 43 | parts := strings.Split(version, ".") 44 | major, err := strconv.Atoi(parts[0]) 45 | if err != nil { 46 | return err 47 | } 48 | if major < 5 { 49 | return fmt.Errorf("redis streams are not supported in version %q", version) 50 | } 51 | 52 | return nil 53 | } 54 | 55 | // incrementMessageID takes in a message ID (e.g. 1564886140363-0) and 56 | // increments the index section (e.g. 1564886140363-1). This is the next valid 57 | // ID value, and it can be used for paging through messages. 58 | func incrementMessageID(id string) (string, error) { 59 | parts := strings.Split(id, "-") 60 | index := parts[1] 61 | parsed, err := strconv.ParseInt(index, 10, 64) 62 | if err != nil { 63 | return "", errors.Wrapf(err, "error parsing message ID %q", id) 64 | } 65 | return fmt.Sprintf("%s-%d", parts[0], parsed+1), nil 66 | } 67 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package redisqueue provides a producer and consumer of a queue that uses Redis 3 | streams (https://redis.io/topics/streams-intro). 4 | 5 | Features 6 | 7 | The features of this package include: 8 | 9 | - A `Producer` struct to make enqueuing messages easy. 10 | - A `Consumer` struct to make processing messages concurrenly. 11 | - Claiming and acknowledging messages if there's no error, so that if a consumer 12 | dies while processing, the message it was working on isn't lost. This 13 | guarantees at least once delivery. 14 | - A "visibility timeout" so that if a message isn't processed in a designated 15 | time frame, it will be be processed by another consumer. 16 | - A max length on the stream so that it doesn't store the messages indefinitely 17 | and run out of memory. 18 | - Graceful handling of Unix signals (`SIGINT` and `SIGTERM`) to let in-flight 19 | messages complete. 20 | - A channel that will surface any errors so you can handle them centrally. 21 | - Graceful handling of panics to avoid crashing the whole process. 22 | - A concurrency setting to control how many goroutines are spawned to process 23 | messages. 24 | - A batch size setting to limit the total messages in flight. 25 | - Support for multiple streams. 26 | 27 | Example 28 | 29 | Here's an example of a producer that inserts 1000 messages into a queue: 30 | 31 | package main 32 | 33 | import ( 34 | "fmt" 35 | 36 | "github.com/robinjoseph08/redisqueue/v2" 37 | ) 38 | 39 | func main() { 40 | p, err := redisqueue.NewProducerWithOptions(&redisqueue.ProducerOptions{ 41 | StreamMaxLength: 10000, 42 | ApproximateMaxLength: true, 43 | }) 44 | if err != nil { 45 | panic(err) 46 | } 47 | 48 | for i := 0; i < 1000; i++ { 49 | err := p.Enqueue(&redisqueue.Message{ 50 | Stream: "redisqueue:test", 51 | Values: map[string]interface{}{ 52 | "index": i, 53 | }, 54 | }) 55 | if err != nil { 56 | panic(err) 57 | } 58 | 59 | if i%100 == 0 { 60 | fmt.Printf("enqueued %d\n", i) 61 | } 62 | } 63 | } 64 | 65 | And here's an example of a consumer that reads the messages off of that queue: 66 | 67 | package main 68 | 69 | import ( 70 | "fmt" 71 | "time" 72 | 73 | "github.com/robinjoseph08/redisqueue/v2" 74 | ) 75 | 76 | func main() { 77 | c, err := redisqueue.NewConsumerWithOptions(&redisqueue.ConsumerOptions{ 78 | VisibilityTimeout: 60 * time.Second, 79 | BlockingTimeout: 5 * time.Second, 80 | ReclaimInterval: 1 * time.Second, 81 | BufferSize: 100, 82 | Concurrency: 10, 83 | }) 84 | if err != nil { 85 | panic(err) 86 | } 87 | 88 | c.Register("redisqueue:test", process) 89 | 90 | go func() { 91 | for err := range c.Errors { 92 | // handle errors accordingly 93 | fmt.Printf("err: %+v\n", err) 94 | } 95 | }() 96 | 97 | fmt.Println("starting") 98 | c.Run() 99 | fmt.Println("stopped") 100 | } 101 | 102 | func process(msg *redisqueue.Message) error { 103 | fmt.Printf("processing message: %v\n", msg.Values["index"]) 104 | return nil 105 | } 106 | */ 107 | package redisqueue 108 | -------------------------------------------------------------------------------- /producer.go: -------------------------------------------------------------------------------- 1 | package redisqueue 2 | 3 | import ( 4 | "github.com/go-redis/redis/v7" 5 | ) 6 | 7 | // ProducerOptions provide options to configure the Producer. 8 | type ProducerOptions struct { 9 | // StreamMaxLength sets the MAXLEN option when calling XADD. This creates a 10 | // capped stream to prevent the stream from taking up memory indefinitely. 11 | // It's important to note though that this isn't the maximum number of 12 | // _completed_ messages, but the maximum number of _total_ messages. This 13 | // means that if all consumers are down, but producers are still enqueuing, 14 | // and the maximum is reached, unprocessed message will start to be dropped. 15 | // So ideally, you'll set this number to be as high as you can makee it. 16 | // More info here: https://redis.io/commands/xadd#capped-streams. 17 | StreamMaxLength int64 18 | // ApproximateMaxLength determines whether to use the ~ with the MAXLEN 19 | // option. This allows the stream trimming to done in a more efficient 20 | // manner. More info here: https://redis.io/commands/xadd#capped-streams. 21 | ApproximateMaxLength bool 22 | // RedisClient supersedes the RedisOptions field, and allows you to inject 23 | // an already-made Redis Client for use in the consumer. This may be either 24 | // the standard client or a cluster client. 25 | RedisClient redis.UniversalClient 26 | // RedisOptions allows you to configure the underlying Redis connection. 27 | // More info here: 28 | // https://pkg.go.dev/github.com/go-redis/redis/v7?tab=doc#Options. 29 | // 30 | // This field is used if RedisClient field is nil. 31 | RedisOptions *RedisOptions 32 | } 33 | 34 | // Producer adds a convenient wrapper around enqueuing messages that will be 35 | // processed later by a Consumer. 36 | type Producer struct { 37 | options *ProducerOptions 38 | redis redis.UniversalClient 39 | } 40 | 41 | var defaultProducerOptions = &ProducerOptions{ 42 | StreamMaxLength: 1000, 43 | ApproximateMaxLength: true, 44 | } 45 | 46 | // NewProducer uses a default set of options to create a Producer. It sets 47 | // StreamMaxLength to 1000 and ApproximateMaxLength to true. In most production 48 | // environments, you'll want to use NewProducerWithOptions. 49 | func NewProducer() (*Producer, error) { 50 | return NewProducerWithOptions(defaultProducerOptions) 51 | } 52 | 53 | // NewProducerWithOptions creates a Producer using custom ProducerOptions. 54 | func NewProducerWithOptions(options *ProducerOptions) (*Producer, error) { 55 | var r redis.UniversalClient 56 | 57 | if options.RedisClient != nil { 58 | r = options.RedisClient 59 | } else { 60 | r = newRedisClient(options.RedisOptions) 61 | } 62 | 63 | if err := redisPreflightChecks(r); err != nil { 64 | return nil, err 65 | } 66 | 67 | return &Producer{ 68 | options: options, 69 | redis: r, 70 | }, nil 71 | } 72 | 73 | // Enqueue takes in a pointer to Message and enqueues it into the stream set at 74 | // msg.Stream. While you can set msg.ID, unless you know what you're doing, you 75 | // should let Redis auto-generate the ID. If an ID is auto-generated, it will be 76 | // set on msg.ID for your reference. msg.Values is also required. 77 | func (p *Producer) Enqueue(msg *Message) error { 78 | args := &redis.XAddArgs{ 79 | ID: msg.ID, 80 | Stream: msg.Stream, 81 | Values: msg.Values, 82 | } 83 | if p.options.ApproximateMaxLength { 84 | args.MaxLenApprox = p.options.StreamMaxLength 85 | } else { 86 | args.MaxLen = p.options.StreamMaxLength 87 | } 88 | id, err := p.redis.XAdd(args).Result() 89 | if err != nil { 90 | return err 91 | } 92 | msg.ID = id 93 | return nil 94 | } 95 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # redisqueue 2 | 3 | ![Version](https://img.shields.io/badge/version-v2.1.0-green.svg) 4 | [![GoDoc](https://godoc.org/github.com/robinjoseph08/redisqueue?status.svg)](https://pkg.go.dev/github.com/robinjoseph08/redisqueue/v2?tab=doc) 5 | [![Build Status](https://travis-ci.org/robinjoseph08/redisqueue.svg?branch=master)](https://travis-ci.org/robinjoseph08/redisqueue) 6 | [![Coverage Status](https://coveralls.io/repos/github/robinjoseph08/redisqueue/badge.svg?branch=master)](https://coveralls.io/github/robinjoseph08/redisqueue?branch=master) 7 | [![Go Report Card](https://goreportcard.com/badge/github.com/robinjoseph08/redisqueue)](https://goreportcard.com/report/github.com/robinjoseph08/redisqueue) 8 | ![License](https://img.shields.io/github/license/robinjoseph08/redisqueue.svg) 9 | 10 | `redisqueue` provides a producer and consumer of a queue that uses [Redis 11 | streams](https://redis.io/topics/streams-intro). 12 | 13 | ## Features 14 | 15 | - A `Producer` struct to make enqueuing messages easy. 16 | - A `Consumer` struct to make processing messages concurrenly. 17 | - Claiming and acknowledging messages if there's no error, so that if a consumer 18 | dies while processing, the message it was working on isn't lost. This 19 | guarantees at least once delivery. 20 | - A "visibility timeout" so that if a message isn't processed in a designated 21 | time frame, it will be be processed by another consumer. 22 | - A max length on the stream so that it doesn't store the messages indefinitely 23 | and run out of memory. 24 | - Graceful handling of Unix signals (`SIGINT` and `SIGTERM`) to let in-flight 25 | messages complete. 26 | - A channel that will surface any errors so you can handle them centrally. 27 | - Graceful handling of panics to avoid crashing the whole process. 28 | - A concurrency setting to control how many goroutines are spawned to process 29 | messages. 30 | - A batch size setting to limit the total messages in flight. 31 | - Support for multiple streams. 32 | 33 | ## Installation 34 | 35 | `redisqueue` requires a Go version with Modules support and uses import 36 | versioning. So please make sure to initialize a Go module before installing 37 | `redisqueue`: 38 | 39 | ```sh 40 | go mod init github.com/my/repo 41 | go get github.com/robinjoseph08/redisqueue/v2 42 | ``` 43 | 44 | Import: 45 | 46 | ```go 47 | import "github.com/robinjoseph08/redisqueue/v2" 48 | ``` 49 | 50 | ## Example 51 | 52 | Here's an example of a producer that inserts 1000 messages into a queue: 53 | 54 | ```go 55 | package main 56 | 57 | import ( 58 | "fmt" 59 | 60 | "github.com/robinjoseph08/redisqueue/v2" 61 | ) 62 | 63 | func main() { 64 | p, err := redisqueue.NewProducerWithOptions(&redisqueue.ProducerOptions{ 65 | StreamMaxLength: 10000, 66 | ApproximateMaxLength: true, 67 | }) 68 | if err != nil { 69 | panic(err) 70 | } 71 | 72 | for i := 0; i < 1000; i++ { 73 | err := p.Enqueue(&redisqueue.Message{ 74 | Stream: "redisqueue:test", 75 | Values: map[string]interface{}{ 76 | "index": i, 77 | }, 78 | }) 79 | if err != nil { 80 | panic(err) 81 | } 82 | 83 | if i%100 == 0 { 84 | fmt.Printf("enqueued %d\n", i) 85 | } 86 | } 87 | } 88 | ``` 89 | 90 | And here's an example of a consumer that reads the messages off of that queue: 91 | 92 | ```go 93 | package main 94 | 95 | import ( 96 | "fmt" 97 | "time" 98 | 99 | "github.com/robinjoseph08/redisqueue/v2" 100 | ) 101 | 102 | func main() { 103 | c, err := redisqueue.NewConsumerWithOptions(&redisqueue.ConsumerOptions{ 104 | VisibilityTimeout: 60 * time.Second, 105 | BlockingTimeout: 5 * time.Second, 106 | ReclaimInterval: 1 * time.Second, 107 | BufferSize: 100, 108 | Concurrency: 10, 109 | }) 110 | if err != nil { 111 | panic(err) 112 | } 113 | 114 | c.Register("redisqueue:test", process) 115 | 116 | go func() { 117 | for err := range c.Errors { 118 | // handle errors accordingly 119 | fmt.Printf("err: %+v\n", err) 120 | } 121 | }() 122 | 123 | fmt.Println("starting") 124 | c.Run() 125 | fmt.Println("stopped") 126 | } 127 | 128 | func process(msg *redisqueue.Message) error { 129 | fmt.Printf("processing message: %v\n", msg.Values["index"]) 130 | return nil 131 | } 132 | ``` 133 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nBpB11F9br+3HUrpgb+fcm5iADzXXYEw= 2 | github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8/go.mod h1:oX5x61PbNXchhh0oikYAH+4Pcfw5LKv21+Jnpr6r6Pc= 3 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 6 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= 8 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 9 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 10 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 11 | github.com/git-chglog/git-chglog v0.0.0-20190611050339-63a4e637021f h1:8l4Aw3Jmx0pLKYMkY+1b6yBPgE+rzRtA5T3vqFyI2Z8= 12 | github.com/git-chglog/git-chglog v0.0.0-20190611050339-63a4e637021f/go.mod h1:Dcsy1kii/xFyNad5JqY/d0GO5mu91sungp5xotbm3Yk= 13 | github.com/go-redis/redis/v7 v7.3.0 h1:3oHqd0W7f/VLKBxeYTEpqdMUsmMectngjM9OtoRoIgg= 14 | github.com/go-redis/redis/v7 v7.3.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= 15 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 16 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 17 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 18 | github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= 19 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 20 | github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= 21 | github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 22 | github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174 h1:WlZsjVhE8Af9IcZDGgJGQpNflI3+MJSBhsgT5PCtzBQ= 23 | github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A= 24 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 25 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 26 | github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= 27 | github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 28 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= 29 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= 30 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 31 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 32 | github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= 33 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 34 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 35 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 36 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 37 | github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= 38 | github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 39 | github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 40 | github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= 41 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 42 | github.com/mattn/goveralls v0.0.2 h1:7eJB6EqsPhRVxvwEXGnqdO2sJI0PTsrWoTMXEk9/OQc= 43 | github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= 44 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= 45 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= 46 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 47 | github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= 48 | github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 49 | github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= 50 | github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 51 | github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= 52 | github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= 53 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 54 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 55 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 56 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 57 | github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= 58 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 59 | github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 60 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 61 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 62 | github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df h1:Y2l28Jr3vOEeYtxfVbMtVfOdAwuUqWaP9fvNKiBVeXY= 63 | github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df/go.mod h1:pnyouUty/nBr/zm3GYwTIt+qFTLWbdjeLjZmJdzJOu8= 64 | github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= 65 | github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= 66 | golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 67 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= 68 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 69 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 70 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= 71 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 72 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= 73 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 74 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 75 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= 76 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 77 | golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 78 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 79 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 80 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= 81 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 82 | golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= 83 | golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 84 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 85 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 86 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 87 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 88 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 89 | golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db h1:9hRk1xeL9LTT3yX/941DqeBz87XgHAQuj+TbimYJuiw= 90 | golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= 91 | gopkg.in/AlecAivazis/survey.v1 v1.8.5 h1:QoEEmn/d5BbuPIL2qvXwzJdttFFhRQFkaq+tEKb7SMI= 92 | gopkg.in/AlecAivazis/survey.v1 v1.8.5/go.mod h1:iBNOmqKz/NUbZx3bA+4hAGLRC7fSK7tgtVDT4tB22XA= 93 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 94 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 95 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 96 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 97 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 98 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 99 | gopkg.in/kyokomi/emoji.v1 v1.5.1 h1:beetH5mWDMzFznJ+Qzd5KVHp79YKhVUMcdO8LpRLeGw= 100 | gopkg.in/kyokomi/emoji.v1 v1.5.1/go.mod h1:N9AZ6hi1jHOPn34PsbpufQZUcKftSD7WgS2pgpmH4Lg= 101 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 102 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 103 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 104 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 105 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 106 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 107 | gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= 108 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 109 | -------------------------------------------------------------------------------- /consumer_test.go: -------------------------------------------------------------------------------- 1 | package redisqueue 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | "time" 7 | 8 | "github.com/go-redis/redis/v7" 9 | "github.com/pkg/errors" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestNewConsumer(t *testing.T) { 15 | t.Run("creates a new consumer", func(tt *testing.T) { 16 | c, err := NewConsumer() 17 | require.NoError(tt, err) 18 | 19 | assert.NotNil(tt, c) 20 | }) 21 | } 22 | 23 | func TestNewConsumerWithOptions(t *testing.T) { 24 | t.Run("creates a new consumer", func(tt *testing.T) { 25 | c, err := NewConsumerWithOptions(&ConsumerOptions{}) 26 | require.NoError(tt, err) 27 | 28 | assert.NotNil(tt, c) 29 | }) 30 | 31 | t.Run("sets defaults for Name, GroupName, BlockingTimeout, and ReclaimTimeout", func(tt *testing.T) { 32 | c, err := NewConsumerWithOptions(&ConsumerOptions{}) 33 | require.NoError(tt, err) 34 | 35 | hostname, err := os.Hostname() 36 | require.NoError(tt, err) 37 | 38 | assert.Equal(tt, hostname, c.options.Name) 39 | assert.Equal(tt, "redisqueue", c.options.GroupName) 40 | assert.Equal(tt, 5*time.Second, c.options.BlockingTimeout) 41 | assert.Equal(tt, 1*time.Second, c.options.ReclaimInterval) 42 | }) 43 | 44 | t.Run("allows override of Name, GroupName, BlockingTimeout, ReclaimTimeout, and RedisClient", func(tt *testing.T) { 45 | rc := newRedisClient(nil) 46 | 47 | c, err := NewConsumerWithOptions(&ConsumerOptions{ 48 | Name: "test_name", 49 | GroupName: "test_group_name", 50 | BlockingTimeout: 10 * time.Second, 51 | ReclaimInterval: 10 * time.Second, 52 | RedisClient: rc, 53 | }) 54 | require.NoError(tt, err) 55 | 56 | assert.Equal(tt, rc, c.redis) 57 | assert.Equal(tt, "test_name", c.options.Name) 58 | assert.Equal(tt, "test_group_name", c.options.GroupName) 59 | assert.Equal(tt, 10*time.Second, c.options.BlockingTimeout) 60 | assert.Equal(tt, 10*time.Second, c.options.ReclaimInterval) 61 | }) 62 | 63 | t.Run("bubbles up errors", func(tt *testing.T) { 64 | _, err := NewConsumerWithOptions(&ConsumerOptions{ 65 | RedisOptions: &RedisOptions{Addr: "localhost:0"}, 66 | }) 67 | require.Error(tt, err) 68 | 69 | assert.Contains(tt, err.Error(), "dial tcp") 70 | }) 71 | } 72 | 73 | func TestRegister(t *testing.T) { 74 | fn := func(msg *Message) error { 75 | return nil 76 | } 77 | 78 | t.Run("set the function", func(tt *testing.T) { 79 | c, err := NewConsumer() 80 | require.NoError(tt, err) 81 | 82 | c.Register(tt.Name(), fn) 83 | 84 | assert.Len(tt, c.consumers, 1) 85 | }) 86 | } 87 | 88 | func TestRegisterWithLastID(t *testing.T) { 89 | fn := func(msg *Message) error { 90 | return nil 91 | } 92 | 93 | tests := []struct { 94 | name string 95 | stream string 96 | id string 97 | want map[string]registeredConsumer 98 | }{ 99 | { 100 | name: "custom_id", 101 | id: "42", 102 | want: map[string]registeredConsumer{ 103 | "test": {id: "42", fn: fn}, 104 | }, 105 | }, 106 | { 107 | name: "no_id", 108 | id: "", 109 | want: map[string]registeredConsumer{ 110 | "test": {id: "0", fn: fn}, 111 | }, 112 | }, 113 | } 114 | 115 | for _, tt := range tests { 116 | t.Run(tt.name, func(t *testing.T) { 117 | c, err := NewConsumer() 118 | require.NoError(t, err) 119 | 120 | c.RegisterWithLastID("test", tt.id, fn) 121 | 122 | assert.Len(t, c.consumers, 1) 123 | assert.Contains(t, c.consumers, "test") 124 | assert.Equal(t, c.consumers["test"].id, tt.want["test"].id) 125 | assert.NotNil(t, c.consumers["test"].fn) 126 | }) 127 | } 128 | } 129 | 130 | func TestRun(t *testing.T) { 131 | t.Run("sends an error if no ConsumerFuncs are registered", func(tt *testing.T) { 132 | c, err := NewConsumer() 133 | require.NoError(tt, err) 134 | 135 | go func() { 136 | err := <-c.Errors 137 | require.Error(tt, err) 138 | assert.Equal(tt, "at least one consumer function needs to be registered", err.Error()) 139 | }() 140 | 141 | c.Run() 142 | }) 143 | 144 | t.Run("calls the ConsumerFunc on for a message", func(tt *testing.T) { 145 | // create a consumer 146 | c, err := NewConsumerWithOptions(&ConsumerOptions{ 147 | VisibilityTimeout: 60 * time.Second, 148 | BlockingTimeout: 10 * time.Millisecond, 149 | BufferSize: 100, 150 | Concurrency: 10, 151 | }) 152 | require.NoError(tt, err) 153 | 154 | // create a producer 155 | p, err := NewProducer() 156 | require.NoError(tt, err) 157 | 158 | // create consumer group 159 | c.redis.XGroupDestroy(tt.Name(), c.options.GroupName) 160 | c.redis.XGroupCreateMkStream(tt.Name(), c.options.GroupName, "$") 161 | 162 | // enqueue a message 163 | err = p.Enqueue(&Message{ 164 | Stream: tt.Name(), 165 | Values: map[string]interface{}{"test": "value"}, 166 | }) 167 | require.NoError(tt, err) 168 | 169 | // register a handler that will assert the message and then shut down 170 | // the consumer 171 | c.Register(tt.Name(), func(m *Message) error { 172 | assert.Equal(tt, "value", m.Values["test"]) 173 | c.Shutdown() 174 | return nil 175 | }) 176 | 177 | // watch for consumer errors 178 | go func() { 179 | err := <-c.Errors 180 | require.NoError(tt, err) 181 | }() 182 | 183 | // run the consumer 184 | c.Run() 185 | }) 186 | 187 | t.Run("reclaims pending messages according to ReclaimInterval", func(tt *testing.T) { 188 | // create a consumer 189 | c, err := NewConsumerWithOptions(&ConsumerOptions{ 190 | VisibilityTimeout: 5 * time.Millisecond, 191 | BlockingTimeout: 10 * time.Millisecond, 192 | ReclaimInterval: 1 * time.Millisecond, 193 | BufferSize: 100, 194 | Concurrency: 10, 195 | }) 196 | require.NoError(tt, err) 197 | 198 | // create a producer 199 | p, err := NewProducer() 200 | require.NoError(tt, err) 201 | 202 | // create consumer group 203 | c.redis.XGroupDestroy(tt.Name(), c.options.GroupName) 204 | c.redis.XGroupCreateMkStream(tt.Name(), c.options.GroupName, "$") 205 | 206 | // enqueue a message 207 | msg := &Message{ 208 | Stream: tt.Name(), 209 | Values: map[string]interface{}{"test": "value"}, 210 | } 211 | err = p.Enqueue(msg) 212 | require.NoError(tt, err) 213 | 214 | // register a handler that will assert the message and then shut down 215 | // the consumer 216 | c.Register(tt.Name(), func(m *Message) error { 217 | assert.Equal(tt, msg.ID, m.ID) 218 | c.Shutdown() 219 | return nil 220 | }) 221 | 222 | // read the message but don't acknowledge it 223 | res, err := c.redis.XReadGroup(&redis.XReadGroupArgs{ 224 | Group: c.options.GroupName, 225 | Consumer: "failed_consumer", 226 | Streams: []string{tt.Name(), ">"}, 227 | Count: 1, 228 | }).Result() 229 | require.NoError(tt, err) 230 | require.Len(tt, res, 1) 231 | require.Len(tt, res[0].Messages, 1) 232 | require.Equal(tt, msg.ID, res[0].Messages[0].ID) 233 | 234 | // wait for more than VisibilityTimeout + ReclaimInterval to ensure that 235 | // the pending message is reclaimed 236 | time.Sleep(6 * time.Millisecond) 237 | 238 | // watch for consumer errors 239 | go func() { 240 | err := <-c.Errors 241 | require.NoError(tt, err) 242 | }() 243 | 244 | // run the consumer 245 | c.Run() 246 | }) 247 | 248 | t.Run("doesn't reclaim if there is no VisibilityTimeout set", func(tt *testing.T) { 249 | // create a consumer 250 | c, err := NewConsumerWithOptions(&ConsumerOptions{ 251 | BlockingTimeout: 10 * time.Millisecond, 252 | ReclaimInterval: 1 * time.Millisecond, 253 | BufferSize: 100, 254 | Concurrency: 10, 255 | }) 256 | require.NoError(tt, err) 257 | 258 | // create a producer 259 | p, err := NewProducerWithOptions(&ProducerOptions{ 260 | StreamMaxLength: 2, 261 | ApproximateMaxLength: false, 262 | }) 263 | require.NoError(tt, err) 264 | 265 | // create consumer group 266 | c.redis.XGroupDestroy(tt.Name(), c.options.GroupName) 267 | c.redis.XGroupCreateMkStream(tt.Name(), c.options.GroupName, "$") 268 | 269 | // enqueue a message 270 | msg1 := &Message{ 271 | Stream: tt.Name(), 272 | Values: map[string]interface{}{"test": "value"}, 273 | } 274 | msg2 := &Message{ 275 | Stream: tt.Name(), 276 | Values: map[string]interface{}{"test": "value2"}, 277 | } 278 | err = p.Enqueue(msg1) 279 | require.NoError(tt, err) 280 | 281 | // register a handler that will assert the message and then shut down 282 | // the consumer 283 | c.Register(tt.Name(), func(m *Message) error { 284 | assert.Equal(tt, msg2.ID, m.ID) 285 | c.Shutdown() 286 | return nil 287 | }) 288 | 289 | // read the message but don't acknowledge it 290 | res, err := c.redis.XReadGroup(&redis.XReadGroupArgs{ 291 | Group: c.options.GroupName, 292 | Consumer: "failed_consumer", 293 | Streams: []string{tt.Name(), ">"}, 294 | Count: 1, 295 | }).Result() 296 | require.NoError(tt, err) 297 | require.Len(tt, res, 1) 298 | require.Len(tt, res[0].Messages, 1) 299 | require.Equal(tt, msg1.ID, res[0].Messages[0].ID) 300 | 301 | // add another message to the stream to let the consumer consume it 302 | err = p.Enqueue(msg2) 303 | require.NoError(tt, err) 304 | 305 | // watch for consumer errors 306 | go func() { 307 | err := <-c.Errors 308 | require.NoError(tt, err) 309 | }() 310 | 311 | // run the consumer 312 | c.Run() 313 | 314 | // check if the pending message is still there 315 | pendingRes, err := c.redis.XPendingExt(&redis.XPendingExtArgs{ 316 | Stream: tt.Name(), 317 | Group: c.options.GroupName, 318 | Start: "-", 319 | End: "+", 320 | Count: 1, 321 | }).Result() 322 | require.NoError(tt, err) 323 | require.Len(tt, pendingRes, 1) 324 | require.Equal(tt, msg1.ID, pendingRes[0].ID) 325 | }) 326 | 327 | t.Run("acknowledges pending messages that have already been deleted", func(tt *testing.T) { 328 | // create a consumer 329 | c, err := NewConsumerWithOptions(&ConsumerOptions{ 330 | VisibilityTimeout: 5 * time.Millisecond, 331 | BlockingTimeout: 10 * time.Millisecond, 332 | ReclaimInterval: 1 * time.Millisecond, 333 | BufferSize: 100, 334 | Concurrency: 10, 335 | }) 336 | require.NoError(tt, err) 337 | 338 | // create a producer 339 | p, err := NewProducerWithOptions(&ProducerOptions{ 340 | StreamMaxLength: 1, 341 | ApproximateMaxLength: false, 342 | }) 343 | require.NoError(tt, err) 344 | 345 | // create consumer group 346 | c.redis.XGroupDestroy(tt.Name(), c.options.GroupName) 347 | c.redis.XGroupCreateMkStream(tt.Name(), c.options.GroupName, "$") 348 | 349 | // enqueue a message 350 | msg := &Message{ 351 | Stream: tt.Name(), 352 | Values: map[string]interface{}{"test": "value"}, 353 | } 354 | err = p.Enqueue(msg) 355 | require.NoError(tt, err) 356 | 357 | // register a noop handler that should never be called 358 | c.Register(tt.Name(), func(m *Message) error { 359 | t.Fail() 360 | return nil 361 | }) 362 | 363 | // read the message but don't acknowledge it 364 | res, err := c.redis.XReadGroup(&redis.XReadGroupArgs{ 365 | Group: c.options.GroupName, 366 | Consumer: "failed_consumer", 367 | Streams: []string{tt.Name(), ">"}, 368 | Count: 1, 369 | }).Result() 370 | require.NoError(tt, err) 371 | require.Len(tt, res, 1) 372 | require.Len(tt, res[0].Messages, 1) 373 | require.Equal(tt, msg.ID, res[0].Messages[0].ID) 374 | 375 | // delete the message 376 | err = c.redis.XDel(tt.Name(), msg.ID).Err() 377 | require.NoError(tt, err) 378 | 379 | // watch for consumer errors 380 | go func() { 381 | err := <-c.Errors 382 | require.NoError(tt, err) 383 | }() 384 | 385 | // in 10ms, shut down the consumer 386 | go func() { 387 | time.Sleep(10 * time.Millisecond) 388 | c.Shutdown() 389 | }() 390 | 391 | // run the consumer 392 | c.Run() 393 | 394 | // check that there are no pending messages 395 | pendingRes, err := c.redis.XPendingExt(&redis.XPendingExtArgs{ 396 | Stream: tt.Name(), 397 | Group: c.options.GroupName, 398 | Start: "-", 399 | End: "+", 400 | Count: 1, 401 | }).Result() 402 | require.NoError(tt, err) 403 | require.Len(tt, pendingRes, 0) 404 | }) 405 | 406 | t.Run("returns an error on a string panic", func(tt *testing.T) { 407 | // create a consumer 408 | c, err := NewConsumerWithOptions(&ConsumerOptions{ 409 | VisibilityTimeout: 60 * time.Second, 410 | BlockingTimeout: 10 * time.Millisecond, 411 | BufferSize: 100, 412 | Concurrency: 10, 413 | }) 414 | require.NoError(tt, err) 415 | 416 | // create a producer 417 | p, err := NewProducer() 418 | require.NoError(tt, err) 419 | 420 | // create consumer group 421 | c.redis.XGroupDestroy(tt.Name(), c.options.GroupName) 422 | c.redis.XGroupCreateMkStream(tt.Name(), c.options.GroupName, "$") 423 | 424 | // enqueue a message 425 | err = p.Enqueue(&Message{ 426 | Stream: tt.Name(), 427 | Values: map[string]interface{}{"test": "value"}, 428 | }) 429 | require.NoError(tt, err) 430 | 431 | // register a handler that will assert the message, shut down the 432 | // consumer, and then panic with a string 433 | c.Register(tt.Name(), func(m *Message) error { 434 | assert.Equal(tt, "value", m.Values["test"]) 435 | c.Shutdown() 436 | panic("this is a panic") 437 | }) 438 | 439 | // watch for the panic 440 | go func() { 441 | err := <-c.Errors 442 | require.Error(tt, err) 443 | assert.Contains(tt, err.Error(), "this is a panic") 444 | }() 445 | 446 | // run the consumer 447 | c.Run() 448 | }) 449 | 450 | t.Run("returns an error on an error panic", func(tt *testing.T) { 451 | // create a consumer 452 | c, err := NewConsumerWithOptions(&ConsumerOptions{ 453 | VisibilityTimeout: 60 * time.Second, 454 | BlockingTimeout: 10 * time.Millisecond, 455 | BufferSize: 100, 456 | Concurrency: 10, 457 | }) 458 | require.NoError(tt, err) 459 | 460 | // create a producer 461 | p, err := NewProducer() 462 | require.NoError(tt, err) 463 | 464 | // create consumer group 465 | c.redis.XGroupDestroy(tt.Name(), c.options.GroupName) 466 | c.redis.XGroupCreateMkStream(tt.Name(), c.options.GroupName, "$") 467 | 468 | // enqueue a message 469 | err = p.Enqueue(&Message{ 470 | Stream: tt.Name(), 471 | Values: map[string]interface{}{"test": "value"}, 472 | }) 473 | require.NoError(tt, err) 474 | 475 | // register a handler that will assert the message, shut down the 476 | // consumer, and then panic with an error 477 | c.Register(tt.Name(), func(m *Message) error { 478 | assert.Equal(tt, "value", m.Values["test"]) 479 | c.Shutdown() 480 | panic(errors.New("this is a panic")) 481 | }) 482 | 483 | // watch for the panic 484 | go func() { 485 | err := <-c.Errors 486 | require.Error(tt, err) 487 | assert.Contains(tt, err.Error(), "this is a panic") 488 | }() 489 | 490 | // run the consumer 491 | c.Run() 492 | }) 493 | } 494 | -------------------------------------------------------------------------------- /consumer.go: -------------------------------------------------------------------------------- 1 | package redisqueue 2 | 3 | import ( 4 | "net" 5 | "os" 6 | "sync" 7 | "time" 8 | 9 | "github.com/go-redis/redis/v7" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | // ConsumerFunc is a type alias for the functions that will be used to handle 14 | // and process Messages. 15 | type ConsumerFunc func(*Message) error 16 | 17 | type registeredConsumer struct { 18 | fn ConsumerFunc 19 | id string 20 | } 21 | 22 | // ConsumerOptions provide options to configure the Consumer. 23 | type ConsumerOptions struct { 24 | // Name sets the name of this consumer. This will be used when fetching from 25 | // Redis. If empty, the hostname will be used. 26 | Name string 27 | // GroupName sets the name of the consumer group. This will be used when 28 | // coordinating in Redis. If empty, the hostname will be used. 29 | GroupName string 30 | // VisibilityTimeout dictates the maximum amount of time a message should 31 | // stay in pending. If there is a message that has been idle for more than 32 | // this duration, the consumer will attempt to claim it. 33 | VisibilityTimeout time.Duration 34 | // BlockingTimeout designates how long the XREADGROUP call blocks for. If 35 | // this is 0, it will block indefinitely. While this is the most efficient 36 | // from a polling perspective, if this call never times out, there is no 37 | // opportunity to yield back to Go at a regular interval. This means it's 38 | // possible that if no messages are coming in, the consumer cannot 39 | // gracefully shutdown. Instead, it's recommended to set this to 1-5 40 | // seconds, or even longer, depending on how long your application can wait 41 | // to shutdown. 42 | BlockingTimeout time.Duration 43 | // ReclaimInterval is the amount of time in between calls to XPENDING to 44 | // attempt to reclaim jobs that have been idle for more than the visibility 45 | // timeout. A smaller duration will result in more frequent checks. This 46 | // will allow messages to be reaped faster, but it will put more load on 47 | // Redis. 48 | ReclaimInterval time.Duration 49 | // BufferSize determines the size of the channel uses to coordinate the 50 | // processing of the messages. This determines the maximum number of 51 | // in-flight messages. 52 | BufferSize int 53 | // Concurrency dictates how many goroutines to spawn to handle the messages. 54 | Concurrency int 55 | // RedisClient supersedes the RedisOptions field, and allows you to inject 56 | // an already-made Redis Client for use in the consumer. This may be either 57 | // the standard client or a cluster client. 58 | RedisClient redis.UniversalClient 59 | // RedisOptions allows you to configure the underlying Redis connection. 60 | // More info here: 61 | // https://pkg.go.dev/github.com/go-redis/redis/v7?tab=doc#Options. 62 | // 63 | // This field is used if RedisClient field is nil. 64 | RedisOptions *RedisOptions 65 | } 66 | 67 | // Consumer adds a convenient wrapper around dequeuing and managing concurrency. 68 | type Consumer struct { 69 | // Errors is a channel that you can receive from to centrally handle any 70 | // errors that may occur either by your ConsumerFuncs or by internal 71 | // processing functions. Because this is an unbuffered channel, you must 72 | // have a listener on it. If you don't parts of the consumer could stop 73 | // functioning when errors occur due to the blocking nature of unbuffered 74 | // channels. 75 | Errors chan error 76 | 77 | options *ConsumerOptions 78 | redis redis.UniversalClient 79 | consumers map[string]registeredConsumer 80 | streams []string 81 | queue chan *Message 82 | wg *sync.WaitGroup 83 | 84 | stopReclaim chan struct{} 85 | stopPoll chan struct{} 86 | stopWorkers chan struct{} 87 | } 88 | 89 | var defaultConsumerOptions = &ConsumerOptions{ 90 | VisibilityTimeout: 60 * time.Second, 91 | BlockingTimeout: 5 * time.Second, 92 | ReclaimInterval: 1 * time.Second, 93 | BufferSize: 100, 94 | Concurrency: 10, 95 | } 96 | 97 | // NewConsumer uses a default set of options to create a Consumer. It sets Name 98 | // to the hostname, GroupName to "redisqueue", VisibilityTimeout to 60 seconds, 99 | // BufferSize to 100, and Concurrency to 10. In most production environments, 100 | // you'll want to use NewConsumerWithOptions. 101 | func NewConsumer() (*Consumer, error) { 102 | return NewConsumerWithOptions(defaultConsumerOptions) 103 | } 104 | 105 | // NewConsumerWithOptions creates a Consumer with custom ConsumerOptions. If 106 | // Name is left empty, it defaults to the hostname; if GroupName is left empty, 107 | // it defaults to "redisqueue"; if BlockingTimeout is 0, it defaults to 5 108 | // seconds; if ReclaimInterval is 0, it defaults to 1 second. 109 | func NewConsumerWithOptions(options *ConsumerOptions) (*Consumer, error) { 110 | hostname, _ := os.Hostname() 111 | 112 | if options.Name == "" { 113 | options.Name = hostname 114 | } 115 | if options.GroupName == "" { 116 | options.GroupName = "redisqueue" 117 | } 118 | if options.BlockingTimeout == 0 { 119 | options.BlockingTimeout = 5 * time.Second 120 | } 121 | if options.ReclaimInterval == 0 { 122 | options.ReclaimInterval = 1 * time.Second 123 | } 124 | 125 | var r redis.UniversalClient 126 | 127 | if options.RedisClient != nil { 128 | r = options.RedisClient 129 | } else { 130 | r = newRedisClient(options.RedisOptions) 131 | } 132 | 133 | if err := redisPreflightChecks(r); err != nil { 134 | return nil, err 135 | } 136 | 137 | return &Consumer{ 138 | Errors: make(chan error), 139 | 140 | options: options, 141 | redis: r, 142 | consumers: make(map[string]registeredConsumer), 143 | streams: make([]string, 0), 144 | queue: make(chan *Message, options.BufferSize), 145 | wg: &sync.WaitGroup{}, 146 | 147 | stopReclaim: make(chan struct{}, 1), 148 | stopPoll: make(chan struct{}, 1), 149 | stopWorkers: make(chan struct{}, options.Concurrency), 150 | }, nil 151 | } 152 | 153 | // RegisterWithLastID is the same as Register, except that it also lets you 154 | // specify the oldest message to receive when first creating the consumer group. 155 | // This can be any valid message ID, "0" for all messages in the stream, or "$" 156 | // for only new messages. 157 | // 158 | // If the consumer group already exists the id field is ignored, meaning you'll 159 | // receive unprocessed messages. 160 | func (c *Consumer) RegisterWithLastID(stream string, id string, fn ConsumerFunc) { 161 | if len(id) == 0 { 162 | id = "0" 163 | } 164 | 165 | c.consumers[stream] = registeredConsumer{ 166 | fn: fn, 167 | id: id, 168 | } 169 | } 170 | 171 | // Register takes in a stream name and a ConsumerFunc that will be called when a 172 | // message comes in from that stream. Register must be called at least once 173 | // before Run is called. If the same stream name is passed in twice, the first 174 | // ConsumerFunc is overwritten by the second. 175 | func (c *Consumer) Register(stream string, fn ConsumerFunc) { 176 | c.RegisterWithLastID(stream, "0", fn) 177 | } 178 | 179 | // Run starts all of the worker goroutines and starts processing from the 180 | // streams that have been registered with Register. All errors will be sent to 181 | // the Errors channel. If Register was never called, an error will be sent and 182 | // Run will terminate early. The same will happen if an error occurs when 183 | // creating the consumer group in Redis. Run will block until Shutdown is called 184 | // and all of the in-flight messages have been processed. 185 | func (c *Consumer) Run() { 186 | if len(c.consumers) == 0 { 187 | c.Errors <- errors.New("at least one consumer function needs to be registered") 188 | return 189 | } 190 | 191 | for stream, consumer := range c.consumers { 192 | c.streams = append(c.streams, stream) 193 | err := c.redis.XGroupCreateMkStream(stream, c.options.GroupName, consumer.id).Err() 194 | // ignoring the BUSYGROUP error makes this a noop 195 | if err != nil && err.Error() != "BUSYGROUP Consumer Group name already exists" { 196 | c.Errors <- errors.Wrap(err, "error creating consumer group") 197 | return 198 | } 199 | } 200 | 201 | for i := 0; i < len(c.consumers); i++ { 202 | c.streams = append(c.streams, ">") 203 | } 204 | 205 | go c.reclaim() 206 | go c.poll() 207 | 208 | stop := newSignalHandler() 209 | go func() { 210 | <-stop 211 | c.Shutdown() 212 | }() 213 | 214 | c.wg.Add(c.options.Concurrency) 215 | 216 | for i := 0; i < c.options.Concurrency; i++ { 217 | go c.work() 218 | } 219 | 220 | c.wg.Wait() 221 | } 222 | 223 | // Shutdown stops new messages from being processed and tells the workers to 224 | // wait until all in-flight messages have been processed, and then they exit. 225 | // The order that things stop is 1) the reclaim process (if it's running), 2) 226 | // the polling process, and 3) the worker processes. 227 | func (c *Consumer) Shutdown() { 228 | c.stopReclaim <- struct{}{} 229 | if c.options.VisibilityTimeout == 0 { 230 | c.stopPoll <- struct{}{} 231 | } 232 | } 233 | 234 | // reclaim runs in a separate goroutine and checks the list of pending messages 235 | // in every stream. For every message, if it's been idle for longer than the 236 | // VisibilityTimeout, it will attempt to claim that message for this consumer. 237 | // If VisibilityTimeout is 0, this function returns early and no messages are 238 | // reclaimed. It checks the list of pending messages according to 239 | // ReclaimInterval. 240 | func (c *Consumer) reclaim() { 241 | if c.options.VisibilityTimeout == 0 { 242 | return 243 | } 244 | 245 | ticker := time.NewTicker(c.options.ReclaimInterval) 246 | 247 | for { 248 | select { 249 | case <-c.stopReclaim: 250 | // once the reclaim process has stopped, stop the polling process 251 | c.stopPoll <- struct{}{} 252 | return 253 | case <-ticker.C: 254 | for stream := range c.consumers { 255 | start := "-" 256 | end := "+" 257 | 258 | for { 259 | res, err := c.redis.XPendingExt(&redis.XPendingExtArgs{ 260 | Stream: stream, 261 | Group: c.options.GroupName, 262 | Start: start, 263 | End: end, 264 | Count: int64(c.options.BufferSize - len(c.queue)), 265 | }).Result() 266 | if err != nil && err != redis.Nil { 267 | c.Errors <- errors.Wrap(err, "error listing pending messages") 268 | break 269 | } 270 | 271 | if len(res) == 0 { 272 | break 273 | } 274 | 275 | msgs := make([]string, 0) 276 | 277 | for _, r := range res { 278 | if r.Idle >= c.options.VisibilityTimeout { 279 | claimres, err := c.redis.XClaim(&redis.XClaimArgs{ 280 | Stream: stream, 281 | Group: c.options.GroupName, 282 | Consumer: c.options.Name, 283 | MinIdle: c.options.VisibilityTimeout, 284 | Messages: []string{r.ID}, 285 | }).Result() 286 | if err != nil && err != redis.Nil { 287 | c.Errors <- errors.Wrapf(err, "error claiming %d message(s)", len(msgs)) 288 | break 289 | } 290 | // If the Redis nil error is returned, it means that 291 | // the message no longer exists in the stream. 292 | // However, it is still in a pending state. This 293 | // could happen if a message was claimed by a 294 | // consumer, that consumer died, and the message 295 | // gets deleted (either through a XDEL call or 296 | // through MAXLEN). Since the message no longer 297 | // exists, the only way we can get it out of the 298 | // pending state is to acknowledge it. 299 | if err == redis.Nil { 300 | err = c.redis.XAck(stream, c.options.GroupName, r.ID).Err() 301 | if err != nil { 302 | c.Errors <- errors.Wrapf(err, "error acknowledging after failed claim for %q stream and %q message", stream, r.ID) 303 | continue 304 | } 305 | } 306 | c.enqueue(stream, claimres) 307 | } 308 | } 309 | 310 | newID, err := incrementMessageID(res[len(res)-1].ID) 311 | if err != nil { 312 | c.Errors <- err 313 | break 314 | } 315 | 316 | start = newID 317 | } 318 | } 319 | } 320 | } 321 | } 322 | 323 | // poll constantly checks the streams using XREADGROUP to see if there are any 324 | // messages for this consumer to process. It blocks for up to 5 seconds instead 325 | // of blocking indefinitely so that it can periodically check to see if Shutdown 326 | // was called. 327 | func (c *Consumer) poll() { 328 | for { 329 | select { 330 | case <-c.stopPoll: 331 | // once the polling has stopped (i.e. there will be no more messages 332 | // put onto c.queue), stop all of the workers 333 | for i := 0; i < c.options.Concurrency; i++ { 334 | c.stopWorkers <- struct{}{} 335 | } 336 | return 337 | default: 338 | res, err := c.redis.XReadGroup(&redis.XReadGroupArgs{ 339 | Group: c.options.GroupName, 340 | Consumer: c.options.Name, 341 | Streams: c.streams, 342 | Count: int64(c.options.BufferSize - len(c.queue)), 343 | Block: c.options.BlockingTimeout, 344 | }).Result() 345 | if err != nil { 346 | if err, ok := err.(net.Error); ok && err.Timeout() { 347 | continue 348 | } 349 | if err == redis.Nil { 350 | continue 351 | } 352 | c.Errors <- errors.Wrap(err, "error reading redis stream") 353 | continue 354 | } 355 | 356 | for _, r := range res { 357 | c.enqueue(r.Stream, r.Messages) 358 | } 359 | } 360 | } 361 | } 362 | 363 | // enqueue takes a slice of XMessages, creates corresponding Messages, and sends 364 | // them on the centralized channel for worker goroutines to process. 365 | func (c *Consumer) enqueue(stream string, msgs []redis.XMessage) { 366 | for _, m := range msgs { 367 | msg := &Message{ 368 | ID: m.ID, 369 | Stream: stream, 370 | Values: m.Values, 371 | } 372 | c.queue <- msg 373 | } 374 | } 375 | 376 | // work is called in a separate goroutine. The number of work goroutines is 377 | // determined by Concurreny. Once it gets a message from the centralized 378 | // channel, it calls the corrensponding ConsumerFunc depending on the stream it 379 | // came from. If no error is returned from the ConsumerFunc, the message is 380 | // acknowledged in Redis. 381 | func (c *Consumer) work() { 382 | defer c.wg.Done() 383 | 384 | for { 385 | select { 386 | case msg := <-c.queue: 387 | err := c.process(msg) 388 | if err != nil { 389 | c.Errors <- errors.Wrapf(err, "error calling ConsumerFunc for %q stream and %q message", msg.Stream, msg.ID) 390 | continue 391 | } 392 | err = c.redis.XAck(msg.Stream, c.options.GroupName, msg.ID).Err() 393 | if err != nil { 394 | c.Errors <- errors.Wrapf(err, "error acknowledging after success for %q stream and %q message", msg.Stream, msg.ID) 395 | continue 396 | } 397 | case <-c.stopWorkers: 398 | return 399 | } 400 | } 401 | } 402 | 403 | func (c *Consumer) process(msg *Message) (err error) { 404 | defer func() { 405 | if r := recover(); r != nil { 406 | if e, ok := r.(error); ok { 407 | err = errors.Wrap(e, "ConsumerFunc panic") 408 | return 409 | } 410 | err = errors.Errorf("ConsumerFunc panic: %v", r) 411 | } 412 | }() 413 | err = c.consumers[msg.Stream].fn(msg) 414 | return 415 | } 416 | --------------------------------------------------------------------------------