├── .github └── workflows │ ├── build.yml │ ├── commitlint.yml │ ├── golangci-lint.yml │ └── release.yml ├── .golangci.yml ├── .prettierrc ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.md ├── RELEASING.md ├── azsqs ├── factory.go └── queue.go ├── azsqs_test.go ├── bench_test.go ├── commitlint.config.js ├── consumer.go ├── consumer_config.go ├── consumer_test.go ├── doc.go ├── example ├── redisexample │ ├── README.md │ ├── consumer │ │ └── main.go │ ├── doc.go │ ├── go.mod │ ├── go.sum │ ├── producer │ │ └── main.go │ └── tasks.go └── sqsexample │ ├── README.md │ ├── consumer │ └── main.go │ ├── go.mod │ ├── go.sum │ ├── producer │ └── main.go │ └── tasks.go ├── example_ratelimit_test.go ├── example_test.go ├── extra └── taskqotel │ ├── go.mod │ ├── go.sum │ └── otel.go ├── go.mod ├── go.sum ├── handler.go ├── internal ├── base │ ├── batcher.go │ └── factory.go ├── error.go ├── log.go ├── msgutil │ └── msgutil.go ├── safe.go ├── unsafe.go └── util.go ├── ironmq ├── factory.go └── queue.go ├── ironmq_test.go ├── memqueue ├── bench_test.go ├── factory.go ├── memqueue_test.go └── queue.go ├── message.go ├── package.json ├── queue.go ├── redisq ├── factory.go └── queue.go ├── redisq_test.go ├── registry.go ├── scripts ├── release.sh └── tag.sh ├── storage.go ├── sysinfo_linux.go ├── sysinfo_other.go ├── task.go ├── taskq.go └── version.go /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [v3] 6 | pull_request: 7 | branches: [v3] 8 | 9 | jobs: 10 | build: 11 | name: build 12 | runs-on: ubuntu-latest 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | go-version: [1.16.x, 1.17.x] 17 | 18 | services: 19 | redis: 20 | image: redis 21 | options: >- 22 | --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 23 | ports: 24 | - 6379:6379 25 | 26 | steps: 27 | - name: Set up ${{ matrix.go-version }} 28 | uses: actions/setup-go@v2 29 | 30 | - name: Checkout code 31 | uses: actions/checkout@v2 32 | 33 | - name: Test 34 | run: make test 35 | -------------------------------------------------------------------------------- /.github/workflows/commitlint.yml: -------------------------------------------------------------------------------- 1 | name: Lint Commit Messages 2 | on: [pull_request] 3 | 4 | jobs: 5 | commitlint: 6 | runs-on: ubuntu-latest 7 | steps: 8 | - uses: actions/checkout@v2 9 | with: 10 | fetch-depth: 0 11 | - uses: wagoid/commitlint-github-action@v4 12 | -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | branches: 8 | - master 9 | - main 10 | pull_request: 11 | 12 | jobs: 13 | golangci: 14 | name: lint 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: golangci-lint 19 | uses: golangci/golangci-lint-action@v2 20 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Releases 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - uses: ncipollo/release-action@v1 14 | with: 15 | body: Check CHANGELOG.md for details 16 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | tests: false 3 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | proseWrap: always 2 | printWidth: 100 3 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## [3.2.9](https://github.com/vmihailenco/taskq/compare/v3.2.8...v3.2.9) (2022-08-24) 2 | 3 | 4 | ### Bug Fixes 5 | 6 | * swapped dst and src arguments in zstd decode call ([e61a842](https://github.com/vmihailenco/taskq/commit/e61a84219a8fe65444da5ca9b19571d2245633f2)) 7 | * Use localStorage for memqueue tests instead of Redis for [#162](https://github.com/vmihailenco/taskq/issues/162) ([b2ec9f5](https://github.com/vmihailenco/taskq/commit/b2ec9f53b0a3182b49c1c1510172e3ab6ac34b85)) 8 | 9 | 10 | ### Features 11 | 12 | * allow set backoff duration for redis scheduler ([f6818a8](https://github.com/vmihailenco/taskq/commit/f6818a888f92e6a78e022aae2083d202bfdd3726)) 13 | 14 | 15 | 16 | ## [3.2.8](https://github.com/vmihailenco/taskq/compare/v3.2.7...v3.2.8) (2021-11-18) 17 | 18 | 19 | ### Bug Fixes 20 | 21 | * ack msg before we delete it ([bac023a](https://github.com/vmihailenco/taskq/commit/bac023a71ba191e60f43ce3ca01a25d08d0a70c2)) 22 | * adding ctx to msg ([819b42b](https://github.com/vmihailenco/taskq/commit/819b42b66bf482187843670a4a2fc288e9173e29)) 23 | 24 | 25 | 26 | ## [3.2.7](https://github.com/vmihailenco/taskq/compare/v3.2.6...v3.2.7) (2021-10-28) 27 | 28 | 29 | ### Bug Fixes 30 | 31 | * **redisq:** rework tests to use redis client ([670be0f](https://github.com/vmihailenco/taskq/commit/670be0f0ba7ee729df4c6e89c0c571340914f936)) 32 | * **redsiq:** call xack inside delete in redsiq ([2f6bd74](https://github.com/vmihailenco/taskq/commit/2f6bd74c006132be6cbec74f9c4808888da34aff)) 33 | 34 | 35 | 36 | ## [3.2.6](https://github.com/vmihailenco/taskq/compare/v3.2.5...v3.2.6) (2021-10-11) 37 | 38 | 39 | ### Bug Fixes 40 | 41 | * introduce interfaces to allow mocking in tests ([6bc8f3b](https://github.com/vmihailenco/taskq/commit/6bc8f3b0462812996c39605c10428b43460696ff)) 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 The github.com/go-msgqueue/msgqueue Authors. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above 11 | copyright notice, this list of conditions and the following disclaimer 12 | in the documentation and/or other materials provided with the 13 | distribution. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ALL_GO_MOD_DIRS := $(shell find . -type f -name 'go.mod' -exec dirname {} \; | sort) 2 | 3 | test: 4 | go test ./... 5 | go test ./... -short -race 6 | go test ./... -run=NONE -bench=. -benchmem 7 | 8 | go_mod_tidy: 9 | go get -u && go mod tidy -go=1.17 10 | set -e; for dir in $(ALL_GO_MOD_DIRS); do \ 11 | echo "go mod tidy in $${dir}"; \ 12 | (cd "$${dir}" && \ 13 | go get -u ./... && \ 14 | go mod tidy -go=1.17); \ 15 | done 16 | 17 | fmt: 18 | gofmt -w -s ./ 19 | goimports -w -local github.com/vmihailenco/taskq ./ 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Golang asynchronous task/job queue with Redis, SQS, IronMQ, and in-memory backends 2 | 3 | ![build workflow](https://github.com/vmihailenco/taskq/actions/workflows/build.yml/badge.svg) 4 | [![PkgGoDev](https://pkg.go.dev/badge/github.com/vmihailenco/taskq/v3)](https://pkg.go.dev/github.com/vmihailenco/taskq/v3) 5 | [![Documentation](https://img.shields.io/badge/bun-documentation-informational)](https://taskq.uptrace.dev/) 6 | [![Chat](https://discordapp.com/api/guilds/752070105847955518/widget.png)](https://discord.gg/rWtp5Aj) 7 | 8 | > taskq is brought to you by :star: [**uptrace/uptrace**](https://github.com/uptrace/uptrace). 9 | > Uptrace is an open source and blazingly fast 10 | > [distributed tracing tool](https://get.uptrace.dev/compare/distributed-tracing-tools.html) powered 11 | > by OpenTelemetry and ClickHouse. Give it a star as well! 12 | 13 | ## Features 14 | 15 | - Redis, SQS, IronMQ, and in-memory backends. 16 | - Automatically scaling number of goroutines used to fetch (fetcher) and process messages (worker). 17 | - Global rate limiting. 18 | - Global limit of workers. 19 | - Call once - deduplicating messages with same name. 20 | - Automatic retries with exponential backoffs. 21 | - Automatic pausing when all messages in queue fail. 22 | - Fallback handler for processing failed messages. 23 | - Message batching. It is used in SQS and IronMQ backends to add/delete messages in batches. 24 | - Automatic message compression using snappy / s2. 25 | 26 | Resources: 27 | 28 | - [**Get started**](https://taskq.uptrace.dev/guide/golang-task-queue.html) 29 | - [Examples](https://github.com/vmihailenco/taskq/tree/v3/example) 30 | - [Discussions](https://github.com/uptrace/bun/discussions) 31 | - [Chat](https://discord.gg/rWtp5Aj) 32 | - [Reference](https://pkg.go.dev/github.com/vmihailenco/taskq/v3) 33 | 34 | ## Getting started 35 | 36 | To get started, see [Golang Task Queue](https://taskq.uptrace.dev/) documentation. 37 | 38 | **Producer**: 39 | 40 | ```go 41 | import ( 42 | "github.com/vmihailenco/taskq/v3" 43 | "github.com/vmihailenco/taskq/v3/redisq" 44 | ) 45 | 46 | // Create a queue factory. 47 | var QueueFactory = redisq.NewFactory() 48 | 49 | // Create a queue. 50 | var MainQueue = QueueFactory.RegisterQueue(&taskq.QueueOptions{ 51 | Name: "api-worker", 52 | Redis: Redis, // go-redis client 53 | }) 54 | 55 | // Register a task. 56 | var CountTask = taskq.RegisterTask(&taskq.TaskOptions{ 57 | Name: "counter", 58 | Handler: func() error { 59 | IncrLocalCounter() 60 | return nil 61 | }, 62 | }) 63 | 64 | ctx := context.Background() 65 | 66 | // And start producing. 67 | for { 68 | // Call the task without any args. 69 | err := MainQueue.Add(CountTask.WithArgs(ctx)) 70 | if err != nil { 71 | panic(err) 72 | } 73 | time.Sleep(time.Second) 74 | } 75 | ``` 76 | 77 | **Consumer**: 78 | 79 | ```go 80 | // Start consuming the queue. 81 | if err := MainQueue.Start(context.Background()); err != nil { 82 | log.Fatal(err) 83 | } 84 | ``` 85 | 86 | ## See also 87 | 88 | - [Golang ORM](https://github.com/uptrace/bun) for PostgreSQL, MySQL, MSSQL, and SQLite 89 | - [Golang PostgreSQL](https://bun.uptrace.dev/postgres/) 90 | - [Golang HTTP router](https://github.com/uptrace/bunrouter) 91 | - [Golang ClickHouse](https://github.com/uptrace/go-clickhouse) 92 | 93 | ## Contributors 94 | 95 | Thanks to all the people who already contributed! 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /RELEASING.md: -------------------------------------------------------------------------------- 1 | # Releasing 2 | 3 | 1. Run `release.sh` script which updates versions in go.mod files and pushes a new branch to GitHub: 4 | 5 | ```shell 6 | TAG=v1.0.0 ./scripts/release.sh 7 | ``` 8 | 9 | 2. Open a pull request and wait for the build to finish. 10 | 11 | 3. Merge the pull request and run `tag.sh` to create tags for packages: 12 | 13 | ```shell 14 | TAG=v1.0.0 ./scripts/tag.sh 15 | ``` 16 | -------------------------------------------------------------------------------- /azsqs/factory.go: -------------------------------------------------------------------------------- 1 | package azsqs 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aws/aws-sdk-go/service/sqs" 7 | 8 | "github.com/vmihailenco/taskq/v3" 9 | "github.com/vmihailenco/taskq/v3/internal/base" 10 | ) 11 | 12 | type factory struct { 13 | base base.Factory 14 | 15 | sqs *sqs.SQS 16 | accountID string 17 | } 18 | 19 | var _ taskq.Factory = (*factory)(nil) 20 | 21 | func NewFactory(sqs *sqs.SQS, accountID string) taskq.Factory { 22 | return &factory{ 23 | sqs: sqs, 24 | accountID: accountID, 25 | } 26 | } 27 | 28 | func (f *factory) RegisterQueue(opt *taskq.QueueOptions) taskq.Queue { 29 | q := NewQueue(f.sqs, f.accountID, opt) 30 | if err := f.base.Register(q); err != nil { 31 | panic(err) 32 | } 33 | return q 34 | } 35 | 36 | func (f *factory) Range(fn func(queue taskq.Queue) bool) { 37 | f.base.Range(fn) 38 | } 39 | 40 | func (f *factory) StartConsumers(ctx context.Context) error { 41 | return f.base.StartConsumers(ctx) 42 | } 43 | 44 | func (f *factory) StopConsumers() error { 45 | return f.base.StopConsumers() 46 | } 47 | 48 | func (f *factory) Close() error { 49 | return f.base.Close() 50 | } 51 | -------------------------------------------------------------------------------- /azsqs/queue.go: -------------------------------------------------------------------------------- 1 | package azsqs 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "strconv" 8 | "strings" 9 | "sync" 10 | "time" 11 | 12 | "github.com/aws/aws-sdk-go/aws" 13 | "github.com/aws/aws-sdk-go/aws/awserr" 14 | "github.com/aws/aws-sdk-go/service/sqs" 15 | 16 | "github.com/vmihailenco/taskq/v3" 17 | "github.com/vmihailenco/taskq/v3/internal" 18 | "github.com/vmihailenco/taskq/v3/internal/base" 19 | "github.com/vmihailenco/taskq/v3/internal/msgutil" 20 | "github.com/vmihailenco/taskq/v3/memqueue" 21 | ) 22 | 23 | const msgSizeLimit = 262144 24 | 25 | const delayUntilAttr = "TaskqDelayUntil" 26 | 27 | type Queue struct { 28 | opt *taskq.QueueOptions 29 | 30 | sqs *sqs.SQS 31 | accountID string 32 | 33 | addQueue *memqueue.Queue 34 | addTask *taskq.Task 35 | addBatcher *base.Batcher 36 | 37 | delQueue *memqueue.Queue 38 | delTask *taskq.Task 39 | delBatcher *base.Batcher 40 | 41 | mu sync.RWMutex 42 | _queueURL string 43 | 44 | consumer *taskq.Consumer 45 | } 46 | 47 | var _ taskq.Queue = (*Queue)(nil) 48 | 49 | func NewQueue(sqs *sqs.SQS, accountID string, opt *taskq.QueueOptions) *Queue { 50 | opt.Init() 51 | 52 | q := &Queue{ 53 | sqs: sqs, 54 | accountID: accountID, 55 | opt: opt, 56 | } 57 | 58 | q.initAddQueue() 59 | q.initDelQueue() 60 | 61 | return q 62 | } 63 | 64 | func (q *Queue) initAddQueue() { 65 | queueName := "azsqs:" + q.opt.Name + ":add" 66 | q.addQueue = memqueue.NewQueue(&taskq.QueueOptions{ 67 | Name: queueName, 68 | BufferSize: 100, 69 | Redis: q.opt.Redis, 70 | }) 71 | q.addTask = taskq.RegisterTask(&taskq.TaskOptions{ 72 | Name: queueName + ":add-message", 73 | Handler: taskq.HandlerFunc(q.addBatcherAdd), 74 | FallbackHandler: msgutil.UnwrapMessageHandler(q.opt.Handler.HandleMessage), 75 | RetryLimit: 3, 76 | MinBackoff: time.Second, 77 | }) 78 | q.addBatcher = base.NewBatcher(q.addQueue.Consumer(), &base.BatcherOptions{ 79 | Handler: q.addBatch, 80 | ShouldBatch: q.shouldBatchAdd, 81 | }) 82 | } 83 | 84 | func (q *Queue) initDelQueue() { 85 | queueName := "azsqs:" + q.opt.Name + ":delete" 86 | q.delQueue = memqueue.NewQueue(&taskq.QueueOptions{ 87 | Name: queueName, 88 | BufferSize: 100, 89 | Redis: q.opt.Redis, 90 | }) 91 | q.delTask = taskq.RegisterTask(&taskq.TaskOptions{ 92 | Name: queueName + ":delete-message", 93 | Handler: taskq.HandlerFunc(q.delBatcherAdd), 94 | RetryLimit: 3, 95 | MinBackoff: time.Second, 96 | }) 97 | q.delBatcher = base.NewBatcher(q.delQueue.Consumer(), &base.BatcherOptions{ 98 | Handler: q.deleteBatch, 99 | ShouldBatch: q.shouldBatchDelete, 100 | }) 101 | } 102 | 103 | func (q *Queue) Name() string { 104 | return q.opt.Name 105 | } 106 | 107 | func (q *Queue) String() string { 108 | return fmt.Sprintf("queue=%q", q.Name()) 109 | } 110 | 111 | func (q *Queue) Options() *taskq.QueueOptions { 112 | return q.opt 113 | } 114 | 115 | func (q *Queue) Consumer() taskq.QueueConsumer { 116 | if q.consumer == nil { 117 | q.consumer = taskq.NewConsumer(q) 118 | } 119 | return q.consumer 120 | } 121 | 122 | func (q *Queue) Len() (int, error) { 123 | params := &sqs.GetQueueAttributesInput{ 124 | QueueUrl: aws.String(q.queueURL()), 125 | AttributeNames: []*string{aws.String("ApproximateNumberOfMessages")}, 126 | } 127 | resp, err := q.sqs.GetQueueAttributes(params) 128 | if err != nil { 129 | return 0, err 130 | } 131 | 132 | prop := resp.Attributes["ApproximateNumberOfMessages"] 133 | return strconv.Atoi(*prop) 134 | } 135 | 136 | // Add adds message to the queue. 137 | func (q *Queue) Add(msg *taskq.Message) error { 138 | if msg.TaskName == "" { 139 | return internal.ErrTaskNameRequired 140 | } 141 | if q.isDuplicate(msg) { 142 | msg.Err = taskq.ErrDuplicate 143 | return nil 144 | } 145 | msg = msgutil.WrapMessage(msg) 146 | msg.TaskName = q.addTask.Name() 147 | return q.addQueue.Add(msg) 148 | } 149 | 150 | func (q *Queue) queueURL() string { 151 | q.mu.RLock() 152 | queueURL := q._queueURL 153 | q.mu.RUnlock() 154 | if queueURL != "" { 155 | return queueURL 156 | } 157 | 158 | q.mu.Lock() 159 | _, _ = q.createQueue() 160 | 161 | queueURL, err := q.getQueueURL() 162 | if err == nil { 163 | q._queueURL = queueURL 164 | } 165 | q.mu.Unlock() 166 | 167 | return queueURL 168 | } 169 | 170 | func (q *Queue) createQueue() (string, error) { 171 | visTimeout := strconv.Itoa(int(q.opt.ReservationTimeout / time.Second)) 172 | in := &sqs.CreateQueueInput{ 173 | QueueName: aws.String(q.Name()), 174 | Attributes: map[string]*string{ 175 | "VisibilityTimeout": &visTimeout, 176 | }, 177 | } 178 | out, err := q.sqs.CreateQueue(in) 179 | if err != nil { 180 | return "", err 181 | } 182 | return *out.QueueUrl, nil 183 | } 184 | 185 | func (q *Queue) getQueueURL() (string, error) { 186 | in := &sqs.GetQueueUrlInput{ 187 | QueueName: aws.String(q.Name()), 188 | QueueOwnerAWSAccountId: &q.accountID, 189 | } 190 | out, err := q.sqs.GetQueueUrl(in) 191 | if err != nil { 192 | return "", err 193 | } 194 | return *out.QueueUrl, nil 195 | } 196 | 197 | func (q *Queue) ReserveN( 198 | ctx context.Context, n int, waitTimeout time.Duration, 199 | ) ([]taskq.Message, error) { 200 | if n > 10 { 201 | n = 10 202 | } 203 | in := &sqs.ReceiveMessageInput{ 204 | QueueUrl: aws.String(q.queueURL()), 205 | MaxNumberOfMessages: aws.Int64(int64(n)), 206 | WaitTimeSeconds: aws.Int64(int64(waitTimeout / time.Second)), 207 | AttributeNames: []*string{aws.String("ApproximateReceiveCount")}, 208 | MessageAttributeNames: []*string{aws.String(delayUntilAttr)}, 209 | } 210 | out, err := q.sqs.ReceiveMessage(in) 211 | if err != nil { 212 | return nil, err 213 | } 214 | 215 | msgs := make([]taskq.Message, len(out.Messages)) 216 | for i, sqsMsg := range out.Messages { 217 | msg := &msgs[i] 218 | 219 | if *sqsMsg.Body != "_" { 220 | b, err := internal.DecodeString(*sqsMsg.Body) 221 | if err != nil { 222 | msg.Err = err 223 | } else { 224 | err = msg.UnmarshalBinary(b) 225 | if err != nil { 226 | msg.Err = err 227 | } 228 | } 229 | } 230 | 231 | msg.ReservationID = *sqsMsg.ReceiptHandle 232 | 233 | if v, ok := sqsMsg.Attributes["ApproximateReceiveCount"]; ok { 234 | var err error 235 | msg.ReservedCount, err = strconv.Atoi(*v) 236 | if err != nil { 237 | msg.Err = err 238 | } 239 | } 240 | 241 | if v, ok := sqsMsg.MessageAttributes[delayUntilAttr]; ok { 242 | until, err := time.Parse(time.RFC3339, *v.StringValue) 243 | if err != nil { 244 | msg.Err = err 245 | } else { 246 | msg.Delay = time.Until(until) 247 | if msg.Delay < 0 { 248 | msg.Delay = 0 249 | } 250 | } 251 | } 252 | } 253 | 254 | return msgs, nil 255 | } 256 | 257 | func (q *Queue) Release(msg *taskq.Message) error { 258 | in := &sqs.ChangeMessageVisibilityInput{ 259 | QueueUrl: aws.String(q.queueURL()), 260 | ReceiptHandle: &msg.ReservationID, 261 | VisibilityTimeout: aws.Int64(int64(msg.Delay / time.Second)), 262 | } 263 | var err error 264 | for i := 0; i < 3; i++ { 265 | _, err = q.sqs.ChangeMessageVisibility(in) 266 | if err == nil { 267 | return nil 268 | } 269 | if i > 0 && 270 | strings.Contains(err.Error(), "Message does not exist") { 271 | return nil 272 | } 273 | if !strings.Contains(err.Error(), "Please try again") { 274 | break 275 | } 276 | } 277 | return err 278 | } 279 | 280 | // Delete deletes the message from the queue. 281 | func (q *Queue) Delete(msg *taskq.Message) error { 282 | msg = msgutil.WrapMessage(msg) 283 | msg.TaskName = q.delTask.Name() 284 | return q.delQueue.Add(msg) 285 | } 286 | 287 | // Purge deletes all messages from the queue using SQS API. 288 | func (q *Queue) Purge() error { 289 | in := &sqs.PurgeQueueInput{ 290 | QueueUrl: aws.String(q.queueURL()), 291 | } 292 | _, err := q.sqs.PurgeQueue(in) 293 | return err 294 | } 295 | 296 | // Close is like CloseTimeout with 30 seconds timeout. 297 | func (q *Queue) Close() error { 298 | return q.CloseTimeout(30 * time.Second) 299 | } 300 | 301 | // CloseTimeout closes the queue waiting for pending messages to be processed. 302 | func (q *Queue) CloseTimeout(timeout time.Duration) error { 303 | if q.consumer != nil { 304 | _ = q.consumer.StopTimeout(timeout) 305 | } 306 | 307 | firstErr := q.addBatcher.Close() 308 | 309 | err := q.addQueue.CloseTimeout(timeout) 310 | if err != nil && firstErr == nil { 311 | firstErr = err 312 | } 313 | 314 | err = q.delBatcher.Close() 315 | if err != nil && firstErr == nil { 316 | firstErr = err 317 | } 318 | 319 | err = q.delQueue.CloseTimeout(timeout) 320 | if err != nil && firstErr == nil { 321 | firstErr = err 322 | } 323 | 324 | return firstErr 325 | } 326 | 327 | func (q *Queue) addBatcherAdd(msg *taskq.Message) error { 328 | return q.addBatcher.Add(msg) 329 | } 330 | 331 | func (q *Queue) addBatch(msgs []*taskq.Message) error { 332 | const maxDelay = 15 * time.Minute 333 | 334 | if len(msgs) == 0 { 335 | return errors.New("azsqs: no messages to add") 336 | } 337 | 338 | in := &sqs.SendMessageBatchInput{ 339 | QueueUrl: aws.String(q.queueURL()), 340 | } 341 | 342 | for i, msg := range msgs { 343 | msg, err := msgutil.UnwrapMessage(msg) 344 | if err != nil { 345 | return err 346 | } 347 | 348 | b, err := msg.MarshalBinary() 349 | if err != nil { 350 | msg.Err = err 351 | internal.Logger.Printf("azsqs: Message.MarshalBinary failed: %s", err) 352 | continue 353 | } 354 | 355 | str := internal.EncodeToString(b) 356 | if str == "" { 357 | str = "_" // SQS requires body. 358 | } 359 | 360 | if len(str) > msgSizeLimit { 361 | internal.Logger.Printf("task=%q: str=%d bytes=%d is larger than %d", 362 | msg.TaskName, len(str), len(b), msgSizeLimit) 363 | } 364 | 365 | entry := &sqs.SendMessageBatchRequestEntry{ 366 | Id: aws.String(strconv.Itoa(i)), 367 | MessageBody: aws.String(str), 368 | } 369 | if msg.Delay <= maxDelay { 370 | entry.DelaySeconds = aws.Int64(int64(msg.Delay / time.Second)) 371 | } else { 372 | entry.DelaySeconds = aws.Int64(int64(maxDelay / time.Second)) 373 | delayUntil := time.Now().Add(msg.Delay - maxDelay) 374 | entry.MessageAttributes = map[string]*sqs.MessageAttributeValue{ 375 | delayUntilAttr: { 376 | DataType: aws.String("String"), 377 | StringValue: aws.String(delayUntil.Format(time.RFC3339)), 378 | }, 379 | } 380 | } 381 | 382 | in.Entries = append(in.Entries, entry) 383 | } 384 | 385 | out, err := q.sqs.SendMessageBatch(in) 386 | if err != nil { 387 | awsErr, ok := err.(awserr.Error) 388 | if ok && awsErr.Code() == "ErrCodeBatchRequestTooLong" && len(msgs) == 1 { 389 | msgs[0].Err = err 390 | msgs[0].ReservedCount = 9999 // don't retry 391 | return err 392 | } 393 | internal.Logger.Printf("azsqs: SendMessageBatch msgs=%d failed: %s", 394 | len(msgs), err) 395 | return err 396 | } 397 | 398 | for _, entry := range out.Failed { 399 | if entry.SenderFault != nil && *entry.SenderFault { 400 | internal.Logger.Printf( 401 | "azsqs: SendMessageBatch failed with code=%s message=%q", 402 | tos(entry.Code), tos(entry.Message)) 403 | continue 404 | } 405 | 406 | msg := findMessageByID(msgs, tos(entry.Id)) 407 | if msg != nil { 408 | msg.Err = fmt.Errorf("%s: %s", tos(entry.Code), tos(entry.Message)) 409 | } else { 410 | internal.Logger.Printf("azsqs: can't find message with id=%s", tos(entry.Id)) 411 | } 412 | } 413 | 414 | return nil 415 | } 416 | 417 | func (q *Queue) shouldBatchAdd(batch []*taskq.Message, msg *taskq.Message) bool { 418 | batch = append(batch, msg) 419 | 420 | const sizeLimit = 250 * 1024 421 | if q.batchSize(batch) > sizeLimit { 422 | return false 423 | } 424 | 425 | const messagesLimit = 10 426 | return len(batch) < messagesLimit 427 | } 428 | 429 | func (q *Queue) batchSize(batch []*taskq.Message) int { 430 | var size int 431 | for _, msg := range batch { 432 | msg, err := msgutil.UnwrapMessage(msg) 433 | if err != nil { 434 | internal.Logger.Printf("azsqs: UnwrapMessage failed: %s", err) 435 | continue 436 | } 437 | 438 | b, err := msg.MarshalBinary() 439 | if err != nil { 440 | internal.Logger.Printf("azsqs: Message.MarshalBinary failed: %s", err) 441 | continue 442 | } 443 | 444 | size += internal.MaxEncodedLen(len(b)) 445 | } 446 | return size 447 | } 448 | 449 | func (q *Queue) delBatcherAdd(msg *taskq.Message) error { 450 | return q.delBatcher.Add(msg) 451 | } 452 | 453 | func (q *Queue) deleteBatch(msgs []*taskq.Message) error { 454 | if len(msgs) == 0 { 455 | return errors.New("azsqs: no messages to delete") 456 | } 457 | 458 | entries := make([]*sqs.DeleteMessageBatchRequestEntry, len(msgs)) 459 | for i, msg := range msgs { 460 | msg, err := msgutil.UnwrapMessage(msg) 461 | if err != nil { 462 | return err 463 | } 464 | 465 | entries[i] = &sqs.DeleteMessageBatchRequestEntry{ 466 | Id: aws.String(strconv.Itoa(i)), 467 | ReceiptHandle: &msg.ReservationID, 468 | } 469 | } 470 | 471 | in := &sqs.DeleteMessageBatchInput{ 472 | QueueUrl: aws.String(q.queueURL()), 473 | Entries: entries, 474 | } 475 | out, err := q.sqs.DeleteMessageBatch(in) 476 | if err != nil { 477 | internal.Logger.Printf("azsqs: DeleteMessageBatch failed: %s", err) 478 | return err 479 | } 480 | 481 | for _, entry := range out.Failed { 482 | if entry.SenderFault != nil && *entry.SenderFault { 483 | internal.Logger.Printf( 484 | "azsqs: DeleteMessageBatch failed with code=%s message=%q", 485 | tos(entry.Code), tos(entry.Message), 486 | ) 487 | continue 488 | } 489 | 490 | msg := findMessageByID(msgs, tos(entry.Id)) 491 | if msg != nil { 492 | msg.Err = fmt.Errorf("%s: %s", tos(entry.Code), tos(entry.Message)) 493 | } else { 494 | internal.Logger.Printf("azsqs: can't find message with id=%s", tos(entry.Id)) 495 | } 496 | } 497 | return nil 498 | } 499 | 500 | func (q *Queue) shouldBatchDelete(batch []*taskq.Message, msg *taskq.Message) bool { 501 | const messagesLimit = 10 502 | return len(batch)+1 < messagesLimit 503 | } 504 | 505 | func (q *Queue) GetAddQueue() *memqueue.Queue { 506 | return q.addQueue 507 | } 508 | 509 | func (q *Queue) GetDeleteQueue() *memqueue.Queue { 510 | return q.delQueue 511 | } 512 | 513 | func (q *Queue) isDuplicate(msg *taskq.Message) bool { 514 | if msg.Name == "" { 515 | return false 516 | } 517 | return q.opt.Storage.Exists(msg.Ctx, msgutil.FullMessageName(q, msg)) 518 | } 519 | 520 | func findMessageByID(msgs []*taskq.Message, id string) *taskq.Message { 521 | i, err := strconv.Atoi(id) 522 | if err != nil { 523 | return nil 524 | } 525 | if i < len(msgs) { 526 | return msgs[i] 527 | } 528 | return nil 529 | } 530 | 531 | func tos(s *string) string { 532 | if s == nil { 533 | return "" 534 | } 535 | return *s 536 | } 537 | -------------------------------------------------------------------------------- /azsqs_test.go: -------------------------------------------------------------------------------- 1 | package taskq_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/aws/aws-sdk-go/aws/session" 8 | "github.com/aws/aws-sdk-go/service/sqs" 9 | 10 | "github.com/vmihailenco/taskq/v3" 11 | "github.com/vmihailenco/taskq/v3/azsqs" 12 | ) 13 | 14 | var accountID string 15 | 16 | func init() { 17 | accountID = os.Getenv("AWS_ACCOUNT_ID") 18 | } 19 | 20 | func awsSQS() *sqs.SQS { 21 | return sqs.New(session.New()) 22 | } 23 | 24 | func azsqsFactory() taskq.Factory { 25 | return azsqs.NewFactory(awsSQS(), accountID) 26 | } 27 | 28 | func TestSQSConsumer(t *testing.T) { 29 | t.Skip() 30 | 31 | testConsumer(t, azsqsFactory(), &taskq.QueueOptions{ 32 | Name: queueName("sqs-consumer"), 33 | }) 34 | } 35 | 36 | func TestSQSUnknownTask(t *testing.T) { 37 | t.Skip() 38 | 39 | testUnknownTask(t, azsqsFactory(), &taskq.QueueOptions{ 40 | Name: queueName("sqs-unknown-task"), 41 | }) 42 | } 43 | 44 | func TestSQSFallback(t *testing.T) { 45 | t.Skip() 46 | 47 | testFallback(t, azsqsFactory(), &taskq.QueueOptions{ 48 | Name: queueName("sqs-fallback"), 49 | }) 50 | } 51 | 52 | func TestSQSDelay(t *testing.T) { 53 | t.Skip() 54 | 55 | testDelay(t, azsqsFactory(), &taskq.QueueOptions{ 56 | Name: queueName("sqs-delay"), 57 | }) 58 | } 59 | 60 | func TestSQSRetry(t *testing.T) { 61 | t.Skip() 62 | 63 | testRetry(t, azsqsFactory(), &taskq.QueueOptions{ 64 | Name: queueName("sqs-retry"), 65 | }) 66 | } 67 | 68 | func TestSQSNamedMessage(t *testing.T) { 69 | t.Skip() 70 | 71 | testNamedMessage(t, azsqsFactory(), &taskq.QueueOptions{ 72 | Name: queueName("sqs-named-message"), 73 | }) 74 | } 75 | 76 | func TestSQSCallOnce(t *testing.T) { 77 | t.Skip() 78 | 79 | testCallOnce(t, azsqsFactory(), &taskq.QueueOptions{ 80 | Name: queueName("sqs-call-once"), 81 | }) 82 | } 83 | 84 | func TestSQSLen(t *testing.T) { 85 | t.Skip() 86 | 87 | testLen(t, azsqsFactory(), &taskq.QueueOptions{ 88 | Name: queueName("sqs-queue-len"), 89 | }) 90 | } 91 | 92 | func TestSQSRateLimit(t *testing.T) { 93 | t.Skip() 94 | 95 | testRateLimit(t, azsqsFactory(), &taskq.QueueOptions{ 96 | Name: queueName("sqs-rate-limit"), 97 | }) 98 | } 99 | 100 | func TestSQSErrorDelay(t *testing.T) { 101 | t.Skip() 102 | 103 | testErrorDelay(t, azsqsFactory(), &taskq.QueueOptions{ 104 | Name: queueName("sqs-delayer"), 105 | }) 106 | } 107 | 108 | func TestSQSWorkerLimit(t *testing.T) { 109 | t.Skip() 110 | 111 | testWorkerLimit(t, azsqsFactory(), &taskq.QueueOptions{ 112 | Name: queueName("sqs-worker-limit"), 113 | }) 114 | } 115 | 116 | func TestSQSInvalidCredentials(t *testing.T) { 117 | t.Skip() 118 | 119 | man := azsqs.NewFactory(awsSQS(), "123") 120 | testInvalidCredentials(t, man, &taskq.QueueOptions{ 121 | Name: queueName("sqs-invalid-credentials"), 122 | }) 123 | } 124 | 125 | func TestSQSBatchConsumerSmallMessage(t *testing.T) { 126 | t.Skip() 127 | 128 | testBatchConsumer(t, azsqsFactory(), &taskq.QueueOptions{ 129 | Name: queueName("sqs-batch-consumer-small-message"), 130 | }, 100) 131 | } 132 | 133 | func TestSQSBatchConsumerLarge(t *testing.T) { 134 | t.Skip() 135 | 136 | testBatchConsumer(t, azsqsFactory(), &taskq.QueueOptions{ 137 | Name: queueName("sqs-batch-processor-large-message"), 138 | }, 64000) 139 | } 140 | -------------------------------------------------------------------------------- /bench_test.go: -------------------------------------------------------------------------------- 1 | package taskq_test 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "testing" 7 | 8 | "github.com/vmihailenco/taskq/v3" 9 | "github.com/vmihailenco/taskq/v3/memqueue" 10 | "github.com/vmihailenco/taskq/v3/redisq" 11 | ) 12 | 13 | func BenchmarkConsumerMemq(b *testing.B) { 14 | benchmarkConsumer(b, memqueue.NewFactory()) 15 | } 16 | 17 | func BenchmarkConsumerRedisq(b *testing.B) { 18 | benchmarkConsumer(b, redisq.NewFactory()) 19 | } 20 | 21 | var ( 22 | once sync.Once 23 | q taskq.Queue 24 | task *taskq.Task 25 | wg sync.WaitGroup 26 | ) 27 | 28 | func benchmarkConsumer(b *testing.B, factory taskq.Factory) { 29 | c := context.Background() 30 | 31 | once.Do(func() { 32 | q = factory.RegisterQueue(&taskq.QueueOptions{ 33 | Name: "bench", 34 | Redis: redisRing(), 35 | }) 36 | 37 | task = taskq.RegisterTask(&taskq.TaskOptions{ 38 | Name: "bench", 39 | Handler: func() { 40 | wg.Done() 41 | }, 42 | }) 43 | 44 | _ = q.Consumer().Start(c) 45 | }) 46 | 47 | b.ResetTimer() 48 | 49 | for i := 0; i < b.N; i++ { 50 | for j := 0; j < 100; j++ { 51 | wg.Add(1) 52 | _ = q.Add(task.WithArgs(c)) 53 | } 54 | wg.Wait() 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /commitlint.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { extends: ["@commitlint/config-conventional"] }; 2 | -------------------------------------------------------------------------------- /consumer_config.go: -------------------------------------------------------------------------------- 1 | package taskq 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "time" 7 | 8 | "github.com/vmihailenco/taskq/v3/internal" 9 | ) 10 | 11 | const numSelectedThreshold = 20 12 | 13 | type perfProfile struct { 14 | start time.Time 15 | processed int 16 | retries int 17 | timing time.Duration 18 | 19 | tps float64 20 | errorRate float64 21 | } 22 | 23 | func (p *perfProfile) Reset(processed, retries int) { 24 | p.start = time.Now() 25 | p.processed = processed 26 | p.retries = retries 27 | } 28 | 29 | func (p *perfProfile) Update(processed, retries int, timing time.Duration) { 30 | processedDiff := processed - p.processed 31 | retriesDiff := retries - p.retries 32 | total := processedDiff + retriesDiff 33 | elapsed := time.Since(p.start) 34 | 35 | elapsedMS := float64(elapsed) / float64(time.Millisecond) 36 | p.tps = float64(processedDiff) / elapsedMS 37 | 38 | if total > 0 { 39 | p.errorRate = float64(retriesDiff) / float64(total) 40 | } else { 41 | p.errorRate = 0 42 | } 43 | 44 | p.timing = timing 45 | } 46 | 47 | func (p *perfProfile) TPS() float64 { 48 | return p.tps 49 | } 50 | 51 | func (p *perfProfile) ErrorRate() float64 { 52 | return p.errorRate 53 | } 54 | 55 | func (p *perfProfile) Timing() time.Duration { 56 | return p.timing 57 | } 58 | 59 | //------------------------------------------------------------------------------ 60 | 61 | type consumerConfig struct { 62 | NumFetcher int32 63 | NumWorker int32 64 | 65 | perfProfile 66 | 67 | NumSelected int 68 | Score float64 69 | } 70 | 71 | func newConsumerConfig(numFetcher, numWorker int32) *consumerConfig { 72 | return &consumerConfig{ 73 | NumFetcher: numFetcher, 74 | NumWorker: numWorker, 75 | } 76 | } 77 | 78 | func (cfg *consumerConfig) SetScore(score float64) { 79 | if cfg.Score == 0 { 80 | cfg.Score = score 81 | } else { 82 | cfg.Score = (cfg.Score + score) / 2 83 | } 84 | } 85 | 86 | func (cfg *consumerConfig) String() string { 87 | return fmt.Sprintf("fetchers=%d workers=%d tps=%f failure=%f timing=%s score=%f selected=%d", 88 | cfg.NumFetcher, cfg.NumWorker, cfg.tps, cfg.ErrorRate(), cfg.timing, cfg.Score, cfg.NumSelected) 89 | } 90 | 91 | func (cfg *consumerConfig) Equal(other *consumerConfig) bool { 92 | if other == nil { 93 | return false 94 | } 95 | return cfg.NumWorker == other.NumWorker && cfg.NumFetcher == other.NumFetcher 96 | } 97 | 98 | func (cfg *consumerConfig) Clone() *consumerConfig { 99 | return &consumerConfig{ 100 | NumWorker: cfg.NumWorker, 101 | NumFetcher: cfg.NumFetcher, 102 | } 103 | } 104 | 105 | //------------------------------------------------------------------------------ 106 | 107 | type configRoulette struct { 108 | opt *QueueOptions 109 | 110 | rnd *rand.Rand 111 | cfgs []*consumerConfig 112 | nextCfg uint 113 | 114 | maxTPS float64 115 | maxTiming time.Duration 116 | bestCfg *consumerConfig 117 | oldBestCfg *consumerConfig 118 | } 119 | 120 | func newConfigRoulette(opt *QueueOptions) *configRoulette { 121 | r := &configRoulette{ 122 | opt: opt, 123 | 124 | rnd: rand.New(rand.NewSource(time.Now().UnixNano())), 125 | } 126 | 127 | cfg := newConsumerConfig(1, int32(opt.MinNumWorker)) 128 | r.resetConfigs(cfg, false) 129 | 130 | return r 131 | } 132 | 133 | func (r *configRoulette) Select(currCfg *consumerConfig, queueEmpty bool) *consumerConfig { 134 | if currCfg != nil { 135 | r.updateScores(currCfg) 136 | r.checkScores(queueEmpty) 137 | } 138 | 139 | var newCfg *consumerConfig 140 | if r.bestCfg == nil || r.rnd.Float64() <= 0.15 { 141 | idx := r.nextCfg % uint(len(r.cfgs)) 142 | newCfg = r.cfgs[idx] 143 | r.nextCfg++ 144 | } else { 145 | newCfg = r.bestCfg 146 | } 147 | 148 | newCfg.NumSelected++ 149 | return newCfg 150 | } 151 | 152 | func (r *configRoulette) updateScores(cfg *consumerConfig) { 153 | var dirty bool 154 | if tps := cfg.TPS(); tps > r.maxTPS { 155 | r.maxTPS = tps 156 | dirty = true 157 | } 158 | if timing := cfg.Timing(); timing > r.maxTiming { 159 | r.maxTiming = timing 160 | dirty = true 161 | } 162 | 163 | cfg.SetScore(r.configScore(cfg)) 164 | 165 | r.bestCfg = nil 166 | for _, c := range r.cfgs { 167 | if c.NumSelected == 0 { 168 | r.bestCfg = nil 169 | break 170 | } 171 | 172 | if dirty && c != cfg { 173 | c.SetScore(r.configScore(c)) 174 | } 175 | 176 | if r.bestCfg == nil || c.Score-r.bestCfg.Score >= 0.01 { 177 | r.bestCfg = c 178 | } 179 | } 180 | } 181 | 182 | func (r *configRoulette) configScore(cfg *consumerConfig) float64 { 183 | tps := cfg.TPS() 184 | if tps == 0 { 185 | return 0 186 | } 187 | 188 | var score float64 189 | 190 | errorRate := 10 * cfg.ErrorRate() 191 | if errorRate > 1 { 192 | errorRate = 1 193 | } 194 | 195 | score += 0.35 * (1 - errorRate) 196 | 197 | if r.maxTPS != 0 { 198 | score += 0.35 * (tps / r.maxTPS) 199 | } 200 | 201 | if r.maxTiming != 0 { 202 | score += 0.3 * (1 - float64(cfg.Timing())/float64(r.maxTiming)) 203 | } 204 | 205 | return score 206 | } 207 | 208 | func (r *configRoulette) checkScores(queueEmpty bool) { 209 | if r.bestCfg == nil || r.bestCfg.Equal(r.oldBestCfg) { 210 | return 211 | } 212 | if r.bestCfg.NumSelected < numSelectedThreshold { 213 | return 214 | } 215 | if queueEmpty && r.bestCfg.NumWorker > r.oldBestCfg.NumWorker { 216 | return 217 | } 218 | 219 | if false { 220 | for _, cfg := range r.cfgs { 221 | internal.Logger.Println("taskq: " + cfg.String()) 222 | } 223 | } 224 | r.resetConfigs(r.bestCfg, queueEmpty) 225 | } 226 | 227 | func (r *configRoulette) resetConfigs(bestCfg *consumerConfig, queueEmpty bool) { 228 | r.genConfigs(bestCfg, queueEmpty) 229 | r.maxTPS = 0 230 | r.maxTiming = 0 231 | r.bestCfg = nil 232 | } 233 | 234 | func (r *configRoulette) genConfigs(bestCfg *consumerConfig, queueEmpty bool) { 235 | r.cfgs = make([]*consumerConfig, 0, 5) 236 | 237 | if bestCfg.NumFetcher > 1 { 238 | r.addConfig(r.withLessFetchers(bestCfg)) 239 | } 240 | 241 | if bestCfg.NumWorker > r.opt.MinNumWorker { 242 | r.addConfig(r.withLessWorkers(bestCfg, bestCfg.NumWorker/4)) 243 | } 244 | 245 | r.oldBestCfg = bestCfg.Clone() 246 | r.addConfig(r.oldBestCfg) 247 | 248 | if !hasFreeSystemResources() { 249 | internal.Logger.Println("taskq: system does not have enough free resources") 250 | return 251 | } 252 | 253 | if queueEmpty { 254 | r.addConfig(r.withMoreWorkers(bestCfg, 2)) 255 | return 256 | } 257 | 258 | if bestCfg.NumWorker < r.opt.MaxNumWorker { 259 | if r.oldBestCfg == nil || significant(bestCfg.Score-r.oldBestCfg.Score) { 260 | r.addConfig(r.withMoreWorkers(bestCfg, r.opt.MaxNumWorker/4)) 261 | } else { 262 | r.addConfig(r.withMoreWorkers(bestCfg, 2)) 263 | } 264 | } 265 | 266 | if bestCfg.NumFetcher < r.opt.MaxNumFetcher { 267 | r.addConfig(r.withMoreFetchers(bestCfg)) 268 | } 269 | } 270 | 271 | func (r *configRoulette) addConfig(cfg *consumerConfig) { 272 | r.cfgs = append(r.cfgs, cfg) 273 | } 274 | 275 | func (r *configRoulette) withLessWorkers(cfg *consumerConfig, n int32) *consumerConfig { 276 | if n <= 0 { 277 | n = 1 278 | } else if n > 10 { 279 | n = 10 280 | } 281 | cfg = cfg.Clone() 282 | cfg.NumWorker -= n 283 | if cfg.NumWorker < r.opt.MinNumWorker { 284 | cfg.NumWorker = r.opt.MinNumWorker 285 | } 286 | return cfg 287 | } 288 | 289 | func (r *configRoulette) withMoreWorkers(cfg *consumerConfig, n int32) *consumerConfig { 290 | if n <= 0 { 291 | n = 1 292 | } else if n > 10 { 293 | n = 10 294 | } 295 | cfg = cfg.Clone() 296 | cfg.NumWorker += n 297 | if cfg.NumWorker > r.opt.MaxNumWorker { 298 | cfg.NumWorker = r.opt.MaxNumWorker 299 | } 300 | return cfg 301 | } 302 | 303 | func (r *configRoulette) withLessFetchers(cfg *consumerConfig) *consumerConfig { 304 | cfg = cfg.Clone() 305 | cfg.NumFetcher-- 306 | return cfg 307 | } 308 | 309 | func (r *configRoulette) withMoreFetchers(cfg *consumerConfig) *consumerConfig { 310 | cfg = cfg.Clone() 311 | cfg.NumFetcher++ 312 | return cfg 313 | } 314 | 315 | func significant(n float64) bool { 316 | return n > 0.05 317 | } 318 | -------------------------------------------------------------------------------- /consumer_test.go: -------------------------------------------------------------------------------- 1 | package taskq_test 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "math/rand" 9 | "runtime" 10 | "strconv" 11 | "strings" 12 | "sync" 13 | "sync/atomic" 14 | "testing" 15 | "time" 16 | 17 | "github.com/go-redis/redis/v8" 18 | "github.com/go-redis/redis_rate/v9" 19 | 20 | "github.com/vmihailenco/taskq/v3" 21 | "github.com/vmihailenco/taskq/v3/redisq" 22 | ) 23 | 24 | const waitTimeout = time.Second 25 | const testTimeout = 30 * time.Second 26 | 27 | func queueName(s string) string { 28 | version := strings.Split(runtime.Version(), " ")[0] 29 | version = strings.Replace(version, ".", "", -1) 30 | return "test-" + s + "-" + version 31 | } 32 | 33 | var ( 34 | ringOnce sync.Once 35 | ring *redis.Ring 36 | ) 37 | 38 | func redisRing() *redis.Ring { 39 | ringOnce.Do(func() { 40 | ring = redis.NewRing(&redis.RingOptions{ 41 | Addrs: map[string]string{"0": ":6379"}, 42 | }) 43 | }) 44 | _ = ring.FlushDB(context.TODO()).Err() 45 | return ring 46 | } 47 | 48 | func testConsumer(t *testing.T, factory taskq.Factory, opt *taskq.QueueOptions) { 49 | c := context.Background() 50 | opt.WaitTimeout = waitTimeout 51 | opt.Redis = redisRing() 52 | 53 | q := factory.RegisterQueue(opt) 54 | defer q.Close() 55 | purge(t, q) 56 | 57 | ch := make(chan time.Time) 58 | task := taskq.RegisterTask(&taskq.TaskOptions{ 59 | Name: nextTaskID(), 60 | Handler: func(hello, world string) error { 61 | if hello != "hello" { 62 | t.Fatalf("got %s, wanted hello", hello) 63 | } 64 | if world != "world" { 65 | t.Fatalf("got %s, wanted world", world) 66 | } 67 | ch <- time.Now() 68 | return nil 69 | }, 70 | }) 71 | 72 | err := q.Add(task.WithArgs(c, "hello", "world")) 73 | if err != nil { 74 | t.Fatal(err) 75 | } 76 | 77 | p := q.Consumer() 78 | if err := p.Start(c); err != nil { 79 | t.Fatal(err) 80 | } 81 | 82 | select { 83 | case <-ch: 84 | case <-time.After(testTimeout): 85 | t.Fatalf("message was not processed") 86 | } 87 | 88 | if err := p.Stop(); err != nil { 89 | t.Fatal(err) 90 | } 91 | 92 | if err := q.Close(); err != nil { 93 | t.Fatal(err) 94 | } 95 | } 96 | 97 | func testConsumerDelete(t *testing.T, factory taskq.Factory, opt *taskq.QueueOptions) { 98 | c := context.Background() 99 | opt.WaitTimeout = waitTimeout 100 | opt.Redis = redisRing() 101 | 102 | red, ok := opt.Redis.(redisq.RedisStreamClient) 103 | if !ok { 104 | log.Fatal(fmt.Errorf("redisq: Redis client must support streams")) 105 | } 106 | 107 | q := factory.RegisterQueue(opt) 108 | defer q.Close() 109 | 110 | purge(t, q) 111 | 112 | ch := make(chan time.Time) 113 | task := taskq.RegisterTask(&taskq.TaskOptions{ 114 | Name: nextTaskID(), 115 | Handler: func() error { 116 | ch <- time.Now() 117 | return nil 118 | }, 119 | }) 120 | 121 | err := q.Add(task.WithArgs(c)) 122 | if err != nil { 123 | t.Fatal(err) 124 | } 125 | 126 | p := q.Consumer() 127 | if err := p.Start(c); err != nil { 128 | t.Fatal(err) 129 | } 130 | 131 | select { 132 | case <-ch: 133 | case <-time.After(testTimeout): 134 | t.Fatalf("message was not processed") 135 | } 136 | 137 | tm := time.Now().Add(opt.ReservationTimeout) 138 | end := strconv.FormatInt(unixMs(tm), 10) 139 | pending, err := red.XPendingExt(context.Background(), &redis.XPendingExtArgs{ 140 | Stream: "taskq:{" + opt.Name + "}:stream", 141 | Group: "taskq", 142 | Start: "-", 143 | End: end, 144 | Count: 100, 145 | }).Result() 146 | 147 | if err != nil { 148 | t.Fatal(err) 149 | } 150 | 151 | if len(pending) > 0 { 152 | t.Fatal("task not acknowledged and still exists in pending list.") 153 | } 154 | 155 | if err := p.Stop(); err != nil { 156 | t.Fatal(err) 157 | } 158 | 159 | if err := q.Close(); err != nil { 160 | t.Fatal(err) 161 | } 162 | } 163 | 164 | func testUnknownTask(t *testing.T, factory taskq.Factory, opt *taskq.QueueOptions) { 165 | c := context.Background() 166 | opt.WaitTimeout = waitTimeout 167 | opt.Redis = redisRing() 168 | 169 | q := factory.RegisterQueue(opt) 170 | defer q.Close() 171 | purge(t, q) 172 | 173 | _ = taskq.RegisterTask(&taskq.TaskOptions{ 174 | Name: nextTaskID(), 175 | Handler: func() {}, 176 | }) 177 | 178 | taskq.SetUnknownTaskOptions(&taskq.TaskOptions{ 179 | Name: "_", 180 | RetryLimit: 1, 181 | }) 182 | 183 | msg := taskq.NewMessage(c) 184 | msg.TaskName = "unknown" 185 | err := q.Add(msg) 186 | if err != nil { 187 | t.Fatal(err) 188 | } 189 | 190 | p := q.Consumer() 191 | if err := p.Start(c); err != nil { 192 | t.Fatal(err) 193 | } 194 | 195 | if err := p.Stop(); err != nil { 196 | t.Fatal(err) 197 | } 198 | 199 | if err := q.Close(); err != nil { 200 | t.Fatal(err) 201 | } 202 | } 203 | 204 | func testFallback(t *testing.T, factory taskq.Factory, opt *taskq.QueueOptions) { 205 | c := context.Background() 206 | opt.WaitTimeout = waitTimeout 207 | opt.Redis = redisRing() 208 | 209 | q := factory.RegisterQueue(opt) 210 | defer q.Close() 211 | purge(t, q) 212 | 213 | ch := make(chan time.Time) 214 | task := taskq.RegisterTask(&taskq.TaskOptions{ 215 | Name: nextTaskID(), 216 | Handler: func() error { 217 | return errors.New("fake error") 218 | }, 219 | FallbackHandler: func(hello, world string) error { 220 | if hello != "hello" { 221 | t.Fatalf("got %s, wanted hello", hello) 222 | } 223 | if world != "world" { 224 | t.Fatalf("got %s, wanted world", world) 225 | } 226 | ch <- time.Now() 227 | return nil 228 | }, 229 | RetryLimit: 1, 230 | }) 231 | 232 | err := q.Add(task.WithArgs(c, "hello", "world")) 233 | if err != nil { 234 | t.Fatal(err) 235 | } 236 | 237 | p := q.Consumer() 238 | p.Start(c) 239 | 240 | select { 241 | case <-ch: 242 | case <-time.After(testTimeout): 243 | t.Fatalf("message was not processed") 244 | } 245 | 246 | if err := p.Stop(); err != nil { 247 | t.Fatal(err) 248 | } 249 | 250 | if err := q.Close(); err != nil { 251 | t.Fatal(err) 252 | } 253 | } 254 | 255 | func testDelay(t *testing.T, factory taskq.Factory, opt *taskq.QueueOptions) { 256 | c := context.Background() 257 | opt.WaitTimeout = waitTimeout 258 | opt.Redis = redisRing() 259 | 260 | q := factory.RegisterQueue(opt) 261 | defer q.Close() 262 | purge(t, q) 263 | 264 | handlerCh := make(chan time.Time, 10) 265 | task := taskq.RegisterTask(&taskq.TaskOptions{ 266 | Name: nextTaskID(), 267 | Handler: func() { 268 | handlerCh <- time.Now() 269 | }, 270 | }) 271 | 272 | start := time.Now() 273 | 274 | msg := task.WithArgs(c) 275 | msg.Delay = 5 * time.Second 276 | err := q.Add(msg) 277 | if err != nil { 278 | t.Fatal(err) 279 | } 280 | 281 | p := q.Consumer() 282 | p.Start(c) 283 | 284 | var tm time.Time 285 | select { 286 | case tm = <-handlerCh: 287 | case <-time.After(testTimeout): 288 | t.Fatalf("message was not processed") 289 | } 290 | 291 | sub := tm.Sub(start) 292 | if !durEqual(sub, msg.Delay) { 293 | t.Fatalf("message was delayed by %s, wanted %s", sub, msg.Delay) 294 | } 295 | 296 | if err := p.Stop(); err != nil { 297 | t.Fatal(err) 298 | } 299 | 300 | if err := q.Close(); err != nil { 301 | t.Fatal(err) 302 | } 303 | } 304 | 305 | func testRetry(t *testing.T, factory taskq.Factory, opt *taskq.QueueOptions) { 306 | c := context.Background() 307 | opt.WaitTimeout = waitTimeout 308 | opt.Redis = redisRing() 309 | 310 | q := factory.RegisterQueue(opt) 311 | defer q.Close() 312 | purge(t, q) 313 | 314 | handlerCh := make(chan time.Time, 10) 315 | task := taskq.RegisterTask(&taskq.TaskOptions{ 316 | Name: nextTaskID(), 317 | Handler: func(hello, world string) error { 318 | if hello != "hello" { 319 | t.Fatalf("got %q, wanted hello", hello) 320 | } 321 | if world != "world" { 322 | t.Fatalf("got %q, wanted world", world) 323 | } 324 | handlerCh <- time.Now() 325 | return errors.New("fake error") 326 | }, 327 | FallbackHandler: func(msg *taskq.Message) error { 328 | handlerCh <- time.Now() 329 | return nil 330 | }, 331 | RetryLimit: 3, 332 | MinBackoff: time.Second, 333 | }) 334 | 335 | err := q.Add(task.WithArgs(c, "hello", "world")) 336 | if err != nil { 337 | t.Fatal(err) 338 | } 339 | 340 | p := q.Consumer() 341 | p.Start(c) 342 | 343 | timings := []time.Duration{0, time.Second, 3 * time.Second, 3 * time.Second} 344 | testTimings(t, handlerCh, timings) 345 | 346 | if err := p.Stop(); err != nil { 347 | t.Fatal(err) 348 | } 349 | 350 | if err := q.Close(); err != nil { 351 | t.Fatal(err) 352 | } 353 | } 354 | 355 | func testNamedMessage(t *testing.T, factory taskq.Factory, opt *taskq.QueueOptions) { 356 | c := context.Background() 357 | opt.WaitTimeout = waitTimeout 358 | opt.Redis = redisRing() 359 | 360 | q := factory.RegisterQueue(opt) 361 | defer q.Close() 362 | purge(t, q) 363 | 364 | ch := make(chan time.Time, 10) 365 | task := taskq.RegisterTask(&taskq.TaskOptions{ 366 | Name: nextTaskID(), 367 | Handler: func(hello string) error { 368 | if hello != "world" { 369 | panic("hello != world") 370 | } 371 | ch <- time.Now() 372 | return nil 373 | }, 374 | }) 375 | 376 | var wg sync.WaitGroup 377 | for i := 0; i < 10; i++ { 378 | wg.Add(1) 379 | go func() { 380 | defer wg.Done() 381 | 382 | msg := task.WithArgs(c, "world") 383 | msg.Name = "the-name" 384 | err := q.Add(msg) 385 | if err != nil { 386 | t.Fatal(err) 387 | } 388 | }() 389 | } 390 | wg.Wait() 391 | 392 | p := q.Consumer() 393 | p.Start(c) 394 | 395 | select { 396 | case <-ch: 397 | case <-time.After(testTimeout): 398 | t.Fatalf("message was not processed") 399 | } 400 | 401 | select { 402 | case <-ch: 403 | t.Fatalf("message was processed twice") 404 | default: 405 | } 406 | 407 | if err := p.Stop(); err != nil { 408 | t.Fatal(err) 409 | } 410 | 411 | if err := q.Close(); err != nil { 412 | t.Fatal(err) 413 | } 414 | } 415 | 416 | func testCallOnce(t *testing.T, factory taskq.Factory, opt *taskq.QueueOptions) { 417 | c := context.Background() 418 | opt.WaitTimeout = waitTimeout 419 | opt.Redis = redisRing() 420 | 421 | q := factory.RegisterQueue(opt) 422 | defer q.Close() 423 | purge(t, q) 424 | 425 | ch := make(chan time.Time, 10) 426 | task := taskq.RegisterTask(&taskq.TaskOptions{ 427 | Name: nextTaskID(), 428 | Handler: func() { 429 | ch <- time.Now() 430 | }, 431 | }) 432 | 433 | go func() { 434 | for i := 0; i < 3; i++ { 435 | for j := 0; j < 10; j++ { 436 | msg := task.WithArgs(c) 437 | msg.OnceInPeriod(500 * time.Millisecond) 438 | 439 | err := q.Add(msg) 440 | if err != nil { 441 | t.Fatal(err) 442 | } 443 | } 444 | time.Sleep(time.Second) 445 | } 446 | }() 447 | 448 | p := q.Consumer() 449 | if err := p.Start(c); err != nil { 450 | t.Fatal(err) 451 | } 452 | 453 | for i := 0; i < 3; i++ { 454 | select { 455 | case <-ch: 456 | case <-time.After(testTimeout): 457 | t.Fatalf("message was not processed") 458 | } 459 | } 460 | 461 | select { 462 | case <-ch: 463 | t.Fatalf("message was processed twice") 464 | case <-time.After(time.Second): 465 | } 466 | 467 | if err := p.Stop(); err != nil { 468 | t.Fatal(err) 469 | } 470 | 471 | if err := q.Close(); err != nil { 472 | t.Fatal(err) 473 | } 474 | } 475 | 476 | func testLen(t *testing.T, factory taskq.Factory, opt *taskq.QueueOptions) { 477 | const N = 10 478 | 479 | c := context.Background() 480 | opt.WaitTimeout = waitTimeout 481 | opt.Redis = redisRing() 482 | 483 | q := factory.RegisterQueue(opt) 484 | defer q.Close() 485 | purge(t, q) 486 | 487 | task := taskq.RegisterTask(&taskq.TaskOptions{ 488 | Name: nextTaskID(), 489 | Handler: func() {}, 490 | }) 491 | 492 | for i := 0; i < N; i++ { 493 | err := q.Add(task.WithArgs(c)) 494 | if err != nil { 495 | t.Fatal(err) 496 | } 497 | } 498 | 499 | eventually(func() error { 500 | n, err := q.Len() 501 | if err != nil { 502 | return err 503 | } 504 | 505 | if n != N { 506 | return fmt.Errorf("got %d messages, wanted %d", n, N) 507 | } 508 | return nil 509 | }, testTimeout) 510 | 511 | if err := q.Close(); err != nil { 512 | t.Fatal(err) 513 | } 514 | } 515 | 516 | func testRateLimit(t *testing.T, factory taskq.Factory, opt *taskq.QueueOptions) { 517 | c := context.Background() 518 | opt.WaitTimeout = waitTimeout 519 | opt.RateLimit = redis_rate.PerSecond(1) 520 | opt.Redis = redisRing() 521 | 522 | q := factory.RegisterQueue(opt) 523 | defer q.Close() 524 | purge(t, q) 525 | 526 | var count int64 527 | task := taskq.RegisterTask(&taskq.TaskOptions{ 528 | Name: nextTaskID(), 529 | Handler: func() { 530 | atomic.AddInt64(&count, 1) 531 | }, 532 | }) 533 | 534 | var wg sync.WaitGroup 535 | for i := 0; i < 10; i++ { 536 | wg.Add(1) 537 | go func() { 538 | defer wg.Done() 539 | 540 | err := q.Add(task.WithArgs(c)) 541 | if err != nil { 542 | t.Fatal(err) 543 | } 544 | }() 545 | } 546 | wg.Wait() 547 | 548 | p := q.Consumer() 549 | p.Start(c) 550 | 551 | time.Sleep(5 * time.Second) 552 | 553 | if n := atomic.LoadInt64(&count); n-5 > 2 { 554 | t.Fatalf("processed %d messages, wanted 5", n) 555 | } 556 | 557 | if err := p.Stop(); err != nil { 558 | t.Fatal(err) 559 | } 560 | 561 | if err := q.Close(); err != nil { 562 | t.Fatal(err) 563 | } 564 | } 565 | 566 | func testErrorDelay(t *testing.T, factory taskq.Factory, opt *taskq.QueueOptions) { 567 | c := context.Background() 568 | opt.WaitTimeout = waitTimeout 569 | opt.Redis = redisRing() 570 | 571 | q := factory.RegisterQueue(opt) 572 | defer q.Close() 573 | purge(t, q) 574 | 575 | handlerCh := make(chan time.Time, 10) 576 | task := taskq.RegisterTask(&taskq.TaskOptions{ 577 | Name: nextTaskID(), 578 | Handler: func() error { 579 | handlerCh <- time.Now() 580 | return RateLimitError("fake error") 581 | }, 582 | MinBackoff: time.Second, 583 | RetryLimit: 3, 584 | }) 585 | 586 | err := q.Add(task.WithArgs(c)) 587 | if err != nil { 588 | t.Fatal(err) 589 | } 590 | 591 | p := q.Consumer() 592 | p.Start(c) 593 | 594 | timings := []time.Duration{0, 3 * time.Second, 3 * time.Second} 595 | testTimings(t, handlerCh, timings) 596 | 597 | if err := p.Stop(); err != nil { 598 | t.Fatal(err) 599 | } 600 | 601 | if err := q.Close(); err != nil { 602 | t.Fatal(err) 603 | } 604 | } 605 | 606 | func testWorkerLimit(t *testing.T, factory taskq.Factory, opt *taskq.QueueOptions) { 607 | ctx := context.Background() 608 | opt.WaitTimeout = waitTimeout 609 | opt.Redis = redisRing() 610 | opt.WorkerLimit = 1 611 | 612 | q := factory.RegisterQueue(opt) 613 | defer q.Close() 614 | purge(t, q) 615 | 616 | ch := make(chan time.Time, 10) 617 | task := taskq.RegisterTask(&taskq.TaskOptions{ 618 | Name: nextTaskID(), 619 | Handler: func() { 620 | ch <- time.Now() 621 | time.Sleep(time.Second) 622 | }, 623 | }) 624 | 625 | for i := 0; i < 3; i++ { 626 | err := q.Add(task.WithArgs(ctx)) 627 | if err != nil { 628 | t.Fatal(err) 629 | } 630 | } 631 | 632 | p1 := taskq.StartConsumer(ctx, q) 633 | p2 := taskq.StartConsumer(ctx, q) 634 | 635 | timings := []time.Duration{0, time.Second, 2 * time.Second} 636 | testTimings(t, ch, timings) 637 | 638 | if err := p1.Stop(); err != nil { 639 | t.Fatal(err) 640 | } 641 | if err := p2.Stop(); err != nil { 642 | t.Fatal(err) 643 | } 644 | } 645 | 646 | func testInvalidCredentials(t *testing.T, factory taskq.Factory, opt *taskq.QueueOptions) { 647 | ctx := context.Background() 648 | opt.WaitTimeout = waitTimeout 649 | opt.Redis = redisRing() 650 | 651 | q := factory.RegisterQueue(opt) 652 | defer q.Close() 653 | 654 | ch := make(chan time.Time, 10) 655 | task := taskq.RegisterTask(&taskq.TaskOptions{ 656 | Name: nextTaskID(), 657 | Handler: func(s1, s2 string) { 658 | if s1 != "hello" { 659 | t.Fatalf("got %q, wanted hello", s1) 660 | } 661 | if s2 != "world" { 662 | t.Fatalf("got %q, wanted world", s1) 663 | } 664 | ch <- time.Now() 665 | }, 666 | }) 667 | 668 | err := q.Add(task.WithArgs(ctx, "hello", "world")) 669 | if err != nil { 670 | t.Fatal(err) 671 | } 672 | 673 | timings := []time.Duration{3 * time.Second} 674 | testTimings(t, ch, timings) 675 | 676 | err = q.Close() 677 | if err != nil { 678 | t.Fatal(err) 679 | } 680 | } 681 | 682 | func testBatchConsumer( 683 | t *testing.T, factory taskq.Factory, opt *taskq.QueueOptions, messageSize int, 684 | ) { 685 | const N = 16 686 | 687 | ctx := context.Background() 688 | opt.WaitTimeout = waitTimeout 689 | opt.Redis = redisRing() 690 | 691 | payload := make([]byte, messageSize) 692 | _, err := rand.Read(payload) 693 | if err != nil { 694 | t.Fatal(err) 695 | } 696 | 697 | var wg sync.WaitGroup 698 | wg.Add(N) 699 | 700 | opt.WaitTimeout = waitTimeout 701 | q := factory.RegisterQueue(opt) 702 | defer q.Close() 703 | purge(t, q) 704 | 705 | task := taskq.RegisterTask(&taskq.TaskOptions{ 706 | Name: nextTaskID(), 707 | Handler: func(s string) { 708 | defer wg.Done() 709 | if s != string(payload) { 710 | t.Fatalf("s != payload") 711 | } 712 | }, 713 | }) 714 | 715 | for i := 0; i < N; i++ { 716 | err := q.Add(task.WithArgs(ctx, payload)) 717 | if err != nil { 718 | t.Fatal(err) 719 | } 720 | } 721 | 722 | p := q.Consumer() 723 | if err := p.Start(ctx); err != nil { 724 | t.Fatal(err) 725 | } 726 | 727 | done := make(chan struct{}) 728 | go func() { 729 | wg.Wait() 730 | close(done) 731 | }() 732 | 733 | select { 734 | case <-done: 735 | case <-time.After(testTimeout): 736 | t.Fatalf("messages were not processed") 737 | } 738 | 739 | if err := p.Stop(); err != nil { 740 | t.Fatal(err) 741 | } 742 | 743 | if err := q.Close(); err != nil { 744 | t.Fatal(err) 745 | } 746 | } 747 | 748 | func durEqual(d1, d2 time.Duration) bool { 749 | return d1 >= d2 && d2-d1 < 3*time.Second 750 | } 751 | 752 | func testTimings(t *testing.T, ch chan time.Time, timings []time.Duration) { 753 | start := time.Now() 754 | for i, timing := range timings { 755 | var tm time.Time 756 | select { 757 | case tm = <-ch: 758 | case <-time.After(testTimeout): 759 | t.Fatalf("message is not processed after %s", 2*timing) 760 | } 761 | since := tm.Sub(start) 762 | if !durEqual(since, timing) { 763 | t.Fatalf("#%d: timing is %s, wanted %s", i+1, since, timing) 764 | } 765 | } 766 | } 767 | 768 | func purge(t *testing.T, q taskq.Queue) { 769 | err := q.Purge() 770 | if err == nil { 771 | return 772 | } 773 | 774 | task := taskq.RegisterTask(&taskq.TaskOptions{ 775 | Name: "*", 776 | Handler: func() {}, 777 | }) 778 | 779 | consumer := taskq.NewConsumer(q) 780 | err = consumer.ProcessAll(context.Background()) 781 | if err != nil { 782 | t.Fatal(err) 783 | } 784 | 785 | taskq.Tasks.Unregister(task) 786 | } 787 | 788 | func eventually(fn func() error, timeout time.Duration) error { 789 | errCh := make(chan error) 790 | done := make(chan struct{}) 791 | exit := make(chan struct{}) 792 | 793 | go func() { 794 | for { 795 | err := fn() 796 | if err == nil { 797 | close(done) 798 | return 799 | } 800 | 801 | select { 802 | case errCh <- err: 803 | default: 804 | } 805 | 806 | select { 807 | case <-exit: 808 | return 809 | case <-time.After(timeout / 100): 810 | } 811 | } 812 | }() 813 | 814 | select { 815 | case <-done: 816 | return nil 817 | case <-time.After(timeout): 818 | close(exit) 819 | select { 820 | case err := <-errCh: 821 | return err 822 | default: 823 | return fmt.Errorf("timeout after %s", timeout) 824 | } 825 | } 826 | } 827 | 828 | var taskID int 829 | 830 | func nextTaskID() string { 831 | id := strconv.Itoa(taskID) 832 | taskID++ 833 | return id 834 | } 835 | 836 | func unixMs(tm time.Time) int64 { 837 | return tm.UnixNano() / int64(time.Millisecond) 838 | } 839 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package taskq implements task/job queue with Redis, SQS, IronMQ, and in-memory backends. 3 | */ 4 | package taskq 5 | -------------------------------------------------------------------------------- /example/redisexample/README.md: -------------------------------------------------------------------------------- 1 | # taskq example using Redis backend 2 | 3 | The example requires Redis Server running on the `:6379`. 4 | 5 | First, start the consumer: 6 | 7 | ```shell 8 | go run consumer/main.go 9 | 10 | ``` 11 | 12 | Then, start the producer: 13 | 14 | ```shell 15 | go run producer/main.go 16 | ``` 17 | -------------------------------------------------------------------------------- /example/redisexample/consumer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "log" 7 | 8 | "github.com/vmihailenco/taskq/example/redisexample" 9 | ) 10 | 11 | func main() { 12 | flag.Parse() 13 | 14 | c := context.Background() 15 | 16 | err := redisexample.QueueFactory.StartConsumers(c) 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | 21 | go redisexample.LogStats() 22 | 23 | sig := redisexample.WaitSignal() 24 | log.Println(sig.String()) 25 | 26 | err = redisexample.QueueFactory.Close() 27 | if err != nil { 28 | log.Fatal(err) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /example/redisexample/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | This package contains an example usage of redis-backed taskq. It includes two programs: 3 | 4 | 1. api that sends messages on the queue 5 | 2. worker that receives messages 6 | 7 | They share common definitions for the queue `MainQueue` with a single task defined on it `CountTask`. api runs in a loop 8 | submitting messages to the queue, for each message it submits it increments a process-local counter. worker starts a 9 | consumer that executes the `CountTask` for each message receives - the task handler also implements a process local 10 | counter. If you run each program in a separate terminal with: 11 | 12 | go run ./examples/redisexample/api/main.go 13 | go run ./examples/redisexample/worker/main.go 14 | 15 | Then api will periodically print the messages sent to the terminal and worker will print the message received. 16 | 17 | If you spawn further workers they will share consumption of the messages produced by api. 18 | */ 19 | package redisexample 20 | -------------------------------------------------------------------------------- /example/redisexample/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vmihailenco/taskq/example/redisexample 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/go-redis/redis/v8 v8.11.5 7 | github.com/vmihailenco/taskq/v3 v3.2.9 8 | ) 9 | 10 | require ( 11 | github.com/bsm/redislock v0.7.2 // indirect 12 | github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 // indirect 13 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 14 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect 15 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 16 | github.com/go-redis/redis_rate/v9 v9.1.2 // indirect 17 | github.com/google/uuid v1.3.0 // indirect 18 | github.com/hashicorp/golang-lru v0.5.4 // indirect 19 | github.com/klauspost/compress v1.15.1 // indirect 20 | github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect 21 | github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect 22 | ) 23 | 24 | replace github.com/vmihailenco/taskq/v3 => ../.. 25 | -------------------------------------------------------------------------------- /example/redisexample/go.sum: -------------------------------------------------------------------------------- 1 | github.com/aws/aws-sdk-go v1.43.45 h1:2708Bj4uV+ym62MOtBnErm/CDX61C4mFe9V2gXy1caE= 2 | github.com/aws/aws-sdk-go v1.43.45/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= 3 | github.com/bsm/redislock v0.7.2 h1:jggqOio8JyX9FJBKIfjF3fTxAu/v7zC5mAID9LveqG4= 4 | github.com/bsm/redislock v0.7.2/go.mod h1:kS2g0Yvlymc9Dz8V3iVYAtLAaSVruYbAFdYBDrmC5WU= 5 | github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 h1:IHZ1Le1ejzkmS7Si7dIzJvYDWe+BIoNmqMnfWHBZSVw= 6 | github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3/go.mod h1:M5XHQLu90v2JNm/bW2tdsYar+5vhV0gEcBcmDBNAN1Y= 7 | github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= 8 | github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 9 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 10 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 11 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 12 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 14 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 15 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= 16 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= 17 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 18 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 19 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 20 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 21 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 22 | github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= 23 | github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= 24 | github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= 25 | github.com/go-redis/redis_rate/v9 v9.1.2 h1:H0l5VzoAtOE6ydd38j8MCq3ABlGLnvvbA1xDSVVCHgQ= 26 | github.com/go-redis/redis_rate/v9 v9.1.2/go.mod h1:oam2de2apSgRG8aJzwJddXbNu91Iyz1m8IKJE2vpvlQ= 27 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= 28 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 29 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 30 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 31 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 32 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 33 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 34 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 35 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 36 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 37 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 38 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 39 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 40 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 41 | github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 42 | github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= 43 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 44 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 45 | github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= 46 | github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 47 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 48 | github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 49 | github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac h1:w5wltlINIIqRTqQ64dASrCo0fM7k9nosPbKCZnkL0W0= 50 | github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac/go.mod h1:gyMTRVO+ZkEy7wQDyD++okPsBN2q127EpuShhHMWG54= 51 | github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74/go.mod h1:qNa9FlAfO0U/qNkzYBMH1JKYRMzC+sP9IcyV4U18l98= 52 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 53 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 54 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 55 | github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= 56 | github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 57 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 58 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 59 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 60 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 61 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 62 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= 63 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 64 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 65 | github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= 66 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= 67 | github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= 68 | github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= 69 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 70 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 71 | github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= 72 | github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= 73 | github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= 74 | github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= 75 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 76 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 77 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 78 | github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= 79 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 80 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 81 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 82 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 83 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 84 | github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= 85 | github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= 86 | github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= 87 | github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= 88 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 89 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 90 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 91 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 92 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 93 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 94 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 95 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 96 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 97 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 98 | golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= 99 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= 100 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 101 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 102 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 103 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 104 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 105 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 106 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 107 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 108 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 109 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 110 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 111 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 112 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 113 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 114 | golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 115 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 116 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 117 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= 118 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 119 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 120 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 121 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 122 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 123 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 124 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 125 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 126 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 127 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 128 | golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 129 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 130 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 131 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 132 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 133 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 134 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 135 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 136 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 137 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 138 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 139 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 140 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 141 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 142 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 143 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 144 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 145 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 146 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 147 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 148 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 149 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 150 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 151 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 152 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 153 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 154 | -------------------------------------------------------------------------------- /example/redisexample/producer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "log" 7 | 8 | "github.com/vmihailenco/taskq/example/redisexample" 9 | ) 10 | 11 | func main() { 12 | flag.Parse() 13 | 14 | go redisexample.LogStats() 15 | 16 | go func() { 17 | for { 18 | err := redisexample.MainQueue.Add(redisexample.CountTask.WithArgs(context.Background())) 19 | if err != nil { 20 | log.Fatal(err) 21 | } 22 | redisexample.IncrLocalCounter() 23 | } 24 | }() 25 | 26 | sig := redisexample.WaitSignal() 27 | log.Println(sig.String()) 28 | } 29 | -------------------------------------------------------------------------------- /example/redisexample/tasks.go: -------------------------------------------------------------------------------- 1 | package redisexample 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "os/signal" 7 | "sync/atomic" 8 | "syscall" 9 | "time" 10 | 11 | "github.com/go-redis/redis/v8" 12 | 13 | "github.com/vmihailenco/taskq/v3" 14 | "github.com/vmihailenco/taskq/v3/redisq" 15 | ) 16 | 17 | var Redis = redis.NewClient(&redis.Options{ 18 | Addr: ":6379", 19 | }) 20 | 21 | var ( 22 | QueueFactory = redisq.NewFactory() 23 | MainQueue = QueueFactory.RegisterQueue(&taskq.QueueOptions{ 24 | Name: "api-worker", 25 | Redis: Redis, 26 | }) 27 | CountTask = taskq.RegisterTask(&taskq.TaskOptions{ 28 | Name: "counter", 29 | Handler: func() error { 30 | IncrLocalCounter() 31 | time.Sleep(time.Millisecond) 32 | return nil 33 | }, 34 | }) 35 | ) 36 | 37 | var counter int32 38 | 39 | func GetLocalCounter() int32 { 40 | return atomic.LoadInt32(&counter) 41 | } 42 | 43 | func IncrLocalCounter() { 44 | atomic.AddInt32(&counter, 1) 45 | } 46 | 47 | func LogStats() { 48 | var prev int32 49 | for range time.Tick(3 * time.Second) { 50 | n := GetLocalCounter() 51 | log.Printf("processed %d tasks (%d/s)", n, (n-prev)/3) 52 | prev = n 53 | } 54 | } 55 | 56 | func WaitSignal() os.Signal { 57 | ch := make(chan os.Signal, 2) 58 | signal.Notify( 59 | ch, 60 | syscall.SIGINT, 61 | syscall.SIGQUIT, 62 | syscall.SIGTERM, 63 | ) 64 | for { 65 | sig := <-ch 66 | switch sig { 67 | case syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM: 68 | return sig 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /example/sqsexample/README.md: -------------------------------------------------------------------------------- 1 | # taskq example using SQS backend 2 | 3 | The example requires `AWS_ACCOUNT_ID`, `AWS_ACCESS_KEY_ID`, and `AWS_SECRET_ACCESS_KEY` environment 4 | variables. 5 | 6 | First, start the consumer: 7 | 8 | ```shell 9 | go run consumer/main.go 10 | 11 | ``` 12 | 13 | Then, start the producer: 14 | 15 | ```shell 16 | go run producer/main.go 17 | ``` 18 | -------------------------------------------------------------------------------- /example/sqsexample/consumer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "log" 7 | 8 | "github.com/vmihailenco/taskq/example/sqsexample" 9 | ) 10 | 11 | func main() { 12 | flag.Parse() 13 | 14 | c := context.Background() 15 | 16 | err := sqsexample.QueueFactory.StartConsumers(c) 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | 21 | go sqsexample.LogStats() 22 | 23 | sig := sqsexample.WaitSignal() 24 | log.Println(sig.String()) 25 | 26 | err = sqsexample.QueueFactory.Close() 27 | if err != nil { 28 | log.Fatal(err) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /example/sqsexample/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vmihailenco/taskq/example/sqsexample 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go v1.43.45 7 | github.com/go-redis/redis/v8 v8.11.5 8 | github.com/vmihailenco/taskq/v3 v3.2.9 9 | ) 10 | 11 | require ( 12 | github.com/bsm/redislock v0.7.2 // indirect 13 | github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 // indirect 14 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 15 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect 16 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 17 | github.com/go-redis/redis_rate/v9 v9.1.2 // indirect 18 | github.com/hashicorp/golang-lru v0.5.4 // indirect 19 | github.com/jmespath/go-jmespath v0.4.0 // indirect 20 | github.com/klauspost/compress v1.15.1 // indirect 21 | github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect 22 | github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect 23 | ) 24 | 25 | replace github.com/vmihailenco/taskq/v3 => ../.. 26 | -------------------------------------------------------------------------------- /example/sqsexample/go.sum: -------------------------------------------------------------------------------- 1 | github.com/aws/aws-sdk-go v1.43.45 h1:2708Bj4uV+ym62MOtBnErm/CDX61C4mFe9V2gXy1caE= 2 | github.com/aws/aws-sdk-go v1.43.45/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= 3 | github.com/bsm/redislock v0.7.2 h1:jggqOio8JyX9FJBKIfjF3fTxAu/v7zC5mAID9LveqG4= 4 | github.com/bsm/redislock v0.7.2/go.mod h1:kS2g0Yvlymc9Dz8V3iVYAtLAaSVruYbAFdYBDrmC5WU= 5 | github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 h1:IHZ1Le1ejzkmS7Si7dIzJvYDWe+BIoNmqMnfWHBZSVw= 6 | github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3/go.mod h1:M5XHQLu90v2JNm/bW2tdsYar+5vhV0gEcBcmDBNAN1Y= 7 | github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= 8 | github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 9 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 10 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 11 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 12 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 14 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 15 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= 16 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= 17 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 18 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 19 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 20 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 21 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 22 | github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= 23 | github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= 24 | github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= 25 | github.com/go-redis/redis_rate/v9 v9.1.2 h1:H0l5VzoAtOE6ydd38j8MCq3ABlGLnvvbA1xDSVVCHgQ= 26 | github.com/go-redis/redis_rate/v9 v9.1.2/go.mod h1:oam2de2apSgRG8aJzwJddXbNu91Iyz1m8IKJE2vpvlQ= 27 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= 28 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 29 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 30 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 31 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 32 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 33 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 34 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 35 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 36 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 37 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 38 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 39 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 40 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 41 | github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 42 | github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= 43 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 44 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 45 | github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= 46 | github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 47 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 48 | github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 49 | github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac h1:w5wltlINIIqRTqQ64dASrCo0fM7k9nosPbKCZnkL0W0= 50 | github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac/go.mod h1:gyMTRVO+ZkEy7wQDyD++okPsBN2q127EpuShhHMWG54= 51 | github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74/go.mod h1:qNa9FlAfO0U/qNkzYBMH1JKYRMzC+sP9IcyV4U18l98= 52 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 53 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 54 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 55 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 56 | github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= 57 | github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 58 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 59 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 60 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 61 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 62 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 63 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= 64 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 65 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 66 | github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= 67 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= 68 | github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= 69 | github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= 70 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 71 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 72 | github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= 73 | github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= 74 | github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= 75 | github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= 76 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 77 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 78 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 79 | github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= 80 | github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= 81 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 82 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 83 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 84 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 85 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 86 | github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= 87 | github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= 88 | github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= 89 | github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= 90 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 91 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 92 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 93 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 94 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 95 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 96 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 97 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 98 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 99 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 100 | golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= 101 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= 102 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 103 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 104 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 105 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 106 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 107 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 108 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 109 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 110 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 111 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 112 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 113 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 114 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 115 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 116 | golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 117 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 118 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 119 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= 120 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 121 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 122 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 123 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 124 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 125 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 126 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 127 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 128 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 129 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 130 | golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 131 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 132 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 133 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 134 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 135 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 136 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 137 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 138 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 139 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 140 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 141 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 142 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 143 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 144 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 145 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 146 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 147 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 148 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 149 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 150 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 151 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 152 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 153 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 154 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 155 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 156 | -------------------------------------------------------------------------------- /example/sqsexample/producer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "log" 7 | 8 | "github.com/vmihailenco/taskq/example/sqsexample" 9 | ) 10 | 11 | func main() { 12 | flag.Parse() 13 | 14 | go sqsexample.LogStats() 15 | 16 | go func() { 17 | for { 18 | err := sqsexample.MainQueue.Add(sqsexample.CountTask.WithArgs(context.Background())) 19 | if err != nil { 20 | log.Fatal(err) 21 | } 22 | sqsexample.IncrLocalCounter() 23 | } 24 | }() 25 | 26 | sig := sqsexample.WaitSignal() 27 | log.Println(sig.String()) 28 | } 29 | -------------------------------------------------------------------------------- /example/sqsexample/tasks.go: -------------------------------------------------------------------------------- 1 | package sqsexample 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "os/signal" 7 | "sync/atomic" 8 | "syscall" 9 | "time" 10 | 11 | "github.com/aws/aws-sdk-go/aws/session" 12 | "github.com/aws/aws-sdk-go/service/sqs" 13 | "github.com/go-redis/redis/v8" 14 | 15 | "github.com/vmihailenco/taskq/v3" 16 | "github.com/vmihailenco/taskq/v3/azsqs" 17 | ) 18 | 19 | var Redis = redis.NewClient(&redis.Options{ 20 | Addr: ":6379", 21 | }) 22 | 23 | var ( 24 | QueueFactory = azsqs.NewFactory(sqs.New(session.New()), os.Getenv("AWS_ACCOUNT_ID")) 25 | MainQueue = QueueFactory.RegisterQueue(&taskq.QueueOptions{ 26 | Name: "api-worker", 27 | Redis: Redis, 28 | }) 29 | CountTask = taskq.RegisterTask(&taskq.TaskOptions{ 30 | Name: "counter", 31 | Handler: func() error { 32 | IncrLocalCounter() 33 | return nil 34 | }, 35 | }) 36 | ) 37 | 38 | var counter int32 39 | 40 | func GetLocalCounter() int32 { 41 | return atomic.LoadInt32(&counter) 42 | } 43 | 44 | func IncrLocalCounter() { 45 | atomic.AddInt32(&counter, 1) 46 | } 47 | 48 | func LogStats() { 49 | var prev int32 50 | for range time.Tick(3 * time.Second) { 51 | n := GetLocalCounter() 52 | log.Printf("processed %d tasks (%d/s)", n, (n-prev)/3) 53 | prev = n 54 | } 55 | } 56 | 57 | func WaitSignal() os.Signal { 58 | ch := make(chan os.Signal, 2) 59 | signal.Notify( 60 | ch, 61 | syscall.SIGINT, 62 | syscall.SIGQUIT, 63 | syscall.SIGTERM, 64 | ) 65 | for { 66 | sig := <-ch 67 | switch sig { 68 | case syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM: 69 | return sig 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /example_ratelimit_test.go: -------------------------------------------------------------------------------- 1 | package taskq_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/vmihailenco/taskq/v3" 9 | "github.com/vmihailenco/taskq/v3/memqueue" 10 | ) 11 | 12 | type RateLimitError string 13 | 14 | func (e RateLimitError) Error() string { 15 | return string(e) 16 | } 17 | 18 | func (RateLimitError) Delay() time.Duration { 19 | return 3 * time.Second 20 | } 21 | 22 | func Example_customRateLimit() { 23 | start := time.Now() 24 | q := memqueue.NewQueue(&taskq.QueueOptions{ 25 | Name: "test", 26 | }) 27 | task := taskq.RegisterTask(&taskq.TaskOptions{ 28 | Name: "Example_customRateLimit", 29 | Handler: func() error { 30 | fmt.Println("retried in", timeSince(start)) 31 | return RateLimitError("calm down") 32 | }, 33 | RetryLimit: 2, 34 | MinBackoff: time.Millisecond, 35 | }) 36 | 37 | ctx := context.Background() 38 | q.Add(task.WithArgs(ctx)) 39 | 40 | // Wait for all messages to be processed. 41 | _ = q.Close() 42 | 43 | // Output: retried in 0s 44 | // retried in 3s 45 | } 46 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | package taskq_test 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "math" 8 | "time" 9 | 10 | "github.com/go-redis/redis_rate/v9" 11 | 12 | "github.com/vmihailenco/taskq/v3" 13 | "github.com/vmihailenco/taskq/v3/memqueue" 14 | ) 15 | 16 | func timeSince(start time.Time) time.Duration { 17 | secs := float64(time.Since(start)) / float64(time.Second) 18 | return time.Duration(math.Floor(secs)) * time.Second 19 | } 20 | 21 | func timeSinceCeil(start time.Time) time.Duration { 22 | secs := float64(time.Since(start)) / float64(time.Second) 23 | return time.Duration(math.Ceil(secs)) * time.Second 24 | } 25 | 26 | func Example_retryOnError() { 27 | start := time.Now() 28 | q := memqueue.NewQueue(&taskq.QueueOptions{ 29 | Name: "test", 30 | }) 31 | task := taskq.RegisterTask(&taskq.TaskOptions{ 32 | Name: "Example_retryOnError", 33 | Handler: func() error { 34 | fmt.Println("retried in", timeSince(start)) 35 | return errors.New("fake error") 36 | }, 37 | RetryLimit: 3, 38 | MinBackoff: time.Second, 39 | }) 40 | 41 | ctx := context.Background() 42 | q.Add(task.WithArgs(ctx)) 43 | 44 | // Wait for all messages to be processed. 45 | _ = q.Close() 46 | 47 | // Output: retried in 0s 48 | // retried in 1s 49 | // retried in 3s 50 | } 51 | 52 | func Example_messageDelay() { 53 | start := time.Now() 54 | q := memqueue.NewQueue(&taskq.QueueOptions{ 55 | Name: "test", 56 | }) 57 | task := taskq.RegisterTask(&taskq.TaskOptions{ 58 | Name: "Example_messageDelay", 59 | Handler: func() { 60 | fmt.Println("processed with delay", timeSince(start)) 61 | }, 62 | }) 63 | 64 | ctx := context.Background() 65 | msg := task.WithArgs(ctx) 66 | msg.Delay = time.Second 67 | _ = q.Add(msg) 68 | 69 | // Wait for all messages to be processed. 70 | _ = q.Close() 71 | 72 | // Output: processed with delay 1s 73 | } 74 | 75 | func Example_rateLimit() { 76 | start := time.Now() 77 | q := memqueue.NewQueue(&taskq.QueueOptions{ 78 | Name: "test", 79 | Redis: redisRing(), 80 | RateLimit: redis_rate.PerSecond(1), 81 | }) 82 | task := taskq.RegisterTask(&taskq.TaskOptions{ 83 | Name: "Example_rateLimit", 84 | Handler: func() {}, 85 | }) 86 | 87 | const n = 5 88 | 89 | ctx := context.Background() 90 | for i := 0; i < n; i++ { 91 | _ = q.Add(task.WithArgs(ctx)) 92 | } 93 | 94 | // Wait for all messages to be processed. 95 | _ = q.Close() 96 | 97 | fmt.Printf("%d msg/s", timeSinceCeil(start)/time.Second/n) 98 | // Output: 1 msg/s 99 | } 100 | 101 | func Example_once() { 102 | q := memqueue.NewQueue(&taskq.QueueOptions{ 103 | Name: "test", 104 | Redis: redisRing(), 105 | RateLimit: redis_rate.PerSecond(1), 106 | }) 107 | task := taskq.RegisterTask(&taskq.TaskOptions{ 108 | Name: "Example_once", 109 | Handler: func(name string) { 110 | fmt.Println("hello", name) 111 | }, 112 | }) 113 | 114 | ctx := context.Background() 115 | for i := 0; i < 10; i++ { 116 | msg := task.WithArgs(ctx, "world") 117 | // Call once in a second. 118 | msg.OnceInPeriod(time.Second) 119 | 120 | _ = q.Add(msg) 121 | } 122 | 123 | // Wait for all messages to be processed. 124 | _ = q.Close() 125 | 126 | // Output: hello world 127 | } 128 | -------------------------------------------------------------------------------- /extra/taskqotel/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vmihailenco/taskq/extra/taskqotel/v3 2 | 3 | go 1.17 4 | 5 | replace github.com/vmihailenco/taskq/v3 => ../.. 6 | 7 | require ( 8 | github.com/vmihailenco/taskq/v3 v3.2.9 9 | go.opentelemetry.io/otel v1.6.3 10 | go.opentelemetry.io/otel/trace v1.6.3 11 | 12 | ) 13 | 14 | require ( 15 | github.com/bsm/redislock v0.7.2 // indirect 16 | github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 // indirect 17 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 18 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 19 | github.com/go-logr/logr v1.2.3 // indirect 20 | github.com/go-logr/stdr v1.2.2 // indirect 21 | github.com/go-redis/redis/v8 v8.11.5 // indirect 22 | github.com/go-redis/redis_rate/v9 v9.1.2 // indirect 23 | github.com/hashicorp/golang-lru v0.5.4 // indirect 24 | github.com/klauspost/compress v1.15.1 // indirect 25 | github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect 26 | github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect 27 | ) 28 | -------------------------------------------------------------------------------- /extra/taskqotel/go.sum: -------------------------------------------------------------------------------- 1 | github.com/aws/aws-sdk-go v1.43.45 h1:2708Bj4uV+ym62MOtBnErm/CDX61C4mFe9V2gXy1caE= 2 | github.com/aws/aws-sdk-go v1.43.45/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= 3 | github.com/bsm/redislock v0.7.2 h1:jggqOio8JyX9FJBKIfjF3fTxAu/v7zC5mAID9LveqG4= 4 | github.com/bsm/redislock v0.7.2/go.mod h1:kS2g0Yvlymc9Dz8V3iVYAtLAaSVruYbAFdYBDrmC5WU= 5 | github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 h1:IHZ1Le1ejzkmS7Si7dIzJvYDWe+BIoNmqMnfWHBZSVw= 6 | github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3/go.mod h1:M5XHQLu90v2JNm/bW2tdsYar+5vhV0gEcBcmDBNAN1Y= 7 | github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= 8 | github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 9 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 10 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 11 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 12 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 14 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 15 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= 16 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= 17 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 18 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 19 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 20 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 21 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 22 | github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 23 | github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= 24 | github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 25 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 26 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 27 | github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= 28 | github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= 29 | github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= 30 | github.com/go-redis/redis_rate/v9 v9.1.2 h1:H0l5VzoAtOE6ydd38j8MCq3ABlGLnvvbA1xDSVVCHgQ= 31 | github.com/go-redis/redis_rate/v9 v9.1.2/go.mod h1:oam2de2apSgRG8aJzwJddXbNu91Iyz1m8IKJE2vpvlQ= 32 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= 33 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 34 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 35 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 36 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 37 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 38 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 39 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 40 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 41 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 42 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 43 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 44 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 45 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 46 | github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 47 | github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= 48 | github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= 49 | github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= 50 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 51 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 52 | github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= 53 | github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 54 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 55 | github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 56 | github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac h1:w5wltlINIIqRTqQ64dASrCo0fM7k9nosPbKCZnkL0W0= 57 | github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac/go.mod h1:gyMTRVO+ZkEy7wQDyD++okPsBN2q127EpuShhHMWG54= 58 | github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74/go.mod h1:qNa9FlAfO0U/qNkzYBMH1JKYRMzC+sP9IcyV4U18l98= 59 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 60 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 61 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 62 | github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= 63 | github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 64 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 65 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 66 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 67 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 68 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 69 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= 70 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 71 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 72 | github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= 73 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= 74 | github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= 75 | github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= 76 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 77 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 78 | github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= 79 | github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= 80 | github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= 81 | github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= 82 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 83 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 84 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 85 | github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= 86 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 87 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 88 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 89 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 90 | github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= 91 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 92 | github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= 93 | github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= 94 | github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= 95 | github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= 96 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 97 | go.opentelemetry.io/otel v1.6.3 h1:FLOfo8f9JzFVFVyU+MSRJc2HdEAXQgm7pIv2uFKRSZE= 98 | go.opentelemetry.io/otel v1.6.3/go.mod h1:7BgNga5fNlF/iZjG06hM3yofffp0ofKCDwSXx1GC4dI= 99 | go.opentelemetry.io/otel/trace v1.6.3 h1:IqN4L+5b0mPNjdXIiZ90Ni4Bl5BRkDQywePLWemd9bc= 100 | go.opentelemetry.io/otel/trace v1.6.3/go.mod h1:GNJQusJlUgZl9/TQBPKU/Y/ty+0iVB5fjhKeJGZPGFs= 101 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 102 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 103 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 104 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 105 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 106 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 107 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 108 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 109 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 110 | golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= 111 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= 112 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 113 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 114 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 115 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 116 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 117 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 118 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 119 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 120 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 121 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 122 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 123 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 124 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 125 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 126 | golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 127 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 128 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 129 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= 130 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 131 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 132 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 133 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 134 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 135 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 136 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 137 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 138 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 139 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 140 | golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 141 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 142 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 143 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 144 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 145 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 146 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 147 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 148 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 149 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 150 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 151 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 152 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 153 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 154 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 155 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 156 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 157 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 158 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 159 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 160 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 161 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 162 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 163 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 164 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 165 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 166 | -------------------------------------------------------------------------------- /extra/taskqotel/otel.go: -------------------------------------------------------------------------------- 1 | package taskqotel 2 | 3 | import ( 4 | "go.opentelemetry.io/otel" 5 | "go.opentelemetry.io/otel/codes" 6 | "go.opentelemetry.io/otel/trace" 7 | 8 | "github.com/vmihailenco/taskq/v3" 9 | ) 10 | 11 | var tracer = otel.Tracer("github.com/vmihailenco/taskq") 12 | 13 | type OpenTelemetryHook struct{} 14 | 15 | var _ taskq.ConsumerHook = (*OpenTelemetryHook)(nil) 16 | 17 | func NewHook() *OpenTelemetryHook { 18 | return new(OpenTelemetryHook) 19 | } 20 | 21 | func (h OpenTelemetryHook) BeforeProcessMessage(evt *taskq.ProcessMessageEvent) error { 22 | evt.Message.Ctx, _ = tracer.Start(evt.Message.Ctx, evt.Message.TaskName) 23 | return nil 24 | } 25 | 26 | func (h OpenTelemetryHook) AfterProcessMessage(evt *taskq.ProcessMessageEvent) error { 27 | ctx := evt.Message.Ctx 28 | 29 | span := trace.SpanFromContext(ctx) 30 | defer span.End() 31 | 32 | if err := evt.Message.Err; err != nil { 33 | span.SetStatus(codes.Error, err.Error()) 34 | span.RecordError(err) 35 | } 36 | 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vmihailenco/taskq/v3 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go v1.43.45 7 | github.com/bsm/redislock v0.7.2 8 | github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 9 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 10 | github.com/go-redis/redis/v8 v8.11.5 11 | github.com/go-redis/redis_rate/v9 v9.1.2 12 | github.com/google/uuid v1.3.0 13 | github.com/hashicorp/golang-lru v0.5.4 14 | github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac 15 | github.com/klauspost/compress v1.15.1 16 | github.com/onsi/ginkgo v1.16.5 17 | github.com/onsi/gomega v1.18.1 18 | github.com/satori/go.uuid v1.2.0 19 | github.com/vmihailenco/msgpack/v5 v5.3.5 20 | ) 21 | 22 | require ( 23 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 24 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 25 | github.com/fsnotify/fsnotify v1.4.9 // indirect 26 | github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74 // indirect 27 | github.com/jmespath/go-jmespath v0.4.0 // indirect 28 | github.com/kr/pretty v0.2.1 // indirect 29 | github.com/nxadm/tail v1.4.8 // indirect 30 | github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect 31 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect 32 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect 33 | golang.org/x/text v0.3.7 // indirect 34 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect 35 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 36 | gopkg.in/yaml.v2 v2.4.0 // indirect 37 | ) 38 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/aws/aws-sdk-go v1.43.45 h1:2708Bj4uV+ym62MOtBnErm/CDX61C4mFe9V2gXy1caE= 2 | github.com/aws/aws-sdk-go v1.43.45/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= 3 | github.com/bsm/redislock v0.7.2 h1:jggqOio8JyX9FJBKIfjF3fTxAu/v7zC5mAID9LveqG4= 4 | github.com/bsm/redislock v0.7.2/go.mod h1:kS2g0Yvlymc9Dz8V3iVYAtLAaSVruYbAFdYBDrmC5WU= 5 | github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 h1:IHZ1Le1ejzkmS7Si7dIzJvYDWe+BIoNmqMnfWHBZSVw= 6 | github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3/go.mod h1:M5XHQLu90v2JNm/bW2tdsYar+5vhV0gEcBcmDBNAN1Y= 7 | github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= 8 | github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 9 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 10 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 11 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 12 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 14 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 15 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= 16 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= 17 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 18 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 19 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 20 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 21 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 22 | github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= 23 | github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= 24 | github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= 25 | github.com/go-redis/redis_rate/v9 v9.1.2 h1:H0l5VzoAtOE6ydd38j8MCq3ABlGLnvvbA1xDSVVCHgQ= 26 | github.com/go-redis/redis_rate/v9 v9.1.2/go.mod h1:oam2de2apSgRG8aJzwJddXbNu91Iyz1m8IKJE2vpvlQ= 27 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= 28 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 29 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 30 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 31 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 32 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 33 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 34 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 35 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 36 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= 37 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 38 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 39 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 40 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 41 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 42 | github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 43 | github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= 44 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 45 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 46 | github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= 47 | github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 48 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 49 | github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 50 | github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac h1:w5wltlINIIqRTqQ64dASrCo0fM7k9nosPbKCZnkL0W0= 51 | github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac/go.mod h1:gyMTRVO+ZkEy7wQDyD++okPsBN2q127EpuShhHMWG54= 52 | github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74 h1:gyfyP8SEIZHs1u2ivTdIbWRtfaKbg5K79d06vnqroJo= 53 | github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74/go.mod h1:qNa9FlAfO0U/qNkzYBMH1JKYRMzC+sP9IcyV4U18l98= 54 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 55 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 56 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 57 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 58 | github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= 59 | github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 60 | github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= 61 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 62 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 63 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 64 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 65 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 66 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 67 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= 68 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 69 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 70 | github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= 71 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= 72 | github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= 73 | github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ= 74 | github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= 75 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 76 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 77 | github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= 78 | github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= 79 | github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= 80 | github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= 81 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 82 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 83 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 84 | github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= 85 | github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= 86 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 87 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 88 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 89 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 90 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 91 | github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= 92 | github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= 93 | github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= 94 | github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= 95 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 96 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 97 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 98 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 99 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 100 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 101 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 102 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 103 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 104 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 105 | golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= 106 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= 107 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 108 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 109 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 110 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 111 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 112 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 113 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 114 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 115 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 116 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 117 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 118 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 119 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 120 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 121 | golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 122 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 123 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 124 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= 125 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 126 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 127 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 128 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 129 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 130 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 131 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 132 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 133 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 134 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 135 | golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 136 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 137 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 138 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 139 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 140 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 141 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 142 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 143 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 144 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 145 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 146 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 147 | google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= 148 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 149 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 150 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 151 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 152 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 153 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 154 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 155 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 156 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 157 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 158 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 159 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 160 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 161 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 162 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 163 | -------------------------------------------------------------------------------- /handler.go: -------------------------------------------------------------------------------- 1 | package taskq 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "reflect" 9 | 10 | "github.com/vmihailenco/msgpack/v5" 11 | ) 12 | 13 | var contextType = reflect.TypeOf((*context.Context)(nil)).Elem() 14 | var messageType = reflect.TypeOf((*Message)(nil)) 15 | var errorType = reflect.TypeOf((*error)(nil)).Elem() 16 | 17 | // Handler is an interface for processing messages. 18 | type Handler interface { 19 | HandleMessage(msg *Message) error 20 | } 21 | 22 | type HandlerFunc func(*Message) error 23 | 24 | func (fn HandlerFunc) HandleMessage(msg *Message) error { 25 | return fn(msg) 26 | } 27 | 28 | type reflectFunc struct { 29 | fv reflect.Value // Kind() == reflect.Func 30 | ft reflect.Type 31 | 32 | acceptsContext bool 33 | returnsError bool 34 | } 35 | 36 | var _ Handler = (*reflectFunc)(nil) 37 | 38 | func NewHandler(fn interface{}) Handler { 39 | if fn == nil { 40 | panic(errors.New("taskq: handler func is nil")) 41 | } 42 | if h, ok := fn.(Handler); ok { 43 | return h 44 | } 45 | 46 | h := reflectFunc{ 47 | fv: reflect.ValueOf(fn), 48 | } 49 | h.ft = h.fv.Type() 50 | if h.ft.Kind() != reflect.Func { 51 | panic(fmt.Sprintf("taskq: got %s, wanted %s", h.ft.Kind(), reflect.Func)) 52 | } 53 | 54 | h.returnsError = returnsError(h.ft) 55 | if acceptsMessage(h.ft) { 56 | if h.returnsError { 57 | return HandlerFunc(fn.(func(*Message) error)) 58 | } 59 | if h.ft.NumOut() == 0 { 60 | theFn := fn.(func(*Message)) 61 | return HandlerFunc(func(msg *Message) error { 62 | theFn(msg) 63 | return nil 64 | }) 65 | } 66 | } 67 | 68 | h.acceptsContext = acceptsContext(h.ft) 69 | return &h 70 | } 71 | 72 | func (h *reflectFunc) HandleMessage(msg *Message) error { 73 | in, err := h.fnArgs(msg) 74 | if err != nil { 75 | return err 76 | } 77 | 78 | out := h.fv.Call(in) 79 | if h.returnsError { 80 | errv := out[h.ft.NumOut()-1] 81 | if !errv.IsNil() { 82 | return errv.Interface().(error) 83 | } 84 | } 85 | 86 | return nil 87 | } 88 | 89 | func (h *reflectFunc) fnArgs(msg *Message) ([]reflect.Value, error) { 90 | in := make([]reflect.Value, h.ft.NumIn()) 91 | inSaved := in 92 | 93 | var inStart int 94 | if h.acceptsContext { 95 | inStart = 1 96 | in[0] = reflect.ValueOf(msg.Ctx) 97 | in = in[1:] 98 | } 99 | 100 | if len(msg.Args) == len(in) { 101 | var hasWrongType bool 102 | for i, arg := range msg.Args { 103 | v := reflect.ValueOf(arg) 104 | inType := h.ft.In(inStart + i) 105 | 106 | if inType.Kind() == reflect.Interface { 107 | if !v.Type().Implements(inType) { 108 | hasWrongType = true 109 | break 110 | } 111 | } else if v.Type() != inType { 112 | hasWrongType = true 113 | break 114 | } 115 | 116 | in[i] = v 117 | } 118 | if !hasWrongType { 119 | return inSaved, nil 120 | } 121 | } 122 | 123 | b, err := msg.MarshalArgs() 124 | if err != nil { 125 | return nil, err 126 | } 127 | 128 | dec := msgpack.NewDecoder(bytes.NewBuffer(b)) 129 | n, err := dec.DecodeArrayLen() 130 | if err != nil { 131 | return nil, err 132 | } 133 | 134 | if n == -1 { 135 | n = 0 136 | } 137 | if n != len(in) { 138 | return nil, fmt.Errorf("taskq: got %d args, wanted %d", n, len(in)) 139 | } 140 | 141 | for i := 0; i < len(in); i++ { 142 | arg := reflect.New(h.ft.In(inStart + i)).Elem() 143 | err = dec.DecodeValue(arg) 144 | if err != nil { 145 | err = fmt.Errorf( 146 | "taskq: decoding arg=%d failed (data=%.100x): %s", i, b, err) 147 | return nil, err 148 | } 149 | in[i] = arg 150 | } 151 | 152 | return inSaved, nil 153 | } 154 | 155 | func acceptsMessage(typ reflect.Type) bool { 156 | return typ.NumIn() == 1 && typ.In(0) == messageType 157 | } 158 | 159 | func acceptsContext(typ reflect.Type) bool { 160 | return typ.NumIn() > 0 && typ.In(0).Implements(contextType) 161 | } 162 | 163 | func returnsError(typ reflect.Type) bool { 164 | n := typ.NumOut() 165 | return n > 0 && typ.Out(n-1) == errorType 166 | } 167 | -------------------------------------------------------------------------------- /internal/base/batcher.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/vmihailenco/taskq/v3" 8 | ) 9 | 10 | type BatcherOptions struct { 11 | Handler func([]*taskq.Message) error 12 | ShouldBatch func([]*taskq.Message, *taskq.Message) bool 13 | 14 | Timeout time.Duration 15 | } 16 | 17 | func (opt *BatcherOptions) init() { 18 | if opt.Timeout == 0 { 19 | opt.Timeout = 3 * time.Second 20 | } 21 | } 22 | 23 | // Batcher collects messages for later batch processing. 24 | type Batcher struct { 25 | consumer taskq.QueueConsumer 26 | opt *BatcherOptions 27 | 28 | timer *time.Timer 29 | 30 | mu sync.Mutex 31 | batch []*taskq.Message 32 | closed bool 33 | } 34 | 35 | func NewBatcher(consumer taskq.QueueConsumer, opt *BatcherOptions) *Batcher { 36 | opt.init() 37 | b := Batcher{ 38 | consumer: consumer, 39 | opt: opt, 40 | } 41 | b.timer = time.AfterFunc(time.Minute, b.onTimeout) 42 | b.timer.Stop() 43 | return &b 44 | } 45 | 46 | func (b *Batcher) flush() { 47 | if len(b.batch) > 0 { 48 | b.process(b.batch) 49 | b.batch = nil 50 | } 51 | } 52 | 53 | func (b *Batcher) Add(msg *taskq.Message) error { 54 | var batch []*taskq.Message 55 | 56 | b.mu.Lock() 57 | 58 | if b.closed { 59 | if len(b.batch) > 0 { 60 | panic("not reached") 61 | } 62 | batch = []*taskq.Message{msg} 63 | } else { 64 | if len(b.batch) == 0 { 65 | b.stopTimer() 66 | b.timer.Reset(b.opt.Timeout) 67 | } 68 | 69 | if b.opt.ShouldBatch(b.batch, msg) { 70 | b.batch = append(b.batch, msg) 71 | } else { 72 | batch = b.batch 73 | b.batch = []*taskq.Message{msg} 74 | } 75 | } 76 | 77 | b.mu.Unlock() 78 | 79 | if len(batch) > 0 { 80 | b.process(batch) 81 | } 82 | 83 | return taskq.ErrAsyncTask 84 | } 85 | 86 | func (b *Batcher) stopTimer() { 87 | if !b.timer.Stop() { 88 | select { 89 | case <-b.timer.C: 90 | default: 91 | } 92 | } 93 | } 94 | 95 | func (b *Batcher) process(batch []*taskq.Message) { 96 | err := b.opt.Handler(batch) 97 | for _, msg := range batch { 98 | if msg.Err == nil { 99 | msg.Err = err 100 | } 101 | b.consumer.Put(msg) 102 | } 103 | } 104 | 105 | func (b *Batcher) onTimeout() { 106 | b.mu.Lock() 107 | b.flush() 108 | b.mu.Unlock() 109 | } 110 | 111 | func (b *Batcher) Close() error { 112 | b.mu.Lock() 113 | defer b.mu.Unlock() 114 | 115 | if b.closed { 116 | return nil 117 | } 118 | b.closed = true 119 | 120 | b.stopTimer() 121 | b.flush() 122 | 123 | return nil 124 | } 125 | -------------------------------------------------------------------------------- /internal/base/factory.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/vmihailenco/taskq/v3" 9 | ) 10 | 11 | type Factory struct { 12 | m sync.Map 13 | } 14 | 15 | func (f *Factory) Register(queue taskq.Queue) error { 16 | name := queue.Name() 17 | _, loaded := f.m.LoadOrStore(name, queue) 18 | if loaded { 19 | return fmt.Errorf("queue=%q already exists", name) 20 | } 21 | return nil 22 | } 23 | 24 | func (f *Factory) Unregister(name string) { 25 | f.m.Delete(name) 26 | } 27 | 28 | func (f *Factory) Reset() { 29 | f.m = sync.Map{} 30 | } 31 | 32 | func (f *Factory) Range(fn func(queue taskq.Queue) bool) { 33 | f.m.Range(func(_, value interface{}) bool { 34 | return fn(value.(taskq.Queue)) 35 | }) 36 | } 37 | 38 | func (f *Factory) StartConsumers(ctx context.Context) error { 39 | return f.forEachQueue(func(q taskq.Queue) error { 40 | return q.Consumer().Start(ctx) 41 | }) 42 | } 43 | 44 | func (f *Factory) StopConsumers() error { 45 | return f.forEachQueue(func(q taskq.Queue) error { 46 | return q.Consumer().Stop() 47 | }) 48 | } 49 | 50 | func (f *Factory) Close() error { 51 | return f.forEachQueue(func(q taskq.Queue) error { 52 | return q.Close() 53 | }) 54 | } 55 | 56 | func (f *Factory) forEachQueue(fn func(taskq.Queue) error) error { 57 | var wg sync.WaitGroup 58 | errCh := make(chan error, 1) 59 | f.Range(func(q taskq.Queue) bool { 60 | wg.Add(1) 61 | go func(q taskq.Queue) { 62 | defer wg.Done() 63 | err := fn(q) 64 | select { 65 | case errCh <- err: 66 | default: 67 | } 68 | }(q) 69 | return true 70 | }) 71 | wg.Wait() 72 | select { 73 | case err := <-errCh: 74 | return err 75 | default: 76 | return nil 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /internal/error.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import "errors" 4 | 5 | var ErrNotSupported = errors.New("not supported") 6 | var ErrTaskNameRequired = errors.New("taskq: Message.TaskName is required") 7 | -------------------------------------------------------------------------------- /internal/log.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "log" 5 | ) 6 | 7 | var Logger *log.Logger 8 | -------------------------------------------------------------------------------- /internal/msgutil/msgutil.go: -------------------------------------------------------------------------------- 1 | package msgutil 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | 7 | "github.com/dgryski/go-farm" 8 | 9 | "github.com/vmihailenco/taskq/v3" 10 | "github.com/vmihailenco/taskq/v3/internal" 11 | ) 12 | 13 | func WrapMessage(msg *taskq.Message) *taskq.Message { 14 | msg0 := taskq.NewMessage(msg.Ctx, msg) 15 | msg0.Name = msg.Name 16 | return msg0 17 | } 18 | 19 | func UnwrapMessage(msg *taskq.Message) (*taskq.Message, error) { 20 | if len(msg.Args) != 1 { 21 | err := fmt.Errorf("UnwrapMessage: got %d args, wanted 1", len(msg.Args)) 22 | return nil, err 23 | } 24 | 25 | msg, ok := msg.Args[0].(*taskq.Message) 26 | if !ok { 27 | err := fmt.Errorf("UnwrapMessage: got %v, wanted *taskq.Message", msg.Args) 28 | return nil, err 29 | } 30 | return msg, nil 31 | } 32 | 33 | func UnwrapMessageHandler(fn interface{}) taskq.HandlerFunc { 34 | if fn == nil { 35 | return nil 36 | } 37 | h := fn.(func(*taskq.Message) error) 38 | return taskq.HandlerFunc(func(msg *taskq.Message) error { 39 | msg, err := UnwrapMessage(msg) 40 | if err != nil { 41 | return err 42 | } 43 | return h(msg) 44 | }) 45 | } 46 | 47 | func FullMessageName(q taskq.Queue, msg *taskq.Message) string { 48 | ln := len(q.Name()) + len(msg.TaskName) 49 | data := make([]byte, 0, ln+len(msg.Name)) 50 | data = append(data, q.Name()...) 51 | data = append(data, msg.TaskName...) 52 | data = append(data, msg.Name...) 53 | 54 | b := make([]byte, 3+8+8) 55 | copy(b, "tq:") 56 | 57 | // Hash message name. 58 | h := farm.Hash64(data[ln:]) 59 | binary.BigEndian.PutUint64(b[3:11], h) 60 | 61 | // Hash queue name and use it as a seed. 62 | seed := farm.Hash64(data[:ln]) 63 | 64 | // Hash everything using the seed. 65 | h = farm.Hash64WithSeed(data, seed) 66 | binary.BigEndian.PutUint64(b[11:19], h) 67 | 68 | return internal.BytesToString(b) 69 | } 70 | -------------------------------------------------------------------------------- /internal/safe.go: -------------------------------------------------------------------------------- 1 | //go:build appengine 2 | // +build appengine 3 | 4 | package internal 5 | 6 | func BytesToString(b []byte) string { 7 | return string(b) 8 | } 9 | 10 | func StringToBytes(s string) []byte { 11 | return []byte(s) 12 | } 13 | -------------------------------------------------------------------------------- /internal/unsafe.go: -------------------------------------------------------------------------------- 1 | //go:build !appengine 2 | // +build !appengine 3 | 4 | package internal 5 | 6 | import ( 7 | "unsafe" 8 | ) 9 | 10 | // BytesToString converts byte slice to string. 11 | func BytesToString(b []byte) string { 12 | return *(*string)(unsafe.Pointer(&b)) 13 | } 14 | 15 | // StringToBytes converts string to byte slice. 16 | func StringToBytes(s string) []byte { 17 | return *(*[]byte)(unsafe.Pointer( 18 | &struct { 19 | string 20 | Cap int 21 | }{s, len(s)}, 22 | )) 23 | } 24 | -------------------------------------------------------------------------------- /internal/util.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "encoding/ascii85" 5 | "errors" 6 | ) 7 | 8 | func MaxEncodedLen(n int) int { 9 | return ascii85.MaxEncodedLen(n) 10 | } 11 | 12 | func EncodeToString(src []byte) string { 13 | dst := make([]byte, MaxEncodedLen(len(src))) 14 | n := ascii85.Encode(dst, src) 15 | dst = dst[:n] 16 | return BytesToString(dst) 17 | } 18 | 19 | func DecodeString(src string) ([]byte, error) { 20 | dst := make([]byte, len(src)) 21 | ndst, nsrc, err := ascii85.Decode(dst, StringToBytes(src), true) 22 | if err != nil { 23 | return nil, err 24 | } 25 | if nsrc != len(src) { 26 | return nil, errors.New("ascii85: src is not fully decoded") 27 | } 28 | return dst[:ndst], nil 29 | } 30 | -------------------------------------------------------------------------------- /ironmq/factory.go: -------------------------------------------------------------------------------- 1 | package ironmq 2 | 3 | import ( 4 | "context" 5 | 6 | iron_config "github.com/iron-io/iron_go3/config" 7 | "github.com/iron-io/iron_go3/mq" 8 | 9 | "github.com/vmihailenco/taskq/v3" 10 | "github.com/vmihailenco/taskq/v3/internal/base" 11 | ) 12 | 13 | type factory struct { 14 | base base.Factory 15 | 16 | cfg *iron_config.Settings 17 | } 18 | 19 | var _ taskq.Factory = (*factory)(nil) 20 | 21 | func (f *factory) RegisterQueue(opt *taskq.QueueOptions) taskq.Queue { 22 | ironq := mq.ConfigNew(opt.Name, f.cfg) 23 | q := NewQueue(ironq, opt) 24 | if err := f.base.Register(q); err != nil { 25 | panic(err) 26 | } 27 | return q 28 | } 29 | 30 | func (f *factory) Range(fn func(taskq.Queue) bool) { 31 | f.base.Range(fn) 32 | } 33 | 34 | func (f *factory) StartConsumers(ctx context.Context) error { 35 | return f.base.StartConsumers(ctx) 36 | } 37 | 38 | func (f *factory) StopConsumers() error { 39 | return f.base.StopConsumers() 40 | } 41 | 42 | func (f *factory) Close() error { 43 | return f.base.Close() 44 | } 45 | 46 | func NewFactory(cfg *iron_config.Settings) taskq.Factory { 47 | return &factory{ 48 | cfg: cfg, 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /ironmq/queue.go: -------------------------------------------------------------------------------- 1 | package ironmq 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "strings" 8 | "time" 9 | 10 | "github.com/iron-io/iron_go3/api" 11 | "github.com/iron-io/iron_go3/mq" 12 | 13 | "github.com/vmihailenco/taskq/v3" 14 | "github.com/vmihailenco/taskq/v3/internal" 15 | "github.com/vmihailenco/taskq/v3/internal/base" 16 | "github.com/vmihailenco/taskq/v3/internal/msgutil" 17 | "github.com/vmihailenco/taskq/v3/memqueue" 18 | ) 19 | 20 | type Queue struct { 21 | opt *taskq.QueueOptions 22 | 23 | q mq.Queue 24 | 25 | addQueue *memqueue.Queue 26 | addTask *taskq.Task 27 | 28 | delQueue *memqueue.Queue 29 | delTask *taskq.Task 30 | delBatcher *base.Batcher 31 | 32 | consumer *taskq.Consumer 33 | } 34 | 35 | var _ taskq.Queue = (*Queue)(nil) 36 | 37 | func NewQueue(mqueue mq.Queue, opt *taskq.QueueOptions) *Queue { 38 | if opt.Name == "" { 39 | opt.Name = mqueue.Name 40 | } 41 | opt.Init() 42 | 43 | q := &Queue{ 44 | q: mqueue, 45 | opt: opt, 46 | } 47 | 48 | q.initAddQueue() 49 | q.initDelQueue() 50 | 51 | return q 52 | } 53 | 54 | func (q *Queue) initAddQueue() { 55 | queueName := "ironmq:" + q.opt.Name + ":add" 56 | q.addQueue = memqueue.NewQueue(&taskq.QueueOptions{ 57 | Name: queueName, 58 | BufferSize: 100, 59 | Redis: q.opt.Redis, 60 | }) 61 | q.addTask = taskq.RegisterTask(&taskq.TaskOptions{ 62 | Name: queueName + ":add-mesage", 63 | Handler: taskq.HandlerFunc(q.add), 64 | FallbackHandler: msgutil.UnwrapMessageHandler(q.opt.Handler.HandleMessage), 65 | RetryLimit: 3, 66 | MinBackoff: time.Second, 67 | }) 68 | } 69 | 70 | func (q *Queue) initDelQueue() { 71 | queueName := "ironmq:" + q.opt.Name + ":delete" 72 | q.delQueue = memqueue.NewQueue(&taskq.QueueOptions{ 73 | Name: queueName, 74 | BufferSize: 100, 75 | Redis: q.opt.Redis, 76 | }) 77 | q.delTask = taskq.RegisterTask(&taskq.TaskOptions{ 78 | Name: queueName + ":delete-message", 79 | Handler: taskq.HandlerFunc(q.delBatcherAdd), 80 | RetryLimit: 3, 81 | MinBackoff: time.Second, 82 | }) 83 | q.delBatcher = base.NewBatcher(q.delQueue.Consumer(), &base.BatcherOptions{ 84 | Handler: q.deleteBatch, 85 | ShouldBatch: q.shouldBatchDelete, 86 | }) 87 | } 88 | 89 | func (q *Queue) Name() string { 90 | return q.q.Name 91 | } 92 | 93 | func (q *Queue) String() string { 94 | return fmt.Sprintf("queue=%q", q.Name()) 95 | } 96 | 97 | func (q *Queue) Options() *taskq.QueueOptions { 98 | return q.opt 99 | } 100 | 101 | func (q *Queue) Consumer() taskq.QueueConsumer { 102 | if q.consumer == nil { 103 | q.consumer = taskq.NewConsumer(q) 104 | } 105 | return q.consumer 106 | } 107 | 108 | func (q *Queue) createQueue() error { 109 | _, err := mq.ConfigCreateQueue(mq.QueueInfo{Name: q.q.Name}, &q.q.Settings) 110 | return err 111 | } 112 | 113 | func (q *Queue) Len() (int, error) { 114 | queueInfo, err := q.q.Info() 115 | if err != nil { 116 | return 0, err 117 | } 118 | return queueInfo.Size, nil 119 | } 120 | 121 | // Add adds message to the queue. 122 | func (q *Queue) Add(msg *taskq.Message) error { 123 | if msg.TaskName == "" { 124 | return internal.ErrTaskNameRequired 125 | } 126 | if q.isDuplicate(msg) { 127 | msg.Err = taskq.ErrDuplicate 128 | return nil 129 | } 130 | msg = msgutil.WrapMessage(msg) 131 | msg.TaskName = q.addTask.Name() 132 | return q.addQueue.Add(msg) 133 | } 134 | 135 | func (q *Queue) ReserveN( 136 | ctx context.Context, n int, waitTimeout time.Duration, 137 | ) ([]taskq.Message, error) { 138 | if n > 100 { 139 | n = 100 140 | } 141 | 142 | reservationSecs := int(q.opt.ReservationTimeout / time.Second) 143 | waitSecs := int(waitTimeout / time.Second) 144 | 145 | mqMsgs, err := q.q.LongPoll(n, reservationSecs, waitSecs, false) 146 | if err != nil { 147 | if v, ok := err.(api.HTTPResponseError); ok && v.StatusCode() == 404 { 148 | if strings.Contains(v.Error(), "Message not found") { 149 | return nil, nil 150 | } 151 | if strings.Contains(v.Error(), "Queue not found") { 152 | _ = q.createQueue() 153 | } 154 | } 155 | return nil, err 156 | } 157 | 158 | msgs := make([]taskq.Message, len(mqMsgs)) 159 | for i, mqMsg := range mqMsgs { 160 | msg := &msgs[i] 161 | 162 | b, err := internal.DecodeString(mqMsg.Body) 163 | if err != nil { 164 | msg.Err = err 165 | } else { 166 | err = msg.UnmarshalBinary(b) 167 | if err != nil { 168 | msg.Err = err 169 | } 170 | } 171 | 172 | msg.ID = mqMsg.Id 173 | msg.ReservationID = mqMsg.ReservationId 174 | msg.ReservedCount = mqMsg.ReservedCount 175 | } 176 | 177 | return msgs, nil 178 | } 179 | 180 | func (q *Queue) Release(msg *taskq.Message) error { 181 | return retry(func() error { 182 | return q.q.ReleaseMessage(msg.ID, msg.ReservationID, int64(msg.Delay/time.Second)) 183 | }) 184 | } 185 | 186 | // Delete deletes the message from the queue. 187 | func (q *Queue) Delete(msg *taskq.Message) error { 188 | err := retry(func() error { 189 | return q.q.DeleteMessage(msg.ID, msg.ReservationID) 190 | }) 191 | if err == nil { 192 | return nil 193 | } 194 | if v, ok := err.(api.HTTPResponseError); ok && v.StatusCode() == 404 { 195 | return nil 196 | } 197 | return err 198 | } 199 | 200 | // Purge deletes all messages from the queue using IronMQ API. 201 | func (q *Queue) Purge() error { 202 | return q.q.Clear() 203 | } 204 | 205 | // Close is like CloseTimeout with 30 seconds timeout. 206 | func (q *Queue) Close() error { 207 | return q.CloseTimeout(30 * time.Second) 208 | } 209 | 210 | // CloseTimeout closes the queue waiting for pending messages to be processed. 211 | func (q *Queue) CloseTimeout(timeout time.Duration) error { 212 | if q.consumer != nil { 213 | _ = q.consumer.StopTimeout(timeout) 214 | } 215 | 216 | firstErr := q.delBatcher.Close() 217 | 218 | err := q.delQueue.CloseTimeout(timeout) 219 | if err != nil && firstErr == nil { 220 | firstErr = err 221 | } 222 | 223 | return firstErr 224 | } 225 | 226 | func (q *Queue) add(msg *taskq.Message) error { 227 | msg, err := msgutil.UnwrapMessage(msg) 228 | if err != nil { 229 | return err 230 | } 231 | 232 | b, err := msg.MarshalBinary() 233 | if err != nil { 234 | return err 235 | } 236 | 237 | id, err := q.q.PushMessage(mq.Message{ 238 | Body: internal.EncodeToString(b), 239 | Delay: int64(msg.Delay / time.Second), 240 | }) 241 | if err != nil { 242 | return err 243 | } 244 | 245 | msg.ID = id 246 | return nil 247 | } 248 | 249 | func (q *Queue) delBatcherAdd(msg *taskq.Message) error { 250 | return q.delBatcher.Add(msg) 251 | } 252 | 253 | func (q *Queue) deleteBatch(msgs []*taskq.Message) error { 254 | if len(msgs) == 0 { 255 | return errors.New("ironmq: no messages to delete") 256 | } 257 | 258 | mqMsgs := make([]mq.Message, len(msgs)) 259 | for i, msg := range msgs { 260 | msg, err := msgutil.UnwrapMessage(msg) 261 | if err != nil { 262 | return err 263 | } 264 | 265 | mqMsgs[i] = mq.Message{ 266 | Id: msg.ID, 267 | ReservationId: msg.ReservationID, 268 | } 269 | } 270 | 271 | err := retry(func() error { 272 | return q.q.DeleteReservedMessages(mqMsgs) 273 | }) 274 | if err != nil { 275 | internal.Logger.Printf("ironmq: DeleteReservedMessages failed: %s", err) 276 | return err 277 | } 278 | 279 | return nil 280 | } 281 | 282 | func (q *Queue) shouldBatchDelete(batch []*taskq.Message, msg *taskq.Message) bool { 283 | const messagesLimit = 10 284 | return len(batch)+1 < messagesLimit 285 | } 286 | 287 | func (q *Queue) isDuplicate(msg *taskq.Message) bool { 288 | if msg.Name == "" { 289 | return false 290 | } 291 | return q.opt.Storage.Exists(msg.Ctx, msgutil.FullMessageName(q, msg)) 292 | } 293 | 294 | func retry(fn func() error) error { 295 | var err error 296 | for i := 0; i < 3; i++ { 297 | err = fn() 298 | if err == nil { 299 | return nil 300 | } 301 | if v, ok := err.(api.HTTPResponseError); ok && v.StatusCode() >= 500 { 302 | continue 303 | } 304 | break 305 | } 306 | return err 307 | } 308 | -------------------------------------------------------------------------------- /ironmq_test.go: -------------------------------------------------------------------------------- 1 | package taskq_test 2 | 3 | import ( 4 | "testing" 5 | 6 | iron_config "github.com/iron-io/iron_go3/config" 7 | 8 | "github.com/vmihailenco/taskq/v3" 9 | "github.com/vmihailenco/taskq/v3/ironmq" 10 | ) 11 | 12 | func ironmqFactory() taskq.Factory { 13 | settings := iron_config.Config("iron_mq") 14 | return ironmq.NewFactory(&settings) 15 | } 16 | 17 | func TestIronmqConsumer(t *testing.T) { 18 | t.Skip() 19 | 20 | testConsumer(t, ironmqFactory(), &taskq.QueueOptions{ 21 | Name: queueName("ironmq-consumer"), 22 | }) 23 | } 24 | 25 | func TestIronmqUnknownTask(t *testing.T) { 26 | t.Skip() 27 | 28 | testUnknownTask(t, ironmqFactory(), &taskq.QueueOptions{ 29 | Name: queueName("ironmq-unknown-task"), 30 | }) 31 | } 32 | 33 | func TestIronmqFallback(t *testing.T) { 34 | t.Skip() 35 | 36 | testFallback(t, ironmqFactory(), &taskq.QueueOptions{ 37 | Name: queueName("ironmq-fallback"), 38 | }) 39 | } 40 | 41 | func TestIronmqDelay(t *testing.T) { 42 | t.Skip() 43 | 44 | testDelay(t, ironmqFactory(), &taskq.QueueOptions{ 45 | Name: queueName("ironmq-delay"), 46 | }) 47 | } 48 | 49 | func TestIronmqRetry(t *testing.T) { 50 | t.Skip() 51 | 52 | testRetry(t, ironmqFactory(), &taskq.QueueOptions{ 53 | Name: queueName("ironmq-retry"), 54 | }) 55 | } 56 | 57 | func TestIronmqNamedMessage(t *testing.T) { 58 | t.Skip() 59 | 60 | testNamedMessage(t, ironmqFactory(), &taskq.QueueOptions{ 61 | Name: queueName("ironmq-named-message"), 62 | }) 63 | } 64 | 65 | func TestIronmqCallOnce(t *testing.T) { 66 | t.Skip() 67 | 68 | testCallOnce(t, ironmqFactory(), &taskq.QueueOptions{ 69 | Name: queueName("ironmq-call-once"), 70 | }) 71 | } 72 | 73 | func TestIronmqLen(t *testing.T) { 74 | t.Skip() 75 | 76 | testLen(t, ironmqFactory(), &taskq.QueueOptions{ 77 | Name: queueName("ironmq-len"), 78 | }) 79 | } 80 | 81 | func TestIronmqRateLimit(t *testing.T) { 82 | t.Skip() 83 | 84 | testRateLimit(t, ironmqFactory(), &taskq.QueueOptions{ 85 | Name: queueName("ironmq-rate-limit"), 86 | }) 87 | } 88 | 89 | func TestIronmqErrorDelay(t *testing.T) { 90 | t.Skip() 91 | 92 | testErrorDelay(t, ironmqFactory(), &taskq.QueueOptions{ 93 | Name: queueName("ironmq-delayer"), 94 | }) 95 | } 96 | 97 | func TestIronmqWorkerLimit(t *testing.T) { 98 | t.Skip() 99 | 100 | testWorkerLimit(t, ironmqFactory(), &taskq.QueueOptions{ 101 | Name: queueName("ironmq-worker-limit"), 102 | }) 103 | } 104 | 105 | func TestIronmqInvalidCredentials(t *testing.T) { 106 | t.Skip() 107 | 108 | settings := &iron_config.Settings{ 109 | ProjectId: "123", 110 | } 111 | factory := ironmq.NewFactory(settings) 112 | testInvalidCredentials(t, factory, &taskq.QueueOptions{ 113 | Name: queueName("ironmq-invalid-credentials"), 114 | }) 115 | } 116 | -------------------------------------------------------------------------------- /memqueue/bench_test.go: -------------------------------------------------------------------------------- 1 | package memqueue_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/vmihailenco/taskq/v3" 8 | "github.com/vmihailenco/taskq/v3/memqueue" 9 | ) 10 | 11 | func BenchmarkCallAsync(b *testing.B) { 12 | taskq.Tasks.Reset() 13 | ctx := context.Background() 14 | 15 | q := memqueue.NewQueue(&taskq.QueueOptions{ 16 | Name: "test", 17 | Storage: taskq.NewLocalStorage(), 18 | }) 19 | defer q.Close() 20 | 21 | task := taskq.RegisterTask(&taskq.TaskOptions{ 22 | Name: "test", 23 | Handler: func() {}, 24 | }) 25 | 26 | b.ResetTimer() 27 | 28 | b.RunParallel(func(pb *testing.PB) { 29 | for pb.Next() { 30 | _ = q.Add(task.WithArgs(ctx)) 31 | } 32 | }) 33 | } 34 | 35 | func BenchmarkNamedMessage(b *testing.B) { 36 | taskq.Tasks.Reset() 37 | ctx := context.Background() 38 | 39 | q := memqueue.NewQueue(&taskq.QueueOptions{ 40 | Name: "test", 41 | Storage: taskq.NewLocalStorage(), 42 | }) 43 | defer q.Close() 44 | 45 | task := taskq.RegisterTask(&taskq.TaskOptions{ 46 | Name: "test", 47 | Handler: func() {}, 48 | }) 49 | 50 | b.ResetTimer() 51 | 52 | b.RunParallel(func(pb *testing.PB) { 53 | for pb.Next() { 54 | msg := task.WithArgs(ctx) 55 | msg.Name = "myname" 56 | q.Add(msg) 57 | } 58 | }) 59 | } 60 | -------------------------------------------------------------------------------- /memqueue/factory.go: -------------------------------------------------------------------------------- 1 | package memqueue 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/vmihailenco/taskq/v3" 7 | "github.com/vmihailenco/taskq/v3/internal/base" 8 | ) 9 | 10 | type factory struct { 11 | base base.Factory 12 | } 13 | 14 | var _ taskq.Factory = (*factory)(nil) 15 | 16 | func NewFactory() taskq.Factory { 17 | return &factory{} 18 | } 19 | 20 | func (f *factory) RegisterQueue(opt *taskq.QueueOptions) taskq.Queue { 21 | q := NewQueue(opt) 22 | if err := f.base.Register(q); err != nil { 23 | panic(err) 24 | } 25 | return q 26 | } 27 | 28 | func (f *factory) Range(fn func(taskq.Queue) bool) { 29 | f.base.Range(fn) 30 | } 31 | 32 | func (f *factory) StartConsumers(ctx context.Context) error { 33 | return f.base.StartConsumers(ctx) 34 | } 35 | 36 | func (f *factory) StopConsumers() error { 37 | return f.base.StopConsumers() 38 | } 39 | 40 | func (f *factory) Close() error { 41 | return f.base.Close() 42 | } 43 | -------------------------------------------------------------------------------- /memqueue/memqueue_test.go: -------------------------------------------------------------------------------- 1 | package memqueue_test 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "sync" 10 | "sync/atomic" 11 | "testing" 12 | "time" 13 | 14 | . "github.com/onsi/ginkgo" 15 | . "github.com/onsi/gomega" 16 | uuid "github.com/satori/go.uuid" 17 | 18 | "github.com/vmihailenco/taskq/v3" 19 | "github.com/vmihailenco/taskq/v3/memqueue" 20 | ) 21 | 22 | func TestMemqueue(t *testing.T) { 23 | RegisterFailHandler(Fail) 24 | RunSpecs(t, "memqueue") 25 | } 26 | 27 | var _ = BeforeSuite(func() { 28 | taskq.SetLogger(log.New(ioutil.Discard, "", 0)) 29 | }) 30 | 31 | var _ = BeforeEach(func() { 32 | taskq.Tasks.Reset() 33 | }) 34 | 35 | var _ = Describe("message with args", func() { 36 | ctx := context.Background() 37 | ch := make(chan bool, 10) 38 | 39 | BeforeEach(func() { 40 | q := memqueue.NewQueue(&taskq.QueueOptions{ 41 | Name: "test", 42 | Storage: taskq.NewLocalStorage(), 43 | }) 44 | task := taskq.RegisterTask(&taskq.TaskOptions{ 45 | Name: "test", 46 | Handler: func(s string, i int) { 47 | Expect(s).To(Equal("string")) 48 | Expect(i).To(Equal(42)) 49 | ch <- true 50 | }, 51 | }) 52 | err := q.Add(task.WithArgs(ctx, "string", 42)) 53 | Expect(err).NotTo(HaveOccurred()) 54 | 55 | err = q.Close() 56 | Expect(err).NotTo(HaveOccurred()) 57 | }) 58 | 59 | It("handler is called with args", func() { 60 | Expect(ch).To(Receive()) 61 | Expect(ch).NotTo(Receive()) 62 | }) 63 | }) 64 | 65 | var _ = Describe("context.Context", func() { 66 | ctx := context.Background() 67 | ch := make(chan bool, 10) 68 | 69 | BeforeEach(func() { 70 | q := memqueue.NewQueue(&taskq.QueueOptions{ 71 | Name: "test", 72 | Storage: taskq.NewLocalStorage(), 73 | }) 74 | task := taskq.RegisterTask(&taskq.TaskOptions{ 75 | Name: "test", 76 | Handler: func(c context.Context, s string, i int) { 77 | Expect(s).To(Equal("string")) 78 | Expect(i).To(Equal(42)) 79 | ch <- true 80 | }, 81 | }) 82 | err := q.Add(task.WithArgs(ctx, "string", 42)) 83 | Expect(err).NotTo(HaveOccurred()) 84 | 85 | err = q.Close() 86 | Expect(err).NotTo(HaveOccurred()) 87 | }) 88 | 89 | It("handler is called with args", func() { 90 | Expect(ch).To(Receive()) 91 | Expect(ch).NotTo(Receive()) 92 | }) 93 | }) 94 | 95 | var _ = Describe("message with invalid number of args", func() { 96 | ctx := context.Background() 97 | ch := make(chan bool, 10) 98 | 99 | BeforeEach(func() { 100 | q := memqueue.NewQueue(&taskq.QueueOptions{ 101 | Name: "test", 102 | Storage: taskq.NewLocalStorage(), 103 | }) 104 | task := taskq.RegisterTask(&taskq.TaskOptions{ 105 | Name: "test", 106 | Handler: func(s string) { 107 | ch <- true 108 | }, 109 | RetryLimit: 1, 110 | }) 111 | q.Consumer().Stop() 112 | 113 | err := q.Add(task.WithArgs(ctx)) 114 | Expect(err).NotTo(HaveOccurred()) 115 | 116 | err = q.Consumer().ProcessOne(ctx) 117 | Expect(err).To(MatchError("taskq: got 0 args, wanted 1")) 118 | 119 | err = q.Consumer().ProcessAll(ctx) 120 | Expect(err).NotTo(HaveOccurred()) 121 | 122 | err = q.Close() 123 | Expect(err).NotTo(HaveOccurred()) 124 | }) 125 | 126 | It("handler is not called", func() { 127 | Expect(ch).NotTo(Receive()) 128 | }) 129 | }) 130 | 131 | var _ = Describe("HandlerFunc", func() { 132 | ctx := context.Background() 133 | ch := make(chan bool, 10) 134 | 135 | BeforeEach(func() { 136 | q := memqueue.NewQueue(&taskq.QueueOptions{ 137 | Name: "test", 138 | Storage: taskq.NewLocalStorage(), 139 | }) 140 | task := taskq.RegisterTask(&taskq.TaskOptions{ 141 | Name: "test", 142 | Handler: func(msg *taskq.Message) error { 143 | Expect(msg.Args).To(Equal([]interface{}{"string", 42})) 144 | ch <- true 145 | return nil 146 | }, 147 | }) 148 | 149 | err := q.Add(task.WithArgs(ctx, "string", 42)) 150 | Expect(err).NotTo(HaveOccurred()) 151 | 152 | err = q.Close() 153 | Expect(err).NotTo(HaveOccurred()) 154 | }) 155 | 156 | It("is called with Message", func() { 157 | Expect(ch).To(Receive()) 158 | Expect(ch).NotTo(Receive()) 159 | }) 160 | }) 161 | 162 | var _ = Describe("message retry timing", func() { 163 | ctx := context.Background() 164 | var q *memqueue.Queue 165 | var task *taskq.Task 166 | backoff := 100 * time.Millisecond 167 | var count int 168 | var ch chan time.Time 169 | 170 | BeforeEach(func() { 171 | count = 0 172 | ch = make(chan time.Time, 10) 173 | q = memqueue.NewQueue(&taskq.QueueOptions{ 174 | Name: "test", 175 | Storage: taskq.NewLocalStorage(), 176 | }) 177 | task = taskq.RegisterTask(&taskq.TaskOptions{ 178 | Name: "test", 179 | Handler: func() error { 180 | ch <- time.Now() 181 | count++ 182 | return fmt.Errorf("fake error #%d", count) 183 | }, 184 | RetryLimit: 3, 185 | MinBackoff: backoff, 186 | }) 187 | }) 188 | 189 | Context("without delay", func() { 190 | var now time.Time 191 | 192 | BeforeEach(func() { 193 | now = time.Now() 194 | _ = q.Add(task.WithArgs(ctx)) 195 | 196 | err := q.Close() 197 | Expect(err).NotTo(HaveOccurred()) 198 | }) 199 | 200 | It("is retried in time", func() { 201 | Expect(ch).To(Receive(BeTemporally("~", now, backoff/10))) 202 | Expect(ch).To(Receive(BeTemporally("~", now.Add(backoff), backoff/10))) 203 | Expect(ch).To(Receive(BeTemporally("~", now.Add(3*backoff), backoff/10))) 204 | Expect(ch).NotTo(Receive()) 205 | }) 206 | }) 207 | 208 | Context("message with delay", func() { 209 | var now time.Time 210 | 211 | BeforeEach(func() { 212 | msg := task.WithArgs(ctx) 213 | msg.Delay = 5 * backoff 214 | now = time.Now().Add(msg.Delay) 215 | 216 | q.Add(msg) 217 | 218 | err := q.Close() 219 | Expect(err).NotTo(HaveOccurred()) 220 | }) 221 | 222 | It("is retried in time", func() { 223 | Expect(ch).To(Receive(BeTemporally("~", now, backoff/10))) 224 | Expect(ch).To(Receive(BeTemporally("~", now.Add(backoff), backoff/10))) 225 | Expect(ch).To(Receive(BeTemporally("~", now.Add(3*backoff), backoff/10))) 226 | Expect(ch).NotTo(Receive()) 227 | }) 228 | }) 229 | }) 230 | 231 | var _ = Describe("failing queue with error handler", func() { 232 | ctx := context.Background() 233 | var q *memqueue.Queue 234 | ch := make(chan bool, 10) 235 | 236 | BeforeEach(func() { 237 | q = memqueue.NewQueue(&taskq.QueueOptions{ 238 | Name: "test", 239 | Storage: taskq.NewLocalStorage(), 240 | }) 241 | task := taskq.RegisterTask(&taskq.TaskOptions{ 242 | Name: "test", 243 | Handler: func() error { 244 | return errors.New("fake error") 245 | }, 246 | FallbackHandler: func() { 247 | ch <- true 248 | }, 249 | RetryLimit: 1, 250 | }) 251 | q.Add(task.WithArgs(ctx)) 252 | 253 | err := q.Close() 254 | Expect(err).NotTo(HaveOccurred()) 255 | }) 256 | 257 | It("error handler is called when handler fails", func() { 258 | Expect(ch).To(Receive()) 259 | Expect(ch).NotTo(Receive()) 260 | }) 261 | }) 262 | 263 | var _ = Describe("named message", func() { 264 | ctx := context.Background() 265 | var count int64 266 | 267 | BeforeEach(func() { 268 | q := memqueue.NewQueue(&taskq.QueueOptions{ 269 | Name: "test", 270 | Storage: taskq.NewLocalStorage(), 271 | }) 272 | task := taskq.RegisterTask(&taskq.TaskOptions{ 273 | Name: "test", 274 | Handler: func() { 275 | atomic.AddInt64(&count, 1) 276 | }, 277 | }) 278 | 279 | name := uuid.NewV4().String() 280 | 281 | var wg sync.WaitGroup 282 | for i := 0; i < 100; i++ { 283 | wg.Add(1) 284 | go func() { 285 | defer GinkgoRecover() 286 | defer wg.Done() 287 | msg := task.WithArgs(ctx) 288 | msg.Name = name 289 | q.Add(msg) 290 | }() 291 | } 292 | wg.Wait() 293 | 294 | err := q.Close() 295 | Expect(err).NotTo(HaveOccurred()) 296 | }) 297 | 298 | It("is processed once", func() { 299 | n := atomic.LoadInt64(&count) 300 | Expect(n).To(Equal(int64(1))) 301 | }) 302 | }) 303 | 304 | var _ = Describe("CallOnce", func() { 305 | ctx := context.Background() 306 | var now time.Time 307 | delay := 3 * time.Second 308 | ch := make(chan time.Time, 10) 309 | 310 | BeforeEach(func() { 311 | now = time.Now() 312 | 313 | q := memqueue.NewQueue(&taskq.QueueOptions{ 314 | Name: "test", 315 | Storage: taskq.NewLocalStorage(), 316 | }) 317 | task := taskq.RegisterTask(&taskq.TaskOptions{ 318 | Name: "test", 319 | Handler: func() error { 320 | ch <- time.Now() 321 | return nil 322 | }, 323 | }) 324 | 325 | var wg sync.WaitGroup 326 | for i := 0; i < 10; i++ { 327 | wg.Add(1) 328 | go func() { 329 | defer GinkgoRecover() 330 | defer wg.Done() 331 | 332 | msg := task.WithArgs(ctx) 333 | msg.OnceInPeriod(delay) 334 | 335 | q.Add(msg) 336 | }() 337 | } 338 | wg.Wait() 339 | 340 | err := q.Close() 341 | Expect(err).NotTo(HaveOccurred()) 342 | }) 343 | 344 | It("processes message once with delay", func() { 345 | Eventually(ch).Should(Receive(BeTemporally(">", now.Add(delay), time.Second))) 346 | Consistently(ch).ShouldNot(Receive()) 347 | }) 348 | }) 349 | 350 | var _ = Describe("stress testing", func() { 351 | const n = 10000 352 | ctx := context.Background() 353 | var count int64 354 | 355 | BeforeEach(func() { 356 | q := memqueue.NewQueue(&taskq.QueueOptions{ 357 | Name: "test", 358 | Storage: taskq.NewLocalStorage(), 359 | }) 360 | task := taskq.RegisterTask(&taskq.TaskOptions{ 361 | Name: "test", 362 | Handler: func() { 363 | atomic.AddInt64(&count, 1) 364 | }, 365 | }) 366 | 367 | for i := 0; i < n; i++ { 368 | q.Add(task.WithArgs(ctx)) 369 | } 370 | 371 | err := q.Close() 372 | Expect(err).NotTo(HaveOccurred()) 373 | }) 374 | 375 | It("handler is called for all messages", func() { 376 | nn := atomic.LoadInt64(&count) 377 | Expect(nn).To(Equal(int64(n))) 378 | }) 379 | }) 380 | 381 | var _ = Describe("stress testing failing queue", func() { 382 | const n = 100000 383 | ctx := context.Background() 384 | var errorCount int64 385 | 386 | BeforeEach(func() { 387 | q := memqueue.NewQueue(&taskq.QueueOptions{ 388 | Name: "test", 389 | PauseErrorsThreshold: -1, 390 | Storage: taskq.NewLocalStorage(), 391 | }) 392 | task := taskq.RegisterTask(&taskq.TaskOptions{ 393 | Name: "test", 394 | Handler: func() error { 395 | return errors.New("fake error") 396 | }, 397 | FallbackHandler: func() { 398 | atomic.AddInt64(&errorCount, 1) 399 | }, 400 | RetryLimit: 1, 401 | }) 402 | 403 | for i := 0; i < n; i++ { 404 | q.Add(task.WithArgs(ctx)) 405 | } 406 | 407 | err := q.Close() 408 | Expect(err).NotTo(HaveOccurred()) 409 | }) 410 | 411 | It("error handler is called for all messages", func() { 412 | nn := atomic.LoadInt64(&errorCount) 413 | Expect(nn).To(Equal(int64(n))) 414 | }) 415 | }) 416 | 417 | var _ = Describe("empty queue", func() { 418 | ctx := context.Background() 419 | var q *memqueue.Queue 420 | var task *taskq.Task 421 | var processed uint32 422 | 423 | BeforeEach(func() { 424 | processed = 0 425 | q = memqueue.NewQueue(&taskq.QueueOptions{ 426 | Name: "test", 427 | Storage: taskq.NewLocalStorage(), 428 | }) 429 | task = taskq.RegisterTask(&taskq.TaskOptions{ 430 | Name: "test", 431 | Handler: func() { 432 | atomic.AddUint32(&processed, 1) 433 | }, 434 | }) 435 | }) 436 | 437 | AfterEach(func() { 438 | _ = q.Close() 439 | }) 440 | 441 | It("can be closed", func() { 442 | err := q.Close() 443 | Expect(err).NotTo(HaveOccurred()) 444 | }) 445 | 446 | It("stops processor", func() { 447 | err := q.Consumer().Stop() 448 | Expect(err).NotTo(HaveOccurred()) 449 | }) 450 | 451 | testEmptyQueue := func() { 452 | It("processes all messages", func() { 453 | err := q.Consumer().ProcessAll(ctx) 454 | Expect(err).NotTo(HaveOccurred()) 455 | }) 456 | 457 | It("processes one message", func() { 458 | err := q.Consumer().ProcessOne(ctx) 459 | Expect(err).To(MatchError("taskq: queue is empty")) 460 | 461 | err = q.Consumer().ProcessAll(ctx) 462 | Expect(err).NotTo(HaveOccurred()) 463 | }) 464 | } 465 | 466 | Context("when processor is stopped", func() { 467 | BeforeEach(func() { 468 | err := q.Consumer().Stop() 469 | Expect(err).NotTo(HaveOccurred()) 470 | }) 471 | 472 | testEmptyQueue() 473 | 474 | Context("when there are messages in the queue", func() { 475 | BeforeEach(func() { 476 | for i := 0; i < 3; i++ { 477 | err := q.Add(task.WithArgs(ctx)) 478 | Expect(err).NotTo(HaveOccurred()) 479 | } 480 | }) 481 | 482 | It("processes all messages", func() { 483 | p := q.Consumer() 484 | 485 | err := p.ProcessAll(ctx) 486 | Expect(err).NotTo(HaveOccurred()) 487 | 488 | n := atomic.LoadUint32(&processed) 489 | Expect(n).To(Equal(uint32(3))) 490 | }) 491 | 492 | It("processes one message", func() { 493 | p := q.Consumer() 494 | 495 | err := p.ProcessOne(ctx) 496 | Expect(err).NotTo(HaveOccurred()) 497 | 498 | n := atomic.LoadUint32(&processed) 499 | Expect(n).To(Equal(uint32(1))) 500 | 501 | err = p.ProcessAll(ctx) 502 | Expect(err).NotTo(HaveOccurred()) 503 | 504 | n = atomic.LoadUint32(&processed) 505 | Expect(n).To(Equal(uint32(3))) 506 | }) 507 | }) 508 | }) 509 | }) 510 | 511 | // slot splits time into equal periods (called slots) and returns 512 | // slot number for provided time. 513 | func slot(period time.Duration) int64 { 514 | tm := time.Now() 515 | periodSec := int64(period / time.Second) 516 | if periodSec == 0 { 517 | return tm.Unix() 518 | } 519 | return tm.Unix() / periodSec 520 | } 521 | -------------------------------------------------------------------------------- /memqueue/queue.go: -------------------------------------------------------------------------------- 1 | package memqueue 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "sync" 8 | "sync/atomic" 9 | "time" 10 | 11 | "github.com/vmihailenco/taskq/v3" 12 | "github.com/vmihailenco/taskq/v3/internal" 13 | "github.com/vmihailenco/taskq/v3/internal/msgutil" 14 | ) 15 | 16 | type scheduler struct { 17 | timerLock sync.Mutex 18 | timerMap map[*taskq.Message]*time.Timer 19 | } 20 | 21 | func (q *scheduler) Schedule(msg *taskq.Message, fn func()) { 22 | q.timerLock.Lock() 23 | defer q.timerLock.Unlock() 24 | 25 | timer := time.AfterFunc(msg.Delay, func() { 26 | // Remove our entry from the map 27 | q.timerLock.Lock() 28 | delete(q.timerMap, msg) 29 | q.timerLock.Unlock() 30 | 31 | fn() 32 | }) 33 | 34 | if q.timerMap == nil { 35 | q.timerMap = make(map[*taskq.Message]*time.Timer) 36 | } 37 | q.timerMap[msg] = timer 38 | } 39 | 40 | func (q *scheduler) Remove(msg *taskq.Message) { 41 | q.timerLock.Lock() 42 | defer q.timerLock.Unlock() 43 | 44 | timer, ok := q.timerMap[msg] 45 | if ok { 46 | timer.Stop() 47 | delete(q.timerMap, msg) 48 | } 49 | } 50 | 51 | func (q *scheduler) Purge() int { 52 | q.timerLock.Lock() 53 | defer q.timerLock.Unlock() 54 | 55 | // Stop all delayed items 56 | for _, timer := range q.timerMap { 57 | timer.Stop() 58 | } 59 | 60 | n := len(q.timerMap) 61 | q.timerMap = nil 62 | 63 | return n 64 | } 65 | 66 | //------------------------------------------------------------------------------ 67 | 68 | const ( 69 | stateRunning = 0 70 | stateClosing = 1 71 | stateClosed = 2 72 | ) 73 | 74 | type Queue struct { 75 | opt *taskq.QueueOptions 76 | 77 | sync bool 78 | noDelay bool 79 | 80 | wg sync.WaitGroup 81 | consumer *taskq.Consumer 82 | 83 | scheduler scheduler 84 | 85 | _state int32 86 | } 87 | 88 | var _ taskq.Queue = (*Queue)(nil) 89 | 90 | func NewQueue(opt *taskq.QueueOptions) *Queue { 91 | return NewQueueMaybeConsumer(true, opt) 92 | } 93 | 94 | func NewQueueMaybeConsumer(startConsumer bool, opt *taskq.QueueOptions) *Queue { 95 | opt.Init() 96 | 97 | q := &Queue{ 98 | opt: opt, 99 | } 100 | 101 | q.consumer = taskq.NewConsumer(q) 102 | if startConsumer { 103 | if err := q.consumer.Start(context.Background()); err != nil { 104 | panic(err) 105 | } 106 | } 107 | 108 | return q 109 | } 110 | 111 | func (q *Queue) Name() string { 112 | return q.opt.Name 113 | } 114 | 115 | func (q *Queue) String() string { 116 | return fmt.Sprintf("queue=%q", q.Name()) 117 | } 118 | 119 | func (q *Queue) Options() *taskq.QueueOptions { 120 | return q.opt 121 | } 122 | 123 | func (q *Queue) Consumer() taskq.QueueConsumer { 124 | return q.consumer 125 | } 126 | 127 | func (q *Queue) SetSync(sync bool) { 128 | q.sync = sync 129 | } 130 | 131 | func (q *Queue) SetNoDelay(noDelay bool) { 132 | q.noDelay = noDelay 133 | } 134 | 135 | // Close is like CloseTimeout with 30 seconds timeout. 136 | func (q *Queue) Close() error { 137 | return q.CloseTimeout(30 * time.Second) 138 | } 139 | 140 | // CloseTimeout closes the queue waiting for pending messages to be processed. 141 | func (q *Queue) CloseTimeout(timeout time.Duration) error { 142 | if !atomic.CompareAndSwapInt32(&q._state, stateRunning, stateClosing) { 143 | return fmt.Errorf("taskq: %s is already closed", q) 144 | } 145 | err := q.WaitTimeout(timeout) 146 | 147 | if !atomic.CompareAndSwapInt32(&q._state, stateClosing, stateClosed) { 148 | panic("not reached") 149 | } 150 | 151 | _ = q.consumer.StopTimeout(timeout) 152 | _ = q.Purge() 153 | 154 | return err 155 | } 156 | 157 | func (q *Queue) WaitTimeout(timeout time.Duration) error { 158 | done := make(chan struct{}, 1) 159 | go func() { 160 | q.wg.Wait() 161 | done <- struct{}{} 162 | }() 163 | 164 | select { 165 | case <-done: 166 | case <-time.After(timeout): 167 | return fmt.Errorf("taskq: %s: messages are not processed after %s", q.consumer, timeout) 168 | } 169 | 170 | return nil 171 | } 172 | 173 | func (q *Queue) Len() (int, error) { 174 | return q.consumer.Len(), nil 175 | } 176 | 177 | // Add adds message to the queue. 178 | func (q *Queue) Add(msg *taskq.Message) error { 179 | if q.closed() { 180 | return fmt.Errorf("taskq: %s is closed", q) 181 | } 182 | if msg.TaskName == "" { 183 | return internal.ErrTaskNameRequired 184 | } 185 | if q.isDuplicate(msg) { 186 | msg.Err = taskq.ErrDuplicate 187 | return nil 188 | } 189 | q.wg.Add(1) 190 | return q.enqueueMessage(msg) 191 | } 192 | 193 | func (q *Queue) enqueueMessage(msg *taskq.Message) error { 194 | if (q.noDelay || q.sync) && msg.Delay > 0 { 195 | msg.Delay = 0 196 | } 197 | msg.ReservedCount++ 198 | 199 | if q.sync { 200 | return q.consumer.Process(msg) 201 | } 202 | 203 | if msg.Delay > 0 { 204 | q.scheduler.Schedule(msg, func() { 205 | // If the queue closed while we were waiting, just return 206 | if q.closed() { 207 | q.wg.Done() 208 | return 209 | } 210 | msg.Delay = 0 211 | _ = q.consumer.Add(msg) 212 | }) 213 | return nil 214 | } 215 | return q.consumer.Add(msg) 216 | } 217 | 218 | func (q *Queue) ReserveN(_ context.Context, _ int, _ time.Duration) ([]taskq.Message, error) { 219 | return nil, internal.ErrNotSupported 220 | } 221 | 222 | func (q *Queue) Release(msg *taskq.Message) error { 223 | // Shallow copy. 224 | clone := *msg 225 | clone.Err = nil 226 | return q.enqueueMessage(&clone) 227 | } 228 | 229 | func (q *Queue) Delete(msg *taskq.Message) error { 230 | q.scheduler.Remove(msg) 231 | q.wg.Done() 232 | return nil 233 | } 234 | 235 | func (q *Queue) DeleteBatch(msgs []*taskq.Message) error { 236 | if len(msgs) == 0 { 237 | return errors.New("taskq: no messages to delete") 238 | } 239 | for _, msg := range msgs { 240 | if err := q.Delete(msg); err != nil { 241 | return err 242 | } 243 | } 244 | return nil 245 | } 246 | 247 | func (q *Queue) Purge() error { 248 | // Purge any messages already in the consumer 249 | err := q.consumer.Purge() 250 | 251 | numPurged := q.scheduler.Purge() 252 | for i := 0; i < numPurged; i++ { 253 | q.wg.Done() 254 | } 255 | 256 | return err 257 | } 258 | 259 | func (q *Queue) closed() bool { 260 | return atomic.LoadInt32(&q._state) == stateClosed 261 | } 262 | 263 | func (q *Queue) isDuplicate(msg *taskq.Message) bool { 264 | if msg.Name == "" { 265 | return false 266 | } 267 | return q.opt.Storage.Exists(msg.Ctx, msgutil.FullMessageName(q, msg)) 268 | } 269 | -------------------------------------------------------------------------------- /message.go: -------------------------------------------------------------------------------- 1 | package taskq 2 | 3 | import ( 4 | "context" 5 | "encoding" 6 | "encoding/binary" 7 | "errors" 8 | "fmt" 9 | "time" 10 | 11 | "github.com/klauspost/compress/s2" 12 | "github.com/klauspost/compress/zstd" 13 | "github.com/vmihailenco/msgpack/v5" 14 | 15 | "github.com/vmihailenco/taskq/v3/internal" 16 | ) 17 | 18 | // ErrDuplicate is returned when adding duplicate message to the queue. 19 | var ErrDuplicate = errors.New("taskq: message with such name already exists") 20 | 21 | // Message is used to create and retrieve messages from a queue. 22 | type Message struct { 23 | Ctx context.Context `msgpack:"-"` 24 | 25 | // SQS/IronMQ message id. 26 | ID string `msgpack:"1,omitempty,alias:ID"` 27 | 28 | // Optional name for the message. Messages with the same name 29 | // are processed only once. 30 | Name string `msgpack:"-"` 31 | 32 | // Delay specifies the duration the queue must wait 33 | // before executing the message. 34 | Delay time.Duration `msgpack:"-"` 35 | 36 | // Args passed to the handler. 37 | Args []interface{} `msgpack:"-"` 38 | 39 | // Binary representation of the args. 40 | ArgsCompression string `msgpack:"2,omitempty,alias:ArgsCompression"` 41 | ArgsBin []byte `msgpack:"3,alias:ArgsBin"` 42 | 43 | // SQS/IronMQ reservation id that is used to release/delete the message. 44 | ReservationID string `msgpack:"-"` 45 | 46 | // The number of times the message has been reserved or released. 47 | ReservedCount int `msgpack:"4,omitempty,alias:ReservedCount"` 48 | 49 | TaskName string `msgpack:"5,alias:TaskName"` 50 | Err error `msgpack:"-"` 51 | 52 | evt *ProcessMessageEvent 53 | marshalBinaryCache []byte 54 | } 55 | 56 | func NewMessage(ctx context.Context, args ...interface{}) *Message { 57 | return &Message{ 58 | Ctx: ctx, 59 | Args: args, 60 | } 61 | } 62 | 63 | func (m *Message) String() string { 64 | return fmt.Sprintf("Message", 65 | m.ID, m.Name, m.ReservedCount) 66 | } 67 | 68 | // SetDelay sets the message delay. 69 | func (m *Message) SetDelay(delay time.Duration) { 70 | m.Delay = delay 71 | } 72 | 73 | // OnceInPeriod uses the period and the args to generate such a message name 74 | // that message with such args is added to the queue once in a given period. 75 | // If args are not provided then message args are used instead. 76 | func (m *Message) OnceInPeriod(period time.Duration, args ...interface{}) { 77 | m.setNameFromArgs(period, args...) 78 | m.SetDelay(period) 79 | } 80 | 81 | func (m *Message) OnceWithDelay(delay time.Duration) { 82 | m.setNameFromArgs(0) 83 | if delay > 0 { 84 | m.SetDelay(delay) 85 | } 86 | } 87 | 88 | func (m *Message) OnceWithSchedule(tm time.Time) { 89 | m.OnceWithDelay(time.Until(tm)) 90 | } 91 | 92 | func (m *Message) setNameFromArgs(period time.Duration, args ...interface{}) { 93 | var b []byte 94 | if len(args) > 0 { 95 | b, _ = msgpack.Marshal(args) 96 | } else { 97 | b, _ = m.MarshalArgs() 98 | } 99 | if period > 0 { 100 | b = appendTimeSlot(b, period) 101 | } 102 | m.Name = internal.BytesToString(b) 103 | } 104 | 105 | var zdec, _ = zstd.NewReader(nil) 106 | 107 | func (m *Message) decompress() ([]byte, error) { 108 | switch m.ArgsCompression { 109 | case "": 110 | return m.ArgsBin, nil 111 | case "zstd": 112 | return zdec.DecodeAll(m.ArgsBin, nil) 113 | case "s2": 114 | return s2.Decode(nil, m.ArgsBin) 115 | default: 116 | return nil, fmt.Errorf("taskq: unsupported compression=%s", m.ArgsCompression) 117 | } 118 | } 119 | 120 | func (m *Message) MarshalArgs() ([]byte, error) { 121 | if m.ArgsBin != nil { 122 | if m.ArgsCompression == "" { 123 | return m.ArgsBin, nil 124 | } 125 | if m.Args == nil { 126 | return m.decompress() 127 | } 128 | } 129 | 130 | b, err := msgpack.Marshal(m.Args) 131 | if err != nil { 132 | return nil, err 133 | } 134 | m.ArgsBin = b 135 | 136 | return b, nil 137 | } 138 | 139 | type messageRaw Message 140 | 141 | var _ encoding.BinaryMarshaler = (*Message)(nil) 142 | 143 | func (m *Message) MarshalBinary() ([]byte, error) { 144 | if m.TaskName == "" { 145 | return nil, internal.ErrTaskNameRequired 146 | } 147 | if m.marshalBinaryCache != nil { 148 | return m.marshalBinaryCache, nil 149 | } 150 | 151 | _, err := m.MarshalArgs() 152 | if err != nil { 153 | return nil, err 154 | } 155 | 156 | if m.ArgsCompression == "" && len(m.ArgsBin) >= 512 { 157 | compressed := s2.Encode(nil, m.ArgsBin) 158 | if len(compressed) < len(m.ArgsBin) { 159 | m.ArgsCompression = "s2" 160 | m.ArgsBin = compressed 161 | } 162 | } 163 | 164 | b, err := msgpack.Marshal((*messageRaw)(m)) 165 | if err != nil { 166 | return nil, err 167 | } 168 | 169 | m.marshalBinaryCache = b 170 | return b, nil 171 | } 172 | 173 | var _ encoding.BinaryUnmarshaler = (*Message)(nil) 174 | 175 | func (m *Message) UnmarshalBinary(b []byte) error { 176 | if err := msgpack.Unmarshal(b, (*messageRaw)(m)); err != nil { 177 | return err 178 | } 179 | 180 | b, err := m.decompress() 181 | if err != nil { 182 | return err 183 | } 184 | 185 | m.ArgsCompression = "" 186 | m.ArgsBin = b 187 | 188 | return nil 189 | } 190 | 191 | func appendTimeSlot(b []byte, period time.Duration) []byte { 192 | l := len(b) 193 | b = append(b, make([]byte, 16)...) 194 | binary.LittleEndian.PutUint64(b[l:], uint64(period)) 195 | binary.LittleEndian.PutUint64(b[l+8:], uint64(timeSlot(period))) 196 | return b 197 | } 198 | 199 | func timeSlot(period time.Duration) int64 { 200 | if period <= 0 { 201 | return 0 202 | } 203 | return time.Now().UnixNano() / int64(period) 204 | } 205 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "taskq", 3 | "version": "3.2.9" 4 | } 5 | -------------------------------------------------------------------------------- /queue.go: -------------------------------------------------------------------------------- 1 | package taskq 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "runtime" 7 | "time" 8 | 9 | "github.com/go-redis/redis_rate/v9" 10 | ) 11 | 12 | type QueueOptions struct { 13 | // Queue name. 14 | Name string 15 | 16 | // Minimum number of goroutines processing messages. 17 | // Default is 1. 18 | MinNumWorker int32 19 | // Maximum number of goroutines processing messages. 20 | // Default is 32 * number of CPUs. 21 | MaxNumWorker int32 22 | // Global limit of concurrently running workers across all servers. 23 | // Overrides MaxNumWorker. 24 | WorkerLimit int32 25 | // Maximum number of goroutines fetching messages. 26 | // Default is 8 * number of CPUs. 27 | MaxNumFetcher int32 28 | 29 | // Number of messages reserved by a fetcher in the queue in one request. 30 | // Default is 10 messages. 31 | ReservationSize int 32 | // Time after which the reserved message is returned to the queue. 33 | // Default is 5 minutes. 34 | ReservationTimeout time.Duration 35 | // Time that a long polling receive call waits for a message to become 36 | // available before returning an empty response. 37 | // Default is 10 seconds. 38 | WaitTimeout time.Duration 39 | // Size of the buffer where reserved messages are stored. 40 | // Default is the same as ReservationSize. 41 | BufferSize int 42 | 43 | // Number of consecutive failures after which queue processing is paused. 44 | // Default is 100 failures. 45 | PauseErrorsThreshold int 46 | 47 | // Processing rate limit. 48 | RateLimit redis_rate.Limit 49 | 50 | // Optional rate limiter. The default is to use Redis. 51 | RateLimiter *redis_rate.Limiter 52 | 53 | // Redis client that is used for storing metadata. 54 | Redis Redis 55 | 56 | // Optional storage interface. The default is to use Redis. 57 | Storage Storage 58 | 59 | // Optional message handler. The default is the global Tasks registry. 60 | Handler Handler 61 | 62 | inited bool 63 | 64 | // ConsumerIdleTimeout Time after which the consumer need to be deleted. 65 | // Default is 6 hour 66 | ConsumerIdleTimeout time.Duration 67 | 68 | // SchedulerBackoffTime is the time of backoff for the scheduler( 69 | // Scheduler was designed to clean zombie Consumer and requeue pending msgs, and so on. 70 | // Default is randomly between 1~1.5s 71 | // We can change it to a bigger value so that it won't slowdown the redis when using redis queue. 72 | // It will be between SchedulerBackoffTime and SchedulerBackoffTime+250ms. 73 | SchedulerBackoffTime time.Duration 74 | } 75 | 76 | func (opt *QueueOptions) Init() { 77 | if opt.inited { 78 | return 79 | } 80 | opt.inited = true 81 | 82 | if opt.Name == "" { 83 | panic("QueueOptions.Name is required") 84 | } 85 | 86 | if opt.WorkerLimit > 0 { 87 | opt.MinNumWorker = opt.WorkerLimit 88 | opt.MaxNumWorker = opt.WorkerLimit 89 | } else { 90 | if opt.MinNumWorker == 0 { 91 | opt.MinNumWorker = 1 92 | } 93 | if opt.MaxNumWorker == 0 { 94 | opt.MaxNumWorker = 32 * int32(runtime.NumCPU()) 95 | } 96 | } 97 | if opt.MaxNumFetcher == 0 { 98 | opt.MaxNumFetcher = 8 * int32(runtime.NumCPU()) 99 | } 100 | 101 | switch opt.PauseErrorsThreshold { 102 | case -1: 103 | opt.PauseErrorsThreshold = 0 104 | case 0: 105 | opt.PauseErrorsThreshold = 100 106 | } 107 | 108 | if opt.ReservationSize == 0 { 109 | opt.ReservationSize = 10 110 | } 111 | if opt.ReservationTimeout == 0 { 112 | opt.ReservationTimeout = 5 * time.Minute 113 | } 114 | if opt.BufferSize == 0 { 115 | opt.BufferSize = opt.ReservationSize 116 | } 117 | if opt.WaitTimeout == 0 { 118 | opt.WaitTimeout = 10 * time.Second 119 | } 120 | 121 | if opt.ConsumerIdleTimeout == 0 { 122 | opt.ConsumerIdleTimeout = 6 * time.Hour 123 | } 124 | 125 | if opt.Storage == nil { 126 | opt.Storage = newRedisStorage(opt.Redis) 127 | } 128 | 129 | if !opt.RateLimit.IsZero() && opt.RateLimiter == nil && opt.Redis != nil { 130 | opt.RateLimiter = redis_rate.NewLimiter(opt.Redis) 131 | } 132 | 133 | if opt.Handler == nil { 134 | opt.Handler = &Tasks 135 | } 136 | } 137 | 138 | //------------------------------------------------------------------------------ 139 | 140 | type Queue interface { 141 | fmt.Stringer 142 | Name() string 143 | Options() *QueueOptions 144 | Consumer() QueueConsumer 145 | 146 | Len() (int, error) 147 | Add(msg *Message) error 148 | ReserveN(ctx context.Context, n int, waitTimeout time.Duration) ([]Message, error) 149 | Release(msg *Message) error 150 | Delete(msg *Message) error 151 | Purge() error 152 | Close() error 153 | CloseTimeout(timeout time.Duration) error 154 | } 155 | 156 | // QueueConsumer reserves messages from the queue, processes them, 157 | // and then either releases or deletes messages from the queue. 158 | type QueueConsumer interface { 159 | // AddHook adds a hook into message processing. 160 | AddHook(hook ConsumerHook) 161 | Queue() Queue 162 | Options() *QueueOptions 163 | Len() int 164 | // Stats returns processor stats. 165 | Stats() *ConsumerStats 166 | Add(msg *Message) error 167 | // Start starts consuming messages in the queue. 168 | Start(ctx context.Context) error 169 | // Stop is StopTimeout with 30 seconds timeout. 170 | Stop() error 171 | // StopTimeout waits workers for timeout duration to finish processing current 172 | // messages and stops workers. 173 | StopTimeout(timeout time.Duration) error 174 | // ProcessAll starts workers to process messages in the queue and then stops 175 | // them when all messages are processed. 176 | ProcessAll(ctx context.Context) error 177 | // ProcessOne processes at most one message in the queue. 178 | ProcessOne(ctx context.Context) error 179 | // Process is low-level API to process message bypassing the internal queue. 180 | Process(msg *Message) error 181 | Put(msg *Message) 182 | // Purge discards messages from the internal queue. 183 | Purge() error 184 | String() string 185 | } 186 | -------------------------------------------------------------------------------- /redisq/factory.go: -------------------------------------------------------------------------------- 1 | package redisq 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/vmihailenco/taskq/v3" 7 | "github.com/vmihailenco/taskq/v3/internal/base" 8 | ) 9 | 10 | type factory struct { 11 | base base.Factory 12 | } 13 | 14 | var _ taskq.Factory = (*factory)(nil) 15 | 16 | func NewFactory() taskq.Factory { 17 | return &factory{} 18 | } 19 | 20 | func (f *factory) RegisterQueue(opt *taskq.QueueOptions) taskq.Queue { 21 | q := NewQueue(opt) 22 | if err := f.base.Register(q); err != nil { 23 | panic(err) 24 | } 25 | return q 26 | } 27 | 28 | func (f *factory) Range(fn func(taskq.Queue) bool) { 29 | f.base.Range(fn) 30 | } 31 | 32 | func (f *factory) StartConsumers(ctx context.Context) error { 33 | return f.base.StartConsumers(ctx) 34 | } 35 | 36 | func (f *factory) StopConsumers() error { 37 | return f.base.StopConsumers() 38 | } 39 | 40 | func (f *factory) Close() error { 41 | return f.base.Close() 42 | } 43 | -------------------------------------------------------------------------------- /redisq/queue.go: -------------------------------------------------------------------------------- 1 | package redisq 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math/rand" 7 | "os" 8 | "strconv" 9 | "strings" 10 | "sync" 11 | "sync/atomic" 12 | "time" 13 | 14 | "github.com/go-redis/redis/v8" 15 | "github.com/google/uuid" 16 | 17 | "github.com/bsm/redislock" 18 | 19 | "github.com/vmihailenco/taskq/v3" 20 | "github.com/vmihailenco/taskq/v3/internal" 21 | "github.com/vmihailenco/taskq/v3/internal/msgutil" 22 | ) 23 | 24 | const batchSize = 100 25 | 26 | type RedisStreamClient interface { 27 | Del(ctx context.Context, keys ...string) *redis.IntCmd 28 | TxPipeline() redis.Pipeliner 29 | 30 | XAdd(ctx context.Context, a *redis.XAddArgs) *redis.StringCmd 31 | XDel(ctx context.Context, stream string, ids ...string) *redis.IntCmd 32 | XLen(ctx context.Context, stream string) *redis.IntCmd 33 | XRangeN(ctx context.Context, stream, start, stop string, count int64) *redis.XMessageSliceCmd 34 | XGroupCreateMkStream(ctx context.Context, stream, group, start string) *redis.StatusCmd 35 | XReadGroup(ctx context.Context, a *redis.XReadGroupArgs) *redis.XStreamSliceCmd 36 | XAck(ctx context.Context, stream, group string, ids ...string) *redis.IntCmd 37 | XPendingExt(ctx context.Context, a *redis.XPendingExtArgs) *redis.XPendingExtCmd 38 | XTrim(ctx context.Context, key string, maxLen int64) *redis.IntCmd 39 | XGroupDelConsumer(ctx context.Context, stream, group, consumer string) *redis.IntCmd 40 | 41 | ZAdd(ctx context.Context, key string, members ...*redis.Z) *redis.IntCmd 42 | ZRangeByScore(ctx context.Context, key string, opt *redis.ZRangeBy) *redis.StringSliceCmd 43 | ZRem(ctx context.Context, key string, members ...interface{}) *redis.IntCmd 44 | XInfoConsumers(ctx context.Context, key string, group string) *redis.XInfoConsumersCmd 45 | } 46 | 47 | type Queue struct { 48 | opt *taskq.QueueOptions 49 | 50 | consumer *taskq.Consumer 51 | 52 | redis RedisStreamClient 53 | wg sync.WaitGroup 54 | 55 | zset string 56 | stream string 57 | streamGroup string 58 | streamConsumer string 59 | schedulerLockPrefix string 60 | 61 | _closed uint32 62 | } 63 | 64 | var _ taskq.Queue = (*Queue)(nil) 65 | 66 | func NewQueue(opt *taskq.QueueOptions) *Queue { 67 | const redisPrefix = "taskq:" 68 | 69 | if opt.WaitTimeout == 0 { 70 | opt.WaitTimeout = time.Second 71 | } 72 | opt.Init() 73 | if opt.Redis == nil { 74 | panic(fmt.Errorf("redisq: Redis client is required")) 75 | } 76 | red, ok := opt.Redis.(RedisStreamClient) 77 | if !ok { 78 | panic(fmt.Errorf("redisq: Redis client must support streams")) 79 | } 80 | 81 | q := &Queue{ 82 | opt: opt, 83 | 84 | redis: red, 85 | 86 | zset: redisPrefix + "{" + opt.Name + "}:zset", 87 | stream: redisPrefix + "{" + opt.Name + "}:stream", 88 | streamGroup: "taskq", 89 | streamConsumer: consumer(), 90 | schedulerLockPrefix: redisPrefix + opt.Name + ":scheduler-lock:", 91 | } 92 | 93 | q.wg.Add(1) 94 | go func() { 95 | defer q.wg.Done() 96 | q.scheduler("delayed", q.scheduleDelayed) 97 | }() 98 | 99 | q.wg.Add(1) 100 | go func() { 101 | defer q.wg.Done() 102 | q.scheduler("pending", q.schedulePending) 103 | }() 104 | 105 | q.wg.Add(1) 106 | go func() { 107 | defer q.wg.Done() 108 | q.scheduler("clean_zombie_consumers", q.cleanZombieConsumers) 109 | }() 110 | 111 | return q 112 | } 113 | 114 | func consumer() string { 115 | s, _ := os.Hostname() 116 | s += ":pid:" + strconv.Itoa(os.Getpid()) 117 | s += ":" + strconv.Itoa(rand.Int()) 118 | return s 119 | } 120 | 121 | func (q *Queue) Name() string { 122 | return q.opt.Name 123 | } 124 | 125 | func (q *Queue) String() string { 126 | return fmt.Sprintf("queue=%q", q.Name()) 127 | } 128 | 129 | func (q *Queue) Options() *taskq.QueueOptions { 130 | return q.opt 131 | } 132 | 133 | func (q *Queue) Consumer() taskq.QueueConsumer { 134 | if q.consumer == nil { 135 | q.consumer = taskq.NewConsumer(q) 136 | } 137 | return q.consumer 138 | } 139 | 140 | func (q *Queue) Len() (int, error) { 141 | n, err := q.redis.XLen(context.TODO(), q.stream).Result() 142 | return int(n), err 143 | } 144 | 145 | // Add adds message to the queue. 146 | func (q *Queue) Add(msg *taskq.Message) error { 147 | return q.add(q.redis, msg) 148 | } 149 | 150 | func (q *Queue) add(pipe RedisStreamClient, msg *taskq.Message) error { 151 | if msg.TaskName == "" { 152 | return internal.ErrTaskNameRequired 153 | } 154 | if q.isDuplicate(msg) { 155 | msg.Err = taskq.ErrDuplicate 156 | return nil 157 | } 158 | 159 | if msg.ID == "" { 160 | u := uuid.New() 161 | msg.ID = internal.BytesToString(u[:]) 162 | } 163 | 164 | body, err := msg.MarshalBinary() 165 | if err != nil { 166 | return err 167 | } 168 | 169 | if msg.Delay > 0 { 170 | tm := time.Now().Add(msg.Delay) 171 | return pipe.ZAdd(msg.Ctx, q.zset, &redis.Z{ 172 | Score: float64(unixMs(tm)), 173 | Member: body, 174 | }).Err() 175 | } 176 | 177 | return pipe.XAdd(msg.Ctx, &redis.XAddArgs{ 178 | Stream: q.stream, 179 | Values: map[string]interface{}{ 180 | "body": body, 181 | }, 182 | }).Err() 183 | } 184 | 185 | func (q *Queue) ReserveN( 186 | ctx context.Context, n int, waitTimeout time.Duration, 187 | ) ([]taskq.Message, error) { 188 | streams, err := q.redis.XReadGroup(ctx, &redis.XReadGroupArgs{ 189 | Streams: []string{q.stream, ">"}, 190 | Group: q.streamGroup, 191 | Consumer: q.streamConsumer, 192 | Count: int64(n), 193 | Block: waitTimeout, 194 | }).Result() 195 | if err != nil { 196 | if err == redis.Nil { // timeout 197 | return nil, nil 198 | } 199 | if strings.HasPrefix(err.Error(), "NOGROUP") { 200 | q.createStreamGroup(ctx) 201 | return q.ReserveN(ctx, n, waitTimeout) 202 | } 203 | return nil, err 204 | } 205 | 206 | stream := &streams[0] 207 | msgs := make([]taskq.Message, len(stream.Messages)) 208 | for i := range stream.Messages { 209 | xmsg := &stream.Messages[i] 210 | msg := &msgs[i] 211 | msg.Ctx = ctx 212 | 213 | err = unmarshalMessage(msg, xmsg) 214 | if err != nil { 215 | msg.Err = err 216 | } 217 | } 218 | 219 | return msgs, nil 220 | } 221 | 222 | func (q *Queue) createStreamGroup(ctx context.Context) { 223 | _ = q.redis.XGroupCreateMkStream(ctx, q.stream, q.streamGroup, "0").Err() 224 | } 225 | 226 | func (q *Queue) Release(msg *taskq.Message) error { 227 | // Make the delete and re-queue operation atomic in case we crash midway 228 | // and lose a message. 229 | pipe := q.redis.TxPipeline() 230 | // When Release a msg, ack it before we delete msg. 231 | if err := pipe.XAck(msg.Ctx, q.stream, q.streamGroup, msg.ID).Err(); err != nil { 232 | return err 233 | } 234 | 235 | err := pipe.XDel(msg.Ctx, q.stream, msg.ID).Err() 236 | if err != nil { 237 | return err 238 | } 239 | 240 | msg.ReservedCount++ 241 | err = q.add(pipe, msg) 242 | if err != nil { 243 | return err 244 | } 245 | 246 | _, err = pipe.Exec(msg.Ctx) 247 | return err 248 | } 249 | 250 | // Delete deletes the message from the queue. 251 | func (q *Queue) Delete(msg *taskq.Message) error { 252 | if err := q.redis.XAck(msg.Ctx, q.stream, q.streamGroup, msg.ID).Err(); err != nil { 253 | return err 254 | } 255 | return q.redis.XDel(msg.Ctx, q.stream, msg.ID).Err() 256 | } 257 | 258 | // Purge deletes all messages from the queue. 259 | func (q *Queue) Purge() error { 260 | ctx := context.TODO() 261 | _ = q.redis.Del(ctx, q.zset).Err() 262 | _ = q.redis.XTrim(ctx, q.stream, 0).Err() 263 | return nil 264 | } 265 | 266 | // Close is like CloseTimeout with 30 seconds timeout. 267 | func (q *Queue) Close() error { 268 | return q.CloseTimeout(30 * time.Second) 269 | } 270 | 271 | // CloseTimeout closes the queue waiting for pending messages to be processed. 272 | func (q *Queue) CloseTimeout(timeout time.Duration) error { 273 | if !atomic.CompareAndSwapUint32(&q._closed, 0, 1) { 274 | return nil 275 | } 276 | 277 | if q.consumer != nil { 278 | _ = q.consumer.StopTimeout(timeout) 279 | } 280 | 281 | _ = q.redis.XGroupDelConsumer( 282 | context.TODO(), q.stream, q.streamGroup, q.streamConsumer).Err() 283 | 284 | return nil 285 | } 286 | 287 | func (q *Queue) closed() bool { 288 | return atomic.LoadUint32(&q._closed) == 1 289 | } 290 | 291 | func (q *Queue) scheduler(name string, fn func(ctx context.Context) (int, error)) { 292 | for { 293 | if q.closed() { 294 | break 295 | } 296 | 297 | ctx := context.TODO() 298 | 299 | var n int 300 | err := q.withRedisLock(ctx, q.schedulerLockPrefix+name, func(ctx context.Context) error { 301 | var err error 302 | n, err = fn(ctx) 303 | return err 304 | }) 305 | if err != nil && err != redislock.ErrNotObtained { 306 | internal.Logger.Printf("redisq: %s failed: %s", name, err) 307 | } 308 | if err != nil || n == 0 { 309 | time.Sleep(q.schedulerBackoff()) 310 | } 311 | } 312 | } 313 | 314 | func (q *Queue) schedulerBackoff() time.Duration { 315 | n := rand.Intn(500) 316 | if q.opt.SchedulerBackoffTime > 0 { 317 | return q.opt.SchedulerBackoffTime + time.Duration(n)*time.Millisecond 318 | } 319 | return time.Duration(n+1000) * time.Millisecond 320 | } 321 | 322 | func (q *Queue) scheduleDelayed(ctx context.Context) (int, error) { 323 | tm := time.Now() 324 | max := strconv.FormatInt(unixMs(tm), 10) 325 | bodies, err := q.redis.ZRangeByScore(ctx, q.zset, &redis.ZRangeBy{ 326 | Min: "-inf", 327 | Max: max, 328 | Count: batchSize, 329 | }).Result() 330 | if err != nil { 331 | return 0, err 332 | } 333 | 334 | pipe := q.redis.TxPipeline() 335 | for _, body := range bodies { 336 | pipe.XAdd(ctx, &redis.XAddArgs{ 337 | Stream: q.stream, 338 | Values: map[string]interface{}{ 339 | "body": body, 340 | }, 341 | }) 342 | pipe.ZRem(ctx, q.zset, body) 343 | } 344 | _, err = pipe.Exec(ctx) 345 | if err != nil { 346 | return 0, err 347 | } 348 | 349 | return len(bodies), nil 350 | } 351 | 352 | func (q *Queue) cleanZombieConsumers(ctx context.Context) (int, error) { 353 | consumers, err := q.redis.XInfoConsumers(ctx, q.stream, q.streamGroup).Result() 354 | if err != nil { 355 | return 0, err 356 | } 357 | 358 | for _, consumer := range consumers { 359 | // skip the current stream consumer 360 | if consumer.Name == q.streamConsumer { 361 | continue 362 | } 363 | 364 | if time.Duration(consumer.Idle)*time.Millisecond > q.opt.ConsumerIdleTimeout { 365 | _ = q.redis.XGroupDelConsumer(ctx, q.stream, q.streamGroup, consumer.Name).Err() 366 | } 367 | } 368 | return 0, nil 369 | } 370 | 371 | // schedulePending schedules pending messages that are older than the `ReservationTimeout`. 372 | // ReservationTimeout is the time after which a message is considered to be not processed and need to be re-enqueue. 373 | func (q *Queue) schedulePending(ctx context.Context) (int, error) { 374 | tm := time.Now().Add(-q.opt.ReservationTimeout) 375 | end := strconv.FormatInt(unixMs(tm), 10) 376 | 377 | pending, err := q.redis.XPendingExt(ctx, &redis.XPendingExtArgs{ 378 | Stream: q.stream, 379 | Group: q.streamGroup, 380 | Start: "-", 381 | End: end, 382 | Count: batchSize, 383 | }).Result() 384 | if err != nil { 385 | if strings.HasPrefix(err.Error(), "NOGROUP") { 386 | q.createStreamGroup(ctx) 387 | 388 | return 0, nil 389 | } 390 | return 0, err 391 | } 392 | 393 | for i := range pending { 394 | xmsgInfo := &pending[i] 395 | id := xmsgInfo.ID 396 | 397 | xmsgs, err := q.redis.XRangeN(ctx, q.stream, id, id, 1).Result() 398 | if err != nil { 399 | return 0, err 400 | } 401 | 402 | if len(xmsgs) != 1 { 403 | err := fmt.Errorf("redisq: can't find pending message id=%q in stream=%q", 404 | id, q.stream) 405 | return 0, err 406 | } 407 | 408 | xmsg := &xmsgs[0] 409 | msg := new(taskq.Message) 410 | msg.Ctx = ctx 411 | err = unmarshalMessage(msg, xmsg) 412 | if err != nil { 413 | return 0, err 414 | } 415 | 416 | err = q.Release(msg) 417 | if err != nil { 418 | return 0, err 419 | } 420 | } 421 | 422 | return len(pending), nil 423 | } 424 | 425 | func (q *Queue) isDuplicate(msg *taskq.Message) bool { 426 | if msg.Name == "" { 427 | return false 428 | } 429 | exists := q.opt.Storage.Exists(msg.Ctx, msgutil.FullMessageName(q, msg)) 430 | return exists 431 | } 432 | 433 | func (q *Queue) withRedisLock( 434 | ctx context.Context, name string, fn func(ctx context.Context) error, 435 | ) error { 436 | lock, err := redislock.Obtain(ctx, q.opt.Redis, name, time.Minute, nil) 437 | if err != nil { 438 | return err 439 | } 440 | 441 | defer func() { 442 | if err := lock.Release(ctx); err != nil { 443 | internal.Logger.Printf("redislock.Release failed: %s", err) 444 | } 445 | }() 446 | 447 | return fn(ctx) 448 | } 449 | 450 | func unixMs(tm time.Time) int64 { 451 | return tm.UnixNano() / int64(time.Millisecond) 452 | } 453 | 454 | func unmarshalMessage(msg *taskq.Message, xmsg *redis.XMessage) error { 455 | body := xmsg.Values["body"].(string) 456 | err := msg.UnmarshalBinary(internal.StringToBytes(body)) 457 | if err != nil { 458 | return err 459 | } 460 | 461 | msg.ID = xmsg.ID 462 | if msg.ReservedCount == 0 { 463 | msg.ReservedCount = 1 464 | } 465 | 466 | return nil 467 | } 468 | -------------------------------------------------------------------------------- /redisq_test.go: -------------------------------------------------------------------------------- 1 | package taskq_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/vmihailenco/taskq/v3" 8 | "github.com/vmihailenco/taskq/v3/redisq" 9 | ) 10 | 11 | func redisqFactory() taskq.Factory { 12 | return redisq.NewFactory() 13 | } 14 | 15 | func TestRedisqConsumer(t *testing.T) { 16 | testConsumer(t, redisqFactory(), &taskq.QueueOptions{ 17 | Name: queueName("redisq-consumer"), 18 | }) 19 | } 20 | 21 | func TestRedisqAckMessage(t *testing.T) { 22 | testConsumerDelete(t, redisqFactory(), &taskq.QueueOptions{ 23 | Name: queueName("redisq-ack-message"), 24 | ReservationTimeout: 1 * time.Second, 25 | }) 26 | } 27 | 28 | func TestRedisqUnknownTask(t *testing.T) { 29 | testUnknownTask(t, redisqFactory(), &taskq.QueueOptions{ 30 | Name: queueName("redisq-unknown-task"), 31 | }) 32 | } 33 | 34 | func TestRedisqFallback(t *testing.T) { 35 | testFallback(t, redisqFactory(), &taskq.QueueOptions{ 36 | Name: queueName("redisq-fallback"), 37 | }) 38 | } 39 | 40 | func TestRedisqDelay(t *testing.T) { 41 | testDelay(t, redisqFactory(), &taskq.QueueOptions{ 42 | Name: queueName("redisq-delay"), 43 | }) 44 | } 45 | 46 | func TestRedisqRetry(t *testing.T) { 47 | testRetry(t, redisqFactory(), &taskq.QueueOptions{ 48 | Name: queueName("redisq-retry"), 49 | }) 50 | } 51 | 52 | func TestRedisqNamedMessage(t *testing.T) { 53 | testNamedMessage(t, redisqFactory(), &taskq.QueueOptions{ 54 | Name: queueName("redisq-named-message"), 55 | }) 56 | } 57 | 58 | func TestRedisqCallOnce(t *testing.T) { 59 | testCallOnce(t, redisqFactory(), &taskq.QueueOptions{ 60 | Name: queueName("redisq-call-once"), 61 | }) 62 | } 63 | 64 | func TestRedisqLen(t *testing.T) { 65 | testLen(t, redisqFactory(), &taskq.QueueOptions{ 66 | Name: queueName("redisq-queue-len"), 67 | }) 68 | } 69 | 70 | func TestRedisqRateLimit(t *testing.T) { 71 | testRateLimit(t, redisqFactory(), &taskq.QueueOptions{ 72 | Name: queueName("redisq-rate-limit"), 73 | }) 74 | } 75 | 76 | func TestRedisqErrorDelay(t *testing.T) { 77 | testErrorDelay(t, redisqFactory(), &taskq.QueueOptions{ 78 | Name: queueName("redisq-delayer"), 79 | }) 80 | } 81 | 82 | func TestRedisqWorkerLimit(t *testing.T) { 83 | testWorkerLimit(t, redisqFactory(), &taskq.QueueOptions{ 84 | Name: queueName("redisq-worker-limit"), 85 | }) 86 | } 87 | 88 | func TestRedisqBatchConsumerSmallMessage(t *testing.T) { 89 | testBatchConsumer(t, redisqFactory(), &taskq.QueueOptions{ 90 | Name: queueName("redisq-batch-consumer-small-message"), 91 | }, 100) 92 | } 93 | 94 | func TestRedisqBatchConsumerLarge(t *testing.T) { 95 | testBatchConsumer(t, redisqFactory(), &taskq.QueueOptions{ 96 | Name: queueName("redisq-batch-processor-large-message"), 97 | }, 64000) 98 | } 99 | -------------------------------------------------------------------------------- /registry.go: -------------------------------------------------------------------------------- 1 | package taskq 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | var Tasks TaskMap 10 | 11 | type TaskMap struct { 12 | m sync.Map 13 | } 14 | 15 | func (r *TaskMap) Get(name string) *Task { 16 | if v, ok := r.m.Load(name); ok { 17 | return v.(*Task) 18 | } 19 | if v, ok := r.m.Load("*"); ok { 20 | return v.(*Task) 21 | } 22 | return nil 23 | } 24 | 25 | func (r *TaskMap) Register(opt *TaskOptions) (*Task, error) { 26 | opt.init() 27 | 28 | task := &Task{ 29 | opt: opt, 30 | handler: NewHandler(opt.Handler), 31 | } 32 | 33 | if opt.FallbackHandler != nil { 34 | task.fallbackHandler = NewHandler(opt.FallbackHandler) 35 | } 36 | 37 | name := task.Name() 38 | _, loaded := r.m.LoadOrStore(name, task) 39 | if loaded { 40 | return nil, fmt.Errorf("task=%q already exists", name) 41 | } 42 | return task, nil 43 | } 44 | 45 | func (r *TaskMap) Unregister(task *Task) { 46 | r.m.Delete(task.Name()) 47 | } 48 | 49 | func (r *TaskMap) Reset() { 50 | r.m = sync.Map{} 51 | } 52 | 53 | func (r *TaskMap) Range(fn func(name string, task *Task) bool) { 54 | r.m.Range(func(key, value interface{}) bool { 55 | return fn(key.(string), value.(*Task)) 56 | }) 57 | } 58 | 59 | func (r *TaskMap) HandleMessage(msg *Message) error { 60 | task := r.Get(msg.TaskName) 61 | if task == nil { 62 | msg.Delay = r.delay(msg, nil, unknownTaskOpt) 63 | return fmt.Errorf("taskq: unknown task=%q", msg.TaskName) 64 | } 65 | 66 | opt := task.Options() 67 | if opt.DeferFunc != nil { 68 | defer opt.DeferFunc() 69 | } 70 | 71 | msgErr := task.HandleMessage(msg) 72 | if msgErr == nil { 73 | return nil 74 | } 75 | 76 | msg.Delay = r.delay(msg, msgErr, opt) 77 | return msgErr 78 | } 79 | 80 | func (r *TaskMap) delay(msg *Message, msgErr error, opt *TaskOptions) time.Duration { 81 | if msg.ReservedCount >= opt.RetryLimit { 82 | return 0 83 | } 84 | if delayer, ok := msgErr.(Delayer); ok { 85 | return delayer.Delay() 86 | } 87 | return exponentialBackoff(opt.MinBackoff, opt.MaxBackoff, msg.ReservedCount) 88 | } 89 | -------------------------------------------------------------------------------- /scripts/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | help() { 6 | cat <<- EOF 7 | Usage: TAG=tag $0 8 | 9 | Updates version in go.mod files and pushes a new brash to GitHub. 10 | 11 | VARIABLES: 12 | TAG git tag, for example, v1.0.0 13 | EOF 14 | exit 0 15 | } 16 | 17 | if [ -z "$TAG" ] 18 | then 19 | printf "TAG is required\n\n" 20 | help 21 | fi 22 | 23 | TAG_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\-[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?(\\+[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?$" 24 | if ! [[ "${TAG}" =~ ${TAG_REGEX} ]]; then 25 | printf "TAG is not valid: ${TAG}\n\n" 26 | exit 1 27 | fi 28 | 29 | TAG_FOUND=`git tag --list ${TAG}` 30 | if [[ ${TAG_FOUND} = ${TAG} ]] ; then 31 | printf "tag ${TAG} already exists\n\n" 32 | exit 1 33 | fi 34 | 35 | if ! git diff --quiet 36 | then 37 | printf "working tree is not clean\n\n" 38 | git status 39 | exit 1 40 | fi 41 | 42 | git checkout v3 43 | 44 | PACKAGE_DIRS=$(find . -mindepth 2 -type f -name 'go.mod' -exec dirname {} \; \ 45 | | sed 's/^\.\///' \ 46 | | sort) 47 | 48 | for dir in $PACKAGE_DIRS 49 | do 50 | sed --in-place \ 51 | "s/vmihailenco\/taskq\([^ ]*\) v.*/vmihailenco\/taskq\1 ${TAG}/" "${dir}/go.mod" 52 | done 53 | 54 | for dir in $PACKAGE_DIRS 55 | do 56 | printf "${dir}: go get -d && go mod tidy\n" 57 | (cd ./${dir} && go get -d && go mod tidy) 58 | done 59 | 60 | sed --in-place "s/\(return \)\"[^\"]*\"/\1\"${TAG#v}\"/" ./version.go 61 | sed --in-place "s/\(\"version\": \)\"[^\"]*\"/\1\"${TAG#v}\"/" ./package.json 62 | 63 | conventional-changelog -p angular -i CHANGELOG.md -s 64 | 65 | git checkout -b release/${TAG} v3 66 | git add -u 67 | git commit -m "chore: release $TAG (release.sh)" 68 | git push origin release/${TAG} 69 | -------------------------------------------------------------------------------- /scripts/tag.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | help() { 6 | cat <<- EOF 7 | Usage: TAG=tag $0 8 | 9 | Creates git tags for public Go packages. 10 | 11 | VARIABLES: 12 | TAG git tag, for example, v1.0.0 13 | EOF 14 | exit 0 15 | } 16 | 17 | if [ -z "$TAG" ] 18 | then 19 | printf "TAG env var is required\n\n"; 20 | help 21 | fi 22 | 23 | if ! grep -Fq "\"${TAG#v}\"" version.go 24 | then 25 | printf "version.go does not contain ${TAG#v}\n" 26 | exit 1 27 | fi 28 | 29 | PACKAGE_DIRS=$(find . -mindepth 2 -type f -name 'go.mod' -exec dirname {} \; \ 30 | | grep -E -v "example|internal" \ 31 | | sed 's/^\.\///' \ 32 | | sort) 33 | 34 | git tag ${TAG} 35 | git push origin ${TAG} 36 | 37 | for dir in $PACKAGE_DIRS 38 | do 39 | printf "tagging ${dir}/${TAG}\n" 40 | git tag ${dir}/${TAG} 41 | git push origin ${dir}/${TAG} 42 | done 43 | -------------------------------------------------------------------------------- /storage.go: -------------------------------------------------------------------------------- 1 | package taskq 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | 8 | "github.com/hashicorp/golang-lru/simplelru" 9 | ) 10 | 11 | type Storage interface { 12 | Exists(ctx context.Context, key string) bool 13 | } 14 | 15 | var _ Storage = (*localStorage)(nil) 16 | var _ Storage = (*redisStorage)(nil) 17 | 18 | // LOCAL 19 | 20 | type localStorage struct { 21 | mu sync.Mutex 22 | cache *simplelru.LRU 23 | } 24 | 25 | func NewLocalStorage() Storage { 26 | return &localStorage{} 27 | } 28 | 29 | func (s *localStorage) Exists(_ context.Context, key string) bool { 30 | s.mu.Lock() 31 | defer s.mu.Unlock() 32 | 33 | if s.cache == nil { 34 | var err error 35 | s.cache, err = simplelru.NewLRU(128000, nil) 36 | if err != nil { 37 | panic(err) 38 | } 39 | } 40 | 41 | _, ok := s.cache.Get(key) 42 | if ok { 43 | return true 44 | } 45 | 46 | s.cache.Add(key, nil) 47 | return false 48 | } 49 | 50 | // REDIS 51 | 52 | type redisStorage struct { 53 | redis Redis 54 | } 55 | 56 | func newRedisStorage(redis Redis) Storage { 57 | return &redisStorage{ 58 | redis: redis, 59 | } 60 | } 61 | 62 | func (s *redisStorage) Exists(ctx context.Context, key string) bool { 63 | val, err := s.redis.SetNX(ctx, key, "", 24*time.Hour).Result() 64 | if err != nil { 65 | return true 66 | } 67 | return !val 68 | } 69 | -------------------------------------------------------------------------------- /sysinfo_linux.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package taskq 5 | 6 | import ( 7 | "runtime" 8 | 9 | "github.com/capnm/sysinfo" 10 | ) 11 | 12 | func hasFreeSystemResources() bool { 13 | si := sysinfo.Get() 14 | free := si.FreeRam + si.BufferRam 15 | 16 | // at least 200MB of RAM is free 17 | if free < 2e5 { 18 | return false 19 | } 20 | 21 | // at least 5% of RAM is free 22 | if float64(free)/float64(si.TotalRam) < 0.05 { 23 | return false 24 | } 25 | 26 | // avg load is not too high 27 | if si.Loads[0] > 1.5*float64(runtime.NumCPU()) { 28 | return false 29 | } 30 | 31 | return true 32 | } 33 | -------------------------------------------------------------------------------- /sysinfo_other.go: -------------------------------------------------------------------------------- 1 | //go:build !linux 2 | // +build !linux 3 | 4 | package taskq 5 | 6 | func hasFreeSystemResources() bool { 7 | return true 8 | } 9 | -------------------------------------------------------------------------------- /task.go: -------------------------------------------------------------------------------- 1 | package taskq 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | var unknownTaskOpt *TaskOptions 10 | 11 | func init() { 12 | SetUnknownTaskOptions(&TaskOptions{ 13 | Name: "unknown", 14 | }) 15 | } 16 | 17 | func SetUnknownTaskOptions(opt *TaskOptions) { 18 | opt.init() 19 | unknownTaskOpt = opt 20 | } 21 | 22 | type TaskOptions struct { 23 | // Task name. 24 | Name string 25 | 26 | // Function called to process a message. 27 | // There are three permitted types of signature: 28 | // 1. A zero-argument function 29 | // 2. A function whose arguments are assignable in type from those which are passed in the message 30 | // 3. A function which takes a single `*Message` argument 31 | // The handler function may also optionally take a Context as a first argument and may optionally return an error. 32 | // If the handler takes a Context, when it is invoked it will be passed the same Context as that which was passed to 33 | // `StartConsumer`. If the handler returns a non-nil error the message processing will fail and will be retried/. 34 | Handler interface{} 35 | // Function called to process failed message after the specified number of retries have all failed. 36 | // The FallbackHandler accepts the same types of function as the Handler. 37 | FallbackHandler interface{} 38 | 39 | // Optional function used by Consumer with defer statement 40 | // to recover from panics. 41 | DeferFunc func() 42 | 43 | // Number of tries/releases after which the message fails permanently 44 | // and is deleted. 45 | // Default is 64 retries. 46 | RetryLimit int 47 | // Minimum backoff time between retries. 48 | // Default is 30 seconds. 49 | MinBackoff time.Duration 50 | // Maximum backoff time between retries. 51 | // Default is 30 minutes. 52 | MaxBackoff time.Duration 53 | 54 | inited bool 55 | } 56 | 57 | func (opt *TaskOptions) init() { 58 | if opt.inited { 59 | return 60 | } 61 | opt.inited = true 62 | 63 | if opt.Name == "" { 64 | panic("TaskOptions.Name is required") 65 | } 66 | if opt.RetryLimit == 0 { 67 | opt.RetryLimit = 64 68 | } 69 | if opt.MinBackoff == 0 { 70 | opt.MinBackoff = 30 * time.Second 71 | } 72 | if opt.MaxBackoff == 0 { 73 | opt.MaxBackoff = 30 * time.Minute 74 | } 75 | } 76 | 77 | type Task struct { 78 | opt *TaskOptions 79 | 80 | handler Handler 81 | fallbackHandler Handler 82 | } 83 | 84 | func RegisterTask(opt *TaskOptions) *Task { 85 | task, err := Tasks.Register(opt) 86 | if err != nil { 87 | panic(err) 88 | } 89 | 90 | return task 91 | } 92 | 93 | func (t *Task) Name() string { 94 | return t.opt.Name 95 | } 96 | 97 | func (t *Task) String() string { 98 | return fmt.Sprintf("task=%q", t.Name()) 99 | } 100 | 101 | func (t *Task) Options() *TaskOptions { 102 | return t.opt 103 | } 104 | 105 | func (t *Task) HandleMessage(msg *Message) error { 106 | if msg.Err != nil { 107 | if t.fallbackHandler != nil { 108 | return t.fallbackHandler.HandleMessage(msg) 109 | } 110 | return nil 111 | } 112 | return t.handler.HandleMessage(msg) 113 | } 114 | 115 | func (t *Task) WithArgs(ctx context.Context, args ...interface{}) *Message { 116 | msg := NewMessage(ctx, args...) 117 | msg.TaskName = t.opt.Name 118 | return msg 119 | } 120 | -------------------------------------------------------------------------------- /taskq.go: -------------------------------------------------------------------------------- 1 | package taskq 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "os" 7 | "time" 8 | 9 | "github.com/go-redis/redis/v8" 10 | 11 | "github.com/vmihailenco/taskq/v3/internal" 12 | ) 13 | 14 | func init() { 15 | SetLogger(log.New(os.Stderr, "taskq: ", log.LstdFlags|log.Lshortfile)) 16 | } 17 | 18 | func SetLogger(logger *log.Logger) { 19 | internal.Logger = logger 20 | } 21 | 22 | // Factory is an interface that abstracts creation of new queues. 23 | // It is implemented in subpackages memqueue, azsqs, and ironmq. 24 | type Factory interface { 25 | RegisterQueue(*QueueOptions) Queue 26 | Range(func(Queue) bool) 27 | StartConsumers(context.Context) error 28 | StopConsumers() error 29 | Close() error 30 | } 31 | 32 | type Redis interface { 33 | Del(ctx context.Context, keys ...string) *redis.IntCmd 34 | SetNX(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.BoolCmd 35 | Pipelined(ctx context.Context, fn func(pipe redis.Pipeliner) error) ([]redis.Cmder, error) 36 | 37 | // Eval Required by redislock 38 | Eval(ctx context.Context, script string, keys []string, args ...interface{}) *redis.Cmd 39 | EvalSha(ctx context.Context, sha1 string, keys []string, args ...interface{}) *redis.Cmd 40 | ScriptExists(ctx context.Context, scripts ...string) *redis.BoolSliceCmd 41 | ScriptLoad(ctx context.Context, script string) *redis.StringCmd 42 | } 43 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package taskq 2 | 3 | // Version is the current release version. 4 | func Version() string { 5 | return "3.2.9" 6 | } 7 | --------------------------------------------------------------------------------