├── .github ├── FUNDING.yml ├── dependabot.yml └── workflows │ ├── codeql.yaml │ ├── go.yml │ └── goreleaser.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yaml ├── LICENSE ├── README.md ├── README.zh-cn.md ├── README.zh-tw.md ├── _example ├── example01 │ ├── go.mod │ ├── go.sum │ └── main.go └── example02 │ ├── go.mod │ ├── go.sum │ └── main.go ├── bearer.yml ├── benchmark_test.go ├── core └── worker.go ├── errors.go ├── go.mod ├── go.sum ├── images ├── _base.d2 ├── flow-01.d2 ├── flow-01.svg ├── flow-02.d2 ├── flow-02.svg ├── flow-03.d2 ├── flow-03.svg ├── nats.svg └── rabbitmq.svg ├── job ├── benchmark_test.go ├── job.go ├── job_test.go ├── option.go └── option_test.go ├── logger.go ├── logger_test.go ├── metric.go ├── metric_test.go ├── mocks ├── mock_queued_message.go ├── mock_task_message.go ├── mock_worker.go └── mocks.go ├── options.go ├── options_test.go ├── pool.go ├── pool_test.go ├── queue.go ├── queue_example_test.go ├── queue_test.go ├── recovery.go ├── ring.go ├── ring_test.go ├── thread.go └── thread_test.go /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 13 | custom: ['https://www.paypal.me/appleboy46'] 14 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: github-actions 4 | directory: / 5 | schedule: 6 | interval: weekly 7 | - package-ecosystem: gomod 8 | directory: / 9 | schedule: 10 | interval: weekly 11 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yaml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | name: "CodeQL" 7 | 8 | on: 9 | push: 10 | branches: [master] 11 | pull_request: 12 | # The branches below must be a subset of the branches above 13 | branches: [master] 14 | schedule: 15 | - cron: "30 1 * * 0" 16 | 17 | jobs: 18 | analyze: 19 | name: Analyze 20 | runs-on: ubuntu-latest 21 | 22 | permissions: 23 | # required for all workflows 24 | security-events: write 25 | 26 | # only required for workflows in private repositories 27 | actions: read 28 | contents: read 29 | 30 | strategy: 31 | fail-fast: false 32 | matrix: 33 | # Override automatic language detection by changing the below list 34 | # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] 35 | # TODO: Enable for javascript later 36 | language: ["go"] 37 | 38 | steps: 39 | - name: Checkout repository 40 | uses: actions/checkout@v4 41 | 42 | # Initializes the CodeQL tools for scanning. 43 | - name: Initialize CodeQL 44 | uses: github/codeql-action/init@v3 45 | with: 46 | languages: ${{ matrix.language }} 47 | # If you wish to specify custom queries, you can do so here or in a config file. 48 | # By default, queries listed here will override any specified in a config file. 49 | # Prefix the list here with "+" to use these queries and those in the config file. 50 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 51 | 52 | - name: Perform CodeQL Analysis 53 | uses: github/codeql-action/analyze@v3 54 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Run Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | 11 | permissions: 12 | actions: read 13 | contents: read 14 | statuses: write 15 | 16 | concurrency: 17 | group: ${{ github.workflow }}-${{ github.ref }} 18 | cancel-in-progress: true 19 | 20 | jobs: 21 | lint: 22 | runs-on: ubuntu-latest 23 | steps: 24 | - name: Checkout Code 25 | uses: actions/checkout@v4 26 | with: 27 | ref: ${{ github.ref }} 28 | - name: Setup go 29 | uses: actions/setup-go@v5 30 | with: 31 | go-version-file: "go.mod" 32 | check-latest: true 33 | - name: Setup golangci-lint 34 | uses: golangci/golangci-lint-action@v8 35 | with: 36 | version: v2.1 37 | args: --verbose 38 | 39 | - name: Bearer 40 | uses: bearer/bearer-action@v2 41 | with: 42 | diff: true 43 | 44 | test: 45 | strategy: 46 | matrix: 47 | os: [ubuntu-latest] 48 | go: [1.23, 1.24] 49 | include: 50 | - os: ubuntu-latest 51 | go-build: ~/.cache/go-build 52 | name: ${{ matrix.os }} @ Go ${{ matrix.go }} 53 | runs-on: ${{ matrix.os }} 54 | env: 55 | GO111MODULE: on 56 | GOPROXY: https://proxy.golang.org 57 | steps: 58 | - name: Set up Go ${{ matrix.go }} 59 | uses: actions/setup-go@v5 60 | with: 61 | go-version: ${{ matrix.go }} 62 | 63 | - name: Checkout Code 64 | uses: actions/checkout@v4 65 | with: 66 | ref: ${{ github.ref }} 67 | 68 | - uses: actions/cache@v4 69 | with: 70 | path: | 71 | ${{ matrix.go-build }} 72 | ~/go/pkg/mod 73 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 74 | restore-keys: | 75 | ${{ runner.os }}-go- 76 | - name: Run Tests 77 | run: | 78 | go test -race -v -covermode=atomic -coverprofile=coverage.out 79 | 80 | - name: Run Benchmark 81 | run: | 82 | go test -v -run=^$ -count 5 -benchmem -bench . ./... 83 | 84 | - name: Upload coverage to Codecov 85 | uses: codecov/codecov-action@v5 86 | with: 87 | flags: ${{ matrix.os }},go-${{ matrix.go }} 88 | -------------------------------------------------------------------------------- /.github/workflows/goreleaser.yml: -------------------------------------------------------------------------------- 1 | name: Goreleaser 2 | 3 | on: 4 | push: 5 | tags: 6 | - "*" 7 | 8 | permissions: 9 | contents: write 10 | 11 | jobs: 12 | goreleaser: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout 16 | uses: actions/checkout@v4 17 | with: 18 | fetch-depth: 0 19 | - name: Set up Go 20 | uses: actions/setup-go@v5 21 | with: 22 | go-version: "^1" 23 | - name: Run GoReleaser 24 | uses: goreleaser/goreleaser-action@v6 25 | with: 26 | # either 'goreleaser' (default) or 'goreleaser-pro' 27 | distribution: goreleaser 28 | version: latest 29 | args: release --clean 30 | env: 31 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | default: none 4 | enable: 5 | - bodyclose 6 | - dogsled 7 | - dupl 8 | - errcheck 9 | - exhaustive 10 | - gochecknoinits 11 | - goconst 12 | - gocritic 13 | - gocyclo 14 | - goprintffuncname 15 | - gosec 16 | - govet 17 | - ineffassign 18 | - lll 19 | - misspell 20 | - nakedret 21 | - noctx 22 | - nolintlint 23 | - staticcheck 24 | - unconvert 25 | - unparam 26 | - unused 27 | - whitespace 28 | exclusions: 29 | generated: lax 30 | presets: 31 | - comments 32 | - common-false-positives 33 | - legacy 34 | - std-error-handling 35 | paths: 36 | - third_party$ 37 | - builtin$ 38 | - examples$ 39 | formatters: 40 | enable: 41 | - gofmt 42 | - gofumpt 43 | - goimports 44 | exclusions: 45 | generated: lax 46 | paths: 47 | - third_party$ 48 | - builtin$ 49 | - examples$ 50 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | builds: 2 | - # If true, skip the build. 3 | # Useful for library projects. 4 | # Default is false 5 | skip: true 6 | 7 | changelog: 8 | # Set it to true if you wish to skip the changelog generation. 9 | # This may result in an empty release notes on GitHub/GitLab/Gitea. 10 | disable: false 11 | 12 | # Changelog generation implementation to use. 13 | # 14 | # Valid options are: 15 | # - `git`: uses `git log`; 16 | # - `github`: uses the compare GitHub API, appending the author login to the changelog. 17 | # - `gitlab`: uses the compare GitLab API, appending the author name and email to the changelog. 18 | # - `github-native`: uses the GitHub release notes generation API, disables the groups feature. 19 | # 20 | # Defaults to `git`. 21 | use: github 22 | 23 | # Sorts the changelog by the commit's messages. 24 | # Could either be asc, desc or empty 25 | # Default is empty 26 | sort: asc 27 | 28 | # Group commits messages by given regex and title. 29 | # Order value defines the order of the groups. 30 | # Proving no regex means all commits will be grouped under the default group. 31 | # Groups are disabled when using github-native, as it already groups things by itself. 32 | # 33 | # Default is no groups. 34 | groups: 35 | - title: Features 36 | regexp: "^.*feat[(\\w-)]*:+.*$" 37 | order: 0 38 | - title: "Bug fixes" 39 | regexp: "^.*fix[(\\w-)]*:+.*$" 40 | order: 1 41 | - title: "Enhancements" 42 | regexp: "^.*chore[(\\w-)]*:+.*$" 43 | order: 2 44 | - title: "Refactor" 45 | regexp: "^.*refactor[(\\w-)]*:+.*$" 46 | order: 3 47 | - title: "Build process updates" 48 | regexp: ^.*?(build|ci)(\(.+\))??!?:.+$ 49 | order: 4 50 | - title: "Documentation updates" 51 | regexp: ^.*?docs?(\(.+\))??!?:.+$ 52 | order: 4 53 | - title: Others 54 | order: 999 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Bo-Yi Wu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Queue 2 | 3 | [![CodeQL](https://github.com/golang-queue/queue/actions/workflows/codeql.yaml/badge.svg)](https://github.com/golang-queue/queue/actions/workflows/codeql.yaml) 4 | [![Run Tests](https://github.com/golang-queue/queue/actions/workflows/go.yml/badge.svg)](https://github.com/golang-queue/queue/actions/workflows/go.yml) 5 | [![codecov](https://codecov.io/gh/golang-queue/queue/branch/master/graph/badge.svg?token=SSo3mHejOE)](https://codecov.io/gh/golang-queue/queue) 6 | 7 | [繁體中文](./README.zh-tw.md) | [简体中文](./README.zh-cn.md) 8 | 9 | Queue is a Golang library designed to help you create and manage a pool of Goroutines (lightweight threads). It allows you to efficiently run multiple tasks in parallel, utilizing the full CPU capacity of your machine. 10 | 11 | ## Features 12 | 13 | - [x] Supports [Circular buffer](https://en.wikipedia.org/wiki/Circular_buffer) queues. 14 | - [x] Integrates with [NSQ](https://nsq.io/) for real-time distributed messaging. 15 | - [x] Integrates with [NATS](https://nats.io/) for adaptive edge and distributed systems. 16 | - [x] Integrates with [Redis Pub/Sub](https://redis.io/docs/manual/pubsub/). 17 | - [x] Integrates with [Redis Streams](https://redis.io/docs/manual/data-types/streams/). 18 | - [x] Integrates with [RabbitMQ](https://www.rabbitmq.com/). 19 | 20 | ## Queue Scenario 21 | 22 | A simple queue service using a ring buffer as the default backend. 23 | 24 | ![queue01](./images/flow-01.svg) 25 | 26 | Easily switch the queue service to use NSQ, NATS, or Redis. 27 | 28 | ![queue02](./images/flow-02.svg) 29 | 30 | Supports multiple producers and consumers. 31 | 32 | ![queue03](./images/flow-03.svg) 33 | 34 | ## Requirements 35 | 36 | Go version **1.22** or above 37 | 38 | ## Installation 39 | 40 | To install the stable version: 41 | 42 | ```sh 43 | go get github.com/golang-queue/queue 44 | ``` 45 | 46 | To install the latest version: 47 | 48 | ```sh 49 | go get github.com/golang-queue/queue@master 50 | ``` 51 | 52 | ## Usage 53 | 54 | ### Basic Usage of Pool (using the Task function) 55 | 56 | By calling the `QueueTask()` method, you can schedule tasks to be executed by workers (Goroutines) in the pool. 57 | 58 | ```go 59 | package main 60 | 61 | import ( 62 | "context" 63 | "fmt" 64 | "time" 65 | 66 | "github.com/golang-queue/queue" 67 | ) 68 | 69 | func main() { 70 | taskN := 100 71 | rets := make(chan string, taskN) 72 | 73 | // initialize the queue pool 74 | q := queue.NewPool(5) 75 | // shut down the service and notify all workers 76 | // wait until all jobs are complete 77 | defer q.Release() 78 | 79 | // assign tasks to the queue 80 | for i := 0; i < taskN; i++ { 81 | go func(i int) { 82 | if err := q.QueueTask(func(ctx context.Context) error { 83 | rets <- fmt.Sprintf("Hi Gopher, handle the job: %02d", +i) 84 | return nil 85 | }); err != nil { 86 | panic(err) 87 | } 88 | }(i) 89 | } 90 | 91 | // wait until all tasks are done 92 | for i := 0; i < taskN; i++ { 93 | fmt.Println("message:", <-rets) 94 | time.Sleep(20 * time.Millisecond) 95 | } 96 | } 97 | ``` 98 | 99 | ### Basic Usage of Pool (using a message queue) 100 | 101 | Define a new message struct and implement the `Bytes()` function to encode the message. Use the `WithFn` function to handle messages from the queue. 102 | 103 | ```go 104 | package main 105 | 106 | import ( 107 | "context" 108 | "encoding/json" 109 | "fmt" 110 | "log" 111 | "time" 112 | 113 | "github.com/golang-queue/queue" 114 | "github.com/golang-queue/queue/core" 115 | ) 116 | 117 | type job struct { 118 | Name string 119 | Message string 120 | } 121 | 122 | func (j *job) Bytes() []byte { 123 | b, err := json.Marshal(j) 124 | if err != nil { 125 | panic(err) 126 | } 127 | return b 128 | } 129 | 130 | func main() { 131 | taskN := 100 132 | rets := make(chan string, taskN) 133 | 134 | // initialize the queue pool 135 | q := queue.NewPool(5, queue.WithFn(func(ctx context.Context, m core.TaskMessage) error { 136 | var v job 137 | if err := json.Unmarshal(m.Payload(), &v); err != nil { 138 | return err 139 | } 140 | 141 | rets <- "Hi, " + v.Name + ", " + v.Message 142 | return nil 143 | })) 144 | // shut down the service and notify all workers 145 | // wait until all jobs are complete 146 | defer q.Release() 147 | 148 | // assign tasks to the queue 149 | for i := 0; i < taskN; i++ { 150 | go func(i int) { 151 | if err := q.Queue(&job{ 152 | Name: "Gopher", 153 | Message: fmt.Sprintf("handle the job: %d", i+1), 154 | }); err != nil { 155 | log.Println(err) 156 | } 157 | }(i) 158 | } 159 | 160 | // wait until all tasks are done 161 | for i := 0; i < taskN; i++ { 162 | fmt.Println("message:", <-rets) 163 | time.Sleep(50 * time.Millisecond) 164 | } 165 | } 166 | ``` 167 | 168 | ## Using NSQ as a Queue 169 | 170 | Refer to the [NSQ documentation](https://github.com/golang-queue/nsq) for more details. 171 | 172 | ```go 173 | package main 174 | 175 | import ( 176 | "context" 177 | "encoding/json" 178 | "fmt" 179 | "log" 180 | "time" 181 | 182 | "github.com/golang-queue/nsq" 183 | "github.com/golang-queue/queue" 184 | "github.com/golang-queue/queue/core" 185 | ) 186 | 187 | type job struct { 188 | Message string 189 | } 190 | 191 | func (j *job) Bytes() []byte { 192 | b, err := json.Marshal(j) 193 | if err != nil { 194 | panic(err) 195 | } 196 | return b 197 | } 198 | 199 | func main() { 200 | taskN := 100 201 | rets := make(chan string, taskN) 202 | 203 | // define the worker 204 | w := nsq.NewWorker( 205 | nsq.WithAddr("127.0.0.1:4150"), 206 | nsq.WithTopic("example"), 207 | nsq.WithChannel("foobar"), 208 | // concurrent job number 209 | nsq.WithMaxInFlight(10), 210 | nsq.WithRunFunc(func(ctx context.Context, m core.TaskMessage) error { 211 | var v job 212 | if err := json.Unmarshal(m.Payload(), &v); err != nil { 213 | return err 214 | } 215 | 216 | rets <- v.Message 217 | return nil 218 | }), 219 | ) 220 | 221 | // define the queue 222 | q := queue.NewPool( 223 | 5, 224 | queue.WithWorker(w), 225 | ) 226 | 227 | // assign tasks to the queue 228 | for i := 0; i < taskN; i++ { 229 | go func(i int) { 230 | q.Queue(&job{ 231 | Message: fmt.Sprintf("handle the job: %d", i+1), 232 | }) 233 | }(i) 234 | } 235 | 236 | // wait until all tasks are done 237 | for i := 0; i < taskN; i++ { 238 | fmt.Println("message:", <-rets) 239 | time.Sleep(50 * time.Millisecond) 240 | } 241 | 242 | // shut down the service and notify all workers 243 | q.Release() 244 | } 245 | ``` 246 | 247 | ## Using NATS as a Queue 248 | 249 | Refer to the [NATS documentation](https://github.com/golang-queue/nats) for more details. 250 | 251 | ```go 252 | package main 253 | 254 | import ( 255 | "context" 256 | "encoding/json" 257 | "fmt" 258 | "log" 259 | "time" 260 | 261 | "github.com/golang-queue/nats" 262 | "github.com/golang-queue/queue" 263 | "github.com/golang-queue/queue/core" 264 | ) 265 | 266 | type job struct { 267 | Message string 268 | } 269 | 270 | func (j *job) Bytes() []byte { 271 | b, err := json.Marshal(j) 272 | if err != nil { 273 | panic(err) 274 | } 275 | return b 276 | } 277 | 278 | func main() { 279 | taskN := 100 280 | rets := make(chan string, taskN) 281 | 282 | // define the worker 283 | w := nats.NewWorker( 284 | nats.WithAddr("127.0.0.1:4222"), 285 | nats.WithSubj("example"), 286 | nats.WithQueue("foobar"), 287 | nats.WithRunFunc(func(ctx context.Context, m core.TaskMessage) error { 288 | var v job 289 | if err := json.Unmarshal(m.Payload(), &v); err != nil { 290 | return err 291 | } 292 | 293 | rets <- v.Message 294 | return nil 295 | }), 296 | ) 297 | 298 | // define the queue 299 | q, err := queue.NewQueue( 300 | queue.WithWorkerCount(10), 301 | queue.WithWorker(w), 302 | ) 303 | if err != nil { 304 | log.Fatal(err) 305 | } 306 | 307 | // start the workers 308 | q.Start() 309 | 310 | // assign tasks to the queue 311 | for i := 0; i < taskN; i++ { 312 | go func(i int) { 313 | q.Queue(&job{ 314 | Message: fmt.Sprintf("handle the job: %d", i+1), 315 | }) 316 | }(i) 317 | } 318 | 319 | // wait until all tasks are done 320 | for i := 0; i < taskN; i++ { 321 | fmt.Println("message:", <-rets) 322 | time.Sleep(50 * time.Millisecond) 323 | } 324 | 325 | // shut down the service and notify all workers 326 | q.Release() 327 | } 328 | ``` 329 | 330 | ## Using Redis (Pub/Sub) as a Queue 331 | 332 | Refer to the [Redis documentation](https://github.com/golang-queue/redisdb) for more details. 333 | 334 | ```go 335 | package main 336 | 337 | import ( 338 | "context" 339 | "encoding/json" 340 | "fmt" 341 | "log" 342 | "time" 343 | 344 | "github.com/golang-queue/queue" 345 | "github.com/golang-queue/queue/core" 346 | "github.com/golang-queue/redisdb" 347 | ) 348 | 349 | type job struct { 350 | Message string 351 | } 352 | 353 | func (j *job) Bytes() []byte { 354 | b, err := json.Marshal(j) 355 | if err != nil { 356 | panic(err) 357 | } 358 | return b 359 | } 360 | 361 | func main() { 362 | taskN := 100 363 | rets := make(chan string, taskN) 364 | 365 | // define the worker 366 | w := redisdb.NewWorker( 367 | redisdb.WithAddr("127.0.0.1:6379"), 368 | redisdb.WithChannel("foobar"), 369 | redisdb.WithRunFunc(func(ctx context.Context, m core.TaskMessage) error { 370 | var v job 371 | if err := json.Unmarshal(m.Payload(), &v); err != nil { 372 | return err 373 | } 374 | 375 | rets <- v.Message 376 | return nil 377 | }), 378 | ) 379 | 380 | // define the queue 381 | q, err := queue.NewQueue( 382 | queue.WithWorkerCount(10), 383 | queue.WithWorker(w), 384 | ) 385 | if err != nil { 386 | log.Fatal(err) 387 | } 388 | 389 | // start the workers 390 | q.Start() 391 | 392 | // assign tasks to the queue 393 | for i := 0; i < taskN; i++ { 394 | go func(i int) { 395 | q.Queue(&job{ 396 | Message: fmt.Sprintf("handle the job: %d", i+1), 397 | }) 398 | }(i) 399 | } 400 | 401 | // wait until all tasks are done 402 | for i := 0; i < taskN; i++ { 403 | fmt.Println("message:", <-rets) 404 | time.Sleep(50 * time.Millisecond) 405 | } 406 | 407 | // shut down the service and notify all workers 408 | q.Release() 409 | } 410 | ``` 411 | -------------------------------------------------------------------------------- /README.zh-cn.md: -------------------------------------------------------------------------------- 1 | # 队列 2 | 3 | [![CodeQL](https://github.com/golang-queue/queue/actions/workflows/codeql.yaml/badge.svg)](https://github.com/golang-queue/queue/actions/workflows/codeql.yaml) 4 | [![Run Tests](https://github.com/golang-queue/queue/actions/workflows/go.yml/badge.svg)](https://github.com/golang-queue/queue/actions/workflows/go.yml) 5 | [![codecov](https://codecov.io/gh/golang-queue/queue/branch/master/graph/badge.svg?token=SSo3mHejOE)](https://codecov.io/gh/golang-queue/queue) 6 | 7 | [English](./README.md) | [繁體中文](./README.zh-tw.md) 8 | 9 | Queue 是一个 Golang 库，帮助您创建和管理 Goroutines（轻量级线程）池。它允许您高效地并行运行多个任务，利用机器的 CPU 容量。 10 | 11 | ## 特性 12 | 13 | - [x] 支持 [循环缓冲区](https://en.wikipedia.org/wiki/Circular_buffer) 队列。 14 | - [x] 集成 [NSQ](https://nsq.io/) 进行实时分布式消息传递。 15 | - [x] 集成 [NATS](https://nats.io/) 以适应边缘和分布式系统。 16 | - [x] 集成 [Redis Pub/Sub](https://redis.io/docs/manual/pubsub/)。 17 | - [x] 集成 [Redis Streams](https://redis.io/docs/manual/data-types/streams/)。 18 | - [x] 集成 [RabbitMQ](https://www.rabbitmq.com/)。 19 | 20 | ## 队列场景 21 | 22 | 使用环形缓冲区作为默认后端的简单队列服务。 23 | 24 | ![queue01](./images/flow-01.svg) 25 | 26 | 轻松切换队列服务以使用 NSQ、NATS 或 Redis。 27 | 28 | ![queue02](./images/flow-02.svg) 29 | 30 | 支持多个生产者和消费者。 31 | 32 | ![queue03](./images/flow-03.svg) 33 | 34 | ## 要求 35 | 36 | Go 版本 **1.22** 或以上 37 | 38 | ## 安装 39 | 40 | 安装稳定版本： 41 | 42 | ```sh 43 | go get github.com/golang-queue/queue 44 | ``` 45 | 46 | 安装最新版本： 47 | 48 | ```sh 49 | go get github.com/golang-queue/queue@master 50 | ``` 51 | 52 | ## 使用 53 | 54 | ### 池的基本用法（使用 Task 函数） 55 | 56 | 通过调用 `QueueTask()` 方法，您可以安排任务由池中的工作者（Goroutines）执行。 57 | 58 | ```go 59 | package main 60 | 61 | import ( 62 | "context" 63 | "fmt" 64 | "time" 65 | 66 | "github.com/golang-queue/queue" 67 | ) 68 | 69 | func main() { 70 | taskN := 100 71 | rets := make(chan string, taskN) 72 | 73 | // 初始化队列池 74 | q := queue.NewPool(5) 75 | // 关闭服务并通知所有工作者 76 | // 等待所有作业完成 77 | defer q.Release() 78 | 79 | // 将任务分配到队列 80 | for i := 0; i < taskN; i++ { 81 | go func(i int) { 82 | if err := q.QueueTask(func(ctx context.Context) error { 83 | rets <- fmt.Sprintf("Hi Gopher, 处理作业: %02d", +i) 84 | return nil 85 | }); err != nil { 86 | panic(err) 87 | } 88 | }(i) 89 | } 90 | 91 | // 等待所有任务完成 92 | for i := 0; i < taskN; i++ { 93 | fmt.Println("消息:", <-rets) 94 | time.Sleep(20 * time.Millisecond) 95 | } 96 | } 97 | ``` 98 | 99 | ### 池的基本用法（使用消息队列） 100 | 101 | 定义一个新的消息结构并实现 `Bytes()` 函数来编码消息。使用 `WithFn` 函数处理来自队列的消息。 102 | 103 | ```go 104 | package main 105 | 106 | import ( 107 | "context" 108 | "encoding/json" 109 | "fmt" 110 | "log" 111 | "time" 112 | 113 | "github.com/golang-queue/queue" 114 | "github.com/golang-queue/queue/core" 115 | ) 116 | 117 | type job struct { 118 | Name string 119 | Message string 120 | } 121 | 122 | func (j *job) Bytes() []byte { 123 | b, err := json.Marshal(j) 124 | if err != nil { 125 | panic(err) 126 | } 127 | return b 128 | } 129 | 130 | func main() { 131 | taskN := 100 132 | rets := make(chan string, taskN) 133 | 134 | // 初始化队列池 135 | q := queue.NewPool(5, queue.WithFn(func(ctx context.Context, m core.TaskMessage) error { 136 | var v job 137 | if err := json.Unmarshal(m.Payload(), &v); err != nil { 138 | return err 139 | } 140 | 141 | rets <- "Hi, " + v.Name + ", " + v.Message 142 | return nil 143 | })) 144 | // 关闭服务并通知所有工作者 145 | // 等待所有作业完成 146 | defer q.Release() 147 | 148 | // 将任务分配到队列 149 | for i := 0; i < taskN; i++ { 150 | go func(i int) { 151 | if err := q.Queue(&job{ 152 | Name: "Gopher", 153 | Message: fmt.Sprintf("处理作业: %d", i+1), 154 | }); err != nil { 155 | log.Println(err) 156 | } 157 | }(i) 158 | } 159 | 160 | // 等待所有任务完成 161 | for i := 0; i < taskN; i++ { 162 | fmt.Println("消息:", <-rets) 163 | time.Sleep(50 * time.Millisecond) 164 | } 165 | } 166 | ``` 167 | 168 | ## 使用 NSQ 作为队列 169 | 170 | 请参阅 [NSQ 文档](https://github.com/golang-queue/nsq) 了解更多详情。 171 | 172 | ```go 173 | package main 174 | 175 | import ( 176 | "context" 177 | "encoding/json" 178 | "fmt" 179 | "log" 180 | "time" 181 | 182 | "github.com/golang-queue/nsq" 183 | "github.com/golang-queue/queue" 184 | "github.com/golang-queue/queue/core" 185 | ) 186 | 187 | type job struct { 188 | Message string 189 | } 190 | 191 | func (j *job) Bytes() []byte { 192 | b, err := json.Marshal(j) 193 | if err != nil { 194 | panic(err) 195 | } 196 | return b 197 | } 198 | 199 | func main() { 200 | taskN := 100 201 | rets := make(chan string, taskN) 202 | 203 | // 定义工作者 204 | w := nsq.NewWorker( 205 | nsq.WithAddr("127.0.0.1:4150"), 206 | nsq.WithTopic("example"), 207 | nsq.WithChannel("foobar"), 208 | // 并发作业数量 209 | nsq.WithMaxInFlight(10), 210 | nsq.WithRunFunc(func(ctx context.Context, m core.TaskMessage) error { 211 | var v job 212 | if err := json.Unmarshal(m.Payload(), &v); err != nil { 213 | return err 214 | } 215 | 216 | rets <- v.Message 217 | return nil 218 | }), 219 | ) 220 | 221 | // 定义队列 222 | q := queue.NewPool( 223 | 5, 224 | queue.WithWorker(w), 225 | ) 226 | 227 | // 将任务分配到队列 228 | for i := 0; i < taskN; i++ { 229 | go func(i int) { 230 | q.Queue(&job{ 231 | Message: fmt.Sprintf("处理作业: %d", i+1), 232 | }) 233 | }(i) 234 | } 235 | 236 | // 等待所有任务完成 237 | for i := 0; i < taskN; i++ { 238 | fmt.Println("消息:", <-rets) 239 | time.Sleep(50 * time.Millisecond) 240 | } 241 | 242 | // 关闭服务并通知所有工作者 243 | q.Release() 244 | } 245 | ``` 246 | 247 | ## 使用 NATS 作为队列 248 | 249 | 请参阅 [NATS 文档](https://github.com/golang-queue/nats) 了解更多详情。 250 | 251 | ```go 252 | package main 253 | 254 | import ( 255 | "context" 256 | "encoding/json" 257 | "fmt" 258 | "log" 259 | "time" 260 | 261 | "github.com/golang-queue/nats" 262 | "github.com/golang-queue/queue" 263 | "github.com/golang-queue/queue/core" 264 | ) 265 | 266 | type job struct { 267 | Message string 268 | } 269 | 270 | func (j *job) Bytes() []byte { 271 | b, err := json.Marshal(j) 272 | if err != nil { 273 | panic(err) 274 | } 275 | return b 276 | } 277 | 278 | func main() { 279 | taskN := 100 280 | rets := make(chan string, taskN) 281 | 282 | // 定义工作者 283 | w := nats.NewWorker( 284 | nats.WithAddr("127.0.0.1:4222"), 285 | nats.WithSubj("example"), 286 | nats.WithQueue("foobar"), 287 | nats.WithRunFunc(func(ctx context.Context, m core.TaskMessage) error { 288 | var v job 289 | if err := json.Unmarshal(m.Payload(), &v); err != nil { 290 | return err 291 | } 292 | 293 | rets <- v.Message 294 | return nil 295 | }), 296 | ) 297 | 298 | // 定义队列 299 | q, err := queue.NewQueue( 300 | queue.WithWorkerCount(10), 301 | queue.WithWorker(w), 302 | ) 303 | if err != nil { 304 | log.Fatal(err) 305 | } 306 | 307 | // 启动工作者 308 | q.Start() 309 | 310 | // 将任务分配到队列 311 | for i := 0; i < taskN; i++ { 312 | go func(i int) { 313 | q.Queue(&job{ 314 | Message: fmt.Sprintf("处理作业: %d", i+1), 315 | }) 316 | }(i) 317 | } 318 | 319 | // 等待所有任务完成 320 | for i := 0; i < taskN; i++ { 321 | fmt.Println("消息:", <-rets) 322 | time.Sleep(50 * time.Millisecond) 323 | } 324 | 325 | // 关闭服务并通知所有工作者 326 | q.Release() 327 | } 328 | ``` 329 | 330 | ## 使用 Redis (Pub/Sub) 作为队列 331 | 332 | 请参阅 [Redis 文档](https://github.com/golang-queue/redisdb) 了解更多详情。 333 | 334 | ```go 335 | package main 336 | 337 | import ( 338 | "context" 339 | "encoding/json" 340 | "fmt" 341 | "log" 342 | "time" 343 | 344 | "github.com/golang-queue/queue" 345 | "github.com/golang-queue/queue/core" 346 | "github.com/golang-queue/redisdb" 347 | ) 348 | 349 | type job struct { 350 | Message string 351 | } 352 | 353 | func (j *job) Bytes() []byte { 354 | b, err := json.Marshal(j) 355 | if err != nil { 356 | panic(err) 357 | } 358 | return b 359 | } 360 | 361 | func main() { 362 | taskN := 100 363 | rets := make(chan string, taskN) 364 | 365 | // 定义工作者 366 | w := redisdb.NewWorker( 367 | redisdb.WithAddr("127.0.0.1:6379"), 368 | redisdb.WithChannel("foobar"), 369 | redisdb.WithRunFunc(func(ctx context.Context, m core.TaskMessage) error { 370 | var v job 371 | if err := json.Unmarshal(m.Payload(), &v); err != nil { 372 | return err 373 | } 374 | 375 | rets <- v.Message 376 | return nil 377 | }), 378 | ) 379 | 380 | // 定义队列 381 | q, err := queue.NewQueue( 382 | queue.WithWorkerCount(10), 383 | queue.WithWorker(w), 384 | ) 385 | if err != nil { 386 | log.Fatal(err) 387 | } 388 | 389 | // 启动工作者 390 | q.Start() 391 | 392 | // 将任务分配到队列 393 | for i := 0; i < taskN; i++ { 394 | go func(i int) { 395 | q.Queue(&job{ 396 | Message: fmt.Sprintf("处理作业: %d", i+1), 397 | }) 398 | }(i) 399 | } 400 | 401 | // 等待所有任务完成 402 | for i := 0; i < taskN; i++ { 403 | fmt.Println("消息:", <-rets) 404 | time.Sleep(50 * time.Millisecond) 405 | } 406 | 407 | // 关闭服务并通知所有工作者 408 | q.Release() 409 | } 410 | ``` 411 | -------------------------------------------------------------------------------- /README.zh-tw.md: -------------------------------------------------------------------------------- 1 | # Queue 2 | 3 | [![CodeQL](https://github.com/golang-queue/queue/actions/workflows/codeql.yaml/badge.svg)](https://github.com/golang-queue/queue/actions/workflows/codeql.yaml) 4 | [![Run Tests](https://github.com/golang-queue/queue/actions/workflows/go.yml/badge.svg)](https://github.com/golang-queue/queue/actions/workflows/go.yml) 5 | [![codecov](https://codecov.io/gh/golang-queue/queue/branch/master/graph/badge.svg?token=SSo3mHejOE)](https://codecov.io/gh/golang-queue/queue) 6 | 7 | [English](./README.md) | [简体中文](./README.zh-cn.md) 8 | 9 | Queue 是一個 Golang 函式庫，幫助你建立和管理 Goroutines（輕量級執行緒）的池。它允許你有效地並行執行多個任務，充分利用機器的 CPU 容量。 10 | 11 | ## 功能 12 | 13 | - [x] 支援 [環形緩衝區](https://en.wikipedia.org/wiki/Circular_buffer) 隊列。 14 | - [x] 與 [NSQ](https://nsq.io/) 整合，用於即時分佈式消息傳遞。 15 | - [x] 與 [NATS](https://nats.io/) 整合，用於自適應邊緣和分佈式系統。 16 | - [x] 與 [Redis Pub/Sub](https://redis.io/docs/manual/pubsub/) 整合。 17 | - [x] 與 [Redis Streams](https://redis.io/docs/manual/data-types/streams/) 整合。 18 | - [x] 與 [RabbitMQ](https://www.rabbitmq.com/) 整合。 19 | 20 | ## 隊列場景 21 | 22 | 使用環形緩衝區作為預設後端的簡單隊列服務。 23 | 24 | ![queue01](./images/flow-01.svg) 25 | 26 | 輕鬆切換隊列服務以使用 NSQ、NATS 或 Redis。 27 | 28 | ![queue02](./images/flow-02.svg) 29 | 30 | 支援多個生產者和消費者。 31 | 32 | ![queue03](./images/flow-03.svg) 33 | 34 | ## 要求 35 | 36 | Go 版本 **1.22** 或以上 37 | 38 | ## 安裝 39 | 40 | 安裝穩定版本： 41 | 42 | ```sh 43 | go get github.com/golang-queue/queue 44 | ``` 45 | 46 | 安裝最新版本： 47 | 48 | ```sh 49 | go get github.com/golang-queue/queue@master 50 | ``` 51 | 52 | ## 使用方法 53 | 54 | ### 池的基本使用（使用 Task 函數） 55 | 56 | 通過調用 `QueueTask()` 方法，你可以安排任務由池中的工作者（Goroutines）執行。 57 | 58 | ```go 59 | package main 60 | 61 | import ( 62 | "context" 63 | "fmt" 64 | "time" 65 | 66 | "github.com/golang-queue/queue" 67 | ) 68 | 69 | func main() { 70 | taskN := 100 71 | rets := make(chan string, taskN) 72 | 73 | // 初始化隊列池 74 | q := queue.NewPool(5) 75 | // 關閉服務並通知所有工作者 76 | // 等待所有工作完成 77 | defer q.Release() 78 | 79 | // 將任務分配給隊列 80 | for i := 0; i < taskN; i++ { 81 | go func(i int) { 82 | if err := q.QueueTask(func(ctx context.Context) error { 83 | rets <- fmt.Sprintf("Hi Gopher, 處理工作: %02d", +i) 84 | return nil 85 | }); err != nil { 86 | panic(err) 87 | } 88 | }(i) 89 | } 90 | 91 | // 等待所有任務完成 92 | for i := 0; i < taskN; i++ { 93 | fmt.Println("message:", <-rets) 94 | time.Sleep(20 * time.Millisecond) 95 | } 96 | } 97 | ``` 98 | 99 | ### 池的基本使用（使用消息隊列） 100 | 101 | 定義一個新的消息結構並實現 `Bytes()` 函數來編碼消息。使用 `WithFn` 函數來處理來自隊列的消息。 102 | 103 | ```go 104 | package main 105 | 106 | import ( 107 | "context" 108 | "encoding/json" 109 | "fmt" 110 | "log" 111 | "time" 112 | 113 | "github.com/golang-queue/queue" 114 | "github.com/golang-queue/queue/core" 115 | ) 116 | 117 | type job struct { 118 | Name string 119 | Message string 120 | } 121 | 122 | func (j *job) Bytes() []byte { 123 | b, err := json.Marshal(j) 124 | if err != nil { 125 | panic(err) 126 | } 127 | return b 128 | } 129 | 130 | func main() { 131 | taskN := 100 132 | rets := make(chan string, taskN) 133 | 134 | // 初始化隊列池 135 | q := queue.NewPool(5, queue.WithFn(func(ctx context.Context, m core.TaskMessage) error { 136 | var v job 137 | if err := json.Unmarshal(m.Payload(), &v); err != nil { 138 | return err 139 | } 140 | 141 | rets <- "Hi, " + v.Name + ", " + v.Message 142 | return nil 143 | })) 144 | // 關閉服務並通知所有工作者 145 | // 等待所有工作完成 146 | defer q.Release() 147 | 148 | // 將任務分配給隊列 149 | for i := 0; i < taskN; i++ { 150 | go func(i int) { 151 | if err := q.Queue(&job{ 152 | Name: "Gopher", 153 | Message: fmt.Sprintf("處理工作: %d", i+1), 154 | }); err != nil { 155 | log.Println(err) 156 | } 157 | }(i) 158 | } 159 | 160 | // 等待所有任務完成 161 | for i := 0; i < taskN; i++ { 162 | fmt.Println("message:", <-rets) 163 | time.Sleep(50 * time.Millisecond) 164 | } 165 | } 166 | ``` 167 | 168 | ## 使用 NSQ 作為隊列 169 | 170 | 請參閱 [NSQ 文檔](https://github.com/golang-queue/nsq) 以獲取更多詳細信息。 171 | 172 | ```go 173 | package main 174 | 175 | import ( 176 | "context" 177 | "encoding/json" 178 | "fmt" 179 | "log" 180 | "time" 181 | 182 | "github.com/golang-queue/nsq" 183 | "github.com/golang-queue/queue" 184 | "github.com/golang-queue/queue/core" 185 | ) 186 | 187 | type job struct { 188 | Message string 189 | } 190 | 191 | func (j *job) Bytes() []byte { 192 | b, err := json.Marshal(j) 193 | if err != nil { 194 | panic(err) 195 | } 196 | return b 197 | } 198 | 199 | func main() { 200 | taskN := 100 201 | rets := make(chan string, taskN) 202 | 203 | // 定義工作者 204 | w := nsq.NewWorker( 205 | nsq.WithAddr("127.0.0.1:4150"), 206 | nsq.WithTopic("example"), 207 | nsq.WithChannel("foobar"), 208 | // 並發工作數量 209 | nsq.WithMaxInFlight(10), 210 | nsq.WithRunFunc(func(ctx context.Context, m core.TaskMessage) error { 211 | var v job 212 | if err := json.Unmarshal(m.Payload(), &v); err != nil { 213 | return err 214 | } 215 | 216 | rets <- v.Message 217 | return nil 218 | }), 219 | ) 220 | 221 | // 定義隊列 222 | q := queue.NewPool( 223 | 5, 224 | queue.WithWorker(w), 225 | ) 226 | 227 | // 將任務分配給隊列 228 | for i := 0; i < taskN; i++ { 229 | go func(i int) { 230 | q.Queue(&job{ 231 | Message: fmt.Sprintf("處理工作: %d", i+1), 232 | }) 233 | }(i) 234 | } 235 | 236 | // 等待所有任務完成 237 | for i := 0; i < taskN; i++ { 238 | fmt.Println("message:", <-rets) 239 | time.Sleep(50 * time.Millisecond) 240 | } 241 | 242 | // 關閉服務並通知所有工作者 243 | q.Release() 244 | } 245 | ``` 246 | 247 | ## 使用 NATS 作為隊列 248 | 249 | 請參閱 [NATS 文檔](https://github.com/golang-queue/nats) 以獲取更多詳細信息。 250 | 251 | ```go 252 | package main 253 | 254 | import ( 255 | "context" 256 | "encoding/json" 257 | "fmt" 258 | "log" 259 | "time" 260 | 261 | "github.com/golang-queue/nats" 262 | "github.com/golang-queue/queue" 263 | "github.com/golang-queue/queue/core" 264 | ) 265 | 266 | type job struct { 267 | Message string 268 | } 269 | 270 | func (j *job) Bytes() []byte { 271 | b, err := json.Marshal(j) 272 | if err != nil { 273 | panic(err) 274 | } 275 | return b 276 | } 277 | 278 | func main() { 279 | taskN := 100 280 | rets := make(chan string, taskN) 281 | 282 | // 定義工作者 283 | w := nats.NewWorker( 284 | nats.WithAddr("127.0.0.1:4222"), 285 | nats.WithSubj("example"), 286 | nats.WithQueue("foobar"), 287 | nats.WithRunFunc(func(ctx context.Context, m core.TaskMessage) error { 288 | var v job 289 | if err := json.Unmarshal(m.Payload(), &v); err != nil { 290 | return err 291 | } 292 | 293 | rets <- v.Message 294 | return nil 295 | }), 296 | ) 297 | 298 | // 定義隊列 299 | q, err := queue.NewQueue( 300 | queue.WithWorkerCount(10), 301 | queue.WithWorker(w), 302 | ) 303 | if err != nil { 304 | log.Fatal(err) 305 | } 306 | 307 | // 啟動工作者 308 | q.Start() 309 | 310 | // 將任務分配給隊列 311 | for i := 0; i < taskN; i++ { 312 | go func(i int) { 313 | q.Queue(&job{ 314 | Message: fmt.Sprintf("處理工作: %d", i+1), 315 | }) 316 | }(i) 317 | } 318 | 319 | // 等待所有任務完成 320 | for i := 0; i < taskN; i++ { 321 | fmt.Println("message:", <-rets) 322 | time.Sleep(50 * time.Millisecond) 323 | } 324 | 325 | // 關閉服務並通知所有工作者 326 | q.Release() 327 | } 328 | ``` 329 | 330 | ## 使用 Redis (Pub/Sub) 作為隊列 331 | 332 | 請參閱 [Redis 文檔](https://github.com/golang-queue/redisdb) 以獲取更多詳細信息。 333 | 334 | ```go 335 | package main 336 | 337 | import ( 338 | "context" 339 | "encoding/json" 340 | "fmt" 341 | "log" 342 | "time" 343 | 344 | "github.com/golang-queue/queue" 345 | "github.com/golang-queue/queue/core" 346 | "github.com/golang-queue/redisdb" 347 | ) 348 | 349 | type job struct { 350 | Message string 351 | } 352 | 353 | func (j *job) Bytes() []byte { 354 | b, err := json.Marshal(j) 355 | if err != nil { 356 | panic(err) 357 | } 358 | return b 359 | } 360 | 361 | func main() { 362 | taskN := 100 363 | rets := make(chan string, taskN) 364 | 365 | // 定義工作者 366 | w := redisdb.NewWorker( 367 | redisdb.WithAddr("127.0.0.1:6379"), 368 | redisdb.WithChannel("foobar"), 369 | redisdb.WithRunFunc(func(ctx context.Context, m core.TaskMessage) error { 370 | var v job 371 | if err := json.Unmarshal(m.Payload(), &v); err != nil { 372 | return err 373 | } 374 | 375 | rets <- v.Message 376 | return nil 377 | }), 378 | ) 379 | 380 | // 定義隊列 381 | q, err := queue.NewQueue( 382 | queue.WithWorkerCount(10), 383 | queue.WithWorker(w), 384 | ) 385 | if err != nil { 386 | log.Fatal(err) 387 | } 388 | 389 | // 啟動工作者 390 | q.Start() 391 | 392 | // 將任務分配給隊列 393 | for i := 0; i < taskN; i++ { 394 | go func(i int) { 395 | q.Queue(&job{ 396 | Message: fmt.Sprintf("處理工作: %d", i+1), 397 | }) 398 | }(i) 399 | } 400 | 401 | // 等待所有任務完成 402 | for i := 0; i < taskN; i++ { 403 | fmt.Println("message:", <-rets) 404 | time.Sleep(50 * time.Millisecond) 405 | } 406 | 407 | // 關閉服務並通知所有工作者 408 | q.Release() 409 | } 410 | ``` 411 | -------------------------------------------------------------------------------- /_example/example01/go.mod: -------------------------------------------------------------------------------- 1 | module example 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/golang-queue/contrib v0.0.1 7 | github.com/golang-queue/queue v0.2.0 8 | ) 9 | 10 | require ( 11 | github.com/jpillora/backoff v1.0.0 // indirect 12 | github.com/rs/zerolog v1.26.1 // indirect 13 | ) 14 | 15 | replace github.com/golang-queue/queue => ../../ 16 | -------------------------------------------------------------------------------- /_example/example01/go.sum: -------------------------------------------------------------------------------- 1 | github.com/appleboy/com v0.2.1 h1:dHAHauX3eYDuheAahI83HIGFxpi0SEb2ZAu9EZ9hbUM= 2 | github.com/appleboy/com v0.2.1/go.mod h1:kByEI3/vzI5GM1+O5QdBHLsXaOsmFsJcOpCSgASi4sg= 3 | github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 7 | github.com/golang-queue/contrib v0.0.1 h1:Jd61HHridQ8ReJ04kgmF3GYCr/oxIUNeJFoDU6Iuxtc= 8 | github.com/golang-queue/contrib v0.0.1/go.mod h1:Vmreu9zyumpRg7ew516NIP3HUUikaAUBAVq35ujulQA= 9 | github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= 10 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 11 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 12 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 13 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 14 | github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= 15 | github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= 16 | github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= 17 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 18 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 19 | github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 20 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 21 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 22 | go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= 23 | go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= 24 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 25 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 26 | golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= 27 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 28 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 29 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 30 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 31 | golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 32 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 33 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 34 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 35 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 36 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 37 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 38 | golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 39 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 40 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 41 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 42 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 43 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 44 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 45 | golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= 46 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 47 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 48 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 49 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 50 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 51 | -------------------------------------------------------------------------------- /_example/example01/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/golang-queue/contrib/zerolog" 9 | "github.com/golang-queue/queue" 10 | ) 11 | 12 | func main() { 13 | taskN := 100 14 | rets := make(chan string, taskN) 15 | 16 | // initial queue pool 17 | q := queue.NewPool(5, queue.WithLogger(zerolog.New())) 18 | // shutdown the service and notify all the worker 19 | // wait all jobs done. 20 | defer q.Release() 21 | 22 | // assign tasks in queue 23 | for i := 0; i < taskN; i++ { 24 | go func(i int) { 25 | if err := q.QueueTask(func(ctx context.Context) error { 26 | rets <- fmt.Sprintf("Hi Gopher, handle the job: %02d", +i) 27 | return nil 28 | }); err != nil { 29 | panic(err) 30 | } 31 | }(i) 32 | } 33 | 34 | // wait until all tasks done 35 | for i := 0; i < taskN; i++ { 36 | fmt.Println("message:", <-rets) 37 | time.Sleep(20 * time.Millisecond) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /_example/example02/go.mod: -------------------------------------------------------------------------------- 1 | module example 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/golang-queue/contrib v1.1.0 7 | github.com/golang-queue/queue v0.2.0 8 | ) 9 | 10 | require ( 11 | github.com/jpillora/backoff v1.0.0 // indirect 12 | github.com/mattn/go-colorable v0.1.14 // indirect 13 | github.com/mattn/go-isatty v0.0.20 // indirect 14 | github.com/rs/zerolog v1.33.0 // indirect 15 | golang.org/x/sys v0.29.0 // indirect 16 | ) 17 | 18 | replace github.com/golang-queue/queue => ../../ 19 | -------------------------------------------------------------------------------- /_example/example02/go.sum: -------------------------------------------------------------------------------- 1 | github.com/appleboy/com v0.2.1 h1:dHAHauX3eYDuheAahI83HIGFxpi0SEb2ZAu9EZ9hbUM= 2 | github.com/appleboy/com v0.2.1/go.mod h1:kByEI3/vzI5GM1+O5QdBHLsXaOsmFsJcOpCSgASi4sg= 3 | github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 7 | github.com/golang-queue/contrib v1.1.0 h1:1vydG/z/CKCg84BPNYAqru5sKxmlUv3ZlbUbRI+Hp+4= 8 | github.com/golang-queue/contrib v1.1.0/go.mod h1:bKxkQxxIEOeTperG3l2kgeLmVLLodF5iOrNING5wleo= 9 | github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= 10 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 11 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 12 | github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= 13 | github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= 14 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 15 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 16 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 17 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 18 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 19 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 20 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 21 | github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= 22 | github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= 23 | github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= 24 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 25 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 26 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 27 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 28 | go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= 29 | go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= 30 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 31 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 32 | golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 33 | golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= 34 | golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 35 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 36 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 37 | -------------------------------------------------------------------------------- /_example/example02/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "time" 9 | 10 | "github.com/golang-queue/contrib/zerolog" 11 | "github.com/golang-queue/queue" 12 | "github.com/golang-queue/queue/core" 13 | ) 14 | 15 | type job struct { 16 | Name string 17 | Message string 18 | } 19 | 20 | func (j *job) Bytes() []byte { 21 | b, err := json.Marshal(j) 22 | if err != nil { 23 | panic(err) 24 | } 25 | return b 26 | } 27 | 28 | func main() { 29 | taskN := 100 30 | rets := make(chan string, taskN) 31 | 32 | // initial queue pool 33 | q := queue.NewPool(5, queue.WithFn(func(ctx context.Context, m core.TaskMessage) error { 34 | var v job 35 | if err := json.Unmarshal(m.Payload(), &v); err != nil { 36 | return err 37 | } 38 | rets <- "Hi, " + v.Name + ", " + v.Message 39 | return nil 40 | }), queue.WithLogger(zerolog.New())) 41 | // shutdown the service and notify all the worker 42 | // wait all jobs done. 43 | defer q.Release() 44 | 45 | // assign tasks in queue 46 | for i := 0; i < taskN; i++ { 47 | go func(i int) { 48 | if err := q.Queue(&job{ 49 | Name: "Gopher", 50 | Message: fmt.Sprintf("handle the job: %d", i+1), 51 | }); err != nil { 52 | log.Println(err) 53 | } 54 | }(i) 55 | } 56 | 57 | // wait until all tasks done 58 | for i := 0; i < taskN; i++ { 59 | fmt.Println("message:", <-rets) 60 | time.Sleep(20 * time.Millisecond) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /bearer.yml: -------------------------------------------------------------------------------- 1 | rule: 2 | # Disable all default rules by setting this value to true. 3 | disable-default-rules: false 4 | # Specify the comma-separated ids of the rules you would like to run; 5 | # skips all other rules. 6 | only-rule: [] 7 | # Specify the comma-separated ids of the rules you would like to skip; 8 | # runs all other rules. 9 | skip-rule: ["go_gosec_unsafe_unsafe", "go_gosec_filesystem_filereadtaint"] 10 | -------------------------------------------------------------------------------- /benchmark_test.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/golang-queue/queue/core" 9 | "github.com/golang-queue/queue/job" 10 | ) 11 | 12 | var count = 1 13 | 14 | type testqueue interface { 15 | Queue(task core.TaskMessage) error 16 | Request() (core.TaskMessage, error) 17 | } 18 | 19 | func testQueue(b *testing.B, pool testqueue) { 20 | message := job.NewTask(func(context.Context) error { 21 | return nil 22 | }, 23 | job.AllowOption{ 24 | RetryCount: job.Int64(100), 25 | RetryDelay: job.Time(30 * time.Millisecond), 26 | Timeout: job.Time(3 * time.Millisecond), 27 | }, 28 | ) 29 | 30 | b.ReportAllocs() 31 | b.ResetTimer() 32 | for n := 0; n < b.N; n++ { 33 | for i := 0; i < count; i++ { 34 | _ = pool.Queue(&message) 35 | _, _ = pool.Request() 36 | } 37 | } 38 | } 39 | 40 | func BenchmarkNewRing(b *testing.B) { 41 | pool := NewRing( 42 | WithQueueSize(b.N*count), 43 | WithLogger(emptyLogger{}), 44 | ) 45 | 46 | testQueue(b, pool) 47 | } 48 | 49 | func BenchmarkQueueTask(b *testing.B) { 50 | w := NewRing() 51 | q, _ := NewQueue( 52 | WithWorker(w), 53 | WithLogger(emptyLogger{}), 54 | ) 55 | b.ReportAllocs() 56 | b.ResetTimer() 57 | 58 | m := job.NewTask(func(context.Context) error { 59 | return nil 60 | }) 61 | 62 | for n := 0; n < b.N; n++ { 63 | if err := q.queue(&m); err != nil { 64 | b.Fatal(err) 65 | } 66 | } 67 | } 68 | 69 | func BenchmarkQueue(b *testing.B) { 70 | w := NewRing() 71 | q, _ := NewQueue( 72 | WithWorker(w), 73 | WithLogger(emptyLogger{}), 74 | ) 75 | b.ReportAllocs() 76 | b.ResetTimer() 77 | 78 | m := job.NewMessage(&mockMessage{ 79 | message: "foo", 80 | }) 81 | 82 | for n := 0; n < b.N; n++ { 83 | if err := q.queue(&m); err != nil { 84 | b.Fatal(err) 85 | } 86 | } 87 | } 88 | 89 | // func BenchmarkRingPayload(b *testing.B) { 90 | // b.ReportAllocs() 91 | 92 | // task := &job.Message{ 93 | // Timeout: 100 * time.Millisecond, 94 | // Payload: []byte(`{"timeout":3600000000000}`), 95 | // } 96 | // w := NewRing( 97 | // WithFn(func(ctx context.Context, m core.TaskMessage) error { 98 | // return nil 99 | // }), 100 | // ) 101 | 102 | // q, _ := NewQueue( 103 | // WithWorker(w), 104 | // WithLogger(emptyLogger{}), 105 | // ) 106 | 107 | // for n := 0; n < b.N; n++ { 108 | // _ = q.run(task) 109 | // } 110 | // } 111 | 112 | func BenchmarkRingWithTask(b *testing.B) { 113 | b.ReportAllocs() 114 | 115 | task := job.Message{ 116 | Timeout: 100 * time.Millisecond, 117 | Task: func(_ context.Context) error { 118 | return nil 119 | }, 120 | } 121 | w := NewRing( 122 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 123 | return nil 124 | }), 125 | ) 126 | 127 | q, _ := NewQueue( 128 | WithWorker(w), 129 | WithLogger(emptyLogger{}), 130 | ) 131 | 132 | for n := 0; n < b.N; n++ { 133 | _ = q.run(&task) 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /core/worker.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | // Worker represents an interface for a worker that processes tasks. 8 | // It provides methods to run tasks, shut down the worker, queue tasks, and request tasks from the queue. 9 | type Worker interface { 10 | // Run starts the worker and processes the given task in the provided context. 11 | // It returns an error if the task cannot be processed. 12 | Run(ctx context.Context, task TaskMessage) error 13 | 14 | // Shutdown stops the worker and performs any necessary cleanup. 15 | // It returns an error if the shutdown process fails. 16 | Shutdown() error 17 | 18 | // Queue adds a task to the worker's queue. 19 | // It returns an error if the task cannot be added to the queue. 20 | Queue(task TaskMessage) error 21 | 22 | // Request retrieves a task from the worker's queue. 23 | // It returns the queued message and an error if the retrieval fails. 24 | Request() (TaskMessage, error) 25 | } 26 | 27 | // QueuedMessage represents an interface for a message that can be queued. 28 | // It requires the implementation of a Bytes method, which returns the message 29 | // content as a slice of bytes. 30 | type QueuedMessage interface { 31 | Bytes() []byte 32 | } 33 | 34 | // TaskMessage represents an interface for a task message that can be queued. 35 | // It embeds the QueuedMessage interface and adds a method to retrieve the payload of the message. 36 | type TaskMessage interface { 37 | QueuedMessage 38 | Payload() []byte 39 | } 40 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import "errors" 4 | 5 | var ( 6 | // ErrNoTaskInQueue there is nothing in the queue 7 | ErrNoTaskInQueue = errors.New("golang-queue: no task in queue") 8 | // ErrQueueHasBeenClosed the current queue is closed 9 | ErrQueueHasBeenClosed = errors.New("golang-queue: queue has been closed") 10 | // ErrMaxCapacity Maximum size limit reached 11 | ErrMaxCapacity = errors.New("golang-queue: maximum size limit reached") 12 | ) 13 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/golang-queue/queue 2 | 3 | go 1.23 4 | 5 | require ( 6 | github.com/appleboy/com v0.3.0 7 | github.com/jpillora/backoff v1.0.0 8 | github.com/stretchr/testify v1.10.0 9 | go.uber.org/goleak v1.3.0 10 | go.uber.org/mock v0.5.2 11 | ) 12 | 13 | require ( 14 | github.com/davecgh/go-spew v1.1.1 // indirect 15 | github.com/kr/text v0.2.0 // indirect 16 | github.com/pmezard/go-difflib v1.0.0 // indirect 17 | gopkg.in/yaml.v3 v3.0.1 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/appleboy/com v0.3.0 h1:omze/tJPyi2YVH+m23GSrCGt90A+4vQNpEYBW+GuSr4= 2 | github.com/appleboy/com v0.3.0/go.mod h1:kByEI3/vzI5GM1+O5QdBHLsXaOsmFsJcOpCSgASi4sg= 3 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= 7 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 8 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 9 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 10 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 11 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 12 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 13 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 14 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 15 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 16 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 17 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 18 | go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= 19 | go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= 20 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 21 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 22 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 23 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 24 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 25 | -------------------------------------------------------------------------------- /images/_base.d2: -------------------------------------------------------------------------------- 1 | classes: { 2 | base: { 3 | style: { 4 | bold: true 5 | font-size: 28 6 | } 7 | } 8 | 9 | person: { 10 | shape: person 11 | } 12 | 13 | animated: { 14 | style: { 15 | animated: true 16 | } 17 | } 18 | 19 | multiple: { 20 | style: { 21 | multiple: true 22 | } 23 | } 24 | 25 | enqueue: { 26 | label: Enqueue Task 27 | } 28 | 29 | dispatch: { 30 | label: Dispatch Task 31 | } 32 | 33 | library: { 34 | style: { 35 | bold: true 36 | font-size: 32 37 | fill: PapayaWhip 38 | fill-pattern: grain 39 | border-radius: 8 40 | font: mono 41 | } 42 | } 43 | 44 | task: { 45 | style: { 46 | bold: true 47 | font-size: 32 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /images/flow-01.d2: -------------------------------------------------------------------------------- 1 | direction: right 2 | 3 | ...@_base 4 | 5 | user01: { 6 | label: User01 7 | class: [base; person; multiple] 8 | } 9 | 10 | user02: { 11 | label: User02 12 | class: [base; person; multiple] 13 | } 14 | 15 | user03: { 16 | label: User03 17 | class: [base; person; multiple] 18 | } 19 | 20 | user01 -> container.task01: { 21 | label: Create Task 22 | class: [base; animated] 23 | } 24 | user02 -> container.task02: { 25 | label: Create Task 26 | class: [base; animated] 27 | } 28 | user03 -> container.task03: { 29 | label: Create Task 30 | class: [base; animated] 31 | } 32 | 33 | container: Application { 34 | direction: right 35 | style: { 36 | bold: true 37 | font-size: 28 38 | } 39 | icon: https://icons.terrastruct.com/dev%2Fgo.svg 40 | 41 | task01: { 42 | icon: https://icons.terrastruct.com/essentials%2F092-graph%20bar.svg 43 | class: [task; multiple] 44 | } 45 | 46 | task02: { 47 | icon: https://icons.terrastruct.com/essentials%2F095-download.svg 48 | class: [task; multiple] 49 | } 50 | 51 | task03: { 52 | icon: https://icons.terrastruct.com/essentials%2F195-attachment.svg 53 | class: [task; multiple] 54 | } 55 | 56 | queue: { 57 | label: Queue Library 58 | icon: https://icons.terrastruct.com/dev%2Fgo.svg 59 | style: { 60 | bold: true 61 | font-size: 32 62 | fill: honeydew 63 | } 64 | 65 | producer: { 66 | label: Producer 67 | class: library 68 | } 69 | 70 | consumer: { 71 | label: Consumer 72 | class: library 73 | } 74 | 75 | database: { 76 | label: Ring\nBuffer 77 | shape: cylinder 78 | style: { 79 | bold: true 80 | font-size: 32 81 | fill-pattern: lines 82 | font: mono 83 | } 84 | } 85 | 86 | producer -> database 87 | database -> consumer 88 | } 89 | 90 | worker01: { 91 | icon: https://icons.terrastruct.com/essentials%2F092-graph%20bar.svg 92 | class: [task] 93 | } 94 | 95 | worker02: { 96 | icon: https://icons.terrastruct.com/essentials%2F095-download.svg 97 | class: [task] 98 | } 99 | 100 | worker03: { 101 | icon: https://icons.terrastruct.com/essentials%2F092-graph%20bar.svg 102 | class: [task] 103 | } 104 | 105 | worker04: { 106 | icon: https://icons.terrastruct.com/essentials%2F195-attachment.svg 107 | class: [task] 108 | } 109 | 110 | task01 -> queue.producer: { 111 | class: [base; enqueue] 112 | } 113 | task02 -> queue.producer: { 114 | class: [base; enqueue] 115 | } 116 | task03 -> queue.producer: { 117 | class: [base; enqueue] 118 | } 119 | queue.consumer -> worker01: { 120 | class: [base; dispatch] 121 | } 122 | queue.consumer -> worker02: { 123 | class: [base; dispatch] 124 | } 125 | queue.consumer -> worker03: { 126 | class: [base; dispatch] 127 | } 128 | queue.consumer -> worker04: { 129 | class: [base; dispatch] 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /images/flow-02.d2: -------------------------------------------------------------------------------- 1 | direction: right 2 | 3 | ...@_base 4 | 5 | user01: User01 { 6 | label: User01 7 | class: [base; person; multiple] 8 | } 9 | 10 | user02: User02 { 11 | label: User02 12 | class: [base; person; multiple] 13 | } 14 | 15 | user03: User03 { 16 | label: User03 17 | class: [base; person; multiple] 18 | } 19 | 20 | user04: User04 { 21 | label: User04 22 | class: [base; person; multiple] 23 | } 24 | 25 | user01 -> container.task01: { 26 | label: Create Task 27 | class: [base; animated] 28 | } 29 | user02 -> container.task02: { 30 | label: Create Task 31 | class: [base; animated] 32 | } 33 | 34 | user03 -> container.task03: { 35 | label: Create Task 36 | class: [base; animated] 37 | } 38 | 39 | user04 -> container.task04: { 40 | label: Create Task 41 | class: [base; animated] 42 | } 43 | 44 | container: Application { 45 | direction: right 46 | style: { 47 | bold: true 48 | font-size: 28 49 | } 50 | icon: https://icons.terrastruct.com/dev%2Fgo.svg 51 | 52 | task01: { 53 | icon: https://icons.terrastruct.com/essentials%2F092-graph%20bar.svg 54 | class: [task; multiple] 55 | } 56 | 57 | task02: { 58 | icon: https://icons.terrastruct.com/essentials%2F095-download.svg 59 | class: [task; multiple] 60 | } 61 | 62 | task03: { 63 | icon: https://icons.terrastruct.com/essentials%2F195-attachment.svg 64 | class: [task; multiple] 65 | } 66 | 67 | task04: { 68 | icon: https://icons.terrastruct.com/essentials%2F213-alarm.svg 69 | class: [task; multiple] 70 | } 71 | 72 | queue: Queue Library { 73 | icon: https://icons.terrastruct.com/dev%2Fgo.svg 74 | style: { 75 | bold: true 76 | font-size: 32 77 | fill: honeydew 78 | } 79 | producer01: { 80 | label: Producer01 81 | class: library 82 | } 83 | 84 | producer02: { 85 | label: Producer02 86 | class: library 87 | } 88 | 89 | producer03: { 90 | label: Producer03 91 | class: library 92 | } 93 | 94 | producer04: { 95 | label: Producer04 96 | class: library 97 | } 98 | 99 | consumer01: { 100 | label: Consumer01 101 | class: library 102 | } 103 | 104 | consumer02: { 105 | label: Consumer02 106 | class: library 107 | } 108 | 109 | consumer03: { 110 | label: Consumer03 111 | class: library 112 | } 113 | 114 | consumer04: { 115 | label: Consumer04 116 | class: library 117 | } 118 | 119 | database: Ring\nBuffer { 120 | shape: cylinder 121 | style: { 122 | bold: true 123 | font-size: 32 124 | fill-pattern: lines 125 | font: mono 126 | } 127 | } 128 | 129 | redis: Redis { 130 | icon: https://icons.terrastruct.com/dev%2Fredis.svg 131 | shape: image 132 | style: { 133 | bold: true 134 | font-size: 32 135 | font: mono 136 | } 137 | } 138 | 139 | rabbitmq: RabbotMQ { 140 | icon: ./rabbitmq.svg 141 | shape: image 142 | style: { 143 | bold: true 144 | font-size: 32 145 | font: mono 146 | } 147 | } 148 | 149 | nats: NATS { 150 | icon: ./nats.svg 151 | shape: image 152 | style: { 153 | bold: true 154 | font-size: 32 155 | font: mono 156 | } 157 | } 158 | 159 | producer01 -> database 160 | producer02 -> redis 161 | producer03 -> rabbitmq 162 | producer04 -> nats 163 | database -> consumer01 164 | redis -> consumer02 165 | rabbitmq -> consumer03 166 | nats -> consumer04 167 | } 168 | 169 | worker01: { 170 | icon: https://icons.terrastruct.com/essentials%2F092-graph%20bar.svg 171 | class: [task] 172 | } 173 | 174 | worker02: { 175 | icon: https://icons.terrastruct.com/essentials%2F095-download.svg 176 | class: [task] 177 | } 178 | 179 | worker03: { 180 | icon: https://icons.terrastruct.com/essentials%2F092-graph%20bar.svg 181 | class: [task] 182 | } 183 | 184 | worker04: { 185 | icon: https://icons.terrastruct.com/essentials%2F213-alarm.svg 186 | class: [task] 187 | } 188 | 189 | task01 -> queue.producer01: { 190 | class: [base; enqueue] 191 | } 192 | task02 -> queue.producer02: { 193 | class: [base; enqueue] 194 | } 195 | task03 -> queue.producer03: { 196 | class: [base; enqueue] 197 | } 198 | task04 -> queue.producer04: { 199 | class: [base; enqueue] 200 | } 201 | queue.consumer01 -> worker01: { 202 | class: [base; dispatch] 203 | } 204 | queue.consumer02 -> worker02: { 205 | class: [base; dispatch] 206 | } 207 | queue.consumer03 -> worker03: { 208 | class: [base; dispatch] 209 | } 210 | queue.consumer04 -> worker04: { 211 | class: [base; dispatch] 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /images/flow-03.d2: -------------------------------------------------------------------------------- 1 | direction: right 2 | 3 | user01: User01 { 4 | shape: person 5 | style: { 6 | multiple: true 7 | bold: true 8 | font-size: 28 9 | } 10 | } 11 | 12 | user02: User02 { 13 | shape: person 14 | style: { 15 | multiple: true 16 | bold: true 17 | font-size: 28 18 | } 19 | } 20 | 21 | user03: User03 { 22 | shape: person 23 | style: { 24 | multiple: true 25 | bold: true 26 | font-size: 28 27 | } 28 | } 29 | 30 | user04: User04 { 31 | shape: person 32 | style: { 33 | multiple: true 34 | bold: true 35 | font-size: 28 36 | } 37 | } 38 | 39 | container_producer_01: Service01 { 40 | direction: right 41 | style: { 42 | bold: true 43 | font-size: 28 44 | } 45 | icon: https://icons.terrastruct.com/dev%2Fgo.svg 46 | 47 | task01: { 48 | icon: https://icons.terrastruct.com/essentials%2F092-graph%20bar.svg 49 | style: { 50 | multiple: true 51 | bold: true 52 | font-size: 32 53 | } 54 | } 55 | 56 | task02: { 57 | icon: https://icons.terrastruct.com/essentials%2F095-download.svg 58 | style: { 59 | multiple: true 60 | bold: true 61 | font-size: 32 62 | } 63 | } 64 | 65 | queue: Queue Library { 66 | icon: https://icons.terrastruct.com/dev%2Fgo.svg 67 | style: { 68 | bold: true 69 | font-size: 28 70 | fill: honeydew 71 | } 72 | 73 | producer: Producer { 74 | style: { 75 | bold: true 76 | font-size: 32 77 | fill: PapayaWhip 78 | fill-pattern: grain 79 | border-radius: 8 80 | font: mono 81 | } 82 | } 83 | } 84 | 85 | task01 -> queue.producer: Enqueue Task { 86 | style: { 87 | animated: true 88 | bold: true 89 | font-size: 28 90 | } 91 | } 92 | 93 | task02 -> queue.producer: Enqueue Task { 94 | style: { 95 | animated: true 96 | bold: true 97 | font-size: 28 98 | } 99 | } 100 | } 101 | 102 | container_producer_02: Service02 { 103 | direction: right 104 | style: { 105 | bold: true 106 | font-size: 28 107 | } 108 | icon: https://icons.terrastruct.com/dev%2Fgo.svg 109 | 110 | task01: { 111 | icon: https://icons.terrastruct.com/essentials%2F195-attachment.svg 112 | style: { 113 | multiple: true 114 | bold: true 115 | font-size: 32 116 | } 117 | } 118 | 119 | task02: { 120 | icon: https://icons.terrastruct.com/essentials%2F213-alarm.svg 121 | style: { 122 | multiple: true 123 | bold: true 124 | font-size: 32 125 | } 126 | } 127 | 128 | queue: Queue Library { 129 | icon: https://icons.terrastruct.com/dev%2Fgo.svg 130 | style: { 131 | bold: true 132 | font-size: 28 133 | fill: honeydew 134 | } 135 | 136 | producer: Producer { 137 | style: { 138 | bold: true 139 | font-size: 32 140 | fill: PapayaWhip 141 | fill-pattern: grain 142 | border-radius: 8 143 | font: mono 144 | } 145 | } 146 | } 147 | 148 | task01 -> queue.producer: Enqueue Task { 149 | style: { 150 | animated: true 151 | bold: true 152 | font-size: 28 153 | } 154 | } 155 | 156 | task02 -> queue.producer: Enqueue Task { 157 | style: { 158 | animated: true 159 | bold: true 160 | font-size: 28 161 | } 162 | } 163 | } 164 | 165 | database: Ring\nBuffer { 166 | shape: cylinder 167 | style: { 168 | bold: true 169 | font-size: 32 170 | fill-pattern: lines 171 | font: mono 172 | } 173 | } 174 | 175 | other_store: "" { 176 | style: { 177 | opacity: 0 178 | } 179 | 180 | redis: Redis { 181 | icon: https://icons.terrastruct.com/dev%2Fredis.svg 182 | shape: image 183 | style: { 184 | bold: true 185 | font-size: 32 186 | font: mono 187 | opacity: 0.7 188 | } 189 | } 190 | 191 | rabbitmq: RabbotMQ { 192 | icon: ./images/rabbitmq.svg 193 | shape: image 194 | style: { 195 | bold: true 196 | font-size: 32 197 | font: mono 198 | } 199 | } 200 | 201 | nats: NATS { 202 | icon: ./images/nats.svg 203 | shape: image 204 | style: { 205 | bold: true 206 | font-size: 32 207 | font: mono 208 | opacity: 0.7 209 | } 210 | } 211 | } 212 | 213 | container_consumer_01: Worker01 { 214 | direction: right 215 | style: { 216 | bold: true 217 | font-size: 28 218 | } 219 | icon: https://icons.terrastruct.com/dev%2Fgo.svg 220 | 221 | task01: { 222 | icon: https://icons.terrastruct.com/essentials%2F092-graph%20bar.svg 223 | style: { 224 | multiple: true 225 | bold: true 226 | font-size: 32 227 | } 228 | } 229 | 230 | task02: { 231 | icon: https://icons.terrastruct.com/essentials%2F095-download.svg 232 | style: { 233 | multiple: true 234 | bold: true 235 | font-size: 32 236 | } 237 | } 238 | 239 | queue: Queue Library { 240 | icon: https://icons.terrastruct.com/dev%2Fgo.svg 241 | style: { 242 | bold: true 243 | font-size: 28 244 | fill: honeydew 245 | } 246 | 247 | consumer: Consumer { 248 | style: { 249 | bold: true 250 | font-size: 32 251 | fill: PapayaWhip 252 | fill-pattern: grain 253 | border-radius: 8 254 | font: mono 255 | } 256 | } 257 | } 258 | 259 | queue.consumer -> task01: Handle Task { 260 | style: { 261 | animated: true 262 | bold: true 263 | font-size: 28 264 | } 265 | } 266 | 267 | queue.consumer -> task02: Handle Task { 268 | style: { 269 | bold: true 270 | font-size: 28 271 | } 272 | } 273 | } 274 | 275 | container_consumer_02: Worker02 { 276 | direction: right 277 | style: { 278 | bold: true 279 | font-size: 28 280 | } 281 | icon: https://icons.terrastruct.com/dev%2Fgo.svg 282 | 283 | task01: { 284 | icon: https://icons.terrastruct.com/essentials%2F095-download.svg 285 | style: { 286 | multiple: true 287 | bold: true 288 | font-size: 32 289 | } 290 | } 291 | 292 | task02: { 293 | icon: https://icons.terrastruct.com/essentials%2F092-graph%20bar.svg 294 | style: { 295 | multiple: true 296 | bold: true 297 | font-size: 32 298 | } 299 | } 300 | 301 | task03: { 302 | icon: https://icons.terrastruct.com/essentials%2F195-attachment.svg 303 | style: { 304 | multiple: true 305 | bold: true 306 | font-size: 32 307 | } 308 | } 309 | 310 | task04: { 311 | icon: https://icons.terrastruct.com/essentials%2F213-alarm.svg 312 | style: { 313 | multiple: true 314 | bold: true 315 | font-size: 32 316 | } 317 | } 318 | 319 | queue01: Queue Library { 320 | icon: https://icons.terrastruct.com/dev%2Fgo.svg 321 | style: { 322 | bold: true 323 | font-size: 28 324 | fill: honeydew 325 | } 326 | 327 | consumer: Consumer { 328 | style: { 329 | bold: true 330 | font-size: 32 331 | fill: PapayaWhip 332 | fill-pattern: grain 333 | border-radius: 8 334 | font: mono 335 | } 336 | } 337 | } 338 | 339 | queue02: Queue Library { 340 | icon: https://icons.terrastruct.com/dev%2Fgo.svg 341 | style: { 342 | bold: true 343 | font-size: 28 344 | fill: honeydew 345 | } 346 | 347 | consumer: Consumer { 348 | style: { 349 | bold: true 350 | font-size: 32 351 | fill: PapayaWhip 352 | fill-pattern: grain 353 | border-radius: 8 354 | font: mono 355 | } 356 | } 357 | } 358 | 359 | queue01.consumer -> task01: Handle Task { 360 | style: { 361 | animated: true 362 | bold: true 363 | font-size: 28 364 | } 365 | } 366 | 367 | queue01.consumer -> task02: Handle Task { 368 | style: { 369 | animated: true 370 | bold: true 371 | font-size: 28 372 | } 373 | } 374 | 375 | queue02.consumer -> task03: Handle Task { 376 | style: { 377 | animated: true 378 | bold: true 379 | font-size: 28 380 | } 381 | } 382 | 383 | queue02.consumer -> task04: Handle Task { 384 | style: { 385 | animated: true 386 | bold: true 387 | font-size: 28 388 | } 389 | } 390 | } 391 | 392 | # connection 393 | 394 | user01 -> container_producer_01.task01: Create Task { 395 | style: { 396 | animated: true 397 | bold: true 398 | font-size: 28 399 | } 400 | } 401 | 402 | user02 -> container_producer_01.task02: Create Task { 403 | style: { 404 | animated: true 405 | bold: true 406 | font-size: 28 407 | } 408 | } 409 | 410 | user03 -> container_producer_02.task01: Create Task { 411 | style: { 412 | animated: true 413 | bold: true 414 | font-size: 28 415 | } 416 | } 417 | 418 | user04 -> container_producer_02.task02: Create Task { 419 | style: { 420 | animated: true 421 | bold: true 422 | font-size: 28 423 | } 424 | } 425 | 426 | container_producer_01.queue.producer -> database: Store Task { 427 | style: { 428 | animated: true 429 | bold: true 430 | font-size: 28 431 | } 432 | } 433 | 434 | container_producer_02.queue.producer -> other_store.rabbitmq: Store Task { 435 | style: { 436 | animated: true 437 | bold: true 438 | font-size: 28 439 | } 440 | } 441 | 442 | container_producer_02.queue.producer -> other_store.redis: "" { 443 | style: { 444 | bold: true 445 | opacity: 0.3 446 | } 447 | } 448 | 449 | container_producer_02.queue.producer -> other_store.nats: "" { 450 | style: { 451 | bold: true 452 | opacity: 0.3 453 | } 454 | } 455 | 456 | database -> container_consumer_01.queue.consumer: Fetch Task { 457 | style: { 458 | animated: true 459 | bold: true 460 | font-size: 28 461 | } 462 | } 463 | 464 | database -> container_consumer_02.queue01.consumer: Fetch Task { 465 | style: { 466 | animated: true 467 | bold: true 468 | font-size: 28 469 | } 470 | } 471 | 472 | other_store.rabbitmq -> container_consumer_02.queue02.consumer: Fetch Task { 473 | style: { 474 | animated: true 475 | bold: true 476 | font-size: 28 477 | } 478 | } 479 | 480 | other_store.redis -> container_consumer_02.queue02.consumer: "" { 481 | style: { 482 | bold: true 483 | font-size: 28 484 | opacity: 0.3 485 | } 486 | } 487 | 488 | other_store.nats -> container_consumer_02.queue02.consumer: "" { 489 | style: { 490 | bold: true 491 | font-size: 28 492 | opacity: 0.3 493 | } 494 | } 495 | -------------------------------------------------------------------------------- /images/nats.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/rabbitmq.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /job/benchmark_test.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func BenchmarkNewTask(b *testing.B) { 10 | b.ReportAllocs() 11 | for i := 0; i < b.N; i++ { 12 | NewTask(func(context.Context) error { 13 | return nil 14 | }, 15 | AllowOption{ 16 | RetryCount: Int64(100), 17 | RetryDelay: Time(30 * time.Millisecond), 18 | Timeout: Time(3 * time.Millisecond), 19 | }, 20 | ) 21 | } 22 | } 23 | 24 | func BenchmarkNewMessage(b *testing.B) { 25 | b.ReportAllocs() 26 | for i := 0; i < b.N; i++ { 27 | _ = NewMessage(mockMessage{ 28 | message: "foo", 29 | }, 30 | AllowOption{ 31 | RetryCount: Int64(100), 32 | RetryDelay: Time(30 * time.Millisecond), 33 | Timeout: Time(3 * time.Millisecond), 34 | }, 35 | ) 36 | } 37 | } 38 | 39 | func BenchmarkNewOption(b *testing.B) { 40 | b.ReportAllocs() 41 | for i := 0; i < b.N; i++ { 42 | _ = NewOptions( 43 | AllowOption{ 44 | RetryCount: Int64(100), 45 | RetryDelay: Time(30 * time.Millisecond), 46 | Timeout: Time(3 * time.Millisecond), 47 | }, 48 | ) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /job/job.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "time" 7 | 8 | "github.com/golang-queue/queue/core" 9 | ) 10 | 11 | // TaskFunc is the task function 12 | type TaskFunc func(context.Context) error 13 | 14 | // Message describes a task and its metadata. 15 | type Message struct { 16 | Task TaskFunc `json:"-" msgpack:"-"` 17 | 18 | // Timeout is the duration the task can be processed by Handler. 19 | // zero if not specified 20 | // default is 60 time.Minute 21 | Timeout time.Duration `json:"timeout" msgpack:"timeout"` 22 | 23 | // Payload is the payload data of the task. 24 | Body []byte `json:"body" msgpack:"body"` 25 | 26 | // RetryCount set count of retry 27 | // default is 0, no retry. 28 | RetryCount int64 `json:"retry_count" msgpack:"retry_count"` 29 | 30 | // RetryDelay set delay between retry 31 | // default is 100ms 32 | RetryDelay time.Duration `json:"retry_delay" msgpack:"retry_delay"` 33 | 34 | // RetryFactor is the multiplying factor for each increment step. 35 | // 36 | // Defaults to 2. 37 | RetryFactor float64 `json:"retry_factor" msgpack:"retry_factor"` 38 | 39 | // Minimum value of the counter. 40 | // 41 | // Defaults to 100 milliseconds. 42 | RetryMin time.Duration `json:"retry_min" msgpack:"retry_min"` 43 | 44 | // Maximum value of the counter. 45 | // 46 | // Defaults to 10 seconds. 47 | RetryMax time.Duration `json:"retry_max" msgpack:"retry_max"` 48 | 49 | // Jitter eases contention by randomizing backoff steps 50 | Jitter bool `json:"jitter" msgpack:"jitter"` 51 | } 52 | 53 | // Payload returns the payload data of the Message. 54 | // It returns the byte slice of the payload. 55 | // 56 | // Returns: 57 | // - A byte slice containing the payload data. 58 | func (m *Message) Payload() []byte { 59 | return m.Body 60 | } 61 | 62 | // Bytes returns the byte slice of the Message struct. 63 | // If the marshalling process encounters an error, the function will panic. 64 | // It returns the marshalled byte slice. 65 | // 66 | // Returns: 67 | // - A byte slice containing the msgpack-encoded data. 68 | func (m *Message) Bytes() []byte { 69 | b, err := json.Marshal(m) 70 | if err != nil { 71 | panic(err) 72 | } 73 | 74 | return b 75 | } 76 | 77 | // NewMessage create new message 78 | func NewMessage(m core.QueuedMessage, opts ...AllowOption) Message { 79 | o := NewOptions(opts...) 80 | 81 | return Message{ 82 | RetryCount: o.retryCount, 83 | RetryDelay: o.retryDelay, 84 | RetryFactor: o.retryFactor, 85 | RetryMin: o.retryMin, 86 | RetryMax: o.retryMax, 87 | Timeout: o.timeout, 88 | Body: m.Bytes(), 89 | } 90 | } 91 | 92 | func NewTask(task TaskFunc, opts ...AllowOption) Message { 93 | o := NewOptions(opts...) 94 | 95 | return Message{ 96 | Timeout: o.timeout, 97 | RetryCount: o.retryCount, 98 | RetryDelay: o.retryDelay, 99 | RetryFactor: o.retryFactor, 100 | RetryMin: o.retryMin, 101 | RetryMax: o.retryMax, 102 | Task: task, 103 | } 104 | } 105 | 106 | // Encode takes a Message struct and marshals it into a byte slice using msgpack. 107 | // If the marshalling process encounters an error, the function will panic. 108 | // It returns the marshalled byte slice. 109 | // 110 | // Parameters: 111 | // - m: A pointer to the Message struct to be encoded. 112 | // 113 | // Returns: 114 | // - A byte slice containing the msgpack-encoded data. 115 | func Encode(m *Message) []byte { 116 | b, err := json.Marshal(m) 117 | if err != nil { 118 | panic(err) 119 | } 120 | 121 | return b 122 | } 123 | 124 | // Decode takes a byte slice and unmarshals it into a Message struct using msgpack. 125 | // If the unmarshalling process encounters an error, the function will panic. 126 | // It returns a pointer to the unmarshalled Message. 127 | // 128 | // Parameters: 129 | // - b: A byte slice containing the msgpack-encoded data. 130 | // 131 | // Returns: 132 | // - A pointer to the decoded Message struct. 133 | func Decode(b []byte) *Message { 134 | var msg Message 135 | err := json.Unmarshal(b, &msg) 136 | if err != nil { 137 | panic(err) 138 | } 139 | 140 | return &msg 141 | } 142 | -------------------------------------------------------------------------------- /job/job_test.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/appleboy/com/bytesconv" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | type mockMessage struct { 12 | message string 13 | } 14 | 15 | func (m mockMessage) Bytes() []byte { 16 | return bytesconv.StrToBytes(m.message) 17 | } 18 | 19 | func TestMessageEncodeDecode(t *testing.T) { 20 | m := NewMessage(&mockMessage{ 21 | message: "foo", 22 | }, 23 | AllowOption{ 24 | RetryCount: Int64(100), 25 | RetryDelay: Time(30 * time.Millisecond), 26 | Timeout: Time(3 * time.Millisecond), 27 | RetryMin: Time(200 * time.Millisecond), 28 | RetryMax: Time(20 * time.Second), 29 | RetryFactor: Float64(4.0), 30 | }, 31 | ) 32 | 33 | out := Decode(m.Bytes()) 34 | 35 | assert.Equal(t, int64(100), out.RetryCount) 36 | assert.Equal(t, 30*time.Millisecond, out.RetryDelay) 37 | assert.Equal(t, 3*time.Millisecond, out.Timeout) 38 | assert.Equal(t, "foo", string(out.Payload())) 39 | assert.Equal(t, 200*time.Millisecond, out.RetryMin) 40 | assert.Equal(t, 20*time.Second, out.RetryMax) 41 | assert.Equal(t, 4.0, out.RetryFactor) 42 | } 43 | -------------------------------------------------------------------------------- /job/option.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import "time" 4 | 5 | // Options is a set of options for the queue 6 | type Options struct { 7 | retryCount int64 8 | retryDelay time.Duration 9 | retryFactor float64 10 | retryMin time.Duration 11 | retryMax time.Duration 12 | jitter bool 13 | 14 | timeout time.Duration 15 | } 16 | 17 | // newDefaultOptions create new default options 18 | func newDefaultOptions() Options { 19 | return Options{ 20 | retryCount: 0, 21 | retryDelay: 0, 22 | retryFactor: 2, 23 | retryMin: 100 * time.Millisecond, 24 | retryMax: 10 * time.Second, 25 | timeout: 60 * time.Minute, 26 | jitter: false, 27 | } 28 | } 29 | 30 | // AllowOption is a function that sets some option on the Options 31 | type AllowOption struct { 32 | RetryCount *int64 33 | RetryDelay *time.Duration 34 | RetryFactor *float64 35 | RetryMin *time.Duration 36 | RetryMax *time.Duration 37 | Jitter *bool 38 | Timeout *time.Duration 39 | } 40 | 41 | // NewOptions create new options 42 | func NewOptions(opts ...AllowOption) Options { 43 | o := newDefaultOptions() 44 | 45 | if len(opts) != 0 { 46 | if opts[0].RetryCount != nil && *opts[0].RetryCount != o.retryCount { 47 | o.retryCount = *opts[0].RetryCount 48 | } 49 | 50 | if opts[0].RetryDelay != nil && *opts[0].RetryDelay != o.retryDelay { 51 | o.retryDelay = *opts[0].RetryDelay 52 | } 53 | 54 | if opts[0].Timeout != nil && *opts[0].Timeout != o.timeout { 55 | o.timeout = *opts[0].Timeout 56 | } 57 | 58 | if opts[0].RetryFactor != nil && *opts[0].RetryFactor != o.retryFactor { 59 | o.retryFactor = *opts[0].RetryFactor 60 | } 61 | 62 | if opts[0].RetryMin != nil && *opts[0].RetryMin != o.retryMin { 63 | o.retryMin = *opts[0].RetryMin 64 | } 65 | 66 | if opts[0].RetryMax != nil && *opts[0].RetryMax != o.retryMax { 67 | o.retryMax = *opts[0].RetryMax 68 | } 69 | 70 | if opts[0].Jitter != nil && *opts[0].Jitter != o.jitter { 71 | o.jitter = *opts[0].Jitter 72 | } 73 | } 74 | 75 | return o 76 | } 77 | 78 | // Int64 is a helper routine that allocates a new int64 value 79 | func Int64(val int64) *int64 { 80 | return &val 81 | } 82 | 83 | // Float64 is a helper routine that allocates a new float64 value 84 | func Float64(val float64) *float64 { 85 | return &val 86 | } 87 | 88 | // Time is a helper routine that allocates a new time value 89 | func Time(v time.Duration) *time.Duration { 90 | return &v 91 | } 92 | 93 | // Bool is a helper routine that allocates a new bool value 94 | func Bool(val bool) *bool { 95 | return &val 96 | } 97 | -------------------------------------------------------------------------------- /job/option_test.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | // TestMessageEncodeDecode test message encode and decode 11 | func TestOptions(t *testing.T) { 12 | o := NewOptions( 13 | AllowOption{ 14 | RetryCount: Int64(100), 15 | RetryDelay: Time(30 * time.Millisecond), 16 | Timeout: Time(3 * time.Millisecond), 17 | Jitter: Bool(true), 18 | }, 19 | ) 20 | 21 | assert.Equal(t, int64(100), o.retryCount) 22 | assert.Equal(t, 30*time.Millisecond, o.retryDelay) 23 | assert.Equal(t, 3*time.Millisecond, o.timeout) 24 | assert.Equal(t, 100*time.Millisecond, o.retryMin) 25 | assert.Equal(t, 10*time.Second, o.retryMax) 26 | assert.Equal(t, 2.0, o.retryFactor) 27 | assert.True(t, o.jitter) 28 | } 29 | -------------------------------------------------------------------------------- /logger.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | ) 8 | 9 | // Logger interface is used throughout gorush 10 | type Logger interface { 11 | Infof(format string, args ...any) 12 | Errorf(format string, args ...any) 13 | Fatalf(format string, args ...any) 14 | Info(args ...any) 15 | Error(args ...any) 16 | Fatal(args ...any) 17 | } 18 | 19 | // NewLogger for simple logger. 20 | func NewLogger() Logger { 21 | return defaultLogger{ 22 | infoLogger: log.New(os.Stderr, "INFO: ", log.Ldate|log.Ltime), 23 | errorLogger: log.New(os.Stderr, "ERROR: ", log.Ldate|log.Ltime), 24 | fatalLogger: log.New(os.Stderr, "FATAL: ", log.Ldate|log.Ltime), 25 | } 26 | } 27 | 28 | type defaultLogger struct { 29 | infoLogger *log.Logger 30 | errorLogger *log.Logger 31 | fatalLogger *log.Logger 32 | } 33 | 34 | func (l defaultLogger) logWithCallerf(logger *log.Logger, format string, args ...any) { 35 | stackInfo := stack(3) // Assuming stack(3) returns caller info string 36 | // Prepend stack info to the arguments and adjust the format string 37 | logger.Printf("%s "+format, append([]any{stackInfo}, args...)...) 38 | } 39 | 40 | func (l defaultLogger) logWithCaller(logger *log.Logger, args ...any) { 41 | stack := stack(3) 42 | logger.Println(append([]any{stack}, args...)...) 43 | } 44 | 45 | func (l defaultLogger) Infof(format string, args ...any) { 46 | l.infoLogger.Printf(format, args...) 47 | } 48 | 49 | func (l defaultLogger) Errorf(format string, args ...any) { 50 | l.errorLogger.Printf(format, args...) 51 | } 52 | 53 | func (l defaultLogger) Fatalf(format string, args ...any) { 54 | l.logWithCallerf(l.fatalLogger, format, args...) 55 | } 56 | 57 | func (l defaultLogger) Info(args ...any) { 58 | l.infoLogger.Println(fmt.Sprint(args...)) 59 | } 60 | 61 | func (l defaultLogger) Error(args ...any) { 62 | l.errorLogger.Println(fmt.Sprint(args...)) 63 | } 64 | 65 | func (l defaultLogger) Fatal(args ...any) { 66 | l.logWithCaller(l.fatalLogger, args...) 67 | } 68 | 69 | // NewEmptyLogger for simple logger. 70 | func NewEmptyLogger() Logger { 71 | return emptyLogger{} 72 | } 73 | 74 | // EmptyLogger no meesgae logger 75 | type emptyLogger struct{} 76 | 77 | func (l emptyLogger) Infof(format string, args ...any) {} 78 | func (l emptyLogger) Errorf(format string, args ...any) {} 79 | func (l emptyLogger) Fatalf(format string, args ...any) {} 80 | func (l emptyLogger) Info(args ...any) {} 81 | func (l emptyLogger) Error(args ...any) {} 82 | func (l emptyLogger) Fatal(args ...any) {} 83 | -------------------------------------------------------------------------------- /logger_test.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | func ExampleNewEmptyLogger() { 4 | l := NewEmptyLogger() 5 | l.Info("test") 6 | l.Infof("test") 7 | l.Error("test") 8 | l.Errorf("test") 9 | l.Fatal("test") 10 | l.Fatalf("test") 11 | // Output: 12 | } 13 | -------------------------------------------------------------------------------- /metric.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import "sync/atomic" 4 | 5 | // Metric interface 6 | type Metric interface { 7 | IncBusyWorker() 8 | DecBusyWorker() 9 | BusyWorkers() int64 10 | SuccessTasks() uint64 11 | FailureTasks() uint64 12 | SubmittedTasks() uint64 13 | CompletedTasks() uint64 14 | IncSuccessTask() 15 | IncFailureTask() 16 | IncSubmittedTask() 17 | } 18 | 19 | var _ Metric = (*metric)(nil) 20 | 21 | type metric struct { 22 | busyWorkers int64 23 | successTasks uint64 24 | failureTasks uint64 25 | submittedTasks uint64 26 | } 27 | 28 | // NewMetric for default metric structure 29 | func NewMetric() Metric { 30 | return &metric{} 31 | } 32 | 33 | func (m *metric) IncBusyWorker() { 34 | atomic.AddInt64(&m.busyWorkers, 1) 35 | } 36 | 37 | func (m *metric) DecBusyWorker() { 38 | atomic.AddInt64(&m.busyWorkers, ^int64(0)) 39 | } 40 | 41 | func (m *metric) BusyWorkers() int64 { 42 | return atomic.LoadInt64(&m.busyWorkers) 43 | } 44 | 45 | func (m *metric) IncSuccessTask() { 46 | atomic.AddUint64(&m.successTasks, 1) 47 | } 48 | 49 | func (m *metric) IncFailureTask() { 50 | atomic.AddUint64(&m.failureTasks, 1) 51 | } 52 | 53 | func (m *metric) IncSubmittedTask() { 54 | atomic.AddUint64(&m.submittedTasks, 1) 55 | } 56 | 57 | func (m *metric) SuccessTasks() uint64 { 58 | return atomic.LoadUint64(&m.successTasks) 59 | } 60 | 61 | func (m *metric) FailureTasks() uint64 { 62 | return atomic.LoadUint64(&m.failureTasks) 63 | } 64 | 65 | func (m *metric) SubmittedTasks() uint64 { 66 | return atomic.LoadUint64(&m.submittedTasks) 67 | } 68 | 69 | func (m *metric) CompletedTasks() uint64 { 70 | return atomic.LoadUint64(&m.successTasks) + atomic.LoadUint64(&m.failureTasks) 71 | } 72 | -------------------------------------------------------------------------------- /metric_test.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | "time" 8 | 9 | "github.com/golang-queue/queue/core" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestMetricData(t *testing.T) { 15 | w := NewRing( 16 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 17 | switch string(m.Payload()) { 18 | case "foo1": 19 | panic("missing something") 20 | case "foo2": 21 | return errors.New("missing something") 22 | case "foo3": 23 | return nil 24 | } 25 | return nil 26 | }), 27 | ) 28 | q, err := NewQueue( 29 | WithWorker(w), 30 | WithWorkerCount(4), 31 | ) 32 | assert.NoError(t, err) 33 | assert.NoError(t, q.Queue(mockMessage{ 34 | message: "foo1", 35 | })) 36 | assert.NoError(t, q.Queue(mockMessage{ 37 | message: "foo2", 38 | })) 39 | assert.NoError(t, q.Queue(mockMessage{ 40 | message: "foo3", 41 | })) 42 | assert.NoError(t, q.Queue(mockMessage{ 43 | message: "foo4", 44 | })) 45 | q.Start() 46 | time.Sleep(50 * time.Millisecond) 47 | assert.Equal(t, uint64(4), q.SubmittedTasks()) 48 | assert.Equal(t, uint64(2), q.SuccessTasks()) 49 | assert.Equal(t, uint64(2), q.FailureTasks()) 50 | assert.Equal(t, uint64(4), q.CompletedTasks()) 51 | q.Release() 52 | } 53 | -------------------------------------------------------------------------------- /mocks/mock_queued_message.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/golang-queue/queue/core (interfaces: QueuedMessage) 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -package=mocks -destination=mock_queued_message.go github.com/golang-queue/queue/core QueuedMessage 7 | // 8 | 9 | // Package mocks is a generated GoMock package. 10 | package mocks 11 | 12 | import ( 13 | reflect "reflect" 14 | 15 | gomock "go.uber.org/mock/gomock" 16 | ) 17 | 18 | // MockQueuedMessage is a mock of QueuedMessage interface. 19 | type MockQueuedMessage struct { 20 | ctrl *gomock.Controller 21 | recorder *MockQueuedMessageMockRecorder 22 | } 23 | 24 | // MockQueuedMessageMockRecorder is the mock recorder for MockQueuedMessage. 25 | type MockQueuedMessageMockRecorder struct { 26 | mock *MockQueuedMessage 27 | } 28 | 29 | // NewMockQueuedMessage creates a new mock instance. 30 | func NewMockQueuedMessage(ctrl *gomock.Controller) *MockQueuedMessage { 31 | mock := &MockQueuedMessage{ctrl: ctrl} 32 | mock.recorder = &MockQueuedMessageMockRecorder{mock} 33 | return mock 34 | } 35 | 36 | // EXPECT returns an object that allows the caller to indicate expected use. 37 | func (m *MockQueuedMessage) EXPECT() *MockQueuedMessageMockRecorder { 38 | return m.recorder 39 | } 40 | 41 | // Bytes mocks base method. 42 | func (m *MockQueuedMessage) Bytes() []byte { 43 | m.ctrl.T.Helper() 44 | ret := m.ctrl.Call(m, "Bytes") 45 | ret0, _ := ret[0].([]byte) 46 | return ret0 47 | } 48 | 49 | // Bytes indicates an expected call of Bytes. 50 | func (mr *MockQueuedMessageMockRecorder) Bytes() *gomock.Call { 51 | mr.mock.ctrl.T.Helper() 52 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bytes", reflect.TypeOf((*MockQueuedMessage)(nil).Bytes)) 53 | } 54 | -------------------------------------------------------------------------------- /mocks/mock_task_message.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/golang-queue/queue/core (interfaces: TaskMessage) 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -package=mocks -destination=mock_task_message.go github.com/golang-queue/queue/core TaskMessage 7 | // 8 | 9 | // Package mocks is a generated GoMock package. 10 | package mocks 11 | 12 | import ( 13 | reflect "reflect" 14 | 15 | gomock "go.uber.org/mock/gomock" 16 | ) 17 | 18 | // MockTaskMessage is a mock of TaskMessage interface. 19 | type MockTaskMessage struct { 20 | ctrl *gomock.Controller 21 | recorder *MockTaskMessageMockRecorder 22 | } 23 | 24 | // MockTaskMessageMockRecorder is the mock recorder for MockTaskMessage. 25 | type MockTaskMessageMockRecorder struct { 26 | mock *MockTaskMessage 27 | } 28 | 29 | // NewMockTaskMessage creates a new mock instance. 30 | func NewMockTaskMessage(ctrl *gomock.Controller) *MockTaskMessage { 31 | mock := &MockTaskMessage{ctrl: ctrl} 32 | mock.recorder = &MockTaskMessageMockRecorder{mock} 33 | return mock 34 | } 35 | 36 | // EXPECT returns an object that allows the caller to indicate expected use. 37 | func (m *MockTaskMessage) EXPECT() *MockTaskMessageMockRecorder { 38 | return m.recorder 39 | } 40 | 41 | // Bytes mocks base method. 42 | func (m *MockTaskMessage) Bytes() []byte { 43 | m.ctrl.T.Helper() 44 | ret := m.ctrl.Call(m, "Bytes") 45 | ret0, _ := ret[0].([]byte) 46 | return ret0 47 | } 48 | 49 | // Bytes indicates an expected call of Bytes. 50 | func (mr *MockTaskMessageMockRecorder) Bytes() *gomock.Call { 51 | mr.mock.ctrl.T.Helper() 52 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bytes", reflect.TypeOf((*MockTaskMessage)(nil).Bytes)) 53 | } 54 | 55 | // Payload mocks base method. 56 | func (m *MockTaskMessage) Payload() []byte { 57 | m.ctrl.T.Helper() 58 | ret := m.ctrl.Call(m, "Payload") 59 | ret0, _ := ret[0].([]byte) 60 | return ret0 61 | } 62 | 63 | // Payload indicates an expected call of Payload. 64 | func (mr *MockTaskMessageMockRecorder) Payload() *gomock.Call { 65 | mr.mock.ctrl.T.Helper() 66 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Payload", reflect.TypeOf((*MockTaskMessage)(nil).Payload)) 67 | } 68 | -------------------------------------------------------------------------------- /mocks/mock_worker.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/golang-queue/queue/core (interfaces: Worker) 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -package=mocks -destination=mock_worker.go github.com/golang-queue/queue/core Worker 7 | // 8 | 9 | // Package mocks is a generated GoMock package. 10 | package mocks 11 | 12 | import ( 13 | context "context" 14 | reflect "reflect" 15 | 16 | core "github.com/golang-queue/queue/core" 17 | gomock "go.uber.org/mock/gomock" 18 | ) 19 | 20 | // MockWorker is a mock of Worker interface. 21 | type MockWorker struct { 22 | ctrl *gomock.Controller 23 | recorder *MockWorkerMockRecorder 24 | } 25 | 26 | // MockWorkerMockRecorder is the mock recorder for MockWorker. 27 | type MockWorkerMockRecorder struct { 28 | mock *MockWorker 29 | } 30 | 31 | // NewMockWorker creates a new mock instance. 32 | func NewMockWorker(ctrl *gomock.Controller) *MockWorker { 33 | mock := &MockWorker{ctrl: ctrl} 34 | mock.recorder = &MockWorkerMockRecorder{mock} 35 | return mock 36 | } 37 | 38 | // EXPECT returns an object that allows the caller to indicate expected use. 39 | func (m *MockWorker) EXPECT() *MockWorkerMockRecorder { 40 | return m.recorder 41 | } 42 | 43 | // Queue mocks base method. 44 | func (m *MockWorker) Queue(arg0 core.TaskMessage) error { 45 | m.ctrl.T.Helper() 46 | ret := m.ctrl.Call(m, "Queue", arg0) 47 | ret0, _ := ret[0].(error) 48 | return ret0 49 | } 50 | 51 | // Queue indicates an expected call of Queue. 52 | func (mr *MockWorkerMockRecorder) Queue(arg0 any) *gomock.Call { 53 | mr.mock.ctrl.T.Helper() 54 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Queue", reflect.TypeOf((*MockWorker)(nil).Queue), arg0) 55 | } 56 | 57 | // Request mocks base method. 58 | func (m *MockWorker) Request() (core.TaskMessage, error) { 59 | m.ctrl.T.Helper() 60 | ret := m.ctrl.Call(m, "Request") 61 | ret0, _ := ret[0].(core.TaskMessage) 62 | ret1, _ := ret[1].(error) 63 | return ret0, ret1 64 | } 65 | 66 | // Request indicates an expected call of Request. 67 | func (mr *MockWorkerMockRecorder) Request() *gomock.Call { 68 | mr.mock.ctrl.T.Helper() 69 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Request", reflect.TypeOf((*MockWorker)(nil).Request)) 70 | } 71 | 72 | // Run mocks base method. 73 | func (m *MockWorker) Run(arg0 context.Context, arg1 core.TaskMessage) error { 74 | m.ctrl.T.Helper() 75 | ret := m.ctrl.Call(m, "Run", arg0, arg1) 76 | ret0, _ := ret[0].(error) 77 | return ret0 78 | } 79 | 80 | // Run indicates an expected call of Run. 81 | func (mr *MockWorkerMockRecorder) Run(arg0, arg1 any) *gomock.Call { 82 | mr.mock.ctrl.T.Helper() 83 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockWorker)(nil).Run), arg0, arg1) 84 | } 85 | 86 | // Shutdown mocks base method. 87 | func (m *MockWorker) Shutdown() error { 88 | m.ctrl.T.Helper() 89 | ret := m.ctrl.Call(m, "Shutdown") 90 | ret0, _ := ret[0].(error) 91 | return ret0 92 | } 93 | 94 | // Shutdown indicates an expected call of Shutdown. 95 | func (mr *MockWorkerMockRecorder) Shutdown() *gomock.Call { 96 | mr.mock.ctrl.T.Helper() 97 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Shutdown", reflect.TypeOf((*MockWorker)(nil).Shutdown)) 98 | } 99 | -------------------------------------------------------------------------------- /mocks/mocks.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | //go:generate mockgen -package=mocks -destination=mock_worker.go github.com/golang-queue/queue/core Worker 4 | //go:generate mockgen -package=mocks -destination=mock_queued_message.go github.com/golang-queue/queue/core QueuedMessage 5 | //go:generate mockgen -package=mocks -destination=mock_task_message.go github.com/golang-queue/queue/core TaskMessage 6 | -------------------------------------------------------------------------------- /options.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "context" 5 | "runtime" 6 | "time" 7 | 8 | "github.com/golang-queue/queue/core" 9 | ) 10 | 11 | var ( 12 | defaultCapacity = 0 13 | defaultWorkerCount = int64(runtime.NumCPU()) 14 | defaultNewLogger = NewLogger() 15 | defaultFn = func(context.Context, core.TaskMessage) error { return nil } 16 | defaultMetric = NewMetric() 17 | ) 18 | 19 | // An Option configures a mutex. 20 | type Option interface { 21 | apply(*Options) 22 | } 23 | 24 | // OptionFunc is a function that configures a queue. 25 | type OptionFunc func(*Options) 26 | 27 | // Apply calls f(option) 28 | func (f OptionFunc) apply(option *Options) { 29 | f(option) 30 | } 31 | 32 | // WithWorkerCount set worker count 33 | func WithWorkerCount(num int64) Option { 34 | return OptionFunc(func(q *Options) { 35 | if num <= 0 { 36 | num = defaultWorkerCount 37 | } 38 | q.workerCount = num 39 | }) 40 | } 41 | 42 | // WithQueueSize set worker count 43 | func WithQueueSize(num int) Option { 44 | return OptionFunc(func(q *Options) { 45 | q.queueSize = num 46 | }) 47 | } 48 | 49 | // WithLogger set custom logger 50 | func WithLogger(l Logger) Option { 51 | return OptionFunc(func(q *Options) { 52 | q.logger = l 53 | }) 54 | } 55 | 56 | // WithMetric set custom Metric 57 | func WithMetric(m Metric) Option { 58 | return OptionFunc(func(q *Options) { 59 | q.metric = m 60 | }) 61 | } 62 | 63 | // WithWorker set custom worker 64 | func WithWorker(w core.Worker) Option { 65 | return OptionFunc(func(q *Options) { 66 | q.worker = w 67 | }) 68 | } 69 | 70 | // WithFn set custom job function 71 | func WithFn(fn func(context.Context, core.TaskMessage) error) Option { 72 | return OptionFunc(func(q *Options) { 73 | q.fn = fn 74 | }) 75 | } 76 | 77 | // WithAfterFn set callback function after job done 78 | func WithAfterFn(afterFn func()) Option { 79 | return OptionFunc(func(q *Options) { 80 | q.afterFn = afterFn 81 | }) 82 | } 83 | 84 | // WithRetryInterval sets the retry interval 85 | func WithRetryInterval(d time.Duration) Option { 86 | return OptionFunc(func(q *Options) { 87 | q.retryInterval = d 88 | }) 89 | } 90 | 91 | // Options for custom args in Queue 92 | type Options struct { 93 | workerCount int64 94 | logger Logger 95 | queueSize int 96 | worker core.Worker 97 | fn func(context.Context, core.TaskMessage) error 98 | afterFn func() 99 | metric Metric 100 | retryInterval time.Duration 101 | } 102 | 103 | // NewOptions initialize the default value for the options 104 | func NewOptions(opts ...Option) *Options { 105 | o := &Options{ 106 | workerCount: defaultWorkerCount, 107 | queueSize: defaultCapacity, 108 | logger: defaultNewLogger, 109 | worker: nil, 110 | fn: defaultFn, 111 | metric: defaultMetric, 112 | retryInterval: time.Second, 113 | } 114 | 115 | // Loop through each option 116 | for _, opt := range opts { 117 | // Call the option giving the instantiated 118 | opt.apply(o) 119 | } 120 | 121 | return o 122 | } 123 | -------------------------------------------------------------------------------- /options_test.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestWithRetryInterval(t *testing.T) { 9 | tests := []struct { 10 | name string 11 | duration time.Duration 12 | want time.Duration 13 | }{ 14 | { 15 | name: "Set 2 seconds retry interval", 16 | duration: 2 * time.Second, 17 | want: 2 * time.Second, 18 | }, 19 | { 20 | name: "Set 500ms retry interval", 21 | duration: 500 * time.Millisecond, 22 | want: 500 * time.Millisecond, 23 | }, 24 | { 25 | name: "Set zero retry interval", 26 | duration: 0, 27 | want: 0, 28 | }, 29 | { 30 | name: "Set negative retry interval", 31 | duration: -1 * time.Second, 32 | want: -1 * time.Second, 33 | }, 34 | } 35 | 36 | for _, tt := range tests { 37 | t.Run(tt.name, func(t *testing.T) { 38 | opts := NewOptions(WithRetryInterval(tt.duration)) 39 | if opts.retryInterval != tt.want { 40 | t.Errorf("WithRetryInterval() = %v, want %v", opts.retryInterval, tt.want) 41 | } 42 | }) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /pool.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | // NewPool initializes a new pool 4 | func NewPool(size int64, opts ...Option) *Queue { 5 | o := []Option{ 6 | WithWorkerCount(size), 7 | WithWorker(NewRing(opts...)), 8 | } 9 | o = append( 10 | o, 11 | opts..., 12 | ) 13 | 14 | q, err := NewQueue(o...) 15 | if err != nil { 16 | panic(err) 17 | } 18 | 19 | q.Start() 20 | 21 | return q 22 | } 23 | -------------------------------------------------------------------------------- /pool_test.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestNewPoolWithQueueTask(t *testing.T) { 11 | totalN := int64(5) 12 | taskN := 100 13 | rets := make(chan struct{}, taskN) 14 | 15 | p := NewPool(totalN) 16 | for i := 0; i < taskN; i++ { 17 | assert.NoError(t, p.QueueTask(func(context.Context) error { 18 | rets <- struct{}{} 19 | return nil 20 | })) 21 | } 22 | 23 | for i := 0; i < taskN; i++ { 24 | <-rets 25 | } 26 | 27 | // shutdown all, and now running worker is 0 28 | p.Release() 29 | assert.Equal(t, int64(0), p.BusyWorkers()) 30 | } 31 | 32 | func TestPoolNumber(t *testing.T) { 33 | p := NewPool(0) 34 | p.Start() 35 | // shutdown all, and now running worker is 0 36 | p.Release() 37 | assert.Equal(t, int64(0), p.BusyWorkers()) 38 | } 39 | -------------------------------------------------------------------------------- /queue.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | // Package queue provides a high-performance, extensible message queue implementation 4 | // supporting multiple workers, job retries, dynamic scaling, and graceful shutdown. 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | "sync" 10 | "sync/atomic" 11 | "time" 12 | 13 | "github.com/golang-queue/queue/core" 14 | "github.com/golang-queue/queue/job" 15 | 16 | "github.com/jpillora/backoff" 17 | ) 18 | 19 | /* 20 | ErrQueueShutdown is returned when an operation is attempted on a queue 21 | that has already been closed and released. 22 | */ 23 | var ErrQueueShutdown = errors.New("queue has been closed and released") 24 | 25 | type ( 26 | // Queue represents a message queue with worker management, job scheduling, 27 | // retry logic, and graceful shutdown capabilities. 28 | Queue struct { 29 | sync.Mutex // Mutex to protect concurrent access to queue state 30 | metric *metric // Metrics collector for tracking queue and worker stats 31 | logger Logger // Logger for queue events and errors 32 | workerCount int64 // Number of worker goroutines to process jobs 33 | routineGroup *routineGroup // Group to manage and wait for goroutines 34 | quit chan struct{} // Channel to signal shutdown to all goroutines 35 | ready chan struct{} // Channel to signal worker readiness 36 | notify chan struct{} // Channel to notify workers of new jobs 37 | worker core.Worker // The worker implementation that processes jobs 38 | stopOnce sync.Once // Ensures shutdown is only performed once 39 | stopFlag int32 // Atomic flag indicating if shutdown has started 40 | afterFn func() // Optional callback after each job execution 41 | retryInterval time.Duration // Interval for retrying job requests 42 | } 43 | ) 44 | 45 | /* 46 | ErrMissingWorker is returned when a queue is created without a worker implementation. 47 | */ 48 | var ErrMissingWorker = errors.New("missing worker module") 49 | 50 | // NewQueue creates and returns a new Queue instance with the provided options. 51 | // Returns an error if no worker is specified. 52 | func NewQueue(opts ...Option) (*Queue, error) { 53 | o := NewOptions(opts...) 54 | q := &Queue{ 55 | routineGroup: newRoutineGroup(), // Manages all goroutines spawned by the queue 56 | quit: make(chan struct{}), // Signals shutdown to all goroutines 57 | ready: make(chan struct{}, 1), // Signals when a worker is ready to process a job 58 | notify: make(chan struct{}, 1), // Notifies workers of new jobs 59 | workerCount: o.workerCount, // Number of worker goroutines 60 | logger: o.logger, // Logger for queue events 61 | worker: o.worker, // Worker implementation 62 | metric: &metric{}, // Metrics collector 63 | afterFn: o.afterFn, // Optional post-job callback 64 | retryInterval: o.retryInterval, // Interval for retrying job requests 65 | } 66 | 67 | if q.worker == nil { 68 | return nil, ErrMissingWorker 69 | } 70 | 71 | return q, nil 72 | } 73 | 74 | // Start launches all worker goroutines and begins processing jobs. 75 | // If workerCount is zero, Start is a no-op. 76 | func (q *Queue) Start() { 77 | q.Lock() 78 | count := q.workerCount 79 | q.Unlock() 80 | if count == 0 { 81 | return 82 | } 83 | q.routineGroup.Run(func() { 84 | q.start() 85 | }) 86 | } 87 | 88 | // Shutdown initiates a graceful shutdown of the queue. 89 | // It signals all goroutines to stop, shuts down the worker, and closes the quit channel. 90 | // Shutdown is idempotent and safe to call multiple times. 91 | func (q *Queue) Shutdown() { 92 | if !atomic.CompareAndSwapInt32(&q.stopFlag, 0, 1) { 93 | return 94 | } 95 | 96 | q.stopOnce.Do(func() { 97 | if q.metric.BusyWorkers() > 0 { 98 | q.logger.Infof("shutdown all tasks: %d workers", q.metric.BusyWorkers()) 99 | } 100 | 101 | if err := q.worker.Shutdown(); err != nil { 102 | q.logger.Error(err) 103 | } 104 | close(q.quit) 105 | }) 106 | } 107 | 108 | // Release performs a graceful shutdown and waits for all goroutines to finish. 109 | func (q *Queue) Release() { 110 | q.Shutdown() 111 | q.Wait() 112 | } 113 | 114 | // BusyWorkers returns the number of workers currently processing jobs. 115 | func (q *Queue) BusyWorkers() int64 { 116 | return q.metric.BusyWorkers() 117 | } 118 | 119 | // SuccessTasks returns the number of successfully completed tasks. 120 | func (q *Queue) SuccessTasks() uint64 { 121 | return q.metric.SuccessTasks() 122 | } 123 | 124 | // FailureTasks returns the number of failed tasks. 125 | func (q *Queue) FailureTasks() uint64 { 126 | return q.metric.FailureTasks() 127 | } 128 | 129 | // SubmittedTasks returns the number of tasks submitted to the queue. 130 | func (q *Queue) SubmittedTasks() uint64 { 131 | return q.metric.SubmittedTasks() 132 | } 133 | 134 | // CompletedTasks returns the total number of completed tasks (success + failure). 135 | func (q *Queue) CompletedTasks() uint64 { 136 | return q.metric.CompletedTasks() 137 | } 138 | 139 | // Wait blocks until all goroutines in the routine group have finished. 140 | func (q *Queue) Wait() { 141 | q.routineGroup.Wait() 142 | } 143 | 144 | // Queue enqueues a single job (core.QueuedMessage) into the queue. 145 | // Accepts job options for customization. 146 | func (q *Queue) Queue(message core.QueuedMessage, opts ...job.AllowOption) error { 147 | data := job.NewMessage(message, opts...) 148 | 149 | return q.queue(&data) 150 | } 151 | 152 | // QueueTask enqueues a single task function into the queue. 153 | // Accepts job options for customization. 154 | func (q *Queue) QueueTask(task job.TaskFunc, opts ...job.AllowOption) error { 155 | data := job.NewTask(task, opts...) 156 | return q.queue(&data) 157 | } 158 | 159 | // queue is an internal helper to enqueue a job.Message into the worker. 160 | // It increments the submitted task metric and notifies workers if possible. 161 | func (q *Queue) queue(m *job.Message) error { 162 | if atomic.LoadInt32(&q.stopFlag) == 1 { 163 | return ErrQueueShutdown 164 | } 165 | 166 | if err := q.worker.Queue(m); err != nil { 167 | return err 168 | } 169 | 170 | q.metric.IncSubmittedTask() 171 | // Notify a worker that a new job is available. 172 | // If the notify channel is full, the worker is busy and we avoid blocking. 173 | select { 174 | case q.notify <- struct{}{}: 175 | default: 176 | } 177 | 178 | return nil 179 | } 180 | 181 | // work executes a single task, handling panics and updating metrics accordingly. 182 | // After execution, it schedules the next worker if needed. 183 | func (q *Queue) work(task core.TaskMessage) { 184 | var err error 185 | // Defer block to handle panics, update metrics, and run afterFn callback. 186 | defer func() { 187 | q.metric.DecBusyWorker() 188 | e := recover() 189 | if e != nil { 190 | q.logger.Fatalf("panic error: %v", e) 191 | } 192 | q.schedule() 193 | 194 | // Update success or failure metrics based on execution result. 195 | if err == nil && e == nil { 196 | q.metric.IncSuccessTask() 197 | } else { 198 | q.metric.IncFailureTask() 199 | } 200 | if q.afterFn != nil { 201 | q.afterFn() 202 | } 203 | }() 204 | 205 | if err = q.run(task); err != nil { 206 | q.logger.Errorf("runtime error: %s", err.Error()) 207 | } 208 | } 209 | 210 | // run dispatches the task to the appropriate handler based on its type. 211 | // Returns an error if the task type is invalid. 212 | func (q *Queue) run(task core.TaskMessage) error { 213 | switch t := task.(type) { 214 | case *job.Message: 215 | return q.handle(t) 216 | default: 217 | return errors.New("invalid task type") 218 | } 219 | } 220 | 221 | // handle executes a job.Message, supporting retries, timeouts, and panic recovery. 222 | // Returns an error if the job fails or times out. 223 | func (q *Queue) handle(m *job.Message) error { 224 | // done: receives the result of the job execution 225 | // panicChan: receives any panic that occurs in the job goroutine 226 | done := make(chan error, 1) 227 | panicChan := make(chan any, 1) 228 | startTime := time.Now() 229 | ctx, cancel := context.WithTimeout(context.Background(), m.Timeout) 230 | defer func() { 231 | cancel() 232 | }() 233 | 234 | // Run the job in a separate goroutine to support timeout and panic recovery. 235 | go func() { 236 | // Defer block to catch panics and send to panicChan 237 | defer func() { 238 | if p := recover(); p != nil { 239 | panicChan <- p 240 | } 241 | }() 242 | 243 | var err error 244 | 245 | // Set up backoff for retry logic 246 | b := &backoff.Backoff{ 247 | Min: m.RetryMin, 248 | Max: m.RetryMax, 249 | Factor: m.RetryFactor, 250 | Jitter: m.Jitter, 251 | } 252 | delay := m.RetryDelay 253 | loop: 254 | for { 255 | // If a custom Task function is provided, use it; otherwise, use the worker's Run method. 256 | if m.Task != nil { 257 | err = m.Task(ctx) 258 | } else { 259 | err = q.worker.Run(ctx, m) 260 | } 261 | 262 | // If no error or no retries left, exit loop. 263 | if err == nil || m.RetryCount == 0 { 264 | break 265 | } 266 | m.RetryCount-- 267 | 268 | // If no fixed retry delay, use backoff. 269 | if m.RetryDelay == 0 { 270 | delay = b.Duration() 271 | } 272 | 273 | select { 274 | case <-time.After(delay): // Wait before retrying 275 | q.logger.Infof("retry remaining times: %d, delay time: %s", m.RetryCount, delay) 276 | case <-ctx.Done(): // Timeout reached 277 | err = ctx.Err() 278 | break loop 279 | } 280 | } 281 | 282 | done <- err 283 | }() 284 | 285 | select { 286 | case p := <-panicChan: 287 | panic(p) 288 | case <-ctx.Done(): // Timeout reached 289 | return ctx.Err() 290 | case <-q.quit: // Queue is shutting down 291 | // Cancel job and wait for remaining time or job completion 292 | cancel() 293 | leftTime := m.Timeout - time.Since(startTime) 294 | select { 295 | case <-time.After(leftTime): 296 | return context.DeadlineExceeded 297 | case err := <-done: // Job finished 298 | return err 299 | case p := <-panicChan: 300 | panic(p) 301 | } 302 | case err := <-done: // Job finished 303 | return err 304 | } 305 | } 306 | 307 | // UpdateWorkerCount dynamically updates the number of worker goroutines. 308 | // Triggers scheduling to adjust to the new worker count. 309 | func (q *Queue) UpdateWorkerCount(num int64) { 310 | q.Lock() 311 | q.workerCount = num 312 | q.Unlock() 313 | q.schedule() 314 | } 315 | 316 | // schedule checks if more workers can be started based on the current busy count. 317 | // If so, it signals readiness to start a new worker. 318 | func (q *Queue) schedule() { 319 | q.Lock() 320 | defer q.Unlock() 321 | if q.BusyWorkers() >= q.workerCount { 322 | return 323 | } 324 | 325 | select { 326 | case q.ready <- struct{}{}: 327 | default: 328 | } 329 | } 330 | 331 | /* 332 | start launches the main worker loop, which manages job scheduling and execution. 333 | 334 | - It uses a ticker to periodically retry job requests if the queue is empty. 335 | - For each available worker slot, it requests a new task from the worker. 336 | - If a task is available, it is sent to the tasks channel and processed by a new goroutine. 337 | - The loop exits when the quit channel is closed. 338 | */ 339 | func (q *Queue) start() { 340 | tasks := make(chan core.TaskMessage, 1) 341 | ticker := time.NewTicker(q.retryInterval) 342 | defer ticker.Stop() 343 | 344 | for { 345 | // Ensure the number of busy workers does not exceed the configured worker count. 346 | q.schedule() 347 | 348 | select { 349 | case <-q.ready: // Wait for a worker slot to become available 350 | case <-q.quit: // Shutdown signal received 351 | return 352 | } 353 | 354 | // Request a task from the worker in a background goroutine. 355 | q.routineGroup.Run(func() { 356 | for { 357 | t, err := q.worker.Request() 358 | if t == nil || err != nil { 359 | if err != nil { 360 | select { 361 | case <-q.quit: 362 | if !errors.Is(err, ErrNoTaskInQueue) { 363 | close(tasks) 364 | return 365 | } 366 | case <-ticker.C: 367 | case <-q.notify: 368 | } 369 | } 370 | } 371 | if t != nil { 372 | tasks <- t 373 | return 374 | } 375 | 376 | select { 377 | case <-q.quit: 378 | if !errors.Is(err, ErrNoTaskInQueue) { 379 | close(tasks) 380 | return 381 | } 382 | default: 383 | } 384 | } 385 | }) 386 | 387 | task, ok := <-tasks 388 | if !ok { 389 | return 390 | } 391 | 392 | // Start processing the new task in a separate goroutine. 393 | q.metric.IncBusyWorker() 394 | q.routineGroup.Run(func() { 395 | q.work(task) 396 | }) 397 | } 398 | } 399 | -------------------------------------------------------------------------------- /queue_example_test.go: -------------------------------------------------------------------------------- 1 | package queue_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "time" 8 | 9 | "github.com/golang-queue/queue" 10 | "github.com/golang-queue/queue/job" 11 | ) 12 | 13 | func ExampleNewPool_queueTask() { 14 | taskN := 7 15 | rets := make(chan int, taskN) 16 | // allocate a pool with 5 goroutines to deal with those tasks 17 | p := queue.NewPool(5) 18 | // don't forget to release the pool in the end 19 | defer p.Release() 20 | 21 | // assign tasks to asynchronous goroutine pool 22 | for i := 0; i < taskN; i++ { 23 | idx := i 24 | if err := p.QueueTask(func(context.Context) error { 25 | // sleep and return the index 26 | time.Sleep(20 * time.Millisecond) 27 | rets <- idx 28 | return nil 29 | }); err != nil { 30 | log.Println(err) 31 | } 32 | } 33 | 34 | // wait until all tasks done 35 | for i := 0; i < taskN; i++ { 36 | fmt.Println("index:", <-rets) 37 | } 38 | 39 | // Unordered output: 40 | // index: 3 41 | // index: 0 42 | // index: 2 43 | // index: 4 44 | // index: 5 45 | // index: 6 46 | // index: 1 47 | } 48 | 49 | func ExampleNewPool_queueTaskTimeout() { 50 | taskN := 7 51 | rets := make(chan int, taskN) 52 | resps := make(chan error, 1) 53 | // allocate a pool with 5 goroutines to deal with those tasks 54 | q := queue.NewPool(5) 55 | // don't forget to release the pool in the end 56 | defer q.Release() 57 | 58 | // assign tasks to asynchronous goroutine pool 59 | for i := 0; i < taskN; i++ { 60 | idx := i 61 | if err := q.QueueTask(func(ctx context.Context) error { 62 | // panic job 63 | if idx == 5 { 64 | panic("system error") 65 | } 66 | // timeout job 67 | if idx == 6 { 68 | time.Sleep(105 * time.Millisecond) 69 | } 70 | select { 71 | case <-ctx.Done(): 72 | resps <- ctx.Err() 73 | default: 74 | } 75 | 76 | rets <- idx 77 | return nil 78 | }, job.AllowOption{ 79 | Timeout: job.Time(100 * time.Millisecond), 80 | }); err != nil { 81 | log.Println(err) 82 | } 83 | } 84 | 85 | // wait until all tasks done 86 | for i := 0; i < taskN-1; i++ { 87 | fmt.Println("index:", <-rets) 88 | } 89 | close(resps) 90 | for e := range resps { 91 | fmt.Println(e.Error()) 92 | } 93 | 94 | fmt.Println("success task count:", q.SuccessTasks()) 95 | fmt.Println("failure task count:", q.FailureTasks()) 96 | fmt.Println("submitted task count:", q.SubmittedTasks()) 97 | 98 | // Unordered output: 99 | // index: 3 100 | // index: 0 101 | // index: 2 102 | // index: 4 103 | // index: 6 104 | // index: 1 105 | // context deadline exceeded 106 | // success task count: 5 107 | // failure task count: 2 108 | // submitted task count: 7 109 | } 110 | -------------------------------------------------------------------------------- /queue_test.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | "time" 8 | 9 | "github.com/appleboy/com/bytesconv" 10 | "github.com/golang-queue/queue/core" 11 | "github.com/golang-queue/queue/job" 12 | "github.com/golang-queue/queue/mocks" 13 | 14 | "github.com/stretchr/testify/assert" 15 | "go.uber.org/goleak" 16 | "go.uber.org/mock/gomock" 17 | ) 18 | 19 | func TestMain(m *testing.M) { 20 | goleak.VerifyTestMain(m) 21 | } 22 | 23 | type mockMessage struct { 24 | message string 25 | } 26 | 27 | func (m mockMessage) Bytes() []byte { 28 | return bytesconv.StrToBytes(m.message) 29 | } 30 | 31 | func (m mockMessage) Payload() []byte { 32 | return bytesconv.StrToBytes(m.message) 33 | } 34 | 35 | func TestNewQueueWithZeroWorker(t *testing.T) { 36 | controller := gomock.NewController(t) 37 | defer controller.Finish() 38 | 39 | q, err := NewQueue() 40 | assert.Error(t, err) 41 | assert.Nil(t, q) 42 | 43 | w := mocks.NewMockWorker(controller) 44 | w.EXPECT().Shutdown().Return(nil) 45 | w.EXPECT().Request().Return(nil, nil).AnyTimes() 46 | q, err = NewQueue( 47 | WithWorker(w), 48 | WithWorkerCount(0), 49 | ) 50 | assert.NoError(t, err) 51 | assert.NotNil(t, q) 52 | 53 | q.Start() 54 | time.Sleep(50 * time.Millisecond) 55 | assert.Equal(t, int64(0), q.BusyWorkers()) 56 | q.Release() 57 | } 58 | 59 | func TestNewQueueWithDefaultWorker(t *testing.T) { 60 | controller := gomock.NewController(t) 61 | defer controller.Finish() 62 | 63 | q, err := NewQueue() 64 | assert.Error(t, err) 65 | assert.Nil(t, q) 66 | 67 | w := mocks.NewMockWorker(controller) 68 | m := mocks.NewMockTaskMessage(controller) 69 | m.EXPECT().Bytes().Return([]byte("test")).AnyTimes() 70 | m.EXPECT().Payload().Return([]byte("test")).AnyTimes() 71 | w.EXPECT().Shutdown().Return(nil) 72 | w.EXPECT().Request().Return(m, nil).AnyTimes() 73 | w.EXPECT().Run(context.Background(), m).Return(nil).AnyTimes() 74 | q, err = NewQueue( 75 | WithWorker(w), 76 | ) 77 | assert.NoError(t, err) 78 | assert.NotNil(t, q) 79 | 80 | q.Start() 81 | q.Release() 82 | assert.Equal(t, int64(0), q.BusyWorkers()) 83 | } 84 | 85 | func TestHandleTimeout(t *testing.T) { 86 | m := &job.Message{ 87 | Timeout: 100 * time.Millisecond, 88 | Body: []byte("foo"), 89 | } 90 | w := NewRing( 91 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 92 | time.Sleep(200 * time.Millisecond) 93 | return nil 94 | }), 95 | ) 96 | 97 | q, err := NewQueue( 98 | WithWorker(w), 99 | ) 100 | assert.NoError(t, err) 101 | assert.NotNil(t, q) 102 | 103 | err = q.handle(m) 104 | assert.Error(t, err) 105 | assert.Equal(t, context.DeadlineExceeded, err) 106 | 107 | done := make(chan error) 108 | go func() { 109 | done <- q.handle(m) 110 | }() 111 | 112 | err = <-done 113 | assert.Error(t, err) 114 | assert.Equal(t, context.DeadlineExceeded, err) 115 | } 116 | 117 | func TestJobComplete(t *testing.T) { 118 | m := &job.Message{ 119 | Timeout: 100 * time.Millisecond, 120 | Body: []byte("foo"), 121 | } 122 | w := NewRing( 123 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 124 | return errors.New("job completed") 125 | }), 126 | ) 127 | 128 | q, err := NewQueue( 129 | WithWorker(w), 130 | ) 131 | assert.NoError(t, err) 132 | assert.NotNil(t, q) 133 | 134 | err = q.handle(m) 135 | assert.Error(t, err) 136 | assert.Equal(t, errors.New("job completed"), err) 137 | 138 | m = &job.Message{ 139 | Timeout: 250 * time.Millisecond, 140 | Body: []byte("foo"), 141 | } 142 | 143 | w = NewRing( 144 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 145 | time.Sleep(200 * time.Millisecond) 146 | return errors.New("job completed") 147 | }), 148 | ) 149 | 150 | q, err = NewQueue( 151 | WithWorker(w), 152 | ) 153 | assert.NoError(t, err) 154 | assert.NotNil(t, q) 155 | 156 | err = q.handle(m) 157 | assert.Error(t, err) 158 | assert.Equal(t, errors.New("job completed"), err) 159 | } 160 | 161 | func TestTaskJobComplete(t *testing.T) { 162 | m := &job.Message{ 163 | Timeout: 100 * time.Millisecond, 164 | Task: func(ctx context.Context) error { 165 | return errors.New("job completed") 166 | }, 167 | } 168 | w := NewRing() 169 | 170 | q, err := NewQueue( 171 | WithWorker(w), 172 | ) 173 | assert.NoError(t, err) 174 | assert.NotNil(t, q) 175 | 176 | err = q.handle(m) 177 | assert.Error(t, err) 178 | assert.Equal(t, errors.New("job completed"), err) 179 | 180 | m = &job.Message{ 181 | Timeout: 250 * time.Millisecond, 182 | Task: func(ctx context.Context) error { 183 | return nil 184 | }, 185 | } 186 | 187 | assert.NoError(t, q.handle(m)) 188 | 189 | // job timeout 190 | m = &job.Message{ 191 | Timeout: 50 * time.Millisecond, 192 | Task: func(ctx context.Context) error { 193 | time.Sleep(60 * time.Millisecond) 194 | return nil 195 | }, 196 | } 197 | assert.Equal(t, context.DeadlineExceeded, q.handle(m)) 198 | } 199 | 200 | func TestMockWorkerAndMessage(t *testing.T) { 201 | controller := gomock.NewController(t) 202 | defer controller.Finish() 203 | 204 | m := mocks.NewMockTaskMessage(controller) 205 | 206 | w := mocks.NewMockWorker(controller) 207 | w.EXPECT().Shutdown().Return(nil) 208 | w.EXPECT().Request().DoAndReturn(func() (core.TaskMessage, error) { 209 | return m, errors.New("nil") 210 | }) 211 | 212 | q, err := NewQueue( 213 | WithWorker(w), 214 | WithWorkerCount(1), 215 | ) 216 | assert.NoError(t, err) 217 | assert.NotNil(t, q) 218 | q.Start() 219 | time.Sleep(50 * time.Millisecond) 220 | q.Release() 221 | } 222 | -------------------------------------------------------------------------------- /recovery.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "runtime" 8 | "strings" 9 | "sync" 10 | ) 11 | 12 | const ( 13 | dunno = "???" 14 | centerDot = "·" 15 | dot = "." 16 | slash = "/" 17 | ) 18 | 19 | // bufferPool is a pool of byte buffers that can be reused to reduce the number 20 | // of allocations and improve performance. It uses sync.Pool to manage a pool 21 | // of reusable *bytes.Buffer objects. When a buffer is requested from the pool, 22 | // if one is available, it is returned; otherwise, a new buffer is created. 23 | // When a buffer is no longer needed, it should be put back into the pool to be 24 | // reused. 25 | var bufferPool = sync.Pool{ 26 | New: func() any { 27 | return new(bytes.Buffer) 28 | }, 29 | } 30 | 31 | // stack captures and returns the current stack trace, skipping the specified number of frames. 32 | // It retrieves the stack trace information, including the file name, line number, and function name, 33 | // and formats it into a byte slice. The function uses a buffer pool to manage memory efficiently. 34 | // 35 | // Parameters: 36 | // - skip: The number of stack frames to skip before recording the trace. 37 | // 38 | // Returns: 39 | // - A byte slice containing the formatted stack trace. 40 | func stack(skip int) []byte { 41 | buf := bufferPool.Get().(*bytes.Buffer) 42 | defer bufferPool.Put(buf) 43 | buf.Reset() 44 | 45 | var lines [][]byte 46 | var lastFile string 47 | for i := skip; ; i++ { 48 | pc, file, line, ok := runtime.Caller(i) 49 | if !ok { 50 | break 51 | } 52 | fmt.Fprintf(buf, "%s:%d (0x%x)\n", file, line, pc) 53 | if file != lastFile { 54 | data, err := os.ReadFile(file) 55 | if err != nil { 56 | continue 57 | } 58 | lines = bytes.Split(data, []byte{'\n'}) 59 | lastFile = file 60 | } 61 | fmt.Fprintf(buf, "\t%s: %s\n", function(pc), source(lines, line)) 62 | } 63 | return buf.Bytes() 64 | } 65 | 66 | // source retrieves the n-th line from the provided slice of byte slices, 67 | // trims any leading and trailing whitespace, and returns it as a byte slice. 68 | // If n is out of range, it returns a default "dunno" byte slice. 69 | // 70 | // Parameters: 71 | // 72 | // lines - a slice of byte slices representing lines of text 73 | // n - the 1-based index of the line to retrieve 74 | // 75 | // Returns: 76 | // 77 | // A byte slice containing the trimmed n-th line, or a default "dunno" byte slice if n is out of range. 78 | func source(lines [][]byte, n int) []byte { 79 | n-- 80 | if n < 0 || n >= len(lines) { 81 | return []byte(dunno) 82 | } 83 | return bytes.TrimSpace(lines[n]) 84 | } 85 | 86 | // function takes a program counter (pc) value and returns the name of the function 87 | // corresponding to that program counter as a byte slice. It uses runtime.FuncForPC 88 | // to retrieve the function information and processes the function name to remove 89 | // any path and package information, returning only the base function name. 90 | func function(pc uintptr) []byte { 91 | fn := runtime.FuncForPC(pc) 92 | if fn == nil { 93 | return []byte(dunno) 94 | } 95 | name := fn.Name() 96 | if lastSlash := strings.LastIndex(name, slash); lastSlash >= 0 { 97 | name = name[lastSlash+1:] 98 | } 99 | if period := strings.Index(name, dot); period >= 0 { 100 | name = name[period+1:] 101 | } 102 | name = strings.ReplaceAll(name, centerDot, dot) 103 | return []byte(name) 104 | } 105 | -------------------------------------------------------------------------------- /ring.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "sync/atomic" 7 | 8 | "github.com/golang-queue/queue/core" 9 | ) 10 | 11 | var _ core.Worker = (*Ring)(nil) 12 | 13 | // Ring represents a simple queue using a buffer channel. 14 | type Ring struct { 15 | sync.Mutex 16 | taskQueue []core.TaskMessage // taskQueue holds the tasks in the ring buffer. 17 | runFunc func(context.Context, core.TaskMessage) error // runFunc is the function responsible for processing tasks. 18 | capacity int // capacity is the maximum number of tasks the queue can hold. 19 | count int // count is the current number of tasks in the queue. 20 | head int // head is the index of the first task in the queue. 21 | tail int // tail is the index where the next task will be added. 22 | exit chan struct{} // exit is used to signal when the queue is shutting down. 23 | logger Logger // logger is used for logging messages. 24 | stopOnce sync.Once // stopOnce ensures the shutdown process only runs once. 25 | stopFlag int32 // stopFlag indicates whether the queue is shutting down. 26 | } 27 | 28 | // Run executes a new task using the provided context and task message. 29 | // It calls the runFunc function, which is responsible for processing the task. 30 | // The context allows for cancellation and timeout control of the task execution. 31 | func (s *Ring) Run(ctx context.Context, task core.TaskMessage) error { 32 | return s.runFunc(ctx, task) 33 | } 34 | 35 | // Shutdown gracefully shuts down the worker. 36 | // It sets the stopFlag to indicate that the queue is shutting down and prevents new tasks from being added. 37 | // If the queue is already shut down, it returns ErrQueueShutdown. 38 | // It waits for all tasks to be processed before completing the shutdown. 39 | func (s *Ring) Shutdown() error { 40 | // Attempt to set the stopFlag from 0 to 1. If it fails, the queue is already shut down. 41 | if !atomic.CompareAndSwapInt32(&s.stopFlag, 0, 1) { 42 | return ErrQueueShutdown 43 | } 44 | 45 | // Ensure the shutdown process only runs once. 46 | s.stopOnce.Do(func() { 47 | s.Lock() 48 | count := s.count 49 | s.Unlock() 50 | // If there are tasks in the queue, wait for them to be processed. 51 | if count > 0 { 52 | <-s.exit 53 | } 54 | }) 55 | return nil 56 | } 57 | 58 | // Queue adds a task to the ring buffer queue. 59 | // It returns an error if the queue is shut down or has reached its maximum capacity. 60 | func (s *Ring) Queue(task core.TaskMessage) error { //nolint:stylecheck 61 | // Check if the queue is shut down 62 | if atomic.LoadInt32(&s.stopFlag) == 1 { 63 | return ErrQueueShutdown 64 | } 65 | // Check if the queue has reached its maximum capacity 66 | if s.capacity > 0 && s.count >= s.capacity { 67 | return ErrMaxCapacity 68 | } 69 | 70 | s.Lock() 71 | // Resize the queue if necessary 72 | if s.count == len(s.taskQueue) { 73 | s.resize(s.count * 2) 74 | } 75 | // Add the task to the queue 76 | s.taskQueue[s.tail] = task 77 | s.tail = (s.tail + 1) % len(s.taskQueue) 78 | s.count++ 79 | s.Unlock() 80 | 81 | return nil 82 | } 83 | 84 | // Request retrieves the next task message from the ring queue. 85 | // If the queue has been stopped and is empty, it signals the exit channel 86 | // and returns an error indicating the queue has been closed. 87 | // If the queue is empty but not stopped, it returns an error indicating 88 | // there are no tasks in the queue. 89 | // If a task is successfully retrieved, it is removed from the queue, 90 | // and the queue may be resized if it is less than half full. 91 | // Returns the task message and nil on success, or an error if the queue 92 | // is empty or has been closed. 93 | func (s *Ring) Request() (core.TaskMessage, error) { 94 | if atomic.LoadInt32(&s.stopFlag) == 1 && s.count == 0 { 95 | select { 96 | case s.exit <- struct{}{}: 97 | default: 98 | } 99 | return nil, ErrQueueHasBeenClosed 100 | } 101 | 102 | s.Lock() 103 | defer s.Unlock() 104 | if s.count == 0 { 105 | return nil, ErrNoTaskInQueue 106 | } 107 | data := s.taskQueue[s.head] 108 | s.taskQueue[s.head] = nil 109 | s.head = (s.head + 1) % len(s.taskQueue) 110 | s.count-- 111 | 112 | if n := len(s.taskQueue) / 2; n >= 2 && s.count <= n { 113 | s.resize(n) 114 | } 115 | 116 | return data, nil 117 | } 118 | 119 | // resize adjusts the size of the ring buffer to the specified capacity n. 120 | // It reallocates the underlying slice to the new size and copies the existing 121 | // elements to the new slice in the correct order. The head and tail pointers 122 | // are updated accordingly to maintain the correct order of elements in the 123 | // resized buffer. 124 | // 125 | // Parameters: 126 | // 127 | // n - the new capacity of the ring buffer. 128 | func (q *Ring) resize(n int) { 129 | nodes := make([]core.TaskMessage, n) 130 | if q.head < q.tail { 131 | copy(nodes, q.taskQueue[q.head:q.tail]) 132 | } else { 133 | copy(nodes, q.taskQueue[q.head:]) 134 | copy(nodes[len(q.taskQueue)-q.head:], q.taskQueue[:q.tail]) 135 | } 136 | 137 | q.tail = q.count % n 138 | q.head = 0 139 | q.taskQueue = nodes 140 | } 141 | 142 | // NewRing creates a new Ring instance with the provided options. 143 | // It initializes the task queue with a default size of 2, sets the capacity 144 | // based on the provided options, and configures the logger and run function. 145 | // The function returns a pointer to the newly created Ring instance. 146 | // 147 | // Parameters: 148 | // 149 | // opts - A variadic list of Option functions to configure the Ring instance. 150 | // 151 | // Returns: 152 | // 153 | // *Ring - A pointer to the newly created Ring instance. 154 | func NewRing(opts ...Option) *Ring { 155 | o := NewOptions(opts...) 156 | w := &Ring{ 157 | taskQueue: make([]core.TaskMessage, 2), 158 | capacity: o.queueSize, 159 | exit: make(chan struct{}), 160 | logger: o.logger, 161 | runFunc: o.fn, 162 | } 163 | 164 | return w 165 | } 166 | -------------------------------------------------------------------------------- /ring_test.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "runtime" 9 | "testing" 10 | "time" 11 | 12 | "github.com/golang-queue/queue/core" 13 | "github.com/golang-queue/queue/job" 14 | "github.com/golang-queue/queue/mocks" 15 | 16 | "github.com/stretchr/testify/assert" 17 | "go.uber.org/mock/gomock" 18 | ) 19 | 20 | func TestMaxCapacity(t *testing.T) { 21 | w := NewRing(WithQueueSize(2)) 22 | 23 | assert.NoError(t, w.Queue(&mockMessage{})) 24 | assert.NoError(t, w.Queue(&mockMessage{})) 25 | assert.Error(t, w.Queue(&mockMessage{})) 26 | 27 | err := w.Queue(&mockMessage{}) 28 | assert.Equal(t, ErrMaxCapacity, err) 29 | } 30 | 31 | func TestCustomFuncAndWait(t *testing.T) { 32 | m := mockMessage{ 33 | message: "foo", 34 | } 35 | w := NewRing( 36 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 37 | time.Sleep(500 * time.Millisecond) 38 | return nil 39 | }), 40 | ) 41 | q, err := NewQueue( 42 | WithWorker(w), 43 | WithWorkerCount(2), 44 | WithLogger(NewLogger()), 45 | ) 46 | assert.NoError(t, err) 47 | assert.NoError(t, q.Queue(m)) 48 | assert.NoError(t, q.Queue(m)) 49 | assert.NoError(t, q.Queue(m)) 50 | assert.NoError(t, q.Queue(m)) 51 | q.Start() 52 | time.Sleep(100 * time.Millisecond) 53 | assert.Equal(t, 2, int(q.metric.BusyWorkers())) 54 | time.Sleep(600 * time.Millisecond) 55 | q.Shutdown() 56 | q.Wait() 57 | // you will see the execute time > 1000ms 58 | } 59 | 60 | func TestEnqueueJobAfterShutdown(t *testing.T) { 61 | m := mockMessage{ 62 | message: "foo", 63 | } 64 | w := NewRing() 65 | q, err := NewQueue( 66 | WithWorker(w), 67 | WithWorkerCount(2), 68 | ) 69 | assert.NoError(t, err) 70 | q.Start() 71 | time.Sleep(50 * time.Millisecond) 72 | q.Shutdown() 73 | // can't queue task after shutdown 74 | err = q.Queue(m) 75 | assert.Error(t, err) 76 | assert.Equal(t, ErrQueueShutdown, err) 77 | q.Wait() 78 | } 79 | 80 | func TestJobReachTimeout(t *testing.T) { 81 | m := mockMessage{ 82 | message: "foo", 83 | } 84 | w := NewRing( 85 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 86 | for { 87 | select { 88 | case <-ctx.Done(): 89 | log.Println("get data:", string(m.Payload())) 90 | if errors.Is(ctx.Err(), context.Canceled) { 91 | log.Println("queue has been shutdown and cancel the job") 92 | } else if errors.Is(ctx.Err(), context.DeadlineExceeded) { 93 | log.Println("job deadline exceeded") 94 | } 95 | return nil 96 | default: 97 | } 98 | time.Sleep(50 * time.Millisecond) 99 | } 100 | }), 101 | ) 102 | q, err := NewQueue( 103 | WithWorker(w), 104 | WithWorkerCount(2), 105 | ) 106 | assert.NoError(t, err) 107 | assert.NoError(t, q.Queue(m, job.AllowOption{Timeout: job.Time(30 * time.Millisecond)})) 108 | q.Start() 109 | time.Sleep(50 * time.Millisecond) 110 | q.Release() 111 | } 112 | 113 | func TestCancelJobAfterShutdown(t *testing.T) { 114 | m := mockMessage{ 115 | message: "foo", 116 | } 117 | w := NewRing( 118 | WithLogger(NewEmptyLogger()), 119 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 120 | for { 121 | select { 122 | case <-ctx.Done(): 123 | log.Println("get data:", string(m.Payload())) 124 | if errors.Is(ctx.Err(), context.Canceled) { 125 | log.Println("queue has been shutdown and cancel the job") 126 | } else if errors.Is(ctx.Err(), context.DeadlineExceeded) { 127 | log.Println("job deadline exceeded") 128 | } 129 | return nil 130 | default: 131 | } 132 | time.Sleep(50 * time.Millisecond) 133 | } 134 | }), 135 | ) 136 | q, err := NewQueue( 137 | WithWorker(w), 138 | WithWorkerCount(2), 139 | ) 140 | assert.NoError(t, err) 141 | assert.NoError(t, q.Queue(m, job.AllowOption{Timeout: job.Time(100 * time.Millisecond)})) 142 | assert.NoError(t, q.Queue(m, job.AllowOption{Timeout: job.Time(100 * time.Millisecond)})) 143 | q.Start() 144 | time.Sleep(10 * time.Millisecond) 145 | assert.Equal(t, int64(2), q.BusyWorkers()) 146 | q.Release() 147 | } 148 | 149 | func TestGoroutineLeak(t *testing.T) { 150 | w := NewRing( 151 | WithLogger(NewLogger()), 152 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 153 | for { 154 | select { 155 | case <-ctx.Done(): 156 | if errors.Is(ctx.Err(), context.Canceled) { 157 | log.Println("queue has been shutdown and cancel the job: " + string(m.Payload())) 158 | } else if errors.Is(ctx.Err(), context.DeadlineExceeded) { 159 | log.Println("job deadline exceeded: " + string(m.Payload())) 160 | } 161 | return nil 162 | default: 163 | log.Println("get data:", string(m.Payload())) 164 | time.Sleep(50 * time.Millisecond) 165 | return nil 166 | } 167 | } 168 | }), 169 | ) 170 | q, err := NewQueue( 171 | WithLogger(NewLogger()), 172 | WithWorker(w), 173 | WithWorkerCount(10), 174 | ) 175 | assert.NoError(t, err) 176 | for i := 0; i < 400; i++ { 177 | m := mockMessage{ 178 | message: fmt.Sprintf("new message: %d", i+1), 179 | } 180 | 181 | assert.NoError(t, q.Queue(m)) 182 | } 183 | 184 | q.Start() 185 | time.Sleep(1 * time.Second) 186 | q.Release() 187 | fmt.Println("number of goroutines:", runtime.NumGoroutine()) 188 | } 189 | 190 | func TestGoroutinePanic(t *testing.T) { 191 | m := mockMessage{ 192 | message: "foo", 193 | } 194 | w := NewRing( 195 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 196 | panic("missing something") 197 | }), 198 | ) 199 | q, err := NewQueue( 200 | WithWorker(w), 201 | WithWorkerCount(2), 202 | ) 203 | assert.NoError(t, err) 204 | assert.NoError(t, q.Queue(m)) 205 | q.Start() 206 | time.Sleep(10 * time.Millisecond) 207 | q.Release() 208 | } 209 | 210 | func TestIncreaseWorkerCount(t *testing.T) { 211 | w := NewRing( 212 | WithLogger(NewEmptyLogger()), 213 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 214 | time.Sleep(500 * time.Millisecond) 215 | return nil 216 | }), 217 | ) 218 | q, err := NewQueue( 219 | WithLogger(NewLogger()), 220 | WithWorker(w), 221 | WithWorkerCount(5), 222 | ) 223 | assert.NoError(t, err) 224 | 225 | for i := 1; i <= 10; i++ { 226 | m := mockMessage{ 227 | message: fmt.Sprintf("new message: %d", i), 228 | } 229 | assert.NoError(t, q.Queue(m)) 230 | } 231 | 232 | q.Start() 233 | time.Sleep(100 * time.Millisecond) 234 | assert.Equal(t, int64(5), q.BusyWorkers()) 235 | q.UpdateWorkerCount(10) 236 | time.Sleep(100 * time.Millisecond) 237 | assert.Equal(t, int64(10), q.BusyWorkers()) 238 | q.Release() 239 | } 240 | 241 | func TestDecreaseWorkerCount(t *testing.T) { 242 | w := NewRing( 243 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 244 | time.Sleep(100 * time.Millisecond) 245 | return nil 246 | }), 247 | ) 248 | q, err := NewQueue( 249 | WithLogger(NewLogger()), 250 | WithWorker(w), 251 | WithWorkerCount(5), 252 | ) 253 | assert.NoError(t, err) 254 | 255 | for i := 1; i <= 10; i++ { 256 | m := mockMessage{ 257 | message: fmt.Sprintf("test message: %d", i), 258 | } 259 | assert.NoError(t, q.Queue(m)) 260 | } 261 | 262 | q.Start() 263 | time.Sleep(20 * time.Millisecond) 264 | assert.Equal(t, int64(5), q.BusyWorkers()) 265 | q.UpdateWorkerCount(3) 266 | time.Sleep(100 * time.Millisecond) 267 | assert.Equal(t, int64(3), q.BusyWorkers()) 268 | time.Sleep(100 * time.Millisecond) 269 | assert.Equal(t, int64(2), q.BusyWorkers()) 270 | q.Release() 271 | } 272 | 273 | func TestHandleAllJobBeforeShutdownRing(t *testing.T) { 274 | controller := gomock.NewController(t) 275 | defer controller.Finish() 276 | 277 | m := mocks.NewMockTaskMessage(controller) 278 | 279 | w := NewRing( 280 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 281 | time.Sleep(10 * time.Millisecond) 282 | return nil 283 | }), 284 | ) 285 | 286 | done := make(chan struct{}) 287 | assert.NoError(t, w.Queue(m)) 288 | assert.NoError(t, w.Queue(m)) 289 | go func() { 290 | assert.NoError(t, w.Shutdown()) 291 | done <- struct{}{} 292 | }() 293 | time.Sleep(50 * time.Millisecond) 294 | task, err := w.Request() 295 | assert.NotNil(t, task) 296 | assert.NoError(t, err) 297 | task, err = w.Request() 298 | assert.NotNil(t, task) 299 | assert.NoError(t, err) 300 | task, err = w.Request() 301 | assert.Nil(t, task) 302 | assert.True(t, errors.Is(err, ErrQueueHasBeenClosed)) 303 | <-done 304 | } 305 | 306 | func TestHandleAllJobBeforeShutdownRingInQueue(t *testing.T) { 307 | controller := gomock.NewController(t) 308 | defer controller.Finish() 309 | 310 | m := mocks.NewMockTaskMessage(controller) 311 | m.EXPECT().Bytes().Return([]byte("test")).AnyTimes() 312 | m.EXPECT().Payload().Return([]byte("test")).AnyTimes() 313 | 314 | messages := make(chan string, 10) 315 | 316 | w := NewRing( 317 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 318 | time.Sleep(10 * time.Millisecond) 319 | messages <- string(m.Payload()) 320 | return nil 321 | }), 322 | ) 323 | 324 | q, err := NewQueue( 325 | WithLogger(NewLogger()), 326 | WithWorker(w), 327 | WithWorkerCount(1), 328 | ) 329 | assert.NoError(t, err) 330 | 331 | assert.NoError(t, q.Queue(m)) 332 | assert.NoError(t, q.Queue(m)) 333 | assert.Len(t, messages, 0) 334 | q.Start() 335 | q.Release() 336 | assert.Len(t, messages, 2) 337 | } 338 | 339 | func TestRetryCountWithNewMessage(t *testing.T) { 340 | controller := gomock.NewController(t) 341 | defer controller.Finish() 342 | 343 | m := mocks.NewMockQueuedMessage(controller) 344 | m.EXPECT().Bytes().Return([]byte("test")).AnyTimes() 345 | 346 | messages := make(chan string, 10) 347 | keep := make(chan struct{}) 348 | count := 1 349 | 350 | w := NewRing( 351 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 352 | if count%3 != 0 { 353 | count++ 354 | return errors.New("count not correct") 355 | } 356 | close(keep) 357 | messages <- string(m.Payload()) 358 | return nil 359 | }), 360 | ) 361 | 362 | q, err := NewQueue( 363 | WithLogger(NewLogger()), 364 | WithWorker(w), 365 | WithWorkerCount(1), 366 | ) 367 | assert.NoError(t, err) 368 | 369 | assert.NoError(t, q.Queue( 370 | m, 371 | job.AllowOption{ 372 | RetryCount: job.Int64(3), 373 | RetryDelay: job.Time(50 * time.Millisecond), 374 | }, 375 | )) 376 | assert.Len(t, messages, 0) 377 | q.Start() 378 | // wait retry twice. 379 | <-keep 380 | q.Release() 381 | assert.Len(t, messages, 1) 382 | } 383 | 384 | func TestRetryCountWithNewTask(t *testing.T) { 385 | messages := make(chan string, 10) 386 | count := 1 387 | 388 | w := NewRing() 389 | 390 | q, err := NewQueue( 391 | WithLogger(NewLogger()), 392 | WithWorker(w), 393 | WithWorkerCount(1), 394 | ) 395 | assert.NoError(t, err) 396 | 397 | keep := make(chan struct{}) 398 | 399 | assert.NoError(t, q.QueueTask( 400 | func(ctx context.Context) error { 401 | if count%3 != 0 { 402 | count++ 403 | return errors.New("count not correct") 404 | } 405 | close(keep) 406 | messages <- "foobar" 407 | return nil 408 | }, 409 | job.AllowOption{ 410 | RetryCount: job.Int64(3), 411 | }, 412 | )) 413 | assert.Len(t, messages, 0) 414 | q.Start() 415 | // wait retry twice. 416 | <-keep 417 | q.Release() 418 | assert.Len(t, messages, 1) 419 | } 420 | 421 | func TestCancelRetryCountWithNewTask(t *testing.T) { 422 | messages := make(chan string, 10) 423 | count := 1 424 | 425 | w := NewRing() 426 | 427 | q, err := NewQueue( 428 | WithLogger(NewLogger()), 429 | WithWorker(w), 430 | WithWorkerCount(1), 431 | ) 432 | assert.NoError(t, err) 433 | 434 | assert.NoError(t, q.QueueTask( 435 | func(ctx context.Context) error { 436 | if count%3 != 0 { 437 | count++ 438 | q.logger.Info("add count") 439 | return errors.New("count not correct") 440 | } 441 | messages <- "foobar" 442 | return nil 443 | }, 444 | job.AllowOption{ 445 | RetryCount: job.Int64(3), 446 | RetryDelay: job.Time(100 * time.Millisecond), 447 | }, 448 | )) 449 | assert.Len(t, messages, 0) 450 | q.Start() 451 | time.Sleep(50 * time.Millisecond) 452 | q.Release() 453 | assert.Len(t, messages, 0) 454 | assert.Equal(t, 2, count) 455 | } 456 | 457 | func TestCancelRetryCountWithNewMessage(t *testing.T) { 458 | controller := gomock.NewController(t) 459 | defer controller.Finish() 460 | 461 | m := mocks.NewMockQueuedMessage(controller) 462 | m.EXPECT().Bytes().Return([]byte("test")).AnyTimes() 463 | 464 | messages := make(chan string, 10) 465 | count := 1 466 | 467 | w := NewRing( 468 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 469 | if count%3 != 0 { 470 | count++ 471 | return errors.New("count not correct") 472 | } 473 | messages <- string(m.Payload()) 474 | return nil 475 | }), 476 | ) 477 | 478 | q, err := NewQueue( 479 | WithLogger(NewLogger()), 480 | WithWorker(w), 481 | WithWorkerCount(1), 482 | ) 483 | assert.NoError(t, err) 484 | 485 | assert.NoError(t, q.Queue( 486 | m, 487 | job.AllowOption{ 488 | RetryCount: job.Int64(3), 489 | RetryDelay: job.Time(100 * time.Millisecond), 490 | }, 491 | )) 492 | assert.Len(t, messages, 0) 493 | q.Start() 494 | time.Sleep(50 * time.Millisecond) 495 | q.Release() 496 | assert.Len(t, messages, 0) 497 | assert.Equal(t, 2, count) 498 | } 499 | 500 | func TestErrNoTaskInQueue(t *testing.T) { 501 | w := NewRing( 502 | WithFn(func(ctx context.Context, m core.TaskMessage) error { 503 | return nil 504 | }), 505 | ) 506 | task, err := w.Request() 507 | assert.Nil(t, task) 508 | assert.Error(t, err) 509 | assert.Equal(t, ErrNoTaskInQueue, err) 510 | } 511 | 512 | func BenchmarkRingQueue(b *testing.B) { 513 | b.Run("queue and request operations", func(b *testing.B) { 514 | w := NewRing(WithQueueSize(1000)) 515 | m := mockMessage{message: "test"} 516 | 517 | b.ResetTimer() 518 | b.RunParallel(func(pb *testing.PB) { 519 | for pb.Next() { 520 | _ = w.Queue(&m) 521 | _, _ = w.Request() 522 | } 523 | }) 524 | }) 525 | 526 | b.Run("concurrent queue operations", func(b *testing.B) { 527 | w := NewRing(WithQueueSize(1000)) 528 | m := mockMessage{message: "test"} 529 | 530 | b.ResetTimer() 531 | b.RunParallel(func(pb *testing.PB) { 532 | for pb.Next() { 533 | _ = w.Queue(&m) 534 | } 535 | }) 536 | }) 537 | 538 | b.Run("resize operations", func(b *testing.B) { 539 | w := NewRing() 540 | m := mockMessage{message: "test"} 541 | 542 | b.ResetTimer() 543 | for i := 0; i < b.N; i++ { 544 | for j := 0; j < 100; j++ { 545 | _ = w.Queue(&m) 546 | } 547 | for j := 0; j < 100; j++ { 548 | _, _ = w.Request() 549 | } 550 | } 551 | }) 552 | } 553 | -------------------------------------------------------------------------------- /thread.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import "sync" 4 | 5 | type routineGroup struct { 6 | waitGroup sync.WaitGroup 7 | } 8 | 9 | func newRoutineGroup() *routineGroup { 10 | return new(routineGroup) 11 | } 12 | 13 | func (g *routineGroup) Run(fn func()) { 14 | g.waitGroup.Add(1) 15 | 16 | go func() { 17 | defer g.waitGroup.Done() 18 | fn() 19 | }() 20 | } 21 | 22 | func (g *routineGroup) Wait() { 23 | g.waitGroup.Wait() 24 | } 25 | -------------------------------------------------------------------------------- /thread_test.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "sync/atomic" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestRoutineGroupRun(t *testing.T) { 10 | t.Run("execute single function", func(t *testing.T) { 11 | g := newRoutineGroup() 12 | var counter int32 13 | 14 | g.Run(func() { 15 | atomic.AddInt32(&counter, 1) 16 | }) 17 | 18 | g.Wait() 19 | 20 | if atomic.LoadInt32(&counter) != 1 { 21 | t.Errorf("expected counter to be 1, got %d", counter) 22 | } 23 | }) 24 | 25 | t.Run("execute multiple functions", func(t *testing.T) { 26 | g := newRoutineGroup() 27 | var counter int32 28 | numRoutines := 10 29 | 30 | for i := 0; i < numRoutines; i++ { 31 | g.Run(func() { 32 | atomic.AddInt32(&counter, 1) 33 | time.Sleep(10 * time.Millisecond) 34 | }) 35 | } 36 | 37 | g.Wait() 38 | 39 | if atomic.LoadInt32(&counter) != int32(numRoutines) { 40 | t.Errorf("expected counter to be %d, got %d", numRoutines, counter) 41 | } 42 | }) 43 | } 44 | --------------------------------------------------------------------------------