├── .gitignore ├── .travis.yml ├── LICENSE ├── MAINTAINERS ├── Makefile ├── README.md ├── amqp ├── amqp.go └── amqp_test.go ├── appveyor.yml ├── common.go ├── example_test.go ├── job.go ├── memory ├── memory.go └── memory_test.go ├── register.go ├── register_test.go └── test └── suite.go /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .ci/ 3 | Makefile.main 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.11.x 5 | - 1.10.x 6 | 7 | go_import_path: gopkg.in/src-d/go-queue.v1 8 | 9 | matrix: 10 | fast_finish: true 11 | allow_failures: 12 | - go: tip 13 | 14 | services: 15 | - docker 16 | 17 | sudo: required 18 | 19 | before_install: 20 | - docker pull rabbitmq:3-management 21 | - docker run -d --name rabbitmq -p 127.0.0.1:5672:5672 rabbitmq:3-management 22 | - docker ps -a 23 | 24 | install: 25 | - make dependencies 26 | 27 | script: 28 | - make test-coverage -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2017 Sourced Technologies, S.L. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /MAINTAINERS: -------------------------------------------------------------------------------- 1 | Javier Fontan (@jfontan) 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Package configuration 2 | PROJECT = go-queue 3 | COMMANDS = 4 | 5 | # Including ci Makefile 6 | CI_REPOSITORY ?= https://github.com/src-d/ci.git 7 | CI_BRANCH ?= v1 8 | CI_PATH ?= .ci 9 | MAKEFILE := $(CI_PATH)/Makefile.main 10 | $(MAKEFILE): 11 | git clone --quiet --depth 1 -b $(CI_BRANCH) $(CI_REPOSITORY) $(CI_PATH); 12 | -include $(MAKEFILE) 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-queue [![GoDoc](https://godoc.org/gopkg.in/src-d/go-queue.v1?status.svg)](https://godoc.org/github.com/src-d/go-queue) [![Build Status](https://travis-ci.org/src-d/go-queue.svg)](https://travis-ci.org/src-d/go-queue) [![Build status](https://ci.appveyor.com/api/projects/status/15cdr1nk890qpk7g?svg=true)](https://ci.appveyor.com/project/mcuadros/go-queue-5ncaj) [![codecov.io](https://codecov.io/github/src-d/go-queue/coverage.svg)](https://codecov.io/github/src-d/go-queue) [![Go Report Card](https://goreportcard.com/badge/github.com/src-d/go-queue)](https://goreportcard.com/report/github.com/src-d/go-queue) 2 | 3 | Queue is a generic interface to abstract the details of implementation of queue 4 | systems. 5 | 6 | Similar to the package [`database/sql`](https://golang.org/pkg/database/sql/), 7 | this package implements a common interface to interact with different queue 8 | systems, in a unified way. 9 | 10 | Currently, only AMQP queues and an in-memory queue are supported. 11 | 12 | Installation 13 | ------------ 14 | 15 | The recommended way to install *go-queue* is: 16 | 17 | ``` 18 | go get -u gopkg.in/src-d/go-queue.v1/... 19 | ``` 20 | 21 | Usage 22 | ----- 23 | 24 | This example shows how to publish and consume a Job from the in-memory 25 | implementation, very useful for unit tests. 26 | 27 | The queue implementations to be supported by the `NewBroker` should be imported 28 | as shows the example. 29 | 30 | ```go 31 | import ( 32 | ... 33 | "gopkg.in/src-d/go-queue.v1" 34 | _ "gopkg.in/src-d/go-queue.v1/memory" 35 | ) 36 | 37 | ... 38 | 39 | b, _ := queue.NewBroker("memory://") 40 | q, _ := b.Queue("test-queue") 41 | 42 | j, _ := queue.NewJob() 43 | if err := j.Encode("hello world!"); err != nil { 44 | log.Fatal(err) 45 | } 46 | 47 | if err := q.Publish(j); err != nil { 48 | log.Fatal(err) 49 | } 50 | 51 | iter, err := q.Consume(1) 52 | if err != nil { 53 | log.Fatal(err) 54 | } 55 | 56 | consumedJob, _ := iter.Next() 57 | 58 | var payload string 59 | _ = consumedJob.Decode(&payload) 60 | 61 | fmt.Println(payload) 62 | // Output: hello world! 63 | ``` 64 | 65 | 66 | Configuration 67 | ------------- 68 | 69 | ### AMQP 70 | 71 | The list of available variables is: 72 | 73 | - `AMQP_BACKOFF_MIN` (default: 20ms): Minimum time to wait for retry the connection or queue channel assignment. 74 | - `AMQP_BACKOFF_MAX` (default: 30s): Maximum time to wait for retry the connection or queue channel assignment. 75 | - `AMQP_BACKOFF_FACTOR` (default: 2): Multiplying factor for each increment step on the retry. 76 | - `AMQP_BURIED_QUEUE_SUFFIX` (default: `.buriedQueue`): Suffix for the buried queue name. 77 | - `AMQP_BURIED_EXCHANGE_SUFFIX` (default: `.buriedExchange`): Suffix for the exchange name. 78 | - `AMQP_BURIED_TIMEOUT` (default: 500): Time in milliseconds to wait for new jobs from the buried queue. 79 | - `AMQP_RETRIES_HEADER` (default: `x-retries`): Message header to set the number of retries. 80 | - `AMQP_ERROR_HEADER` (default: `x-error-type`): Message header to set the error type. 81 | 82 | License 83 | ------- 84 | Apache License Version 2.0, see [LICENSE](LICENSE) -------------------------------------------------------------------------------- /amqp/amqp.go: -------------------------------------------------------------------------------- 1 | package amqp 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "os" 7 | "strings" 8 | "sync" 9 | "sync/atomic" 10 | "time" 11 | 12 | "gopkg.in/src-d/go-queue.v1" 13 | 14 | "github.com/jpillora/backoff" 15 | "github.com/kelseyhightower/envconfig" 16 | "github.com/streadway/amqp" 17 | "gopkg.in/src-d/go-errors.v1" 18 | "gopkg.in/src-d/go-log.v1" 19 | ) 20 | 21 | func init() { 22 | err := envconfig.Process("amqp", &DefaultConfiguration) 23 | if err != nil { 24 | panic(err) 25 | } 26 | 27 | queue.Register("amqp", func(uri string) (queue.Broker, error) { 28 | return New(uri) 29 | }) 30 | } 31 | 32 | // DefaultConfiguration contains the default configuration initialized from 33 | // environment variables. 34 | var DefaultConfiguration Configuration 35 | 36 | // Configuration AMQP configuration settings, this settings are set using the 37 | // environment variables. 38 | type Configuration struct { 39 | BuriedQueueSuffix string `envconfig:"BURIED_QUEUE_SUFFIX" default:".buriedQueue"` 40 | BuriedExchangeSuffix string `envconfig:"BURIED_EXCHANGE_SUFFIX" default:".buriedExchange"` 41 | BuriedTimeout int `envconfig:"BURIED_TIMEOUT" default:"500"` 42 | 43 | RetriesHeader string `envconfig:"RETRIES_HEADER" default:"x-retries"` 44 | ErrorHeader string `envconfig:"ERROR_HEADER" default:"x-error-type"` 45 | 46 | BackoffMin time.Duration `envconfig:"BACKOFF_MIN" default:"200ms"` 47 | BackoffMax time.Duration `envconfig:"BACKOFF_MAX" default:"30s"` 48 | BackoffFactor float64 `envconfig:"BACKOFF_FACTOR" default:"2"` 49 | } 50 | 51 | var consumerSeq uint64 52 | 53 | var ( 54 | ErrConnectionFailed = errors.NewKind("failed to connect to RabbitMQ: %s") 55 | ErrOpenChannel = errors.NewKind("failed to open a channel: %s") 56 | ErrRetrievingHeader = errors.NewKind("error retrieving '%s' header from message %s") 57 | ErrRepublishingJobs = errors.NewKind("couldn't republish some jobs : %s") 58 | ) 59 | 60 | // Broker implements the queue.Broker interface for AMQP, such as RabbitMQ. 61 | type Broker struct { 62 | mut sync.RWMutex 63 | conn *amqp.Connection 64 | ch *amqp.Channel 65 | connErrors chan *amqp.Error 66 | chErrors chan *amqp.Error 67 | stop chan struct{} 68 | backoff *backoff.Backoff 69 | } 70 | 71 | type connection interface { 72 | connection() *amqp.Connection 73 | channel() *amqp.Channel 74 | } 75 | 76 | // New creates a new AMQPBroker. 77 | func New(url string) (queue.Broker, error) { 78 | conn, err := amqp.Dial(url) 79 | if err != nil { 80 | return nil, ErrConnectionFailed.New(err) 81 | } 82 | 83 | ch, err := conn.Channel() 84 | if err != nil { 85 | return nil, ErrOpenChannel.New(err) 86 | } 87 | 88 | b := &Broker{ 89 | conn: conn, 90 | ch: ch, 91 | stop: make(chan struct{}), 92 | backoff: &backoff.Backoff{ 93 | Min: DefaultConfiguration.BackoffMin, 94 | Max: DefaultConfiguration.BackoffMax, 95 | Factor: DefaultConfiguration.BackoffFactor, 96 | Jitter: false, 97 | }, 98 | } 99 | 100 | b.connErrors = make(chan *amqp.Error, 1) 101 | b.conn.NotifyClose(b.connErrors) 102 | 103 | b.chErrors = make(chan *amqp.Error, 1) 104 | b.ch.NotifyClose(b.chErrors) 105 | 106 | go b.manageConnection(url) 107 | 108 | return b, nil 109 | } 110 | 111 | func (b *Broker) manageConnection(url string) { 112 | for { 113 | select { 114 | case err := <-b.connErrors: 115 | log.Errorf(err, "amqp connection error - reconnecting") 116 | if err == nil { 117 | break 118 | } 119 | b.reconnect(url) 120 | 121 | case err := <-b.chErrors: 122 | log.Errorf(err, "amqp channel error - reopening channel") 123 | if err == nil { 124 | break 125 | } 126 | b.reopenChannel() 127 | 128 | case <-b.stop: 129 | return 130 | } 131 | } 132 | } 133 | 134 | func (b *Broker) reconnect(url string) { 135 | b.backoff.Reset() 136 | 137 | b.mut.Lock() 138 | defer b.mut.Unlock() 139 | // open a new connection and channel 140 | b.conn = b.tryConnection(url) 141 | b.connErrors = make(chan *amqp.Error, 1) 142 | b.conn.NotifyClose(b.connErrors) 143 | 144 | b.ch = b.tryChannel(b.conn) 145 | b.chErrors = make(chan *amqp.Error, 1) 146 | b.ch.NotifyClose(b.chErrors) 147 | } 148 | 149 | func (b *Broker) reopenChannel() { 150 | b.backoff.Reset() 151 | 152 | b.mut.Lock() 153 | defer b.mut.Unlock() 154 | // open a new channel 155 | b.ch = b.tryChannel(b.conn) 156 | b.chErrors = make(chan *amqp.Error, 1) 157 | b.ch.NotifyClose(b.chErrors) 158 | } 159 | 160 | func (b *Broker) tryConnection(url string) *amqp.Connection { 161 | for { 162 | conn, err := amqp.Dial(url) 163 | if err == nil { 164 | b.backoff.Reset() 165 | return conn 166 | } 167 | d := b.backoff.Duration() 168 | 169 | log.Errorf(err, "error connecting to amqp, reconnecting in %s", d) 170 | time.Sleep(d) 171 | } 172 | } 173 | 174 | func (b *Broker) tryChannel(conn *amqp.Connection) *amqp.Channel { 175 | for { 176 | ch, err := conn.Channel() 177 | if err == nil { 178 | b.backoff.Reset() 179 | return ch 180 | } 181 | d := b.backoff.Duration() 182 | 183 | log.Errorf(err, "error creatting channel, new retry in %s", d) 184 | time.Sleep(d) 185 | } 186 | } 187 | 188 | func (b *Broker) connection() *amqp.Connection { 189 | b.mut.RLock() 190 | conn := b.conn 191 | b.mut.RUnlock() 192 | return conn 193 | } 194 | 195 | func (b *Broker) channel() *amqp.Channel { 196 | b.mut.RLock() 197 | ch := b.ch 198 | b.mut.RUnlock() 199 | return ch 200 | } 201 | 202 | func (b *Broker) newBuriedQueue(mainQueueName string) (q amqp.Queue, rex string, err error) { 203 | ch, err := b.conn.Channel() 204 | if err != nil { 205 | return 206 | } 207 | 208 | buriedName := mainQueueName + DefaultConfiguration.BuriedQueueSuffix 209 | rex = mainQueueName + DefaultConfiguration.BuriedExchangeSuffix 210 | 211 | if err = ch.ExchangeDeclare(rex, "fanout", true, false, false, false, nil); err != nil { 212 | return 213 | } 214 | 215 | q, err = b.ch.QueueDeclare( 216 | buriedName, 217 | true, 218 | false, 219 | false, 220 | false, 221 | nil, 222 | ) 223 | 224 | if err != nil { 225 | return 226 | } 227 | 228 | if err = ch.QueueBind(buriedName, "", rex, true, nil); err != nil { 229 | return 230 | } 231 | 232 | return 233 | } 234 | 235 | // Queue returns the queue with the given name. 236 | func (b *Broker) Queue(name string) (queue.Queue, error) { 237 | buriedQueue, rex, err := b.newBuriedQueue(name) 238 | if err != nil { 239 | return nil, err 240 | } 241 | 242 | q, err := b.channel().QueueDeclare( 243 | name, // name 244 | true, // durable 245 | false, // delete when unused 246 | false, // exclusive 247 | false, // no-wait 248 | amqp.Table{ 249 | "x-dead-letter-exchange": rex, 250 | "x-dead-letter-routing-key": name, 251 | "x-max-priority": uint8(queue.PriorityUrgent), 252 | }, 253 | ) 254 | 255 | if err != nil { 256 | return nil, err 257 | } 258 | 259 | return &Queue{ 260 | conn: b, 261 | queue: q, 262 | buriedQueue: &Queue{conn: b, queue: buriedQueue}, 263 | }, nil 264 | } 265 | 266 | // Close closes all the connections managed by the broker. 267 | func (b *Broker) Close() error { 268 | close(b.stop) 269 | 270 | if err := b.channel().Close(); err != nil { 271 | return err 272 | } 273 | 274 | return b.connection().Close() 275 | } 276 | 277 | // Queue implements the Queue interface for the AMQP. 278 | type Queue struct { 279 | conn connection 280 | queue amqp.Queue 281 | buriedQueue *Queue 282 | } 283 | 284 | // Publish publishes the given Job to the Queue. 285 | func (q *Queue) Publish(j *queue.Job) (err error) { 286 | if j == nil || j.Size() == 0 { 287 | return queue.ErrEmptyJob.New() 288 | } 289 | 290 | headers := amqp.Table{} 291 | if j.Retries > 0 { 292 | headers[DefaultConfiguration.RetriesHeader] = j.Retries 293 | } 294 | 295 | if j.ErrorType != "" { 296 | headers[DefaultConfiguration.ErrorHeader] = j.ErrorType 297 | } 298 | 299 | for { 300 | err = q.conn.channel().Publish( 301 | "", // exchange 302 | q.queue.Name, // routing key 303 | false, // mandatory 304 | false, 305 | amqp.Publishing{ 306 | DeliveryMode: amqp.Persistent, 307 | MessageId: j.ID, 308 | Priority: uint8(j.Priority), 309 | Timestamp: j.Timestamp, 310 | ContentType: j.ContentType, 311 | Body: j.Raw, 312 | Headers: headers, 313 | }, 314 | ) 315 | if err == nil { 316 | break 317 | } 318 | 319 | log.Errorf(err, "publishing to %s", q.queue.Name) 320 | if err != amqp.ErrClosed { 321 | break 322 | } 323 | } 324 | 325 | return err 326 | } 327 | 328 | // PublishDelayed publishes the given Job with a given delay. Delayed messages 329 | // will not go into the buried queue if they fail. 330 | func (q *Queue) PublishDelayed(j *queue.Job, delay time.Duration) error { 331 | if j == nil || j.Size() == 0 { 332 | return queue.ErrEmptyJob.New() 333 | } 334 | 335 | ttl := delay / time.Millisecond 336 | delayedQueue, err := q.conn.channel().QueueDeclare( 337 | j.ID, // name 338 | true, // durable 339 | true, // delete when unused 340 | false, // exclusive 341 | false, // no-wait 342 | amqp.Table{ 343 | "x-dead-letter-exchange": "", 344 | "x-dead-letter-routing-key": q.queue.Name, 345 | "x-message-ttl": int64(ttl), 346 | "x-expires": int64(ttl) * 2, 347 | "x-max-priority": uint8(queue.PriorityUrgent), 348 | }, 349 | ) 350 | if err != nil { 351 | return err 352 | } 353 | 354 | for { 355 | err = q.conn.channel().Publish( 356 | "", // exchange 357 | delayedQueue.Name, 358 | false, 359 | false, 360 | amqp.Publishing{ 361 | DeliveryMode: amqp.Persistent, 362 | MessageId: j.ID, 363 | Priority: uint8(j.Priority), 364 | Timestamp: j.Timestamp, 365 | ContentType: j.ContentType, 366 | Body: j.Raw, 367 | }, 368 | ) 369 | if err == nil { 370 | break 371 | } 372 | 373 | log.Errorf(err, "delay publishing to %s", q.queue.Name) 374 | if err != amqp.ErrClosed { 375 | break 376 | } 377 | } 378 | 379 | return err 380 | } 381 | 382 | type jobErr struct { 383 | job *queue.Job 384 | err error 385 | } 386 | 387 | // RepublishBuried will republish in the main queue those jobs that timed out without Ack 388 | // or were Rejected with requeue = False and makes comply return true. 389 | func (q *Queue) RepublishBuried(conditions ...queue.RepublishConditionFunc) error { 390 | if q.buriedQueue == nil { 391 | return fmt.Errorf("buriedQueue is nil, called RepublishBuried on the internal buried queue?") 392 | } 393 | 394 | // enforce prefetching only one job 395 | iter, err := q.buriedQueue.Consume(1) 396 | if err != nil { 397 | return err 398 | } 399 | 400 | defer iter.Close() 401 | 402 | timeout := time.Duration(DefaultConfiguration.BuriedTimeout) * time.Millisecond 403 | 404 | var notComplying []*queue.Job 405 | var errorsPublishing []*jobErr 406 | for { 407 | j, err := iter.(*JobIter).nextWithTimeout(timeout) 408 | if err != nil { 409 | return err 410 | } 411 | 412 | if j == nil { 413 | log.Debugf("no more jobs in the buried queue") 414 | 415 | break 416 | } 417 | 418 | if err = j.Ack(); err != nil { 419 | return err 420 | } 421 | 422 | if queue.RepublishConditions(conditions).Comply(j) { 423 | start := time.Now() 424 | if err = q.Publish(j); err != nil { 425 | log.With(log.Fields{ 426 | "duration": time.Since(start), 427 | "id": j.ID, 428 | }).Errorf(err, "error publishing job") 429 | 430 | errorsPublishing = append(errorsPublishing, &jobErr{j, err}) 431 | } else { 432 | log.With(log.Fields{ 433 | "duration": time.Since(start), 434 | "id": j.ID, 435 | }).Debugf("job republished") 436 | } 437 | } else { 438 | log.With(log.Fields{ 439 | "id": j.ID, 440 | "error-type": j.ErrorType, 441 | "content-type": j.ContentType, 442 | "retries": j.Retries, 443 | }).Debugf("job does not comply with conditions") 444 | 445 | notComplying = append(notComplying, j) 446 | } 447 | } 448 | 449 | log.Debugf("rejecting %v non complying jobs", len(notComplying)) 450 | 451 | for i, job := range notComplying { 452 | start := time.Now() 453 | 454 | if err = job.Reject(true); err != nil { 455 | return err 456 | } 457 | 458 | log.With(log.Fields{ 459 | "duration": time.Since(start), 460 | "id": job.ID, 461 | }).Debugf("job rejected (%v/%v)", i, len(notComplying)) 462 | } 463 | 464 | return q.handleRepublishErrors(errorsPublishing) 465 | } 466 | 467 | func (q *Queue) handleRepublishErrors(list []*jobErr) error { 468 | if len(list) > 0 { 469 | stringErrors := []string{} 470 | for i, je := range list { 471 | stringErrors = append(stringErrors, je.err.Error()) 472 | start := time.Now() 473 | 474 | if err := q.buriedQueue.Publish(je.job); err != nil { 475 | return err 476 | } 477 | 478 | log.With(log.Fields{ 479 | "duration": time.Since(start), 480 | "id": je.job.ID, 481 | }).Debugf("job reburied (%v/%v)", i, len(list)) 482 | } 483 | 484 | return ErrRepublishingJobs.New(strings.Join(stringErrors, ": ")) 485 | } 486 | 487 | return nil 488 | } 489 | 490 | // Transaction executes the given callback inside a transaction. 491 | func (q *Queue) Transaction(txcb queue.TxCallback) error { 492 | ch, err := q.conn.connection().Channel() 493 | if err != nil { 494 | return ErrOpenChannel.New(err) 495 | } 496 | 497 | defer ch.Close() 498 | 499 | if err := ch.Tx(); err != nil { 500 | return err 501 | } 502 | 503 | txQueue := &Queue{ 504 | conn: &Broker{ 505 | conn: q.conn.connection(), 506 | ch: ch, 507 | }, 508 | queue: q.queue, 509 | } 510 | 511 | err = txcb(txQueue) 512 | if err != nil { 513 | if err := ch.TxRollback(); err != nil { 514 | return err 515 | } 516 | 517 | return err 518 | } 519 | 520 | return ch.TxCommit() 521 | } 522 | 523 | // Consume implements the Queue interface. The advertisedWindow value 524 | // is the maximum number of unacknowledged jobs 525 | func (q *Queue) Consume(advertisedWindow int) (queue.JobIter, error) { 526 | ch, err := q.conn.connection().Channel() 527 | if err != nil { 528 | return nil, ErrOpenChannel.New(err) 529 | } 530 | 531 | if err := ch.Qos(advertisedWindow, 0, false); err != nil { 532 | return nil, err 533 | } 534 | 535 | id := q.consumeID() 536 | c, err := ch.Consume( 537 | q.queue.Name, // queue 538 | id, // consumer 539 | false, // autoAck 540 | false, // exclusive 541 | false, // noLocal 542 | false, // noWait 543 | nil, // args 544 | ) 545 | if err != nil { 546 | return nil, err 547 | } 548 | 549 | return &JobIter{id: id, ch: ch, c: c}, nil 550 | } 551 | 552 | func (q *Queue) consumeID() string { 553 | return fmt.Sprintf("%s-%s-%d", 554 | os.Args[0], 555 | q.queue.Name, 556 | atomic.AddUint64(&consumerSeq, 1), 557 | ) 558 | } 559 | 560 | // JobIter implements the JobIter interface for AMQP. 561 | type JobIter struct { 562 | id string 563 | ch *amqp.Channel 564 | c <-chan amqp.Delivery 565 | } 566 | 567 | // Next returns the next job in the iter. 568 | func (i *JobIter) Next() (*queue.Job, error) { 569 | d, ok := <-i.c 570 | if !ok { 571 | return nil, queue.ErrAlreadyClosed.New() 572 | } 573 | 574 | return fromDelivery(&d) 575 | } 576 | 577 | func (i *JobIter) nextWithTimeout(timeout time.Duration) (*queue.Job, error) { 578 | select { 579 | case d, ok := <-i.c: 580 | if !ok { 581 | return nil, queue.ErrAlreadyClosed.New() 582 | } 583 | 584 | return fromDelivery(&d) 585 | case <-time.After(timeout): 586 | return nil, nil 587 | } 588 | } 589 | 590 | // Close closes the channel of the JobIter. 591 | func (i *JobIter) Close() error { 592 | if err := i.ch.Cancel(i.id, false); err != nil { 593 | return err 594 | } 595 | 596 | return i.ch.Close() 597 | } 598 | 599 | // Acknowledger implements the Acknowledger for AMQP. 600 | type Acknowledger struct { 601 | ack amqp.Acknowledger 602 | id uint64 603 | } 604 | 605 | // Ack signals acknowledgement. 606 | func (a *Acknowledger) Ack() error { 607 | return a.ack.Ack(a.id, false) 608 | } 609 | 610 | // Reject signals rejection. If requeue is false, the job will go to the buried 611 | // queue until Queue.RepublishBuried() is called. 612 | func (a *Acknowledger) Reject(requeue bool) error { 613 | return a.ack.Reject(a.id, requeue) 614 | } 615 | 616 | func fromDelivery(d *amqp.Delivery) (*queue.Job, error) { 617 | j, err := queue.NewJob() 618 | if err != nil { 619 | return nil, err 620 | } 621 | 622 | j.ID = d.MessageId 623 | j.Priority = queue.Priority(d.Priority) 624 | j.Timestamp = d.Timestamp 625 | j.ContentType = d.ContentType 626 | j.Acknowledger = &Acknowledger{d.Acknowledger, d.DeliveryTag} 627 | j.Raw = d.Body 628 | 629 | if retries, ok := d.Headers[DefaultConfiguration.RetriesHeader]; ok { 630 | switch r := retries.(type) { 631 | case int16: 632 | j.Retries = int32(r) 633 | 634 | case int32: 635 | j.Retries = int32(r) 636 | 637 | case int64: 638 | if r <= math.MaxInt32 { 639 | j.Retries = int32(r) 640 | } else { 641 | j.Retries = 0 642 | } 643 | 644 | default: 645 | err = d.Reject(false) 646 | if err != nil { 647 | return nil, ErrRetrievingHeader.Wrap( 648 | err, 649 | DefaultConfiguration.RetriesHeader, 650 | d.MessageId, 651 | ) 652 | } 653 | 654 | return nil, ErrRetrievingHeader.New( 655 | DefaultConfiguration.RetriesHeader, 656 | d.MessageId, 657 | ) 658 | } 659 | } 660 | 661 | if errorType, ok := d.Headers[DefaultConfiguration.ErrorHeader]; ok { 662 | errorType, ok := errorType.(string) 663 | if !ok { 664 | err = d.Reject(false) 665 | if err != nil { 666 | return nil, ErrRetrievingHeader.Wrap( 667 | err, 668 | DefaultConfiguration.ErrorHeader, 669 | d.MessageId, 670 | ) 671 | } 672 | 673 | return nil, ErrRetrievingHeader.New( 674 | DefaultConfiguration.ErrorHeader, 675 | d.MessageId, 676 | ) 677 | } 678 | 679 | j.ErrorType = errorType 680 | } 681 | 682 | return j, nil 683 | } 684 | -------------------------------------------------------------------------------- /amqp/amqp_test.go: -------------------------------------------------------------------------------- 1 | package amqp 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "testing" 9 | "time" 10 | 11 | "gopkg.in/src-d/go-queue.v1" 12 | "gopkg.in/src-d/go-queue.v1/test" 13 | 14 | "github.com/streadway/amqp" 15 | "github.com/stretchr/testify/assert" 16 | "github.com/stretchr/testify/require" 17 | "github.com/stretchr/testify/suite" 18 | ) 19 | 20 | // RabbitMQ reconnect tests require running docker. 21 | // If `docker ps` command returned an error we skip some of the tests. 22 | var ( 23 | dockerIsRunning bool 24 | dockerCmdOutput string 25 | inAppVeyor bool 26 | ) 27 | 28 | func init() { 29 | cmd := exec.Command("docker", "ps") 30 | b, err := cmd.CombinedOutput() 31 | 32 | dockerCmdOutput, dockerIsRunning = string(b), (err == nil) 33 | inAppVeyor = os.Getenv("APPVEYOR") == "True" 34 | } 35 | 36 | func TestAMQPSuite(t *testing.T) { 37 | suite.Run(t, new(AMQPSuite)) 38 | } 39 | 40 | type AMQPSuite struct { 41 | test.QueueSuite 42 | } 43 | 44 | const testAMQPURI = "amqp://127.0.0.1:5672" 45 | 46 | func (s *AMQPSuite) SetupSuite() { 47 | s.BrokerURI = testAMQPURI 48 | } 49 | 50 | func TestDefaultConfig(t *testing.T) { 51 | assert.Equal(t, DefaultConfiguration.BuriedExchangeSuffix, ".buriedExchange") 52 | } 53 | 54 | func TestNewAMQPBroker_bad_url(t *testing.T) { 55 | assert := assert.New(t) 56 | 57 | b, err := New("badurl") 58 | assert.Error(err) 59 | assert.Nil(b) 60 | } 61 | 62 | func sendJobs(assert *assert.Assertions, n int, p queue.Priority, q queue.Queue) { 63 | for i := 0; i < n; i++ { 64 | j, err := queue.NewJob() 65 | assert.NoError(err) 66 | j.SetPriority(p) 67 | err = j.Encode(i) 68 | assert.NoError(err) 69 | err = q.Publish(j) 70 | assert.NoError(err) 71 | } 72 | } 73 | 74 | func TestAMQPPriorities(t *testing.T) { 75 | assert := assert.New(t) 76 | 77 | broker, err := New(testAMQPURI) 78 | assert.NoError(err) 79 | if !assert.NotNil(broker) { 80 | return 81 | } 82 | 83 | name := test.NewName() 84 | q, err := broker.Queue(name) 85 | assert.NoError(err) 86 | assert.NotNil(q) 87 | 88 | // Send 50 low priority jobs 89 | sendJobs(assert, 50, queue.PriorityLow, q) 90 | 91 | // Send 50 high priority jobs 92 | sendJobs(assert, 50, queue.PriorityUrgent, q) 93 | 94 | // Receive and collect priorities 95 | iter, err := q.Consume(1) 96 | assert.NoError(err) 97 | assert.NotNil(iter) 98 | 99 | sumFirst := uint(0) 100 | sumLast := uint(0) 101 | 102 | for i := 0; i < 100; i++ { 103 | j, err := iter.Next() 104 | assert.NoError(err) 105 | assert.NoError(j.Ack()) 106 | 107 | if i < 50 { 108 | sumFirst += uint(j.Priority) 109 | } else { 110 | sumLast += uint(j.Priority) 111 | } 112 | } 113 | 114 | assert.True(sumFirst > sumLast) 115 | assert.Equal(uint(queue.PriorityUrgent)*50, sumFirst) 116 | assert.Equal(uint(queue.PriorityLow)*50, sumLast) 117 | } 118 | 119 | func TestAMQPHeaders(t *testing.T) { 120 | broker, err := queue.NewBroker(testAMQPURI) 121 | require.NoError(t, err) 122 | defer func() { require.NoError(t, broker.Close()) }() 123 | 124 | q, err := broker.Queue(test.NewName()) 125 | require.NoError(t, err) 126 | 127 | tests := []struct { 128 | name string 129 | retries int32 130 | errorType string 131 | }{ 132 | { 133 | name: fmt.Sprintf("with %s and %s headers", 134 | DefaultConfiguration.RetriesHeader, DefaultConfiguration.ErrorHeader), 135 | retries: int32(10), 136 | errorType: "error-test", 137 | }, 138 | { 139 | name: fmt.Sprintf("with %s header", DefaultConfiguration.RetriesHeader), 140 | retries: int32(10), 141 | errorType: "", 142 | }, 143 | { 144 | name: fmt.Sprintf("with %s headers", DefaultConfiguration.ErrorHeader), 145 | retries: int32(0), 146 | errorType: "error-test", 147 | }, 148 | { 149 | name: "with no headers", 150 | retries: int32(0), 151 | errorType: "", 152 | }, 153 | } 154 | 155 | for i, test := range tests { 156 | job, err := queue.NewJob() 157 | require.NoError(t, err) 158 | 159 | job.Retries = test.retries 160 | job.ErrorType = test.errorType 161 | 162 | require.NoError(t, job.Encode(i)) 163 | require.NoError(t, q.Publish(job)) 164 | } 165 | 166 | jobIter, err := q.Consume(len(tests)) 167 | require.NoError(t, err) 168 | 169 | for _, test := range tests { 170 | t.Run(test.name, func(t *testing.T) { 171 | job, err := jobIter.Next() 172 | require.NoError(t, err) 173 | require.NotNil(t, job) 174 | 175 | require.Equal(t, test.retries, job.Retries) 176 | require.Equal(t, test.errorType, job.ErrorType) 177 | }) 178 | } 179 | } 180 | 181 | func TestAMQPHeaderRetriesType(t *testing.T) { 182 | broker, err := queue.NewBroker(testAMQPURI) 183 | require.NoError(t, err) 184 | defer func() { require.NoError(t, broker.Close()) }() 185 | 186 | q, err := broker.Queue(test.NewName()) 187 | require.NoError(t, err) 188 | 189 | qa, ok := q.(*Queue) 190 | require.True(t, ok) 191 | 192 | tests := []struct { 193 | name string 194 | retries interface{} 195 | }{ 196 | { 197 | name: "int16", 198 | retries: int16(42), 199 | }, 200 | { 201 | name: "int32", 202 | retries: int32(42), 203 | }, 204 | { 205 | name: "int64", 206 | retries: int64(42), 207 | }, 208 | } 209 | 210 | for _, test := range tests { 211 | headers := amqp.Table{} 212 | headers[DefaultConfiguration.RetriesHeader] = test.retries 213 | err := qa.conn.channel().Publish( 214 | "", // exchange 215 | qa.queue.Name, // routing key 216 | false, // mandatory 217 | false, 218 | amqp.Publishing{ 219 | DeliveryMode: amqp.Persistent, 220 | MessageId: "id", 221 | Priority: uint8(queue.PriorityNormal), 222 | Timestamp: time.Now(), 223 | ContentType: "application/msgpack", 224 | Body: []byte("gaxSZXBvc2l0b3J5SUTEEAFmXSlGxxOsFGMLs/gl7Qw="), 225 | Headers: headers, 226 | }, 227 | ) 228 | require.NoError(t, err) 229 | } 230 | 231 | jobIter, err := q.Consume(len(tests)) 232 | require.NoError(t, err) 233 | 234 | for _, test := range tests { 235 | t.Run(test.name, func(t *testing.T) { 236 | job, err := jobIter.Next() 237 | require.NoError(t, err) 238 | require.NotNil(t, job) 239 | 240 | require.Equal(t, int32(42), job.Retries) 241 | }) 242 | } 243 | } 244 | 245 | func TestAMQPRepublishBuried(t *testing.T) { 246 | broker, err := queue.NewBroker(testAMQPURI) 247 | require.NoError(t, err) 248 | defer func() { require.NoError(t, broker.Close()) }() 249 | 250 | queueName := test.NewName() 251 | q, err := broker.Queue(queueName) 252 | require.NoError(t, err) 253 | 254 | amqpQueue, ok := q.(*Queue) 255 | require.True(t, ok) 256 | 257 | buried := amqpQueue.buriedQueue 258 | 259 | tests := []struct { 260 | name string 261 | payload string 262 | }{ 263 | {name: "message 1", payload: "payload 1"}, 264 | {name: "message 2", payload: "republish"}, 265 | {name: "message 3", payload: "payload 3"}, 266 | {name: "message 3", payload: "payload 4"}, 267 | } 268 | 269 | for _, utest := range tests { 270 | job, err := queue.NewJob() 271 | require.NoError(t, err) 272 | 273 | job.Raw = []byte(utest.payload) 274 | 275 | err = buried.Publish(job) 276 | require.NoError(t, err) 277 | time.Sleep(1 * time.Second) 278 | } 279 | 280 | var condition queue.RepublishConditionFunc = func(j *queue.Job) bool { 281 | return string(j.Raw) == "republish" 282 | } 283 | 284 | err = q.RepublishBuried(condition) 285 | require.NoError(t, err) 286 | 287 | jobIter, err := q.Consume(1) 288 | require.NoError(t, err) 289 | defer func() { require.NoError(t, jobIter.Close()) }() 290 | 291 | job, err := jobIter.Next() 292 | require.NoError(t, err) 293 | require.Equal(t, string(job.Raw), "republish") 294 | } 295 | 296 | func TestReconnect(t *testing.T) { 297 | if inAppVeyor || !dockerIsRunning { 298 | t.Skip() 299 | } 300 | 301 | broker, err := queue.NewBroker(testAMQPURI) 302 | require.NoError(t, err) 303 | defer func() { broker.Close() }() 304 | 305 | queueName := test.NewName() 306 | q, err := broker.Queue(queueName) 307 | require.NoError(t, err) 308 | 309 | ctx, cancel := context.WithCancel(context.Background()) 310 | defer cancel() 311 | go rabbitHiccup(ctx, 5*time.Second) 312 | 313 | tests := []struct { 314 | name string 315 | payload string 316 | }{ 317 | {name: "message 1", payload: "payload 1"}, 318 | {name: "message 2", payload: "payload 2"}, 319 | {name: "message 3", payload: "payload 3"}, 320 | {name: "message 3", payload: "payload 4"}, 321 | } 322 | 323 | for _, test := range tests { 324 | job, err := queue.NewJob() 325 | require.NoError(t, err) 326 | 327 | job.Raw = []byte(test.payload) 328 | 329 | err = q.Publish(job) 330 | require.NoError(t, err) 331 | time.Sleep(100 * time.Millisecond) 332 | } 333 | 334 | n := len(tests) 335 | jobIter, err := q.Consume(1) 336 | require.NoError(t, err) 337 | defer func() { jobIter.Close() }() 338 | 339 | for i := 0; i < n; i++ { 340 | if job, err := jobIter.Next(); err != nil { 341 | t.Log(err) 342 | 343 | job, err = queue.NewJob() 344 | require.NoError(t, err) 345 | job.Raw = []byte("check connection - retry till we connect") 346 | err = q.Publish(job) 347 | require.NoError(t, err) 348 | break 349 | } else { 350 | t.Log(string(job.Raw)) 351 | } 352 | } 353 | } 354 | 355 | // rabbitHiccup restarts rabbitmq every interval 356 | // it requires the RabbitMQ running in docker container: 357 | // docker run --name rabbitmq -d -p 127.0.0.1:5672:5672 rabbitmq:3-management 358 | func rabbitHiccup(ctx context.Context, interval time.Duration) error { 359 | cmd := exec.Command("docker", "restart", "rabbitmq") 360 | err := cmd.Start() 361 | for err == nil { 362 | select { 363 | case <-ctx.Done(): 364 | err = ctx.Err() 365 | 366 | case <-time.After(interval): 367 | err = cmd.Start() 368 | } 369 | } 370 | 371 | return err 372 | } 373 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | version: "{build}" 2 | image: Visual Studio 2017 3 | platform: x64 4 | 5 | clone_folder: c:\gopath\src\gopkg.in\src-d\go-queue.v1 6 | 7 | environment: 8 | GOPATH: c:\gopath 9 | RABBITMQ_VERSION: any 10 | 11 | cache: 12 | - C:\Users\appveyor\AppData\Local\Temp\chocolatey\ -> appveyor.yml 13 | 14 | install: 15 | - set PATH=%GOPATH%\bin;C:\go\bin;C:\mingw-w64\x86_64-7.2.0-posix-seh-rt_v5-rev1\mingw64\bin;C:\msys64\usr\bin;%PATH%;"C:\Program Files\Git\mingw64\bin" 16 | - mingw32-make ci-install 17 | 18 | test_script: 19 | - mingw32-make test 20 | 21 | build: off 22 | deploy: off -------------------------------------------------------------------------------- /common.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "io" 5 | "time" 6 | 7 | "gopkg.in/src-d/go-errors.v1" 8 | ) 9 | 10 | // Priority represents a priority level. 11 | type Priority uint8 12 | 13 | const ( 14 | // PriorityUrgent represents an urgent priority level. 15 | PriorityUrgent Priority = 8 16 | // PriorityNormal represents a normal priority level. 17 | PriorityNormal Priority = 4 18 | // PriorityLow represents a low priority level. 19 | PriorityLow Priority = 0 20 | ) 21 | 22 | var ( 23 | // ErrAlreadyClosed is the error returned when trying to close an already closed 24 | // connection. 25 | ErrAlreadyClosed = errors.NewKind("already closed") 26 | // ErrEmptyJob is the error returned when an empty job is published. 27 | ErrEmptyJob = errors.NewKind("invalid empty job") 28 | // ErrTxNotSupported is the error returned when the transaction receives a 29 | // callback does not know how to handle. 30 | ErrTxNotSupported = errors.NewKind("transactions not supported") 31 | ) 32 | 33 | // Broker represents a message broker. 34 | type Broker interface { 35 | // Queue returns a Queue from the Broker with the given name. 36 | Queue(string) (Queue, error) 37 | // Close closes the connection of the Broker. 38 | Close() error 39 | } 40 | 41 | // TxCallback is a function to be called in a transaction. 42 | type TxCallback func(q Queue) error 43 | 44 | // RepublishConditionFunc is a function used to filter jobs to republish. 45 | type RepublishConditionFunc func(job *Job) bool 46 | 47 | // RepublishConditions alias of a list RepublishConditionFunc 48 | type RepublishConditions []RepublishConditionFunc 49 | 50 | // Comply checks if the Job matches any of the defined conditions. 51 | func (c RepublishConditions) Comply(job *Job) bool { 52 | if len(c) == 0 { 53 | return true 54 | } 55 | 56 | for _, condition := range c { 57 | if condition(job) { 58 | return true 59 | } 60 | } 61 | 62 | return false 63 | } 64 | 65 | // Queue represents a message queue. 66 | type Queue interface { 67 | // Publish publishes the given Job to the queue. 68 | Publish(*Job) error 69 | // PublishDelayed publishes the given Job to the queue with a given delay. 70 | PublishDelayed(*Job, time.Duration) error 71 | // Transaction executes the passed TxCallback inside a transaction. 72 | Transaction(TxCallback) error 73 | // Consume returns a JobIter for the queue. It receives the maximum 74 | // number of unacknowledged jobs the iterator will allow at any given 75 | // time (see the Acknowledger interface). 76 | Consume(advertisedWindow int) (JobIter, error) 77 | // RepublishBuried republishes to the main queue those jobs complying 78 | // one of the conditions, leaving the rest of them in the buried queue. 79 | RepublishBuried(conditions ...RepublishConditionFunc) error 80 | } 81 | 82 | // JobIter represents an iterator over a set of Jobs. 83 | type JobIter interface { 84 | // Next returns the next Job in the iterator. It should block until 85 | // a new job becomes available or while too many undelivered jobs have 86 | // been already returned (see the argument to Queue.Consume). Returns 87 | // ErrAlreadyClosed if the iterator is closed. 88 | Next() (*Job, error) 89 | io.Closer 90 | } 91 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | package queue_test 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | 7 | "gopkg.in/src-d/go-queue.v1" 8 | _ "gopkg.in/src-d/go-queue.v1/memory" 9 | ) 10 | 11 | func ExampleMemoryQueue() { 12 | b, err := queue.NewBroker("memory://") 13 | if err != nil { 14 | log.Fatal(err) 15 | } 16 | 17 | q, err := b.Queue("test-queue") 18 | if err != nil { 19 | log.Fatal(err) 20 | } 21 | 22 | j, err := queue.NewJob() 23 | if err != nil { 24 | log.Fatal(err) 25 | } 26 | 27 | if err := j.Encode("hello world!"); err != nil { 28 | log.Fatal(err) 29 | } 30 | 31 | if err := q.Publish(j); err != nil { 32 | log.Fatal(err) 33 | } 34 | 35 | iter, err := q.Consume(1) 36 | 37 | consumedJob, err := iter.Next() 38 | if err != nil { 39 | log.Fatal(err) 40 | } 41 | 42 | var payload string 43 | if err := consumedJob.Decode(&payload); err != nil { 44 | log.Fatal(err) 45 | } 46 | 47 | fmt.Println(payload) 48 | // Output: hello world! 49 | } 50 | -------------------------------------------------------------------------------- /job.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "gopkg.in/src-d/go-errors.v1" 8 | 9 | "github.com/satori/go.uuid" 10 | "gopkg.in/vmihailenco/msgpack.v2" 11 | ) 12 | 13 | const msgpackContentType = "application/msgpack" 14 | 15 | // Job contains the information for a job to be published to a queue. 16 | type Job struct { 17 | // ID of the job. 18 | ID string 19 | // Priority is the priority level. 20 | Priority Priority 21 | // Timestamp is the time of creation. 22 | Timestamp time.Time 23 | // Retries is the number of times this job can be processed before being rejected. 24 | Retries int32 25 | // ErrorType is the kind of error that made the job fail. 26 | ErrorType string 27 | // ContentType of the job 28 | ContentType string 29 | // Raw content of the Job 30 | Raw []byte 31 | // Acknowledger is the acknowledgement management system for the job. 32 | Acknowledger Acknowledger 33 | } 34 | 35 | // Acknowledger represents the object in charge of acknowledgement 36 | // management for a job. When a job is acknowledged using any of the 37 | // functions in this interface, it will be considered delivered by the 38 | // queue. 39 | type Acknowledger interface { 40 | // Ack is called when the Job has finished. 41 | Ack() error 42 | // Reject is called if the job has errored. The parameter indicates 43 | // whether the job should be put back in the queue or not. 44 | Reject(requeue bool) error 45 | } 46 | 47 | // NewJob creates a new Job with default values, a new unique ID and current 48 | // timestamp. 49 | func NewJob() (*Job, error) { 50 | u, err := uuid.NewV4() 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | return &Job{ 56 | ID: u.String(), 57 | Priority: PriorityNormal, 58 | Timestamp: time.Now(), 59 | ContentType: msgpackContentType, 60 | }, nil 61 | } 62 | 63 | // SetPriority sets job priority 64 | func (j *Job) SetPriority(priority Priority) { 65 | j.Priority = priority 66 | } 67 | 68 | // Encode encodes the payload to the wire format used. 69 | func (j *Job) Encode(payload interface{}) error { 70 | var err error 71 | j.Raw, err = encode(msgpackContentType, &payload) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | return nil 77 | } 78 | 79 | // Decode decodes the payload from the wire format. 80 | func (j *Job) Decode(payload interface{}) error { 81 | return decode(msgpackContentType, j.Raw, &payload) 82 | } 83 | 84 | // ErrCantAck is the error returned when the Job does not come from a queue 85 | var ErrCantAck = errors.NewKind("can't acknowledge this message, it does not come from a queue") 86 | 87 | // Ack is called when the job is finished. 88 | func (j *Job) Ack() error { 89 | if j.Acknowledger == nil { 90 | return ErrCantAck.New() 91 | } 92 | return j.Acknowledger.Ack() 93 | } 94 | 95 | // Reject is called when the job errors. The parameter is true if and only if the 96 | // job should be put back in the queue. 97 | func (j *Job) Reject(requeue bool) error { 98 | if j.Acknowledger == nil { 99 | return ErrCantAck.New() 100 | } 101 | return j.Acknowledger.Reject(requeue) 102 | } 103 | 104 | // Size returns the size of the message. 105 | func (j *Job) Size() int { 106 | return len(j.Raw) 107 | } 108 | 109 | func encode(mime string, p interface{}) ([]byte, error) { 110 | switch mime { 111 | case msgpackContentType: 112 | return msgpack.Marshal(p) 113 | default: 114 | return nil, fmt.Errorf("unknown content type: %s", mime) 115 | } 116 | } 117 | 118 | func decode(mime string, r []byte, p interface{}) error { 119 | switch mime { 120 | case msgpackContentType: 121 | return msgpack.Unmarshal(r, p) 122 | default: 123 | return fmt.Errorf("unknown content type: %s", mime) 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /memory/memory.go: -------------------------------------------------------------------------------- 1 | package memory 2 | 3 | import ( 4 | "io" 5 | "sync" 6 | "time" 7 | 8 | "gopkg.in/src-d/go-queue.v1" 9 | ) 10 | 11 | func init() { 12 | queue.Register("memory", func(uri string) (queue.Broker, error) { 13 | return New(), nil 14 | }) 15 | 16 | queue.Register("memoryfinite", func(uri string) (queue.Broker, error) { 17 | return NewFinite(true), nil 18 | }) 19 | } 20 | 21 | // Broker is a in-memory implementation of Broker. 22 | type Broker struct { 23 | queues map[string]queue.Queue 24 | finite bool 25 | } 26 | 27 | // New creates a new Broker for an in-memory queue. 28 | func New() queue.Broker { 29 | return NewFinite(false) 30 | } 31 | 32 | // NewFinite creates a new Broker for an in-memory queue. The argument 33 | // specifies if the JobIter stops on EOF or not. 34 | func NewFinite(finite bool) queue.Broker { 35 | return &Broker{ 36 | queues: make(map[string]queue.Queue), 37 | finite: finite, 38 | } 39 | } 40 | 41 | // Queue returns the queue with the given name. 42 | func (b *Broker) Queue(name string) (queue.Queue, error) { 43 | if _, ok := b.queues[name]; !ok { 44 | b.queues[name] = &Queue{ 45 | jobs: make([]*queue.Job, 0, 10), 46 | finite: b.finite, 47 | } 48 | } 49 | 50 | return b.queues[name], nil 51 | } 52 | 53 | // Close closes the connection in the Broker. 54 | func (b *Broker) Close() error { 55 | return nil 56 | } 57 | 58 | // Queue implements a queue.Queue interface. 59 | type Queue struct { 60 | jobs []*queue.Job 61 | buriedJobs []*queue.Job 62 | sync.RWMutex 63 | idx int 64 | publishImmediately bool 65 | finite bool 66 | } 67 | 68 | // Publish publishes a Job to the queue. 69 | func (q *Queue) Publish(j *queue.Job) error { 70 | if j == nil || j.Size() == 0 { 71 | return queue.ErrEmptyJob.New() 72 | } 73 | 74 | q.Lock() 75 | defer q.Unlock() 76 | q.jobs = append(q.jobs, j) 77 | return nil 78 | } 79 | 80 | // PublishDelayed publishes a Job to the queue with a given delay. 81 | func (q *Queue) PublishDelayed(j *queue.Job, delay time.Duration) error { 82 | if j == nil || j.Size() == 0 { 83 | return queue.ErrEmptyJob.New() 84 | } 85 | 86 | if q.publishImmediately { 87 | return q.Publish(j) 88 | } 89 | go func() { 90 | time.Sleep(delay) 91 | q.Publish(j) 92 | }() 93 | return nil 94 | } 95 | 96 | // RepublishBuried implements the Queue interface. 97 | func (q *Queue) RepublishBuried(conditions ...queue.RepublishConditionFunc) error { 98 | for _, job := range q.buriedJobs { 99 | if queue.RepublishConditions(conditions).Comply(job) { 100 | job.ErrorType = "" 101 | if err := q.Publish(job); err != nil { 102 | return err 103 | } 104 | } 105 | } 106 | return nil 107 | } 108 | 109 | // Transaction calls the given callback inside a transaction. 110 | func (q *Queue) Transaction(txcb queue.TxCallback) error { 111 | txQ := &Queue{jobs: make([]*queue.Job, 0, 10), publishImmediately: true} 112 | if err := txcb(txQ); err != nil { 113 | return err 114 | } 115 | 116 | q.jobs = append(q.jobs, txQ.jobs...) 117 | return nil 118 | } 119 | 120 | // Consume implements Queue. The advertisedWindow value is the maximum number of 121 | // unacknowledged jobs. Use 0 for an infinite window. 122 | func (q *Queue) Consume(advertisedWindow int) (queue.JobIter, error) { 123 | jobIter := JobIter{ 124 | q: q, 125 | RWMutex: &q.RWMutex, 126 | finite: q.finite, 127 | } 128 | 129 | if advertisedWindow > 0 { 130 | jobIter.chn = make(chan struct{}, advertisedWindow) 131 | } 132 | 133 | return &jobIter, nil 134 | } 135 | 136 | // JobIter implements a queue.JobIter interface. 137 | type JobIter struct { 138 | q *Queue 139 | closed bool 140 | finite bool 141 | chn chan struct{} 142 | *sync.RWMutex 143 | } 144 | 145 | // Acknowledger implements a queue.Acknowledger interface. 146 | type Acknowledger struct { 147 | q *Queue 148 | j *queue.Job 149 | chn chan struct{} 150 | } 151 | 152 | // Ack is called when the Job has finished. 153 | func (a *Acknowledger) Ack() error { 154 | a.release() 155 | return nil 156 | } 157 | 158 | // Reject is called when the Job has errored. The argument indicates whether the Job 159 | // should be put back in queue or not. If requeue is false, the job will go to the buried 160 | // queue until Queue.RepublishBuried() is called. 161 | func (a *Acknowledger) Reject(requeue bool) error { 162 | defer a.release() 163 | 164 | if !requeue { 165 | // Send to the buried queue for later republishing 166 | a.q.buriedJobs = append(a.q.buriedJobs, a.j) 167 | return nil 168 | } 169 | 170 | return a.q.Publish(a.j) 171 | } 172 | 173 | func (a *Acknowledger) release() { 174 | if a.chn != nil { 175 | <-a.chn 176 | } 177 | } 178 | 179 | func (i *JobIter) isClosed() bool { 180 | i.RLock() 181 | defer i.RUnlock() 182 | return i.closed 183 | } 184 | 185 | // Next returns the next job in the iter. 186 | func (i *JobIter) Next() (*queue.Job, error) { 187 | i.acquire() 188 | for { 189 | if i.isClosed() { 190 | i.release() 191 | return nil, queue.ErrAlreadyClosed.New() 192 | } 193 | 194 | j, err := i.next() 195 | if err == nil { 196 | return j, nil 197 | } 198 | 199 | if err == io.EOF && i.finite { 200 | i.release() 201 | return nil, err 202 | } 203 | 204 | time.Sleep(1 * time.Second) 205 | } 206 | } 207 | 208 | func (i *JobIter) next() (*queue.Job, error) { 209 | i.Lock() 210 | defer i.Unlock() 211 | if len(i.q.jobs) <= i.q.idx { 212 | return nil, io.EOF 213 | } 214 | 215 | j := i.q.jobs[i.q.idx] 216 | j.Acknowledger = &Acknowledger{j: j, q: i.q, chn: i.chn} 217 | i.q.idx++ 218 | 219 | return j, nil 220 | } 221 | 222 | // Close closes the iter. 223 | func (i *JobIter) Close() error { 224 | i.Lock() 225 | defer i.Unlock() 226 | i.closed = true 227 | return nil 228 | } 229 | 230 | func (i *JobIter) acquire() { 231 | if i.chn != nil { 232 | i.chn <- struct{}{} 233 | } 234 | } 235 | 236 | func (i *JobIter) release() { 237 | if i.chn != nil { 238 | <-i.chn 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /memory/memory_test.go: -------------------------------------------------------------------------------- 1 | package memory 2 | 3 | import ( 4 | "io" 5 | "testing" 6 | 7 | "gopkg.in/src-d/go-queue.v1" 8 | "gopkg.in/src-d/go-queue.v1/test" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/suite" 12 | ) 13 | 14 | func TestMemorySuite(t *testing.T) { 15 | suite.Run(t, new(MemorySuite)) 16 | } 17 | 18 | type MemorySuite struct { 19 | test.QueueSuite 20 | } 21 | 22 | func (s *MemorySuite) SetupSuite() { 23 | s.BrokerURI = "memory://" 24 | } 25 | 26 | func (s *MemorySuite) TestIntegration() { 27 | assert := assert.New(s.T()) 28 | 29 | qName := test.NewName() 30 | q, err := s.Broker.Queue(qName) 31 | assert.NoError(err) 32 | assert.NotNil(q) 33 | 34 | j, err := queue.NewJob() 35 | assert.NoError(err) 36 | 37 | j.Encode(true) 38 | err = q.Publish(j) 39 | assert.NoError(err) 40 | 41 | for i := 0; i < 100; i++ { 42 | job, err := queue.NewJob() 43 | assert.NoError(err) 44 | 45 | job.Encode(true) 46 | err = q.Publish(job) 47 | assert.NoError(err) 48 | } 49 | 50 | advertisedWindow := 0 // ignored by memory brokers 51 | iter, err := q.Consume(advertisedWindow) 52 | assert.NoError(err) 53 | 54 | retrievedJob, err := iter.Next() 55 | assert.NoError(err) 56 | assert.NoError(retrievedJob.Ack()) 57 | 58 | var payload bool 59 | err = retrievedJob.Decode(&payload) 60 | assert.NoError(err) 61 | assert.True(payload) 62 | 63 | assert.Equal(j.Priority, retrievedJob.Priority) 64 | assert.Equal(j.Timestamp.Second(), retrievedJob.Timestamp.Second()) 65 | 66 | err = iter.Close() 67 | assert.NoError(err) 68 | } 69 | 70 | func (s *MemorySuite) TestFinite() { 71 | assert := assert.New(s.T()) 72 | 73 | b, err := queue.NewBroker("memoryfinite://") 74 | assert.NoError(err) 75 | 76 | qName := test.NewName() 77 | q, err := b.Queue(qName) 78 | assert.NoError(err) 79 | assert.NotNil(q) 80 | 81 | j, err := queue.NewJob() 82 | assert.NoError(err) 83 | 84 | j.Encode(true) 85 | err = q.Publish(j) 86 | assert.NoError(err) 87 | 88 | advertisedWindow := 0 // ignored by memory brokers 89 | iter, err := q.Consume(advertisedWindow) 90 | assert.NoError(err) 91 | 92 | retrievedJob, err := iter.Next() 93 | assert.NoError(err) 94 | assert.NoError(retrievedJob.Ack()) 95 | 96 | retrievedJob, err = iter.Next() 97 | assert.Equal(io.EOF, err) 98 | assert.Nil(retrievedJob) 99 | } 100 | -------------------------------------------------------------------------------- /register.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "net/url" 5 | 6 | "gopkg.in/src-d/go-errors.v0" 7 | ) 8 | 9 | var ( 10 | // ErrUnsupportedProtocol is the error returned when a Broker does not know 11 | // how to connect to a given URI. 12 | ErrUnsupportedProtocol = errors.NewKind("unsupported protocol: %s") 13 | // ErrMalformedURI is the error returned when a Broker does not know 14 | // how to parse a given URI. 15 | ErrMalformedURI = errors.NewKind("malformed connection URI: %s") 16 | 17 | register = make(map[string]BrokerBuilder, 0) 18 | ) 19 | 20 | // BrokerBuilder instantiates a new Broker based on the given uri. 21 | type BrokerBuilder func(uri string) (Broker, error) 22 | 23 | // Register registers a new BrokerBuilder to be used by NewBroker, this function 24 | // should be used in an init function in the implementation packages such as 25 | // `amqp` and `memory`. 26 | func Register(name string, b BrokerBuilder) { 27 | register[name] = b 28 | } 29 | 30 | // NewBroker creates a new Broker based on the given URI. In order to register 31 | // different implementations the package should be imported, example: 32 | // 33 | // import _ "gopkg.in/src-d/go-queue.v1/amqp" 34 | func NewBroker(uri string) (Broker, error) { 35 | url, err := url.Parse(uri) 36 | if err != nil { 37 | return nil, ErrMalformedURI.Wrap(err, uri) 38 | } 39 | 40 | if url.Scheme == "" { 41 | return nil, ErrMalformedURI.New(uri) 42 | } 43 | 44 | b, ok := register[url.Scheme] 45 | if !ok { 46 | return nil, ErrUnsupportedProtocol.New(url.Scheme) 47 | } 48 | 49 | return b(uri) 50 | } 51 | -------------------------------------------------------------------------------- /register_test.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestNewBroker(t *testing.T) { 10 | assert := assert.New(t) 11 | 12 | b, err := NewBroker("badproto://badurl") 13 | assert.True(ErrUnsupportedProtocol.Is(err)) 14 | assert.Nil(b) 15 | 16 | b, err = NewBroker("foo://host%10") 17 | assert.Error(err) 18 | assert.Nil(b) 19 | } 20 | -------------------------------------------------------------------------------- /test/suite.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "math/rand" 7 | "strconv" 8 | "sync" 9 | "sync/atomic" 10 | "testing" 11 | "time" 12 | 13 | "gopkg.in/src-d/go-queue.v1" 14 | 15 | "github.com/stretchr/testify/assert" 16 | "github.com/stretchr/testify/suite" 17 | ) 18 | 19 | var testRand *rand.Rand 20 | 21 | func init() { 22 | testRand = rand.New(rand.NewSource(time.Now().UnixNano())) 23 | } 24 | 25 | func NewName() string { 26 | return fmt.Sprintf("queue_tests_%d", testRand.Int()) 27 | } 28 | 29 | type QueueSuite struct { 30 | suite.Suite 31 | r rand.Rand 32 | 33 | TxNotSupported bool 34 | BrokerURI string 35 | 36 | Broker queue.Broker 37 | } 38 | 39 | func (s *QueueSuite) SetupTest() { 40 | b, err := queue.NewBroker(s.BrokerURI) 41 | if !s.NoError(err) { 42 | s.FailNow(err.Error()) 43 | } 44 | 45 | s.Broker = b 46 | } 47 | 48 | func (s *QueueSuite) TearDownTest() { 49 | s.NoError(s.Broker.Close()) 50 | } 51 | 52 | func (s *QueueSuite) TestConsume_empty() { 53 | assert := assert.New(s.T()) 54 | 55 | qName := NewName() 56 | q, err := s.Broker.Queue(qName) 57 | assert.NoError(err) 58 | assert.NotNil(q) 59 | 60 | advertisedWindow := 1 61 | iter, err := q.Consume(advertisedWindow) 62 | assert.NoError(err) 63 | assert.NotNil(iter) 64 | 65 | assert.NoError(iter.Close()) 66 | } 67 | 68 | func (s *QueueSuite) TestJobIter_Next_closed() { 69 | assert := assert.New(s.T()) 70 | 71 | qName := NewName() 72 | q, err := s.Broker.Queue(qName) 73 | assert.NoError(err) 74 | assert.NotNil(q) 75 | 76 | advertisedWindow := 1 77 | iter, err := q.Consume(advertisedWindow) 78 | assert.NoError(err) 79 | assert.NotNil(iter) 80 | 81 | done := s.checkNextClosed(iter) 82 | assert.NoError(iter.Close()) 83 | <-done 84 | } 85 | 86 | func (s *QueueSuite) TestJobIter_Next_empty() { 87 | assert := assert.New(s.T()) 88 | 89 | qName := NewName() 90 | q, err := s.Broker.Queue(qName) 91 | assert.NoError(err) 92 | assert.NotNil(q) 93 | 94 | advertisedWindow := 1 95 | iter, err := q.Consume(advertisedWindow) 96 | assert.NoError(err) 97 | assert.NotNil(iter) 98 | 99 | nJobs := 0 100 | 101 | done := make(chan struct{}) 102 | go func() { 103 | j, err := iter.Next() 104 | assert.NoError(err) 105 | assert.NotNil(j) 106 | 107 | nJobs += 1 108 | done <- struct{}{} 109 | }() 110 | 111 | time.Sleep(50 * time.Millisecond) 112 | 113 | assert.Equal(0, nJobs) 114 | 115 | j, err := queue.NewJob() 116 | assert.NoError(err) 117 | 118 | err = j.Encode(1) 119 | assert.NoError(err) 120 | 121 | err = q.Publish(j) 122 | assert.NoError(err) 123 | 124 | <-done 125 | 126 | assert.Equal(1, nJobs) 127 | assert.NoError(iter.Close()) 128 | } 129 | 130 | func (s *QueueSuite) TestJob_Reject_no_requeue() { 131 | assert := assert.New(s.T()) 132 | 133 | qName := NewName() 134 | q, err := s.Broker.Queue(qName) 135 | assert.NoError(err) 136 | assert.NotNil(q) 137 | 138 | j, err := queue.NewJob() 139 | assert.NoError(err) 140 | 141 | err = j.Encode(1) 142 | assert.NoError(err) 143 | 144 | err = q.Publish(j) 145 | assert.NoError(err) 146 | 147 | advertisedWindow := 1 148 | iter, err := q.Consume(advertisedWindow) 149 | assert.NoError(err) 150 | assert.NotNil(iter) 151 | 152 | j, err = iter.Next() 153 | assert.NoError(err) 154 | assert.NotNil(j) 155 | 156 | err = j.Reject(false) 157 | assert.NoError(err) 158 | 159 | done := s.checkNextClosed(iter) 160 | time.Sleep(50 * time.Millisecond) 161 | assert.NoError(iter.Close()) 162 | <-done 163 | } 164 | 165 | func (s *QueueSuite) TestJob_Reject_requeue() { 166 | assert := assert.New(s.T()) 167 | 168 | qName := NewName() 169 | q, err := s.Broker.Queue(qName) 170 | assert.NoError(err) 171 | assert.NotNil(q) 172 | 173 | j, err := queue.NewJob() 174 | assert.NoError(err) 175 | 176 | err = j.Encode(1) 177 | assert.NoError(err) 178 | 179 | err = q.Publish(j) 180 | assert.NoError(err) 181 | 182 | advertisedWindow := 1 183 | iter, err := q.Consume(advertisedWindow) 184 | assert.NoError(err) 185 | assert.NotNil(iter) 186 | 187 | j, err = iter.Next() 188 | assert.NoError(err) 189 | assert.NotNil(j) 190 | 191 | err = j.Reject(true) 192 | assert.NoError(err) 193 | 194 | j, err = iter.Next() 195 | assert.NoError(err) 196 | assert.NotNil(j) 197 | 198 | assert.NoError(iter.Close()) 199 | } 200 | 201 | func (s *QueueSuite) TestPublish_nil() { 202 | assert := assert.New(s.T()) 203 | 204 | qName := NewName() 205 | q, err := s.Broker.Queue(qName) 206 | assert.NoError(err) 207 | assert.NotNil(q) 208 | 209 | err = q.Publish(nil) 210 | assert.True(queue.ErrEmptyJob.Is(err)) 211 | } 212 | 213 | func (s *QueueSuite) TestPublish_empty() { 214 | assert := assert.New(s.T()) 215 | 216 | qName := NewName() 217 | q, err := s.Broker.Queue(qName) 218 | assert.NoError(err) 219 | assert.NotNil(q) 220 | 221 | err = q.Publish(&queue.Job{}) 222 | assert.True(queue.ErrEmptyJob.Is(err)) 223 | } 224 | 225 | func (s *QueueSuite) TestPublishDelayed_nil() { 226 | assert := assert.New(s.T()) 227 | 228 | qName := NewName() 229 | q, err := s.Broker.Queue(qName) 230 | assert.NoError(err) 231 | assert.NotNil(q) 232 | 233 | err = q.PublishDelayed(nil, time.Second) 234 | assert.True(queue.ErrEmptyJob.Is(err)) 235 | } 236 | 237 | func (s *QueueSuite) TestPublishDelayed_empty() { 238 | assert := assert.New(s.T()) 239 | 240 | qName := NewName() 241 | q, err := s.Broker.Queue(qName) 242 | assert.NoError(err) 243 | assert.NotNil(q) 244 | 245 | err = q.PublishDelayed(&queue.Job{}, time.Second) 246 | assert.True(queue.ErrEmptyJob.Is(err)) 247 | } 248 | 249 | func (s *QueueSuite) TestPublishAndConsume_immediate_ack() { 250 | assert := assert.New(s.T()) 251 | 252 | qName := NewName() 253 | q, err := s.Broker.Queue(qName) 254 | assert.NoError(err) 255 | assert.NotNil(q) 256 | 257 | var ( 258 | ids []string 259 | priorities []queue.Priority 260 | timestamps []time.Time 261 | ) 262 | for i := 0; i < 100; i++ { 263 | j, err := queue.NewJob() 264 | assert.NoError(err) 265 | err = j.Encode(i) 266 | assert.NoError(err) 267 | err = q.Publish(j) 268 | assert.NoError(err) 269 | ids = append(ids, j.ID) 270 | priorities = append(priorities, j.Priority) 271 | timestamps = append(timestamps, j.Timestamp) 272 | } 273 | 274 | advertisedWindow := 1 275 | iter, err := q.Consume(advertisedWindow) 276 | assert.NoError(err) 277 | assert.NotNil(iter) 278 | 279 | for i := 0; i < 100; i++ { 280 | j, err := iter.Next() 281 | assert.NoError(err) 282 | assert.NoError(j.Ack()) 283 | 284 | var payload int 285 | assert.NoError(j.Decode(&payload)) 286 | assert.Equal(i, payload) 287 | 288 | assert.Equal(ids[i], j.ID) 289 | assert.Equal(priorities[i], j.Priority) 290 | assert.Equal(timestamps[i].Unix(), j.Timestamp.Unix()) 291 | } 292 | 293 | done := s.checkNextClosed(iter) 294 | assert.NoError(iter.Close()) 295 | <-done 296 | } 297 | 298 | func (s *QueueSuite) TestConsumersCanShareJobIteratorConcurrently() { 299 | assert := assert.New(s.T()) 300 | const ( 301 | nConsumers int = 10 302 | nJobs int = nConsumers 303 | advertisedWindow int = nConsumers 304 | ) 305 | queue := s.newQueueWithJobs(nJobs) 306 | 307 | // the iter will be shared by all consumers 308 | iter, err := queue.Consume(advertisedWindow) 309 | assert.NoError(err) 310 | assert.NotNil(iter) 311 | 312 | // attempt to start several consumers concurrently 313 | // that never Ack or Reject their jobs 314 | var allStarted sync.WaitGroup 315 | allStarted.Add(nConsumers) 316 | for i := 0; i < nConsumers; i++ { 317 | go func() { 318 | _, err := iter.Next() 319 | assert.NoError(err) 320 | allStarted.Done() 321 | }() 322 | } 323 | 324 | // send true to the done channel when all consumers has started 325 | done := make(chan bool) 326 | go func() { 327 | allStarted.Wait() 328 | done <- true 329 | }() 330 | 331 | // wait until all consumers have started or fail after a give up period 332 | giveUp := time.After(1 * time.Second) 333 | select { 334 | case <-done: 335 | // nop, all consumers started concurrently just fine. 336 | case <-giveUp: 337 | assert.FailNow("Give up waiting for consumers to start") 338 | } 339 | } 340 | 341 | // newQueueWithJobs creates and return a new queue with n jobs in it. 342 | func (s *QueueSuite) newQueueWithJobs(n int) queue.Queue { 343 | assert := assert.New(s.T()) 344 | 345 | q, err := s.Broker.Queue(NewName()) 346 | assert.NoError(err) 347 | 348 | for i := 0; i < n; i++ { 349 | job, err := queue.NewJob() 350 | assert.NoError(err) 351 | err = job.Encode(i) 352 | assert.NoError(err) 353 | err = q.Publish(job) 354 | assert.NoError(err) 355 | } 356 | 357 | return q 358 | } 359 | 360 | func (s *QueueSuite) TestDelayed() { 361 | assert := assert.New(s.T()) 362 | 363 | delay := 1 * time.Second 364 | 365 | qName := NewName() 366 | q, err := s.Broker.Queue(qName) 367 | assert.NoError(err) 368 | assert.NotNil(q) 369 | 370 | j, err := queue.NewJob() 371 | assert.NoError(err) 372 | err = j.Encode("hello") 373 | assert.NoError(err) 374 | 375 | start := time.Now() 376 | err = q.PublishDelayed(j, delay) 377 | assert.NoError(err) 378 | 379 | advertisedWindow := 1 380 | iter, err := q.Consume(advertisedWindow) 381 | assert.NoError(err) 382 | 383 | var since time.Duration 384 | for { 385 | j, err := iter.Next() 386 | assert.NoError(err) 387 | if j == nil { 388 | time.Sleep(300 * time.Millisecond) 389 | continue 390 | } 391 | 392 | since = time.Since(start) 393 | 394 | var payload string 395 | assert.NoError(j.Decode(&payload)) 396 | assert.Equal("hello", payload) 397 | break 398 | } 399 | 400 | assert.True(since >= delay) 401 | } 402 | 403 | func (s *QueueSuite) TestTransaction_Error() { 404 | if s.TxNotSupported { 405 | s.T().Skip("transactions not supported") 406 | } 407 | 408 | assert := assert.New(s.T()) 409 | 410 | qName := NewName() 411 | q, err := s.Broker.Queue(qName) 412 | assert.NoError(err) 413 | assert.NotNil(q) 414 | 415 | err = q.Transaction(func(qu queue.Queue) error { 416 | job, err := queue.NewJob() 417 | assert.NoError(err) 418 | assert.NoError(job.Encode("goodbye")) 419 | assert.NoError(qu.Publish(job)) 420 | return errors.New("foo") 421 | }) 422 | assert.Error(err) 423 | 424 | advertisedWindow := 1 425 | i, err := q.Consume(advertisedWindow) 426 | assert.NoError(err) 427 | 428 | done := s.checkNextClosed(i) 429 | time.Sleep(50 * time.Millisecond) 430 | assert.NoError(i.Close()) 431 | <-done 432 | } 433 | 434 | func (s *QueueSuite) TestTransaction() { 435 | if s.TxNotSupported { 436 | s.T().Skip("transactions not supported") 437 | } 438 | 439 | assert := assert.New(s.T()) 440 | 441 | qName := NewName() 442 | q, err := s.Broker.Queue(qName) 443 | assert.NoError(err) 444 | assert.NotNil(q) 445 | 446 | err = q.Transaction(func(q queue.Queue) error { 447 | job, err := queue.NewJob() 448 | assert.NoError(err) 449 | assert.NoError(job.Encode("hello")) 450 | assert.NoError(q.Publish(job)) 451 | return nil 452 | }) 453 | assert.NoError(err) 454 | 455 | advertisedWindow := 1 456 | iter, err := q.Consume(advertisedWindow) 457 | assert.NoError(err) 458 | j, err := iter.Next() 459 | assert.NoError(err) 460 | assert.NotNil(j) 461 | var payload string 462 | assert.NoError(j.Decode(&payload)) 463 | assert.Equal("hello", payload) 464 | assert.NoError(iter.Close()) 465 | } 466 | 467 | func (s *QueueSuite) TestTransaction_not_supported() { 468 | assert := assert.New(s.T()) 469 | 470 | if !s.TxNotSupported { 471 | s.T().Skip("transactions supported") 472 | } 473 | 474 | qName := NewName() 475 | q, err := s.Broker.Queue(qName) 476 | assert.NoError(err) 477 | assert.NotNil(q) 478 | 479 | err = q.Transaction(nil) 480 | assert.True(queue.ErrTxNotSupported.Is(err)) 481 | } 482 | 483 | func (s *QueueSuite) TestRetryQueue() { 484 | assert := assert.New(s.T()) 485 | 486 | qName := NewName() 487 | q, err := s.Broker.Queue(qName) 488 | assert.NoError(err) 489 | assert.NotNil(q) 490 | 491 | // 1: Publish jobs to the main queue. 492 | j1, err := queue.NewJob() 493 | assert.NoError(err) 494 | err = j1.Encode(1) 495 | assert.NoError(err) 496 | 497 | err = q.Publish(j1) 498 | assert.NoError(err) 499 | 500 | j2, err := queue.NewJob() 501 | assert.NoError(err) 502 | err = j2.Encode(2) 503 | assert.NoError(err) 504 | err = q.Publish(j2) 505 | assert.NoError(err) 506 | 507 | // 2: consume and reject them. 508 | advertisedWindow := 1 509 | iterMain, err := q.Consume(advertisedWindow) 510 | assert.NoError(err) 511 | assert.NotNil(iterMain) 512 | 513 | jReject1, err := iterMain.Next() 514 | assert.NoError(err) 515 | assert.NotNil(jReject1) 516 | // Jobs should go to the retry queue when rejected with requeue = false 517 | err = jReject1.Reject(false) 518 | assert.NoError(err) 519 | 520 | jReject2, err := iterMain.Next() 521 | assert.NoError(err) 522 | assert.NotNil(jReject2) 523 | err = jReject2.Reject(false) 524 | assert.NoError(err) 525 | 526 | // 3. republish the jobs in the retry queue. 527 | err = q.RepublishBuried() 528 | assert.NoError(err) 529 | 530 | // 4. re-read the jobs on the main queue. 531 | var payload int 532 | jRepub1, err := iterMain.Next() 533 | assert.NoError(jRepub1.Decode(&payload)) 534 | assert.Equal(1, payload) 535 | assert.NoError(jRepub1.Ack()) 536 | 537 | jRepub2, err := iterMain.Next() 538 | assert.NoError(jRepub2.Decode(&payload)) 539 | assert.Equal(2, payload) 540 | assert.NoError(jRepub2.Ack()) 541 | 542 | done := s.checkNextClosed(iterMain) 543 | assert.NoError(iterMain.Close()) 544 | iterMain.Close() 545 | <-done 546 | } 547 | 548 | func (s *QueueSuite) TestConcurrent() { 549 | testCases := []int{1, 2, 13, 150} 550 | 551 | for _, advertisedWindow := range testCases { 552 | s.T().Run(strconv.Itoa(advertisedWindow), func(t *testing.T) { 553 | assert := assert.New(t) 554 | 555 | qName := NewName() 556 | q, err := s.Broker.Queue(qName) 557 | assert.NoError(err) 558 | assert.NotNil(q) 559 | 560 | var continueWG sync.WaitGroup 561 | continueWG.Add(1) 562 | 563 | var calledWG sync.WaitGroup 564 | 565 | var calls int32 566 | atomic.StoreInt32(&calls, 0) 567 | 568 | iter, err := q.Consume(advertisedWindow) 569 | assert.NoError(err) 570 | 571 | go func() { 572 | for { 573 | j, err := iter.Next() 574 | if queue.ErrAlreadyClosed.Is(err) { 575 | return 576 | } 577 | assert.NoError(err) 578 | if j == nil { 579 | time.Sleep(300 * time.Millisecond) 580 | continue 581 | } 582 | 583 | go func() { 584 | // Removes 1 from calledWG, and gets locked 585 | // until continueWG is released 586 | atomic.AddInt32(&calls, 1) 587 | 588 | calledWG.Done() 589 | continueWG.Wait() 590 | 591 | assert.NoError(j.Ack()) 592 | }() 593 | } 594 | }() 595 | 596 | assert.EqualValues(0, atomic.LoadInt32(&calls)) 597 | calledWG.Add(advertisedWindow) 598 | 599 | // Enqueue some jobs, 3 * advertisedWindow 600 | for i := 0; i < advertisedWindow*3; i++ { 601 | j, err := queue.NewJob() 602 | assert.NoError(err) 603 | err = j.Encode(i) 604 | assert.NoError(err) 605 | err = q.Publish(j) 606 | assert.NoError(err) 607 | } 608 | 609 | // The first batch of calls should be exactly advertisedWindow 610 | calledWG.Wait() 611 | assert.EqualValues(advertisedWindow, atomic.LoadInt32(&calls)) 612 | 613 | // Let the iterator go though all the jobs, should be 3*advertisedWindow 614 | calledWG.Add(2 * advertisedWindow) 615 | continueWG.Done() 616 | calledWG.Wait() 617 | assert.EqualValues(3*advertisedWindow, atomic.LoadInt32(&calls)) 618 | }) 619 | } 620 | } 621 | 622 | func (s *QueueSuite) checkNextClosed(iter queue.JobIter) chan struct{} { 623 | assert := assert.New(s.T()) 624 | 625 | done := make(chan struct{}) 626 | go func() { 627 | j, err := iter.Next() 628 | assert.True(queue.ErrAlreadyClosed.Is(err)) 629 | assert.Nil(j) 630 | done <- struct{}{} 631 | }() 632 | return done 633 | } 634 | --------------------------------------------------------------------------------