├── .gitignore ├── Gopkg.lock ├── Gopkg.toml ├── LICENSE ├── README.md ├── config.go ├── connection.go ├── job.go ├── queue.go └── topic.go /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | cmd 3 | bin -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc" 6 | name = "github.com/Sirupsen/logrus" 7 | packages = ["."] 8 | pruneopts = "UT" 9 | revision = "3e01752db0189b9157070a0e1668a620f9a85da2" 10 | version = "v1.0.6" 11 | 12 | [[projects]] 13 | digest = "1:2209584c0f7c9b68c23374e659357ab546e1b70eec2761f03280f69a8fd23d77" 14 | name = "github.com/cenkalti/backoff" 15 | packages = ["."] 16 | pruneopts = "UT" 17 | revision = "2ea60e5f094469f9e65adb9cd103795b73ae743e" 18 | version = "v2.0.0" 19 | 20 | [[projects]] 21 | digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747" 22 | name = "github.com/pkg/errors" 23 | packages = ["."] 24 | pruneopts = "UT" 25 | revision = "645ef00459ed84a119197bfb8d8205042c6df63d" 26 | version = "v0.8.0" 27 | 28 | [[projects]] 29 | branch = "master" 30 | digest = "1:08e00568d99ec12096ba60887632eb2b94ed8b3c23e2ed90eb263e12eacf8f3a" 31 | name = "github.com/streadway/amqp" 32 | packages = ["."] 33 | pruneopts = "UT" 34 | revision = "e5adc2ada8b8efff032bf61173a233d143e9318e" 35 | 36 | [[projects]] 37 | branch = "master" 38 | digest = "1:3f3a05ae0b95893d90b9b3b5afdb79a9b3d96e4e36e099d841ae602e4aca0da8" 39 | name = "golang.org/x/crypto" 40 | packages = ["ssh/terminal"] 41 | pruneopts = "UT" 42 | revision = "0e37d006457bf46f9e6692014ba72ef82c33022c" 43 | 44 | [[projects]] 45 | branch = "master" 46 | digest = "1:76ee51c3f468493aff39dbacc401e8831fbb765104cbf613b89bef01cf4bad70" 47 | name = "golang.org/x/net" 48 | packages = ["context"] 49 | pruneopts = "UT" 50 | revision = "26e67e76b6c3f6ce91f7c52def5af501b4e0f3a2" 51 | 52 | [[projects]] 53 | branch = "master" 54 | digest = "1:49edbccdd6fb213dcf45486b0c939b788740451e5f2ff77aa42ca742d10da449" 55 | name = "golang.org/x/sys" 56 | packages = [ 57 | "unix", 58 | "windows", 59 | ] 60 | pruneopts = "UT" 61 | revision = "1561086e645b2809fb9f8a1e2a38160bf8d53bf4" 62 | 63 | [solve-meta] 64 | analyzer-name = "dep" 65 | analyzer-version = 1 66 | input-imports = [ 67 | "github.com/Sirupsen/logrus", 68 | "github.com/cenkalti/backoff", 69 | "github.com/pkg/errors", 70 | "github.com/streadway/amqp", 71 | ] 72 | solver-name = "gps-cdcl" 73 | solver-version = 1 74 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | # Gopkg.toml example 2 | # 3 | # Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md 4 | # for detailed Gopkg.toml documentation. 5 | # 6 | # required = ["github.com/user/thing/cmd/thing"] 7 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] 8 | # 9 | # [[constraint]] 10 | # name = "github.com/user/project" 11 | # version = "1.0.0" 12 | # 13 | # [[constraint]] 14 | # name = "github.com/user/project2" 15 | # branch = "dev" 16 | # source = "github.com/myfork/project2" 17 | # 18 | # [[override]] 19 | # name = "github.com/x/y" 20 | # version = "2.4.0" 21 | # 22 | # [prune] 23 | # non-go = false 24 | # go-tests = true 25 | # unused-packages = true 26 | 27 | 28 | [prune] 29 | go-tests = true 30 | unused-packages = true 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Arturo Vergara , Mazing Studio 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Hop 2 | 3 | [![GoDoc](https://godoc.org/github.com/mazingstudio/hop?status.svg)](https://godoc.org/github.com/mazingstudio/hop) 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/mazingstudio/hop)](https://goreportcard.com/report/github.com/mazingstudio/hop) 5 | [![license](https://img.shields.io/github/license/mashape/apistatus.svg)]() 6 | 7 | _An AMQP client wrapper that provides easy work queue semantics_ 8 | 9 | ## Introduction 10 | 11 | Hop consists of a simple set of abstractions over AMQP concepts that allow you to think in terms of jobs and topics, rather than queues, exchanges and routing. 12 | 13 | ### Jobs 14 | 15 | The most basic abstraction in Hop are Jobs, which are work units that may be marked as done or failed by your worker processes. When a Job is completed succesfully, it gets removed from the queue. If a Job fails, on the other hand, the worker has the option of requeueing it or dropping it altogether. If a worker dies in the middle of processing a Job, the Job is placed back into the queue. All of this maps to AMQP messages and `ack` and `reject` mechanics. 16 | 17 | ### Topics 18 | 19 | Similarly to the beanstalkd concept of tubes, Hop introduces Topics, which are distinct queues into which producers can put work units, and from which workers can pull them for processing. Underneath, a Topic is a mapping to an AMQP TCP channel, and a queue with the same name as the Topic. Note that Topics are _not_ the same as AMQP exchange topics. 20 | 21 | ### The Work Queue 22 | 23 | The third Hop abstraction is the Work Queue, which is nothing more than the grouping of Topics a worker or producer can interact with. The Work Queue abtracts over the TCP connection to the AMQP broker and performs such tasks as dialing, graceful shutdown, and declaration of new channels. Underneath, it maps to an AMQP direct exchange. 24 | 25 | ## Usage 26 | 27 | ```go 28 | package main 29 | 30 | import ( 31 | log "github.com/Sirupsen/logrus" 32 | "github.com/mazingstudio/hop" 33 | ) 34 | 35 | func main() { 36 | // Initialize a WorkQueue with the default configuration 37 | wq, err := hop.DefaultQueue("amqp://guest:guest@localhost:5672/") 38 | if err != nil { 39 | log.Fatalf("error creating queue: %s", err) 40 | } 41 | // Make sure to gracefully shut down 42 | defer wq.Close() 43 | 44 | // Get the "tasks" Topic handle 45 | tasks, err := wq.GetTopic("tasks") 46 | if err != nil { 47 | log.Fatalf("error getting topic: %s", err) 48 | } 49 | 50 | // Put a Job into the "tasks" Topic 51 | err = tasks.Put([]byte("Hello")) 52 | if err != nil { 53 | log.Fatalf("error putting: %s", err) 54 | } 55 | 56 | // Pull the Job from the Topic 57 | hello, err := tasks.Pull() 58 | if err != nil { 59 | log.Fatalf("error pulling: %s", err) 60 | } 61 | // This should print "hello" 62 | log.Infof("job body: %s", hello.Body()) 63 | 64 | // Mark the Job as failed and requeue 65 | hello.Fail(true) 66 | 67 | // Pull the Job again 68 | hello2, err := tasks.Pull() 69 | if err != nil { 70 | log.Fatalf("error pulling: %s", err) 71 | } 72 | log.Infof("job body: %s", hello.Body()) 73 | 74 | // Mark the Job as done 75 | hello2.Done() 76 | } 77 | ``` 78 | 79 | ## License & Third Party Code 80 | 81 | Hop uses [`streadway/amqp`](https://github.com/streadway/amqp) internally. 82 | 83 | Hop is distributed under the MIT License. Please refer to `LICENSE` for more details. -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package hop 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/streadway/amqp" 7 | ) 8 | 9 | // Config represents a WorkQueue configuration. 10 | type Config struct { 11 | // ExchangeName is used when naming the underlying exchange for the work 12 | // queue. Please DO NOT use a pre-existing exchange name, as this would not 13 | // guarantee that the pre-existing exchange has the necessary properties 14 | // to provide work queue semantics. 15 | ExchangeName string 16 | // Persist indicates whether the underlying queues and messages should be 17 | // saved to disk to survive server restarts and crashes 18 | Persistent bool 19 | // MaxConnectionRetry indicates for how long dialing should be retried 20 | // during connection recovery 21 | MaxConnectionRetry time.Duration 22 | // AMQPConfig is optionally used to set up the internal connection to the 23 | // AMQP broker. If nil, default values (determined by the underlying 24 | // library) are used. 25 | AMQPConfig *amqp.Config 26 | } 27 | 28 | // DefaultConfig are the default values used when initializing the default 29 | // queue. 30 | var DefaultConfig = &Config{ 31 | ExchangeName: "hop.exchange", 32 | Persistent: false, 33 | MaxConnectionRetry: 15 * time.Minute, 34 | } 35 | -------------------------------------------------------------------------------- /connection.go: -------------------------------------------------------------------------------- 1 | package hop 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/cenkalti/backoff" 7 | "github.com/pkg/errors" 8 | "github.com/streadway/amqp" 9 | ) 10 | 11 | type connection struct { 12 | addr string 13 | config *Config 14 | conn *amqp.Connection 15 | mu sync.Mutex 16 | } 17 | 18 | func newConnection(addr string, config *Config) *connection { 19 | return &connection{ 20 | addr: addr, 21 | config: config, 22 | } 23 | } 24 | 25 | func (c *connection) connect() error { 26 | c.mu.Lock() 27 | defer c.mu.Unlock() 28 | conn, err := c.dialWithBackOff() 29 | if err != nil { 30 | return errors.Wrap(err, "queue connection failed permanently") 31 | } 32 | c.conn = conn 33 | return nil 34 | } 35 | 36 | func (c *connection) dialWithBackOff() (*amqp.Connection, error) { 37 | var conn *amqp.Connection 38 | connect := func() error { 39 | var err error 40 | if c.config.AMQPConfig != nil { 41 | conn, err = amqp.DialConfig(c.addr, *c.config.AMQPConfig) 42 | return err 43 | } 44 | conn, err = amqp.Dial(c.addr) 45 | return err 46 | } 47 | b := backoff.NewExponentialBackOff() 48 | b.MaxElapsedTime = c.config.MaxConnectionRetry 49 | err := backoff.Retry(connect, b) 50 | if err != nil { 51 | return nil, errors.Wrap(err, "error dialing") 52 | } 53 | return conn, nil 54 | } 55 | 56 | func (c *connection) newChannel() interface{} { 57 | var ( 58 | ch *amqp.Channel 59 | err error 60 | ) 61 | for err == nil { 62 | ch, err = c.conn.Channel() 63 | if err != nil { 64 | err = c.connect() 65 | continue 66 | } 67 | return ch 68 | } 69 | return errors.Wrap(err, "failed to initialize channel") 70 | } 71 | 72 | func (c *connection) Close() error { 73 | return c.conn.Close() 74 | } 75 | -------------------------------------------------------------------------------- /job.go: -------------------------------------------------------------------------------- 1 | package hop 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/streadway/amqp" 6 | ) 7 | 8 | // Job represents a work unit taken from the work queue. 9 | type Job struct { 10 | topic *Topic 11 | del *amqp.Delivery 12 | ch *amqp.Channel 13 | } 14 | 15 | // Done marks the Job for queue removal. 16 | func (j *Job) Done() error { 17 | defer j.topic.queue.putChannel(j.ch) 18 | err := j.ch.Ack(j.del.DeliveryTag, false) 19 | if err != nil { 20 | return errors.Wrap(err, "error acknowledging delivery") 21 | } 22 | return nil 23 | } 24 | 25 | // Fail marks the Job as failed. If requeue is true, the Job will be added 26 | // back to the queue; otherwise, it will be dropped. 27 | func (j *Job) Fail(requeue bool) error { 28 | defer j.topic.queue.putChannel(j.ch) 29 | err := j.ch.Nack(j.del.DeliveryTag, false, requeue) 30 | if err != nil { 31 | return errors.Wrap(err, "error rejecting delivery") 32 | } 33 | return nil 34 | } 35 | 36 | // Body returns the Job's body 37 | func (j *Job) Body() []byte { 38 | return j.del.Body 39 | } 40 | 41 | // Delivery returns the Job's underlying delivery. Use this if you need more 42 | // control over the AMQP message. 43 | func (j *Job) Delivery() *amqp.Delivery { 44 | return j.del 45 | } 46 | -------------------------------------------------------------------------------- /queue.go: -------------------------------------------------------------------------------- 1 | package hop 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/pkg/errors" 7 | "github.com/streadway/amqp" 8 | ) 9 | 10 | // WorkQueue is an abstraction over an AMQP connection that automatically 11 | // performs management and configuration to provide easy work queue semantics. 12 | type WorkQueue struct { 13 | conn *connection 14 | config *Config 15 | channelPool sync.Pool 16 | } 17 | 18 | const exchangeKind = "direct" 19 | 20 | // DefaultQueue creates a queue with the default configuration and connection 21 | // parameters. 22 | func DefaultQueue(addr string) (*WorkQueue, error) { 23 | return ConnectQueue(addr, DefaultConfig) 24 | } 25 | 26 | // ConnectQueue dials using the provided address string and creates a queue 27 | // with the passed configuration. 28 | func ConnectQueue(addr string, config *Config) (*WorkQueue, error) { 29 | conn := newConnection(addr, config) 30 | err := conn.connect() 31 | if err != nil { 32 | return nil, errors.Wrap(err, "error connecting") 33 | } 34 | return newQueue(conn, config) 35 | } 36 | 37 | func newQueue(conn *connection, config *Config) (*WorkQueue, error) { 38 | q := &WorkQueue{ 39 | conn: conn, 40 | config: config, 41 | channelPool: sync.Pool{ 42 | New: conn.newChannel, 43 | }, 44 | } 45 | if err := q.init(); err != nil { 46 | return nil, errors.Wrap(err, "failed to initialize queue") 47 | } 48 | return q, nil 49 | } 50 | 51 | func (q *WorkQueue) init() error { 52 | ch, err := q.getChannel() 53 | if err != nil { 54 | return errors.Wrap(err, "error getting management channel") 55 | } 56 | defer q.putChannel(ch) 57 | 58 | err = ch.ExchangeDeclare(q.config.ExchangeName, exchangeKind, 59 | q.config.Persistent, false, false, false, nil) 60 | if err != nil { 61 | return errors.Wrap(err, "error declaring exchange") 62 | } 63 | return nil 64 | } 65 | 66 | // Close gracefully closes the connection to the message broker. 67 | func (q *WorkQueue) Close() error { 68 | err := q.conn.Close() 69 | if err != nil { 70 | return errors.Wrap(err, "error closing connection") 71 | } 72 | return nil 73 | } 74 | 75 | // --- Channels --- 76 | 77 | func (q *WorkQueue) getChannel() (*amqp.Channel, error) { 78 | res := q.channelPool.Get() 79 | switch v := res.(type) { 80 | case error: 81 | return nil, errors.Wrap(v, "channel retrieval failed permanently") 82 | case *amqp.Channel: 83 | return v, nil 84 | } 85 | return nil, nil 86 | } 87 | 88 | func (q *WorkQueue) putChannel(ch *amqp.Channel) { 89 | q.channelPool.Put(ch) 90 | } 91 | -------------------------------------------------------------------------------- /topic.go: -------------------------------------------------------------------------------- 1 | package hop 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/streadway/amqp" 6 | ) 7 | 8 | // Topic represents a named "tube" from which jobs can be exclusively pulled and 9 | // put into. Underneath, each topic corresponds to a queue bound to a direct 10 | // exchange, and each topic instance manages its own channel. 11 | type Topic struct { 12 | queue *WorkQueue 13 | name string 14 | config *Config 15 | } 16 | 17 | // GetTopic returns a "tube" handle, from which to pull and put jobs into. 18 | // Thanks to AMQP protocol rules, Topic declarations are idempotent, meaning 19 | // that Topics only get created if they don't already exist. 20 | func (q *WorkQueue) GetTopic(name string) (*Topic, error) { 21 | ch, err := q.getChannel() 22 | if err != nil { 23 | return nil, errors.Wrap(err, "error getting channel") 24 | } 25 | defer q.putChannel(ch) 26 | _, err = ch.QueueDeclare(name, q.config.Persistent, 27 | false, false, false, nil) 28 | if err != nil { 29 | return nil, errors.Wrap(err, "error declaring queue") 30 | } 31 | err = ch.QueueBind(name, name, q.config.ExchangeName, true, nil) 32 | if err != nil { 33 | return nil, errors.Wrap(err, "unable to bind queue") 34 | } 35 | return &Topic{ 36 | queue: q, 37 | name: name, 38 | config: q.config, 39 | }, nil 40 | } 41 | 42 | // Name returns the Topic's name. 43 | func (t *Topic) Name() string { 44 | return t.name 45 | } 46 | 47 | // Pull blocks until a Job is available to consume and returns it. 48 | func (t *Topic) Pull() (*Job, error) { 49 | var ( 50 | del amqp.Delivery 51 | ok bool 52 | err error 53 | ) 54 | ch, err := t.queue.getChannel() 55 | if err != nil { 56 | return nil, errors.Wrap(err, "error getting channel") 57 | } 58 | for !ok { 59 | del, ok, err = ch.Get(t.name, false) 60 | if err != nil { 61 | return nil, errors.Wrap(err, "error getting message") 62 | } 63 | } 64 | return &Job{ 65 | topic: t, 66 | del: &del, 67 | ch: ch, 68 | }, nil 69 | } 70 | 71 | // Put places a Job in the queue with the given body. Use PutPublishing if you 72 | // need more control over your messages. 73 | func (t *Topic) Put(body []byte) error { 74 | deliveryMode := amqp.Transient 75 | if t.config.Persistent { 76 | deliveryMode = amqp.Persistent 77 | } 78 | p := &amqp.Publishing{ 79 | DeliveryMode: deliveryMode, 80 | Body: body, 81 | } 82 | return t.PutPublishing(p) 83 | } 84 | 85 | // PutPublishing puts an AMQP message into the queue. Use this if you need more 86 | // granularity in your message parameters. 87 | func (t *Topic) PutPublishing(p *amqp.Publishing) error { 88 | ch, err := t.queue.getChannel() 89 | if err != nil { 90 | return errors.Wrap(err, "error getting channel") 91 | } 92 | defer t.queue.putChannel(ch) 93 | err = ch.Publish(t.config.ExchangeName, t.name, false, false, *p) 94 | if err != nil { 95 | return errors.Wrap(err, "error publishing message") 96 | } 97 | return nil 98 | } 99 | --------------------------------------------------------------------------------