├── History.md ├── Readme.md ├── consumer.go └── consumer_test.go /History.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/segmentio/go-queue/42d4d0f688cb55368bda7c29f833d0ad233ca28f/History.md -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # queue 2 | 3 | The `queue` package wraps NSQ consumers to help reduce the amount of boilerplate 4 | that each program has to implement to get rolling. It does not attempt to abstract NSQ away entirely. 5 | 6 | View the go-nsq [documentation](http://godoc.org/github.com/bitly/go-nsq#Config) for configuration options. 7 | 8 | ## Example 9 | 10 | ```go 11 | c := queue.NewConsumer("events", "ingestion") 12 | 13 | c.Set("nsqd", ":5001") 14 | c.Set("concurrency", 15) 15 | c.Set("max_attempts", 10) 16 | c.Set("max_in_flight", 150) 17 | c.Set("default_requeue_delay", "15s") 18 | 19 | c.Start(nsq.HandlerFunc(func(msg *nsq.Message) error { 20 | // do something 21 | return nil 22 | })) 23 | ``` 24 | 25 | ## Usage 26 | 27 | #### type Consumer 28 | 29 | ```go 30 | type Consumer struct {} 31 | ``` 32 | 33 | Consumer convenience layer. 34 | 35 | #### func NewConsumer 36 | 37 | ```go 38 | func NewConsumer(topic, channel string) *Consumer 39 | ``` 40 | NewConsumer returns a new consumer of `topic` and `channel`. 41 | 42 | #### func (*Consumer) Set 43 | 44 | ```go 45 | func (c *Consumer) Set(option string, value interface{}) 46 | ``` 47 | Set `option` to `value`, any error will be returned in `.Start()`. 48 | 49 | Custom options implemented: 50 | 51 | - `topic` consumer topic 52 | - `channel` consumer channel 53 | - `nsqd` nsqd address 54 | - `nsqds` nsqd addresses 55 | - `nsqlookupd` nsqlookupd address 56 | - `nsqlookupds` nsqlookupd addresses 57 | - `concurrency` concurrent handlers [1] 58 | 59 | #### func (*Consumer) SetLogger 60 | 61 | ```go 62 | func (c *Consumer) SetLogger(log logger, level nsq.LogLevel) 63 | ``` 64 | SetLogger replaces the default logger. 65 | 66 | #### func (*Consumer) Start 67 | 68 | ```go 69 | func (c *Consumer) Start(handler nsq.Handler) error 70 | ``` 71 | Start consumer with `handler`. 72 | 73 | #### func (*Consumer) Stop 74 | 75 | ```go 76 | func (c *Consumer) Stop() error 77 | ``` 78 | Stop and wait. 79 | 80 | # License 81 | 82 | MIT -------------------------------------------------------------------------------- /consumer.go: -------------------------------------------------------------------------------- 1 | // 2 | // The queue package wraps NSQ consumers to help 3 | // reduce the amount of boilerplate that each 4 | // program has to implement to get rolling. It does not attempt 5 | // to abstract NSQ away entirely. 6 | // 7 | package queue 8 | 9 | import ( 10 | "fmt" 11 | "log" 12 | "os" 13 | 14 | "github.com/nsqio/go-nsq" 15 | ) 16 | 17 | // Logger interface. 18 | type logger interface { 19 | Output(int, string) error 20 | } 21 | 22 | // Consumer convenience layer. 23 | type Consumer struct { 24 | client *nsq.Consumer 25 | config *nsq.Config 26 | nsqds []string 27 | nsqlookupds []string 28 | concurrency int 29 | channel string 30 | topic string 31 | level nsq.LogLevel 32 | log logger 33 | err error 34 | } 35 | 36 | // NewConsumer returns a new consumer of `topic` and `channel`. 37 | func NewConsumer(topic, channel string) *Consumer { 38 | return &Consumer{ 39 | log: log.New(os.Stderr, "", log.LstdFlags), 40 | config: nsq.NewConfig(), 41 | level: nsq.LogLevelInfo, 42 | channel: channel, 43 | topic: topic, 44 | concurrency: 1, 45 | } 46 | } 47 | 48 | // SetLogger replaces the default logger. 49 | func (c *Consumer) SetLogger(log logger, level nsq.LogLevel) { 50 | c.level = level 51 | c.log = log 52 | } 53 | 54 | // SetMap applies all `options`. 55 | func (c *Consumer) SetMap(options map[string]interface{}) { 56 | for k, v := range options { 57 | c.Set(k, v) 58 | } 59 | } 60 | 61 | // Set `option` to `value`, any error will be returned in `.Start()`. 62 | // 63 | // Custom options implemented: 64 | // 65 | // - `topic` consumer topic 66 | // - `channel` consumer channel 67 | // - `nsqd` nsqd address 68 | // - `nsqds` nsqd addresses 69 | // - `nsqlookupd` nsqlookupd address 70 | // - `nsqlookupds` nsqlookupd addresses 71 | // - `concurrency` concurrent handlers [1] 72 | // 73 | // 74 | func (c *Consumer) Set(option string, value interface{}) { 75 | switch option { 76 | case "topic": 77 | c.topic = value.(string) 78 | case "channel": 79 | c.channel = value.(string) 80 | case "concurrency": 81 | c.concurrency = value.(int) 82 | case "nsqd": 83 | c.nsqds = []string{value.(string)} 84 | case "nsqlookupd": 85 | c.nsqlookupds = []string{value.(string)} 86 | case "nsqds": 87 | s, err := strings(value) 88 | if err != nil { 89 | c.err = fmt.Errorf("%q: %v", option, err) 90 | return 91 | } 92 | c.nsqds = s 93 | case "nsqlookupds": 94 | s, err := strings(value) 95 | if err != nil { 96 | c.err = fmt.Errorf("%q: %v", option, err) 97 | return 98 | } 99 | c.nsqlookupds = s 100 | default: 101 | err := c.config.Set(option, value) 102 | if err != nil { 103 | c.err = err 104 | } 105 | } 106 | } 107 | 108 | // Start consumer with `handler`. 109 | func (c *Consumer) Start(handler nsq.Handler) error { 110 | if c.err != nil { 111 | return c.err 112 | } 113 | 114 | client, err := nsq.NewConsumer(c.topic, c.channel, c.config) 115 | if err != nil { 116 | return err 117 | } 118 | c.client = client 119 | 120 | client.SetLogger(c.log, c.level) 121 | client.AddConcurrentHandlers(handler, c.concurrency) 122 | 123 | return c.connect() 124 | } 125 | 126 | // Stop and wait. 127 | func (c *Consumer) Stop() error { 128 | c.client.Stop() 129 | <-c.client.StopChan 130 | return nil 131 | } 132 | 133 | // Connect to the configure nsqd(s) or nsqlookupd(s). 134 | func (c *Consumer) connect() error { 135 | if len(c.nsqds) == 0 && len(c.nsqlookupds) == 0 { 136 | return fmt.Errorf(`at least one "nsqd" or "nsqlookupd" address must be configured`) 137 | } 138 | 139 | if len(c.nsqds) > 0 { 140 | err := c.client.ConnectToNSQDs(c.nsqds) 141 | if err != nil { 142 | return err 143 | } 144 | } 145 | 146 | if len(c.nsqlookupds) > 0 { 147 | err := c.client.ConnectToNSQLookupds(c.nsqlookupds) 148 | if err != nil { 149 | return err 150 | } 151 | } 152 | 153 | return nil 154 | } 155 | 156 | // Returns a slice of strings or error. 157 | // 158 | // Primarily to allow for []interface{} from parsing configuration files. 159 | func strings(v interface{}) ([]string, error) { 160 | switch v.(type) { 161 | case []string: 162 | return v.([]string), nil 163 | case []interface{}: 164 | var ret []string 165 | for _, e := range v.([]interface{}) { 166 | s, ok := e.(string) 167 | 168 | if !ok { 169 | return nil, fmt.Errorf("string expected, got %v", e) 170 | } 171 | 172 | ret = append(ret, s) 173 | } 174 | return ret, nil 175 | default: 176 | return nil, fmt.Errorf("strings expected") 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /consumer_test.go: -------------------------------------------------------------------------------- 1 | package queue_test 2 | 3 | import "github.com/segmentio/go-queue" 4 | import "github.com/bmizerany/assert" 5 | import "github.com/nsqio/go-nsq" 6 | import "testing" 7 | import "bytes" 8 | import "log" 9 | 10 | func check(err error) { 11 | if err != nil { 12 | panic(err) 13 | } 14 | } 15 | 16 | func TestConsumer_Start_ConfigErrors(t *testing.T) { 17 | c := queue.NewConsumer("events", "ingestion") 18 | 19 | c.Set("nsqd", ":5001") 20 | c.Set("concurrency", 5) 21 | c.Set("max_attempts", 10) 22 | c.Set("max_in_flight", "oh noes") 23 | c.Set("default_requeue_delay", "15s") 24 | 25 | err := c.Start(nil) 26 | assert.Equal(t, `failed to coerce option max_in_flight (oh noes) - strconv.ParseInt: parsing "oh noes": invalid syntax`, err.Error()) 27 | } 28 | 29 | func TestConsumer_Start_ConfigMissingDaemon(t *testing.T) { 30 | c := queue.NewConsumer("events", "ingestion") 31 | 32 | c.Set("concurrency", 5) 33 | c.Set("max_attempts", 10) 34 | 35 | err := c.Start(nil) 36 | assert.Equal(t, `at least one "nsqd" or "nsqlookupd" address must be configured`, err.Error()) 37 | } 38 | 39 | func TestConsumer_Start_Handler(t *testing.T) { 40 | done := make(chan bool) 41 | b := new(bytes.Buffer) 42 | l := log.New(b, "", 0) 43 | 44 | c := queue.NewConsumer("events", "ingestion") 45 | c.SetLogger(l, nsq.LogLevelDebug) 46 | 47 | c.Set("nsqd", ":5001") 48 | c.Set("nsqds", []interface{}{":5001"}) 49 | c.Set("concurrency", 5) 50 | c.Set("max_attempts", 10) 51 | c.Set("max_in_flight", 150) 52 | c.Set("default_requeue_delay", "15s") 53 | 54 | err := c.Start(nsq.HandlerFunc(func(msg *nsq.Message) error { 55 | done <- true 56 | return nil 57 | })) 58 | 59 | assert.Equal(t, nil, err) 60 | 61 | go func() { 62 | p, err := nsq.NewProducer(":5001", nsq.NewConfig()) 63 | check(err) 64 | p.Publish("events", []byte("hello")) 65 | }() 66 | 67 | <-done 68 | assert.Equal(t, nil, c.Stop()) 69 | } 70 | --------------------------------------------------------------------------------