├── .gitignore ├── History.md ├── Readme.md ├── broadcast ├── broadcast.go ├── broadcast_test.go ├── conn.go └── mocks │ ├── Conn.go │ └── RedisConn.go ├── circle.yml ├── docker-compose.yml ├── list ├── list.go └── list_test.go ├── main.go ├── pubsub ├── pubsub.go └── pubsub_test.go ├── ratelimit ├── ratelimit.go └── ratelimit_test.go ├── template ├── template.go └── template_test.go └── vendor └── vendor.json /.gitignore: -------------------------------------------------------------------------------- 1 | vendor/*/ 2 | nsq_to_redis 3 | -------------------------------------------------------------------------------- /History.md: -------------------------------------------------------------------------------- 1 | 2 | 2.1.0 / 2018-01-08 3 | ================== 4 | 5 | * [SCH-175] Update to buffer and pipeline commands (#16) 6 | * Fix CI (#18) 7 | * Fix --list-size (#15) 8 | 9 | v2.0.1 / 2017-04-26 10 | =================== 11 | 12 | * Fix regression introduced in `v2.0.0` where messages were incorrectly published to Redis. It is not recommended you use v2.0.0! 13 | 14 | v2.0.0 / 2017-04-24 15 | =================== 16 | 17 | * Speed up JSON parsing by using `json.RawMessage` instead of `map[string]interface{}` 18 | * Speed up templating by writing custom templating on top of [`gjson`](https://github.com/tidwall/gjson) instead of using [`go-interpolate`](https://github.com/segmentio/go-interpolate). 19 | * Internal: Migrate to govendor. 20 | 21 | 22 | v1.5.0 / 2016-01-29 23 | ================== 24 | 25 | * add ratelimit 26 | * list: typo 27 | * add Message 28 | 29 | 30 | v1.4.0 / 2016-01-25 31 | ================== 32 | 33 | * Flush once for each message 34 | 35 | 36 | v1.3.0 / 2016-01-22 37 | ================== 38 | 39 | * add --max-idle 40 | 41 | v1.2.0 / 2016-01-22 42 | ================== 43 | 44 | * add optional metrics (see `--statsd` & `--statsd-prefix`) 45 | * fixing nsqlookupd detection 46 | 47 | v1.1.0 / 2015-08-25 48 | ================== 49 | 50 | * add nsqds argument that overrides nsqlookupd 51 | 52 | 53 | v1.0.2 / 2015-05-06 54 | =================== 55 | 56 | * use a different redis lib, seems less buggy 57 | 58 | v1.0.0 / 2015-05-06 59 | =================== 60 | 61 | * add --idle-timeout 62 | 63 | v0.2.0 / 2015-02-19 64 | =================== 65 | 66 | * add Broadcast to deliver to delegates 67 | 68 | v0.1.0 / 2015-02-19 69 | =================== 70 | 71 | * add capped list support 72 | * add --list-size 73 | * add --list 74 | * rename Relay to PubSub 75 | 76 | v0.0.2 / 2015-02-18 77 | =================== 78 | 79 | - update go-interpolate for trailing lit bugfix 80 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | 2 | # nsq_to_redis 3 | 4 | > **Note** 5 | > Segment has paused maintenance on this project, but may return it to an active status in the future. Issues and pull requests from external contributors are not being considered, although internal contributions may appear from time to time. The project remains available under its open source license for anyone to use. 6 | 7 | Publish NSQ messages to Redis Publish/Subscribe: 8 | 9 | ``` 10 | $ nsq_to_redis --topic events --publish "projects:{projectId}" 11 | ``` 12 | 13 | Write to capped lists: 14 | 15 | ``` 16 | $ nsq_to_redis --topic events --list "events:{projectId}" --list-size 100 17 | ``` 18 | 19 | # License 20 | 21 | MIT 22 | -------------------------------------------------------------------------------- /broadcast/broadcast.go: -------------------------------------------------------------------------------- 1 | package broadcast 2 | 3 | import ( 4 | "encoding/json" 5 | "sync" 6 | "time" 7 | 8 | "github.com/bitly/go-nsq" 9 | "github.com/garyburd/redigo/redis" 10 | "github.com/segmentio/go-log" 11 | "github.com/segmentio/go-stats" 12 | "github.com/segmentio/nsq_to_redis/ratelimit" 13 | "github.com/segmentio/statsdclient" 14 | "github.com/tidwall/gjson" 15 | ) 16 | 17 | type RedisPool interface { 18 | Get() redis.Conn 19 | } 20 | 21 | // Handler is a message handler. 22 | type Handler interface { 23 | Handle(Conn, *Message) error 24 | } 25 | 26 | // Message is a parsed message. 27 | type Message struct { 28 | ID nsq.MessageID 29 | JSON json.RawMessage 30 | } 31 | 32 | // Options for broadcast. 33 | type Options struct { 34 | Redis RedisPool 35 | Metrics *statsd.Client 36 | Ratelimiter *ratelimit.Ratelimiter 37 | RatelimitKey string 38 | Log *log.Logger 39 | FlushInterval time.Duration 40 | } 41 | 42 | // Broadcast consumer distributes messages to N handlers. 43 | type Broadcast struct { 44 | Done chan struct{} 45 | *Options 46 | 47 | handlers []Handler 48 | stats *stats.Stats 49 | 50 | // Single connection, channel and mutex are used when applying a flush interval 51 | conn Conn 52 | mutex sync.Mutex 53 | } 54 | 55 | // New broadcast consumer. 56 | func New(o *Options) *Broadcast { 57 | stats := stats.New() 58 | go stats.TickEvery(10 * time.Second) 59 | 60 | broadcast := Broadcast{ 61 | stats: stats, 62 | Options: o, 63 | Done: make(chan struct{}, 1), 64 | } 65 | 66 | if o.FlushInterval < 0 { 67 | panic("FlushInterval must not be a negative duration") 68 | } else if o.FlushInterval > 0 { 69 | conn := NewConn(o.Redis.Get()) 70 | broadcast.conn = conn 71 | broadcast.flushOnInterval(conn) 72 | } 73 | 74 | return &broadcast 75 | } 76 | 77 | // Add handler. 78 | func (b *Broadcast) Add(h Handler) { 79 | b.handlers = append(b.handlers, h) 80 | } 81 | 82 | // HandleMessage parses distributes messages to each delegate. 83 | func (b *Broadcast) HandleMessage(msg *nsq.Message) error { 84 | start := time.Now() 85 | 86 | // parse 87 | m := new(Message) 88 | m.ID = msg.ID 89 | err := json.Unmarshal(msg.Body, &m.JSON) 90 | if err != nil { 91 | b.Log.Error("error parsing json: %s", err) 92 | return nil 93 | } 94 | 95 | // ratelimit 96 | if b.rateExceeded(m) { 97 | b.stats.Incr("ratelimit.discard") 98 | b.Metrics.Incr("counts.ratelimit.discard") 99 | b.Log.Debug("ratelimit exceeded, discarding message") 100 | return nil 101 | } 102 | 103 | conn, done := b.getConn() 104 | defer done() 105 | 106 | for _, h := range b.handlers { 107 | err := h.Handle(conn, m) 108 | if err != nil { 109 | return err 110 | } 111 | } 112 | 113 | if b.FlushInterval == 0 { 114 | if err := b.flush(conn); err != nil { 115 | return err 116 | } 117 | } 118 | 119 | b.Metrics.Duration("timers.broadcast", time.Since(start)) 120 | return nil 121 | } 122 | 123 | // Flushes all messages, then sends on the Done channel. 124 | func (b *Broadcast) Stop() { 125 | if b.FlushInterval > 0 { 126 | conn, done := b.getConn() 127 | defer done() 128 | b.flush(conn) 129 | } 130 | 131 | b.Done <- struct{}{} 132 | } 133 | 134 | func (b *Broadcast) getConn() (conn Conn, done func()) { 135 | if b.FlushInterval == 0 { 136 | db := b.Redis.Get() 137 | conn = NewConn(db) 138 | done = func() { 139 | db.Close() 140 | } 141 | } else { 142 | b.mutex.Lock() 143 | conn = b.conn 144 | done = func() { 145 | b.mutex.Unlock() 146 | } 147 | } 148 | 149 | return conn, done 150 | } 151 | 152 | func (b *Broadcast) flushOnInterval(conn Conn) { 153 | go func() { 154 | for range time.Tick(b.FlushInterval) { 155 | b.mutex.Lock() 156 | b.flush(conn) 157 | b.mutex.Unlock() 158 | } 159 | }() 160 | } 161 | 162 | func (b *Broadcast) flush(conn Conn) error { 163 | err := conn.Flush() 164 | if err != nil { 165 | b.Metrics.Incr("errors.flush") 166 | b.Log.Error("flush: %s", err) 167 | return err 168 | } 169 | 170 | return nil 171 | } 172 | 173 | // rateExceeded returns true if the given message 174 | // rate was exceeded. The method returns false 175 | // if ratelimit was not configured or exceeded. 176 | func (b *Broadcast) rateExceeded(msg *Message) bool { 177 | if b.Ratelimiter != nil { 178 | k := gjson.Get(string(msg.JSON), b.RatelimitKey).String() 179 | return b.Ratelimiter.Exceeded(k) 180 | } 181 | 182 | return false 183 | } 184 | 185 | // NewMessage returns a Message or an error if unable to do so. 186 | // Used primarily by tests. 187 | func NewMessage(id, contents string) (*Message, error) { 188 | nsqId := [nsq.MsgIDLength]byte{} 189 | copy(nsqId[:], id[:nsq.MsgIDLength]) 190 | 191 | m := new(Message) 192 | m.ID = nsqId 193 | err := json.Unmarshal([]byte(contents), &m.JSON) 194 | if err != nil { 195 | return nil, err 196 | } 197 | 198 | return m, nil 199 | } 200 | -------------------------------------------------------------------------------- /broadcast/broadcast_test.go: -------------------------------------------------------------------------------- 1 | package broadcast 2 | 3 | import ( 4 | "io/ioutil" 5 | "sync/atomic" 6 | "testing" 7 | "time" 8 | 9 | nsq "github.com/bitly/go-nsq" 10 | "github.com/bmizerany/assert" 11 | "github.com/garyburd/redigo/redis" 12 | "github.com/segmentio/go-log" 13 | "github.com/segmentio/nsq_to_redis/broadcast/mocks" 14 | "github.com/segmentio/nsq_to_redis/ratelimit" 15 | statsd "github.com/segmentio/statsdclient" 16 | "github.com/stretchr/testify/mock" 17 | ) 18 | 19 | func Benchmark1handler(b *testing.B) { benchmarkBroadcast(1, b) } 20 | func Benchmark2Handlers(b *testing.B) { benchmarkBroadcast(2, b) } 21 | 22 | func benchmarkBroadcast(n int, b *testing.B) { 23 | pool := &mockRedisPool{} 24 | pool.On("Get").Return(mocks.NewNoOpRedisConn()) 25 | 26 | broadcast := New(&Options{ 27 | Redis: pool, 28 | Metrics: statsd.NewClient(ioutil.Discard), 29 | Log: log.Log, 30 | Ratelimiter: ratelimit.New(10, 500), 31 | RatelimitKey: "projectId", 32 | }) 33 | 34 | expectedMessage, err := NewMessage("nsq__message__id", `{"projectId":"gy2d"}`) 35 | if err != nil { 36 | b.Error(err) 37 | } 38 | 39 | for i := 0; i < n; i++ { 40 | h := &mockHandler{} 41 | h.On("Handle", mock.Anything, expectedMessage).Return(nil) 42 | broadcast.Add(h) 43 | } 44 | 45 | nsqMsg := nsq.NewMessage(newNSQMessageId("nsq__message__id"), []byte(`{"projectId":"gy2d"}`)) 46 | 47 | for n := 0; n < b.N; n++ { 48 | broadcast.HandleMessage(nsqMsg) 49 | } 50 | } 51 | 52 | func TestBroadcast(t *testing.T) { 53 | pool := &redis.Pool{ 54 | IdleTimeout: 1 * time.Minute, 55 | MaxIdle: 15, 56 | MaxActive: 100, 57 | Dial: func() (redis.Conn, error) { 58 | return redis.Dial("tcp", ":6379") 59 | }, 60 | } 61 | 62 | b := New(&Options{ 63 | Redis: pool, 64 | Metrics: statsd.NewClient(ioutil.Discard), 65 | Log: log.Log, 66 | Ratelimiter: ratelimit.New(10, 500), 67 | RatelimitKey: "projectId", 68 | }) 69 | 70 | expectedMessage, err := NewMessage("nsq__message__id", `{"projectId":"gy2d"}`) 71 | assert.Equal(t, nil, err) 72 | 73 | h1 := &mockHandler{} 74 | h1.On("Handle", mock.Anything, expectedMessage).Times(1).Return(nil) 75 | h2 := &mockHandler{} 76 | h2.On("Handle", mock.Anything, expectedMessage).Times(1).Return(nil) 77 | 78 | b.Add(h1) 79 | b.Add(h2) 80 | 81 | nsqMsg := nsq.NewMessage(newNSQMessageId("nsq__message__id"), []byte(`{"projectId":"gy2d"}`)) 82 | b.HandleMessage(nsqMsg) 83 | 84 | h1.AssertExpectations(t) 85 | h2.AssertExpectations(t) 86 | } 87 | 88 | func TestBroadcastInvalidFlushInterval(t *testing.T) { 89 | assert.Panic(t, "FlushInterval must not be a negative duration", func() { 90 | New(&Options{FlushInterval: -1 * time.Hour}) 91 | }) 92 | } 93 | 94 | func TestBroadcastWithoutFlushInterval(t *testing.T) { 95 | pool := getMockPool() 96 | broadcast := New(&Options{ 97 | Redis: pool, 98 | Metrics: statsd.NewClient(ioutil.Discard), 99 | Log: log.Log, 100 | }) 101 | 102 | sendMessages(broadcast) 103 | 104 | mockConn := pool.Get().(*mocks.NoOpRedisConn) 105 | assert.Equal(t, 2, int(atomic.LoadUint64(&mockConn.Flushes))) 106 | } 107 | 108 | func TestBroadcastWithFlushInterval(t *testing.T) { 109 | pool := getMockPool() 110 | broadcast := New(&Options{ 111 | Redis: pool, 112 | Metrics: statsd.NewClient(ioutil.Discard), 113 | Log: log.Log, 114 | FlushInterval: 2 * time.Millisecond, 115 | }) 116 | 117 | sendMessages(broadcast) 118 | 119 | mockConn := pool.Get().(*mocks.NoOpRedisConn) 120 | assert.Equal(t, 0, int(atomic.LoadUint64(&mockConn.Flushes))) 121 | <-time.After(3 * time.Millisecond) 122 | assert.Equal(t, 1, int(atomic.LoadUint64(&mockConn.Flushes))) 123 | } 124 | 125 | func TestBroadcastWithFlushIntervalStop(t *testing.T) { 126 | pool := getMockPool() 127 | broadcast := New(&Options{ 128 | Redis: pool, 129 | Metrics: statsd.NewClient(ioutil.Discard), 130 | Log: log.Log, 131 | FlushInterval: 10 * time.Second, 132 | }) 133 | 134 | sendMessages(broadcast) 135 | 136 | mockConn := pool.Get().(*mocks.NoOpRedisConn) 137 | assert.Equal(t, 0, int(atomic.LoadUint64(&mockConn.Flushes))) 138 | broadcast.Stop() 139 | <-broadcast.Done 140 | assert.Equal(t, 1, int(atomic.LoadUint64(&mockConn.Flushes))) 141 | } 142 | 143 | func getMockPool() RedisPool { 144 | pool := &mockRedisPool{} 145 | pool.On("Get").Return(mocks.NewNoOpRedisConn()) 146 | return pool 147 | } 148 | 149 | func sendMessages(broadcast *Broadcast) { 150 | nsqMsg := nsq.NewMessage(newNSQMessageId("nsq__message__id"), []byte(`{"projectId":"gy2d"}`)) 151 | broadcast.HandleMessage(nsqMsg) 152 | broadcast.HandleMessage(nsqMsg) 153 | } 154 | 155 | func newNSQMessageId(id string) nsq.MessageID { 156 | nsqId := [nsq.MsgIDLength]byte{} 157 | copy(nsqId[:], id[:nsq.MsgIDLength]) 158 | return nsqId 159 | } 160 | 161 | type mockHandler struct { 162 | mock.Mock 163 | } 164 | 165 | // Handle provides a mock function with given fields: _a0, _a1 166 | func (_m *mockHandler) Handle(_a0 Conn, _a1 *Message) error { 167 | ret := _m.Called(_a0, _a1) 168 | 169 | var r0 error 170 | if rf, ok := ret.Get(0).(func(Conn, *Message) error); ok { 171 | r0 = rf(_a0, _a1) 172 | } else { 173 | r0 = ret.Error(0) 174 | } 175 | 176 | return r0 177 | } 178 | 179 | type mockRedisPool struct { 180 | mock.Mock 181 | } 182 | 183 | // Get provides a mock function with given fields: 184 | func (_m *mockRedisPool) Get() redis.Conn { 185 | ret := _m.Called() 186 | 187 | var r0 redis.Conn 188 | if rf, ok := ret.Get(0).(func() redis.Conn); ok { 189 | r0 = rf() 190 | } else { 191 | r0 = ret.Get(0).(redis.Conn) 192 | } 193 | 194 | return r0 195 | } 196 | -------------------------------------------------------------------------------- /broadcast/conn.go: -------------------------------------------------------------------------------- 1 | package broadcast 2 | 3 | import "github.com/garyburd/redigo/redis" 4 | 5 | type Conn interface { 6 | Send(cmd string, args ...interface{}) error 7 | Flush() error 8 | } 9 | 10 | // Conn is a single threaded 11 | // buffer for redis commands. 12 | type conn struct { 13 | conn redis.Conn 14 | pending int 15 | } 16 | 17 | // NewConn returns a new Conn. 18 | func NewConn(c redis.Conn) Conn { 19 | return &conn{conn: c} 20 | } 21 | 22 | // Send sends the given command. 23 | func (c *conn) Send(cmd string, args ...interface{}) error { 24 | err := c.conn.Send(cmd, args...) 25 | c.pending++ 26 | return err 27 | } 28 | 29 | // Flush will flush the redis buffers 30 | // and receive all responses from redis. 31 | func (c *conn) Flush() error { 32 | err := c.conn.Flush() 33 | if err != nil { 34 | return err 35 | } 36 | 37 | for i := 0; i < c.pending; i++ { 38 | _, err := c.conn.Receive() 39 | if err != nil { 40 | return err 41 | } 42 | } 43 | 44 | c.pending = 0 45 | 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /broadcast/mocks/Conn.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import "github.com/stretchr/testify/mock" 4 | 5 | type Conn struct { 6 | mock.Mock 7 | } 8 | 9 | // Send provides a mock function with given fields: cmd, args 10 | func (_m *Conn) Send(cmd string, args ...interface{}) error { 11 | ret := _m.Called(cmd, args) 12 | 13 | var r0 error 14 | if rf, ok := ret.Get(0).(func(string, ...interface{}) error); ok { 15 | r0 = rf(cmd, args...) 16 | } else { 17 | r0 = ret.Error(0) 18 | } 19 | 20 | return r0 21 | } 22 | 23 | // Flush provides a mock function with given fields: 24 | func (_m *Conn) Flush() error { 25 | ret := _m.Called() 26 | 27 | var r0 error 28 | if rf, ok := ret.Get(0).(func() error); ok { 29 | r0 = rf() 30 | } else { 31 | r0 = ret.Error(0) 32 | } 33 | 34 | return r0 35 | } 36 | -------------------------------------------------------------------------------- /broadcast/mocks/RedisConn.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "sync/atomic" 5 | 6 | "github.com/garyburd/redigo/redis" 7 | ) 8 | 9 | func NewNoOpRedisConn() redis.Conn { 10 | return &NoOpRedisConn{} 11 | } 12 | 13 | type NoOpRedisConn struct { 14 | Flushes uint64 15 | } 16 | 17 | func (c *NoOpRedisConn) Close() error { 18 | return nil 19 | } 20 | 21 | func (c *NoOpRedisConn) Err() error { 22 | return nil 23 | } 24 | 25 | func (c *NoOpRedisConn) Do(commandName string, args ...interface{}) (reply interface{}, err error) { 26 | return nil, nil 27 | } 28 | 29 | func (c *NoOpRedisConn) Send(commandName string, args ...interface{}) error { 30 | return nil 31 | } 32 | 33 | func (c *NoOpRedisConn) Flush() error { 34 | atomic.AddUint64(&c.Flushes, 1) 35 | return nil 36 | } 37 | 38 | func (c *NoOpRedisConn) Receive() (reply interface{}, err error) { 39 | return nil, nil 40 | } 41 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | 2 | machine: 3 | services: 4 | - docker 5 | 6 | dependencies: 7 | override: 8 | - docker pull segment/golang:latest 9 | 10 | test: 11 | override: 12 | - > 13 | docker run 14 | $(env | grep -E '^CIRCLE_|^DOCKER_|^CIRCLECI=|^CI=' | sed 's/^/--env /g' | tr "\\n" " ") 15 | --rm 16 | --tty 17 | --interactive 18 | --name go 19 | --net host 20 | --volume /var/run/docker.sock:/run/docker.sock 21 | --volume ${GOPATH%%:*}/src:/go/src 22 | --volume ${PWD}:/go/src/github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME} 23 | --workdir /go/src/github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME} 24 | segment/golang:latest 25 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | redis: 3 | image: redis 4 | ports: 5 | - "6379:6379" 6 | -------------------------------------------------------------------------------- /list/list.go: -------------------------------------------------------------------------------- 1 | package list 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/segmentio/go-log" 7 | "github.com/segmentio/go-stats" 8 | "github.com/segmentio/nsq_to_redis/broadcast" 9 | "github.com/segmentio/nsq_to_redis/template" 10 | "github.com/segmentio/statsdclient" 11 | ) 12 | 13 | // Options for List. 14 | type Options struct { 15 | Format string // Redis list key format 16 | Metrics *statsd.Client // Metrics 17 | Log *log.Logger // Logger 18 | Size int64 // List size 19 | } 20 | 21 | // List writes messages to capped lists. 22 | type List struct { 23 | template *template.T 24 | stats *stats.Stats 25 | *Options 26 | } 27 | 28 | // New list with options. 29 | func New(options *Options) (*List, error) { 30 | r := &List{ 31 | Options: options, 32 | stats: stats.New(), 33 | } 34 | 35 | tmpl, err := template.New(r.Format) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | r.template = tmpl 41 | go r.stats.TickEvery(10 * time.Second) 42 | 43 | return r, nil 44 | } 45 | 46 | // HandleMessage expects parsed json messages from NSQ, 47 | // applies them against the key template to produce a 48 | // key name, and writes to the list. 49 | func (l *List) Handle(c broadcast.Conn, msg *broadcast.Message) error { 50 | start := time.Now() 51 | 52 | key, err := l.template.Eval(string(msg.JSON)) 53 | if err != nil { 54 | l.Log.Error("evaluating template: %s", err) 55 | return nil 56 | } 57 | 58 | l.Log.Info("pushing %s to %s", msg.ID, key) 59 | l.Log.Debug("contents %s %s", msg.ID, msg.JSON) 60 | 61 | err = c.Send("LPUSH", key, []byte(msg.JSON)) 62 | if err != nil { 63 | l.Log.Error("lpush: %s", err) 64 | } 65 | 66 | err = c.Send("LTRIM", key, 0, l.Size-1) 67 | if err != nil { 68 | l.Log.Error("ltrim: %s", err) 69 | } 70 | 71 | l.Metrics.Duration("timers.pushed", time.Since(start)) 72 | l.Metrics.Incr("counts.pushed") 73 | l.stats.Incr("pushed") 74 | return nil 75 | } 76 | -------------------------------------------------------------------------------- /list/list_test.go: -------------------------------------------------------------------------------- 1 | package list 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | 7 | "github.com/bmizerany/assert" 8 | "github.com/garyburd/redigo/redis" 9 | goredis "github.com/go-redis/redis" 10 | "github.com/segmentio/go-log" 11 | "github.com/segmentio/nsq_to_redis/broadcast" 12 | "github.com/segmentio/nsq_to_redis/broadcast/mocks" 13 | statsd "github.com/segmentio/statsdclient" 14 | ) 15 | 16 | func BenchmarkList(b *testing.B) { 17 | l := log.Log.New("list_benchmark") 18 | l.SetLevel(log.ERROR) 19 | 20 | list, err := New(&Options{ 21 | Format: "stream_benchmark:persist:{projectId}:ingress", 22 | Log: l, 23 | Metrics: statsd.NewClient(ioutil.Discard), 24 | Size: 50, 25 | }) 26 | if err != nil { 27 | b.Error(err) 28 | } 29 | 30 | conn := &mocks.Conn{} 31 | conn.On("Send", "LPUSH", []interface{}{ 32 | "stream_benchmark:persist:gy2d:ingress", 33 | []byte(`{"projectId":"gy2d"}`), 34 | }).Return(nil) 35 | conn.On("Send", "LTRIM", []interface{}{ 36 | "stream_benchmark:persist:gy2d:ingress", 37 | 0, 38 | int64(49), 39 | }).Return(nil) 40 | 41 | msg, err := broadcast.NewMessage("nsq_message_id_1", `{"projectId":"gy2d"}`) 42 | 43 | for i := 0; i < b.N; i++ { 44 | list.Handle(conn, msg) 45 | } 46 | } 47 | 48 | func TestList(t *testing.T) { 49 | list, err := New(&Options{ 50 | Format: "stream:persist:{projectId}:ingress", 51 | Log: log.Log.New("list_test"), 52 | Metrics: statsd.NewClient(ioutil.Discard), 53 | Size: 50, 54 | }) 55 | assert.Equal(t, nil, err) 56 | 57 | cPublish, err := redis.Dial("tcp", ":6379") 58 | assert.Equal(t, nil, err) 59 | defer cPublish.Close() 60 | 61 | conn := broadcast.NewConn(cPublish) 62 | broadcastMessage, err := broadcast.NewMessage("nsq_message_id_1", `{"projectId":"gy2d"}`) 63 | 64 | err = list.Handle(conn, broadcastMessage) 65 | assert.Equal(t, nil, err) 66 | err = conn.Flush() 67 | assert.Equal(t, nil, err) 68 | 69 | client := goredis.NewClient(&goredis.Options{ 70 | Addr: "localhost:6379", 71 | }) 72 | defer client.LTrim("stream:persist:gy2d:ingress", 0, 0) 73 | 74 | vals, err := client.LRange("stream:persist:gy2d:ingress", 0, -1).Result() 75 | assert.Equal(t, nil, err) 76 | assert.Equal(t, vals, []string{`{"projectId":"gy2d"}`}) 77 | } 78 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "strconv" 6 | "time" 7 | 8 | "github.com/bitly/go-nsq" 9 | "github.com/garyburd/redigo/redis" 10 | "github.com/segmentio/go-log" 11 | "github.com/segmentio/nsq_to_redis/broadcast" 12 | "github.com/segmentio/nsq_to_redis/list" 13 | "github.com/segmentio/nsq_to_redis/pubsub" 14 | "github.com/segmentio/nsq_to_redis/ratelimit" 15 | "github.com/segmentio/statsdclient" 16 | "github.com/tj/docopt" 17 | "github.com/tj/go-gracefully" 18 | ) 19 | 20 | var version = "2.1.0" 21 | 22 | const usage = ` 23 | Usage: 24 | nsq_to_redis 25 | --topic name [--channel name] 26 | [--max-attempts n] [--max-in-flight n] 27 | [--statsd addr] 28 | [--statsd-prefix prefix] 29 | [--lookupd-http-address addr...] 30 | [--nsqd-tcp-address addr...] 31 | [--redis-address addr] 32 | [--flush-interval t] 33 | [--max-idle n] 34 | [--idle-timeout t] 35 | [--list name] [--list-size n] 36 | [--publish name] 37 | [--level name] 38 | [--ratelimit-key key] 39 | [--ratelimit-max-rate n] 40 | [--ratelimit-max-keys n] 41 | 42 | nsq_to_redis -h | --help 43 | nsq_to_redis --version 44 | 45 | Options: 46 | --lookupd-http-address addr nsqlookupd addresses [default: :4161] 47 | --nsqd-tcp-address addr nsqd tcp addresses 48 | --redis-address addr redis address [default: :6379] 49 | --max-attempts n nsq max message attempts [default: 5] 50 | --max-in-flight n nsq messages in-flight [default: 250] 51 | --flush-interval t time to buffer redis commands before flushing [default: 0s] 52 | --max-idle n redis max idle connections [default: 15] 53 | --idle-timeout t idle connection timeout [default: 1m] 54 | --list-size n   redis list size [default: 100] 55 | --list name   redis list template 56 | --publish name   redis channel template 57 | --topic name   nsq consumer topic name 58 | --channel name   nsq consumer channel name [default: nsq_to_redis] 59 | --level name log level [default: info] 60 | --statsd addr tcp address [default: ] 61 | --statsd-prefix prefix prefix for statsd [default: nsq_to_redis.] 62 | --ratelimit-key key a key to use for ratelimits, for example "api_key" [default: ] 63 | --ratelimit-max-rate n max writes for each key per second, N <= 0 will not limit [default: 0] 64 | --ratelimit-max-keys n max keys to keep in memory (lru cache) [default: 500] 65 | -h, --help output help information 66 | -v, --version output version 67 | 68 | ` 69 | 70 | func main() { 71 | args, err := docopt.Parse(usage, nil, true, version, false) 72 | if err != nil { 73 | log.Fatalf("error parsing arguments: %s", err) 74 | } 75 | 76 | lookupds := args["--lookupd-http-address"].([]string) 77 | channel := args["--channel"].(string) 78 | topic := args["--topic"].(string) 79 | 80 | var metrics *statsd.Client 81 | if addr := args["--statsd"].(string); addr != "" { 82 | metrics, err = statsd.Dial(addr) 83 | } else { 84 | metrics = statsd.NewClient(ioutil.Discard) 85 | } 86 | metrics.Prefix(args["--statsd-prefix"].(string)) 87 | 88 | idleTimeout, err := time.ParseDuration(args["--idle-timeout"].(string)) 89 | if err != nil { 90 | log.Fatalf("error parsing idle timeout: %s", err) 91 | } 92 | 93 | flushInterval, err := time.ParseDuration(args["--flush-interval"].(string)) 94 | if err != nil { 95 | log.Fatalf("error parsing flush-interval: %s", err) 96 | } 97 | if flushInterval < 0 { 98 | log.Fatalf("flush-interval must not be a negative value") 99 | } 100 | 101 | maxIdle, err := strconv.Atoi(args["--max-idle"].(string)) 102 | if err != nil { 103 | log.Fatalf("error parsing max-idle: %s", err) 104 | } 105 | if maxIdle > 100 { 106 | log.Fatalf("max-idle must be below 100") 107 | } 108 | 109 | pool := &redis.Pool{ 110 | IdleTimeout: idleTimeout, 111 | MaxIdle: maxIdle, 112 | MaxActive: 100, 113 | Dial: dial(args["--redis-address"].(string)), 114 | TestOnBorrow: ping, 115 | } 116 | 117 | broadcast := broadcast.New(&broadcast.Options{ 118 | Redis: pool, 119 | Metrics: metrics, 120 | Log: log.Log, 121 | Ratelimiter: ratelimiter(args), 122 | RatelimitKey: args["--ratelimit-key"].(string), 123 | FlushInterval: flushInterval, 124 | }) 125 | config := config(args) 126 | 127 | consumer, err := nsq.NewConsumer(topic, channel, config) 128 | if err != nil { 129 | log.Fatalf("error starting consumer: %s", err) 130 | } 131 | 132 | log.SetLevelString(args["--level"].(string)) 133 | 134 | // Pub/Sub support. 135 | if format, ok := args["--publish"].(string); ok { 136 | log.Info("publishing to %q", format) 137 | pubsub, err := pubsub.New(&pubsub.Options{ 138 | Format: format, 139 | Log: log.Log, 140 | Metrics: metrics, 141 | }) 142 | 143 | if err != nil { 144 | log.Fatalf("error starting pubsub: %s", err) 145 | } 146 | 147 | broadcast.Add(pubsub) 148 | } 149 | 150 | // Capped list support. 151 | if format, ok := args["--list"].(string); ok { 152 | size, err := strconv.Atoi(args["--list-size"].(string)) 153 | if err != nil { 154 | log.Fatalf("error parsing --list-size: %s", err) 155 | } 156 | 157 | log.Info("listing to %q (size=%d)", format, size) 158 | list, err := list.New(&list.Options{ 159 | Format: format, 160 | Log: log.Log, 161 | Metrics: metrics, 162 | Size: int64(size), 163 | }) 164 | 165 | if err != nil { 166 | log.Fatalf("error starting list: %s", err) 167 | } 168 | 169 | broadcast.Add(list) 170 | } 171 | 172 | consumer.AddConcurrentHandlers(broadcast, maxIdle) 173 | nsqds := args["--nsqd-tcp-address"].([]string) 174 | 175 | if len(nsqds) > 0 { 176 | err = consumer.ConnectToNSQDs(nsqds) 177 | } else { 178 | err = consumer.ConnectToNSQLookupds(lookupds) 179 | } 180 | if err != nil { 181 | log.Fatalf("error connecting to nsqds: %s", err) 182 | } 183 | 184 | gracefully.Shutdown() 185 | 186 | log.Info("stopping") 187 | consumer.Stop() 188 | <-consumer.StopChan 189 | broadcast.Stop() 190 | <-broadcast.Done 191 | log.Info("bye :)") 192 | } 193 | 194 | // Parse NSQ configuration from args. 195 | func config(args map[string]interface{}) *nsq.Config { 196 | config := nsq.NewConfig() 197 | 198 | n, err := strconv.Atoi(args["--max-attempts"].(string)) 199 | if err != nil { 200 | log.Fatalf("error parsing --max-attempts: %s", err) 201 | } 202 | config.MaxAttempts = uint16(n) 203 | 204 | n, err = strconv.Atoi(args["--max-in-flight"].(string)) 205 | if err != nil { 206 | log.Fatalf("error parsing --max-in-flight: %s", err) 207 | } 208 | config.MaxInFlight = n 209 | 210 | return config 211 | } 212 | 213 | // Parse Ratelimiter configuration and return 214 | // a new ratelimiter or nil. 215 | func ratelimiter(args map[string]interface{}) *ratelimit.Ratelimiter { 216 | rate, err := strconv.Atoi(args["--ratelimit-max-rate"].(string)) 217 | if err != nil { 218 | log.Fatalf("error parsing --ratelimit-max-rate: %s", err) 219 | } 220 | 221 | if rate <= 0 { 222 | return nil 223 | } 224 | 225 | keys, err := strconv.Atoi(args["--ratelimit-max-keys"].(string)) 226 | if err != nil { 227 | log.Fatalf("error parsing --ratelimit-max-keys: %s", err) 228 | } 229 | 230 | return ratelimit.New(rate, keys) 231 | } 232 | 233 | // Dialer. 234 | func dial(addr string) func() (redis.Conn, error) { 235 | return func() (redis.Conn, error) { 236 | return redis.Dial("tcp", addr) 237 | } 238 | } 239 | 240 | // Idle connection test. 241 | func ping(client redis.Conn, t time.Time) error { 242 | _, err := client.Do("PING") 243 | return err 244 | } 245 | -------------------------------------------------------------------------------- /pubsub/pubsub.go: -------------------------------------------------------------------------------- 1 | package pubsub 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/segmentio/go-log" 7 | "github.com/segmentio/go-stats" 8 | "github.com/segmentio/nsq_to_redis/broadcast" 9 | "github.com/segmentio/nsq_to_redis/template" 10 | "github.com/segmentio/statsdclient" 11 | ) 12 | 13 | // Options for PubSub. 14 | type Options struct { 15 | Format string // Redis publish channel format 16 | Log *log.Logger // Logger 17 | Metrics *statsd.Client // Metrics 18 | } 19 | 20 | // PubSub publishes messages to a formatted channel. 21 | type PubSub struct { 22 | template *template.T 23 | stats *stats.Stats 24 | *Options 25 | } 26 | 27 | // New pubsub with options. 28 | func New(options *Options) (*PubSub, error) { 29 | p := &PubSub{ 30 | Options: options, 31 | stats: stats.New(), 32 | } 33 | 34 | tmpl, err := template.New(p.Format) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | p.template = tmpl 40 | go p.stats.TickEvery(10 * time.Second) 41 | 42 | return p, nil 43 | } 44 | 45 | // HandleMessage expects parsed json messages from NSQ, 46 | // applies them against the publish channel template to 47 | // produce the channel name, and then publishes to Redis. 48 | func (p *PubSub) Handle(c broadcast.Conn, msg *broadcast.Message) error { 49 | start := time.Now() 50 | 51 | channel, err := p.template.Eval(string(msg.JSON)) 52 | if err != nil { 53 | p.Log.Error("evaluating template: %s", err) 54 | return nil 55 | } 56 | 57 | p.Log.Info("publish %s to %s", msg.ID, channel) 58 | p.Log.Debug("contents %s %s", msg.ID, msg.JSON) 59 | 60 | err = c.Send("PUBLISH", channel, []byte(msg.JSON)) 61 | if err != nil { 62 | p.Log.Error("publish: %s", err) 63 | return err 64 | } 65 | 66 | p.Metrics.Duration("timers.published", time.Since(start)) 67 | p.Metrics.Incr("counts.published") 68 | p.stats.Incr("published") 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /pubsub/pubsub_test.go: -------------------------------------------------------------------------------- 1 | package pubsub 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | 7 | "github.com/bmizerany/assert" 8 | "github.com/garyburd/redigo/redis" 9 | "github.com/segmentio/go-log" 10 | "github.com/segmentio/nsq_to_redis/broadcast" 11 | "github.com/segmentio/nsq_to_redis/broadcast/mocks" 12 | statsd "github.com/segmentio/statsdclient" 13 | ) 14 | 15 | func TestPubSub(t *testing.T) { 16 | pubSub, err := New(&Options{ 17 | Format: "stream:project:{projectId}:ingress", 18 | Log: log.Log.New("pubsub_test"), 19 | Metrics: statsd.NewClient(ioutil.Discard), 20 | }) 21 | assert.Equal(t, nil, err) 22 | 23 | cPublish, err := redis.Dial("tcp", ":6379") 24 | assert.Equal(t, nil, err) 25 | defer cPublish.Close() 26 | 27 | conn := broadcast.NewConn(cPublish) 28 | broadcastMessage, err := broadcast.NewMessage("nsq__message__id", `{"projectId":"gy2d"}`) 29 | if err != nil { 30 | t.Error(err) 31 | } 32 | 33 | cSubscribe, err := redis.Dial("tcp", ":6379") 34 | assert.Equal(t, nil, err) 35 | defer cSubscribe.Close() 36 | 37 | psc := redis.PubSubConn{Conn: cSubscribe} 38 | psc.Subscribe("stream:project:gy2d:ingress") 39 | defer psc.Close() 40 | 41 | msgC := make(chan redis.Message, 10) 42 | go func() { 43 | for { 44 | switch v := psc.Receive().(type) { 45 | case redis.Message: 46 | msgC <- v 47 | } 48 | } 49 | }() 50 | 51 | err = pubSub.Handle(conn, broadcastMessage) 52 | assert.Equal(t, nil, err) 53 | err = conn.Flush() 54 | assert.Equal(t, nil, err) 55 | 56 | msg := <-msgC 57 | assert.Equal(t, "stream:project:gy2d:ingress", msg.Channel) 58 | assert.Equal(t, `{"projectId":"gy2d"}`, string(msg.Data)) 59 | } 60 | 61 | func BenchmarkPubSub(b *testing.B) { 62 | l := log.Log.New("pubsub_benchmark") 63 | l.SetLevel(log.ERROR) 64 | 65 | pubSub, err := New(&Options{ 66 | Format: "stream:project:{projectId}:ingress", 67 | Log: l, 68 | Metrics: statsd.NewClient(ioutil.Discard), 69 | }) 70 | if err != nil { 71 | b.Error(err) 72 | } 73 | 74 | conn := &mocks.Conn{} 75 | conn.On("Send", "PUBLISH", []interface{}{ 76 | "stream:project:gy2d:ingress", 77 | []byte(`{"projectId":"gy2d"}`), 78 | }).Return(nil) 79 | msg, err := broadcast.NewMessage("nsq_message_id_1", `{"projectId":"gy2d"}`) 80 | 81 | for i := 0; i < b.N; i++ { 82 | pubSub.Handle(conn, msg) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /ratelimit/ratelimit.go: -------------------------------------------------------------------------------- 1 | package ratelimit 2 | 3 | import ( 4 | "github.com/hashicorp/golang-lru" 5 | "github.com/juju/ratelimit" 6 | ) 7 | 8 | // Ratelimiter implements a simple rate limiter. 9 | type Ratelimiter struct { 10 | keys *lru.Cache 11 | rate int64 12 | } 13 | 14 | // New initializes a new Ratelimiter 15 | // with rate per second and lru key cache size. 16 | func New(rate, size int) *Ratelimiter { 17 | c, _ := lru.New(size) 18 | return &Ratelimiter{ 19 | keys: c, 20 | rate: int64(rate), 21 | } 22 | } 23 | 24 | // Exceeded returns true if the given `key` 25 | // has exceeded rate per second. 26 | func (rl *Ratelimiter) Exceeded(key string) bool { 27 | if b, ok := rl.keys.Get(key); ok { 28 | b := b.(*ratelimit.Bucket) 29 | _, took := b.TakeMaxDuration(1, 0) 30 | return !took 31 | } 32 | 33 | rate := float64(rl.rate) 34 | cap := rl.rate 35 | b := ratelimit.NewBucketWithRate(rate, cap) 36 | rl.keys.Add(key, b) 37 | b.Take(1) 38 | return false 39 | } 40 | -------------------------------------------------------------------------------- /ratelimit/ratelimit_test.go: -------------------------------------------------------------------------------- 1 | package ratelimit 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/bmizerany/assert" 8 | ) 9 | 10 | func TestExceeded(t *testing.T) { 11 | rl := New(1, 500) 12 | assert.Equal(t, rl.Exceeded("some-key"), false) 13 | assert.Equal(t, rl.Exceeded("some-key"), true) 14 | time.Sleep(time.Second) // :( 15 | assert.Equal(t, rl.Exceeded("some-key"), false) 16 | assert.Equal(t, rl.Exceeded("some-key"), true) 17 | } 18 | 19 | func TestMaxKeys(t *testing.T) { 20 | rl := New(1, 1) 21 | rl.Exceeded("a") 22 | rl.Exceeded("b") 23 | assert.Equal(t, rl.keys.Len(), 1) 24 | } 25 | -------------------------------------------------------------------------------- /template/template.go: -------------------------------------------------------------------------------- 1 | // Package template provides a simple templating functionality. 2 | // Given a template in the format "foo:{bar}" and data `{"bar":"b"}`, 3 | // eval returns "foo:b". 4 | // It allows multiple variables to be supplied. Given the format "{foo}:{bar}" 5 | // and data `{"foo": "f", "bar": b}`, eval returns "f:b". 6 | // It allows nested variables. Given the format "foo:{bar.baz}" and data 7 | // and data `{"bar": { "baz" : "b"}}`, eval returns "foo:b". 8 | package template 9 | 10 | import ( 11 | "bytes" 12 | "errors" 13 | 14 | "github.com/tidwall/gjson" 15 | ) 16 | 17 | var ErrMissingClosingBrace = errors.New("missing '}'") 18 | 19 | type T struct { 20 | nodes []node 21 | } 22 | 23 | // Returns a new template. 24 | func New(format string) (*T, error) { 25 | var b bytes.Buffer 26 | state := sLiteral 27 | var nodes []node 28 | 29 | for i := 0; i < len(format); i++ { 30 | c := format[i] 31 | switch state { 32 | case sLiteral: 33 | switch c { 34 | case '{': 35 | nodes = append(nodes, node{ 36 | literal: b.String(), 37 | }) 38 | b.Reset() 39 | state = sVariable 40 | default: 41 | b.WriteByte(c) 42 | } 43 | case sVariable: 44 | switch c { 45 | case '}': 46 | nodes = append(nodes, node{ 47 | variable: b.String(), 48 | }) 49 | b.Reset() 50 | state = sLiteral 51 | break 52 | default: 53 | b.WriteByte(c) 54 | } 55 | } 56 | } 57 | 58 | if state == sVariable { 59 | return nil, ErrMissingClosingBrace 60 | } 61 | 62 | if b.Len() != 0 { 63 | nodes = append(nodes, node{ 64 | literal: b.String(), 65 | }) 66 | } 67 | 68 | return &T{nodes}, nil 69 | } 70 | 71 | const ( 72 | sLiteral = iota 73 | sVariable 74 | ) 75 | 76 | func (t *T) Eval(data string) (string, error) { 77 | var b bytes.Buffer 78 | for _, n := range t.nodes { 79 | b.WriteString(n.Eval(data)) 80 | } 81 | return b.String(), nil 82 | } 83 | 84 | type node struct { 85 | variable string 86 | literal string 87 | } 88 | 89 | func (n node) Eval(data string) string { 90 | if n.variable == "" { 91 | return n.literal 92 | } 93 | 94 | return gjson.Get(data, n.variable).String() 95 | } 96 | -------------------------------------------------------------------------------- /template/template_test.go: -------------------------------------------------------------------------------- 1 | package template_test 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "testing" 7 | 8 | interpolate "github.com/segmentio/go-interpolate" 9 | "github.com/segmentio/nsq_to_redis/template" 10 | ) 11 | 12 | func TestNewInvalid(t *testing.T) { 13 | testCases := []struct { 14 | format string 15 | }{ 16 | {"{"}, 17 | {"{foo"}, 18 | {"da:sad{foo"}, 19 | } 20 | 21 | for _, tc := range testCases { 22 | t.Run(fmt.Sprintf("%q", tc.format), func(t *testing.T) { 23 | _, err := template.New(tc.format) 24 | if err != template.ErrMissingClosingBrace { 25 | t.Error("expected New to fail because closing brace was missing but didn't") 26 | } 27 | }) 28 | } 29 | } 30 | 31 | func TestValidTemplates(t *testing.T) { 32 | testCases := []struct { 33 | format string 34 | data string 35 | expected string 36 | }{ 37 | {"}", `{}`, "}"}, 38 | {"", `{}`, ""}, 39 | {"foo:bar:baz", `{}`, "foo:bar:baz"}, 40 | {"foo:{bar}", `{"bar":"baz"}`, "foo:baz"}, 41 | {"{foo}:{bar}:baz", `{"foo":"f","bar":"b"}`, "f:b:baz"}, 42 | {"foo:{bar.baz}", `{"bar":{"baz":"b"}}`, "foo:b"}, 43 | {"stream:{foo}:{bar}", `{"foo":"f","bar":"b"}`, "stream:f:b"}, 44 | {"stream:project:{projectId}:ingress", `{}`, "stream:project:null:ingress"}, 45 | {"integration-errors:project:{projectId}:ingress", `{"projectId":"p"}`, "integration-errors:project:p:ingress"}, 46 | {"stream:project:{projectId}:ingress", `{"projectId":"foo"}`, "stream:project:foo:ingress"}, 47 | {"stream:persist:{projectId}:ingress", `{"projectId":"foo"}`, "stream:persist:foo:ingress"}, 48 | } 49 | 50 | for _, tc := range testCases { 51 | t.Run(fmt.Sprintf("%q.Eval(`%s`)", tc.format, tc.data), func(t *testing.T) { 52 | tmpl, err := template.New(tc.format) 53 | if err != nil { 54 | t.Error("expected New to not error, but did: %v", err) 55 | } 56 | 57 | got, err := tmpl.Eval(tc.data) 58 | if err != nil { 59 | t.Errorf("expected Eval to not error, but did: %v", err) 60 | } 61 | 62 | if got != tc.expected { 63 | t.Errorf("expected eval to return %q, but got: %q", tc.expected, got) 64 | } 65 | }) 66 | } 67 | } 68 | 69 | var benchmarkData = []byte(`{ 70 | "anonymousId": "075100b8-0011-4c87-8731-450e5e41858a", 71 | "context": { 72 | "app": { 73 | "build": 175, 74 | "name": "Car Loan Calculator", 75 | "namespace": "com.boondoggle.autocalc", 76 | "version": "1.7.5" 77 | }, 78 | "device": { 79 | "adTrackingEnabled": true, 80 | "advertisingId": "khisaldas>", 81 | "id": "dsadas", 82 | "manufacturer": "samsung", 83 | "model": "SAMSUNG-SM-G930A", 84 | "name": "heroqlteatt", 85 | "type": "android" 86 | }, 87 | "library": { 88 | "name": "analytics-android", 89 | "version": "4.2.4" 90 | }, 91 | "locale": "en-US", 92 | "network": { 93 | "bluetooth": false, 94 | "carrier": "AT&T", 95 | "cellular": false, 96 | "wifi": true 97 | }, 98 | "os": { 99 | "name": "Android", 100 | "version": "7.0" 101 | }, 102 | "screen": { 103 | "density": 3, 104 | "height": 1920, 105 | "width": 1080 106 | }, 107 | "timezone": "America/Chicago", 108 | "traits": { 109 | "anonymousId": "89b2-d-sads-da-sda-d12bdsad" 110 | }, 111 | "userAgent": "Dalvik/2.1.0 (Linux; U; Android 7.0; SAMSUNG-SM-G930A Build/NRD90M)", 112 | "ip": "99.185.26.86" 113 | }, 114 | "event": "Application Launched", 115 | "integrations": { 116 | "Mixpanel": false, 117 | "MoEngage": false 118 | }, 119 | "messageId": "ef5b39cc-ad0a-4a80-8b43-18573a2b6b15", 120 | "properties": { 121 | "event_id": "33e95185-596c-4097-a597-e3cb3282d686", 122 | "event_number": 34, 123 | "via_analytics_android": "true" 124 | }, 125 | "timestamp": "2017-04-14T21:49:19.549Z", 126 | "type": "track", 127 | "writeKey": "xNb9e1cMp52JPelx7gvUo5kwR8vPxjcF", 128 | "sentAt": "2017-04-14T21:49:18.000Z", 129 | "receivedAt": "2017-04-14T21:49:19.549Z", 130 | "originalTimestamp": "2017-04-14T16:49:18-0500" 131 | }`) 132 | 133 | func BenchmarkTemplate(b *testing.B) { 134 | tmpl, err := template.New("stream:project:{projectId}:ingress") 135 | if err != nil { 136 | b.Error(err) 137 | } 138 | 139 | for i := 0; i < b.N; i++ { 140 | tmpl.Eval(string(benchmarkData)) 141 | } 142 | } 143 | 144 | func BenchmarkInterpolate(b *testing.B) { 145 | tmpl, err := interpolate.New("stream:project:{projectId}:ingress") 146 | if err != nil { 147 | b.Error(err) 148 | } 149 | 150 | for i := 0; i < b.N; i++ { 151 | var m map[string]interface{} 152 | json.Unmarshal(benchmarkData, &m) 153 | tmpl.Eval(m) 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /vendor/vendor.json: -------------------------------------------------------------------------------- 1 | { 2 | "comment": "", 3 | "ignore": "test", 4 | "package": [ 5 | { 6 | "checksumSHA1": "HRmUbYibH9FXhP1oWbrLSut7sk8=", 7 | "path": "github.com/bitly/go-nsq", 8 | "revision": "a3aee1d8e104a99d8fedffe2c45832df6a96d735", 9 | "revisionTime": "2015-07-15T15:41:33Z" 10 | }, 11 | { 12 | "checksumSHA1": "yPT+niKHgZg7Fhwg5QXoYczvwRY=", 13 | "path": "github.com/bmizerany/assert", 14 | "revision": "b7ed37b82869576c289d7d97fb2bbd8b64a0cb28", 15 | "revisionTime": "2016-06-11T22:19:34Z" 16 | }, 17 | { 18 | "checksumSHA1": "lUBE/ZMjzN5VJkT6lgQw3WI72lo=", 19 | "path": "github.com/dustin/go-humanize", 20 | "revision": "2fcb5204cdc65b4bec9fd0a87606bb0d0e3c54e8", 21 | "revisionTime": "2016-07-21T06:51:13Z" 22 | }, 23 | { 24 | "checksumSHA1": "2UmMbNHc8FBr98mJFN1k8ISOIHk=", 25 | "path": "github.com/garyburd/redigo/internal", 26 | "revision": "b8dc90050f24c1a73a52f107f3f575be67b21b7c", 27 | "revisionTime": "2016-05-25T16:57:06Z" 28 | }, 29 | { 30 | "checksumSHA1": "507OiSqTxfGCje7xDT5eq9CCaNQ=", 31 | "path": "github.com/garyburd/redigo/redis", 32 | "revision": "b8dc90050f24c1a73a52f107f3f575be67b21b7c", 33 | "revisionTime": "2016-05-25T16:57:06Z" 34 | }, 35 | { 36 | "checksumSHA1": "NuLhBJ8GppcXqPK4D5sCGn0IcX8=", 37 | "path": "github.com/go-redis/redis", 38 | "revision": "dd76391eb986c2ea0aba41e9edde61ff7e0d4b28", 39 | "revisionTime": "2017-04-05T12:32:37Z" 40 | }, 41 | { 42 | "checksumSHA1": "XUqPA2V7/z8JyWnCPclCztcXO7M=", 43 | "path": "github.com/go-redis/redis/internal", 44 | "revision": "dd76391eb986c2ea0aba41e9edde61ff7e0d4b28", 45 | "revisionTime": "2017-04-05T12:32:37Z" 46 | }, 47 | { 48 | "checksumSHA1": "GQZsUVg/+6UpQAYpc4luMvMutSI=", 49 | "path": "github.com/go-redis/redis/internal/consistenthash", 50 | "revision": "dd76391eb986c2ea0aba41e9edde61ff7e0d4b28", 51 | "revisionTime": "2017-04-05T12:32:37Z" 52 | }, 53 | { 54 | "checksumSHA1": "VP0K8vvf716n6xuzAZTvpz+wTwc=", 55 | "path": "github.com/go-redis/redis/internal/hashtag", 56 | "revision": "dd76391eb986c2ea0aba41e9edde61ff7e0d4b28", 57 | "revisionTime": "2017-04-05T12:32:37Z" 58 | }, 59 | { 60 | "checksumSHA1": "wz1pQuujYv3lDTXq8PKQIKRYl54=", 61 | "path": "github.com/go-redis/redis/internal/pool", 62 | "revision": "dd76391eb986c2ea0aba41e9edde61ff7e0d4b28", 63 | "revisionTime": "2017-04-05T12:32:37Z" 64 | }, 65 | { 66 | "checksumSHA1": "lEtS56d84Jw7raXpyu/XkIpTLLU=", 67 | "path": "github.com/go-redis/redis/internal/proto", 68 | "revision": "dd76391eb986c2ea0aba41e9edde61ff7e0d4b28", 69 | "revisionTime": "2017-04-05T12:32:37Z" 70 | }, 71 | { 72 | "checksumSHA1": "d9PxF1XQGLMJZRct2R8qVM/eYlE=", 73 | "path": "github.com/hashicorp/golang-lru", 74 | "revision": "0a025b7e63adc15a622f29b0b2c4c3848243bbf6", 75 | "revisionTime": "2016-08-13T22:13:03Z" 76 | }, 77 | { 78 | "checksumSHA1": "9hffs0bAIU6CquiRhKQdzjHnKt0=", 79 | "path": "github.com/hashicorp/golang-lru/simplelru", 80 | "revision": "0a025b7e63adc15a622f29b0b2c4c3848243bbf6", 81 | "revisionTime": "2016-08-13T22:13:03Z" 82 | }, 83 | { 84 | "checksumSHA1": "+NJzbj9fa71GPyEhyYcB2urBiXY=", 85 | "path": "github.com/juju/ratelimit", 86 | "revision": "acf38b000a03e4ab89e40f20f1e548f4e6ac7f72", 87 | "revisionTime": "2017-03-14T01:17:55Z" 88 | }, 89 | { 90 | "checksumSHA1": "vbjzNeqTEIPV1W6wT5PB3OwN9Ns=", 91 | "path": "github.com/kr/pretty", 92 | "revision": "737b74a46c4bf788349f72cb256fed10aea4d0ac", 93 | "revisionTime": "2016-07-08T21:57:48Z" 94 | }, 95 | { 96 | "checksumSHA1": "uulQHQ7IsRKqDudBC8Go9J0gtAc=", 97 | "path": "github.com/kr/text", 98 | "revision": "7cafcd837844e784b526369c9bce262804aebc60", 99 | "revisionTime": "2016-05-04T02:26:26Z" 100 | }, 101 | { 102 | "checksumSHA1": "IwuaGWkq9sh9hzmmgkyHvCEMGzM=", 103 | "path": "github.com/mreiferson/go-snappystream", 104 | "revision": "028eae7ab5c4c9e2d1cb4c4ca1e53259bbe7e504", 105 | "revisionTime": "2015-04-16T23:44:20Z" 106 | }, 107 | { 108 | "checksumSHA1": "5mGTJWVBk+2KdbBQoJWMtGoJJgU=", 109 | "path": "github.com/mreiferson/go-snappystream/snappy-go", 110 | "revision": "028eae7ab5c4c9e2d1cb4c4ca1e53259bbe7e504", 111 | "revisionTime": "2015-04-16T23:44:20Z" 112 | }, 113 | { 114 | "checksumSHA1": "dkMvDo5Ni3X6LsIT3vnPYw21LDw=", 115 | "path": "github.com/segmentio/go-interpolate", 116 | "revision": "ed5d043b681f7100b24ed5d6337d22d3221fefe7", 117 | "revisionTime": "2017-04-26T20:41:03Z" 118 | }, 119 | { 120 | "checksumSHA1": "10xF3ucjbvs5iUifJY8hgg6CSpw=", 121 | "path": "github.com/segmentio/go-log", 122 | "revision": "9c7ea0f744953ff5220087609a8525790dc01d8b", 123 | "revisionTime": "2014-12-05T20:14:27Z" 124 | }, 125 | { 126 | "checksumSHA1": "NgMMOiZjk3Q2e0RgmGjgC1cLRUE=", 127 | "path": "github.com/segmentio/go-stats", 128 | "revision": "4a4d67ba5b419d69de649bc6022860a20ca30e34", 129 | "revisionTime": "2015-04-11T19:54:18Z" 130 | }, 131 | { 132 | "checksumSHA1": "UqnxpFiLan1XEonWFLoF4y0CFsY=", 133 | "path": "github.com/segmentio/statsdclient", 134 | "revision": "1694aafe4881d04277b5f2c5835cc504ae53dbc1", 135 | "revisionTime": "2018-01-09T07:05:06Z" 136 | }, 137 | { 138 | "checksumSHA1": "Ntxt39Uh4Bwm20WFS93Xkzzi12E=", 139 | "path": "github.com/tidwall/gjson", 140 | "revision": "6e0babc7e842e3080bdc53d94084167e2ac8db65", 141 | "revisionTime": "2017-04-14T18:13:32Z" 142 | }, 143 | { 144 | "checksumSHA1": "qmePMXEDYGwkAfT9QvtMC58JN/E=", 145 | "path": "github.com/tidwall/match", 146 | "revision": "173748da739a410c5b0b813b956f89ff94730b4c", 147 | "revisionTime": "2016-08-30T17:39:30Z" 148 | }, 149 | { 150 | "checksumSHA1": "LfLTyk4ox5wH0YL4sD5rcbrVWBw=", 151 | "path": "github.com/tj/docopt", 152 | "revision": "c8470e45692f168e8b380c5d625327e756d7d0a9", 153 | "revisionTime": "2014-08-07T22:00:17Z" 154 | }, 155 | { 156 | "checksumSHA1": "t1cla+OlP0NpAWUjx/SPYAR2m3k=", 157 | "path": "github.com/tj/go-gracefully", 158 | "revision": "005c1d102f1bf3158f9ba29c12d68a73ac15f321", 159 | "revisionTime": "2014-12-27T06:10:38Z" 160 | } 161 | ], 162 | "rootPath": "github.com/segmentio/nsq_to_redis" 163 | } 164 | --------------------------------------------------------------------------------