├── .gitignore
├── README.md
├── hekad.toml
└── input.go


/.gitignore:
--------------------------------------------------------------------------------
1 | .vagrant
2 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | heka-redis
 2 | ==========
 3 | 
 4 | Redis PubSub input plugin for [Mozilla Heka](http://hekad.readthedocs.org/)
 5 | 
 6 | See [Building *hekad* with External Plugins](http://hekad.readthedocs.org/en/latest/installing.html#build-include-externals)
 7 | for compiling in plugins.
 8 | 
 9 | Basically, you'll need to edit the cmake/plugin_loader.cmake file and add
10 | 
11 |     add_external_plugin(git https://github.com/victorcoder/heka-redis master)
12 | 
13 | And build heka
14 | 
15 | Debug
16 | =====
17 | 
18 | Add this to your hekad.toml file to see log output of your subscriptions:
19 | 
20 | ```toml
21 | [PayloadEncoder]
22 | append_newlines = false
23 | prefix_ts = true
24 | ts_format = "2006/01/02 3:04:05PM MST"
25 | 
26 | [debug]
27 | type = "LogOutput"
28 | message_matcher = "Type == 'redis_pub_sub'"
29 | encoder = "PayloadEncoder"
30 | ```
31 | 


--------------------------------------------------------------------------------
/hekad.toml:
--------------------------------------------------------------------------------
1 | [RedisPubSubInput]
2 | address = ":6379" # Redis server address
3 | channel = "*"
4 | 


--------------------------------------------------------------------------------
/input.go:
--------------------------------------------------------------------------------
  1 | package heka_redis
  2 | 
  3 | import (
  4 | 	"errors"
  5 | 	"fmt"
  6 | 	"github.com/garyburd/redigo/redis"
  7 | 	"github.com/mozilla-services/heka/pipeline"
  8 | 	"time"
  9 | )
 10 | 
 11 | type RedisPubSubInputConfig struct {
 12 | 	Address     string `toml:"address"`
 13 | 	Channel     string `toml:"channel"`
 14 | 	DecoderName string `toml:"decoder"`
 15 | }
 16 | 
 17 | type RedisPubSubInput struct {
 18 | 	conf *RedisPubSubInputConfig
 19 | 	conn redis.Conn
 20 | }
 21 | 
 22 | func (rpsi *RedisPubSubInput) ConfigStruct() interface{} {
 23 | 	return &RedisPubSubInputConfig{":6379", "*", ""}
 24 | }
 25 | 
 26 | func (rpsi *RedisPubSubInput) Init(config interface{}) error {
 27 | 	rpsi.conf = config.(*RedisPubSubInputConfig)
 28 | 
 29 | 	var err error
 30 | 	rpsi.conn, err = redis.Dial("tcp", rpsi.conf.Address)
 31 | 	if err != nil {
 32 | 		return fmt.Errorf("connecting to - %s", err.Error())
 33 | 	}
 34 | 
 35 | 	return nil
 36 | }
 37 | 
 38 | func (rpsi *RedisPubSubInput) Run(ir pipeline.InputRunner, h pipeline.PluginHelper) error {
 39 | 	var (
 40 | 		dRunner pipeline.DecoderRunner
 41 | 		decoder pipeline.Decoder
 42 | 		pack    *pipeline.PipelinePack
 43 | 		e       error
 44 | 		ok      bool
 45 | 	)
 46 | 	// Get the InputRunner's chan to receive empty PipelinePacks
 47 | 	packSupply := ir.InChan()
 48 | 
 49 | 	if rpsi.conf.DecoderName != "" {
 50 | 		if dRunner, ok = h.DecoderRunner(rpsi.conf.DecoderName, fmt.Sprintf("%s-%s", ir.Name(), rpsi.conf.DecoderName)); !ok {
 51 | 			return fmt.Errorf("Decoder not found: %s", rpsi.conf.DecoderName)
 52 | 		}
 53 | 		decoder = dRunner.Decoder()
 54 | 	}
 55 | 
 56 | 	//Connect to the channel
 57 | 	psc := redis.PubSubConn{Conn: rpsi.conn}
 58 | 	psc.PSubscribe(rpsi.conf.Channel)
 59 | 
 60 | 	for {
 61 | 		switch n := psc.Receive().(type) {
 62 | 		case redis.PMessage:
 63 | 			// Grab an empty PipelinePack from the InputRunner
 64 | 			pack = <-packSupply
 65 | 			pack.Message.SetType("redis_pub_sub")
 66 | 			pack.Message.SetLogger(n.Channel)
 67 | 			pack.Message.SetPayload(string(n.Data))
 68 | 			pack.Message.SetTimestamp(time.Now().UnixNano())
 69 | 			var packs []*pipeline.PipelinePack
 70 | 			if decoder == nil {
 71 | 				packs = []*pipeline.PipelinePack{pack}
 72 | 			} else {
 73 | 				packs, e = decoder.Decode(pack)
 74 | 			}
 75 | 			if packs != nil {
 76 | 				for _, p := range packs {
 77 | 					ir.Inject(p)
 78 | 				}
 79 | 			} else {
 80 | 				if e != nil {
 81 | 					ir.LogError(fmt.Errorf("Couldn't parse Redis message: %s", n.Data))
 82 | 				}
 83 | 				pack.Recycle(nil)
 84 | 			}
 85 | 		case redis.Subscription:
 86 | 			ir.LogMessage(fmt.Sprintf("Subscription: %s %s %d\n", n.Kind, n.Channel, n.Count))
 87 | 			if n.Count == 0 {
 88 | 				return errors.New("No channel to subscribe")
 89 | 			}
 90 | 		case error:
 91 | 			ir.LogError(fmt.Errorf("error: %v\n", n))
 92 | 			return n
 93 | 		}
 94 | 	}
 95 | 
 96 | 	return nil
 97 | }
 98 | 
 99 | func (rpsi *RedisPubSubInput) Stop() {
100 | 	rpsi.conn.Close()
101 | }
102 | 
103 | func init() {
104 | 	pipeline.RegisterPlugin("RedisPubSubInput", func() interface{} {
105 | 		return new(RedisPubSubInput)
106 | 	})
107 | }
108 | 


--------------------------------------------------------------------------------