├── LICENSE ├── README.md ├── buffer └── buffer.go ├── example.config.json ├── input ├── input.go └── lumberjack │ └── lumberjack.go ├── main.go ├── output ├── elasticsearch │ └── elasticsearch.go ├── output.go ├── tcp │ └── tcp.go └── websocket │ ├── template.go │ └── websocket.go ├── parser └── parser.go └── server ├── config.go └── server.go /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Hailo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Logslam - A lumberjack => logstash indexer in Go 2 | 3 | Logslam is a lightweight lumberjack compliant log indexer. It accepts the lumberjack v1 protocol and indexes logs in elasticsearch. It was written with the intention of being a small and efficient replacement for logstash on AWS EC2. It does not attempt to replicate all of logstash's features, the goal was to simply replace it for Hailo's specific use case in the ELK stack. 4 | 5 | ## Supported IO 6 | 7 | ### Inputs 8 | 9 | - Lumberjack V1 Protocol 10 | 11 | ### Outputs 12 | 13 | - TCP Streaming 14 | - WebSocket Streaming 15 | - Elasticsearch 16 | 17 | ## Getting Started 18 | 19 | ### 1. Create config 20 | 21 | Create a config file specifying location of ssl crt/key, bind addresses for inputs/outputs and elasticsearch hosts. 22 | 23 | An example config can be found in example.config.json: 24 | ``` 25 | { 26 | "inputs": { 27 | "lumberjack": { 28 | "host": ":7200", 29 | "ssl_key": "lumberjack.key", 30 | "ssl_crt": "lumberjack.crt" 31 | } 32 | }, 33 | "outputs": { 34 | "tcp": { 35 | "host": ":7201" 36 | }, 37 | "websocket": { 38 | "host": ":7202" 39 | }, 40 | "elasticsearch": { 41 | "hosts": [ 42 | "localhost:9200" 43 | ] 44 | } 45 | } 46 | } 47 | ``` 48 | 49 | ### 2. Run the server 50 | 51 | ``` 52 | $ go get 53 | $ $GOPATH/bin/logslam -config=example.config.json 54 | 2015/01/20 20:59:03 Starting server 55 | 2015/01/20 20:59:03 Starting buffer 56 | 2015/01/20 20:59:03 Starting input lumberjack 57 | 2015/01/20 20:59:03 Starting output tcp 58 | 2015/01/20 20:59:03 Starting output websocket 59 | 2015/01/20 20:59:03 Starting output elasticsearch 60 | ``` 61 | 62 | ### Streaming logs via TCP 63 | 64 | ``` 65 | nc localhost 7201 66 | ``` 67 | 68 | ### Streaming logs via WebSocket 69 | 70 | ``` 71 | Connect to http://localhost:7202 in a browser. 72 | A list of known sources will be displayed. 73 | ``` 74 | -------------------------------------------------------------------------------- /buffer/buffer.go: -------------------------------------------------------------------------------- 1 | package buffer 2 | 3 | import ( 4 | "log" 5 | "time" 6 | ) 7 | 8 | const ( 9 | bufSize = 100 10 | ) 11 | 12 | type Sender interface { 13 | AddSubscriber(string, chan *Event) error 14 | DelSubscriber(string) error 15 | } 16 | 17 | // Taken from https://github.com/elasticsearch/logstash-forwarder/blob/master/event.go 18 | type Event struct { 19 | Source string `json:"source,omitempty"` 20 | Offset int64 `json:"offset,omitempty"` 21 | Line uint64 `json:"line,omitempty"` 22 | Text *string `json:"text,omitempty"` 23 | Fields *map[string]string 24 | } 25 | 26 | // subscriber is some host that wants to receive events 27 | type subscriber struct { 28 | Host string 29 | Send chan *Event 30 | } 31 | 32 | type Buffer struct { 33 | send chan *Event 34 | subscribers map[string]*subscriber 35 | add chan *subscriber 36 | del chan string 37 | term chan bool 38 | ticker *time.Ticker 39 | } 40 | 41 | func New() *Buffer { 42 | return &Buffer{ 43 | ticker: time.NewTicker(time.Duration(10) * time.Millisecond), 44 | send: make(chan *Event, bufSize), 45 | subscribers: make(map[string]*subscriber), 46 | add: make(chan *subscriber, 1), 47 | del: make(chan string, 1), 48 | term: make(chan bool, 1), 49 | } 50 | } 51 | 52 | func (b *Buffer) AddSubscriber(host string, ch chan *Event) error { 53 | b.add <- &subscriber{host, ch} 54 | return nil 55 | } 56 | 57 | func (b *Buffer) DelSubscriber(host string) error { 58 | b.del <- host 59 | return nil 60 | } 61 | 62 | func (b *Buffer) Publish(event *Event) { 63 | for _, sub := range b.subscribers { 64 | select { 65 | case sub.Send <- event: 66 | case <-b.ticker.C: 67 | } 68 | } 69 | } 70 | 71 | func (b *Buffer) Send(event *Event) { 72 | b.send <- event 73 | } 74 | 75 | func (b *Buffer) Start() { 76 | for { 77 | select { 78 | case e := <-b.send: 79 | b.Publish(e) 80 | case s := <-b.add: 81 | if _, ok := b.subscribers[s.Host]; ok { 82 | log.Printf("A subscriber is already registered for %s\n", s.Host) 83 | continue 84 | } 85 | b.subscribers[s.Host] = s 86 | case h := <-b.del: 87 | delete(b.subscribers, h) 88 | case <-b.term: 89 | log.Println("Received on term chan") 90 | break 91 | } 92 | } 93 | } 94 | func (b *Buffer) Stop() { 95 | b.term <- true 96 | } 97 | -------------------------------------------------------------------------------- /example.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "inputs": { 3 | "lumberjack": { 4 | "host": ":7200", 5 | "ssl_key": "lumberjack.key", 6 | "ssl_crt": "lumberjack.crt" 7 | } 8 | }, 9 | "outputs": { 10 | "tcp": { 11 | "host": ":7201" 12 | }, 13 | "websocket": { 14 | "host": ":7202" 15 | }, 16 | "elasticsearch": { 17 | "hosts": [ 18 | "localhost:9200" 19 | ] 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /input/input.go: -------------------------------------------------------------------------------- 1 | package input 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/hailocab/logslam/parser" 8 | ) 9 | 10 | type Input interface { 11 | Init(json.RawMessage, parser.Receiver) error 12 | Start() error 13 | Stop() error 14 | } 15 | 16 | var ( 17 | inputs = make(map[string]Input) 18 | ) 19 | 20 | func Register(name string, input Input) error { 21 | if _, ok := inputs[name]; ok { 22 | return fmt.Errorf("Input %s already exists", name) 23 | } 24 | inputs[name] = input 25 | return nil 26 | } 27 | 28 | func Load(name string) (Input, error) { 29 | input, ok := inputs[name] 30 | if !ok { 31 | return nil, fmt.Errorf("Input %s not found", name) 32 | } 33 | return input, nil 34 | } 35 | -------------------------------------------------------------------------------- /input/lumberjack/lumberjack.go: -------------------------------------------------------------------------------- 1 | package lumberjack 2 | 3 | import ( 4 | "crypto/tls" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "net" 9 | 10 | "github.com/hailocab/logslam/input" 11 | "github.com/hailocab/logslam/parser" 12 | ) 13 | 14 | type Config struct { 15 | Host string `json:"host"` 16 | SSLCrt string `json:"ssl_crt"` 17 | SSLKey string `json:"ssl_key"` 18 | } 19 | 20 | type LJServer struct { 21 | Config *Config 22 | r parser.Receiver 23 | term chan bool 24 | } 25 | 26 | func init() { 27 | input.Register("lumberjack", &LJServer{ 28 | term: make(chan bool, 1), 29 | }) 30 | } 31 | 32 | // lumberConn handles an incoming connection from a lumberjack client 33 | func lumberConn(c net.Conn, r parser.Receiver) { 34 | defer c.Close() 35 | log.Printf("[%s] accepting lumberjack connection", c.RemoteAddr().String()) 36 | parser.New(c, r).Parse() 37 | log.Printf("[%s] closing lumberjack connection", c.RemoteAddr().String()) 38 | } 39 | 40 | func (lj *LJServer) Init(config json.RawMessage, r parser.Receiver) error { 41 | var ljConfig *Config 42 | if err := json.Unmarshal(config, &ljConfig); err != nil { 43 | return fmt.Errorf("Error parsing lumberjack config: %v", err) 44 | } 45 | 46 | lj.Config = ljConfig 47 | lj.r = r 48 | 49 | return nil 50 | } 51 | 52 | func (lj *LJServer) Start() error { 53 | cert, err := tls.LoadX509KeyPair(lj.Config.SSLCrt, lj.Config.SSLKey) 54 | if err != nil { 55 | return fmt.Errorf("Error loading keys: %v", err) 56 | } 57 | 58 | conn, err := net.Listen("tcp", lj.Config.Host) 59 | if err != nil { 60 | return fmt.Errorf("Listener failed: %v", err) 61 | } 62 | 63 | config := tls.Config{Certificates: []tls.Certificate{cert}} 64 | 65 | ln := tls.NewListener(conn, &config) 66 | 67 | for { 68 | select { 69 | case <-lj.term: 70 | log.Println("Lumberjack server received term signal") 71 | return nil 72 | default: 73 | conn, err := ln.Accept() 74 | if err != nil { 75 | log.Printf("Error accepting connection: %v", err) 76 | continue 77 | } 78 | go lumberConn(conn, lj.r) 79 | } 80 | } 81 | 82 | return nil 83 | } 84 | 85 | func (lj *LJServer) Stop() error { 86 | lj.term <- true 87 | return nil 88 | } 89 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | _ "github.com/hailocab/logslam/input/lumberjack" 9 | _ "github.com/hailocab/logslam/output/elasticsearch" 10 | _ "github.com/hailocab/logslam/output/tcp" 11 | _ "github.com/hailocab/logslam/output/websocket" 12 | "github.com/hailocab/logslam/server" 13 | ) 14 | 15 | var ( 16 | config string 17 | ) 18 | 19 | func init() { 20 | flag.StringVar(&config, "config", "", "Path to the config file") 21 | flag.Parse() 22 | 23 | if len(config) == 0 { 24 | fmt.Fprintln(os.Stderr, "Require a config file") 25 | flag.PrintDefaults() 26 | os.Exit(1) 27 | } 28 | } 29 | 30 | func main() { 31 | srv, err := server.New(config) 32 | if err != nil { 33 | fmt.Fprintln(os.Stderr, err.Error()) 34 | os.Exit(1) 35 | } 36 | 37 | srv.Start() 38 | } 39 | -------------------------------------------------------------------------------- /output/elasticsearch/elasticsearch.go: -------------------------------------------------------------------------------- 1 | package elasticsearch 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "log" 9 | "strconv" 10 | "time" 11 | 12 | "github.com/hailocab/elastigo/api" 13 | "github.com/hailocab/logslam/buffer" 14 | "github.com/hailocab/logslam/output" 15 | ) 16 | 17 | const ( 18 | defaultHost = "127.0.0.1" 19 | defaultIndexPrefix = "logstash" 20 | esFlushInterval = 5 21 | esMaxConns = 20 22 | esRecvBuffer = 100 23 | esSendBuffer = 100 24 | ) 25 | 26 | type Indexer struct { 27 | events int 28 | buffer *bytes.Buffer 29 | } 30 | 31 | type Config struct { 32 | Hosts []string `json:"hosts"` 33 | } 34 | 35 | type ESServer struct { 36 | host string 37 | hosts []string 38 | b buffer.Sender 39 | term chan bool 40 | } 41 | 42 | func init() { 43 | output.Register("elasticsearch", &ESServer{ 44 | host: fmt.Sprintf("%s:%d", defaultHost, time.Now().Unix()), 45 | term: make(chan bool, 1), 46 | }) 47 | } 48 | 49 | func indexName(idx string) string { 50 | if len(idx) == 0 { 51 | idx = defaultIndexPrefix 52 | } 53 | 54 | return fmt.Sprintf("%s-%s", idx, time.Now().Format("2006.01.02")) 55 | } 56 | 57 | func bulkSend(buf *bytes.Buffer) error { 58 | _, err := api.DoCommand("POST", "/_bulk", buf) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | return nil 64 | } 65 | 66 | func indexDoc(ev *buffer.Event) *map[string]interface{} { 67 | f := *ev.Fields 68 | host := f["host"] 69 | file := f["file"] 70 | timestamp := f["timestamp"] 71 | message := strconv.Quote(*ev.Text) 72 | 73 | delete(f, "timestamp") 74 | delete(f, "line") 75 | delete(f, "host") 76 | delete(f, "file") 77 | 78 | return &map[string]interface{}{ 79 | "@type": f["type"], 80 | "@message": &message, 81 | "@source_path": file, 82 | "@source_host": host, 83 | "@timestamp": timestamp, 84 | "@fields": &f, 85 | "@source": ev.Source, 86 | } 87 | } 88 | 89 | func (i *Indexer) writeBulk(index string, _type string, data interface{}) error { 90 | w := `{"index":{"_index":"%s","_type":"%s"}}` 91 | 92 | i.buffer.WriteString(fmt.Sprintf(w, index, _type)) 93 | i.buffer.WriteByte('\n') 94 | 95 | switch v := data.(type) { 96 | case *bytes.Buffer: 97 | io.Copy(i.buffer, v) 98 | case []byte: 99 | i.buffer.Write(v) 100 | case string: 101 | i.buffer.WriteString(v) 102 | default: 103 | body, err := json.Marshal(data) 104 | if err != nil { 105 | log.Printf("Error writing bulk data: %v", err) 106 | return err 107 | } 108 | i.buffer.Write(body) 109 | } 110 | i.buffer.WriteByte('\n') 111 | return nil 112 | } 113 | 114 | func (i *Indexer) flush() { 115 | if i.events == 0 { 116 | return 117 | } 118 | 119 | log.Printf("Flushing %d event(s) to elasticsearch", i.events) 120 | for j := 0; j < 3; j++ { 121 | if err := bulkSend(i.buffer); err != nil { 122 | log.Printf("Failed to index event (will retry): %v", err) 123 | time.Sleep(time.Duration(50) * time.Millisecond) 124 | continue 125 | } 126 | break 127 | } 128 | 129 | i.buffer.Reset() 130 | i.events = 0 131 | } 132 | 133 | func (i *Indexer) index(ev *buffer.Event) { 134 | doc := indexDoc(ev) 135 | idx := indexName("") 136 | typ := (*ev.Fields)["type"] 137 | 138 | i.events++ 139 | i.writeBulk(idx, typ, doc) 140 | 141 | if i.events < esSendBuffer { 142 | return 143 | } 144 | 145 | log.Printf("Flushing %d event(s) to elasticsearch", i.events) 146 | for j := 0; j < 3; j++ { 147 | if err := bulkSend(i.buffer); err != nil { 148 | log.Printf("Failed to index event (will retry): %v", err) 149 | time.Sleep(time.Duration(50) * time.Millisecond) 150 | continue 151 | } 152 | break 153 | } 154 | 155 | i.buffer.Reset() 156 | i.events = 0 157 | } 158 | 159 | func (e *ESServer) Init(config json.RawMessage, b buffer.Sender) error { 160 | var esConfig *Config 161 | if err := json.Unmarshal(config, &esConfig); err != nil { 162 | return fmt.Errorf("Error parsing elasticsearch config: %v", err) 163 | } 164 | 165 | e.hosts = esConfig.Hosts 166 | e.b = b 167 | 168 | return nil 169 | } 170 | 171 | func (es *ESServer) Start() error { 172 | api.SetHosts(es.hosts) 173 | 174 | // Add the client as a subscriber 175 | r := make(chan *buffer.Event, esRecvBuffer) 176 | es.b.AddSubscriber(es.host, r) 177 | defer es.b.DelSubscriber(es.host) 178 | 179 | // Create indexer 180 | idx := &Indexer{0, bytes.NewBuffer(nil)} 181 | 182 | // Loop events and publish to elasticsearch 183 | tick := time.NewTicker(time.Duration(esFlushInterval) * time.Second) 184 | 185 | for { 186 | select { 187 | case ev := <-r: 188 | idx.index(ev) 189 | case <-tick.C: 190 | idx.flush() 191 | case <-es.term: 192 | tick.Stop() 193 | log.Println("Elasticsearch received term signal") 194 | break 195 | } 196 | } 197 | 198 | return nil 199 | } 200 | 201 | func (es *ESServer) Stop() error { 202 | es.term <- true 203 | return nil 204 | } 205 | -------------------------------------------------------------------------------- /output/output.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/hailocab/logslam/buffer" 8 | ) 9 | 10 | type Output interface { 11 | Init(json.RawMessage, buffer.Sender) error 12 | Start() error 13 | Stop() error 14 | } 15 | 16 | var ( 17 | outputs = make(map[string]Output) 18 | ) 19 | 20 | func Register(name string, output Output) error { 21 | if _, ok := outputs[name]; ok { 22 | return fmt.Errorf("Output %s already exists", name) 23 | } 24 | outputs[name] = output 25 | return nil 26 | } 27 | 28 | func Load(name string) (Output, error) { 29 | output, ok := outputs[name] 30 | if !ok { 31 | return nil, fmt.Errorf("Output %s not found", name) 32 | } 33 | return output, nil 34 | } 35 | -------------------------------------------------------------------------------- /output/tcp/tcp.go: -------------------------------------------------------------------------------- 1 | package tcp 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "net" 8 | 9 | "github.com/hailocab/logslam/buffer" 10 | "github.com/hailocab/logslam/output" 11 | ) 12 | 13 | const ( 14 | recvBuffer = 100 15 | ) 16 | 17 | type Config struct { 18 | Host string `json:"host"` 19 | } 20 | 21 | type TCPServer struct { 22 | host string 23 | b buffer.Sender 24 | term chan bool 25 | } 26 | 27 | func init() { 28 | output.Register("tcp", &TCPServer{ 29 | term: make(chan bool, 1), 30 | }) 31 | } 32 | 33 | // lumberConn handles an incoming connection from a lumberjack client 34 | func (s *TCPServer) accept(c net.Conn) { 35 | defer func() { 36 | s.b.DelSubscriber(c.RemoteAddr().String()) 37 | log.Printf("[%s] closing tcp connection", c.RemoteAddr().String()) 38 | c.Close() 39 | }() 40 | 41 | log.Printf("[%s] accepting tcp connection", c.RemoteAddr().String()) 42 | 43 | // Add the client as a subscriber 44 | r := make(chan *buffer.Event, recvBuffer) 45 | s.b.AddSubscriber(c.RemoteAddr().String(), r) 46 | 47 | for { 48 | select { 49 | case ev := <-r: 50 | _, err := c.Write([]byte(fmt.Sprintf("%s %s\n", ev.Source, *ev.Text))) 51 | if err != nil { 52 | log.Printf("[%s] error sending event to tcp connection: %v", c.RemoteAddr().String(), err) 53 | return 54 | } 55 | } 56 | } 57 | 58 | } 59 | 60 | func (s *TCPServer) Init(config json.RawMessage, b buffer.Sender) error { 61 | var tcpConfig *Config 62 | if err := json.Unmarshal(config, &tcpConfig); err != nil { 63 | return fmt.Errorf("Error parsing tcp config: %v", err) 64 | } 65 | 66 | s.host = tcpConfig.Host 67 | s.b = b 68 | return nil 69 | } 70 | 71 | func (s *TCPServer) Start() error { 72 | ln, err := net.Listen("tcp", s.host) 73 | if err != nil { 74 | return fmt.Errorf("TCPServer: listener failed: %v", err) 75 | } 76 | 77 | for { 78 | select { 79 | case <-s.term: 80 | log.Println("TCPServer received term signal") 81 | return nil 82 | default: 83 | conn, err := ln.Accept() 84 | if err != nil { 85 | log.Println("Error accepting tcp connection: %v", err) 86 | continue 87 | } 88 | go s.accept(conn) 89 | } 90 | } 91 | 92 | return nil 93 | } 94 | 95 | func (s *TCPServer) Stop() error { 96 | s.term <- true 97 | return nil 98 | } 99 | -------------------------------------------------------------------------------- /output/websocket/template.go: -------------------------------------------------------------------------------- 1 | package websocket 2 | 3 | var index = ` 4 | 5 | 6 | 7 | 8 | Logs 9 | 19 | 20 | 21 |

Log Files

22 | 27 | 28 | 29 | ` 30 | var logs = ` 31 | 32 | 33 | 34 | 35 | {{.Source}} 36 | 47 | 48 | 58 | 59 | 60 |

{{.Source}}

61 | 62 | 63 | 64 | ` 65 | -------------------------------------------------------------------------------- /output/websocket/websocket.go: -------------------------------------------------------------------------------- 1 | package websocket 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "sync" 9 | "text/template" 10 | "time" 11 | 12 | "code.google.com/p/go.net/websocket" 13 | "github.com/hailocab/logslam/buffer" 14 | "github.com/hailocab/logslam/output" 15 | ) 16 | 17 | const ( 18 | recvBuffer = 100 19 | ) 20 | 21 | type Config struct { 22 | Host string `json:"host"` 23 | } 24 | 25 | type WebSocketServer struct { 26 | host string 27 | b buffer.Sender 28 | term chan bool 29 | 30 | mtx sync.RWMutex 31 | logs map[string]time.Time 32 | } 33 | 34 | var ( 35 | indexTemplate, _ = template.New("index").Parse(index) 36 | logsTemplate, _ = template.New("logs").Parse(logs) 37 | ) 38 | 39 | func init() { 40 | output.Register("websocket", &WebSocketServer{ 41 | logs: make(map[string]time.Time), 42 | term: make(chan bool, 1), 43 | }) 44 | } 45 | 46 | func (ws *WebSocketServer) wslogsHandler(w *websocket.Conn) { 47 | source := w.Request().FormValue("source") 48 | host := fmt.Sprintf("%s/%d", w.RemoteAddr().String(), time.Now().UnixNano()) 49 | 50 | defer func() { 51 | log.Printf("[%s] closing websocket conn", w.RemoteAddr().String()) 52 | ws.b.DelSubscriber(host) 53 | w.Close() 54 | }() 55 | 56 | log.Printf("[%s] accepting websocket conn", w.RemoteAddr().String()) 57 | 58 | r := make(chan *buffer.Event, recvBuffer) 59 | ws.b.AddSubscriber(host, r) 60 | 61 | for { 62 | select { 63 | case ev := <-r: 64 | if len(source) > 0 { 65 | if ev.Source != source { 66 | continue 67 | } 68 | } 69 | 70 | err := websocket.Message.Send(w, *ev.Text) 71 | if err != nil { 72 | log.Printf("[%s] error sending ws message: %v", w.RemoteAddr().String(), err.Error()) 73 | return 74 | } 75 | } 76 | } 77 | } 78 | 79 | func (ws *WebSocketServer) logsHandler(w http.ResponseWriter, r *http.Request) { 80 | source := "*" 81 | host := fmt.Sprintf("ws://%s/wslogs", r.Host) 82 | 83 | if src := r.FormValue("source"); len(src) > 0 { 84 | source = src 85 | host = fmt.Sprintf("%s?source=%s", host, src) 86 | } 87 | 88 | logsTemplate.Execute(w, struct{ Source, Server string }{source, host}) 89 | } 90 | 91 | func (ws *WebSocketServer) indexHandler(w http.ResponseWriter, r *http.Request) { 92 | ws.mtx.RLock() 93 | defer ws.mtx.RUnlock() 94 | indexTemplate.Execute(w, ws.logs) 95 | } 96 | 97 | func (ws *WebSocketServer) logListMaintainer() { 98 | defer func() { 99 | ws.b.DelSubscriber("logList") 100 | }() 101 | 102 | r := make(chan *buffer.Event, recvBuffer) 103 | ws.b.AddSubscriber("logList", r) 104 | 105 | ticker := time.NewTicker(time.Duration(600) * time.Second) 106 | 107 | for { 108 | select { 109 | case ev := <-r: 110 | ws.mtx.Lock() 111 | ws.logs[ev.Source] = time.Now() 112 | ws.mtx.Unlock() 113 | case <-ticker.C: 114 | t := time.Now() 115 | ws.mtx.Lock() 116 | for log, ttl := range ws.logs { 117 | if t.Sub(ttl).Seconds() > 600 { 118 | delete(ws.logs, log) 119 | } 120 | } 121 | ws.mtx.Unlock() 122 | } 123 | } 124 | } 125 | 126 | func (ws *WebSocketServer) Init(config json.RawMessage, b buffer.Sender) error { 127 | var wsConfig *Config 128 | if err := json.Unmarshal(config, &wsConfig); err != nil { 129 | return fmt.Errorf("Error parsing websocket config: %v", err) 130 | } 131 | 132 | ws.host = wsConfig.Host 133 | ws.b = b 134 | return nil 135 | } 136 | 137 | func (ws *WebSocketServer) Start() error { 138 | http.Handle("/wslogs", websocket.Handler(ws.wslogsHandler)) 139 | http.HandleFunc("/logs", ws.logsHandler) 140 | http.HandleFunc("/", ws.indexHandler) 141 | 142 | go ws.logListMaintainer() 143 | 144 | err := http.ListenAndServe(ws.host, nil) 145 | if err != nil { 146 | return fmt.Errorf("Error starting websocket server: %v", err) 147 | } 148 | 149 | return nil 150 | } 151 | 152 | func (ws *WebSocketServer) Stop() error { 153 | ws.term <- true 154 | return nil 155 | } 156 | -------------------------------------------------------------------------------- /parser/parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "bytes" 5 | "compress/zlib" 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | "log" 10 | "net" 11 | "strconv" 12 | "time" 13 | 14 | "github.com/hailocab/logslam/buffer" 15 | ) 16 | 17 | const ( 18 | ack = "ACKMSG" 19 | maxKeyLen = 100 * 1024 * 1024 // 100 mb 20 | maxValueLen = 250 * 1024 * 1024 // 250 mb 21 | ) 22 | 23 | type Receiver interface { 24 | Send(*buffer.Event) 25 | } 26 | 27 | type Parser struct { 28 | Conn net.Conn 29 | Recv Receiver 30 | wlen, plen uint32 31 | buffer io.Reader 32 | } 33 | 34 | func New(c net.Conn, r Receiver) *Parser { 35 | return &Parser{ 36 | Conn: c, 37 | Recv: r, 38 | } 39 | } 40 | 41 | // ack acknowledges that the payload was received successfully 42 | func (p *Parser) ack() error { 43 | if _, err := p.Conn.Write([]byte(ack)); err != nil { 44 | return err 45 | } 46 | 47 | return nil 48 | } 49 | 50 | // readKV parses key value pairs from within the payload 51 | func (p *Parser) readKV() ([]byte, []byte, error) { 52 | var klen, vlen uint32 53 | 54 | // Read key len 55 | binary.Read(p.buffer, binary.BigEndian, &klen) 56 | 57 | if klen > maxKeyLen { 58 | return nil, nil, fmt.Errorf("key exceeds max len %d, got %d bytes", maxKeyLen, klen) 59 | } 60 | 61 | // Read key 62 | key := make([]byte, klen) 63 | _, err := p.buffer.Read(key) 64 | if err != nil { 65 | return nil, nil, err 66 | } 67 | 68 | // Read value len 69 | binary.Read(p.buffer, binary.BigEndian, &vlen) 70 | if vlen > maxValueLen { 71 | return nil, nil, fmt.Errorf("value exceeds max len %d, got %d bytes", maxValueLen, vlen) 72 | } 73 | 74 | // Read value 75 | value := make([]byte, vlen) 76 | _, err = p.buffer.Read(value) 77 | if err != nil { 78 | return nil, nil, err 79 | } 80 | 81 | return key, value, nil 82 | } 83 | 84 | // read parses the compressed data frame 85 | func (p *Parser) read() error { 86 | var s, f uint32 87 | var k, v []byte 88 | var err error 89 | 90 | r, err := zlib.NewReader(p.Conn) 91 | if err != nil { 92 | return err 93 | } 94 | defer r.Close() 95 | 96 | // Decompress 97 | buff := new(bytes.Buffer) 98 | io.Copy(buff, r) 99 | p.buffer = buff 100 | 101 | b := make([]byte, 2) 102 | for i := uint32(0); i < p.wlen; i++ { 103 | n, err := buff.Read(b) 104 | if err == io.EOF { 105 | return err 106 | } 107 | 108 | if n == 0 { 109 | continue 110 | } 111 | 112 | switch string(b) { 113 | case "1D": // window size 114 | binary.Read(buff, binary.BigEndian, &s) 115 | binary.Read(buff, binary.BigEndian, &f) 116 | 117 | var ev buffer.Event 118 | fields := make(map[string]string) 119 | fields["timestamp"] = time.Now().Format(time.RFC3339Nano) 120 | 121 | for j := uint32(0); j < f; j++ { 122 | if k, v, err = p.readKV(); err != nil { 123 | return err 124 | } 125 | fields[string(k)] = string(v) 126 | } 127 | 128 | ev.Source = fmt.Sprintf("lumberjack://%s%s", fields["host"], fields["file"]) 129 | ev.Offset, _ = strconv.ParseInt(fields["offset"], 10, 64) 130 | ev.Line = uint64(s) 131 | t := fields["line"] 132 | ev.Text = &t 133 | ev.Fields = &fields 134 | 135 | // Send to the receiver which is a buffer. We block because... 136 | p.Recv.Send(&ev) 137 | default: 138 | return fmt.Errorf("unknown type") 139 | } 140 | } 141 | 142 | return nil 143 | } 144 | 145 | // Parse initialises the read loop and begins parsing the incoming request 146 | func (p *Parser) Parse() { 147 | b := make([]byte, 2) 148 | 149 | Read: 150 | for { 151 | n, err := p.Conn.Read(b) 152 | switch { 153 | case err == io.EOF: 154 | break Read 155 | case n == 0: 156 | goto Read 157 | } 158 | 159 | switch string(b) { 160 | case "1W": // window length 161 | binary.Read(p.Conn, binary.BigEndian, &p.wlen) 162 | case "1C": // frame length 163 | binary.Read(p.Conn, binary.BigEndian, &p.plen) 164 | if err := p.read(); err != nil { 165 | log.Printf("[%s] error parsing %v", p.Conn.RemoteAddr().String(), err) 166 | break Read 167 | } 168 | 169 | if err := p.ack(); err != nil { 170 | log.Printf("[%s] error acking %v", p.Conn.RemoteAddr().String(), err) 171 | break Read 172 | } 173 | default: 174 | // This really shouldn't happen 175 | log.Printf("[%s] Received unknown type", p.Conn.RemoteAddr().String(), err) 176 | break Read 177 | } 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /server/config.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | ) 8 | 9 | type Config struct { 10 | Inputs map[string]json.RawMessage `json:"inputs"` 11 | Outputs map[string]json.RawMessage `json:"outputs"` 12 | } 13 | 14 | func LoadConfig(file string) (*Config, error) { 15 | b, err := ioutil.ReadFile(file) 16 | if err != nil { 17 | return nil, fmt.Errorf("Could not read config file %s: %v", file, err) 18 | } 19 | 20 | var conf *Config 21 | err = json.Unmarshal(b, &conf) 22 | if err != nil { 23 | return nil, fmt.Errorf("Failed to parse config %s: %v", file, err) 24 | } 25 | 26 | return conf, nil 27 | } 28 | -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "os/signal" 7 | "sync" 8 | "syscall" 9 | 10 | "github.com/hailocab/logslam/buffer" 11 | "github.com/hailocab/logslam/input" 12 | "github.com/hailocab/logslam/output" 13 | ) 14 | 15 | type Server struct { 16 | Config *Config 17 | Buffer *buffer.Buffer 18 | 19 | mtx sync.Mutex 20 | inputs map[string]input.Input 21 | outputs map[string]output.Output 22 | } 23 | 24 | func signalCatcher() chan os.Signal { 25 | c := make(chan os.Signal, 1) 26 | signal.Notify(c, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) 27 | return c 28 | } 29 | 30 | func New(configFile string) (*Server, error) { 31 | config, err := LoadConfig(configFile) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | return &Server{ 37 | Config: config, 38 | Buffer: buffer.New(), 39 | inputs: make(map[string]input.Input), 40 | outputs: make(map[string]output.Output), 41 | }, nil 42 | } 43 | 44 | func (s *Server) Start() { 45 | log.Println("Starting server") 46 | 47 | // Start buffer 48 | log.Println("Starting buffer") 49 | go s.Buffer.Start() 50 | 51 | s.mtx.Lock() 52 | 53 | // Start inputs 54 | for name, config := range s.Config.Inputs { 55 | in, err := input.Load(name) 56 | if err != nil { 57 | log.Println(err.Error) 58 | continue 59 | } 60 | 61 | if err := in.Init(config, s.Buffer); err != nil { 62 | log.Printf("Failed to init %s input: %v", name, err) 63 | continue 64 | } 65 | 66 | go func(name string, in input.Input) { 67 | log.Printf("Starting input %s", name) 68 | if err := in.Start(); err != nil { 69 | log.Printf("Error starting input %s: %v", name, err) 70 | } 71 | }(name, in) 72 | 73 | s.inputs[name] = in 74 | } 75 | 76 | // Start ouputs 77 | for name, config := range s.Config.Outputs { 78 | out, err := output.Load(name) 79 | if err != nil { 80 | log.Println(err.Error) 81 | continue 82 | } 83 | 84 | if err := out.Init(config, s.Buffer); err != nil { 85 | log.Printf("Failed to init %s output: %v", name, err) 86 | continue 87 | } 88 | 89 | go func(name string, out output.Output) { 90 | log.Printf("Starting output %s", name) 91 | if err := out.Start(); err != nil { 92 | log.Printf("Error starting output %s: %v", name, err) 93 | } 94 | }(name, out) 95 | 96 | s.outputs[name] = out 97 | } 98 | 99 | s.mtx.Unlock() 100 | 101 | // Wait for kill signal 102 | <-signalCatcher() 103 | log.Printf("Received quit signal") 104 | 105 | // Stop Server 106 | s.Stop() 107 | } 108 | 109 | func (s *Server) Stop() { 110 | log.Println("Stopping server") 111 | 112 | s.mtx.Lock() 113 | 114 | // stop inputs 115 | for name, in := range s.inputs { 116 | log.Printf("Stopping input %s", name) 117 | if err := in.Stop(); err != nil { 118 | log.Printf("Error stopping %s input: %v", name, err) 119 | } 120 | } 121 | 122 | // stop ouputs 123 | for name, out := range s.outputs { 124 | log.Printf("Stopping output %s", name) 125 | if err := out.Stop(); err != nil { 126 | log.Printf("Error stopping %s output: %v", name, err) 127 | } 128 | } 129 | 130 | s.mtx.Unlock() 131 | 132 | log.Println("Stopping buffer") 133 | s.Buffer.Stop() 134 | } 135 | --------------------------------------------------------------------------------