├── LICENSE ├── Makefile ├── README.md ├── config.go ├── config.sample.json ├── entry.go ├── main.go ├── receiver.go ├── shipper.go └── shipper_test.go /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 GoCardless 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION=0.1.0 2 | 3 | .PHONY: build rpm deb 4 | 5 | build: 6 | go build -o logjam *.go 7 | 8 | rpm deb: build 9 | fpm -s dir -t $@ -n logjam -v $(VERSION) \ 10 | --description 'a log shipping tool' \ 11 | --maintainer 'GoCardless Engineering ' \ 12 | --url 'https://github.com/gocardless/logjam' \ 13 | logjam=/usr/bin/logjam 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Logjam 2 | ------ 3 | 4 | Logjam is a log forwarder designed to listen on a local port, receive log entries over UDP, and forward 5 | these messages on to a log collection server (such as logstash). 6 | 7 | The motivation for logjam was a move to containerising our applications, and a need to get logs from these 8 | applications out of the containers. We configure logjam to listen on the `docker0` (172.16.42.1) interface which is 9 | accessible to applications running within docker. 10 | 11 | Logjam supports collecting logs using the following methods: 12 | 13 | - UDP Socket 14 | - File 15 | 16 | Logjam will pipe all entries as a JSON object terminated with "\n" to a remote server. 17 | 18 | Usage 19 | ----- 20 | 21 | ### Config File 22 | 23 | { 24 | "bind": "127.0.0.1", // interface on host to bind (0.0.0.0 for all) 25 | "port": 1470, // port to listen on locally 26 | "server": "10.1.1.10:1470", // logstash server to forward to 27 | "buffer": "/tmp/buffer.log", // file to use for on-disk buffer 28 | "buffer_size": 1024, // entries to keep in memory buffer 29 | "truncate": 3600 // clean out disk buffer every x seconds 30 | } 31 | 32 | ### Execution 33 | 34 | $ logjam --config config.json 35 | 36 | Pipeline 37 | -------- 38 | 39 | ![logjam pipeline](https://i.imgur.com/4e5sowg.png) 40 | 41 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | ) 7 | 8 | type Config struct { 9 | Bind string `json:"bind"` // interface to bind to (0.0.0.0 for all) 10 | Port int `json:"port"` // listen port 11 | Server string `json:"server"` // remote server to publish logs to 12 | DiskBufferPath string `json:"buffer"` // path for disk buffer 13 | BufferSize int `json:"buffer_size"` // queue length of memory bufer 14 | TruncatePeriod int `json:"truncate"` // cleanup buffer every x seconds 15 | Files []string `json:"files"` // files to include in publish 16 | } 17 | 18 | // read config file 19 | func ReadConfigFile(path string) (*Config, error) { 20 | file, err := ioutil.ReadFile(path) 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | var cfg Config 26 | err = json.Unmarshal(file, &cfg) 27 | return &cfg, err 28 | } 29 | -------------------------------------------------------------------------------- /config.sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "bind": "127.0.0.1", 3 | "port": 1470, 4 | "server": "127.0.0.1:1471", 5 | "buffer": "/tmp/buffer.log", 6 | "buffer_size": 1024, 7 | "truncate": 3600 8 | } 9 | -------------------------------------------------------------------------------- /entry.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | ) 7 | 8 | var Hostname string = "unknown" 9 | 10 | type Entry struct { 11 | Host string `json:"host"` 12 | Message string `json:"message"` 13 | } 14 | 15 | func (e *Entry) ToJSON() []byte { 16 | e.Host = Hostname 17 | dump, _ := json.Marshal(e) 18 | return dump 19 | } 20 | 21 | func init() { 22 | Hostname, _ = os.Hostname() 23 | } 24 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | "time" 7 | ) 8 | 9 | var ( 10 | ConfigPath = flag.String("config", "/etc/log-stream.json", "Config File Path") 11 | ) 12 | 13 | func main() { 14 | flag.Parse() 15 | 16 | config, err := ReadConfigFile(*ConfigPath) 17 | if err != nil { 18 | log.Fatalf("Config: Error: %s\n", err) 19 | } 20 | 21 | r := NewReceiver(config.Bind, config.Port, config.BufferSize) 22 | 23 | s, err := NewShipper("udp", config.Server) 24 | if err != nil { 25 | log.Fatalf("Shipper: Error: %s\n", err) 26 | } 27 | log.Printf("Shipper: Connected: %s\n", config.Server) 28 | 29 | go r.WriteToFile(config.DiskBufferPath) 30 | go s.Ship(config.DiskBufferPath) 31 | go s.TruncateEvery(config.DiskBufferPath, time.Duration(config.TruncatePeriod)*time.Second) 32 | 33 | // Ship Files 34 | for _, path := range config.Files { 35 | t, err := r.TailFile(path) 36 | if err != nil { 37 | log.Fatalf("Tail: Error: %s\n", err) 38 | } 39 | 40 | go r.ListenToTail(t) 41 | } 42 | 43 | // Ship Socket 44 | err = r.ListenAndServe() 45 | if err != nil { 46 | log.Fatalf("Receiver: Error: %s\n", err) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /receiver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "log" 6 | "net" 7 | "os" 8 | "regexp" 9 | 10 | "github.com/ActiveState/tail" 11 | ) 12 | 13 | var ( 14 | ValidJSON = regexp.MustCompile("\\{.*\\}") 15 | ) 16 | 17 | type Receiver struct { 18 | Host string // listen address 19 | Port int // listen port 20 | 21 | messages chan []byte // incomming messages 22 | } 23 | 24 | // create a new receiver server 25 | func NewReceiver(host string, port, bufferSize int) Receiver { 26 | return Receiver{ 27 | Host: host, 28 | Port: port, 29 | 30 | messages: make(chan []byte, bufferSize), 31 | } 32 | } 33 | 34 | func (r *Receiver) ListenAndServe() error { 35 | addr := net.UDPAddr{Port: r.Port, IP: net.ParseIP(r.Host)} 36 | conn, err := net.ListenUDP("udp", &addr) 37 | if err != nil { 38 | return err 39 | } 40 | defer conn.Close() 41 | 42 | scanner := bufio.NewScanner(conn) 43 | 44 | for scanner.Scan() { 45 | b := scanner.Bytes() 46 | 47 | if !ValidJSON.Match(b) { 48 | log.Printf("Receiver: Error: Invalid Message\n") 49 | continue 50 | } 51 | 52 | r.messages <- b 53 | } 54 | 55 | if err = scanner.Err(); err != nil { 56 | return err 57 | } 58 | return nil 59 | } 60 | 61 | // tail a file on disk 62 | func (r *Receiver) TailFile(path string) (*tail.Tail, error) { 63 | t, err := tail.TailFile(path, tail.Config{Follow: true, ReOpen: true}) 64 | return t, err 65 | } 66 | 67 | // listen for tail events as if they were entries on the network socket 68 | func (r *Receiver) ListenToTail(t *tail.Tail) { 69 | for line := range t.Lines { 70 | m := Entry{Message: line.Text} 71 | r.messages <- m.ToJSON() 72 | } 73 | } 74 | 75 | // write entries on messages channel to filename 76 | func (r *Receiver) WriteToFile(filename string) { 77 | file, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND, 0644) 78 | if err != nil { 79 | log.Printf("Writer: Error: %s\n", err) 80 | return 81 | } 82 | defer file.Close() 83 | 84 | for { 85 | file.Write(<-r.messages) 86 | file.Write([]byte("\n")) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /shipper.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net" 6 | "os" 7 | "time" 8 | 9 | "github.com/ActiveState/tail" 10 | ) 11 | 12 | type Shipper struct { 13 | net.Conn 14 | } 15 | 16 | // create a new shipper client 17 | func NewShipper(proto string, addr string) (*Shipper, error) { 18 | conn, err := net.Dial(proto, addr) 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | return &Shipper{conn}, nil 24 | } 25 | 26 | // write to socket with exponential backoff in milliseconds 27 | func (s *Shipper) WriteWithBackoff(p []byte, initial int) { 28 | var timeout time.Duration = time.Duration(initial) * time.Millisecond 29 | 30 | for { 31 | _, err := s.Write(p) 32 | if err != nil { 33 | timeout = timeout * 2 34 | time.Sleep(timeout) 35 | continue 36 | } 37 | 38 | return 39 | } 40 | } 41 | 42 | // ship entries to remote log server 43 | func (s *Shipper) Ship(filename string) { 44 | t, err := tail.TailFile(filename, tail.Config{Follow: true, ReOpen: true}) 45 | if err != nil { 46 | log.Printf("Shipper: Error: %s\n", err) 47 | return 48 | } 49 | 50 | for line := range t.Lines { 51 | s.WriteWithBackoff([]byte(line.Text), 125) 52 | } 53 | } 54 | 55 | // truncate a file every period 56 | func (s *Shipper) TruncateEvery(filename string, period time.Duration) { 57 | for { 58 | time.Sleep(period) 59 | 60 | file, err := os.OpenFile(filename, os.O_WRONLY|os.O_TRUNC, 0644) 61 | if err != nil { 62 | log.Printf("Shipper: Truncate: Error: %s\n", err) 63 | continue 64 | } 65 | file.Close() 66 | 67 | log.Printf("Shipper: Truncate: %s\n", filename) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /shipper_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "net" 6 | "testing" 7 | ) 8 | 9 | type StubConn struct { 10 | net.Conn 11 | 12 | buffer *bytes.Buffer 13 | } 14 | 15 | func (sc StubConn) Write(p []byte) (int, error) { 16 | return sc.buffer.Write(p) 17 | } 18 | 19 | func TestWriteWithBackoff(t *testing.T) { 20 | conn := StubConn{buffer: new(bytes.Buffer)} 21 | s := Shipper{conn} 22 | 23 | s.WriteWithBackoff([]byte("hello"), 125) 24 | 25 | if bytes.Compare(conn.buffer.Bytes(), []byte("hello")) != 0 { 26 | t.Fatal("Write Mismatch") 27 | } 28 | } 29 | --------------------------------------------------------------------------------