├── example
    ├── Procfile
    ├── config
    │   └── fluentd.conf
    ├── README.md
    └── server.go
├── message.go
├── LICENSE
├── README.md
├── config.go
├── config_test.go
└── logger.go


/example/Procfile:
--------------------------------------------------------------------------------
1 | fluent: fluentd -c config/fluentd.conf
2 | gin: go run server.go
3 | 


--------------------------------------------------------------------------------
/example/config/fluentd.conf:
--------------------------------------------------------------------------------
1 | <source>
2 | type forward
3 | port 24224
4 | </source>
5 | 
6 | <match **.*>
7 | type stdout
8 | </match>
9 | 


--------------------------------------------------------------------------------
/example/README.md:
--------------------------------------------------------------------------------
 1 | # Example
 2 | 
 3 | Sample application for fluent-logger-go using [gin](https://github.com/gin-gonic/gin).
 4 | 
 5 | ## Setup
 6 | 
 7 | ```
 8 | $ go get github.com/gin-gonic/gin
 9 | $ gem install fluentd
10 | $ gem install foreman
11 | ```
12 | 
13 | ## Launch
14 | 
15 | ```
16 | $ foreman start
17 | ```
18 | 
19 | If you visit http://localhost:3000/foo, { "id": "foo" } is logged to fluentd.
20 | 


--------------------------------------------------------------------------------
/message.go:
--------------------------------------------------------------------------------
 1 | package fluent
 2 | 
 3 | import (
 4 | 	"time"
 5 | 
 6 | 	"github.com/ugorji/go/codec"
 7 | )
 8 | 
 9 | var (
10 | 	mh codec.MsgpackHandle
11 | )
12 | 
13 | type message struct {
14 | 	tag  string
15 | 	time time.Time
16 | 	data interface{}
17 | }
18 | 
19 | func (m *message) toMsgpack() ([]byte, error) {
20 | 	pack := []byte{}
21 | 	encoder := codec.NewEncoderBytes(&pack, &mh)
22 | 
23 | 	rawMessage := []interface{}{m.tag, m.time.Unix(), m.data}
24 | 	err := encoder.Encode(rawMessage)
25 | 
26 | 	return pack, err
27 | }
28 | 


--------------------------------------------------------------------------------
/example/server.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"github.com/gin-gonic/gin"
 5 | 	"github.com/k0kubun/fluent-logger-go"
 6 | )
 7 | 
 8 | var (
 9 | 	logger *fluent.Logger
10 | )
11 | 
12 | func logId(c *gin.Context) {
13 | 	id := c.Params.ByName("id")
14 | 	logger.Post("logId", map[string]string{"id": id})
15 | 	c.String(200, "OK")
16 | }
17 | 
18 | func main() {
19 | 	logger = fluent.NewLogger(fluent.Config{
20 | 		BufferLength: 3 * 1024,
21 | 	})
22 | 
23 | 	r := gin.Default()
24 | 	r.GET("/:id", logId)
25 | 	r.Run(":3000")
26 | }
27 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2014 Takashi Kokubun
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # fluent-logger-go
 2 | 
 3 | Goroutine based asynchronous event logger for Fluentd
 4 | 
 5 | ## Project status
 6 | 
 7 | Not maintained. Please use [fluent/fluent-logger-golang](https://github.com/fluent/fluent-logger-golang) instead.
 8 | 
 9 | ## Features
10 | 
11 | * Channel based non-blocking logging interface
12 | * Queued events are periodically sent to fluentd altogether
13 | 
14 | ## Installation
15 | 
16 | ```
17 | $ go get github.com/k0kubun/fluent-logger-go
18 | ```
19 | 
20 | ## Usage
21 | 
22 | ```go
23 | package main
24 | 
25 | import "github.com/gin-gonic/gin"
26 | import "github.com/k0kubun/fluent-logger-go"
27 | 
28 | var logger *fluent.Logger
29 | 
30 | func logId(c *gin.Context) {
31 | 	id := c.Params.ByName("id")
32 | 	logger.Post("idlog", map[string]string{"id": id})
33 | 	c.String(200, "Logged id")
34 | }
35 | 
36 | func main() {
37 | 	logger = fluent.NewLogger(fluent.Config{})
38 | 	r := gin.Default()
39 | 	r.GET("/:id", logId)
40 | 	r.Run(":3000")
41 | }
42 | ```
43 | 
44 | ### Documentation
45 | 
46 | API documentation can be found here: https://godoc.org/github.com/k0kubun/fluent-logger-go
47 | 
48 | ## Related projects
49 | 
50 | You would like to use another one for some use case.
51 | 
52 | * [t-k/fluent-logger-golang](https://github.com/t-k/fluent-logger-golang)
53 | 


--------------------------------------------------------------------------------
/config.go:
--------------------------------------------------------------------------------
 1 | package fluent
 2 | 
 3 | import "time"
 4 | 
 5 | const (
 6 | 	DefaultFluentHost            = "127.0.0.1"
 7 | 	DefaultFluentPort            = 24224
 8 | 	DefaultChannelLength         = 1000
 9 | 	DefaultBufferLength          = 10 * 1024
10 | 	DefaultMaxTrialForConnection = 10
11 | 	DefaultConnectionTimeout     = 3 * time.Second
12 | 	DefaultBufferingTimeout      = 100 * time.Millisecond
13 | 	DefaultTagPrefix             = ""
14 | )
15 | 
16 | var (
17 | 	intDefault      int
18 | 	stringDefault   string
19 | 	durationDefault time.Duration
20 | )
21 | 
22 | // Config is just for fluent.NewLogger() argument.
23 | type Config struct {
24 | 	// You can customize fluentd host and port.
25 | 	FluentHost string
26 | 	FluentPort int
27 | 
28 | 	// If buffered channel's length is equal to ChannelLength, main thread blocks.
29 | 	ChannelLength int
30 | 
31 | 	// If all posted messages' size reaches BufferLength, logger flushes all logs.
32 | 	BufferLength int
33 | 
34 | 	// Retry connection with fluentd MaxTrialForConnection times.
35 | 	MaxTrialForConnection int
36 | 
37 | 	// Wait for connection until ConnectionTimeout.
38 | 	ConnectionTimeout time.Duration
39 | 
40 | 	// Logger flushes its buffer on each BufferingTimeout interval.
41 | 	BufferingTimeout time.Duration
42 | 
43 | 	// Tag prefix. When set TagPrefix to "foo" and post with "bar.baz",
44 | 	// you'll get "foo.bar.baz" tag.
45 | 	TagPrefix string
46 | }
47 | 
48 | func (c *Config) applyDefaultValues() {
49 | 	assignIfDefault(&c.FluentHost, DefaultFluentHost)
50 | 	assignIfDefault(&c.FluentPort, DefaultFluentPort)
51 | 	assignIfDefault(&c.ChannelLength, DefaultChannelLength)
52 | 	assignIfDefault(&c.BufferLength, DefaultBufferLength)
53 | 	assignIfDefault(&c.MaxTrialForConnection, DefaultMaxTrialForConnection)
54 | 	assignIfDefault(&c.ConnectionTimeout, DefaultConnectionTimeout)
55 | 	assignIfDefault(&c.BufferingTimeout, DefaultBufferingTimeout)
56 | 	assignIfDefault(&c.TagPrefix, DefaultTagPrefix)
57 | }
58 | 
59 | func assignIfDefault(target interface{}, DefaultValue interface{}) {
60 | 	switch target.(type) {
61 | 	case *string:
62 | 		ptr := target.(*string)
63 | 		if *ptr == stringDefault {
64 | 			*ptr = DefaultValue.(string)
65 | 		}
66 | 	case *int:
67 | 		ptr := target.(*int)
68 | 		if *ptr == intDefault {
69 | 			*ptr = DefaultValue.(int)
70 | 		}
71 | 	case *time.Duration:
72 | 		ptr := target.(*time.Duration)
73 | 		if *ptr == durationDefault {
74 | 			*ptr = DefaultValue.(time.Duration)
75 | 		}
76 | 	}
77 | }
78 | 


--------------------------------------------------------------------------------
/config_test.go:
--------------------------------------------------------------------------------
 1 | package fluent
 2 | 
 3 | import (
 4 | 	"testing"
 5 | 	"time"
 6 | )
 7 | 
 8 | func TestApplyDefaultValues(t *testing.T) {
 9 | 	config := Config{}
10 | 
11 | 	assertEqual(t, config.FluentHost, stringDefault)
12 | 	assertEqual(t, config.FluentPort, intDefault)
13 | 	assertEqual(t, config.ChannelLength, intDefault)
14 | 	assertEqual(t, config.BufferLength, intDefault)
15 | 	assertEqual(t, config.MaxTrialForConnection, intDefault)
16 | 	assertEqual(t, config.ConnectionTimeout, durationDefault)
17 | 	assertEqual(t, config.BufferingTimeout, durationDefault)
18 | 	assertEqual(t, config.TagPrefix, stringDefault)
19 | 
20 | 	config.applyDefaultValues()
21 | 	assertEqual(t, config.FluentHost, DefaultFluentHost)
22 | 	assertEqual(t, config.FluentPort, DefaultFluentPort)
23 | 	assertEqual(t, config.ChannelLength, DefaultChannelLength)
24 | 	assertEqual(t, config.BufferLength, DefaultBufferLength)
25 | 	assertEqual(t, config.MaxTrialForConnection, DefaultMaxTrialForConnection)
26 | 	assertEqual(t, config.ConnectionTimeout, DefaultConnectionTimeout)
27 | 	assertEqual(t, config.BufferingTimeout, DefaultBufferingTimeout)
28 | 	assertEqual(t, config.TagPrefix, DefaultTagPrefix)
29 | 
30 | 	config = Config{
31 | 		FluentHost:            "localhost",
32 | 		FluentPort:            80,
33 | 		ChannelLength:         1,
34 | 		BufferLength:          2,
35 | 		MaxTrialForConnection: 3,
36 | 		ConnectionTimeout:     2 * time.Second,
37 | 		BufferingTimeout:      2 * time.Second,
38 | 		TagPrefix:             "prefix",
39 | 	}
40 | 
41 | 	config.applyDefaultValues()
42 | 	assertEqual(t, config.FluentHost, "localhost")
43 | 	assertEqual(t, config.FluentPort, 80)
44 | 	assertEqual(t, config.ChannelLength, 1)
45 | 	assertEqual(t, config.BufferLength, 2)
46 | 	assertEqual(t, config.MaxTrialForConnection, 3)
47 | 	assertEqual(t, config.ConnectionTimeout, 2*time.Second)
48 | 	assertEqual(t, config.BufferingTimeout, 2*time.Second)
49 | 	assertEqual(t, config.TagPrefix, "prefix")
50 | }
51 | 
52 | func assertEqual(t *testing.T, actual interface{}, expect interface{}) {
53 | 	switch actual.(type) {
54 | 	case string:
55 | 		if actual.(string) != expect.(string) {
56 | 			t.Errorf("expected %s, but got %s\n", expect.(string), actual.(string))
57 | 		}
58 | 	case int:
59 | 		if actual.(int) != expect.(int) {
60 | 			t.Errorf("expected %d, but got %d\n", expect.(int), actual.(int))
61 | 		}
62 | 	case time.Duration:
63 | 		if actual.(time.Duration) != expect.(time.Duration) {
64 | 			t.Errorf("expected %d, but got %d\n", expect.(time.Duration), actual.(time.Duration))
65 | 		}
66 | 	}
67 | }
68 | 


--------------------------------------------------------------------------------
/logger.go:
--------------------------------------------------------------------------------
  1 | package fluent
  2 | 
  3 | import (
  4 | 	"errors"
  5 | 	"log"
  6 | 	"net"
  7 | 	"strconv"
  8 | 	"time"
  9 | )
 10 | 
 11 | // Logger owns asynchronous logging to fluentd.
 12 | type Logger struct {
 13 | 	config   Config
 14 | 	postCh   chan message
 15 | 	buffer   []byte
 16 | 	conn     net.Conn
 17 | 	ticker   *time.Ticker
 18 | 	logError bool
 19 | }
 20 | 
 21 | // NewLogger() launches a goroutine to log and returns logger.
 22 | // Logger has a channel to interact with the goroutine.
 23 | func NewLogger(config Config) *Logger {
 24 | 	config.applyDefaultValues()
 25 | 
 26 | 	logger := &Logger{
 27 | 		config:   config,
 28 | 		postCh:   make(chan message, config.ChannelLength),
 29 | 		buffer:   make([]byte, 0, config.BufferLength),
 30 | 		ticker:   time.NewTicker(config.BufferingTimeout),
 31 | 		logError: true,
 32 | 	}
 33 | 	logger.connect()
 34 | 	go logger.loop()
 35 | 
 36 | 	return logger
 37 | }
 38 | 
 39 | // You can send message to logger's goroutine via channel.
 40 | // This logging is executed asynchronously.
 41 | func (l *Logger) Post(tag string, data interface{}) {
 42 | 	tag = l.prependTagPrefix(tag)
 43 | 	l.postCh <- message{tag: tag, time: time.Now(), data: data}
 44 | }
 45 | 
 46 | // You can send message immediately to fluentd.
 47 | func (l *Logger) Log(tag string, data interface{}) error {
 48 | 	tag = l.prependTagPrefix(tag)
 49 | 	msg := &message{tag: tag, time: time.Now(), data: data}
 50 | 	pack, err := msg.toMsgpack()
 51 | 	if err != nil {
 52 | 		return err
 53 | 	}
 54 | 
 55 | 	l.buffer = append(l.buffer, pack...)
 56 | 	return l.sendMessage()
 57 | }
 58 | 
 59 | func (l *Logger) loop() {
 60 | 	for {
 61 | 		select {
 62 | 		case msg := <-l.postCh:
 63 | 			pack, err := msg.toMsgpack()
 64 | 			if err != nil {
 65 | 				log.Printf("message pack dump error: " + err.Error())
 66 | 				continue
 67 | 			}
 68 | 
 69 | 			l.buffer = append(l.buffer, pack...)
 70 | 			if len(l.buffer) > l.config.BufferLength {
 71 | 				l.sendMessage()
 72 | 			}
 73 | 		case <-l.ticker.C:
 74 | 			l.sendMessage()
 75 | 		}
 76 | 	}
 77 | }
 78 | 
 79 | func (l *Logger) sendMessage() error {
 80 | 	if len(l.buffer) == 0 {
 81 | 		return errors.New("Buffer is empty")
 82 | 	}
 83 | 
 84 | 	l.connect()
 85 | 	if l.conn == nil {
 86 | 		return errors.New("Failed to establish connection with fluentd")
 87 | 	}
 88 | 
 89 | 	_, err := l.conn.Write(l.buffer)
 90 | 
 91 | 	if err == nil {
 92 | 		l.buffer = l.buffer[0:0]
 93 | 	} else {
 94 | 		log.Printf("failed to send message: " + err.Error())
 95 | 		l.conn.Close()
 96 | 		l.conn = nil
 97 | 	}
 98 | 	return err
 99 | }
100 | 
101 | func (l *Logger) connect() {
102 | 	if l.conn != nil {
103 | 		return
104 | 	}
105 | 
106 | 	var err error
107 | 	for i := 0; i < l.config.MaxTrialForConnection; i++ {
108 | 		l.conn, err = net.DialTimeout(
109 | 			"tcp",
110 | 			l.config.FluentHost+":"+strconv.Itoa(l.config.FluentPort),
111 | 			l.config.ConnectionTimeout,
112 | 		)
113 | 
114 | 		if err == nil {
115 | 			l.logError = true
116 | 			return
117 | 		}
118 | 	}
119 | 
120 | 	if l.logError {
121 | 		log.Printf("failed to establish connection with fluentd: " + err.Error())
122 | 		l.logError = false
123 | 	}
124 | }
125 | 
126 | func (l *Logger) prependTagPrefix(tag string) string {
127 | 	if l.config.TagPrefix != "" {
128 | 		tag = l.config.TagPrefix + "." + tag
129 | 	}
130 | 	return tag
131 | }
132 | 


--------------------------------------------------------------------------------