├── .gitignore
├── .gitmodules
├── LICENSE
├── Makefile
├── README.md
├── go.mod
├── go.sum
├── lib
    ├── cloudwatchlogs
    │   ├── client.go
    │   ├── init.go
    │   ├── region.go
    │   └── writer.go
    ├── datadog
    │   ├── init.go
    │   └── writer.go
    ├── destination.go
    ├── error.go
    ├── group.go
    ├── journald
    │   ├── init.go
    │   └── reader.go
    ├── log.go
    ├── logdna
    │   ├── init.go
    │   └── writer.go
    ├── loggly
    │   ├── init.go
    │   └── writer.go
    ├── message.go
    ├── message_test.go
    ├── reader.go
    ├── source.go
    ├── statsd
    │   ├── init.go
    │   ├── writer.go
    │   └── writer_test.go
    ├── store.go
    ├── stream.go
    ├── stream_test.go
    ├── syslog
    │   ├── init.go
    │   ├── pool
    │   │   ├── pool.go
    │   │   └── pool_test.go
    │   ├── writer.go
    │   └── writer_test.go
    └── writer.go
├── main.go
└── main_linux.go


/.gitignore:
--------------------------------------------------------------------------------
 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects)
 2 | *.o
 3 | *.a
 4 | *.so
 5 | 
 6 | # Folders
 7 | _obj
 8 | _test
 9 | vendor/*/
10 | bin/
11 | 
12 | # Architecture specific extensions/prefixes
13 | *.[568vq]
14 | [568vq].out
15 | 
16 | *.cgo1.go
17 | *.cgo2.c
18 | _cgo_defun.c
19 | _cgo_gotypes.go
20 | _cgo_export.*
21 | 
22 | _testmain.go
23 | 
24 | *.exe
25 | *.test
26 | *.prof
27 | 
28 | /ecs-logs
29 | 
30 | # Emacs
31 | *~
32 | 
33 | # Packages
34 | *.deb
35 | 


--------------------------------------------------------------------------------
/.gitmodules:
--------------------------------------------------------------------------------
 1 | [submodule "vendor/github.com/aws/aws-sdk-go"]
 2 | 	path = vendor/github.com/aws/aws-sdk-go
 3 | 	url = https://github.com/aws/aws-sdk-go
 4 | [submodule "vendor/github.com/coreos/go-systemd"]
 5 | 	path = vendor/github.com/coreos/go-systemd
 6 | 	url = https://github.com/coreos/go-systemd
 7 | [submodule "vendor/github.com/statsd/client"]
 8 | 	path = vendor/github.com/statsd/client
 9 | 	url = https://github.com/statsd/client
10 | [submodule "vendor/github.com/statsd/datadog"]
11 | 	path = vendor/github.com/statsd/datadog
12 | 	url = https://github.com/statsd/datadog
13 | [submodule "vendor/github.com/visionmedia/go-debug"]
14 | 	path = vendor/github.com/visionmedia/go-debug
15 | 	url = https://github.com/visionmedia/go-debug
16 | [submodule "vendor/github.com/segmentio/ecs-logs-go"]
17 | 	path = vendor/github.com/segmentio/ecs-logs-go
18 | 	url = https://github.com/segmentio/ecs-logs-go
19 | [submodule "vendor/github.com/apex/log"]
20 | 	path = vendor/github.com/apex/log
21 | 	url = https://github.com/apex/log
22 | [submodule "vendor/github.com/go-ini/ini"]
23 | 	path = vendor/github.com/go-ini/ini
24 | 	url = https://github.com/go-ini/ini
25 | [submodule "vendor/github.com/jmespath/go-jmespath"]
26 | 	path = vendor/github.com/jmespath/go-jmespath
27 | 	url = https://github.com/jmespath/go-jmespath
28 | [submodule "vendor/github.com/coreos/pkg"]
29 | 	path = vendor/github.com/coreos/pkg
30 | 	url = https://github.com/coreos/pkg
31 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2016 Segment
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | VERSION := unmaintained
 2 | LDFLAGS := "-X main.version=$(VERSION)"
 3 | REPO := github.com/segmentio/ecs-logs
 4 | SOURCES := $(git ls-files *.go)
 5 | DOCKER_TAG := segment/ecs-logs:v$(VERSION)
 6 | 
 7 | default: bin/ecs-logs-linux-amd64
 8 | 
 9 | bin/ecs-logs-linux-amd64: $(SOURCES)
10 | 	env GOOS=linux GOARCH=amd64 go build -ldflags $(LDFLAGS) -o $@ $(REPO)
11 | 
12 | vendor:
13 | 	go mod vendor
14 | 
15 | test:
16 | 	go test $(shell go list ./...)
17 | 
18 | image:
19 | 	docker build -t $(DOCKER_TAG) -t segment/ecs-logs:latest .
20 | 
21 | push_image:
22 | 	docker push $(DOCKER_TAG)
23 | 	docker push segment/ecs-logs:latest
24 | 
25 | clean:
26 | 	-rm -f bin/* *.deb
27 | 
28 | .PHONY: test clean deb upload_deb image push_image
29 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # ⚠️ Unmaintained ⚠️
  2 | 
  3 | This repository is unmaintained, but left as a historical relic for any wishing to adapt it. Godspeed!
  4 | 
  5 | # ecs-logs
  6 | ecs-logs was a log forwarder for services ran by ecs-agent.
  7 | 
  8 | ## Quick Start
  9 | 
 10 | The simplest way to use ecs-logs is to run it as a docker container, you'll want
 11 | to configure the ecs-agent to forward services logs to journald first, here's
 12 | how to do it:
 13 | 
 14 | - run ecs-agent with `ECS_AVAILABLE_LOGGING_DRIVERS=["journald"]` set in the
 15 | environment variables.
 16 | 
 17 | - configure the task definitions of your services to use the journald driver by
 18 | adding the following:
 19 | ```js
 20 | "logConfiguration": {
 21 |   "logDriver": "journald",
 22 |   "options": {
 23 |     "tag": "<your service name>"
 24 |   }
 25 | }
 26 | ```
 27 | *Note: The tag is important here since it will be used as the group name for the
 28 | CloudWatch logs*
 29 | 
 30 | Once you have ecs-agent properly configured (you should be able to see your ECS
 31 | services logs in the journal), you can start ecs-logs this way:
 32 | ```
 33 | docker run -t -i -v /run/log/journal:/run/log/journal:ro \
 34 |     segment/ecs-logs:latest -src journald -dst cloudwatchlogs
 35 | ```
 36 | That's it! The services logs should now be showing up in CloudWatch Logs.
 37 | 
 38 | ### Docker Image
 39 | 
 40 | - https://hub.docker.com/r/segment/ecs-logs
 41 | 
 42 | ### Sources
 43 | 
 44 | Sources are log streams from which events are read by ecs-logs and forwarded to
 45 | the destinations.  
 46 | The log events can be JSON formatted with the following structure:
 47 | ```js
 48 | {
 49 |   "level": "<debug | info | notice | warn | error | crit | alert | emerg>",
 50 |   "time": "<iso8601 time representation>",
 51 |   "info": {
 52 |     "host": "<hostname>",
 53 |     "source": "<file:line:function>",
 54 |     "id": "<message id>",
 55 |     "pid": <PID>,
 56 |     "gid": <GID>,
 57 |     "uid": <UID>,
 58 |     "errors": [
 59 |       {
 60 |         "type": "<error type>",
 61 |         "error": "<error message>",
 62 |         "errno": <errno value>,
 63 |         "stack": [...]
 64 |       },
 65 |       ...
 66 |     ]
 67 |   },
 68 |   "data": {
 69 |     ...
 70 |   },
 71 |   "message": "<log message>"
 72 | }
 73 | ```
 74 | All fields are optional and ecs-logs will assume defaults if some are missing.
 75 | 
 76 | - **stdin**
 77 | 
 78 | The default source that ecs-logs uses is *stdin*, in most cases this is not what
 79 | will be used in production but can be useful for development and testing
 80 | purposes.  
 81 | Because no metadata can be passed to log messages read from *stdin*, this source
 82 | expects a to read a stream of JSON-formatted objects with this following
 83 | structure:
 84 | ```js
 85 | {
 86 |   "group": "<log group>",
 87 |   "stream": "<log stream>",
 88 |   "event": {
 89 |     ...
 90 |   }
 91 | }
 92 | ```
 93 | Where *group* and *stream* will be used to identify where the log event belong
 94 | and *event* must be a JSON object with the structure defined above.
 95 | 
 96 | - **journald**
 97 | 
 98 | This journald source is what is usually used for production deployments since
 99 | ECS can be easily configured to send docker containers logs to the journal which
100 | then acts as a centralize logging system on host.
101 | 
102 | The journald source expects to find the *CONTAINER_TAG* and *CONTAINER_NAME*
103 | metadata on log events, which it uses to set the group and stream to which the
104 | log event will be sent.
105 | 
106 | You can override the stream name by setting the `JOURNALD_STREAM_NAME` environment
107 | variable with a different journald metadata field to read the stream name from.
108 | 
109 | The log message can be either plain text or JSON formatted. When ecs-logs fails
110 | to parse a JSON message, either because the content is not JSON or because the
111 | format is not something it understands, it will generate a log event where the
112 | *message* field is set to the full log message, for example:
113 | 
114 | *log message:*
115 | ```
116 | 2016-07-05T09:08:12.123Z - INFO - hello!
117 | ```
118 | *log event:*
119 | ```js
120 | {
121 |   "level": "NONE",
122 |   "time": "<time at which the message was sent to the journal>",
123 |   "info": {
124 |     "host": "<hostname>"
125 |   },
126 |   "data": { },
127 |   "message": "2016-07-05T09:08:12.123Z - INFO - hello!"
128 | }
129 | ```
130 | 
131 | ### Usage on OSX
132 | 
133 | If you're developing on OSX it may be inconvenient to not have the system
134 | journal available for testing. One way that this can be worked around is using
135 | the *stdin* source and piping your service's logs through [jq](https://stedolan.github.io/jq/)
136 | to pack well formatted messages.  
137 | Here's an example:
138 | ```shell
139 | ... | jq '. | {group: "<group>", stream: "<stream>", event: .}' | ecs-logs -src stdin -dst ...
140 | ```
141 | *Note that it requires your service to output JSON formatted logs with a
142 | structure that ecs-logs recognize.*
143 | 
144 | ### Proxy
145 | 
146 | To send your logs through a proxy, you can set the `HTTP_PROXY`, `HTTPS_PROXY` or `SOCKS_PROXY` environment variable.
147 | 
148 | `NO_PROXY` can also be set to filter networks where the proxy should not be used.
149 | 
150 | `SOCKS_PROXY` should follow the format `host:port`. If incorrect, the variable will be ignored.
151 | 


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
 1 | module github.com/segmentio/ecs-logs
 2 | 
 3 | go 1.13
 4 | 
 5 | require (
 6 | 	github.com/apex/log v0.0.0-20160721172613-2dafa85a923a
 7 | 	github.com/aws/aws-sdk-go v1.2.10
 8 | 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
 9 | 	github.com/coreos/go-systemd v0.0.0-20160728000419-fa8411dcbcba
10 | 	github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf // indirect
11 | 	github.com/go-ini/ini v1.18.0 // indirect
12 | 	github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 // indirect
13 | 	github.com/jpillora/backoff v0.0.0-20170918002102-8eab2debe79d
14 | 	github.com/kr/pretty v0.2.0 // indirect
15 | 	github.com/segmentio/ecs-logs-go v0.0.0-20170303021009-2f43d53e6e42
16 | 	github.com/segmentio/jutil v0.0.0-20160802072905-2da69de91201
17 | 	github.com/smartystreets/goconvey v1.6.4 // indirect
18 | 	github.com/statsd/client v0.0.2-0.20140909221056-dd7e89e9895a
19 | 	github.com/statsd/client-interface v0.0.0-20140909221041-d354afbbdb2b // indirect
20 | 	github.com/statsd/datadog v0.0.0-20160116190012-12a17042af0a
21 | 	github.com/stretchr/testify v1.5.1 // indirect
22 | 	github.com/visionmedia/go-debug v0.0.0-20180109164601-bfacf9d8a444 // indirect
23 | 	golang.org/x/net v0.0.0-20190311183353-d8887717615a
24 | )
25 | 


--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
 1 | github.com/apex/log v0.0.0-20160721172613-2dafa85a923a h1:sLu94priuZDMpv9CO1jBlnZCDiUq/8H43JukFS58PsY=
 2 | github.com/apex/log v0.0.0-20160721172613-2dafa85a923a/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY=
 3 | github.com/aws/aws-sdk-go v1.2.10 h1:dwFX88TDeGSjGKrUHPcP9FGDfEZuvhP1MHz0jU2WITY=
 4 | github.com/aws/aws-sdk-go v1.2.10/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k=
 5 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 6 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 7 | github.com/coreos/go-systemd v0.0.0-20160728000419-fa8411dcbcba h1:CaLnY81M6xLZHuBqYYdPwF4UDDD5tT+LEtJ8GH6V2dA=
 8 | github.com/coreos/go-systemd v0.0.0-20160728000419-fa8411dcbcba/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 9 | github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf h1:CAKfRE2YtTUIjjh1bkBtyYFaUT/WmOqsJjgtihT0vMI=
10 | github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
11 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
12 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
13 | github.com/go-ini/ini v1.18.0 h1:HpIzEs0R6Hg8q2WOyOP7zGxjOnwmGP6AS6tpedBuMlM=
14 | github.com/go-ini/ini v1.18.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
15 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
16 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
17 | github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE=
18 | github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
19 | github.com/jpillora/backoff v0.0.0-20170918002102-8eab2debe79d h1:ix3WmphUvN0GDd0DO9MH0v6/5xTv+Xm1bPN+1UJn58k=
20 | github.com/jpillora/backoff v0.0.0-20170918002102-8eab2debe79d/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
21 | github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
22 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
23 | github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
24 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
25 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
26 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
27 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
28 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
29 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
30 | github.com/segmentio/ecs-logs-go v0.0.0-20170303021009-2f43d53e6e42 h1:4XRht19Jhw+5KkpEqavDei9k3V5nCBBFt4/Y9DJzq2M=
31 | github.com/segmentio/ecs-logs-go v0.0.0-20170303021009-2f43d53e6e42/go.mod h1:e7m7JFEKlTTMX3dG86CPzf/1m2PXYG08DMxsgjBg0oQ=
32 | github.com/segmentio/jutil v0.0.0-20160802072905-2da69de91201 h1:lxhcUPVRbVQWx78fhhQ69CmCoAeWVmhxcL1et4mnzz8=
33 | github.com/segmentio/jutil v0.0.0-20160802072905-2da69de91201/go.mod h1:S++rU4fLQerZI/2lL/NxYfTsg2qdyCdMFO6za5i8Egg=
34 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
35 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
36 | github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
37 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
38 | github.com/statsd/client v0.0.2-0.20140909221056-dd7e89e9895a h1:HGWd7e6uedtxsE6isrQCnudFVmNteQL8TGtP4oaZkuQ=
39 | github.com/statsd/client v0.0.2-0.20140909221056-dd7e89e9895a/go.mod h1:JOTpbI2aoG6EcTGG0OqCOs2JKUB/MRv8LhNwVXMPu00=
40 | github.com/statsd/client-interface v0.0.0-20140909221041-d354afbbdb2b h1:roBsfyylCInl9m6YrKX1lgk1trGJSdqpZ/AVMpEoSfE=
41 | github.com/statsd/client-interface v0.0.0-20140909221041-d354afbbdb2b/go.mod h1:t9B0pYbEC7s1j9FN7M69aEPZV0H3YMkjasoXYC0d7WY=
42 | github.com/statsd/datadog v0.0.0-20160116190012-12a17042af0a h1:7cDoUfvetNtWnPFmKrq3v1NviTBO5TIpMCSNaFaCWFY=
43 | github.com/statsd/datadog v0.0.0-20160116190012-12a17042af0a/go.mod h1:+IXbTZka0rLdG+LVc85cj9/oSxBRK99CorsxWX/U09c=
44 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
45 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
46 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
47 | github.com/visionmedia/go-debug v0.0.0-20180109164601-bfacf9d8a444 h1:omAc9LPzvfCMXi9UuEB9gbnSVXvz3Bft2zlKZn5Ww7Y=
48 | github.com/visionmedia/go-debug v0.0.0-20180109164601-bfacf9d8a444/go.mod h1:7f/NuZ7w/RrrDGVKvezeak02MX7QbLs4Njo/I+GPxe0=
49 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
50 | golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
51 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
52 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
53 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
54 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
55 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
56 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
57 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
58 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
59 | 


--------------------------------------------------------------------------------
/lib/cloudwatchlogs/client.go:
--------------------------------------------------------------------------------
  1 | package cloudwatchlogs
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"sync"
  6 | 
  7 | 	"github.com/aws/aws-sdk-go/aws"
  8 | 	"github.com/aws/aws-sdk-go/aws/awserr"
  9 | 	"github.com/aws/aws-sdk-go/aws/session"
 10 | 	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
 11 | 	"github.com/segmentio/ecs-logs/lib"
 12 | )
 13 | 
 14 | type client struct {
 15 | 	cmtx   sync.Mutex
 16 | 	client *cloudwatchlogs.CloudWatchLogs
 17 | 
 18 | 	wmtx    sync.Mutex
 19 | 	writers map[string]*writer
 20 | }
 21 | 
 22 | func newClient() *client {
 23 | 	return &client{
 24 | 		writers: make(map[string]*writer, 100),
 25 | 	}
 26 | }
 27 | 
 28 | func (c *client) Open(group string, stream string) (w lib.Writer, err error) {
 29 | 	var client *cloudwatchlogs.CloudWatchLogs
 30 | 	var token string
 31 | 	var writer = c.get(group, stream)
 32 | 
 33 | 	w = writer
 34 | 
 35 | 	writer.mutex.Lock()
 36 | 	defer writer.mutex.Unlock()
 37 | 
 38 | 	if len(writer.token) != 0 {
 39 | 		// The writer already has a token, this means the log group and streams
 40 | 		// have been created for that writer already.
 41 | 		return
 42 | 	}
 43 | 
 44 | 	if client, err = c.getAwsClient(); err != nil {
 45 | 		return
 46 | 	}
 47 | 
 48 | 	if token, err = createGroupAndStream(client, group, stream); err != nil {
 49 | 		// Creating the log group or stream failed, this writer cannot be used.
 50 | 		c.remove(group, stream)
 51 | 		return
 52 | 	}
 53 | 
 54 | 	writer.token = token
 55 | 	return
 56 | }
 57 | 
 58 | func (c *client) Close(group string, stream string) {
 59 | 	c.remove(group, stream)
 60 | }
 61 | 
 62 | func (c *client) get(group string, stream string) (w *writer) {
 63 | 	key := joinGroupStream(group, stream)
 64 | 	c.wmtx.Lock()
 65 | 
 66 | 	if w = c.writers[key]; w == nil {
 67 | 		w = &writer{
 68 | 			group:  group,
 69 | 			stream: stream,
 70 | 			parent: c,
 71 | 		}
 72 | 		c.writers[key] = w
 73 | 	}
 74 | 
 75 | 	c.wmtx.Unlock()
 76 | 	return
 77 | }
 78 | 
 79 | func (c *client) remove(group string, stream string) {
 80 | 	key := joinGroupStream(group, stream)
 81 | 	c.wmtx.Lock()
 82 | 	delete(c.writers, key)
 83 | 	c.wmtx.Unlock()
 84 | }
 85 | 
 86 | func (c *client) getAwsClient() (client *cloudwatchlogs.CloudWatchLogs, err error) {
 87 | 	c.cmtx.Lock()
 88 | 	defer c.cmtx.Unlock()
 89 | 
 90 | 	if client = c.client; client == nil {
 91 | 		if client, err = openAwsClient(); err != nil {
 92 | 			return
 93 | 		}
 94 | 		c.client = client
 95 | 	}
 96 | 
 97 | 	return
 98 | }
 99 | 
100 | func openAwsClient() (client *cloudwatchlogs.CloudWatchLogs, err error) {
101 | 	var region string
102 | 
103 | 	if region, err = getAwsRegion(); err != nil {
104 | 		return
105 | 	}
106 | 
107 | 	client = cloudwatchlogs.New(session.New(&aws.Config{
108 | 		Region: aws.String(region),
109 | 	}))
110 | 	return
111 | }
112 | 
113 | func createGroupAndStream(client *cloudwatchlogs.CloudWatchLogs, group string, stream string) (token string, err error) {
114 | 	var result *cloudwatchlogs.DescribeLogStreamsOutput
115 | 
116 | 	if _, err := client.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
117 | 		LogGroupName: aws.String(group),
118 | 	}); err != nil && !isAlreadyExists(err) {
119 | 		return "", err
120 | 	}
121 | 
122 | 	_, err = client.CreateLogStream(&cloudwatchlogs.CreateLogStreamInput{
123 | 		LogGroupName:  aws.String(group),
124 | 		LogStreamName: aws.String(stream),
125 | 	})
126 | 	if err == nil {
127 | 		// Log stream successfully created.  No token need be provided.
128 | 		return "", nil
129 | 	} else if !isAlreadyExists(err) {
130 | 		return "", err
131 | 	}
132 | 
133 | 	if result, err = client.DescribeLogStreams(&cloudwatchlogs.DescribeLogStreamsInput{
134 | 		Limit:               aws.Int64(1),
135 | 		LogGroupName:        aws.String(group),
136 | 		LogStreamNamePrefix: aws.String(stream),
137 | 	}); err != nil {
138 | 		if isThrottled(err) {
139 | 			// The documentation says that we can only make 5 calls per second to
140 | 			// this endpoint, but we need the sequence token in order to send events
141 | 			// to streams that already exist.
142 | 			//
143 | 			// If we fail to fetch the stream description we still move on without
144 | 			// a token and let the retry logic around PutLogEvents attempt to handle
145 | 			// the issue.
146 | 			return "", nil
147 | 		}
148 | 		return "", err
149 | 	}
150 | 
151 | 	// This should be an invariant
152 | 	if len(result.LogStreams) == 0 {
153 | 		return "", fmt.Errorf("Assertion failure: Log stream %s: %s not found",
154 | 			group, stream)
155 | 	}
156 | 
157 | 	return aws.StringValue(result.LogStreams[0].UploadSequenceToken), nil
158 | }
159 | 
160 | func joinGroupStream(group string, stream string) string {
161 | 	return group + ":" + stream
162 | }
163 | 
164 | func isAwsErrorCode(err error, code string) bool {
165 | 	if err, ok := err.(awserr.Error); ok && err.Code() == code {
166 | 		return true
167 | 	}
168 | 	return false
169 | }
170 | 
171 | func isAlreadyExists(err error) bool {
172 | 	return isAwsErrorCode(err, "ResourceAlreadyExistsException")
173 | }
174 | 
175 | func isThrottled(err error) bool {
176 | 	return isAwsErrorCode(err, "ThrottlingException")
177 | }
178 | 


--------------------------------------------------------------------------------
/lib/cloudwatchlogs/init.go:
--------------------------------------------------------------------------------
1 | package cloudwatchlogs
2 | 
3 | import "github.com/segmentio/ecs-logs/lib"
4 | 
5 | func init() {
6 | 	lib.RegisterDestination("cloudwatchlogs", newClient())
7 | }
8 | 


--------------------------------------------------------------------------------
/lib/cloudwatchlogs/region.go:
--------------------------------------------------------------------------------
 1 | package cloudwatchlogs
 2 | 
 3 | import (
 4 | 	"encoding/json"
 5 | 	"net/http"
 6 | 	"os"
 7 | 	"sync"
 8 | )
 9 | 
10 | type document struct {
11 | 	Region string `json:"region"`
12 | }
13 | 
14 | func getAwsRegion() (region string, err error) {
15 | 	var res *http.Response
16 | 	var doc document
17 | 
18 | 	if region = __getAwsRegion(); len(region) != 0 {
19 | 		return
20 | 	}
21 | 
22 | 	regmtx.Lock()
23 | 	defer regmtx.Unlock()
24 | 
25 | 	if region = regvar; len(region) != 0 {
26 | 		return
27 | 	}
28 | 
29 | 	if region = os.Getenv("AWS_REGION"); len(region) != 0 {
30 | 		goto saveRegion
31 | 	}
32 | 
33 | 	if region = os.Getenv("AWS_DEFAULT_REGION"); len(region) != 0 {
34 | 		goto saveRegion
35 | 	}
36 | 
37 | 	if res, err = http.Get("http://169.254.169.254/latest/dynamic/instance-identity/document"); err != nil {
38 | 		return
39 | 	}
40 | 	defer res.Body.Close()
41 | 
42 | 	if err = json.NewDecoder(res.Body).Decode(&doc); err != nil {
43 | 		return
44 | 	}
45 | 
46 | 	region = doc.Region
47 | saveRegion:
48 | 	regvar = region
49 | 	return
50 | }
51 | 
52 | func __getAwsRegion() (region string) {
53 | 	regmtx.RLock()
54 | 
55 | 	if len(regvar) != 0 {
56 | 		region = regvar
57 | 	}
58 | 
59 | 	regmtx.RUnlock()
60 | 	return
61 | }
62 | 
63 | var (
64 | 	regmtx sync.RWMutex
65 | 	regvar string
66 | )
67 | 


--------------------------------------------------------------------------------
/lib/cloudwatchlogs/writer.go:
--------------------------------------------------------------------------------
  1 | package cloudwatchlogs
  2 | 
  3 | import (
  4 | 	"errors"
  5 | 	"strings"
  6 | 	"sync"
  7 | 
  8 | 	"github.com/aws/aws-sdk-go/aws"
  9 | 	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
 10 | 	"github.com/segmentio/ecs-logs/lib"
 11 | )
 12 | 
 13 | type writer struct {
 14 | 	mutex  sync.Mutex
 15 | 	group  string
 16 | 	stream string
 17 | 	token  string
 18 | 	parent *client
 19 | }
 20 | 
 21 | func (w *writer) Close() error {
 22 | 	return nil
 23 | }
 24 | 
 25 | func (w *writer) WriteMessage(msg lib.Message) error {
 26 | 	return w.WriteMessageBatch(lib.MessageBatch{msg})
 27 | }
 28 | 
 29 | func (w *writer) WriteMessageBatch(batch lib.MessageBatch) (err error) {
 30 | 	if len(batch) == 0 {
 31 | 		return
 32 | 	}
 33 | 
 34 | 	var token *string
 35 | 	var result *cloudwatchlogs.PutLogEventsOutput
 36 | 	var events = make([]*cloudwatchlogs.InputLogEvent, len(batch))
 37 | 
 38 | 	for i, msg := range batch {
 39 | 		events[i] = &cloudwatchlogs.InputLogEvent{
 40 | 			Message:   aws.String(msg.Event.String()),
 41 | 			Timestamp: aws.Int64(aws.TimeUnixMilli(msg.Event.Time)),
 42 | 		}
 43 | 	}
 44 | 
 45 | 	// Because of the logic imposed by the AWS API we can only submit one upload
 46 | 	// request per log stream at a time due to the sequence token being unique
 47 | 	// and usable only once.
 48 | 	w.mutex.Lock()
 49 | 	defer w.mutex.Unlock()
 50 | 
 51 | 	if w.parent == nil {
 52 | 		// Another goroutine has invalidated this writer, giving up.
 53 | 		err = errInvalidWriter
 54 | 		return
 55 | 	}
 56 | 
 57 | 	if len(w.token) != 0 {
 58 | 		token = aws.String(w.token)
 59 | 	}
 60 | 
 61 | 	for attempt := 1; true; attempt++ {
 62 | 		if result, err = w.parent.client.PutLogEvents(&cloudwatchlogs.PutLogEventsInput{
 63 | 			LogEvents:     events,
 64 | 			LogGroupName:  aws.String(w.group),
 65 | 			LogStreamName: aws.String(w.stream),
 66 | 			SequenceToken: token,
 67 | 		}); err == nil {
 68 | 			break
 69 | 		}
 70 | 
 71 | 		// The AWS Go SDK doesn't expose the error type but does return the
 72 | 		// token in the error message so we attempt to extract it from there
 73 | 		// and let the retry logic resubmit the event batch.
 74 | 		//
 75 | 		// See: https://forums.aws.amazon.com/message.jspa?messageID=676912
 76 | 		if token = parseInvalidSequenceTokenException(err); attempt < 3 && token != nil {
 77 | 			err = nil
 78 | 			continue
 79 | 		}
 80 | 
 81 | 		// See: https://github.com/segmentio/ecs-logs/issues/68
 82 | 		if token = parseDataAlreadyAcceptedException(err); token != nil {
 83 | 			err = nil
 84 | 			w.token = *token
 85 | 			return
 86 | 		}
 87 | 
 88 | 		// The documentation says we have to provide the sequence token when
 89 | 		// uploading events to CloudWatchLogs, if an error is returned here
 90 | 		// it's likely the token we have is either invalid or something worse
 91 | 		// happened.
 92 | 		// We remove the writer from it's parent client so a new writer will
 93 | 		// be created.
 94 | 		w.parent.remove(w.group, w.stream)
 95 | 		w.parent = nil
 96 | 		return
 97 | 	}
 98 | 
 99 | 	w.token = aws.StringValue(result.NextSequenceToken)
100 | 	return
101 | }
102 | 
103 | func getRetryTokenFromMessage(msg string) (token *string) {
104 | 	if lines := strings.Split(msg, "\n"); len(lines) != 0 {
105 | 		msg = lines[0]
106 | 	}
107 | 
108 | 	parts := strings.Split(msg, ":")
109 | 
110 | 	if len(parts) < 3 {
111 | 		return
112 | 	}
113 | 
114 | 	s := strings.TrimSpace(parts[2])
115 | 	token = &s
116 | 	return
117 | }
118 | 
119 | func parseInvalidSequenceTokenException(err error) (token *string) {
120 | 	msg := err.Error()
121 | 
122 | 	if !strings.HasPrefix(msg, "InvalidSequenceTokenException:") {
123 | 		return
124 | 	}
125 | 
126 | 	return getRetryTokenFromMessage(msg)
127 | }
128 | 
129 | func parseDataAlreadyAcceptedException(err error) (token *string) {
130 | 	msg := err.Error()
131 | 
132 | 	if !strings.HasPrefix(msg, "DataAlreadyAcceptedException:") {
133 | 		return
134 | 	}
135 | 
136 | 	return getRetryTokenFromMessage(msg)
137 | }
138 | 
139 | var (
140 | 	errInvalidWriter = errors.New("the writer was invalidated by another goroutine")
141 | )
142 | 


--------------------------------------------------------------------------------
/lib/datadog/init.go:
--------------------------------------------------------------------------------
1 | package datadog
2 | 
3 | import "github.com/segmentio/ecs-logs/lib"
4 | 
5 | func init() {
6 | 	lib.RegisterDestination("datadog", lib.DestinationFunc(NewWriter))
7 | }
8 | 


--------------------------------------------------------------------------------
/lib/datadog/writer.go:
--------------------------------------------------------------------------------
 1 | package datadog
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"net/url"
 6 | 	"os"
 7 | 	"strings"
 8 | 
 9 | 	"github.com/segmentio/ecs-logs-go"
10 | 	"github.com/segmentio/ecs-logs/lib"
11 | 	"github.com/segmentio/ecs-logs/lib/statsd"
12 | 	"github.com/statsd/datadog"
13 | )
14 | 
15 | func NewWriter(group string, stream string) (w lib.Writer, err error) {
16 | 	var c statsd.WriterConfig
17 | 	var s string
18 | 	var u *url.URL
19 | 
20 | 	if s = os.Getenv("DATADOG_URL"); len(s) != 0 {
21 | 		if u, err = url.Parse(s); err != nil {
22 | 			err = fmt.Errorf("invalid datadog URL: %s", err)
23 | 			return
24 | 		}
25 | 
26 | 		if u.Scheme != "udp" {
27 | 			err = fmt.Errorf("invalid datadog URL: only the UDP protocol is supported but %s was found", u.Scheme)
28 | 			return
29 | 		}
30 | 
31 | 		c.Address = u.Host
32 | 	}
33 | 
34 | 	c.Group = group
35 | 	c.Stream = stream
36 | 	c.Dial = dialUdpClient
37 | 
38 | 	return statsd.DialWriter(c)
39 | }
40 | 
41 | type client struct {
42 | 	*datadog.Client
43 | }
44 | 
45 | func dialUdpClient(addr string, group string, stream string) (statsd.Client, error) {
46 | 	if dd, err := datadog.Dial(addr); err != nil {
47 | 		return nil, err
48 | 	} else {
49 | 		dd.SetPrefix("ecs-logs.")
50 | 		dd.SetTags("group:"+group, "stream:"+stream)
51 | 		return client{dd}, nil
52 | 	}
53 | }
54 | 
55 | func (c client) IncrEvents(level ecslogs.Level, value int) error {
56 | 	return c.Client.IncrBy("events.count", value, "level:"+strings.ToLower(level.String()))
57 | }
58 | 


--------------------------------------------------------------------------------
/lib/destination.go:
--------------------------------------------------------------------------------
 1 | package lib
 2 | 
 3 | import (
 4 | 	"os"
 5 | 	"sort"
 6 | 	"sync"
 7 | )
 8 | 
 9 | type Destination interface {
10 | 	Open(group string, stream string) (Writer, error)
11 | 
12 | 	Close(group string, stream string)
13 | }
14 | 
15 | type DestinationFunc func(group string, stream string) (Writer, error)
16 | 
17 | func (f DestinationFunc) Open(group string, stream string) (Writer, error) {
18 | 	return f(group, stream)
19 | }
20 | 
21 | func (f DestinationFunc) Close(group string, stream string) {}
22 | 
23 | func RegisterDestination(name string, destination Destination) {
24 | 	dstmtx.Lock()
25 | 	dstmap[name] = destination
26 | 	dstmtx.Unlock()
27 | }
28 | 
29 | func DeregisterDestination(name string) {
30 | 	dstmtx.Lock()
31 | 	delete(dstmap, name)
32 | 	dstmtx.Unlock()
33 | }
34 | 
35 | func GetDestination(name string) (destination Destination) {
36 | 	dstmtx.RLock()
37 | 	destination = dstmap[name]
38 | 	dstmtx.RUnlock()
39 | 	return
40 | }
41 | 
42 | func GetDestinations(names ...string) (destinations []Destination) {
43 | 	destinations = make([]Destination, 0, len(names))
44 | 
45 | 	for _, name := range names {
46 | 		if destination := GetDestination(name); destination != nil {
47 | 			destinations = append(destinations, destination)
48 | 		}
49 | 	}
50 | 
51 | 	return
52 | }
53 | 
54 | func DestinationsAvailable() (destinations []string) {
55 | 	dstmtx.RLock()
56 | 	destinations = make([]string, 0, len(dstmap))
57 | 
58 | 	for name := range dstmap {
59 | 		destinations = append(destinations, name)
60 | 	}
61 | 
62 | 	dstmtx.RUnlock()
63 | 	sort.Strings(destinations)
64 | 	return
65 | }
66 | 
67 | var (
68 | 	dstmtx sync.RWMutex
69 | 	dstmap = map[string]Destination{
70 | 		"stdout": DestinationFunc(func(_ string, _ string) (Writer, error) {
71 | 			return NewMessageEncoder(os.Stdout), nil
72 | 		}),
73 | 	}
74 | )
75 | 


--------------------------------------------------------------------------------
/lib/error.go:
--------------------------------------------------------------------------------
 1 | package lib
 2 | 
 3 | import "strings"
 4 | 
 5 | type ErrorList []error
 6 | 
 7 | func AppendError(err error, other error) error {
 8 | 	if err == nil {
 9 | 		return other
10 | 	}
11 | 	switch e := err.(type) {
12 | 	case ErrorList:
13 | 		return append(e, other)
14 | 	default:
15 | 		return ErrorList{err, other}
16 | 	}
17 | }
18 | 
19 | func (err ErrorList) Error() string {
20 | 	s := make([]string, len(err))
21 | 
22 | 	for i, e := range err {
23 | 		s[i] = e.Error()
24 | 	}
25 | 
26 | 	return strings.Join(s, "\n")
27 | }
28 | 


--------------------------------------------------------------------------------
/lib/group.go:
--------------------------------------------------------------------------------
 1 | package lib
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"time"
 6 | )
 7 | 
 8 | type Group struct {
 9 | 	name      string
10 | 	streams   map[string]*Stream
11 | 	createdOn time.Time
12 | 	updatedOn time.Time
13 | }
14 | 
15 | func NewGroup(name string, now time.Time) *Group {
16 | 	return &Group{
17 | 		name:      name,
18 | 		streams:   make(map[string]*Stream),
19 | 		createdOn: now,
20 | 		updatedOn: now,
21 | 	}
22 | }
23 | 
24 | func (group *Group) String() string {
25 | 	return fmt.Sprintf("group { name = %s }", group.Name())
26 | }
27 | 
28 | func (group *Group) Name() string {
29 | 	return group.name
30 | }
31 | 
32 | func (group *Group) Add(msg Message, now time.Time) (stream *Stream) {
33 | 	if stream = group.streams[msg.Stream]; stream == nil {
34 | 		stream = NewStream(group.Name(), msg.Stream, now)
35 | 		group.streams[msg.Stream] = stream
36 | 	}
37 | 
38 | 	stream.Add(msg, now)
39 | 	group.updatedOn = now
40 | 	return
41 | }
42 | 
43 | func (group *Group) HasExpired(timeout time.Duration, now time.Time) bool {
44 | 	return len(group.streams) == 0 && now.Sub(group.updatedOn) >= timeout
45 | }
46 | 
47 | func (group *Group) RemoveExpired(timeout time.Duration, now time.Time) (streams []*Stream) {
48 | 	for name, stream := range group.streams {
49 | 		if stream.HasExpired(timeout, now) {
50 | 			streams = append(streams, stream)
51 | 			delete(group.streams, name)
52 | 		}
53 | 	}
54 | 	return
55 | }
56 | 
57 | func (group *Group) ForEach(f func(*Stream)) {
58 | 	for _, stream := range group.streams {
59 | 		f(stream)
60 | 	}
61 | }
62 | 


--------------------------------------------------------------------------------
/lib/journald/init.go:
--------------------------------------------------------------------------------
 1 | // +build linux
 2 | 
 3 | package journald
 4 | 
 5 | import "github.com/segmentio/ecs-logs/lib"
 6 | 
 7 | func init() {
 8 | 	lib.RegisterSource("journald", lib.SourceFunc(NewReader))
 9 | }
10 | 


--------------------------------------------------------------------------------
/lib/journald/reader.go:
--------------------------------------------------------------------------------
  1 | // +build linux
  2 | 
  3 | package journald
  4 | 
  5 | import (
  6 | 	"encoding/json"
  7 | 	"fmt"
  8 | 	"io"
  9 | 	"os"
 10 | 	"strconv"
 11 | 	"strings"
 12 | 	"sync/atomic"
 13 | 	"time"
 14 | 
 15 | 	"github.com/coreos/go-systemd/sdjournal"
 16 | 	"github.com/segmentio/ecs-logs-go"
 17 | 	"github.com/segmentio/ecs-logs/lib"
 18 | )
 19 | 
 20 | func NewReader() (r lib.Reader, err error) {
 21 | 	var j *sdjournal.Journal
 22 | 
 23 | 	if j, err = sdjournal.NewJournal(); err != nil {
 24 | 		return
 25 | 	}
 26 | 
 27 | 	if err = j.SeekTail(); err != nil {
 28 | 		j.Close()
 29 | 		return
 30 | 	}
 31 | 
 32 | 	var streamName string
 33 | 	if streamName = os.Getenv("JOURNALD_STREAM_NAME"); len(streamName) == 0 {
 34 | 		streamName = "CONTAINER_NAME"
 35 | 	}
 36 | 
 37 | 	r = &reader{Journal: j, streamName: streamName}
 38 | 	return
 39 | }
 40 | 
 41 | type reader struct {
 42 | 	streamName string
 43 | 	stopped    int32
 44 | 	*sdjournal.Journal
 45 | }
 46 | 
 47 | func (r *reader) Close() (err error) {
 48 | 	atomic.StoreInt32(&r.stopped, 1)
 49 | 	return
 50 | }
 51 | 
 52 | func (r *reader) ReadMessage() (msg lib.Message, err error) {
 53 | 	for atomic.LoadInt32(&r.stopped) == 0 {
 54 | 		var cur int
 55 | 		var ok bool
 56 | 
 57 | 		if cur, err = r.Next(); err != nil {
 58 | 			return
 59 | 		}
 60 | 
 61 | 		if cur == 0 {
 62 | 			r.Wait(1 * time.Second)
 63 | 			continue
 64 | 		}
 65 | 
 66 | 		if msg, ok, err = r.getMessage(); ok || err != nil {
 67 | 			return
 68 | 		}
 69 | 	}
 70 | 
 71 | 	r.Journal.Close()
 72 | 	err = io.EOF
 73 | 	return
 74 | }
 75 | 
 76 | func (r *reader) getMessage() (msg lib.Message, ok bool, err error) {
 77 | 	if msg.Group, err = r.GetDataValue("CONTAINER_TAG"); len(msg.Group) == 0 {
 78 | 		// No CONTAINER_TAG, this must be a journal message from a process that
 79 | 		// isn't running in a docker container.
 80 | 		err = nil
 81 | 		return
 82 | 	}
 83 | 
 84 | 	if msg.Stream, err = r.GetDataValue(r.streamName); err != nil {
 85 | 		// Fallback to CONTAINER_NAME
 86 | 		if msg.Stream, err = r.GetDataValue("CONTAINER_NAME"); err != nil {
 87 | 
 88 | 			// There's a CONTAINER_TAG but no CONTAINER_NAME, something is seriously
 89 | 			// wrong here, the log docker log driver is misbehaving.
 90 | 			err = fmt.Errorf("missing CONTAINER_NAME in message with CONTAINER_TAG=%s", msg.Group)
 91 | 
 92 | 			return
 93 | 		}
 94 | 	}
 95 | 
 96 | 	msg.Stream = sanitizeStreamName(msg.Stream)
 97 | 
 98 | 	if s := r.getString("MESSAGE"); len(s) != 0 {
 99 | 		d := json.NewDecoder(strings.NewReader(s))
100 | 		d.UseNumber()
101 | 
102 | 		if d.Decode(&msg.Event) != nil {
103 | 			msg.Event.Message = s
104 | 		}
105 | 	}
106 | 
107 | 	if msg.Event.Level == ecslogs.NONE {
108 | 		msg.Event.Level = r.getPriority()
109 | 	}
110 | 
111 | 	if len(msg.Event.Info.Host) == 0 {
112 | 		msg.Event.Info.Host = r.getString("_HOSTNAME")
113 | 	}
114 | 
115 | 	if len(msg.Event.Info.Source) == 0 {
116 | 		msg.Event.Info.Source = (ecslogs.FuncInfo{
117 | 			File: r.getString("CODE_FILE"),
118 | 			Func: r.getString("CODE_FUNC"),
119 | 			Line: r.getInt("CODE_LINE"),
120 | 		}).String()
121 | 	}
122 | 
123 | 	if len(msg.Event.Info.ID) == 0 {
124 | 		msg.Event.Info.ID = r.getString("MESSAGE_ID")
125 | 	}
126 | 
127 | 	if msg.Event.Info.PID == 0 {
128 | 		msg.Event.Info.PID = r.getInt("_PID")
129 | 	}
130 | 
131 | 	if msg.Event.Info.GID == 0 {
132 | 		msg.Event.Info.GID = r.getInt("_GID")
133 | 	}
134 | 
135 | 	if msg.Event.Info.UID == 0 {
136 | 		msg.Event.Info.UID = r.getInt("_UID")
137 | 	}
138 | 
139 | 	if msg.Event.Time == (time.Time{}) {
140 | 		msg.Event.Time = r.getTime()
141 | 	}
142 | 
143 | 	ok = true
144 | 	return
145 | }
146 | 
147 | func (r *reader) getInt(k string) (v int) {
148 | 	v, _ = strconv.Atoi(r.getString(k))
149 | 	return
150 | }
151 | 
152 | func (r *reader) getTime() (t time.Time) {
153 | 	if u, e := r.GetRealtimeUsec(); e == nil {
154 | 		t = time.Unix(int64(u/1000000), int64((u%1000000)*1000))
155 | 	} else {
156 | 		t = time.Now()
157 | 	}
158 | 	return
159 | }
160 | 
161 | func (r *reader) getPriority() (p ecslogs.Level) {
162 | 	if v, e := strconv.Atoi(r.getString("PRIORITY")); e != nil {
163 | 		p = ecslogs.INFO
164 | 	} else {
165 | 		p = ecslogs.MakeLevel(v)
166 | 	}
167 | 	return
168 | }
169 | 
170 | func (r *reader) getString(k string) (s string) {
171 | 	s, _ = r.GetDataValue(k)
172 | 	return
173 | }
174 | 
175 | func sanitizeStreamName(name string) string {
176 | 	name = strings.Replace(name, ":", "/", -1)
177 | 	name = strings.Replace(name, "*", "/", -1)
178 | 	max := len(name)
179 | 	if max > 512 {
180 | 		max = 512
181 | 	}
182 | 	return name[:max]
183 | }
184 | 


--------------------------------------------------------------------------------
/lib/log.go:
--------------------------------------------------------------------------------
 1 | package lib
 2 | 
 3 | import (
 4 | 	"github.com/apex/log"
 5 | 	"github.com/segmentio/ecs-logs-go"
 6 | 	"github.com/segmentio/ecs-logs-go/apex"
 7 | )
 8 | 
 9 | type LogLevel log.Level
10 | 
11 | func (lvl *LogLevel) Set(s string) error {
12 | 	if l, e := log.ParseLevel(s); e != nil {
13 | 		return e
14 | 	} else {
15 | 		*lvl = LogLevel(l)
16 | 		return nil
17 | 	}
18 | }
19 | 
20 | func (lvl LogLevel) Get() interface{} {
21 | 	return lvl
22 | }
23 | 
24 | func (lvl LogLevel) String() string {
25 | 	return log.Level(lvl).String()
26 | }
27 | 
28 | type LogHandler struct {
29 | 	Group    string
30 | 	Stream   string
31 | 	Hostname string
32 | 	Queue    *MessageQueue
33 | }
34 | 
35 | func (h *LogHandler) HandleLog(entry *log.Entry) (err error) {
36 | 	msg := Message{
37 | 		Group:  h.Group,
38 | 		Stream: h.Stream,
39 | 		Event:  apex_ecslogs.MakeEvent(entry),
40 | 	}
41 | 
42 | 	if len(msg.Event.Info.Host) == 0 {
43 | 		msg.Event.Info.Host = h.Hostname
44 | 	}
45 | 
46 | 	if len(msg.Event.Info.Source) == 0 {
47 | 		if pc, ok := ecslogs.GuessCaller(0, 10, "github.com/segmentio/ecs-logs/lib", "github.com/apex/log"); ok {
48 | 			if info, ok := ecslogs.GetFuncInfo(pc); ok {
49 | 				msg.Event.Info.Source = info.String()
50 | 			}
51 | 		}
52 | 	}
53 | 
54 | 	h.Queue.Push(msg)
55 | 	h.Queue.Notify()
56 | 	return
57 | }
58 | 


--------------------------------------------------------------------------------
/lib/logdna/init.go:
--------------------------------------------------------------------------------
1 | package logdna
2 | 
3 | import "github.com/segmentio/ecs-logs/lib"
4 | 
5 | func init() {
6 | 	lib.RegisterDestination("logdna", lib.DestinationFunc(NewWriter))
7 | }
8 | 


--------------------------------------------------------------------------------
/lib/logdna/writer.go:
--------------------------------------------------------------------------------
  1 | package logdna
  2 | 
  3 | import (
  4 | 	"crypto/tls"
  5 | 	"fmt"
  6 | 	"net"
  7 | 	"net/url"
  8 | 	"os"
  9 | 	"strings"
 10 | 
 11 | 	"github.com/apex/log"
 12 | 	"github.com/segmentio/ecs-logs/lib"
 13 | 	"github.com/segmentio/ecs-logs/lib/syslog"
 14 | )
 15 | 
 16 | func NewWriter(group string, stream string) (w lib.Writer, err error) {
 17 | 	var endpoint string
 18 | 	var protocol string
 19 | 	var address string
 20 | 	var token string
 21 | 	var tags string
 22 | 	var template string
 23 | 	var timeFormat string
 24 | 	var socksProxy string
 25 | 
 26 | 	if endpoint, err = getEndpoint(); err != nil {
 27 | 		return
 28 | 	}
 29 | 
 30 | 	if protocol, address, token, tags, err = parseEndpoint(endpoint, group, stream); err != nil {
 31 | 		return
 32 | 	}
 33 | 
 34 | 	if template = os.Getenv("LOGDNA_TEMPLATE"); len(template) == 0 {
 35 | 		template = "<{{.PRIVAL}}>1 {{.TIMESTAMP}} {{.HOSTNAME}} {{.GROUP}} {{.STREAM}} {{.MSGID}} [{{.TAG}}] {{.MSG}}"
 36 | 		if len(token) != 0 {
 37 | 			template = "<key:" + token + "> " + template
 38 | 		}
 39 | 	}
 40 | 
 41 | 	if timeFormat = os.Getenv("LOGDNA_TIME_FORMAT"); len(timeFormat) == 0 {
 42 | 		timeFormat = "2016-02-10T09:28:01.982-08:00"
 43 | 	}
 44 | 
 45 | 	if socksProxy = os.Getenv("SOCKS_PROXY"); len(socksProxy) > 0 {
 46 | 		if _, _, err = net.SplitHostPort(socksProxy); err != nil {
 47 | 			log.WithFields(log.Fields{
 48 | 				"SOCKS_PROXY": socksProxy,
 49 | 			}).Warn("bad format, proxy setting will be ignored")
 50 | 			socksProxy = ""
 51 | 		}
 52 | 	}
 53 | 
 54 | 	return syslog.DialWriter(syslog.WriterConfig{
 55 | 		Network:    protocol,
 56 | 		Address:    address,
 57 | 		Template:   template,
 58 | 		TimeFormat: timeFormat,
 59 | 		Tag:        fmt.Sprintf("logdna@48950 %s", tags),
 60 | 		TLS: &tls.Config{
 61 | 			InsecureSkipVerify: true,
 62 | 		},
 63 | 		SocksProxy: socksProxy,
 64 | 	})
 65 | }
 66 | 
 67 | func getEndpoint() (endpoint string, err error) {
 68 | 	var token string
 69 | 
 70 | 	if endpoint = os.Getenv("LOGDNA_URL"); len(endpoint) != 0 {
 71 | 		return
 72 | 	}
 73 | 
 74 | 	if token = os.Getenv("LOGDNA_TOKEN"); len(token) != 0 {
 75 | 		endpoint = "tls://syslog-a.logdna.com:6514"
 76 | 		return
 77 | 	}
 78 | 
 79 | 	err = fmt.Errorf("missing LOGDNA_URL or LOGDNA_TOKEN environment variable")
 80 | 	return
 81 | }
 82 | 
 83 | func parseEndpoint(endpoint string, group string, stream string) (protocol string, address string, token string, tags string, err error) {
 84 | 	var u *url.URL
 85 | 	var q url.Values
 86 | 
 87 | 	if u, err = url.Parse(endpoint); err != nil {
 88 | 		err = fmt.Errorf("invalid logdna endpoint, %s: %s", err, endpoint)
 89 | 		return
 90 | 	}
 91 | 
 92 | 	if q, err = url.ParseQuery(u.RawQuery); err != nil {
 93 | 		err = fmt.Errorf("invalid query string in logdna endpoint, %s: %s", err, endpoint)
 94 | 		return
 95 | 	}
 96 | 
 97 | 	if protocol, err = extractProtocol(u); err != nil {
 98 | 		return
 99 | 	}
100 | 
101 | 	if address, err = extractAddress(u); err != nil {
102 | 		return
103 | 	}
104 | 
105 | 	if token, err = extractToken(u); err != nil {
106 | 		return
107 | 	}
108 | 	tags = extractTags(u, q)
109 | 	return
110 | }
111 | 
112 | func extractProtocol(u *url.URL) (protocol string, err error) {
113 | 	switch u.Scheme {
114 | 	case "tcp", "tls":
115 | 		protocol = u.Scheme
116 | 	case "":
117 | 		err = fmt.Errorf("missing protocol in logdna endpoint: %s", u)
118 | 	default:
119 | 		err = fmt.Errorf("unsupported protocol in logdna endpoint, must be one of 'tcp' or 'tls': %s", u)
120 | 	}
121 | 	return
122 | }
123 | 
124 | func extractAddress(u *url.URL) (address string, err error) {
125 | 	if len(u.Host) == 0 {
126 | 		err = fmt.Errorf("missing host in logdna endpoint: %s", u)
127 | 	} else {
128 | 		address = u.Host
129 | 	}
130 | 	return
131 | }
132 | 
133 | func extractToken(u *url.URL) (token string, err error) {
134 | 	if u.User != nil {
135 | 		token = u.User.Username()
136 | 	}
137 | 
138 | 	if len(token) == 0 {
139 | 		err = fmt.Errorf("missing token parameter in logdna endpoint: %s", u)
140 | 	}
141 | 
142 | 	return
143 | }
144 | 
145 | func extractTags(u *url.URL, q url.Values) string {
146 | 	return fmt.Sprintf("tags=\"%s\"", strings.Join(q["tag"], ","))
147 | }
148 | 


--------------------------------------------------------------------------------
/lib/loggly/init.go:
--------------------------------------------------------------------------------
1 | package loggly
2 | 
3 | import "github.com/segmentio/ecs-logs/lib"
4 | 
5 | func init() {
6 | 	lib.RegisterDestination("loggly", lib.DestinationFunc(NewWriter))
7 | }
8 | 


--------------------------------------------------------------------------------
/lib/loggly/writer.go:
--------------------------------------------------------------------------------
  1 | package loggly
  2 | 
  3 | import (
  4 | 	"crypto/tls"
  5 | 	"fmt"
  6 | 	"net"
  7 | 	"net/url"
  8 | 	"os"
  9 | 	"strings"
 10 | 
 11 | 	"github.com/apex/log"
 12 | 	"github.com/segmentio/ecs-logs/lib"
 13 | 	"github.com/segmentio/ecs-logs/lib/syslog"
 14 | )
 15 | 
 16 | func NewWriter(group string, stream string) (w lib.Writer, err error) {
 17 | 	var endpoint string
 18 | 	var protocol string
 19 | 	var address string
 20 | 	var token string
 21 | 	var pen string
 22 | 	var tags string
 23 | 	var template string
 24 | 	var timeFormat string
 25 | 	var socksProxy string
 26 | 
 27 | 	if endpoint, err = getEndpoint(); err != nil {
 28 | 		return
 29 | 	}
 30 | 
 31 | 	if protocol, address, token, pen, tags, err = parseEndpoint(endpoint, group, stream); err != nil {
 32 | 		return
 33 | 	}
 34 | 
 35 | 	if template = os.Getenv("LOGGLY_TEMPLATE"); len(template) == 0 {
 36 | 		template = "<{{.PRIVAL}}>1 {{.TIMESTAMP}} {{.HOSTNAME}} {{.GROUP}} {{.PROCID}} {{.MSGID}} [{{.TAG}}] {{.MSG}}"
 37 | 	}
 38 | 
 39 | 	if timeFormat = os.Getenv("LOGGLY_TIME_FORMAT"); len(timeFormat) == 0 {
 40 | 		timeFormat = "2006-01-02T15:04:05.999Z07:00"
 41 | 	}
 42 | 
 43 | 	if socksProxy = os.Getenv("SOCKS_PROXY"); len(socksProxy) > 0 {
 44 | 		if _, _, err = net.SplitHostPort(socksProxy); err != nil {
 45 | 			log.WithFields(log.Fields{
 46 | 				"SOCKS_PROXY": socksProxy,
 47 | 			}).Warn("bad format, proxy setting will be ignored")
 48 | 			socksProxy = ""
 49 | 		}
 50 | 	}
 51 | 
 52 | 	return syslog.DialWriter(syslog.WriterConfig{
 53 | 		Network:    protocol,
 54 | 		Address:    address,
 55 | 		Template:   template,
 56 | 		TimeFormat: timeFormat,
 57 | 		Tag:        fmt.Sprintf("%s@%s %s", token, pen, tags),
 58 | 		TLS: &tls.Config{
 59 | 			InsecureSkipVerify: true,
 60 | 		},
 61 | 		SocksProxy: socksProxy,
 62 | 	})
 63 | }
 64 | 
 65 | func getEndpoint() (endpoint string, err error) {
 66 | 	var token string
 67 | 
 68 | 	if endpoint = os.Getenv("LOGGLY_URL"); len(endpoint) != 0 {
 69 | 		return
 70 | 	}
 71 | 
 72 | 	if token = os.Getenv("LOGGLY_TOKEN"); len(token) != 0 {
 73 | 		endpoint = "tls://" + token + "@logs-01.loggly.com:6514"
 74 | 		return
 75 | 	}
 76 | 
 77 | 	err = fmt.Errorf("missing LOGGLY_URL or LOGGLY_TOKEN environment variable")
 78 | 	return
 79 | }
 80 | 
 81 | func parseEndpoint(endpoint string, group string, stream string) (protocol string, address string, token string, pen string, tags string, err error) {
 82 | 	var u *url.URL
 83 | 	var q url.Values
 84 | 
 85 | 	if u, err = url.Parse(endpoint); err != nil {
 86 | 		err = fmt.Errorf("invalid loggly endpoint, %s: %s", err, endpoint)
 87 | 		return
 88 | 	}
 89 | 
 90 | 	if q, err = url.ParseQuery(u.RawQuery); err != nil {
 91 | 		err = fmt.Errorf("invalid query string in loggly endpoint, %s: %s", err, endpoint)
 92 | 		return
 93 | 	}
 94 | 
 95 | 	if protocol, err = extractProtocol(u); err != nil {
 96 | 		return
 97 | 	}
 98 | 
 99 | 	if address, err = extractAddress(u); err != nil {
100 | 		return
101 | 	}
102 | 
103 | 	if token, err = extractToken(u); err != nil {
104 | 		return
105 | 	}
106 | 
107 | 	pen = extractPEN(u)
108 | 	tags = extractTags(u, q, group, stream)
109 | 	return
110 | }
111 | 
112 | func extractProtocol(u *url.URL) (protocol string, err error) {
113 | 	switch u.Scheme {
114 | 	case "tcp", "tls":
115 | 		protocol = u.Scheme
116 | 	case "":
117 | 		err = fmt.Errorf("missing protocol in loggly endpoint: %s", u)
118 | 	default:
119 | 		err = fmt.Errorf("unsupported protocol in loggly endpoint, must be one of 'tcp' or 'tls': %s", u)
120 | 	}
121 | 	return
122 | }
123 | 
124 | func extractAddress(u *url.URL) (address string, err error) {
125 | 	if len(u.Host) == 0 {
126 | 		err = fmt.Errorf("missing host in loggly endpoint: %s", u)
127 | 	} else {
128 | 		address = u.Host
129 | 	}
130 | 	return
131 | }
132 | 
133 | func extractToken(u *url.URL) (token string, err error) {
134 | 	if u.User != nil {
135 | 		token = u.User.Username()
136 | 	}
137 | 
138 | 	if len(token) == 0 {
139 | 		err = fmt.Errorf("missing token parameter in loggly endpoint: %s", u)
140 | 	}
141 | 
142 | 	return
143 | }
144 | 
145 | func extractPEN(u *url.URL) (pen string) {
146 | 	if u.User != nil {
147 | 		pen, _ = u.User.Password()
148 | 	}
149 | 
150 | 	if len(pen) == 0 {
151 | 		pen = "41058"
152 | 	}
153 | 
154 | 	return
155 | }
156 | 
157 | func extractTags(u *url.URL, q url.Values, group string, stream string) string {
158 | 	tags := q["tag"]
159 | 	list := make([]string, 0, len(tags)+2)
160 | 	list = append(list, group, stream)
161 | 	list = append(list, tags...)
162 | 
163 | 	for i, tag := range list {
164 | 		list[i] = fmt.Sprintf("tag=%#v", tag)
165 | 	}
166 | 
167 | 	return strings.Join(list, " ")
168 | }
169 | 


--------------------------------------------------------------------------------
/lib/message.go:
--------------------------------------------------------------------------------
 1 | package lib
 2 | 
 3 | import (
 4 | 	"encoding/json"
 5 | 	"sync"
 6 | 
 7 | 	"github.com/segmentio/ecs-logs-go"
 8 | 	"github.com/segmentio/jutil"
 9 | )
10 | 
11 | type Message struct {
12 | 	Group  string        `json:"group,omitempty"`
13 | 	Stream string        `json:"stream,omitempty"`
14 | 	Event  ecslogs.Event `json:"event,omitempty"`
15 | }
16 | 
17 | func (m Message) Bytes() []byte {
18 | 	b, _ := json.Marshal(m)
19 | 	return b
20 | }
21 | 
22 | func (m Message) String() string {
23 | 	return string(m.Bytes())
24 | }
25 | 
26 | func (m Message) ContentLength() int {
27 | 	n, _ := jutil.Length(m.Event)
28 | 	return n
29 | }
30 | 
31 | type MessageBatch []Message
32 | 
33 | func (list MessageBatch) Swap(i int, j int) {
34 | 	list[i], list[j] = list[j], list[i]
35 | }
36 | 
37 | func (list MessageBatch) Less(i int, j int) bool {
38 | 	return list[i].Event.Time.Before(list[j].Event.Time)
39 | }
40 | 
41 | func (list MessageBatch) Len() int {
42 | 	return len(list)
43 | }
44 | 
45 | type MessageQueue struct {
46 | 	C      <-chan struct{}
47 | 	signal chan struct{}
48 | 	mutex  sync.Mutex
49 | 	batch  MessageBatch
50 | }
51 | 
52 | func NewMessageQueue() *MessageQueue {
53 | 	c := make(chan struct{}, 1)
54 | 	return &MessageQueue{
55 | 		C:      c,
56 | 		signal: c,
57 | 		batch:  make(MessageBatch, 0, 100),
58 | 	}
59 | }
60 | 
61 | func (q *MessageQueue) Push(msg Message) {
62 | 	q.mutex.Lock()
63 | 	q.batch = append(q.batch, msg)
64 | 	q.mutex.Unlock()
65 | }
66 | 
67 | func (q *MessageQueue) Notify() {
68 | 	select {
69 | 	default:
70 | 	case q.signal <- struct{}{}:
71 | 	}
72 | }
73 | 
74 | func (q *MessageQueue) Flush() (batch MessageBatch) {
75 | 	q.mutex.Lock()
76 | 	batch = make(MessageBatch, len(q.batch))
77 | 	copy(batch, q.batch)
78 | 	q.batch = q.batch[:0]
79 | 	q.mutex.Unlock()
80 | 	return
81 | }
82 | 


--------------------------------------------------------------------------------
/lib/message_test.go:
--------------------------------------------------------------------------------
  1 | package lib
  2 | 
  3 | import (
  4 | 	"errors"
  5 | 	"fmt"
  6 | 	"io"
  7 | 	"reflect"
  8 | 	"testing"
  9 | 	"time"
 10 | 
 11 | 	"github.com/segmentio/ecs-logs-go"
 12 | )
 13 | 
 14 | func TestMessageString(t *testing.T) {
 15 | 	d := time.Date(2016, 6, 13, 12, 23, 42, 123456000, time.Local)
 16 | 	m := Message{
 17 | 		Group:  "abc",
 18 | 		Stream: "0123456789",
 19 | 		Event: ecslogs.Event{
 20 | 			Level:   ecslogs.INFO,
 21 | 			Time:    d,
 22 | 			Message: "Hello World!",
 23 | 			Info:    ecslogs.EventInfo{Host: "localhost"},
 24 | 			Data:    ecslogs.EventData{},
 25 | 		},
 26 | 	}
 27 | 
 28 | 	ref := fmt.Sprintf(
 29 | 		`{"group":"abc","stream":"0123456789","event":{"level":"INFO","time":"%s","info":{"host":"localhost"},"data":{},"message":"Hello World!"}}`,
 30 | 		d.Format(time.RFC3339Nano),
 31 | 	)
 32 | 
 33 | 	if s := m.String(); s != ref {
 34 | 		t.Errorf("invalid string representation of the message:\n - expected: %s\n - found:    %s", ref, s)
 35 | 	}
 36 | }
 37 | 
 38 | func TestMessageEncoderDecoder(t *testing.T) {
 39 | 	batch := MessageBatch{
 40 | 		Message{
 41 | 			Group:  "abc",
 42 | 			Stream: "0123456789",
 43 | 			Event: ecslogs.Event{
 44 | 				Level:   ecslogs.INFO,
 45 | 				Time:    time.Date(2016, 6, 13, 12, 23, 42, 123456789, time.UTC),
 46 | 				Message: "Hello World!",
 47 | 			},
 48 | 		},
 49 | 		Message{
 50 | 			Group:  "abc",
 51 | 			Stream: "0123456789",
 52 | 			Event: ecslogs.Event{
 53 | 				Level:   ecslogs.INFO,
 54 | 				Time:    time.Date(2016, 6, 13, 12, 24, 42, 123456789, time.UTC),
 55 | 				Message: "How are you doing?",
 56 | 			},
 57 | 		},
 58 | 	}
 59 | 
 60 | 	r, w := io.Pipe()
 61 | 	e := NewMessageEncoder(w)
 62 | 	d := NewMessageDecoder(r)
 63 | 
 64 | 	// This goroutine encodes the batch of message using the encoder that
 65 | 	// outputs the serialized messages to the write end of the pipe.
 66 | 	go func() {
 67 | 		defer w.Close()
 68 | 		if err := e.WriteMessageBatch(batch); err != nil {
 69 | 			t.Error(err)
 70 | 		}
 71 | 	}()
 72 | 
 73 | 	// This loop reads the messages written to the pipe and rebuilds a list
 74 | 	// of messages until EOF is reached. The orignal batch and list are then
 75 | 	// compred to ensure they are the same.
 76 | 	var list MessageBatch
 77 | 	for {
 78 | 		if msg, err := d.ReadMessage(); err != nil {
 79 | 			if err == io.EOF {
 80 | 				break
 81 | 			}
 82 | 			t.Error(err)
 83 | 			return
 84 | 		} else {
 85 | 			list = append(list, msg)
 86 | 		}
 87 | 	}
 88 | 
 89 | 	if err := d.Close(); err != nil {
 90 | 		t.Error(err)
 91 | 	}
 92 | 
 93 | 	if !reflect.DeepEqual(batch, list) {
 94 | 		t.Errorf("invalid list decoded after encoding batch:\n- %v\n- %v", batch, list)
 95 | 	}
 96 | }
 97 | 
 98 | func TestMessageEncoderWriteMessageBatchError(t *testing.T) {
 99 | 	batch := MessageBatch{
100 | 		Message{
101 | 			Group:  "abc",
102 | 			Stream: "0123456789",
103 | 			Event: ecslogs.Event{
104 | 				Level:   ecslogs.INFO,
105 | 				Time:    time.Date(2016, 6, 13, 12, 23, 42, 123456789, time.UTC),
106 | 				Message: "Hello World!",
107 | 			},
108 | 		},
109 | 		Message{
110 | 			Group:  "abc",
111 | 			Stream: "0123456789",
112 | 			Event: ecslogs.Event{
113 | 				Level:   ecslogs.INFO,
114 | 				Time:    time.Date(2016, 6, 13, 12, 24, 42, 123456789, time.UTC),
115 | 				Message: "How are you doing?",
116 | 			},
117 | 		},
118 | 	}
119 | 
120 | 	x := errors.New("ERR")
121 | 	e := NewMessageEncoder(errorWriter{x})
122 | 
123 | 	if err := e.WriteMessageBatch(batch); err != x {
124 | 		t.Errorf("expected error (%s) but got %s", x, err)
125 | 	}
126 | }
127 | 
128 | // The errorWriter type is used to mock message encoders with a writer that
129 | // always returns an error so we can test error cases.
130 | type errorWriter struct {
131 | 	err error
132 | }
133 | 
134 | func (w errorWriter) Write(b []byte) (int, error) {
135 | 	return 0, w.err
136 | }
137 | 


--------------------------------------------------------------------------------
/lib/reader.go:
--------------------------------------------------------------------------------
 1 | package lib
 2 | 
 3 | import (
 4 | 	"encoding/json"
 5 | 	"io"
 6 | )
 7 | 
 8 | type Reader interface {
 9 | 	io.Closer
10 | 
11 | 	ReadMessage() (Message, error)
12 | }
13 | 
14 | func NewMessageDecoder(r io.Reader) Reader {
15 | 	return decoder{
16 | 		j: json.NewDecoder(r),
17 | 		r: r,
18 | 	}
19 | }
20 | 
21 | type decoder struct {
22 | 	j *json.Decoder
23 | 	r io.Reader
24 | }
25 | 
26 | func (d decoder) Close() (err error) {
27 | 	if c, ok := d.r.(io.Closer); ok {
28 | 		err = c.Close()
29 | 	}
30 | 	return
31 | }
32 | 
33 | func (d decoder) ReadMessage() (msg Message, err error) {
34 | 	err = d.j.Decode(&msg)
35 | 	return
36 | }
37 | 


--------------------------------------------------------------------------------
/lib/source.go:
--------------------------------------------------------------------------------
  1 | package lib
  2 | 
  3 | import (
  4 | 	"io"
  5 | 	"os"
  6 | 	"sort"
  7 | 	"sync"
  8 | )
  9 | 
 10 | type Source interface {
 11 | 	Open() (Reader, error)
 12 | }
 13 | 
 14 | type SourceFunc func() (Reader, error)
 15 | 
 16 | func (f SourceFunc) Open() (Reader, error) {
 17 | 	return f()
 18 | }
 19 | 
 20 | func RegisterSource(name string, source Source) {
 21 | 	srcmtx.Lock()
 22 | 	srcmap[name] = source
 23 | 	srcmtx.Unlock()
 24 | }
 25 | 
 26 | func DeregisterSource(name string) {
 27 | 	srcmtx.Lock()
 28 | 	delete(srcmap, name)
 29 | 	srcmtx.Unlock()
 30 | }
 31 | 
 32 | func GetSource(name string) (source Source) {
 33 | 	srcmtx.RLock()
 34 | 	source = srcmap[name]
 35 | 	srcmtx.RUnlock()
 36 | 	return
 37 | }
 38 | 
 39 | func GetSources(names ...string) (sources []Source) {
 40 | 	sources = make([]Source, 0, len(names))
 41 | 
 42 | 	for _, name := range names {
 43 | 		if source := GetSource(name); source != nil {
 44 | 			sources = append(sources, source)
 45 | 		}
 46 | 	}
 47 | 
 48 | 	return
 49 | }
 50 | 
 51 | func SourcesAvailable() (sources []string) {
 52 | 	srcmtx.RLock()
 53 | 	sources = make([]string, 0, len(srcmap))
 54 | 
 55 | 	for name := range srcmap {
 56 | 		sources = append(sources, name)
 57 | 	}
 58 | 
 59 | 	srcmtx.RUnlock()
 60 | 	sort.Strings(sources)
 61 | 	return
 62 | }
 63 | 
 64 | var (
 65 | 	srcmtx sync.RWMutex
 66 | 	srcmap = map[string]Source{
 67 | 		"stdin": SourceFunc(func() (Reader, error) {
 68 | 			// On some platforms closing stdin doesn't cause pending read
 69 | 			// operations to abort, resulting in a blocking call that never
 70 | 			// returns.
 71 | 			//
 72 | 			// To work around this limitation we start a goroutine that is
 73 | 			// responsible for reading from stdin and sending the bytes through
 74 | 			// an in-memory pipe. When the pipe is closed it properly cancels
 75 | 			// all pending reads which is the behavior we expect.
 76 | 			//
 77 | 			// There probably is a small performance cost to adding this extra
 78 | 			// step but the stdin source shouldn't be used in production
 79 | 			// environments so it shouldn't be a problem in practice.
 80 | 			//
 81 | 			// Note that the goroutine reading from stdin is likely gonna be
 82 | 			// leaked... This is OK in the ecs-logs use case because only one
 83 | 			// stdin reader will be instantiated.
 84 | 			r, w := io.Pipe()
 85 | 			go pipe(w, os.Stdin)
 86 | 			// We use the Close method of the write end of the pipe so when it's
 87 | 			// called the read end will start returning io.EOF to indicate a
 88 | 			// graceful shutdown.
 89 | 			return NewMessageDecoder(struct {
 90 | 				io.Reader
 91 | 				io.Closer
 92 | 			}{r, w}), nil
 93 | 		}),
 94 | 	}
 95 | )
 96 | 
 97 | func pipe(w *io.PipeWriter, r io.Reader) {
 98 | 	_, err := io.Copy(w, r)
 99 | 
100 | 	if err == nil {
101 | 		err = io.EOF
102 | 	}
103 | 
104 | 	w.CloseWithError(err)
105 | }
106 | 


--------------------------------------------------------------------------------
/lib/statsd/init.go:
--------------------------------------------------------------------------------
1 | package statsd
2 | 
3 | import "github.com/segmentio/ecs-logs/lib"
4 | 
5 | func init() {
6 | 	lib.RegisterDestination("statsd", lib.DestinationFunc(NewWriter))
7 | }
8 | 


--------------------------------------------------------------------------------
/lib/statsd/writer.go:
--------------------------------------------------------------------------------
  1 | package statsd
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"io"
  6 | 	"net/url"
  7 | 	"os"
  8 | 	"strings"
  9 | 
 10 | 	"github.com/segmentio/ecs-logs-go"
 11 | 	"github.com/segmentio/ecs-logs/lib"
 12 | 	"github.com/statsd/client"
 13 | )
 14 | 
 15 | type Client interface {
 16 | 	io.Closer
 17 | 
 18 | 	Flush() error
 19 | 
 20 | 	IncrEvents(ecslogs.Level, int) error
 21 | }
 22 | 
 23 | type WriterConfig struct {
 24 | 	Address string
 25 | 	Group   string
 26 | 	Stream  string
 27 | 	Dial    func(addr string, group string, stream string) (Client, error)
 28 | }
 29 | 
 30 | func NewWriter(group string, stream string) (w lib.Writer, err error) {
 31 | 	var c WriterConfig
 32 | 	var s string
 33 | 	var u *url.URL
 34 | 
 35 | 	if s = os.Getenv("STATSD_URL"); len(s) != 0 {
 36 | 		if u, err = url.Parse(s); err != nil {
 37 | 			err = fmt.Errorf("invalid statsd URL: %s", err)
 38 | 			return
 39 | 		}
 40 | 
 41 | 		if u.Scheme != "udp" {
 42 | 			err = fmt.Errorf("invalid statsd URL: only the UDP protocol is supported but %s was found", u.Scheme)
 43 | 			return
 44 | 		}
 45 | 
 46 | 		c.Address = u.Host
 47 | 	}
 48 | 
 49 | 	c.Group = group
 50 | 	c.Stream = stream
 51 | 
 52 | 	return DialWriter(c)
 53 | }
 54 | 
 55 | func DialWriter(config WriterConfig) (w lib.Writer, err error) {
 56 | 	var client Client
 57 | 
 58 | 	if len(config.Address) == 0 {
 59 | 		config.Address = "localhost:8125"
 60 | 	}
 61 | 
 62 | 	if config.Dial == nil {
 63 | 		config.Dial = dial
 64 | 	}
 65 | 
 66 | 	if client, err = config.Dial(config.Address, config.Group, config.Stream); err != nil {
 67 | 		return
 68 | 	}
 69 | 
 70 | 	w = writer{client}
 71 | 	return
 72 | }
 73 | 
 74 | func dial(addr string, group string, stream string) (Client, error) {
 75 | 	if cli, err := statsd.Dial(addr); err != nil {
 76 | 		return nil, err
 77 | 	} else {
 78 | 		cli.Prefix("ecs-logs." + group + ".")
 79 | 		return client{cli}, nil
 80 | 	}
 81 | }
 82 | 
 83 | type client struct {
 84 | 	*statsd.Client
 85 | }
 86 | 
 87 | func (c client) IncrEvents(level ecslogs.Level, value int) error {
 88 | 	return c.IncrBy(strings.ToLower(level.String()), value)
 89 | }
 90 | 
 91 | type writer struct {
 92 | 	client Client
 93 | }
 94 | 
 95 | type metric struct {
 96 | 	value int
 97 | }
 98 | 
 99 | func (w writer) Close() error {
100 | 	return w.client.Close()
101 | }
102 | 
103 | func (w writer) WriteMessage(msg lib.Message) error {
104 | 	return w.WriteMessageBatch(lib.MessageBatch{msg})
105 | }
106 | 
107 | func (w writer) WriteMessageBatch(batch lib.MessageBatch) error {
108 | 	return sendMetrics(w.client, extractMetrics(batch))
109 | }
110 | 
111 | func extractMetrics(batch lib.MessageBatch) map[ecslogs.Level]*metric {
112 | 	metrics := make(map[ecslogs.Level]*metric, 10)
113 | 
114 | 	for _, msg := range batch {
115 | 		if m := metrics[msg.Event.Level]; m == nil {
116 | 			m = &metric{value: 1}
117 | 			metrics[msg.Event.Level] = m
118 | 		} else {
119 | 			m.value++
120 | 		}
121 | 	}
122 | 
123 | 	return metrics
124 | }
125 | 
126 | func sendMetrics(client Client, metrics map[ecslogs.Level]*metric) (err error) {
127 | 	for lvl, met := range metrics {
128 | 		if e := client.IncrEvents(lvl, met.value); e != nil {
129 | 			err = lib.AppendError(err, e)
130 | 		}
131 | 	}
132 | 	if e := client.Flush(); e != nil {
133 | 		err = lib.AppendError(err, e)
134 | 	}
135 | 	return
136 | }
137 | 


--------------------------------------------------------------------------------
/lib/statsd/writer_test.go:
--------------------------------------------------------------------------------
 1 | package statsd
 2 | 
 3 | import (
 4 | 	"testing"
 5 | 
 6 | 	"github.com/segmentio/ecs-logs-go"
 7 | 	"github.com/segmentio/ecs-logs/lib"
 8 | )
 9 | 
10 | func TestExtractMetrics(t *testing.T) {
11 | 	batch := lib.MessageBatch{
12 | 		lib.Message{
13 | 			Event: ecslogs.Event{Level: ecslogs.INFO},
14 | 		},
15 | 		lib.Message{
16 | 			Event: ecslogs.Event{Level: ecslogs.INFO},
17 | 		},
18 | 		lib.Message{
19 | 			Event: ecslogs.Event{Level: ecslogs.WARN},
20 | 		},
21 | 		lib.Message{
22 | 			Event: ecslogs.Event{Level: ecslogs.INFO},
23 | 		},
24 | 		lib.Message{
25 | 			Event: ecslogs.Event{Level: ecslogs.INFO},
26 | 		},
27 | 		lib.Message{
28 | 			Event: ecslogs.Event{Level: ecslogs.ERROR},
29 | 		},
30 | 		lib.Message{
31 | 			Event: ecslogs.Event{Level: ecslogs.WARN},
32 | 		},
33 | 	}
34 | 
35 | 	metrics := extractMetrics(batch)
36 | 
37 | 	if len(metrics) != 3 {
38 | 		t.Errorf("invalid metrics count: %d != %d", len(metrics), 3)
39 | 	}
40 | 
41 | 	countInfo := 0
42 | 	countWarn := 0
43 | 	countError := 0
44 | 
45 | 	for lvl, m := range metrics {
46 | 		switch lvl {
47 | 		case ecslogs.INFO:
48 | 			countInfo += m.value
49 | 
50 | 		case ecslogs.WARN:
51 | 			countWarn += m.value
52 | 
53 | 		case ecslogs.ERROR:
54 | 			countError += m.value
55 | 
56 | 		default:
57 | 			t.Errorf("invalid metric level: %s", lvl)
58 | 			continue
59 | 		}
60 | 	}
61 | 
62 | 	if countInfo != 4 {
63 | 		t.Error("invalid info count:", countInfo)
64 | 	}
65 | 
66 | 	if countWarn != 2 {
67 | 		t.Error("invalid warning count:", countWarn)
68 | 	}
69 | 
70 | 	if countError != 1 {
71 | 		t.Error("invalid error count:", countError)
72 | 	}
73 | }
74 | 


--------------------------------------------------------------------------------
/lib/store.go:
--------------------------------------------------------------------------------
 1 | package lib
 2 | 
 3 | import "time"
 4 | 
 5 | type Store struct {
 6 | 	groups map[string]*Group
 7 | }
 8 | 
 9 | func NewStore() *Store {
10 | 	return &Store{
11 | 		groups: make(map[string]*Group, 100),
12 | 	}
13 | }
14 | 
15 | func (store *Store) Add(msg Message, now time.Time) (group *Group, stream *Stream) {
16 | 	if group = store.groups[msg.Group]; group == nil {
17 | 		group = NewGroup(msg.Group, now)
18 | 		store.groups[msg.Group] = group
19 | 	}
20 | 
21 | 	stream = group.Add(msg, now)
22 | 	return
23 | }
24 | 
25 | func (store *Store) RemoveExpired(timeout time.Duration, now time.Time) (streams []*Stream) {
26 | 	for name, group := range store.groups {
27 | 		streams = append(streams, group.RemoveExpired(timeout, now)...)
28 | 
29 | 		if group.HasExpired(timeout, now) {
30 | 			delete(store.groups, name)
31 | 		}
32 | 	}
33 | 	return
34 | }
35 | 
36 | func (store *Store) ForEach(f func(*Group)) {
37 | 	for _, group := range store.groups {
38 | 		f(group)
39 | 	}
40 | }
41 | 


--------------------------------------------------------------------------------
/lib/stream.go:
--------------------------------------------------------------------------------
  1 | package lib
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"time"
  6 | )
  7 | 
  8 | type Stream struct {
  9 | 	group     string
 10 | 	name      string
 11 | 	bytes     int
 12 | 	messages  MessageBatch
 13 | 	createdOn time.Time
 14 | 	updatedOn time.Time
 15 | 	flushedOn time.Time
 16 | }
 17 | 
 18 | type StreamLimits struct {
 19 | 	MaxCount int
 20 | 	MaxBytes int
 21 | 	MaxTime  time.Duration
 22 | 	Force    bool
 23 | }
 24 | 
 25 | func NewStream(group string, name string, now time.Time) *Stream {
 26 | 	return &Stream{
 27 | 		group:     group,
 28 | 		name:      name,
 29 | 		messages:  make(MessageBatch, 0, 1000),
 30 | 		createdOn: now,
 31 | 		updatedOn: now,
 32 | 		flushedOn: now,
 33 | 	}
 34 | }
 35 | 
 36 | func (stream *Stream) String() string {
 37 | 	return fmt.Sprintf("stream { group = %#v, name = %#v }", stream.Group(), stream.Name())
 38 | }
 39 | 
 40 | func (stream *Stream) Group() string {
 41 | 	return stream.group
 42 | }
 43 | 
 44 | func (stream *Stream) Name() string {
 45 | 	return stream.name
 46 | }
 47 | 
 48 | func (stream *Stream) Add(msg Message, now time.Time) {
 49 | 	stream.bytes += msg.ContentLength()
 50 | 	stream.messages = append(stream.messages, msg)
 51 | 	stream.updatedOn = now
 52 | }
 53 | 
 54 | func (stream *Stream) HasExpired(timeout time.Duration, now time.Time) bool {
 55 | 	return len(stream.messages) == 0 && now.Sub(stream.updatedOn) >= timeout
 56 | }
 57 | 
 58 | func (stream *Stream) Flush(limits StreamLimits, now time.Time) (list MessageBatch, reason string) {
 59 | 	if stream.bytes >= limits.MaxBytes {
 60 | 		return stream.flushDueToBytesLimit(limits.MaxBytes, now), "max batch size exceeded"
 61 | 	}
 62 | 
 63 | 	if len(stream.messages) >= limits.MaxCount {
 64 | 		return stream.flushDueToCountLimit(limits.MaxCount, now), "max message count exceeded"
 65 | 	}
 66 | 
 67 | 	if now.Sub(stream.flushedOn) >= limits.MaxTime {
 68 | 		return stream.flushDueToTimeLimit(now), "time limit exceeded"
 69 | 	}
 70 | 
 71 | 	if limits.Force {
 72 | 		return stream.flushDueToForcedFlushing(now), "forced flushing"
 73 | 	}
 74 | 
 75 | 	return
 76 | }
 77 | 
 78 | func (stream *Stream) flushDueToBytesLimit(maxBytes int, now time.Time) MessageBatch {
 79 | 	count := 0
 80 | 	bytes := 0
 81 | 
 82 | 	for _, msg := range stream.messages {
 83 | 		length := msg.ContentLength()
 84 | 
 85 | 		if (bytes + length) > maxBytes {
 86 | 			break
 87 | 		}
 88 | 
 89 | 		bytes += length
 90 | 		count += 1
 91 | 	}
 92 | 
 93 | 	if count == 0 {
 94 | 		count = 1
 95 | 	}
 96 | 
 97 | 	return stream.flush(count, now)
 98 | }
 99 | 
100 | func (stream *Stream) flushDueToCountLimit(maxCount int, now time.Time) MessageBatch {
101 | 	return stream.flush(maxCount, now)
102 | }
103 | 
104 | func (stream *Stream) flushDueToTimeLimit(now time.Time) MessageBatch {
105 | 	return stream.flush(len(stream.messages), now)
106 | }
107 | 
108 | func (stream *Stream) flushDueToForcedFlushing(now time.Time) MessageBatch {
109 | 	return stream.flush(len(stream.messages), now)
110 | }
111 | 
112 | func (stream *Stream) flush(count int, now time.Time) (msglist MessageBatch) {
113 | 	msglist, stream.messages = splitMessageListHead(stream.messages, count)
114 | 	stream.bytes -= messageListBytes(msglist)
115 | 	stream.flushedOn = now
116 | 	return
117 | }
118 | 
119 | func splitMessageListHead(list MessageBatch, count int) (head MessageBatch, tail MessageBatch) {
120 | 	head = make(MessageBatch, count)
121 | 	tail = make(MessageBatch, len(list)-count, cap(list))
122 | 	copy(head, list[:count])
123 | 	copy(tail, list[count:])
124 | 	return
125 | }
126 | 
127 | func messageListBytes(list MessageBatch) (bytes int) {
128 | 	for _, msg := range list {
129 | 		bytes += msg.ContentLength()
130 | 	}
131 | 	return
132 | }
133 | 


--------------------------------------------------------------------------------
/lib/stream_test.go:
--------------------------------------------------------------------------------
  1 | package lib
  2 | 
  3 | import (
  4 | 	"reflect"
  5 | 	"testing"
  6 | 	"time"
  7 | 
  8 | 	"github.com/segmentio/ecs-logs-go"
  9 | )
 10 | 
 11 | func TestSplitMessageListHead(t *testing.T) {
 12 | 	tests := []struct {
 13 | 		list  MessageBatch
 14 | 		count int
 15 | 	}{
 16 | 		{
 17 | 			list:  MessageBatch{},
 18 | 			count: 0,
 19 | 		},
 20 | 		{
 21 | 			list: MessageBatch{
 22 | 				Message{Group: "A"},
 23 | 				Message{Group: "B"},
 24 | 				Message{Group: "C"},
 25 | 			},
 26 | 			count: 0,
 27 | 		},
 28 | 		{
 29 | 			list: MessageBatch{
 30 | 				Message{Group: "A"},
 31 | 				Message{Group: "B"},
 32 | 				Message{Group: "C"},
 33 | 			},
 34 | 			count: 1,
 35 | 		},
 36 | 		{
 37 | 			list: MessageBatch{
 38 | 				Message{Group: "A"},
 39 | 				Message{Group: "B"},
 40 | 				Message{Group: "C"},
 41 | 			},
 42 | 			count: 3,
 43 | 		},
 44 | 	}
 45 | 
 46 | 	for _, test := range tests {
 47 | 		head, tail := splitMessageListHead(test.list, test.count)
 48 | 
 49 | 		if !reflect.DeepEqual(head, test.list[:test.count]) {
 50 | 			t.Errorf("invalid head:\n- expected: %v\n- found:    %v", test.list[:test.count], head)
 51 | 		}
 52 | 
 53 | 		if !reflect.DeepEqual(tail, test.list[test.count:]) {
 54 | 			t.Errorf("invalid tail:\n- expected: %v\n- found:    %v", test.list[test.count:], tail)
 55 | 		}
 56 | 	}
 57 | }
 58 | 
 59 | func TestStreamName(t *testing.T) {
 60 | 	ts := time.Now()
 61 | 	st := NewStream("A", "0123456789", ts)
 62 | 
 63 | 	if s := st.Name(); s != "0123456789" {
 64 | 		t.Error("invalid stream name:", s)
 65 | 	}
 66 | }
 67 | 
 68 | func TestStreamString(t *testing.T) {
 69 | 	ts := time.Now()
 70 | 	st := NewStream("A", "0123456789", ts)
 71 | 
 72 | 	if s := st.String(); s != `stream { group = "A", name = "0123456789" }` {
 73 | 		t.Error("invalid stream name:", s)
 74 | 	}
 75 | }
 76 | 
 77 | func TestStreamExpired(t *testing.T) {
 78 | 	ts := time.Now()
 79 | 	st := NewStream("A", "0123456789", ts)
 80 | 
 81 | 	if !st.HasExpired(1*time.Second, ts.Add(2*time.Second)) {
 82 | 		t.Error("new stream should be considered expired because it has no messages and wasn't updated recently")
 83 | 	}
 84 | }
 85 | 
 86 | func TestStreamNotExpiredDueToMessages(t *testing.T) {
 87 | 	ts := time.Now()
 88 | 	st := NewStream("A", "0123456789", ts)
 89 | 	st.Add(Message{
 90 | 		Group:  "A",
 91 | 		Stream: "0123456789",
 92 | 		Event: ecslogs.Event{
 93 | 			Time:    ts,
 94 | 			Message: "Hello World!",
 95 | 		},
 96 | 	}, ts)
 97 | 
 98 | 	if st.HasExpired(1*time.Second, ts.Add(2*time.Second)) {
 99 | 		t.Error("non-empty streams should not be expired")
100 | 	}
101 | }
102 | 
103 | func TestStreamNotExpiredDueToTimeout(t *testing.T) {
104 | 	ts := time.Now()
105 | 	st := NewStream("A", "0123456789", ts)
106 | 
107 | 	if st.HasExpired(1*time.Second, ts) {
108 | 		t.Error("streams that were updated recently should not be expired")
109 | 	}
110 | }
111 | 
112 | func TestStreamBytes(t *testing.T) {
113 | 	ts := time.Now()
114 | 	st := NewStream("A", "0123456789", ts)
115 | 	m1 := Message{
116 | 		Group:  "A",
117 | 		Stream: "0123456789",
118 | 		Event: ecslogs.Event{
119 | 			Time:    ts,
120 | 			Message: "Hello World!",
121 | 		},
122 | 	}
123 | 	m2 := Message{
124 | 		Group:  "A",
125 | 		Stream: "0123456789",
126 | 		Event: ecslogs.Event{
127 | 			Time:    ts,
128 | 			Message: "How are you?",
129 | 		},
130 | 	}
131 | 	m3 := Message{
132 | 		Group:  "A",
133 | 		Stream: "0123456789",
134 | 		Event: ecslogs.Event{
135 | 			Time:    ts,
136 | 			Message: "Well",
137 | 		},
138 | 	}
139 | 
140 | 	st.Add(m1, ts)
141 | 	st.Add(m2, ts)
142 | 	st.Add(m3, ts)
143 | 
144 | 	if bytes := m1.ContentLength() + m2.ContentLength() + m3.ContentLength(); bytes != st.bytes {
145 | 		t.Errorf("invalid stream bytes count: %d != %d", st.bytes, bytes)
146 | 	}
147 | 
148 | 	// Flush the first two messages, there should be one message left in the
149 | 	// stream and the number of bytes should match the length of the third
150 | 	// message.
151 | 	list, _ := st.Flush(StreamLimits{
152 | 		MaxBytes: m1.ContentLength() + m2.ContentLength(),
153 | 	}, ts)
154 | 
155 | 	if !reflect.DeepEqual(list, MessageBatch{m1, m2}) {
156 | 		t.Error("invalid list of messages flushed from stream:", list)
157 | 	}
158 | 
159 | 	if !reflect.DeepEqual(st.messages, MessageBatch{m3}) {
160 | 		t.Error("invalid list of messages left in stream:", st.messages)
161 | 	}
162 | 
163 | 	if st.bytes != m3.ContentLength() {
164 | 		t.Error("invalid stream bytes count left in stream:", st.bytes)
165 | 	}
166 | 
167 | 	// Flush the last message with a limit that's shorter than the current byte
168 | 	// count in the stream.
169 | 	list, _ = st.Flush(StreamLimits{
170 | 		MaxBytes: m3.ContentLength() - 1,
171 | 	}, ts)
172 | 
173 | 	if !reflect.DeepEqual(list, MessageBatch{m3}) {
174 | 		t.Error("invalid list of messages flushed from stream:", list)
175 | 	}
176 | 
177 | 	if len(st.messages) != 0 {
178 | 		t.Error("invalid list of messages left in stream:", st.messages)
179 | 	}
180 | 
181 | 	if st.bytes != 0 {
182 | 		t.Error("invalid stream bytes count left in stream:", st.bytes)
183 | 	}
184 | }
185 | 


--------------------------------------------------------------------------------
/lib/syslog/init.go:
--------------------------------------------------------------------------------
1 | package syslog
2 | 
3 | import "github.com/segmentio/ecs-logs/lib"
4 | 
5 | func init() {
6 | 	lib.RegisterDestination("syslog", lib.DestinationFunc(NewWriter))
7 | }
8 | 


--------------------------------------------------------------------------------
/lib/syslog/pool/pool.go:
--------------------------------------------------------------------------------
  1 | package pool
  2 | 
  3 | import (
  4 | 	"io"
  5 | 	"time"
  6 | 
  7 | 	"github.com/jpillora/backoff"
  8 | )
  9 | 
 10 | // A LimitedConnPool is a connection pool, with the property that
 11 | // the total number of live connections is limited to the size
 12 | // parameter passed to NewLimited. In order to make this guarantee,
 13 | // we assume that all new connections are introduced by calls to the
 14 | // Get method, and returned to the pool by calls to Close().
 15 | type LimitedConnPool struct {
 16 | 	conns  chan *conn    // Available connections in the pool
 17 | 	live   chan struct{} // Keep a count of living connections (in our hands, or the client)
 18 | 	signal chan struct{} // Used to wake up the connection producer
 19 | 	err    chan error    // Send dial errors back to the client
 20 | }
 21 | 
 22 | // conn wraps an io.WriteCloser, marking the connection as dead
 23 | // on any write error and changing the meaning of Close.
 24 | // Closing a conn calls the io.WriteCloser's Close method if
 25 | // the conn is marked dead, or returns it to the pool otherwise.
 26 | type conn struct {
 27 | 	conn io.WriteCloser
 28 | 	pool *LimitedConnPool
 29 | 	dead bool
 30 | }
 31 | 
 32 | func (w *conn) Write(p []byte) (int, error) {
 33 | 	n, err := w.conn.Write(p)
 34 | 	if err != nil {
 35 | 		w.dead = true
 36 | 	}
 37 | 	return n, err
 38 | }
 39 | 
 40 | func (w *conn) Close() error {
 41 | 	return w.pool.put(w)
 42 | }
 43 | 
 44 | type bufferedWriter interface {
 45 | 	Flush() error
 46 | }
 47 | 
 48 | func (w *conn) Flush() error {
 49 | 	if t, ok := w.conn.(bufferedWriter); ok {
 50 | 		return t.Flush()
 51 | 	}
 52 | 	return nil
 53 | }
 54 | 
 55 | // NewLimited returns a new LimitedConnPool with the given size limit and dial function.
 56 | func NewLimited(size int, dial func() (io.WriteCloser, error)) (*LimitedConnPool, error) {
 57 | 	// Tentative first try - if this doesn't work, we assume it never will
 58 | 	// and fail to initialize. This is admittedly not great, but we rely on
 59 | 	// unreachable addresses failing immediately in our syslog package, which,
 60 | 	// if no address is specified, attempts a number of fallback addresses
 61 | 	// until one succeeds.
 62 | 	w, err := dial()
 63 | 	if err != nil {
 64 | 		return nil, err
 65 | 	}
 66 | 
 67 | 	p := LimitedConnPool{
 68 | 		conns:  make(chan *conn, size),
 69 | 		live:   make(chan struct{}, size),
 70 | 		signal: make(chan struct{}),
 71 | 
 72 | 		// try to make this large enough to avoid dropping
 73 | 		// errors if clients only check errors occasionally
 74 | 		err: make(chan error, size),
 75 | 	}
 76 | 
 77 | 	p.conns <- &conn{
 78 | 		conn: w,
 79 | 		pool: &p,
 80 | 	}
 81 | 	p.live <- struct{}{}
 82 | 
 83 | 	// keep p.conns populated
 84 | 	go func() {
 85 | 		// TODO: it would be nice if the client could control
 86 | 		// backoff, but doing it here seems sufficient for now.
 87 | 		backoff := &backoff.Backoff{
 88 | 			Factor: 2,
 89 | 			Min:    10 * time.Millisecond,
 90 | 			Max:    10 * time.Second,
 91 | 		}
 92 | 		for range p.signal {
 93 | 			for len(p.live) < size {
 94 | 				w, err := dial()
 95 | 				if err != nil {
 96 | 					select {
 97 | 					case p.err <- err:
 98 | 					default:
 99 | 						// error channel is full, drop this error.
100 | 					}
101 | 					time.Sleep(backoff.Duration())
102 | 					continue
103 | 				}
104 | 				backoff.Reset()
105 | 				p.conns <- &conn{
106 | 					conn: w,
107 | 					pool: &p,
108 | 				}
109 | 				p.live <- struct{}{}
110 | 			}
111 | 		}
112 | 	}()
113 | 
114 | 	// kick off the producer
115 | 	p.signal <- struct{}{}
116 | 
117 | 	return &p, nil
118 | }
119 | 
120 | func (p *LimitedConnPool) Close() {
121 | 	// Important to close this first, so the dialer doesn't loop again.
122 | 	close(p.signal)
123 | 
124 | 	// Close all the underlying connections
125 | 	close(p.conns)
126 | 	for c := range p.conns {
127 | 		c.conn.Close()
128 | 	}
129 | 
130 | 	// Close the error channel, allowing any client error handling
131 | 	// range loops to finish
132 | 	close(p.err)
133 | }
134 | 
135 | // put returns a connection to the pool. If the connection is dead,
136 | // it is removed from the pool so that a new connection can be dialed.
137 | func (p *LimitedConnPool) put(w *conn) error {
138 | 	if w.dead {
139 | 		// decrement the live connection count
140 | 		<-p.live
141 | 
142 | 		// signal the connection dialer if necessary
143 | 		select {
144 | 		case p.signal <- struct{}{}:
145 | 		default:
146 | 		}
147 | 
148 | 		return w.conn.Close()
149 | 	}
150 | 	p.conns <- w
151 | 	return nil
152 | }
153 | 
154 | // Get retrieves a connection from the pool, if available.
155 | // A new connection will only be dialed if the total number
156 | // of live connections is below the configured size limit.
157 | // Closing the returned io.WriteCloser automatically returns
158 | // the connection to the pool.
159 | func (p *LimitedConnPool) Get() io.WriteCloser {
160 | 	return <-p.conns
161 | }
162 | 
163 | // Errors returns a channel of errors encountered when dialing new
164 | // connections. Errors will be dropped if this channel is not
165 | // consumed.
166 | func (p *LimitedConnPool) Errors() <-chan error {
167 | 	return p.err
168 | }
169 | 


--------------------------------------------------------------------------------
/lib/syslog/pool/pool_test.go:
--------------------------------------------------------------------------------
 1 | package pool
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"io"
 6 | 	"io/ioutil"
 7 | 	"math/rand"
 8 | 	"os"
 9 | 	"sync"
10 | 	"sync/atomic"
11 | 	"testing"
12 | 	"time"
13 | )
14 | 
15 | func TestPool(t *testing.T) {
16 | 	const poolSize = 100
17 | 
18 | 	dir, err := ioutil.TempDir("", "pool_test")
19 | 	if err != nil {
20 | 		t.Fatal(err)
21 | 	}
22 | 	defer os.RemoveAll(dir)
23 | 
24 | 	var newConnections int
25 | 	dial := func() (io.WriteCloser, error) {
26 | 		newConnections++
27 | 		return ioutil.TempFile(dir, "")
28 | 	}
29 | 
30 | 	p, err := NewLimited(poolSize, dial)
31 | 	if err != nil {
32 | 		t.Fatal(err)
33 | 	}
34 | 	defer p.Close()
35 | 
36 | 	// consume errors from the pool
37 | 	var failures uint64
38 | 	go func() {
39 | 		for err := range p.Errors() {
40 | 			atomic.AddUint64(&failures, 1)
41 | 			fmt.Fprintln(os.Stderr, err.Error())
42 | 		}
43 | 	}()
44 | 
45 | 	// report pool stats once per second
46 | 	go func() {
47 | 		start := time.Now()
48 | 		for range time.Tick(1 * time.Second) {
49 | 			t.Logf("T+%02.2vs: live=%d pool=%d failures=%d\n", time.Since(start).Seconds(), len(p.live), len(p.conns), atomic.LoadUint64(&failures))
50 | 		}
51 | 	}()
52 | 
53 | 	var wg sync.WaitGroup
54 | 
55 | 	// start 2*poolSize writer goroutines
56 | 	for i := 0; i < 2*poolSize; i++ {
57 | 		start := time.Now()
58 | 		stop := 10 * time.Second
59 | 		if testing.Short() {
60 | 			stop = 2 * time.Second
61 | 		}
62 | 		wg.Add(1)
63 | 		go func() {
64 | 			for time.Since(start) < stop {
65 | 				// wait a bit before getting a conn from the pool
66 | 				time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
67 | 				w := p.Get()
68 | 
69 | 				// random number of writes, with random waits in between
70 | 				for j := 0; j < rand.Intn(10); j++ {
71 | 					time.Sleep(time.Duration(rand.Intn(10)) * time.Millisecond)
72 | 					if _, err := w.Write([]byte("test\n")); err != nil {
73 | 						t.Error(err)
74 | 					}
75 | 				}
76 | 
77 | 				// return the conn to the pool
78 | 				if err := w.Close(); err != nil {
79 | 					t.Error(err)
80 | 				}
81 | 			}
82 | 			wg.Done()
83 | 		}()
84 | 	}
85 | 
86 | 	wg.Wait()
87 | 
88 | 	if newConnections != poolSize {
89 | 		t.Errorf("dialed %d connections, want %d", newConnections, poolSize)
90 | 	}
91 | }
92 | 


--------------------------------------------------------------------------------
/lib/syslog/writer.go:
--------------------------------------------------------------------------------
  1 | package syslog
  2 | 
  3 | import (
  4 | 	"bufio"
  5 | 	"bytes"
  6 | 	"crypto/tls"
  7 | 	"fmt"
  8 | 	"io"
  9 | 	"net"
 10 | 	"net/url"
 11 | 	"os"
 12 | 	"strconv"
 13 | 	"strings"
 14 | 	"sync"
 15 | 	"text/template"
 16 | 	"time"
 17 | 
 18 | 	"github.com/segmentio/ecs-logs/lib"
 19 | 	"github.com/segmentio/ecs-logs/lib/syslog/pool"
 20 | 
 21 | 	"golang.org/x/net/proxy"
 22 | )
 23 | 
 24 | const DefaultTemplate = "<{{.PRIVAL}}>{{.TIMESTAMP}} {{.GROUP}}[{{.STREAM}}]: {{.MSG}}"
 25 | 
 26 | const (
 27 | 	poolSize    = 20
 28 | 	dialTimeout = 10 * time.Second
 29 | )
 30 | 
 31 | var (
 32 | 	connPoolsLock sync.Mutex
 33 | 	connPools     map[string]*pool.LimitedConnPool
 34 | )
 35 | 
 36 | type WriterConfig struct {
 37 | 	Network    string
 38 | 	Address    string
 39 | 	Template   string
 40 | 	TimeFormat string
 41 | 	Tag        string
 42 | 	TLS        *tls.Config
 43 | 	SocksProxy string
 44 | }
 45 | 
 46 | // dialOpts is used to determine whether writers can share
 47 | // the same connection pool.
 48 | type dialOpts struct {
 49 | 	network    string
 50 | 	address    string
 51 | 	tls        *tls.Config
 52 | 	socksProxy string
 53 | }
 54 | 
 55 | // BUG: the generated key does not capture the TLS config,
 56 | // so we are assuming that all otherwise identical dialOpts
 57 | // have the same TLS config.
 58 | func (o *dialOpts) key() string {
 59 | 	return fmt.Sprintf("%s:%s:%s", o.network, o.address, o.socksProxy)
 60 | }
 61 | 
 62 | func init() {
 63 | 	connPools = make(map[string]*pool.LimitedConnPool)
 64 | }
 65 | 
 66 | func NewWriter(group, stream string) (lib.Writer, error) {
 67 | 	var c WriterConfig
 68 | 
 69 | 	if s := os.Getenv("SYSLOG_URL"); len(s) != 0 {
 70 | 		u, err := url.Parse(s)
 71 | 		if err != nil {
 72 | 			return nil, fmt.Errorf("invalid syslog URL: %s", err)
 73 | 		}
 74 | 
 75 | 		c.Network = u.Scheme
 76 | 		c.Address = u.Host
 77 | 	}
 78 | 
 79 | 	c.Template = os.Getenv("SYSLOG_TEMPLATE")
 80 | 	c.TimeFormat = os.Getenv("SYSLOG_TIME_FORMAT")
 81 | 
 82 | 	return DialWriter(c)
 83 | }
 84 | 
 85 | func DialWriter(config WriterConfig) (lib.Writer, error) {
 86 | 	var netopts, addropts []string
 87 | 
 88 | 	if len(config.Network) != 0 {
 89 | 		netopts = []string{config.Network}
 90 | 	} else if len(config.Address) == 0 || strings.HasPrefix(config.Address, "/") {
 91 | 		// When starting with a '/' we assume it's gonna be a file path,
 92 | 		// otherwise we fallback to trying a TLS connection so we don't
 93 | 		// implicitly send logs over an unsecured link.
 94 | 		config.Network = "unix"
 95 | 		netopts = []string{"unixgram", "unix"}
 96 | 	} else {
 97 | 		netopts = []string{"tls"}
 98 | 	}
 99 | 
100 | 	if len(config.Address) != 0 {
101 | 		addropts = []string{config.Address}
102 | 	} else if strings.HasPrefix(config.Network, "unix") {
103 | 		// This was copied from the standard log/syslog package, they do the same
104 | 		// and try to guess at runtime which socket syslogd is using.
105 | 		addropts = []string{"/dev/log", "/var/run/syslog", "/var/run/log"}
106 | 	} else {
107 | 		// The config doesn't point to a unix domain socket, falling back to trying
108 | 		// to connect to syslogd over a network interface.
109 | 		addropts = []string{"localhost:514"}
110 | 	}
111 | 
112 | 	// Try various fallbacks if no hints were given
113 | 	var w *writer
114 | 	var err error
115 | 	for _, n := range netopts {
116 | 		for _, a := range addropts {
117 | 			opts := dialOpts{
118 | 				network:    n,
119 | 				address:    a,
120 | 				tls:        config.TLS,
121 | 				socksProxy: config.SocksProxy,
122 | 			}
123 | 			if w, err = newWriter(opts, config); err == nil {
124 | 				return w, nil
125 | 			}
126 | 		}
127 | 	}
128 | 
129 | 	return nil, err
130 | }
131 | 
132 | type multiError []error
133 | 
134 | func (m multiError) Error() string {
135 | 	s := "encountered errors:\n"
136 | 	for err := range m {
137 | 		s += fmt.Sprintf("\t%v\n", err)
138 | 	}
139 | 	return s
140 | }
141 | 
142 | type writer struct {
143 | 	// configuration
144 | 	timefmt string
145 | 	tpl     *template.Template
146 | 	tag     string
147 | 
148 | 	// connection state
149 | 	pool    *pool.LimitedConnPool
150 | 	backend io.WriteCloser
151 | 
152 | 	// buffered i/o
153 | 	buf   bytes.Buffer
154 | 	out   func(*writer, message) error
155 | 	flush func() error
156 | }
157 | 
158 | func newWriter(opts dialOpts, cfg WriterConfig) (*writer, error) {
159 | 	var out func(*writer, message) error
160 | 	var flush func() error
161 | 
162 | 	if cfg.TimeFormat == "" {
163 | 		cfg.TimeFormat = time.Stamp
164 | 	}
165 | 
166 | 	if cfg.Template == "" {
167 | 		cfg.Template = DefaultTemplate
168 | 	}
169 | 
170 | 	p, err := getPool(opts)
171 | 	if err != nil {
172 | 		return nil, err
173 | 	}
174 | 
175 | 	backend := p.Get()
176 | 	switch b := backend.(type) {
177 | 	case bufferedWriter:
178 | 		out, flush = (*writer).directWrite, b.Flush
179 | 	default:
180 | 		out, flush = (*writer).bufferedWrite, func() error { return nil }
181 | 	}
182 | 
183 | 	// Check for errors reported by the pool when dialing
184 | 	errc := p.Errors()
185 | 	var errs multiError
186 | loop:
187 | 	for {
188 | 		select {
189 | 		case err := <-errc:
190 | 			errs = append(errs, err)
191 | 		default:
192 | 			break loop
193 | 		}
194 | 	}
195 | 	if errs != nil {
196 | 		return nil, errs
197 | 	}
198 | 
199 | 	return &writer{
200 | 		timefmt: cfg.TimeFormat,
201 | 		tpl:     newWriterTemplate(cfg.Template),
202 | 		tag:     cfg.Tag,
203 | 
204 | 		backend: backend,
205 | 		pool:    p,
206 | 
207 | 		flush: flush,
208 | 		out:   out,
209 | 	}, nil
210 | }
211 | 
212 | // getPool returns a connection pool for the given configuration.
213 | func getPool(opts dialOpts) (*pool.LimitedConnPool, error) {
214 | 	connPoolsLock.Lock()
215 | 	defer connPoolsLock.Unlock()
216 | 
217 | 	key := opts.key()
218 | 	p, ok := connPools[key]
219 | 	if !ok {
220 | 		// dial closes over opts
221 | 		dial := func() (io.WriteCloser, error) {
222 | 			return dialWriter(opts.network, opts.address, opts.tls, opts.socksProxy)
223 | 		}
224 | 		var err error
225 | 		p, err = pool.NewLimited(poolSize, dial)
226 | 		if err != nil {
227 | 			return nil, err
228 | 		}
229 | 		connPools[key] = p
230 | 	}
231 | 
232 | 	return p, nil
233 | }
234 | 
235 | func newWriterTemplate(format string) *template.Template {
236 | 	if !strings.HasSuffix(format, "\n") {
237 | 		format += "\n"
238 | 	}
239 | 	t := template.New("syslog")
240 | 	template.Must(t.Parse(format))
241 | 	return t
242 | }
243 | 
244 | func (w *writer) Close() (err error) {
245 | 	return w.backend.Close()
246 | }
247 | 
248 | func (w *writer) WriteMessageBatch(batch lib.MessageBatch) error {
249 | 	for _, msg := range batch {
250 | 		if err := w.write(msg); err != nil {
251 | 			return err
252 | 		}
253 | 	}
254 | 	if err := w.flush(); err != nil {
255 | 		return err
256 | 	}
257 | 	return nil
258 | }
259 | 
260 | func (w *writer) WriteMessage(msg lib.Message) error {
261 | 	if err := w.write(msg); err != nil {
262 | 		return err
263 | 	}
264 | 	if err := w.flush(); err != nil {
265 | 		return err
266 | 	}
267 | 
268 | 	return nil
269 | }
270 | 
271 | func (w *writer) write(msg lib.Message) (err error) {
272 | 	m := message{
273 | 		PRIVAL:    int(msg.Event.Level-1) + 8, // +8 is for user-level messages facility
274 | 		HOSTNAME:  msg.Event.Info.Host,
275 | 		MSGID:     msg.Event.Info.ID,
276 | 		GROUP:     msg.Group,
277 | 		STREAM:    msg.Stream,
278 | 		TIMESTAMP: msg.Event.Time.Format(w.timefmt),
279 | 		TAG:       w.tag,
280 | 	}
281 | 
282 | 	if len(m.HOSTNAME) == 0 {
283 | 		m.HOSTNAME = "-"
284 | 	}
285 | 
286 | 	if len(m.MSGID) == 0 {
287 | 		m.MSGID = "-"
288 | 	}
289 | 
290 | 	if msg.Event.Info.PID == 0 {
291 | 		m.PROCID = "-"
292 | 	} else {
293 | 		m.PROCID = strconv.Itoa(msg.Event.Info.PID)
294 | 	}
295 | 
296 | 	m.MSG = msg.Event.String()
297 | 	return w.out(w, m)
298 | }
299 | 
300 | func (w *writer) directWrite(m message) (err error) {
301 | 	return w.tpl.Execute(w.backend, m)
302 | }
303 | 
304 | func (w *writer) bufferedWrite(m message) (err error) {
305 | 	w.buf.Reset()
306 | 	w.tpl.Execute(&w.buf, m)
307 | 	_, err = w.backend.Write(w.buf.Bytes())
308 | 	return
309 | }
310 | 
311 | type message struct {
312 | 	PRIVAL    int
313 | 	HOSTNAME  string
314 | 	PROCID    string
315 | 	MSGID     string
316 | 	GROUP     string
317 | 	STREAM    string
318 | 	TAG       string
319 | 	MSG       string
320 | 	TIMESTAMP string
321 | }
322 | 
323 | type bufferedWriter interface {
324 | 	Flush() error
325 | }
326 | 
327 | type bufferedConn struct {
328 | 	buf  *bufio.Writer
329 | 	conn net.Conn
330 | }
331 | 
332 | func (c bufferedConn) Close() error                { return c.conn.Close() }
333 | func (c bufferedConn) Flush() error                { return c.buf.Flush() }
334 | func (c bufferedConn) Write(b []byte) (int, error) { return c.buf.Write(b) }
335 | 
336 | func dialWriter(network, address string, config *tls.Config, socksProxy string) (w io.WriteCloser, err error) {
337 | 	var conn, rawConn net.Conn
338 | 	var dial func(string, string) (net.Conn, error)
339 | 	var socksDialer proxy.Dialer
340 | 
341 | 	dialer := net.Dialer{
342 | 		Timeout: dialTimeout,
343 | 	}
344 | 	if network == "tls" {
345 | 		network = "tcp"
346 | 		dial = func(network, address string) (net.Conn, error) {
347 | 			return tls.DialWithDialer(&dialer, network, address, config)
348 | 		}
349 | 	} else {
350 | 		dial = dialer.Dial
351 | 	}
352 | 
353 | 	if socksProxy != "" {
354 | 		if socksDialer, err = proxy.SOCKS5(network, socksProxy, nil, proxy.Direct); err != nil {
355 | 			return
356 | 		}
357 | 
358 | 		dial = func(network, address string) (conn net.Conn, err error) {
359 | 			if config == nil {
360 | 				conn, err = socksDialer.Dial(network, address)
361 | 			} else {
362 | 				rawConn, err = socksDialer.Dial(network, address)
363 | 				if err != nil {
364 | 					return nil, err
365 | 				}
366 | 
367 | 				tlsConn := tls.Client(rawConn, config)
368 | 				if err = tlsConn.Handshake(); err == nil {
369 | 					conn = tlsConn
370 | 				}
371 | 			}
372 | 			return
373 | 		}
374 | 	}
375 | 
376 | 	for attempt := 1; true; attempt++ {
377 | 		if conn, err = dial(network, address); err == nil {
378 | 			break
379 | 		}
380 | 
381 | 		if attempt == 3 {
382 | 			return
383 | 		}
384 | 
385 | 		err = nil
386 | 		time.Sleep(1 * time.Second)
387 | 	}
388 | 
389 | 	if err == nil {
390 | 		switch network {
391 | 		case "udp", "udp4", "udp6", "unixgram", "unixpacket":
392 | 			w = conn
393 | 		default:
394 | 			w = bufferedConn{
395 | 				conn: conn,
396 | 				buf:  bufio.NewWriter(conn),
397 | 			}
398 | 		}
399 | 	}
400 | 
401 | 	return
402 | }
403 | 


--------------------------------------------------------------------------------
/lib/syslog/writer_test.go:
--------------------------------------------------------------------------------
  1 | package syslog
  2 | 
  3 | import (
  4 | 	"crypto/tls"
  5 | 	"fmt"
  6 | 	"math/rand"
  7 | 	"net/url"
  8 | 	"os"
  9 | 	"testing"
 10 | 	"time"
 11 | 
 12 | 	ecslogs "github.com/segmentio/ecs-logs-go"
 13 | 	"github.com/segmentio/ecs-logs/lib"
 14 | )
 15 | 
 16 | const testGoroutines = 50
 17 | 
 18 | func TestWriter(t *testing.T) {
 19 | 	_, ok := os.LookupEnv("SYSLOG_URL")
 20 | 	if !ok {
 21 | 		t.Skip("Skipping test that relies on syslog")
 22 | 	}
 23 | 	// Start a bunch of workers who open and close writers like crazy.
 24 | 	errc := make(chan error, testGoroutines)
 25 | 	start := time.Now()
 26 | 	for i := 0; i < testGoroutines; i++ {
 27 | 		go func(i int) {
 28 | 			for {
 29 | 				time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
 30 | 				w, err := NewWriter("foo", "bar")
 31 | 				if err != nil {
 32 | 					errc <- err
 33 | 				}
 34 | 				for j := 0; j < rand.Intn(10); j++ {
 35 | 					d := time.Duration(rand.Intn(30)) * time.Millisecond
 36 | 					time.Sleep(d)
 37 | 					err = w.WriteMessage(lib.Message{
 38 | 						Group:  "foo",
 39 | 						Stream: "bar",
 40 | 						Event:  ecslogs.MakeEvent(ecslogs.INFO, fmt.Sprintf("slept %v", d)),
 41 | 					})
 42 | 					if err != nil {
 43 | 						errc <- err
 44 | 					}
 45 | 
 46 | 				}
 47 | 
 48 | 				if err := w.Close(); err != nil {
 49 | 					errc <- err
 50 | 				}
 51 | 
 52 | 				if time.Since(start) >= 5*time.Second {
 53 | 					// signal to the main goroutine that we
 54 | 					// are exiting with no errors.
 55 | 					errc <- nil
 56 | 					return
 57 | 				}
 58 | 			}
 59 | 		}(i)
 60 | 	}
 61 | 
 62 | 	for i := 0; i < testGoroutines; i++ {
 63 | 		if err := <-errc; err != nil {
 64 | 			t.Error(err)
 65 | 		}
 66 | 	}
 67 | }
 68 | 
 69 | func TestGetPool(t *testing.T) {
 70 | 	s, ok := os.LookupEnv("SYSLOG_URL")
 71 | 	if !ok {
 72 | 		t.Skip("Skipping test that relies on syslog")
 73 | 	}
 74 | 	u, err := url.Parse(s)
 75 | 	if err != nil {
 76 | 		t.Fatal(err)
 77 | 	}
 78 | 
 79 | 	opts := dialOpts{
 80 | 		network: u.Scheme,
 81 | 		address: u.Host,
 82 | 		tls: &tls.Config{
 83 | 			InsecureSkipVerify: true,
 84 | 		},
 85 | 		socksProxy: "",
 86 | 	}
 87 | 	pool1, err := getPool(opts)
 88 | 	if err != nil {
 89 | 		t.Fatal(err)
 90 | 	}
 91 | 	defer pool1.Close()
 92 | 
 93 | 	// Create a new dialOpts identical to the old one
 94 | 	opts = dialOpts{
 95 | 		network: u.Scheme,
 96 | 		address: u.Host,
 97 | 		tls: &tls.Config{
 98 | 			InsecureSkipVerify: true,
 99 | 		},
100 | 		socksProxy: "",
101 | 	}
102 | 	pool2, err := getPool(opts)
103 | 	if err != nil {
104 | 		t.Fatal(err)
105 | 	}
106 | 
107 | 	if pool1 != pool2 {
108 | 		t.Error("pools did not match")
109 | 	}
110 | }
111 | 
112 | func BenchmarkNewWriter(b *testing.B) {
113 | 	for i := 0; i < b.N; i++ {
114 | 		w, err := NewWriter("foo", "bar")
115 | 		if err != nil {
116 | 			b.Fatal(err)
117 | 		}
118 | 		for j := 0; j < 10; j++ {
119 | 			err := w.WriteMessage(lib.Message{
120 | 				Group:  "foo",
121 | 				Stream: "bar",
122 | 				Event:  ecslogs.MakeEvent(ecslogs.INFO, "test"),
123 | 			})
124 | 			if err != nil {
125 | 				b.Error(err)
126 | 			}
127 | 		}
128 | 		if err := w.Close(); err != nil {
129 | 			b.Fatal(err)
130 | 		}
131 | 	}
132 | }
133 | 
134 | func BenchmarkWrite(b *testing.B) {
135 | 	w, err := NewWriter("foo", "bar")
136 | 	if err != nil {
137 | 		b.Fatal(err)
138 | 	}
139 | 	for i := 0; i < b.N; i++ {
140 | 		err := w.WriteMessage(lib.Message{
141 | 			Group:  "foo",
142 | 			Stream: "bar",
143 | 			Event:  ecslogs.MakeEvent(ecslogs.INFO, "test"),
144 | 		})
145 | 		if err != nil {
146 | 			b.Error(err)
147 | 		}
148 | 	}
149 | 	if err := w.Close(); err != nil {
150 | 		b.Fatal(err)
151 | 	}
152 | }
153 | 


--------------------------------------------------------------------------------
/lib/writer.go:
--------------------------------------------------------------------------------
 1 | package lib
 2 | 
 3 | import (
 4 | 	"encoding/json"
 5 | 	"io"
 6 | )
 7 | 
 8 | type Writer interface {
 9 | 	io.Closer
10 | 
11 | 	WriteMessage(Message) error
12 | 
13 | 	WriteMessageBatch(MessageBatch) error
14 | }
15 | 
16 | func NewMessageEncoder(w io.Writer) Writer {
17 | 	return encoder{
18 | 		j: json.NewEncoder(w),
19 | 		w: w,
20 | 	}
21 | }
22 | 
23 | type encoder struct {
24 | 	j *json.Encoder
25 | 	w io.Writer
26 | }
27 | 
28 | func (e encoder) Close() (err error) {
29 | 	return
30 | }
31 | 
32 | func (e encoder) WriteMessage(msg Message) (err error) {
33 | 	err = e.j.Encode(msg)
34 | 	return
35 | }
36 | 
37 | func (e encoder) WriteMessageBatch(batch MessageBatch) (err error) {
38 | 	for _, msg := range batch {
39 | 		if err = e.WriteMessage(msg); err != nil {
40 | 			return
41 | 		}
42 | 	}
43 | 	return
44 | }
45 | 


--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"flag"
  5 | 	"fmt"
  6 | 	"io"
  7 | 	"os"
  8 | 	"os/signal"
  9 | 	"sort"
 10 | 	"strings"
 11 | 	"sync"
 12 | 	"sync/atomic"
 13 | 	"syscall"
 14 | 	"time"
 15 | 
 16 | 	"net/http"
 17 | 	_ "net/http/pprof"
 18 | 
 19 | 	"github.com/apex/log"
 20 | 	"github.com/apex/log/handlers/cli"
 21 | 	"github.com/apex/log/handlers/multi"
 22 | 	"github.com/segmentio/ecs-logs-go"
 23 | 	"github.com/segmentio/ecs-logs/lib"
 24 | 
 25 | 	_ "github.com/segmentio/ecs-logs/lib/cloudwatchlogs"
 26 | 	_ "github.com/segmentio/ecs-logs/lib/datadog"
 27 | 	_ "github.com/segmentio/ecs-logs/lib/logdna"
 28 | 	_ "github.com/segmentio/ecs-logs/lib/loggly"
 29 | 	_ "github.com/segmentio/ecs-logs/lib/statsd"
 30 | 	_ "github.com/segmentio/ecs-logs/lib/syslog"
 31 | )
 32 | 
 33 | type source struct {
 34 | 	lib.Source
 35 | 	name string
 36 | }
 37 | 
 38 | type destination struct {
 39 | 	lib.Destination
 40 | 	name string
 41 | }
 42 | 
 43 | type reader struct {
 44 | 	lib.Reader
 45 | 	name string
 46 | }
 47 | 
 48 | func main() {
 49 | 	var err error
 50 | 	var src string
 51 | 	var dst string
 52 | 	var hostname string
 53 | 	var level = lib.LogLevel(log.InfoLevel)
 54 | 	var maxBytes int
 55 | 	var maxCount int
 56 | 	var flushTimeout time.Duration
 57 | 	var cacheTimeout time.Duration
 58 | 	var profileAddr string
 59 | 
 60 | 	hostname, _ = os.Hostname()
 61 | 
 62 | 	flag.StringVar(&src, "src", "stdin", "A comma separated list of log sources from which messages will be read ["+strings.Join(lib.SourcesAvailable(), ", ")+"]")
 63 | 	flag.StringVar(&dst, "dst", "stdout", "A comma separated list of log destinations to which messages will be written ["+strings.Join(lib.DestinationsAvailable(), ", ")+"]")
 64 | 	flag.StringVar(&hostname, "hostname", hostname, "The hostname advertised by ecs-logs")
 65 | 	flag.Var(&level, "log-level", "The minimum level of log messages shown by ecs-logs")
 66 | 	flag.IntVar(&maxBytes, "max-batch-bytes", 1000000, "The maximum size in bytes of a message batch")
 67 | 	flag.IntVar(&maxCount, "max-batch-size", 10000, "The maximum number of messages in a batch")
 68 | 	flag.DurationVar(&flushTimeout, "flush-timeout", 5*time.Second, "How often messages will be flushed")
 69 | 	flag.DurationVar(&cacheTimeout, "cache-timeout", 5*time.Minute, "How to wait before clearing unused internal cache")
 70 | 	flag.StringVar(&profileAddr, "pprof-addr", "", "Address to serve profile information")
 71 | 	flag.Parse()
 72 | 
 73 | 	logger := &lib.LogHandler{
 74 | 		Group:    "ecs-logs",
 75 | 		Stream:   hostname,
 76 | 		Hostname: hostname,
 77 | 		Queue:    lib.NewMessageQueue(),
 78 | 	}
 79 | 	log.SetLevel(log.Level(level))
 80 | 	log.SetHandler(multi.New(cli.New(os.Stderr), logger))
 81 | 
 82 | 	// serve profiles if address is configured
 83 | 	if profileAddr != "" {
 84 | 		go func() {
 85 | 			if err := http.ListenAndServe(profileAddr, nil); err != nil {
 86 | 				log.Errorf("pprof: %v", err)
 87 | 			}
 88 | 		}()
 89 | 	}
 90 | 
 91 | 	var store = lib.NewStore()
 92 | 	var sources []source
 93 | 	var readers []reader
 94 | 	var dests []destination
 95 | 
 96 | 	if len(hostname) == 0 {
 97 | 		log.Fatal("no hostname configured")
 98 | 	}
 99 | 
100 | 	if sources = getSources(strings.Split(src, ",")); len(sources) == 0 {
101 | 		log.Fatal("no or invalid log sources")
102 | 	}
103 | 
104 | 	if dests = getDestinations(strings.Split(dst, ",")); len(dests) == 0 {
105 | 		log.Fatal("no or invalid log destinations")
106 | 	}
107 | 
108 | 	if readers, err = openSources(sources); err != nil {
109 | 		log.WithError(err).Fatal("failed to open log sources readers")
110 | 	}
111 | 
112 | 	join := &sync.WaitGroup{}
113 | 
114 | 	limits := lib.StreamLimits{
115 | 		MaxCount: maxCount,
116 | 		MaxBytes: maxBytes,
117 | 		MaxTime:  flushTimeout,
118 | 	}
119 | 
120 | 	expchan := time.Tick(flushTimeout / 2)
121 | 	msgchan := make(chan lib.Message, len(readers))
122 | 	sigchan := make(chan os.Signal, 1)
123 | 	counter := int32(len(readers))
124 | 	startReaders(readers, msgchan, &counter, hostname)
125 | 	setupSignals(sigchan)
126 | 
127 | 	for _, s := range sources {
128 | 		log.WithField("source", s.name).Info("source enabled")
129 | 	}
130 | 
131 | 	for _, d := range dests {
132 | 		log.WithField("destination", d.name).Info("destination enabled")
133 | 	}
134 | 
135 | 	for {
136 | 		select {
137 | 		case msg, ok := <-msgchan:
138 | 			now := time.Now()
139 | 
140 | 			if !ok {
141 | 				log.Info("waiting for all write operations to complete")
142 | 				limits.Force = true
143 | 				flushAll(dests, store, limits, now, join)
144 | 				flushQueue(dests, store, logger.Queue, limits, now, join)
145 | 				join.Wait()
146 | 				return
147 | 			}
148 | 
149 | 			_, stream := store.Add(msg, now)
150 | 			flush(dests, stream, limits, now, join)
151 | 
152 | 		case <-logger.Queue.C:
153 | 			now := time.Now()
154 | 			flushQueue(dests, store, logger.Queue, limits, now, join)
155 | 
156 | 		case <-expchan:
157 | 			now := time.Now()
158 | 			flushAll(dests, store, limits, now, join)
159 | 			removeExpired(dests, store, cacheTimeout, now)
160 | 
161 | 		case sig := <-sigchan:
162 | 			log.WithFields(log.Fields{"signal": sig.String()}).Info("closing message readers")
163 | 			stopReaders(readers)
164 | 		}
165 | 	}
166 | }
167 | 
168 | func getSources(names []string) (sources []source) {
169 | 	for i, src := range lib.GetSources(names...) {
170 | 		sources = append(sources, source{
171 | 			Source: src,
172 | 			name:   names[i],
173 | 		})
174 | 	}
175 | 
176 | 	if len(sources) != len(names) {
177 | 	search:
178 | 		for _, name := range names {
179 | 			for _, source := range sources {
180 | 				if name == source.name {
181 | 					continue search
182 | 				}
183 | 			}
184 | 			log.WithFields(log.Fields{"source": name}).Warn("source disabled")
185 | 		}
186 | 	}
187 | 
188 | 	return
189 | }
190 | 
191 | func getDestinations(names []string) (destinations []destination) {
192 | 	for i, dst := range lib.GetDestinations(names...) {
193 | 		destinations = append(destinations, destination{
194 | 			Destination: dst,
195 | 			name:        names[i],
196 | 		})
197 | 	}
198 | 
199 | 	if len(destinations) != len(names) {
200 | 	search:
201 | 		for _, name := range names {
202 | 			for _, destination := range destinations {
203 | 				if name == destination.name {
204 | 					continue search
205 | 				}
206 | 			}
207 | 			log.WithFields(log.Fields{"destination": name}).Warn("destination disabled")
208 | 		}
209 | 	}
210 | 
211 | 	return
212 | }
213 | 
214 | func openSources(sources []source) (readers []reader, err error) {
215 | 	readers = make([]reader, 0, len(sources))
216 | 
217 | 	for _, source := range sources {
218 | 		if r, e := source.Open(); e != nil {
219 | 			log.WithFields(log.Fields{
220 | 				"source": source.name,
221 | 				"error":  e,
222 | 			}).Error("failed to open log source")
223 | 		} else {
224 | 			readers = append(readers, reader{
225 | 				Reader: r,
226 | 				name:   source.name,
227 | 			})
228 | 		}
229 | 	}
230 | 
231 | 	if len(readers) == 0 {
232 | 		err = fmt.Errorf("no sources to read from")
233 | 	}
234 | 
235 | 	return
236 | }
237 | 
238 | func setupSignals(sigchan chan<- os.Signal) {
239 | 	signal.Notify(sigchan, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM)
240 | }
241 | 
242 | func startReaders(readers []reader, msgchan chan<- lib.Message, counter *int32, hostname string) {
243 | 	for _, reader := range readers {
244 | 		go read(reader, msgchan, counter, hostname)
245 | 	}
246 | }
247 | 
248 | func stopReaders(readers []reader) {
249 | 	for _, reader := range readers {
250 | 		reader.Close()
251 | 	}
252 | }
253 | 
254 | func term(c chan<- lib.Message, counter *int32) {
255 | 	if atomic.AddInt32(counter, -1) == 0 {
256 | 		close(c)
257 | 	}
258 | }
259 | 
260 | func read(r reader, c chan<- lib.Message, counter *int32, hostname string) {
261 | 	defer term(c, counter)
262 | 	for {
263 | 		var msg lib.Message
264 | 		var err error
265 | 
266 | 		if msg, err = r.ReadMessage(); err != nil {
267 | 			if err == io.EOF {
268 | 				log.WithFields(log.Fields{
269 | 					"reader": r.name,
270 | 				}).Info("the message reader was closed")
271 | 			} else {
272 | 				log.WithFields(log.Fields{
273 | 					"reader": r.name,
274 | 					"error":  err,
275 | 				}).Error("the message reader failed")
276 | 			}
277 | 			return
278 | 		}
279 | 
280 | 		if len(msg.Group) == 0 {
281 | 			log.WithFields(log.Fields{
282 | 				"reader":  r.name,
283 | 				"missing": "group",
284 | 			}).Warn("dropping message because the a required field wasn't set")
285 | 			continue
286 | 		}
287 | 
288 | 		if len(msg.Stream) == 0 {
289 | 			log.WithFields(log.Fields{
290 | 				"reader":  r.name,
291 | 				"missing": "stream",
292 | 			}).Warn("dropping message because the a required field wasn't set")
293 | 			continue
294 | 		}
295 | 
296 | 		if len(msg.Event.Info.Host) == 0 {
297 | 			msg.Event.Info.Host = hostname
298 | 		}
299 | 
300 | 		if msg.Event.Time == (time.Time{}) {
301 | 			msg.Event.Time = time.Now()
302 | 		}
303 | 
304 | 		if msg.Event.Data == nil {
305 | 			msg.Event.Data = ecslogs.EventData{}
306 | 		}
307 | 
308 | 		c <- msg
309 | 	}
310 | }
311 | 
312 | func write(dest destination, group, stream string, batch lib.MessageBatch, join *sync.WaitGroup) {
313 | 	defer join.Done()
314 | 
315 | 	var writer lib.Writer
316 | 	var err error
317 | 
318 | 	if writer, err = dest.Open(group, stream); err != nil {
319 | 		logDropBatch(dest.name, group, stream, err, batch)
320 | 		return
321 | 	}
322 | 	defer writer.Close()
323 | 
324 | 	if err = writer.WriteMessageBatch(batch); err != nil {
325 | 		logDropBatch(dest.name, group, stream, err, batch)
326 | 		return
327 | 	}
328 | }
329 | 
330 | func flush(dests []destination, stream *lib.Stream, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
331 | 	for {
332 | 		batch, reason := stream.Flush(limits, now)
333 | 
334 | 		if len(batch) == 0 {
335 | 			break
336 | 		}
337 | 
338 | 		// Ensure all messages in the batch are sorted. Checking if the batch is
339 | 		// sorted is an optimization since in most cases the batch will be sorted
340 | 		// because we're reading events that are generated live (checking for a
341 | 		// sorted list is O(N) vs O(N*log(N)) for sorting it).
342 | 		// There are cases where some log entries do appear unordered and this is
343 | 		// causing issues with CloudWatchLogs.
344 | 		if !sort.IsSorted(batch) {
345 | 			sort.Stable(batch)
346 | 		}
347 | 
348 | 		log.WithFields(log.Fields{
349 | 			"group":  stream.Group(),
350 | 			"stream": stream.Name(),
351 | 			"count":  len(batch),
352 | 			"reason": reason,
353 | 		}).Info("flushing message batch")
354 | 
355 | 		for _, dest := range dests {
356 | 			join.Add(1)
357 | 			go write(dest, stream.Group(), stream.Name(), batch, join)
358 | 		}
359 | 	}
360 | }
361 | 
362 | func flushAll(dests []destination, store *lib.Store, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
363 | 	store.ForEach(func(group *lib.Group) {
364 | 		group.ForEach(func(stream *lib.Stream) {
365 | 			flush(dests, stream, limits, now, join)
366 | 		})
367 | 	})
368 | }
369 | 
370 | func flushQueue(dests []destination, store *lib.Store, queue *lib.MessageQueue, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
371 | 	streams := make(map[string]*lib.Stream)
372 | 
373 | 	for _, msg := range queue.Flush() {
374 | 		_, stream := store.Add(msg, now)
375 | 		key := stream.Group() + ":" + stream.Name()
376 | 
377 | 		if streams[key] == nil {
378 | 			streams[key] = stream
379 | 		}
380 | 	}
381 | 
382 | 	for _, stream := range streams {
383 | 		flush(dests, stream, limits, now, join)
384 | 	}
385 | }
386 | 
387 | func removeExpired(dests []destination, store *lib.Store, cacheTimeout time.Duration, now time.Time) {
388 | 	for _, stream := range store.RemoveExpired(cacheTimeout, now) {
389 | 		for _, dest := range dests {
390 | 			dest.Close(stream.Group(), stream.Name())
391 | 		}
392 | 		log.WithFields(log.Fields{
393 | 			"group":  stream.Group(),
394 | 			"stream": stream.Name(),
395 | 		}).Info("removed expired stream")
396 | 	}
397 | }
398 | 
399 | func logDropBatch(dest string, group string, stream string, err error, batch lib.MessageBatch) {
400 | 	log.WithFields(log.Fields{
401 | 		"group":       group,
402 | 		"stream":      stream,
403 | 		"destination": dest,
404 | 		"error":       err,
405 | 		"count":       len(batch),
406 | 	}).Error("dropping message batch")
407 | 
408 | 	for _, msg := range batch {
409 | 		log.WithFields(log.Fields{
410 | 			"group":  msg.Group,
411 | 			"stream": msg.Stream,
412 | 			"event":  msg.Event,
413 | 		}).Debug("dropped")
414 | 	}
415 | }
416 | 


--------------------------------------------------------------------------------
/main_linux.go:
--------------------------------------------------------------------------------
1 | package main
2 | 
3 | import (
4 | 	_ "github.com/segmentio/ecs-logs/lib/journald"
5 | )
6 | 


--------------------------------------------------------------------------------