├── .github └── workflows │ └── go.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Procfile ├── README.md ├── client.go ├── client_test.go ├── go.mod ├── go.sum ├── logproc.go ├── logproc_test.go ├── server.go └── server_test.go /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: 1.15 20 | 21 | - name: Build 22 | run: go build -v ./... 23 | 24 | - name: Test 25 | run: go test -v ./... 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | pkg 2 | bin 3 | heroku-datadog-drain-golang 4 | vendor/ 5 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.15 as builder 2 | 3 | ENV APP_VERSION 1.2.0 4 | 5 | RUN mkdir -p /usr/src/app 6 | 7 | COPY . /usr/src/app 8 | 9 | RUN cd /usr/src/app && \ 10 | go get ./... && \ 11 | go install 12 | 13 | FROM scratch 14 | COPY --from=builder /go/bin/heroku-datadog-drain-golang . 15 | CMD ["./heroku-datadog-drain-golang"] 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015-2016 Apiary 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: heroku-datadog-drain-golang 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://github.com/apiaryio/heroku-datadog-drain-golang/workflows/Go/badge.svg)](https://github.com/apiaryio/heroku-datadog-drain-golang/actions/workflows/go.yml) 2 | 3 | # Heroku Datadog Drain 4 | 5 | Golang version of [NodeJS](https://github.com/ozinc/heroku-datadog-drain) 6 | 7 | Funnel metrics from multiple Heroku apps into Datadog using statsd. 8 | 9 | ## Supported Heroku metrics: 10 | 11 | - Heroku Router response times, status codes, etc. 12 | - Application errors 13 | - Custom metrics 14 | - Heroku Dyno [runtime metrics](https://devcenter.heroku.com/articles/log-runtime-metrics) 15 | - [Heroku Redis Metrics](https://devcenter.heroku.com/articles/heroku-redis-metrics-logs) 16 | - (beta) Heroku Runtime Language Metrics - we add support for [golang](https://devcenter.heroku.com/articles/language-runtime-metrics-go#getting-started) used in Heroku, next step add this to send to Datadog too for self monitoring app. 17 | 18 | ## Get Started 19 | 20 | ### Clone the Github repository 21 | 22 | ```bash 23 | git clone git@github.com:apiaryio/heroku-datadog-drain-golang.git 24 | cd heroku-datadog-drain-golang 25 | ``` 26 | 27 | ### Setup Heroku, specify the app(s) you'll be monitoring and create a password for each. 28 | 29 | ``` 30 | heroku create 31 | heroku config:set ALLOWED_APPS= _PASSWORD= 32 | ``` 33 | 34 | > **OPTIONAL**: Setup Heroku build packs, including the [Datadog buildpack with Agent V6](https://github.com/DataDog/heroku-buildpack-datadog.git). 35 | > If you already have a StatsD client running, see the STATSD_URL configuration option below. 36 | 37 | ``` 38 | heroku buildpacks:add heroku/go 39 | heroku buildpacks:add --index 1 https://github.com/DataDog/heroku-buildpack-datadog.git 40 | heroku config:set HEROKU_APP_NAME=$(heroku apps:info|grep ===|cut -d' ' -f2) 41 | 42 | heroku config:add DD_API_KEY= 43 | ``` 44 | 45 | > **DANGER**: Original [miketheman heroku-buildpack-datadog project is deprecated](https://github.com/miketheman/heroku-buildpack-datadog) and datadog have own [buildpack](https://docs.datadoghq.com/agent/basic_agent_usage/heroku/) that isn't backward compatible. You have change `DATADOG_API_KEY` to `DD_API_KEY` during upgrade 46 | 47 | Don't forget [set right golang version](https://devcenter.heroku.com/articles/go-support#go-versions). 48 | 49 | ``` 50 | heroku config:set GOVERSION=go1.12 51 | ``` 52 | 53 | You can use specific settings for [Go modules](https://github.com/heroku/heroku-buildpack-go#go-module-specifics) 54 | 55 | ### Deploy to Heroku. 56 | 57 | ``` 58 | git push heroku master 59 | heroku ps:scale web=1 60 | ``` 61 | 62 | ### Add the Heroku log drain using the app slug and password created above. 63 | 64 | ``` 65 | heroku drains:add https://:@.herokuapp.com/ --app 66 | ``` 67 | 68 | ## Configuration 69 | 70 | ```bash 71 | STATSD_URL=.. # Required. Set to: localhost:8125 72 | DD_API_KEY=... # Required. Datadog API Key - https://app.datadoghq.com/account/settings#api 73 | ALLOWED_APPS=my-app,.. # Required. Comma seperated list of app names 74 | _PASSWORD=.. # Required. One per allowed app where corresponds to an app name from ALLOWED_APPS 75 | _TAGS=mytag,.. # Optional. Comma seperated list of default tags for each app 76 | _PREFIX=.. # Optional. String to be prepended to all metrics from a given app 77 | DATADOG_DRAIN_DEBUG=.. # Optional. If DEBUG is set, a lot of stuff will be logged :) 78 | EXCLUDED_TAGS: path,host # Optional. Recommended to solve problem with tags limit (1000) 79 | ``` 80 | 81 | Note that the capitalized `` and `` appearing above indicate that your application name and slug should also be in full caps. For example, to set the password for an application named `my-app`, you would need to specify `heroku config:set ALLOWED_APPS=my-app MY-APP_PASSWORD=example_password` 82 | 83 | The rationale for `EXCLUDED_TAGS` is that the `path=` tag in Heroku logs includes the full HTTP path - including, for instance, query parameters. This makes very easy to swarm Datadog with numerous distinct tag/value pairs; and Datadog has a hard limit of 1000 such distinct pairs. When the limit is breached, they blacklist the entire metric. 84 | 85 | ## Heroku settings 86 | 87 | You need use Standard dynos and better and enable `log-runtime-metrics` in heroku labs for every application. 88 | 89 | ```bash 90 | heroku labs:enable log-runtime-metrics -a APP_NAME 91 | ``` 92 | 93 | This adds basic metrics (cpu, memory etc.) into logs. 94 | 95 | ## Custom Metrics 96 | 97 | If you want to log some custom metrics just format the log line like following: 98 | 99 | ``` 100 | app web.1 - info: responseLogger: metric#tag#route=/parser metric#request_id=11747467-f4ce-4b06-8c99-92be968a02e3 metric#request_length=541 metric#response_length=5163 metric#parser_time=5ms metric#eventLoop.count=606 metric#eventLoop.avg_ms=515.503300330033 metric#eventLoop.p50_ms=0.8805309734513275 metric#eventLoop.p95_ms=3457.206896551724 metric#eventLoop.p99_ms=3457.206896551724 metric#eventLoop.max_ms=5008 101 | ``` 102 | 103 | We support: 104 | 105 | - `metric#` and `sample#` for gauges 106 | - `metric#tag` for tags. 107 | - `count#` for counter increments 108 | - `measure#` for histograms 109 | 110 | more info [here](https://docs.datadoghq.com/guides/dogstatsd/#data-types) 111 | 112 | ## Overriding prefix and tags with drain query params 113 | 114 | To change the prefix use the drain of form: 115 | `https://:@.herokuapp.com?prefix=abcd.` 116 | 117 | To change tags use the drain of form: 118 | `https://:@.herokuapp.com?tags=xyz,abcd` 119 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "regexp" 6 | "sort" 7 | "strconv" 8 | "strings" 9 | 10 | statsd "github.com/DataDog/datadog-go/statsd" 11 | log "github.com/Sirupsen/logrus" 12 | ) 13 | 14 | const sampleRate = 1.0 15 | 16 | const ( 17 | routerMsg int = iota 18 | scalingMsg 19 | sampleMsg 20 | metricsTag 21 | releaseMsg 22 | ) 23 | 24 | var routerMetricsKeys = []string{"dyno", "method", "status", "path", "host", "code", "desc", "at"} 25 | var sampleMetricsKeys = []string{"source"} 26 | var scalingMetricsKeys = []string{"mailer", "web"} 27 | var customMetricsKeys = []string{"media_type", "output_type", "route"} 28 | 29 | type Client struct { 30 | *statsd.Client 31 | ExcludedTags map[string]bool 32 | } 33 | 34 | var statusCode *regexp.Regexp = regexp.MustCompile(`^(?P\d)\d\d`) 35 | 36 | func statsdClient(addr string) (*Client, error) { 37 | 38 | c, err := statsd.New(addr) 39 | return &Client{c, make(map[string]bool)}, err 40 | } 41 | 42 | func (c *Client) sendToStatsd(in chan *logMetrics) { 43 | 44 | var data *logMetrics 45 | var ok bool 46 | for { 47 | data, ok = <-in 48 | 49 | if !ok { //Exit, channel was closed 50 | return 51 | } 52 | 53 | log.WithFields(log.Fields{ 54 | "type": data.typ, 55 | "app": data.app, 56 | "tags": data.tags, 57 | "prefix": data.prefix, 58 | }).Debug("logMetrics received") 59 | 60 | if data.typ == routerMsg { 61 | c.sendRouterMsg(data) 62 | } else if data.typ == sampleMsg { 63 | c.sendSampleMsg(data) 64 | } else if data.typ == scalingMsg { 65 | c.sendEvents(*data.app, "heroku", data.events, *data.tags) 66 | c.sendScalingMsg(data) 67 | } else if data.typ == metricsTag { 68 | c.sendMetricsWithTags(data) 69 | } else if data.typ == releaseMsg { 70 | c.sendEvents(*data.app, "app", data.events, *data.tags) 71 | } else { 72 | log.WithField("type", data.typ).Warn("Unknown log message") 73 | } 74 | } 75 | } 76 | 77 | func (c *Client) sendEvents(app string, namespace string, events []string, tags []string) { 78 | for _, v := range events { 79 | event := statsd.NewEvent(namespace+"/api: "+app, v) 80 | event.Tags = tags 81 | c.Event(event) 82 | log.WithFields(log.Fields{ 83 | "type": "event", 84 | "app": app, 85 | "value": v, 86 | }).Info("Event sent") 87 | } 88 | } 89 | 90 | func (c *Client) extractTags(tags []string, permittedTags []string, metrics map[string]logValue) []string { 91 | for _, mk := range permittedTags { 92 | if c.ExcludedTags[mk] { 93 | continue 94 | } 95 | if v, ok := metrics[mk]; ok { 96 | tags = append(tags, mk+":"+v.Val) 97 | } 98 | } 99 | sort.Strings(tags) 100 | return tags 101 | } 102 | 103 | func addStatusFamilyToTags(data *logMetrics, tags []string) []string { 104 | if val, ok := data.metrics["status"]; ok { 105 | match := statusCode.FindStringSubmatch(val.Val) 106 | if len(match) > 1 { 107 | tags = append(tags, "statusFamily:"+match[1]+"xx") 108 | } 109 | } 110 | return tags 111 | } 112 | 113 | func (c *Client) sendRouterMsg(data *logMetrics) { 114 | tags := c.extractTags(*data.tags, routerMetricsKeys, data.metrics) 115 | tags = addStatusFamilyToTags(data, tags) 116 | 117 | log.WithFields(log.Fields{ 118 | "app": *data.app, 119 | "tags": *data.tags, 120 | "prefix": *data.prefix, 121 | }).Debug("sendRouterMsg") 122 | 123 | conn, err := strconv.ParseFloat(data.metrics["connect"].Val, 10) 124 | if err != nil { 125 | log.WithFields(log.Fields{ 126 | "type": "router", 127 | "err": err, 128 | "metric": "connect", 129 | }).Info("Could not parse metric value") 130 | return 131 | } 132 | serv, err := strconv.ParseFloat(data.metrics["service"].Val, 10) 133 | if err != nil { 134 | log.WithFields(log.Fields{ 135 | "type": "router", 136 | "metric": "service", 137 | "err": err, 138 | }).Info("Could not parse metric value") 139 | return 140 | } 141 | 142 | bytes, err := strconv.ParseFloat(data.metrics["bytes"].Val, 10) 143 | if err != nil { 144 | log.WithFields(log.Fields{ 145 | "type": "router", 146 | "metric": "bytes", 147 | "err": err, 148 | }).Info("Could not parse metric value") 149 | return 150 | } 151 | // https://devcenter.heroku.com/articles/http-routing 152 | err = c.Histogram(*data.prefix+"heroku.router.response.bytes", bytes, tags, sampleRate) 153 | if err != nil { 154 | log.WithField("error", err).Info("Failed to send Histogram") 155 | } 156 | err = c.Histogram(*data.prefix+"heroku.router.request.connect", conn, tags, sampleRate) 157 | if err != nil { 158 | log.WithField("error", err).Info("Failed to send Histogram") 159 | } 160 | err = c.Histogram(*data.prefix+"heroku.router.request.service", serv, tags, sampleRate) 161 | if err != nil { 162 | log.WithField("error", err).Info("Failed to send Histogram") 163 | } 164 | if data.metrics["at"].Val == "error" { 165 | err = c.Count(*data.prefix+"heroku.router.error", 1, tags, 0.1) 166 | if err != nil { 167 | log.WithField("error", err).Info("Failed to send Count") 168 | } 169 | } 170 | } 171 | 172 | func (c *Client) sendSampleMsg(data *logMetrics) { 173 | tags := c.extractTags(*data.tags, sampleMetricsKeys, data.metrics) 174 | 175 | log.WithFields(log.Fields{ 176 | "app": *data.app, 177 | "tags": tags, 178 | "prefix": *data.prefix, 179 | }).Debug("sendSampleMsg") 180 | 181 | for k, v := range data.metrics { 182 | if strings.Index(k, "#") != -1 { 183 | m := strings.Replace(strings.Split(k, "#")[1], "_", ".", -1) 184 | vnum, err := strconv.ParseFloat(v.Val, 10) 185 | if err == nil { 186 | err = c.Gauge(*data.prefix+"heroku.dyno."+m, vnum, tags, sampleRate) 187 | if err != nil { 188 | log.WithField("error", err).Info("Failed to send Gauge") 189 | } 190 | } else { 191 | log.WithFields(log.Fields{ 192 | "type": "sample", 193 | "metric": k, 194 | "err": err, 195 | }).Info("Could not parse metric value") 196 | } 197 | } 198 | } 199 | } 200 | 201 | func (c *Client) sendScalingMsg(data *logMetrics) { 202 | tags := *data.tags 203 | 204 | log.WithFields(log.Fields{ 205 | "app": *data.app, 206 | "tags": tags, 207 | "prefix": *data.prefix, 208 | }).Debug("sendScalingMsg") 209 | 210 | for _, mk := range scalingMetricsKeys { 211 | if v, ok := data.metrics[mk]; ok { 212 | vnum, err := strconv.ParseFloat(v.Val, 10) 213 | if err == nil { 214 | err = c.Gauge(*data.prefix+"heroku.dyno."+mk, vnum, tags, sampleRate) 215 | if err != nil { 216 | log.WithField("error", err).Info("Failed to send Gauge") 217 | } 218 | } else { 219 | log.WithFields(log.Fields{ 220 | "type": "scaling", 221 | "metric": mk, 222 | "err": err, 223 | }).Info("Could not parse metric value") 224 | } 225 | } 226 | } 227 | } 228 | 229 | func (c *Client) sendMetric(metricType string, metricName string, value float64, tags []string) error { 230 | switch metricType { 231 | case "metric", "sample": 232 | return c.Gauge(metricName, value, tags, sampleRate) 233 | case "measure": 234 | return c.Histogram(metricName, value, tags, sampleRate) 235 | case "count": 236 | return c.Count(metricName, int64(value), tags, sampleRate) 237 | default: 238 | return errors.New("Unknown metric type" + metricType) 239 | } 240 | } 241 | 242 | func (c *Client) sendMetricsWithTags(data *logMetrics) { 243 | tags := *data.tags 244 | 245 | Tags: 246 | for k, v := range data.metrics { 247 | if strings.Index(k, "tag#") != -1 { 248 | if _, err := strconv.Atoi(v.Val); err != nil { 249 | m := strings.Replace(strings.Split(k, "tag#")[1], "_", ".", -1) 250 | for _, mk := range customMetricsKeys { 251 | if m == mk { 252 | tags = append(tags, mk+":"+v.Val) 253 | continue Tags 254 | } 255 | } 256 | } 257 | } 258 | } 259 | 260 | log.WithFields(log.Fields{ 261 | "app": *data.app, 262 | "tags": tags, 263 | "prefix": *data.prefix, 264 | }).Debug("sendMetricTag") 265 | 266 | for k, v := range data.metrics { 267 | if strings.Index(k, "#") != -1 { 268 | if vnum, err := strconv.ParseFloat(v.Val, 10); err == nil { 269 | keySplit := strings.Split(k, "#") 270 | metricType := keySplit[0] 271 | m := strings.Replace(keySplit[1], "_", ".", -1) 272 | err = c.sendMetric(metricType, *data.prefix+"app.metric."+m, vnum, tags) 273 | if err != nil { 274 | log.WithField("error", err).Warning("Failed to send Gauge") 275 | } 276 | } else { 277 | log.WithFields(log.Fields{ 278 | "type": "metrics", 279 | "metric": k, 280 | "err": err, 281 | }).Debug("Could not parse metric value") 282 | } 283 | } 284 | } 285 | } 286 | -------------------------------------------------------------------------------- /client_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | ) 7 | 8 | var app = "test" 9 | var tags = []string{"tag1", "tag2"} 10 | var prefix = "prefix." 11 | var events = []string{""} 12 | 13 | var statsdTests = []struct { 14 | cnt int 15 | m logMetrics 16 | Expected []string 17 | }{ 18 | { 19 | cnt: 3, 20 | m: logMetrics{ 21 | routerMsg, 22 | &app, 23 | &tags, 24 | &prefix, 25 | map[string]logValue{ 26 | "at": {"info", ""}, 27 | "path": {"/foo", ""}, 28 | "connect": {"1", "ms"}, 29 | "service": {"37", "ms"}, 30 | "status": {"401", ""}, 31 | "bytes": {"244000", ""}, 32 | "garbage": {"bar", ""}, 33 | }, 34 | events, 35 | }, 36 | Expected: []string{ 37 | "prefix.heroku.router.response.bytes:244000.000000|h|#at:info,status:401,tag1,tag2,statusFamily:4xx", 38 | "prefix.heroku.router.request.connect:1.000000|h|#at:info,status:401,tag1,tag2,statusFamily:4xx", 39 | "prefix.heroku.router.request.service:37.000000|h|#at:info,status:401,tag1,tag2,statusFamily:4xx", 40 | }, 41 | }, 42 | { 43 | cnt: 1, 44 | m: logMetrics{ 45 | metricsTag, 46 | &app, 47 | &tags, 48 | &prefix, 49 | map[string]logValue{ 50 | "metric#load_avg_2m": {"0.01", ""}, 51 | }, 52 | events, 53 | }, 54 | Expected: []string{ 55 | "prefix.app.metric.load.avg.2m:0.010000|g|#tag1,tag2", 56 | }, 57 | }, 58 | { 59 | cnt: 1, 60 | m: logMetrics{ 61 | metricsTag, 62 | &app, 63 | &tags, 64 | &prefix, 65 | map[string]logValue{ 66 | "sample#load_avg_1m": {"0.01", ""}, 67 | }, 68 | events, 69 | }, 70 | Expected: []string{ 71 | "prefix.app.metric.load.avg.1m:0.010000|g|#tag1,tag2", 72 | }, 73 | }, 74 | { 75 | cnt: 1, 76 | m: logMetrics{ 77 | metricsTag, 78 | &app, 79 | &tags, 80 | &prefix, 81 | map[string]logValue{ 82 | "count#clicks": {"1", ""}, 83 | }, 84 | events, 85 | }, 86 | Expected: []string{ 87 | "prefix.app.metric.clicks:1|c|#tag1,tag2", 88 | }, 89 | }, 90 | { 91 | cnt: 1, 92 | m: logMetrics{ 93 | metricsTag, 94 | &app, 95 | &tags, 96 | &prefix, 97 | map[string]logValue{ 98 | "measure#temperature": {"1.3", ""}, 99 | }, 100 | events, 101 | }, 102 | Expected: []string{ 103 | "prefix.app.metric.temperature:1.300000|h|#tag1,tag2", 104 | }, 105 | }, 106 | { 107 | cnt: 1, 108 | m: logMetrics{ 109 | sampleMsg, 110 | &app, 111 | &tags, 112 | &prefix, 113 | map[string]logValue{ 114 | "source": {"web1", ""}, 115 | "sample#load_avg_1m": {"0.01", ""}, 116 | }, 117 | events, 118 | }, 119 | Expected: []string{ 120 | "prefix.heroku.dyno.load.avg.1m:0.010000|g|#source:web1,tag1,tag2", 121 | }, 122 | }, 123 | { 124 | cnt: 3, 125 | m: logMetrics{ 126 | scalingMsg, 127 | &app, 128 | &tags, 129 | &prefix, 130 | map[string]logValue{ 131 | "mailer": {"1", ""}, 132 | "web": {"3", ""}, 133 | }, 134 | []string{ 135 | "Scaling dynos mailer=1 web=3 by foo@bar", 136 | }, 137 | }, 138 | Expected: []string{ 139 | "_e{16,39}:heroku/api: test|Scaling dynos mailer=1 web=3 by foo@bar|#tag1,tag2", 140 | "prefix.heroku.dyno.mailer:1.000000|g|#tag1,tag2", 141 | "prefix.heroku.dyno.web:3.000000|g|#tag1,tag2", 142 | }, 143 | }, 144 | { 145 | cnt: 1, 146 | m: logMetrics{ 147 | releaseMsg, 148 | &app, 149 | &tags, 150 | &prefix, 151 | map[string]logValue{}, 152 | []string{ 153 | "Release v1 created by foo@bar", 154 | }, 155 | }, 156 | Expected: []string{ 157 | "_e{13,29}:app/api: test|Release v1 created by foo@bar|#tag1,tag2", 158 | }, 159 | }, 160 | } 161 | 162 | func TestStatsdClient(t *testing.T) { 163 | 164 | addr := "localhost:1201" 165 | udpAddr, err := net.ResolveUDPAddr("udp", addr) 166 | if err != nil { 167 | t.Fatal(err) 168 | } 169 | 170 | server, err := net.ListenUDP("udp", udpAddr) 171 | if err != nil { 172 | t.Fatal(err) 173 | } 174 | defer server.Close() 175 | 176 | c, err := statsdClient(addr) 177 | if err != nil { 178 | t.Fatal(err) 179 | } 180 | 181 | c.ExcludedTags["path"] = true 182 | 183 | out := make(chan *logMetrics) 184 | defer close(out) 185 | go c.sendToStatsd(out) 186 | 187 | bytes := make([]byte, 1024) 188 | for _, tt := range statsdTests { 189 | out <- &tt.m 190 | for i := 0; i < tt.cnt; i++ { 191 | n, err := server.Read(bytes) 192 | if err != nil { 193 | t.Fatal(err) 194 | } 195 | message := bytes[:n] 196 | if string(message) != tt.Expected[i] { 197 | t.Errorf("Expected: %s. Actual: %s", tt.Expected[i], string(message)) 198 | } 199 | } 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | // +heroku goVersion go1.12 2 | module github.com/apiaryio/heroku-datadog-drain-golang 3 | 4 | require ( 5 | github.com/DataDog/datadog-go v0.0.0-20170427165718-0ddda6bee211 6 | github.com/Sirupsen/logrus v0.11.5 7 | github.com/davecgh/go-spew v1.1.1 // indirect 8 | github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7 // indirect 9 | github.com/gin-gonic/gin v0.0.0-20170702092826-d459835d2b07 10 | github.com/golang/protobuf v0.0.0-20160106020635-2402d76f3d41 // indirect 11 | github.com/heroku/x v0.0.0-20171004170240-705849e307dd 12 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 13 | github.com/kr/pretty v0.1.0 // indirect 14 | github.com/mattn/go-isatty v0.0.3 // indirect 15 | github.com/pmezard/go-difflib v1.0.0 // indirect 16 | github.com/stretchr/testify v1.2.2 // indirect 17 | github.com/ugorji/go v0.0.0-20171231121548-ccfe18359b55 // indirect 18 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a // indirect 19 | golang.org/x/sys v0.0.0-20160415135844-f64b50fbea64 // indirect 20 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect 21 | gopkg.in/go-playground/assert.v1 v1.2.1 // indirect 22 | gopkg.in/go-playground/validator.v8 v8.15.1 // indirect 23 | gopkg.in/yaml.v2 v2.0.0-20160912165603-31c299268d30 // indirect 24 | ) 25 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/DataDog/datadog-go v0.0.0-20170427165718-0ddda6bee211 h1:hOSWYZBOWXZwYN+huYLFKsb4f6uohHgpx6cLlAXOqjA= 2 | github.com/DataDog/datadog-go v0.0.0-20170427165718-0ddda6bee211/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= 3 | github.com/Sirupsen/logrus v0.11.5 h1:aIMrrsnipdTlAieMe7FC/iiuJ0+ELiXCT4YiVQiK9j8= 4 | github.com/Sirupsen/logrus v0.11.5/go.mod h1:rmk17hk6i8ZSAJkSDa7nOxamrG+SP4P0mm+DAvExv4U= 5 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 6 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7 h1:AzN37oI0cOS+cougNAV9szl6CVoj2RYwzS3DpUQNtlY= 8 | github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= 9 | github.com/gin-gonic/gin v0.0.0-20170702092826-d459835d2b07 h1:cZPJWzd2oNeoS0oJM2TlN9rl0OnCgUr10gC8Q4mH+6M= 10 | github.com/gin-gonic/gin v0.0.0-20170702092826-d459835d2b07/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= 11 | github.com/golang/protobuf v0.0.0-20160106020635-2402d76f3d41 h1:BIDtr9YECqqvixqxNnfN1Dp4dlRZB2nS68tywI+YZj4= 12 | github.com/golang/protobuf v0.0.0-20160106020635-2402d76f3d41/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 13 | github.com/heroku/x v0.0.0-20171004170240-705849e307dd h1:zn29UrzyUeQgqxBGXIwQqQJf75IiK4aeCtO5q1V2Vyo= 14 | github.com/heroku/x v0.0.0-20171004170240-705849e307dd/go.mod h1:opmAyjmIGn9/Y+9Nia6eIaktIXIoMhhFXEFbHLMsX3Y= 15 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= 16 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 17 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 18 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 19 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 20 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 21 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 22 | github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= 23 | github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 24 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 25 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 26 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 27 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 28 | github.com/ugorji/go v0.0.0-20171231121548-ccfe18359b55 h1:1IR8KZO9eYKhBHbeUWCb10PVIr7dAcglEhe43mEmInQ= 29 | github.com/ugorji/go v0.0.0-20171231121548-ccfe18359b55/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= 30 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= 31 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 32 | golang.org/x/sys v0.0.0-20160415135844-f64b50fbea64 h1:bCcPub4lKv9/pHMpgldrftsXW84TsZrI/QPdbxiaE/4= 33 | golang.org/x/sys v0.0.0-20160415135844-f64b50fbea64/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 34 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 35 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 36 | gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= 37 | gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= 38 | gopkg.in/go-playground/validator.v8 v8.15.1 h1:IrBBTcgklCPw+7tjCGdAErvsk+44VaIeS/4T1utPQ+I= 39 | gopkg.in/go-playground/validator.v8 v8.15.1/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= 40 | gopkg.in/yaml.v2 v2.0.0-20160912165603-31c299268d30 h1:mNnzt76aN10kG6/XNojKVKR8VDIWvEp4mlj5kyRf6hk= 41 | gopkg.in/yaml.v2 v2.0.0-20160912165603-31c299268d30/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 42 | -------------------------------------------------------------------------------- /logproc.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "regexp" 6 | "strings" 7 | 8 | log "github.com/Sirupsen/logrus" 9 | "github.com/kr/logfmt" 10 | ) 11 | 12 | type logValue struct { 13 | Val string 14 | Unit string // (e.g. ms, MB, etc) 15 | } 16 | 17 | type logMetrics struct { 18 | typ int 19 | app *string 20 | tags *[]string 21 | prefix *string 22 | metrics map[string]logValue 23 | events []string 24 | } 25 | 26 | var dynoNumber *regexp.Regexp = regexp.MustCompile(`\.\d+$`) 27 | 28 | func (lm *logMetrics) HandleLogfmt(key, val []byte) error { 29 | 30 | i := bytes.LastIndexFunc(val, isDigit) 31 | if i == -1 { 32 | lm.metrics[string(key)] = logValue{string(val), ""} 33 | } else { 34 | lm.metrics[string(key)] = logValue{string(val[:i+1]), string(val[i+1:])} 35 | } 36 | 37 | log.WithFields(log.Fields{ 38 | "key": string(key), 39 | "val": lm.metrics[string(key)].Val, 40 | "unit": lm.metrics[string(key)].Unit, 41 | }).Debug("logMetric") 42 | 43 | return nil 44 | } 45 | 46 | // return true if r is an ASCII digit only, as opposed to unicode.IsDigit. 47 | func isDigit(r rune) bool { 48 | return '0' <= r && r <= '9' 49 | } 50 | 51 | func parseMetrics(typ int, ld *logData, data *string, out chan *logMetrics) { 52 | var myslice []string 53 | lm := logMetrics{typ, ld.app, ld.tags, ld.prefix, make(map[string]logValue, 5), myslice} 54 | 55 | if typ == releaseMsg { 56 | events := append(lm.events, *data) 57 | lm.events = events 58 | out <- &lm 59 | return 60 | } 61 | 62 | if err := logfmt.Unmarshal([]byte(*data), &lm); err != nil { 63 | log.WithFields(log.Fields{ 64 | "err": err, 65 | }).Warn() 66 | return 67 | } 68 | if source, ok := lm.metrics["source"]; ok { 69 | tags := append(*lm.tags, "type:"+dynoNumber.ReplaceAllString(source.Val, "")) 70 | lm.tags = &tags 71 | } 72 | out <- &lm 73 | } 74 | 75 | var scalingRe = regexp.MustCompile("Scaled to (.*) by user .*") 76 | var scaledDynoRe = regexp.MustCompile("([^@ ]*)@([^: ]*):([^ ]*)") 77 | 78 | func parseScalingMessage(ld *logData, message *string, out chan *logMetrics) { 79 | if scalingInfo := scalingRe.FindStringSubmatch(*message); scalingInfo != nil { 80 | scaledDynoInfos := scaledDynoRe.FindAllStringSubmatch(scalingInfo[1], -1) 81 | logValues := make(map[string]logValue) 82 | for _, dynoInfo := range scaledDynoInfos { 83 | dynoName := dynoInfo[1] 84 | count := dynoInfo[2] 85 | dynoType := dynoInfo[3] 86 | log.WithFields(log.Fields{ 87 | "dynoName": dynoName, 88 | "count": count, 89 | "dynoType": dynoType, 90 | }).Debug() 91 | logValues[dynoName] = logValue{count, dynoType} 92 | } 93 | events := []string{*message} 94 | lm := logMetrics{scalingMsg, ld.app, ld.tags, ld.prefix, logValues, events} 95 | out <- &lm 96 | } else { 97 | log.WithFields(log.Fields{ 98 | "err": "Scaling message not matched", 99 | "message": *message, 100 | }).Warn() 101 | } 102 | } 103 | 104 | func logProcess(in chan *logData, out chan *logMetrics) { 105 | 106 | var data *logData 107 | var ok bool 108 | for { 109 | data, ok = <-in 110 | 111 | if !ok { //Exit, channel was closed 112 | return 113 | } 114 | 115 | log.Debugln(*data.line) 116 | output := strings.Split(*data.line, " - ") 117 | redisMetrics := strings.Split(output[0], " app[heroku-redis]: source=REDIS") 118 | if len(output) < 2 && len(redisMetrics) == 1 { 119 | continue 120 | } 121 | headers := strings.Split(strings.TrimSpace(output[0]), " ") 122 | if len(headers) >= 6 { 123 | headers = headers[3:6] 124 | } 125 | 126 | log.WithField("headers", headers).Debug("Line headers") 127 | if len(redisMetrics) > 1 { 128 | parseMetrics(sampleMsg, data, &redisMetrics[1], out) 129 | } else if headers[1] == "heroku" { 130 | if headers[2] == "router" { 131 | parseMetrics(routerMsg, data, &output[1], out) 132 | } else { 133 | parseMetrics(sampleMsg, data, &output[1], out) 134 | } 135 | } else if headers[1] == "app" { 136 | if headers[2] == "api" { 137 | if strings.HasPrefix(output[1], "Release") { 138 | parseMetrics(releaseMsg, data, &output[1], out) 139 | } else { 140 | parseScalingMessage(data, &output[1], out) 141 | } 142 | } else { 143 | dynoType := dynoNumber.ReplaceAllString(headers[2], "") 144 | tags := append(*data.tags, "source:"+headers[2], "type:"+dynoType) 145 | data.tags = &tags 146 | parseMetrics(metricsTag, data, &output[1], out) 147 | } 148 | } 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /logproc_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | log "github.com/Sirupsen/logrus" 8 | ) 9 | 10 | func TestLogProc(t *testing.T) { 11 | 12 | lines := strings.Split(`255 <158>1 2015-04-02T11:52:34.520012+00:00 host heroku router - at=info method=POST path="/users" host=myapp.com request_id=c1806361-2081-42e7-a8aa-92b6808eac8e fwd="24.76.242.18" dyno=web.1 connect=1ms service=37ms status=201 bytes=828 13 | 229 <45>1 2015-04-02T11:48:16.839257+00:00 host heroku web.1 - source=web.1 dyno=heroku.35930502.b9de5fce-44b7-4287-99a7-504519070cba sample#load_avg_1m=0.01 sample#load_avg_5m=0.02 sample#load_avg_15m=0.03 14 | 222 <134>1 2017-05-13T15:35:33.787162+00:00 host app api - Scaled to mailer@3:Performance-L web@5:Standard-2X by user someuser@gmail.com 15 | 222 <134>1 2015-04-07T16:01:43.517062+00:00 host heroku api - this_is="broken 16 | 222 <134>1 2015-04-07T16:01:43.517062+00:00 host app api - Release v138 created by user foo@bar 17 | 222 <134>1 2019-10-08T22:21:03+00:00 app[heroku-redis]: source=REDIS addon=redis-lively-37272 sample#active-connections=14 sample#load-avg-1m=0.505 sample#load-avg-5m=0.87 sample#load-avg-15m=0.77 sample#read-iops=0 sample#write-iops=38.19 sample#memory-total=15664212kB sample#memory-free=9796400kB sample#memory-cached=3751668kB sample#memory-redis=2131576bytes sample#hit-rate=0.78062 sample#evicted-keys=0`, "\n") 18 | 19 | app := "test" 20 | tags := []string{"tag1", "tag2"} 21 | prefix := "prefix." 22 | s := loadServerCtx() 23 | s.in = make(chan *logData, 3) 24 | defer close(s.in) 25 | s.out = make(chan *logMetrics, 3) 26 | defer close(s.out) 27 | 28 | go logProcess(s.in, s.out) 29 | 30 | for i, l := range lines { 31 | log.WithField("line", l).Debug("Sending") 32 | s.in <- &logData{&app, &tags, &prefix, &lines[i]} 33 | } 34 | 35 | res := <-s.out 36 | if res.typ != routerMsg { 37 | t.Error("result must be ROUTE") 38 | } 39 | 40 | res = <-s.out 41 | if res.typ != sampleMsg { 42 | t.Error("result must be SAMPLE") 43 | } 44 | 45 | res = <-s.out 46 | if res.typ != scalingMsg { 47 | t.Error("result must be SCALE") 48 | } 49 | 50 | res = <-s.out 51 | if res.typ != releaseMsg { 52 | t.Error("result must be RELEASE") 53 | } 54 | 55 | res = <-s.out 56 | if res.typ != sampleMsg { 57 | t.Error("result must be SAMPLE") 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "os" 6 | "strings" 7 | 8 | log "github.com/Sirupsen/logrus" 9 | "github.com/gin-gonic/gin" 10 | 11 | _ "github.com/heroku/x/hmetrics/onload" 12 | ) 13 | 14 | const bufferLen = 500 15 | 16 | type logData struct { 17 | app *string 18 | tags *[]string 19 | prefix *string 20 | line *string 21 | } 22 | 23 | type ServerCtx struct { 24 | Port string 25 | AllowedApps []string 26 | AppPasswd map[string]string 27 | AppTags map[string][]string 28 | AppPrefix map[string]string 29 | StatsdUrl string 30 | Debug bool 31 | in chan *logData 32 | out chan *logMetrics 33 | } 34 | 35 | //Load configuration from envrionment variables, see list below 36 | //ALLOWED_APPS=my-app,.. Required. 37 | //Comma seperated list of app names 38 | 39 | //_PASSWORD=.. Required. 40 | //One per allowed app where corresponds to an app name from ALLOWED_APPS 41 | 42 | //_TAGS=mytag,.. Optional. 43 | // Comma seperated list of default tags for each app 44 | 45 | //_PREFIX=yee Optional. 46 | //String to be prepended to all metrics from a given app 47 | 48 | //STATSD_URL=.. Required. Default: localhost:8125 49 | //DATADOG_DRAIN_DEBUG= Optional. If DEBUG is set, a lot of stuff w 50 | func loadServerCtx() *ServerCtx { 51 | 52 | s := &ServerCtx{"8080", 53 | nil, 54 | make(map[string]string), 55 | make(map[string][]string), 56 | make(map[string]string), 57 | "localhost:8125", 58 | false, 59 | nil, 60 | nil, 61 | } 62 | port := os.Getenv("PORT") 63 | if port != "" { 64 | s.Port = port 65 | } 66 | 67 | allApps := os.Getenv("ALLOWED_APPS") 68 | if allApps != "" { 69 | apps := strings.Split(allApps, ",") 70 | log.WithField("apps", apps).Info("ALLOWED_APPS loaded.") 71 | for _, app := range apps { 72 | name := strings.ToUpper(app) 73 | s.AllowedApps = append(s.AllowedApps, app) 74 | s.AppPasswd[app] = os.Getenv(name + "_PASSWORD") 75 | if s.AppPasswd[app] == "" { 76 | log.WithField("app", app).Warn("App is allowed but no password set") 77 | } 78 | tags := os.Getenv(name + "_TAGS") 79 | if tags != "" { 80 | s.AppTags[app] = strings.Split(tags, ",") 81 | } 82 | prefix := os.Getenv(name + "_PREFIX") 83 | if strings.Index(prefix, ".") == -1 { 84 | s.AppPrefix[app] = prefix + "." 85 | } else { 86 | s.AppPrefix[app] = prefix 87 | } 88 | } 89 | } else { 90 | log.Warn("No Allowed apps set, nobody can access this service!") 91 | } 92 | 93 | statsd := os.Getenv("STATSD_URL") 94 | if statsd != "" { 95 | s.StatsdUrl = statsd 96 | } 97 | 98 | if os.Getenv("DATADOG_DRAIN_DEBUG") != "" { 99 | s.Debug = true 100 | } 101 | 102 | log.WithFields(log.Fields{ 103 | "port": s.Port, 104 | "AlloweApps": s.AllowedApps, 105 | "AppPasswords": "************", 106 | "AppTags": s.AppTags, 107 | "AppPrefix": s.AppPrefix, 108 | "StatsdUrl": s.StatsdUrl, 109 | "Debug": s.Debug, 110 | }).Info("Configuration loaded") 111 | 112 | return s 113 | } 114 | 115 | func init() { 116 | // Output to stderr instead of stdout 117 | log.SetOutput(os.Stderr) 118 | 119 | // Only log the Info severity or above. 120 | log.SetLevel(log.InfoLevel) 121 | } 122 | 123 | func (s *ServerCtx) getTags(c *gin.Context, app string) []string { 124 | requestTags := c.DefaultQuery("tags", "") 125 | if requestTags == "" { 126 | return s.AppTags[app]; 127 | } else { 128 | return strings.Split(requestTags, ","); 129 | } 130 | } 131 | 132 | func (s *ServerCtx) processLogs(c *gin.Context) { 133 | app := c.MustGet(gin.AuthUserKey).(string) 134 | tags := s.getTags(c, app) 135 | prefix := c.DefaultQuery("prefix", s.AppPrefix[app]) 136 | 137 | scanner := bufio.NewScanner(c.Request.Body) 138 | for scanner.Scan() { 139 | line := scanner.Text() 140 | log.WithField("line", line).Debug("LINE") 141 | s.in <- &logData{&app, &tags, &prefix, &line} 142 | } 143 | if err := scanner.Err(); err != nil { 144 | log.Error(err) 145 | } 146 | 147 | c.String(200, "OK") 148 | } 149 | 150 | func main() { 151 | gin.SetMode(gin.ReleaseMode) 152 | 153 | s := loadServerCtx() 154 | if s.Debug { 155 | log.SetLevel(log.DebugLevel) 156 | gin.SetMode(gin.DebugMode) 157 | } 158 | 159 | c, err := statsdClient(s.StatsdUrl) 160 | if err != nil { 161 | log.WithField("statsdUrl", s.StatsdUrl).Fatal("Could not connect to statsd") 162 | } 163 | 164 | if v := os.Getenv("EXCLUDED_TAGS"); v != "" { 165 | for _, t := range strings.Split(v, ",") { 166 | c.ExcludedTags[t] = true 167 | } 168 | } 169 | 170 | r := gin.Default() 171 | r.GET("/status", func(c *gin.Context) { 172 | c.String(200, "OK") 173 | }) 174 | 175 | if len(s.AppPasswd) > 0 { 176 | auth := r.Group("/", gin.BasicAuth(s.AppPasswd)) 177 | auth.POST("/", s.processLogs) 178 | } 179 | 180 | s.in = make(chan *logData, bufferLen) 181 | defer close(s.in) 182 | s.out = make(chan *logMetrics, bufferLen) 183 | defer close(s.out) 184 | go logProcess(s.in, s.out) 185 | go c.sendToStatsd(s.out) 186 | log.Infoln("Server ready ...") 187 | r.Run(":" + s.Port) 188 | 189 | } 190 | -------------------------------------------------------------------------------- /server_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "io/ioutil" 7 | "net" 8 | "net/http" 9 | "net/http/httptest" 10 | "testing" 11 | "time" 12 | 13 | "github.com/gin-gonic/gin" 14 | ) 15 | 16 | var fullTests = []struct { 17 | cnt int 18 | Req string 19 | Expected []string 20 | }{ 21 | { 22 | cnt: 3, 23 | Req: `255 <158>1 2015-04-02T11:52:34.520012+00:00 host heroku router - at=info method=POST path="/users" host=myapp.com request_id=c1806361-2081-42e7-a8aa-92b6808eac8e fwd="24.76.242.18" dyno=web.1 connect=1ms service=37ms status=201 bytes=828`, 24 | Expected: []string{ 25 | "heroku.router.response.bytes:828.000000|h|#at:info,dyno:web.1,host:myapp.com,method:POST,path:/users,status:201,statusFamily:2xx", 26 | "heroku.router.request.connect:1.000000|h|#at:info,dyno:web.1,host:myapp.com,method:POST,path:/users,status:201,statusFamily:2xx", 27 | "heroku.router.request.service:37.000000|h|#at:info,dyno:web.1,host:myapp.com,method:POST,path:/users,status:201,statusFamily:2xx", 28 | }, 29 | }, 30 | { 31 | cnt: 1, 32 | Req: `229 <45>1 2015-04-02T11:48:16.839257+00:00 host heroku web.1 - source=web.1 dyno=heroku.35930502.b9de5fce-44b7-4287-99a7-504519070cba sample#load_avg_1m=0.01`, 33 | Expected: []string{ 34 | "heroku.dyno.load.avg.1m:0.010000|g|#source:web.1,type:web", 35 | }, 36 | }, 37 | { 38 | cnt: 3, 39 | Req: `222 <134>1 2015-04-07T16:01:43.517062+00:00 host app api - Scaled to web@3:Performance-L mailer@1:Standard-2X by user someuser@gmail.com`, 40 | Expected: []string{ 41 | "_e{16,77}:heroku/api: test|Scaled to web@3:Performance-L mailer@1:Standard-2X by user someuser@gmail.com", 42 | "heroku.dyno.mailer:1.000000|g", 43 | "heroku.dyno.web:3.000000|g", 44 | }, 45 | }, 46 | { 47 | cnt: 1, 48 | Req: `222 <134>1 2015-04-07T16:01:43.517062+00:00 host app api - Release v1 created by foo@bar`, 49 | Expected: []string{ 50 | "_e{13,29}:app/api: test|Release v1 created by foo@bar", 51 | }, 52 | }, 53 | { 54 | cnt: 9, 55 | Req: `452 <134>1 2015-04-07T16:01:43.517062+00:00 host app web.1 - info: responseLogger: metric#tag#route=/parser metric#request_id=11747467-f4ce-4b06-8c99-92be968a02e3 metric#request_length=541 metric#response_length=5163 metric#parser_time=5ms metric#eventLoop.count=606 metric#eventLoop.avg_ms=515.503300330033 metric#eventLoop.p50_ms=0.8805309734513275 metric#eventLoop.p95_ms=3457.206896551724 metric#eventLoop.p99_ms=3457.206896551724 metric#eventLoop.max_ms=5008`, 56 | Expected: []string{ 57 | "app.metric.request.length:541.000000|g|#source:web.1,type:web,route:/parser", 58 | "app.metric.response.length:5163.000000|g|#source:web.1,type:web,route:/parser", 59 | "app.metric.parser.time:5.000000|g|#source:web.1,type:web,route:/parser", 60 | "app.metric.eventLoop.count:606.000000|g|#source:web.1,type:web,route:/parser", 61 | "app.metric.eventLoop.avg.ms:515.503300|g|#source:web.1,type:web,route:/parser", 62 | "app.metric.eventLoop.p50.ms:0.880531|g|#source:web.1,type:web,route:/parser", 63 | "app.metric.eventLoop.p95.ms:3457.206897|g|#source:web.1,type:web,route:/parser", 64 | "app.metric.eventLoop.p99.ms:3457.206897|g|#source:web.1,type:web,route:/parser", 65 | "app.metric.eventLoop.max.ms:5008.000000|g|#source:web.1,type:web,route:/parser", 66 | }, 67 | }, 68 | } 69 | 70 | func TestStatusRequest(t *testing.T) { 71 | 72 | r := gin.New() 73 | r.GET("/status", func(c *gin.Context) { 74 | c.String(200, "OK") 75 | }) 76 | 77 | req, _ := http.NewRequest("GET", "/status", nil) 78 | resp := httptest.NewRecorder() 79 | r.ServeHTTP(resp, req) 80 | 81 | body, err := ioutil.ReadAll(resp.Body) 82 | if err != nil { 83 | t.Error(err) 84 | } 85 | 86 | if string(body) != "OK" { 87 | t.Error("resp body should match") 88 | } 89 | 90 | if resp.Code != 200 { 91 | t.Error("should get a 200") 92 | } 93 | } 94 | 95 | func basicAuth(username, password string) string { 96 | auth := username + ":" + password 97 | return base64.StdEncoding.EncodeToString([]byte(auth)) 98 | } 99 | 100 | func TestLogRequest(t *testing.T) { 101 | 102 | s := loadServerCtx() 103 | s.AllowedApps = append(s.AllowedApps, "test") 104 | s.AppPasswd["test"] = "pass" 105 | 106 | s.in = make(chan *logData) 107 | defer close(s.in) 108 | s.out = make(chan *logMetrics) 109 | defer close(s.out) 110 | 111 | go logProcess(s.in, s.out) 112 | 113 | r := gin.New() 114 | auth := r.Group("/", gin.BasicAuth(s.AppPasswd)) 115 | auth.POST("/", s.processLogs) 116 | 117 | req, _ := http.NewRequest("POST", "/", bytes.NewBuffer([]byte("LINE of text\nAnother line\n"))) 118 | req.SetBasicAuth("test", "pass") 119 | resp := httptest.NewRecorder() 120 | r.ServeHTTP(resp, req) 121 | 122 | body, err := ioutil.ReadAll(resp.Body) 123 | if err != nil { 124 | t.Error(err) 125 | } 126 | if string(body) != "OK" { 127 | t.Error("resp body should match") 128 | } 129 | 130 | if resp.Code != 200 { 131 | t.Error("should get a 200") 132 | } 133 | 134 | } 135 | 136 | func TestFull(t *testing.T) { 137 | 138 | s := loadServerCtx() 139 | s.AllowedApps = append(s.AllowedApps, "test") 140 | s.AppPasswd["test"] = "pass" 141 | 142 | s.in = make(chan *logData) 143 | defer close(s.in) 144 | s.out = make(chan *logMetrics) 145 | defer close(s.out) 146 | 147 | addr := "localhost:1201" 148 | udpAddr, err := net.ResolveUDPAddr("udp", addr) 149 | if err != nil { 150 | t.Fatal(err) 151 | } 152 | 153 | server, err := net.ListenUDP("udp", udpAddr) 154 | if err != nil { 155 | t.Fatal(err) 156 | } 157 | defer server.Close() 158 | 159 | c, err := statsdClient(addr) 160 | if err != nil { 161 | t.Fatal(err) 162 | } 163 | 164 | go logProcess(s.in, s.out) 165 | go c.sendToStatsd(s.out) 166 | 167 | r := gin.New() 168 | auth := r.Group("/", gin.BasicAuth(s.AppPasswd)) 169 | auth.POST("/", s.processLogs) 170 | 171 | data := make([]byte, 1024) 172 | for _, tt := range fullTests { 173 | req, _ := http.NewRequest("POST", "/", bytes.NewBuffer([]byte(tt.Req))) 174 | req.SetBasicAuth("test", "pass") 175 | resp := httptest.NewRecorder() 176 | r.ServeHTTP(resp, req) 177 | 178 | body, err := ioutil.ReadAll(resp.Body) 179 | if err != nil { 180 | t.Error(err) 181 | } 182 | if string(body) != "OK" { 183 | t.Error("resp body should match") 184 | } 185 | if resp.Code != 200 { 186 | t.Error("should get a 200") 187 | } 188 | if tt.cnt != len(tt.Expected) { 189 | t.Error("Count of expected results isn't equal to inputs") 190 | } 191 | for i := 0; i < tt.cnt; i++ { 192 | n, err := server.Read(data) 193 | if err != nil { 194 | t.Fatal(err) 195 | } 196 | message := data[:n] 197 | findEqual := false 198 | 199 | for j := 0; j < len(tt.Expected); j++ { 200 | if string(message) == tt.Expected[j] { 201 | findEqual = true 202 | } 203 | } 204 | if findEqual == false { 205 | t.Errorf("Expected: %s. Actual: %s", tt.Expected[i], string(message)) 206 | } 207 | } 208 | } 209 | 210 | time.Sleep(1 * time.Second) 211 | } 212 | --------------------------------------------------------------------------------