├── event-in-k4.png ├── .gitignore ├── example ├── logstash.conf ├── docker-compose.yml └── README.md ├── README.md ├── LICENSE ├── redis.go └── redis_test.go /event-in-k4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rtoma/logspout-redis-logstash/HEAD/event-in-k4.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | modules.go 2 | Dockerfile.example 3 | 4 | .idea 5 | pkg/ 6 | src/ 7 | target/ 8 | .cache/ 9 | .golangbuilder.sh 10 | -------------------------------------------------------------------------------- /example/logstash.conf: -------------------------------------------------------------------------------- 1 | input { 2 | redis { 3 | host => "redis" 4 | data_type => "list" 5 | key => "logstash" 6 | codec => "json" 7 | } 8 | } 9 | filter { 10 | if [docker][image] =~ /kibana/ { 11 | json { 12 | source => "message" 13 | target => "kibana" 14 | } 15 | date { 16 | match => ['[kibana][@timestamp]', 'ISO8601'] 17 | } 18 | mutate { 19 | rename => ['[kibana][message]', 'message'] 20 | remove => ['[kibana][@timestamp]'] 21 | } 22 | } 23 | } 24 | output { 25 | elasticsearch { 26 | hosts => "elasticsearch" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /example/docker-compose.yml: -------------------------------------------------------------------------------- 1 | kibana: 2 | image: kibana:4.5.4 3 | links: 4 | - elasticsearch 5 | ports: 6 | - 5601:5601 7 | 8 | elasticsearch: 9 | image: elasticsearch:2.3.5 10 | ports: 11 | - 9200:9200 12 | 13 | redis: 14 | image: redis:3.0.7 15 | ports: 16 | - 6379:6379 17 | 18 | logstash: 19 | image: logstash:2.3.4 20 | command: 'logstash -f /logstash.conf -v' 21 | volumes: 22 | - ./logstash.conf:/logstash.conf 23 | links: 24 | - redis 25 | - elasticsearch 26 | 27 | logspout: 28 | image: rtoma/logspout-redis-logstash:0.1.8 29 | command: 'redis://redis' 30 | environment: 31 | - DEBUG=true 32 | # - REDIS_PASSWORD=secret 33 | - REDIS_KEY=logstash 34 | - REDIS_DOCKER_HOST=macbookpro 35 | # - REDIS_LOGSTASH_TYPE=docker 36 | - DEDOT_LABELS=true 37 | volumes: 38 | - /var/run/docker.sock:/var/run/docker.sock:ro 39 | links: 40 | - redis 41 | -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- 1 | # Example 2 | 3 | This directory contains everything you need to try out the logspout-redis-logstash adapter. 4 | 5 | This directory contains a docker-compose.yml that will allow you to quickly spin up linked & configured containers running these applications: 6 | 7 | - logspout, with redis-logstash adapter 8 | - redis 9 | - logstash 10 | - elasticsearch 11 | - kibana4 12 | 13 | 14 | # Requirements 15 | 16 | Have installed: 17 | 18 | - docker 19 | - docker-compose 20 | 21 | On OSX: 22 | 23 | - boot2docker 24 | 25 | 26 | # Quickstart 27 | 28 | Do a ```docker-compose up``` and all containers will start. Missing images will be pulled. 29 | 30 | On OSX do ```open http://$(boot2docker ip):5601``` to open Kibana in your browser. 31 | 32 | On other hosts: go to http://\:5601. 33 | 34 | 35 | # What, what, what? 36 | 37 | Logspout captures stdout/stderr events from all running containers and pushes 38 | them to a Redis list. 39 | 40 | Logstash consumes events from the redis list, does some magic when events come 41 | from Kibana and stores the events in Elasticsearch. 42 | 43 | Kibana allows humans to search the events. 44 | 45 | 46 | # Docker image 47 | 48 | If you want to use logspout with redis-logstash adapter you can use the image on Docker Hub. 49 | 50 | Get it using: 51 | 52 | ``` 53 | docker pull rtoma/logspout-redis-logstash 54 | ``` -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # logspout-redis-logstash 2 | [Logspout](https://github.com/gliderlabs/logspout) adapter for writing Docker container stdout/stderr logs to Redis in Logstash jsonevent layout. 3 | 4 | Since v0.1.4 JSON input is supported, enabling you to add structure to your logs. 5 | 6 | 7 | ## Docker image available 8 | 9 | Logspout including this adapter is available on [Docker Hub](https://registry.hub.docker.com/r/rtoma/logspout-redis-logstash/). Pull it with: 10 | 11 | ``` 12 | $ docker pull rtoma/logspout-redis-logstash 13 | ``` 14 | 15 | ## How to use the Docker image 16 | 17 | ``` 18 | $ docker run -d --name logspout \ 19 | -v /var/run/docker.sock:/var/run/docker.sock:ro \ 20 | rtoma/logspout-redis-logstash \ 21 | redis://?key=bla&... 22 | ``` 23 | 24 | ## Configuration 25 | 26 | Some configuration can be passed via container environment keys. 27 | 28 | Some can be passed via route options (e.g. `logspout redis://?key=foo&password=secret`). 29 | 30 | This table shows all configuration parameters: 31 | 32 | | Parameter | Default | Environment key | Route option key | 33 | |-----------|---------|-----------------|------------------| 34 | | Enable debug, if set debug logging will be printed | disabled | DEBUG | debug | 35 | | Redis password, if set this will force the adapter to execute a Redis AUTH command | none | REDIS_PASSWORD | password | 36 | | Redis key, events will be pushed to this Redis list object | 'logspout' | REDIS_KEY | key | 37 | | Redis database, if set the adapter will execute a Redis SELECT command | 0 | REDIS_DATABASE | database | 38 | | Docker host, will add a docker.host=\ field to the event, allowing you to add the hostname of your host, identifying where your container was running (think mesos) | none | REDIS\_DOCKER\_HOST | docker_host | 39 | | Use Layout v0, what Logstash json format is used. With v0 JSON input support is disabled. | false (meaning we use v1) | REDIS\_USE\_V0\_LAYOUT | use_v0_layout | 40 | | Logstash type, if set the event will get a @type property | none | REDIS\_LOGSTASH\_TYPE | logstash_type | 41 | | If true, will replace all "." in container labels with "_". You need to set this if you are using Elasticsearch 2.x | false | DEDOT_LABELS | dedot_labels | 42 | | Mute errors (to avoid error storm), disable by setting to other than 'true' | true | MUTE\_ERRORS | mute_errors | 43 | | Redis connection timeout | 100 ms | CONNECT\_TIMEOUT | connect_timeout | 44 | | Redis read timeout | 300 ms | READ\_TIMEOUT | read_timeout | 45 | | Redis write timeout | 500 ms | WRITE\_TIMEOUT | write_timeout | 46 | 47 | Note on timeouts: Logspout [stops tailing a container log](https://github.com/gliderlabs/logspout/blob/90302f046f740e3d77dda04f9a4387caed6f7f8d/router/pump.go#L288) if an adapter (like this one) takes longer than 1.0 second to process an event. That's why the sum of our default timeouts is a safe 900 ms. 48 | 49 | 50 | ## JSON input support 51 | 52 | **Note:** this does not work when using the Logstash v0 layout. 53 | 54 | Since v0.1.4 JSON input is supported, enabling you to add structure to your logs. Big words, but this is what it does. 55 | 56 | Imagine your docker application to emit a log to stdout like: 57 | 58 | ``` 59 | {"message":"I was very busy","items_processed":42,"elapsed_time_ms":123} 60 | ``` 61 | 62 | Logspout-redis-logstash will recognize this log as a JSON string and will embed the fields into the JSON document. This is how the final JSON document looks like that will be send to Redis: 63 | 64 | ``` 65 | { 66 | "@timestamp": "2016-03-30T09:54:24.587277635Z", 67 | "host": "ef5fe78b6a8e", 68 | "message": "I was very busy", 69 | "docker": { ... }, 70 | "event": { 71 | "elapsed_time_ms": 123, 72 | "items_processed": 42 73 | } 74 | } 75 | ``` 76 | 77 | What just happened? 78 | 79 | - the `message` field is set with the value from our input document. 80 | - the document contains a `event` hash filled with all input fields (ex message) 81 | 82 | ### Logtypes 83 | 84 | Using the `logtype` field in the input JSON doc, allows you to control the name of the field hash. This was added to give you some control over different logtypes you may want to implement. 85 | 86 | Usecase: imagine you have an application that handles HTTP requests and wants to emit acceslog and applicationlog events. These events are different and you want to handle them differently. 87 | 88 | We currently support two types: 89 | 90 | - accesslog 91 | - applog 92 | 93 | Example input JSON to illustrate this "data wrangling" feature: 94 | 95 | ``` 96 | {"logtype":"applog","message":"something went bOOm!","level":"ERROR","line":42,"file":"source.go"} 97 | ``` 98 | 99 | Which results in this JSON doc for Redis: 100 | 101 | ``` 102 | { 103 | "@timestamp": "2016-03-30T10:03:00.016733514Z", 104 | "host": "16cecd099d78", 105 | "message": "something went bOOm!", 106 | "docker": { ... }, 107 | "logtype": "applog", 108 | "applog": { 109 | "file": "source.go", 110 | "level": "ERROR", 111 | "line": 42 112 | } 113 | } 114 | ``` 115 | 116 | See what happened? 117 | 118 | - the `logtype` field is added and is set with the value from our input doc. 119 | - the `event` hash is now called `applog`, because of the logtype 120 | 121 | If you'd used `"logtype":"accesslog"` the JSON doc would have looked like: 122 | 123 | ``` 124 | { 125 | ... , 126 | "logtype": "accesslog", 127 | "accesslog": { 128 | "file": "source.go", 129 | "level": "ERROR", 130 | "line": 42 131 | } 132 | } 133 | ``` 134 | 135 | ## Building 136 | 137 | Use `./build.sh ` to build a Docker image for a tagged version. 138 | If you developing, use `./build.sh -d ` to build a Docker image from local sources. 139 | 140 | 141 | ## Contribution 142 | 143 | Want to add features? Feel welcome to submit a pull request! 144 | 145 | If you are unable to code, feel free to create a issue describing your feature request or bug report. 146 | 147 | 148 | ## Changelog 149 | 150 | ### 0.1.8, 0.1.9 and 0.1.10 151 | 152 | - Improved build system 153 | 154 | ### 0.1.7 155 | 156 | - Added parameter for dedotting Docker labels (required for ES 2.x). Thanks to adepretis! 157 | 158 | ### 0.1.6 159 | 160 | - Added parameters for Redis connection timeouts 161 | - Error muting can now be disabled for troubleshooting 162 | - Added tracing id in error messages 163 | - Logging of successful rpush after error 164 | 165 | 166 | ### 0.1.5 167 | 168 | - Added support for Docker labels. Thanks to teemupo! 169 | 170 | ### 0.1.4 171 | 172 | - Added support for JSON input. See the paragraph for more information. Thanks to dickiedick62! 173 | 174 | ### 0.1.3 175 | 176 | - Refactoring of configuration possiblities 177 | - Introducing the 'Redis database' configuration parameter, allowing you to push events to a specific Redis database number. 178 | 179 | ### 0.1.2 180 | 181 | - Bugfix: edge case when using a non-Hub image with port number (e.g. my.registry.com:443/my/image:tag) 182 | - Added unit tests to test for regression 183 | 184 | ### 0.1.1 185 | 186 | - The Redis adapter will now reconnect if Redis is unavailable or returns an error. Only 1 reconnect is attempted per event, so if it fails the event gets dropped. Thanks to @rogierlommers 187 | - You can now specify a @type field value. Only if specified this field will be added to the event document. Thanks to @dkhunt27 188 | 189 | ### 0.1.0 190 | 191 | - initial version 192 | 193 | 194 | ## ELK integration 195 | 196 | Try out logspout with redis-logstash adapter in a full ELK stack. A docker-compose.yml can be found in the example/ directory. 197 | 198 | When logspout with adapter is running. Executing something like: 199 | 200 | ``` 201 | docker run --rm centos:7 echo hello from a container 202 | ``` 203 | 204 | Will result in a corresponding event in Elasticsearch. Below is a screenshot from Kibana4: 205 | 206 | ![](event-in-k4.png) 207 | 208 | 209 | ## Credits 210 | 211 | Thanks to [Gliderlabs](https://github.com/gliderlabs) for creating Logspout! 212 | 213 | Much thanks to all contributors. 214 | 215 | For other credits see the header of the redis.go source file. 216 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /redis.go: -------------------------------------------------------------------------------- 1 | // Based on: 2 | // - https://github.com/looplab/logspout-logstash/blob/master/logstash.go 3 | // - https://github.com/gettyimages/logspout-kafka/blob/master/kafka.go 4 | // - https://github.com/gliderlabs/logspout/pull/41/files 5 | // - https://github.com/fsouza/go-dockerclient/blob/master/container.go#L222 6 | 7 | package redis 8 | 9 | import ( 10 | "encoding/json" 11 | "errors" 12 | "fmt" 13 | "log" 14 | "os" 15 | "strconv" 16 | "strings" 17 | "time" 18 | 19 | "github.com/garyburd/redigo/redis" 20 | "github.com/gliderlabs/logspout/router" 21 | ) 22 | 23 | const ( 24 | NO_MESSAGE_PROVIDED = "no message" 25 | LOGTYPE_APPLICATIONLOG = "applog" 26 | LOGTYPE_ACCESSLOG = "accesslog" 27 | DEFAULT_CONNECT_TIMEOUT = 100 28 | DEFAULT_READ_TIMEOUT = 300 29 | DEFAULT_WRITE_TIMEOUT = 500 30 | ) 31 | 32 | type RedisAdapter struct { 33 | route *router.Route 34 | pool *redis.Pool 35 | key string 36 | docker_host string 37 | use_v0 bool 38 | logstash_type string 39 | dedot_labels bool 40 | mute_errors bool 41 | msg_counter int 42 | } 43 | 44 | type DockerFields struct { 45 | Name string `json:"name"` 46 | CID string `json:"cid"` 47 | Image string `json:"image"` 48 | ImageTag string `json:"image_tag,omitempty"` 49 | Source string `json:"source"` 50 | DockerHost string `json:"docker_host,omitempty"` 51 | Labels map[string]string `json:"labels,omitempty"` 52 | } 53 | 54 | type LogstashFields struct { 55 | Docker DockerFields `json:"docker"` 56 | } 57 | 58 | type LogstashMessageV0 struct { 59 | Type string `json:"@type,omitempty"` 60 | Timestamp string `json:"@timestamp"` 61 | Sourcehost string `json:"@source_host"` 62 | Message string `json:"@message"` 63 | Fields LogstashFields `json:"@fields"` 64 | } 65 | 66 | type LogstashMessageV1 struct { 67 | Type string `json:"@type,omitempty"` 68 | Timestamp string `json:"@timestamp"` 69 | Sourcehost string `json:"host"` 70 | Message string `json:"message"` 71 | Fields DockerFields `json:"docker"` 72 | Logtype string `json:"logtype,omitempty"` 73 | // Only one of the following 3 is initialized and used, depending on the incoming json:logtype 74 | LogtypeAccessfields map[string]interface{} `json:"accesslog,omitempty"` 75 | LogtypeAppfields map[string]interface{} `json:"applog,omitempty"` 76 | LogtypeEventfields map[string]interface{} `json:"event,omitempty"` 77 | } 78 | 79 | func init() { 80 | router.AdapterFactories.Register(NewRedisAdapter, "redis") 81 | } 82 | 83 | func NewRedisAdapter(route *router.Route) (router.LogAdapter, error) { 84 | // add port if missing 85 | address := route.Address 86 | if !strings.Contains(address, ":") { 87 | address = address + ":6379" 88 | } 89 | 90 | // get our config keys, first from the route options (e.g. redis://?opt1=val&opt1=val&...) 91 | // if route option is missing, attempt to get the value from the environment 92 | key := getopt(route.Options, "key", "REDIS_KEY", "logspout") 93 | password := getopt(route.Options, "password", "REDIS_PASSWORD", "") 94 | docker_host := getopt(route.Options, "docker_host", "REDIS_DOCKER_HOST", "") 95 | use_v0 := getopt(route.Options, "use_v0_layout", "REDIS_USE_V0_LAYOUT", "") != "" 96 | logstash_type := getopt(route.Options, "logstash_type", "REDIS_LOGSTASH_TYPE", "") 97 | dedot_labels := getopt(route.Options, "dedot_labels", "DEDOT_LABELS", "false") == "true" 98 | debug := getopt(route.Options, "debug", "DEBUG", "") != "" 99 | mute_errors := getopt(route.Options, "mute_errors", "MUTE_ERRORS", "true") == "true" 100 | 101 | connect_timeout := getintopt(route.Options, "connect_timeout", "CONNECT_TIMEOUT", DEFAULT_CONNECT_TIMEOUT) 102 | read_timeout := getintopt(route.Options, "read_timeout", "READ_TIMEOUT", DEFAULT_READ_TIMEOUT) 103 | write_timeout := getintopt(route.Options, "write_timeout", "WRITE_TIMEOUT", DEFAULT_WRITE_TIMEOUT) 104 | 105 | database_s := getopt(route.Options, "database", "REDIS_DATABASE", "0") 106 | database, err := strconv.Atoi(database_s) 107 | if err != nil { 108 | return nil, errorf("Invalid Redis database number specified: %s. Please verify & fix", database_s) 109 | } 110 | 111 | if debug { 112 | log.Printf("Using Redis server '%s', dbnum: %d, password?: %t, pushkey: '%s', v0 layout?: %t, logstash type: '%s'\n", 113 | address, database, password != "", key, use_v0, logstash_type) 114 | log.Printf("Dedotting docker labels: %t", dedot_labels) 115 | log.Printf("Timeouts set, connect: %dms, read: %dms, write: %dms\n", connect_timeout, read_timeout, write_timeout) 116 | } 117 | if connect_timeout+read_timeout+write_timeout > 950 { 118 | log.Printf("WARN: sum of connect, read & write timeouts > 950 ms. You risk loosing container logs as Logspout stops pumping logs after a 1.0 second timeout.") 119 | } 120 | 121 | pool := newRedisConnectionPool(address, password, database, connect_timeout, read_timeout, write_timeout) 122 | 123 | // lets test the water 124 | conn := pool.Get() 125 | defer conn.Close() 126 | res, err := conn.Do("PING") 127 | if err != nil { 128 | return nil, errorf("Cannot connect to Redis server %s: %v", address, err) 129 | } 130 | if debug { 131 | log.Printf("Redis connect successful, got response: %s\n", res) 132 | } 133 | 134 | return &RedisAdapter{ 135 | route: route, 136 | pool: pool, 137 | key: key, 138 | docker_host: docker_host, 139 | use_v0: use_v0, 140 | logstash_type: logstash_type, 141 | dedot_labels: dedot_labels, 142 | mute_errors: mute_errors, 143 | msg_counter: 0, 144 | }, nil 145 | } 146 | 147 | func (a *RedisAdapter) Stream(logstream chan *router.Message) { 148 | conn := a.pool.Get() 149 | defer conn.Close() 150 | 151 | mute := false 152 | 153 | for m := range logstream { 154 | a.msg_counter += 1 155 | msg_id := fmt.Sprintf("%s#%d", m.Container.ID[0:12], a.msg_counter) 156 | 157 | js, err := createLogstashMessage(m, a.docker_host, a.use_v0, a.logstash_type, a.dedot_labels) 158 | if err != nil { 159 | if a.mute_errors { 160 | if !mute { 161 | log.Printf("redis[%s]: error on json.Marshal (muting until recovered): %s\n", msg_id, err) 162 | mute = true 163 | } 164 | } else { 165 | log.Printf("redis[%s]: error on json.Marshal: %s\n", msg_id, err) 166 | } 167 | continue 168 | } 169 | _, err = conn.Do("RPUSH", a.key, js) 170 | if err != nil { 171 | if a.mute_errors { 172 | if !mute { 173 | log.Printf("redis[%s]: error on rpush (muting until restored): %s\n", msg_id, err) 174 | } 175 | } else { 176 | log.Printf("redis[%s]: error on rpush: %s\n", msg_id, err) 177 | } 178 | mute = true 179 | 180 | // first close old connection 181 | conn.Close() 182 | 183 | // next open new connection 184 | conn = a.pool.Get() 185 | 186 | // since message is already marshaled, send again 187 | _, err = conn.Do("RPUSH", a.key, js) 188 | if err != nil { 189 | conn.Close() 190 | if !a.mute_errors { 191 | log.Printf("redis[%s]: error on rpush (retry): %s\n", msg_id, err) 192 | } 193 | } else { 194 | log.Printf("redis[%s]: successful retry rpush after error\n", msg_id) 195 | mute = false 196 | } 197 | 198 | continue 199 | } else { 200 | if mute { 201 | log.Printf("redis[%s]: successful rpush after error\n", msg_id) 202 | mute = false 203 | } 204 | } 205 | } 206 | } 207 | 208 | func errorf(format string, a ...interface{}) (err error) { 209 | err = fmt.Errorf(format, a...) 210 | if os.Getenv("DEBUG") != "" { 211 | fmt.Println(err.Error()) 212 | } 213 | return 214 | } 215 | 216 | func getopt(options map[string]string, optkey string, envkey string, default_value string) (value string) { 217 | value = options[optkey] 218 | if value == "" { 219 | value = os.Getenv(envkey) 220 | if value == "" { 221 | value = default_value 222 | } 223 | } 224 | return 225 | } 226 | func getintopt(options map[string]string, optkey string, envkey string, default_value int) (value int) { 227 | value_s := options[optkey] 228 | if value_s == "" { 229 | value_s = os.Getenv(envkey) 230 | } 231 | if value_s == "" { 232 | value = default_value 233 | } else { 234 | var err error 235 | value, err = strconv.Atoi(value_s) 236 | if err != nil { 237 | log.Printf("Invalid value for integer paramater %s: %s - using default: %d\n", optkey, value_s, default_value) 238 | value = default_value 239 | } 240 | } 241 | return 242 | } 243 | 244 | func newRedisConnectionPool(server, password string, database int, connect_timeout int, read_timeout int, write_timeout int) *redis.Pool { 245 | return &redis.Pool{ 246 | MaxIdle: 1, 247 | IdleTimeout: 240 * time.Second, 248 | Dial: func() (redis.Conn, error) { 249 | c, err := redis.Dial("tcp", server, 250 | redis.DialConnectTimeout(time.Duration(connect_timeout)*time.Millisecond), 251 | redis.DialReadTimeout(time.Duration(read_timeout)*time.Millisecond), 252 | redis.DialWriteTimeout(time.Duration(write_timeout)*time.Millisecond)) 253 | if err != nil { 254 | return nil, err 255 | } 256 | if password != "" { 257 | if _, err := c.Do("AUTH", password); err != nil { 258 | c.Close() 259 | return nil, err 260 | } 261 | } 262 | if database > 0 { 263 | if _, err := c.Do("SELECT", database); err != nil { 264 | c.Close() 265 | return nil, err 266 | } 267 | } 268 | return c, nil 269 | }, 270 | TestOnBorrow: func(c redis.Conn, t time.Time) error { 271 | _, err := c.Do("PING") 272 | if err != nil { 273 | log.Println("redis: test on borrow failed: ", err) 274 | } 275 | return err 276 | }, 277 | } 278 | } 279 | 280 | func splitImage(image_tag string) (image string, tag string) { 281 | colon := strings.LastIndex(image_tag, ":") 282 | sep := strings.LastIndex(image_tag, "/") 283 | if colon > -1 && sep < colon { 284 | image = image_tag[0:colon] 285 | tag = image_tag[colon+1:] 286 | } else { 287 | image = image_tag 288 | } 289 | return 290 | } 291 | 292 | func dedotLabels(labels map[string]string) map[string]string { 293 | for key, _ := range labels { 294 | if strings.Contains(key, ".") { 295 | dedotted_label := strings.Replace(key, ".", "_", -1) 296 | labels[dedotted_label] = labels[key] 297 | delete(labels, key) 298 | } 299 | } 300 | 301 | return labels 302 | } 303 | 304 | func createLogstashMessage(m *router.Message, docker_host string, use_v0 bool, logstash_type string, dedot_labels bool) ([]byte, error) { 305 | image, image_tag := splitImage(m.Container.Config.Image) 306 | cid := m.Container.ID[0:12] 307 | name := m.Container.Name[1:] 308 | timestamp := m.Time.UTC().Format(time.RFC3339Nano) 309 | 310 | if use_v0 { 311 | msg := LogstashMessageV0{} 312 | 313 | msg.Type = logstash_type 314 | msg.Timestamp = timestamp 315 | msg.Message = m.Data 316 | msg.Sourcehost = m.Container.Config.Hostname 317 | msg.Fields.Docker.CID = cid 318 | msg.Fields.Docker.Name = name 319 | msg.Fields.Docker.Image = image 320 | msg.Fields.Docker.ImageTag = image_tag 321 | msg.Fields.Docker.Source = m.Source 322 | msg.Fields.Docker.DockerHost = docker_host 323 | 324 | // see https://github.com/rtoma/logspout-redis-logstash/issues/11 325 | if dedot_labels { 326 | msg.Fields.Docker.Labels = dedotLabels(m.Container.Config.Labels) 327 | } else { 328 | msg.Fields.Docker.Labels = m.Container.Config.Labels 329 | } 330 | 331 | return json.Marshal(msg) 332 | } else { 333 | msg := LogstashMessageV1{} 334 | 335 | msg.Type = logstash_type 336 | msg.Timestamp = timestamp 337 | msg.Sourcehost = m.Container.Config.Hostname 338 | msg.Fields.CID = cid 339 | msg.Fields.Name = name 340 | msg.Fields.Image = image 341 | msg.Fields.ImageTag = image_tag 342 | msg.Fields.Source = m.Source 343 | msg.Fields.DockerHost = docker_host 344 | 345 | // see https://github.com/rtoma/logspout-redis-logstash/issues/11 346 | if dedot_labels { 347 | msg.Fields.Labels = dedotLabels(m.Container.Config.Labels) 348 | } else { 349 | msg.Fields.Labels = m.Container.Config.Labels 350 | } 351 | 352 | // Check if the message to log itself is json 353 | if validJsonMessage(strings.TrimSpace(m.Data)) { 354 | // So it is, include it in the LogstashmessageV1 355 | err := msg.UnmarshalDynamicJSON([]byte(m.Data)) 356 | if err != nil { 357 | // Can't unmarshall the json (invalid?), put it in message 358 | msg.Message = m.Data 359 | } else if msg.Message == "" { 360 | msg.Message = NO_MESSAGE_PROVIDED 361 | } 362 | } else { 363 | // Regular logging (no json) 364 | msg.Message = m.Data 365 | } 366 | return json.Marshal(msg) 367 | } 368 | 369 | } 370 | 371 | func validJsonMessage(s string) bool { 372 | 373 | if !strings.HasPrefix(s, "{") || !strings.HasSuffix(s, "}") { 374 | return false 375 | } 376 | return true 377 | } 378 | 379 | func (d *LogstashMessageV1) UnmarshalDynamicJSON(data []byte) error { 380 | var dynMap map[string]interface{} 381 | 382 | if d == nil { 383 | return errors.New("RawString: UnmarshalJSON on nil pointer") 384 | } 385 | 386 | if err := json.Unmarshal(data, &dynMap); err != nil { 387 | return err 388 | } 389 | 390 | // Take logtype of the hash, but only if it is a valid logtype 391 | if _, ok := dynMap["logtype"].(string); ok { 392 | if dynMap["logtype"].(string) == LOGTYPE_APPLICATIONLOG || dynMap["logtype"].(string) == LOGTYPE_ACCESSLOG { 393 | d.Logtype = dynMap["logtype"].(string) 394 | delete(dynMap, "logtype") 395 | } 396 | } 397 | // Take message out of the hash 398 | if _, ok := dynMap["message"]; ok { 399 | d.Message = dynMap["message"].(string) 400 | delete(dynMap, "message") 401 | } 402 | 403 | // Only initialize the "used" hash in struct 404 | if d.Logtype == LOGTYPE_APPLICATIONLOG { 405 | d.LogtypeAppfields = make(map[string]interface{}, 0) 406 | } else if d.Logtype == LOGTYPE_ACCESSLOG { 407 | d.LogtypeAccessfields = make(map[string]interface{}, 0) 408 | } else { 409 | d.LogtypeEventfields = make(map[string]interface{}, 0) 410 | } 411 | 412 | // Fill the right hash based on logtype 413 | for key, val := range dynMap { 414 | if d.Logtype == LOGTYPE_APPLICATIONLOG { 415 | d.LogtypeAppfields[key] = val 416 | } else if d.Logtype == LOGTYPE_ACCESSLOG { 417 | d.LogtypeAccessfields[key] = val 418 | } else { 419 | d.LogtypeEventfields[key] = val 420 | } 421 | } 422 | 423 | return nil 424 | } 425 | -------------------------------------------------------------------------------- /redis_test.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | //"log" 7 | "testing" 8 | "time" 9 | 10 | "github.com/fsouza/go-dockerclient" 11 | "github.com/gliderlabs/logspout/router" 12 | "github.com/jmoiron/jsonq" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestSplitImage(t *testing.T) { 17 | assert := assert.New(t) 18 | 19 | image, tag := splitImage("bla") 20 | assert.Equal("bla", image) 21 | assert.Equal("", tag) 22 | 23 | image, tag = splitImage("foo:latest") 24 | assert.Equal("foo", image) 25 | assert.Equal("latest", tag) 26 | 27 | image, tag = splitImage("foo/bar:latest") 28 | assert.Equal("foo/bar", image) 29 | assert.Equal("latest", tag) 30 | 31 | image, tag = splitImage("my.registry.host/some/image:1.3.4") 32 | assert.Equal("my.registry.host/some/image", image) 33 | assert.Equal("1.3.4", tag) 34 | 35 | image, tag = splitImage("my.registry.host:443/path/to/image:3.1.4") 36 | assert.Equal("my.registry.host:443/path/to/image", image) 37 | assert.Equal("3.1.4", tag) 38 | 39 | image, tag = splitImage("my.registry.host:443/path/to/image") 40 | assert.Equal("my.registry.host:443/path/to/image", image) 41 | assert.Equal("", tag) 42 | 43 | } 44 | 45 | func TestCreateLogstashMessageV1(t *testing.T) { 46 | 47 | assert := assert.New(t) 48 | 49 | m := router.Message{ 50 | Container: &docker.Container{ 51 | ID: "6feffd9428dc", 52 | Name: "/my_app", 53 | Config: &docker.Config{ 54 | Hostname: "container_hostname", 55 | Image: "my.registry.host:443/path/to/image:1234", 56 | Labels: map[string]string{"label_1": "abc", "label_2": "def"}, 57 | }, 58 | }, 59 | Source: "stdout", 60 | Data: "hello world", 61 | Time: time.Unix(int64(1453818496), 595000000), 62 | } 63 | 64 | msg, _ := createLogstashMessage(&m, "tst-mesos-slave-001", false, "my-type", false) 65 | jq := makeQuery(msg) 66 | 67 | assert.Equal("my-type", getString(jq, "@type")) 68 | assert.Equal("2016-01-26T14:28:16.595Z", getString(jq, "@timestamp")) 69 | assert.Equal("container_hostname", getString(jq, "host")) 70 | assert.Equal("hello world", getString(jq, "message")) 71 | assert.Equal("my_app", getString(jq, "docker", "name")) 72 | assert.Equal("6feffd9428dc", getString(jq, "docker", "cid")) 73 | assert.Equal("my.registry.host:443/path/to/image", getString(jq, "docker", "image")) 74 | assert.Equal("1234", getString(jq, "docker", "image_tag")) 75 | assert.Equal("stdout", getString(jq, "docker", "source")) 76 | assert.Equal("tst-mesos-slave-001", getString(jq, "docker", "docker_host")) 77 | assert.Equal("abc", getString(jq, "docker", "labels", "label_1")) 78 | assert.Equal("def", getString(jq, "docker", "labels", "label_2")) 79 | 80 | } 81 | 82 | func TestCreateLogstashMessageV0(t *testing.T) { 83 | 84 | assert := assert.New(t) 85 | 86 | m := router.Message{ 87 | Container: &docker.Container{ 88 | ID: "f00ffd9428dc", 89 | Name: "/my_db", 90 | Config: &docker.Config{ 91 | Hostname: "container_hostname", 92 | Image: "my.registry.host:443/path/to/image:4321", 93 | Labels: map[string]string{"label_1": "abc", "label_2": "def"}, 94 | }, 95 | }, 96 | Source: "stderr", 97 | Data: "cruel world", 98 | Time: time.Unix(int64(1453813310), 1000000), 99 | } 100 | 101 | msg, _ := createLogstashMessage(&m, "tst-mesos-slave-001", true, "some-type", false) 102 | jq := makeQuery(msg) 103 | 104 | assert.Equal("some-type", getString(jq, "@type")) 105 | assert.Equal("2016-01-26T13:01:50.001Z", getString(jq, "@timestamp")) 106 | assert.Equal("container_hostname", getString(jq, "@source_host")) 107 | assert.Equal("cruel world", getString(jq, "@message")) 108 | assert.Equal("my_db", getString(jq, "@fields", "docker", "name")) 109 | assert.Equal("f00ffd9428dc", getString(jq, "@fields", "docker", "cid")) 110 | assert.Equal("my.registry.host:443/path/to/image", getString(jq, "@fields", "docker", "image")) 111 | assert.Equal("4321", getString(jq, "@fields", "docker", "image_tag")) 112 | assert.Equal("stderr", getString(jq, "@fields", "docker", "source")) 113 | assert.Equal("tst-mesos-slave-001", getString(jq, "@fields", "docker", "docker_host")) 114 | assert.Equal("", getString(jq, "@fields", "decode_error")) 115 | assert.Equal("abc", getString(jq, "@fields", "docker", "labels", "label_1")) 116 | assert.Equal("def", getString(jq, "@fields", "docker", "labels", "label_2")) 117 | 118 | } 119 | 120 | func TestCreateLogstashMessageOptionalType(t *testing.T) { 121 | 122 | assert := assert.New(t) 123 | 124 | m := router.Message{ 125 | Container: &docker.Container{ 126 | ID: "f00ffd9428dc", 127 | Name: "/my_db", 128 | Config: &docker.Config{ 129 | Hostname: "container_hostname", 130 | Image: "my.registry.host:443/path/to/image:4321", 131 | }, 132 | }, 133 | Source: "stderr", 134 | Data: "cruel world", 135 | Time: time.Unix(int64(1453813330), 0), 136 | } 137 | 138 | msg, _ := createLogstashMessage(&m, "tst-mesos-slave-001", true, "", false) 139 | jq := makeQuery(msg) 140 | //log.Printf("Standard message: %s", msg) 141 | 142 | assert.Equal("", getString(jq, "@type")) 143 | 144 | } 145 | 146 | func TestCreateLogstashMessageWithJsonData(t *testing.T) { 147 | 148 | assert := assert.New(t) 149 | 150 | m := router.Message{ 151 | Container: &docker.Container{ 152 | ID: "6feffd9428dc", 153 | Name: "/my_app", 154 | Config: &docker.Config{ 155 | Hostname: "container_hostname", 156 | Image: "my.registry.host:443/path/to/image:1234", 157 | }, 158 | }, 159 | Source: "stdout", 160 | Data: `{"logtype": "applog", "message":"something happened", "level": "DEBUG", "file": "debug.go", "line": 42}`, 161 | Time: time.Unix(int64(1453818496), 595000000), 162 | } 163 | 164 | msg, _ := createLogstashMessage(&m, "tst-mesos-slave-001", false, "my-type", false) 165 | jq := makeQuery(msg) 166 | 167 | assert.Equal("something happened", getString(jq, "message")) 168 | assert.Equal("applog", getString(jq, "logtype")) 169 | assert.Equal("DEBUG", getString(jq, "applog", "level")) 170 | assert.Equal("debug.go", getString(jq, "applog", "file")) 171 | assert.Equal(42, getInt(jq, "applog", "line")) 172 | 173 | } 174 | 175 | func TestCreateLogstashMessageWithJsonDataAndNoMessage(t *testing.T) { 176 | 177 | assert := assert.New(t) 178 | 179 | m := router.Message{ 180 | Container: &docker.Container{ 181 | ID: "6feffd9428dc", 182 | Name: "/my_app", 183 | Config: &docker.Config{ 184 | Hostname: "container_hostname", 185 | Image: "my.registry.host:443/path/to/image:1234", 186 | }, 187 | }, 188 | Source: "stdout", 189 | Data: `{ "logtype": "applog", "level": "DEBUG", "file": "debug.go", "line": 14}`, 190 | Time: time.Unix(int64(1453818496), 595000000), 191 | } 192 | 193 | msg, _ := createLogstashMessage(&m, "tst-mesos-slave-001", false, "my-type", false) 194 | jq := makeQuery(msg) 195 | 196 | assert.Equal("no message", getString(jq, "message")) 197 | 198 | } 199 | 200 | func TestCreateLogstashMessageWithJsonDataAndNoLogtype(t *testing.T) { 201 | 202 | assert := assert.New(t) 203 | 204 | m := router.Message{ 205 | Container: &docker.Container{ 206 | ID: "6feffd9428dc", 207 | Name: "/my_app", 208 | Config: &docker.Config{ 209 | Hostname: "container_hostname", 210 | Image: "my.registry.host:443/path/to/image:1234", 211 | }, 212 | }, 213 | Source: "stdout", 214 | Data: `{ "message":"here i am!", "level": "DEBUG", "file": "debug.go", "line": 42}`, 215 | Time: time.Unix(int64(1453818496), 595000000), 216 | } 217 | 218 | msg, _ := createLogstashMessage(&m, "tst-mesos-slave-001", false, "my-type", false) 219 | jq := makeQuery(msg) 220 | 221 | assert.Equal("here i am!", getString(jq, "message")) 222 | assert.Equal("", getString(jq, "logtype")) 223 | assert.Equal("DEBUG", getString(jq, "event", "level")) 224 | assert.Equal("debug.go", getString(jq, "event", "file")) 225 | assert.Equal(42, getInt(jq, "event", "line")) 226 | 227 | } 228 | 229 | func TestCreateLogstashMessageWithJsonDataAndUnknownLogtype(t *testing.T) { 230 | 231 | assert := assert.New(t) 232 | 233 | m := router.Message{ 234 | Container: &docker.Container{ 235 | ID: "6feffd9428dc", 236 | Name: "/my_app", 237 | Config: &docker.Config{ 238 | Hostname: "container_hostname", 239 | Image: "my.registry.host:443/path/to/image:1234", 240 | }, 241 | }, 242 | Source: "stdout", 243 | Data: `{ "logtype": "nolog", "message":"here i am again!", "level": "INFO", "file": "bla.go", "line": 24}`, 244 | Time: time.Unix(int64(1453818496), 595000000), 245 | } 246 | 247 | msg, _ := createLogstashMessage(&m, "tst-mesos-slave-001", false, "my-type", false) 248 | jq := makeQuery(msg) 249 | //log.Printf("Dynamic message: %s", msg) 250 | 251 | assert.Equal("here i am again!", getString(jq, "message")) 252 | assert.Equal("", getString(jq, "logtype")) 253 | assert.Equal("nolog", getString(jq, "event", "logtype")) 254 | assert.Equal("INFO", getString(jq, "event", "level")) 255 | assert.Equal("bla.go", getString(jq, "event", "file")) 256 | assert.Equal(24, getInt(jq, "event", "line")) 257 | 258 | } 259 | 260 | func TestCreateLogstashMessageWithJsonDataAndAccesLogtype(t *testing.T) { 261 | 262 | assert := assert.New(t) 263 | 264 | m := router.Message{ 265 | Container: &docker.Container{ 266 | ID: "6feffd9428dc", 267 | Name: "/my_app", 268 | Config: &docker.Config{ 269 | Hostname: "container_hostname", 270 | Image: "my.registry.host:443/path/to/image:1234", 271 | }, 272 | }, 273 | Source: "stdout", 274 | Data: `{"agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36","auth":"-","bytes":3488,"client":"[::1]:50393","cookies":"JSESSIONID=eqsjg19bvla01dst8smi6d0f; bol.workbench.remember=emicklei; dev_appserver_login=test@example.com:False:185804764220139124118; _ga=GA1.1.1754835192.1422042636; ","httpversion":"HTTP/1.1","ident":"-","jsession_id":"","logtype":"accesslog","message":"/internal/apidocs.json/v1/policies","mime":"application/json","referrer":"http://localhost:9191/internal/apidocs/","response":200,"site":"localhost:9191","ssl":"false","time_in_sec":0,"time_in_usec":613,"unique_id":"","verb":"GET"}`, 275 | Time: time.Unix(int64(1453818496), 595000000), 276 | } 277 | 278 | msg, _ := createLogstashMessage(&m, "tst-mesos-slave-001", false, "my-type", false) 279 | jq := makeQuery(msg) 280 | //log.Printf("Dynamic message: %s", msg) 281 | 282 | assert.Equal("accesslog", getString(jq, "logtype")) 283 | assert.Equal("/internal/apidocs.json/v1/policies", getString(jq, "message")) 284 | assert.Equal(200, getInt(jq, "accesslog", "response")) 285 | assert.Equal(3488, getInt(jq, "accesslog", "bytes")) 286 | 287 | } 288 | 289 | func TestCreateLogstashMessageWithJsonDataAndNumericLogtype(t *testing.T) { 290 | 291 | assert := assert.New(t) 292 | 293 | m := router.Message{ 294 | Container: &docker.Container{ 295 | ID: "6feffd9428dc", 296 | Name: "/my_app", 297 | Config: &docker.Config{ 298 | Hostname: "container_hostname", 299 | Image: "my.registry.host:443/path/to/image:1234", 300 | }, 301 | }, 302 | Source: "stdout", 303 | Data: `{ "logtype": 1, "message":"here i am!", "level": "DEBUG", "file": "debug.go", "line": 42}`, 304 | Time: time.Unix(int64(1453818496), 595000000), 305 | } 306 | 307 | msg, _ := createLogstashMessage(&m, "tst-mesos-slave-001", false, "my-type", false) 308 | jq := makeQuery(msg) 309 | //log.Printf("Dynamic message: %s", msg) 310 | 311 | assert.Equal("", getString(jq, "logtype")) 312 | assert.Equal(42, getInt(jq, "event", "line")) 313 | 314 | } 315 | 316 | func TestCreateLogstashMessageWithInvalidJsonData(t *testing.T) { 317 | 318 | assert := assert.New(t) 319 | 320 | m := router.Message{ 321 | Container: &docker.Container{ 322 | ID: "6feffd9428dc", 323 | Name: "/my_app", 324 | Config: &docker.Config{ 325 | Hostname: "container_hostname", 326 | Image: "my.registry.host:443/path/to/image:1234", 327 | }, 328 | }, 329 | Source: "stdout", 330 | Data: `{ "logtype": 1, "message":"here i am!", ": "DEBUG", "file": "debug.go", "line": 42}`, 331 | Time: time.Unix(int64(1453818496), 595000000), 332 | } 333 | 334 | msg, _ := createLogstashMessage(&m, "tst-mesos-slave-001", false, "my-type", false) 335 | jq := makeQuery(msg) 336 | //log.Printf("Dynamic message invalid json: %s", msg) 337 | 338 | assert.Equal("", getString(jq, "logtype")) 339 | 340 | } 341 | 342 | func TestCreateLogstashMessageV0WithDeDottingEnabled(t *testing.T) { 343 | 344 | assert := assert.New(t) 345 | 346 | m := router.Message{ 347 | Container: &docker.Container{ 348 | ID: "f00ffd9428dc", 349 | Name: "/my_db", 350 | Config: &docker.Config{ 351 | Hostname: "container_hostname", 352 | Image: "my.registry.host:443/path/to/image:4321", 353 | Labels: map[string]string{"label.1.2.3": "abc", "label.3.2.1": "def"}, 354 | }, 355 | }, 356 | Source: "stderr", 357 | Data: "cruel world", 358 | Time: time.Unix(int64(1453813310), 1000000), 359 | } 360 | 361 | msg, _ := createLogstashMessage(&m, "tst-mesos-slave-001", true, "some-type", true) 362 | jq := makeQuery(msg) 363 | //log.Printf("%s", msg) 364 | 365 | assert.Equal("abc", getString(jq, "@fields", "docker", "labels", "label_1_2_3")) 366 | assert.Equal("def", getString(jq, "@fields", "docker", "labels", "label_3_2_1")) 367 | 368 | } 369 | 370 | func TestCreateLogstashMessageV1WithDeDottingEnabled(t *testing.T) { 371 | 372 | assert := assert.New(t) 373 | 374 | m := router.Message{ 375 | Container: &docker.Container{ 376 | ID: "f00ffd9428dc", 377 | Name: "/my_db", 378 | Config: &docker.Config{ 379 | Hostname: "container_hostname", 380 | Image: "my.registry.host:443/path/to/image:4321", 381 | Labels: map[string]string{"label.1.2.3": "abc", "label.3.2.1": "def"}, 382 | }, 383 | }, 384 | Source: "stderr", 385 | Data: "cruel world", 386 | Time: time.Unix(int64(1453813310), 1000000), 387 | } 388 | 389 | msg, _ := createLogstashMessage(&m, "tst-mesos-slave-001", false, "some-type", true) 390 | jq := makeQuery(msg) 391 | 392 | assert.Equal("abc", getString(jq, "docker", "labels", "label_1_2_3")) 393 | assert.Equal("def", getString(jq, "docker", "labels", "label_3_2_1")) 394 | 395 | } 396 | 397 | func TestCreateLogstashMessageV0WithDeDottingDefaultDisabled(t *testing.T) { 398 | 399 | assert := assert.New(t) 400 | 401 | m := router.Message{ 402 | Container: &docker.Container{ 403 | ID: "f00ffd9428dc", 404 | Name: "/my_db", 405 | Config: &docker.Config{ 406 | Hostname: "container_hostname", 407 | Image: "my.registry.host:443/path/to/image:4321", 408 | Labels: map[string]string{"label.1.2.3": "abc", "label.3.2.1": "def"}, 409 | }, 410 | }, 411 | Source: "stderr", 412 | Data: "cruel world", 413 | Time: time.Unix(int64(1453813310), 1000000), 414 | } 415 | 416 | msg, _ := createLogstashMessage(&m, "tst-mesos-slave-001", true, "some-type", false) 417 | jq := makeQuery(msg) 418 | 419 | assert.Equal("abc", getString(jq, "@fields", "docker", "labels", "label.1.2.3")) 420 | assert.Equal("def", getString(jq, "@fields", "docker", "labels", "label.3.2.1")) 421 | 422 | } 423 | 424 | func TestCreateLogstashMessageV1WithDeDottingDefaultDisabled(t *testing.T) { 425 | 426 | assert := assert.New(t) 427 | 428 | m := router.Message{ 429 | Container: &docker.Container{ 430 | ID: "f00ffd9428dc", 431 | Name: "/my_db", 432 | Config: &docker.Config{ 433 | Hostname: "container_hostname", 434 | Image: "my.registry.host:443/path/to/image:4321", 435 | Labels: map[string]string{"label.1.2.3": "abc", "label.3.2.1": "def"}, 436 | }, 437 | }, 438 | Source: "stderr", 439 | Data: "cruel world", 440 | Time: time.Unix(int64(1453813310), 1000000), 441 | } 442 | 443 | msg, _ := createLogstashMessage(&m, "tst-mesos-slave-001", false, "some-type", false) 444 | jq := makeQuery(msg) 445 | //log.Printf("%s", msg) 446 | 447 | assert.Equal("abc", getString(jq, "docker", "labels", "label.1.2.3")) 448 | assert.Equal("def", getString(jq, "docker", "labels", "label.3.2.1")) 449 | 450 | } 451 | 452 | func TestValidJsonMessageNoJson(t *testing.T) { 453 | assert := assert.New(t) 454 | 455 | js := `whateverthefuckever` 456 | assert.False(validJsonMessage(js)) 457 | 458 | } 459 | 460 | func TestValidJsonMessageJson(t *testing.T) { 461 | assert := assert.New(t) 462 | 463 | js := `{"message":"foo"}` 464 | assert.True(validJsonMessage(js)) 465 | 466 | } 467 | 468 | func getInt(j *jsonq.JsonQuery, s ...string) int { 469 | v, _ := j.Int(s...) 470 | return v 471 | } 472 | 473 | func getString(j *jsonq.JsonQuery, s ...string) string { 474 | v, _ := j.String(s...) 475 | return v 476 | } 477 | 478 | func makeQuery(msg []byte) *jsonq.JsonQuery { 479 | data := map[string]interface{}{} 480 | dec := json.NewDecoder(bytes.NewReader(msg)) 481 | dec.Decode(&data) 482 | jq := jsonq.NewQuery(data) 483 | return jq 484 | } 485 | --------------------------------------------------------------------------------