├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── Dockerfile ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── ci └── build.sh ├── daemonset ├── nonrbac │ └── fluentd.yaml └── rbac │ └── fluentd.yaml ├── entrypoint.sh ├── fluent-plugin-kubernetes_sumologic.gemspec ├── lib └── fluent │ └── plugin │ └── filter_kubernetes_sumologic.rb ├── screenshots ├── container.png ├── docker.png ├── kubelet.png └── kubernetes.png └── test ├── helper.rb └── plugin └── test_filter_kubernetes_sumologic.rb /.gitignore: -------------------------------------------------------------------------------- 1 | # Ruby ignores 2 | *.bridgesupport 3 | *.gem 4 | *.rbc 5 | .dat* 6 | .bundle/* 7 | .ruby-gemset 8 | .ruby-version 9 | .rvmrc 10 | bin/* 11 | coverage 12 | Gemfile.lock 13 | TAGS 14 | 15 | # Mac OSX ignores 16 | .DS_Store 17 | 18 | # VS Code ignores 19 | .vscode/* 20 | 21 | # idea ignores 22 | .idea 23 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | lang: ruby 2 | services: docker 3 | before_install: 4 | - rm -f ./Gemfile.lock 5 | - gem install bundler 6 | script: ci/build.sh 7 | deploy: 8 | provider: rubygems 9 | skip_cleanup: true 10 | api_key: 11 | secure: utz+1f2fW1CTu3rTcvpqz6kcQQak4+1Fb4MvlCHKE4nqjC2ujhxjagzXydFwBVuyrw+/VvFfNshZuO5Drz3qpvcLAqPd7ptru3bnYDiwMTFBiVg3SSVtTFphlNKuwNwM4alT/7Jhi3eW0LVXYAXPA3nhh7fOkTbJb9J2FcKD1/ty1ybgFVeUJlaJ8pcqA0Q4PD/dW/6CMkzt4x2CDgsE+MsufFg2SqGelxty1vWIk4oBa6PsDgAoauX8d9KUsa5MRFi7OWTrZviDKwfHRrTb99jW/k+4iQX0+gwIWDOQwTrajQreNIShvwciiA71MdQ60uFNPQYLretrt2C1cIyeB6Vyt/ozGREo6JfLpCNiwwS2PmWknDsa4nQFytrE6wH9qRQz89Q8vkr4dMlneVIgUDkQBVe7nVuatKOBMwN+ZxHLoDdVcGzLz2wWIzc7FcIiUWFphOiWmZB8G4+St+zOyVCTM25qHby9xs0DDveXyCLXCL2CCip7lG7ZnBaWrmW43YvLKKip2NGmlmi6oecsvT8IbCMxJoj5hGgJXbzYC8dAJ0ZATuVbQuzn94hhPwWWOLkcJ3xqg8jzEwoiMhYvzjvywTlKlc+lgglXIvQVfA4HXmttYX1WkbXuQD4CtdUgdB5JznsaVQo3n9UwD1iiAk/kuwWyKOOC+AmSX2SKlb0= 12 | gem: fluent-plugin-kubernetes_sumologic 13 | on: 14 | tags: true 15 | repo: SumoLogic/fluentd-kubernetes-sumologic 16 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | ## 2.4.0 4 | - Improve Ruby GC to fix #121 [(#125)](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/125) 5 | - Migrate FluentD Conf files to ConfigMap to make customizing FluentD conf simpler [(#124)](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/124) 6 | - Add /u01 mount for running fluentd plugin under OCI OKE [(#129)](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/129) 7 | - Add support for log metadata [(#133)](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/133) 8 | 9 | ## 2.3.1 10 | - Support %{pod_id} interpolation in source metadata [(#110)](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/110) 11 | - Support forward input as source [(#113)](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/113) 12 | 13 | 14 | ## 2.3.0 15 | - Polish "reduce metadata" feature with keeping `namespace_name` [(#108)](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/108) 16 | - Bug fix - missing params in `containers.**` [(#109)](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/109) 17 | 18 | ## 2.2.0 19 | - bump base image to 1.3.2 20 | - upgrade Sumo Logic FluentD Output plugin to 1.4.0, expose timestamp_key configuration 21 | 22 | ## 2.1.0 23 | - [Correctly remove the dynamic replicaset from the pod_name](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/100) 24 | - [Control adding time and stream](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/96) 25 | - [Reduce k8s metadata logs](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/92) 26 | 27 | ## 2.0.0 28 | - remove duplicate control plane logs to resolve #79. Considered a breaking change as this requires the Kubernetes App in Sumo Logic to be updated as it previously used these logs. Please [see here](README.md#upgrading-to-v200) for more information. 29 | - set kubernetes metadata filter plugin logging to warn to remove noisy logs 30 | - upgrade docker image to fluentd 1.2.6 31 | - add rewrite-tag-filter and prometheus plugins 32 | - test for existence of labels key in kubernetes metadata 33 | 34 | ## 1.2.0 35 | - change version scheme 36 | - [Add labels to k8s_metadata](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/83) 37 | - [Add source_host to conf.d/systemd/source.containers.conf](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/84) 38 | - minor refactoring and testing 39 | - upgrade fluent-plugin-sumologic_output to latest (1.3.1) 40 | - expose additional configuration options to provide more control over fluent-plugin-kubernetes_metadata_filter using default values (fixes #80 and #35) 41 | 42 | ## 1.18 43 | - add FluentD S3 Output plugin to give more output options. 44 | 45 | ## 1.17 46 | - [fix #69 by upgrading sumologic fluentd output to 1.1.1 which contains fix to handle json parsing correctly](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/commit/498bd09fc2f353f0986ef3a9c583e4fb8ce7b401) 47 | 48 | ## 1.16 49 | -[Enable ability to configure enable_stat_watcher on in_tail plugins to workaround issue where inotify may hang.](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/commit/b963866aea0587d079913f87ca1a60b4d8afb982) 50 | 51 | ## 1.15 52 | -[Expose environment variable to configure proxy_uri in sumo out configuration](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/issues/65) 53 | -[Allow container path for in_tail plugin to be configured via environment variable. Default to current value for backwards compatibility.](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/issues/64) 54 | 55 | ## 1.14 56 | -[kubernetes_sumologic plugin always adds a timestamp field to json logs](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/issues/61) 57 | 58 | ## 1.13 59 | -[Add ntp and timesyncd to systemd services](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/59#pullrequestreview-115374648) 60 | -[Updates to new Audit Logging format](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/commit/90cc454927055cd337a91942b285e6b57264e8c5) 61 | 62 | ## 1.12 63 | -[Allow hand picked data to be sent to sumo when certain exclusions apply](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/58) 64 | 65 | ## 1.11 66 | -[upgrade metadata filter plugin to address #54](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/commit/eb7a69ce5b36a9d8150bd0a0b62f98fb18d35367) 67 | 68 | ## 1.10 69 | - [add missing exclude options to kubelet logs](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/commit/ced3304f1e64173554a8dee6367c785945f3ce99) 70 | - [Use multi-stage build to eliminate build tools from final image](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/57) 71 | 72 | ## 1.9 73 | - base docker image was quite stale, this release bumps to 1.1.3 and anchors the plugins to specific versions. Same versions from 1.8 used, this just enforces consistency. 74 | - ensure read_from_head in systemd respects setting via environment variable 75 | 76 | ## 1.8 77 | - [Change default FLUSH_INTERVAL to 5s](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/commit/7b100306d6c84335ee0d4ec6724a3218e8028893) 78 | - [Handle possible timeout from Concat Plugin and ensure all logs resume the flow thru rest of pipeline when this occurs](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/commit/7b100306d6c84335ee0d4ec6724a3218e8028893) 79 | - [Allow the time_key field to be defined via environment variables](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/53) 80 | 81 | ## 1.7 82 | - [Fix typo in sumologic kubernetes filter](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/51) 83 | 84 | ## v1.6 85 | - upgrade fluentd-sumo_output plugin to latest 1.0, adds support for millisecond precision. 86 | - add support for kubernetes audit log 87 | 88 | ## v1.5 89 | 90 | - [Setting up tolerations](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/43) 91 | - [add some etcdadm services](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/41) 92 | - [Add RBAC permissions ](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/40) 93 | 94 | ## v1.4 95 | 96 | - [add support for multi-line log messages ](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/33) 97 | 98 | ## v1.3 99 | 100 | - [fix empty? checks on nil values by switching defaults to empty strings](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/32) 101 | 102 | ## v1.2 103 | 104 | - [Enable monitoring endpoints](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/28) 105 | 106 | ## v1.1 107 | 108 | - [Add support for systemd](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/pull/21). 109 | 110 | ## v1.0 111 | 112 | - Initial tag 113 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM fluent/fluentd:v1.3.2-debian AS builder 2 | 3 | ENV PATH /home/fluent/.gem/ruby/2.3.0/bin:$PATH 4 | 5 | COPY ./fluent-plugin-kubernetes_sumologic*.gem ./ 6 | 7 | # New fluent image dynamically creates user in entrypoint 8 | RUN [ -f /bin/entrypoint.sh ] && /bin/entrypoint.sh echo || : && \ 9 | apt-get update && \ 10 | apt-get install -y build-essential ruby-dev libffi-dev libsystemd-dev && \ 11 | gem install fluent-plugin-s3 -v 1.1.4 && \ 12 | gem install fluent-plugin-systemd -v 0.3.1 && \ 13 | gem install fluent-plugin-record-reformer -v 0.9.1 && \ 14 | gem install fluent-plugin-kubernetes_metadata_filter -v 1.0.2 && \ 15 | gem install fluent-plugin-sumologic_output -v 1.4.0 && \ 16 | gem install fluent-plugin-concat -v 2.3.0 && \ 17 | gem install fluent-plugin-rewrite-tag-filter -v 2.1.0 && \ 18 | gem install fluent-plugin-prometheus -v 1.1.0 && \ 19 | gem install fluent-plugin-kubernetes_sumologic && \ 20 | rm -rf /home/fluent/.gem/ruby/2.3.0/cache/*.gem && \ 21 | gem sources -c && \ 22 | apt-get remove --purge -y build-essential ruby-dev libffi-dev libsystemd-dev && \ 23 | rm -rf /var/lib/apt/lists/* 24 | 25 | FROM fluent/fluentd:v1.3.2-debian 26 | 27 | WORKDIR /home/fluent 28 | ENV PATH /home/fluent/.gem/ruby/2.3.0/bin:$PATH 29 | 30 | RUN mkdir -p /mnt/pos 31 | EXPOSE 24284 32 | 33 | RUN mkdir -p /fluentd/etc && \ 34 | mkdir -p /fluentd/plugins 35 | 36 | # Default settings 37 | ENV LOG_FORMAT "json" 38 | ENV FLUSH_INTERVAL "5s" 39 | ENV NUM_THREADS "1" 40 | ENV SOURCE_CATEGORY "%{namespace}/%{pod_name}" 41 | ENV SOURCE_CATEGORY_PREFIX "kubernetes/" 42 | ENV SOURCE_CATEGORY_REPLACE_DASH "/" 43 | ENV SOURCE_NAME "%{namespace}.%{pod}.%{container}" 44 | ENV KUBERNETES_META "true" 45 | ENV KUBERNETES_META_REDUCE "false" 46 | ENV READ_FROM_HEAD "true" 47 | ENV FLUENTD_SOURCE "file" 48 | ENV FLUENTD_USER_CONFIG_DIR "/fluentd/conf.d/user" 49 | ENV MULTILINE_START_REGEXP "/^\w{3} \d{1,2}, \d{4}/" 50 | ENV CONCAT_SEPARATOR "" 51 | ENV AUDIT_LOG_PATH "/mnt/log/kube-apiserver-audit.log" 52 | ENV TIME_KEY "time" 53 | ENV ADD_TIMESTAMP "true" 54 | ENV TIMESTAMP_KEY "timestamp" 55 | ENV ADD_STREAM "true" 56 | ENV ADD_TIME "true" 57 | ENV CONTAINER_LOGS_PATH "/mnt/log/containers/*.log" 58 | ENV ENABLE_STAT_WATCHER "true" 59 | ENV K8S_METADATA_FILTER_WATCH "true" 60 | ENV K8S_METADATA_FILTER_VERIFY_SSL "true" 61 | ENV K8S_METADATA_FILTER_BEARER_CACHE_SIZE "1000" 62 | ENV K8S_METADATA_FILTER_BEARER_CACHE_TTL "3600" 63 | ENV VERIFY_SSL "true" 64 | ENV FORWARD_INPUT_BIND "0.0.0.0" 65 | ENV FORWARD_INPUT_PORT "24224" 66 | 67 | COPY --from=builder /var/lib/gems /var/lib/gems 68 | COPY ./entrypoint.sh /fluentd/ 69 | 70 | ENTRYPOINT ["/fluentd/entrypoint.sh"] 71 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | group :test do 4 | gem 'codecov' 5 | gem 'simplecov' 6 | gem 'webmock' 7 | end 8 | 9 | gemspec -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/SumoLogic/fluentd-kubernetes-sumologic.svg?branch=master)](https://travis-ci.org/SumoLogic/fluentd-kubernetes-sumologic) [![Gem Version](https://badge.fury.io/rb/fluent-plugin-kubernetes_sumologic.svg)](https://badge.fury.io/rb/fluent-plugin-kubernetes_sumologic) [![Docker Pulls](https://img.shields.io/docker/pulls/sumologic/fluentd-kubernetes-sumologic.svg)](https://hub.docker.com/r/sumologic/fluentd-kubernetes-sumologic) [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/SumoLogic/fluentd-output-sumologic/issues) 2 | 3 | ## Deprecation notice 4 | Sumo Logic has ended development of this project, which has been replaced by the officially supported [Sumo Logic Kubernetes Collection](https://github.com/SumoLogic/sumologic-kubernetes-collection) project. If you are planning to set up collection from a new cluster to Sumo Logic, we highly recommend that you use the supported Sumo Logic Kubernetes Collection project rather than this one. We will continue to support bug fixes for this project, but we expect to end support in summer 2020. Please consult the Kubernetes Collection project for instructions on how to [migrate existing instances](https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/master/deploy/docs/Migration_Steps.md) of this project to our new, fully supported solution. 5 | 6 | This page describes the Sumo Kubernetes [Fluentd](http://www.fluentd.org/) plugin. 7 | 8 | ## Support 9 | The code in this repository has been developed in collaboration with the Sumo Logic community and is not supported via standard Sumo Logic Support channels. For any issues or questions please submit an issue within the GitHub repository. The maintainers of this project will work directly with the community to answer any questions, address bugs, or review any requests for new features. 10 | 11 | ## Installation 12 | 13 | The plugin runs as a Kubernetes [DaemonSet](http://kubernetes.io/docs/admin/daemons/); it runs an instance of the plugin on each host in a cluster. Each plugin instance pulls system, kubelet, docker daemon, and container logs from the host and sends them, in JSON or text format, to an HTTP endpoint on a hosted collector in the [Sumo](http://www.sumologic.com) service. Note the plugin with default configuration requires Kubernetes >=1.8. See [the section below on running this on Kubernetes <1.8](#running-on-kubernetes-versions-<1.8) 14 | 15 | - [Step 1 Create hosted collector and HTTP source in Sumo](#step-1--create-hosted-collector-and-http-source-in-sumo) 16 | - [Step 2 Create a Kubernetes secret](#step-2--create-a-kubernetes-secret) 17 | - [Step 3 Install the Sumo Kubernetes FluentD plugin](#step-3--install-the-sumo-kubernetes-fluentd-plugin) 18 | * [Option A Install plugin using kubectl](#option-a--install-plugin-using-kubectl) 19 | * [Option B Helm chart](#option-b--helm-chart) 20 | - [Environment variables](#environment-variables) 21 | + [Override environment variables using annotations](#override-environment-variables-using-annotations) 22 | + [Exclude data using annotations](#exclude-data-using-annotations) 23 | + [Include excluded using annotations](#include-excluded-using-annotations) 24 | - [Step 4 Set up Heapster for metric collection](#step-4-set-up-heapster-for-metric-collection) 25 | * [Kubernetes ConfigMap](#kubernetes-configmap) 26 | * [Kubernetes Service](#kubernetes-service) 27 | * [Kubernetes Deployment](#kubernetes-deployment) 28 | - [Log data](#log-data) 29 | * [Docker](#docker) 30 | * [Kubelet](#kubelet) 31 | * [Containers](#containers) 32 | - [Taints and Tolerations](#taints-and-tolerations) 33 | - [Running On OpenShift](#running-on-openshift) 34 | 35 | 36 | 37 | ![deployment](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/blob/master/screenshots/kubernetes.png) 38 | 39 | # Step 1 Create hosted collector and HTTP source in Sumo 40 | 41 | In this step you create, on the Sumo service, an HTTP endpoint to receive your logs. This process involves creating an HTTP source on a hosted collector in Sumo. In Sumo, collectors use sources to receive data. 42 | 43 | 1. If you don’t already have a Sumo account, you can create one by clicking the **Free Trial** button on https://www.sumologic.com/. 44 | 2. Create a hosted collector, following the instructions on [Configure a Hosted Collector](https://help.sumologic.com/Send-Data/Hosted-Collectors/Configure-a-Hosted-Collector) in Sumo help. (If you already have a Sumo hosted collector that you want to use, skip this step.) 45 | 3. Create an HTTP source on the collector you created in the previous step. For instructions, see [HTTP Logs and Metrics Source](https://help.sumologic.com/Send-Data/Sources/02Sources-for-Hosted-Collectors/HTTP-Source) in Sumo help. 46 | 4. When you have configured the HTTP source, Sumo will display the URL of the HTTP endpoint. Make a note of the URL. You will use it when you configure the Kubernetes service to send data to Sumo. 47 | 48 | # Step 2 Create a Kubernetes secret 49 | 50 | Create a secret in Kubernetes with the HTTP source URL. If you want to change the secret name, you must modify the Kubernetes manifest accordingly. 51 | 52 | `kubectl create secret generic sumologic --from-literal=collector-url=INSERT_HTTP_URL` 53 | 54 | You should see the confirmation message 55 | 56 | `secret "sumologic" created.` 57 | 58 | # Step 3 Install the Sumo Kubernetes FluentD plugin 59 | 60 | Follow the instructions in Option A below to install the plugin using `kubectl`. If you prefer to use a Helm chart, see Option B. 61 | 62 | Before you start, see [Environment variables](#environment-variables) for information about settings you can customize, and how to use annotations to override selected environment variables and exclude data from being sent to Sumo. 63 | 64 | ## Option A Install plugin using kubectl 65 | 66 | See the sample Kubernetes DaemonSet and Role in [fluentd.yaml](/daemonset/rbac/fluentd.yaml). 67 | 68 | 1. Clone the [GitHub repo](https://github.com/SumoLogic/fluentd-kubernetes-sumologic). 69 | 70 | 2. In `fluentd-kubernetes-sumologic`, install the chart using `kubectl`. 71 | 72 | Which `.yaml` file you should use depends on whether or not you are running RBAC for authorization. RBAC is enabled by default as of Kubernetes 1.6. Note the plugin with default configuration requires Kubernetes >=1.8. See the section below on [running this on Kubernetes <1.8](#running-on-kubernetes-versions-<1.8) 73 | 74 | **Non-RBAC (Kubernetes 1.5 and below)** 75 | 76 | `kubectl create -f /daemonset/nonrbac/fluentd.yaml` 77 | 78 | **RBAC (Kubernetes 1.6 and above)**

`kubectl create -f /daemonset/rbac/fluentd.yaml` 79 | 80 | 81 | **Note** if you modified the command in Step 2 to use a different name, update the `.yaml` file to use the correct secret. 82 | 83 | Logs should begin flowing into Sumo within a few minutes of plugin installation. 84 | 85 | ## Option B Helm chart 86 | If you use Helm to manage your Kubernetes resources, there is a Helm chart for the plugin at https://github.com/kubernetes/charts/tree/master/stable/sumologic-fluentd. 87 | 88 | # Environment variables 89 | 90 | Environment | Variable Description 91 | ----------- | -------------------- 92 | `AUDIT_LOG_PATH`|Define the path to the [Kubernetes Audit Log](https://kubernetes.io/docs/tasks/debug-application-cluster/audit/)

Default: `/mnt/log/kube-apiserver-audit.log` 93 | `CONCAT_SEPARATOR` |The character to use to delimit lines within the final concatenated message. Most multi-line messages contain a newline at the end of each line.

Default: "" 94 | `EXCLUDE_CONTAINER_REGEX` |A regular expression for containers. Matching containers will be excluded from Sumo. The logs will still be sent to FluentD. 95 | `EXCLUDE_FACILITY_REGEX`|A regular expression for syslog [facilities](https://en.wikipedia.org/wiki/Syslog#Facility). Matching facilities will be excluded from Sumo. The logs will still be sent to FluentD. 96 | `EXCLUDE_HOST_REGEX`|A regular expression for hosts. Matching hosts will be excluded from Sumo. The logs will still be sent to FluentD. 97 | `EXCLUDE_NAMESPACE_REGEX`|A regular expression for `namespaces`. Matching `namespaces` will be excluded from Sumo. The logs will still be sent to FluentD. 98 | `EXCLUDE_PATH`|Files matching this pattern will be ignored by the `in_tail` plugin, and will not be sent to Kubernetes or Sumo. This can be a comma-separated list as well. See [in_tail](http://docs.fluentd.org/v0.12/articles/in_tail#excludepath) documentation for more information.

For example, defining `EXCLUDE_PATH` as shown below excludes all files matching `/var/log/containers/*.log`,

`...`

`env:`
  - `name: EXCLUDE_PATH`
  `value: "[\"/var/log/containers/*.log\"]"` 99 | `EXCLUDE_POD_REGEX`|A regular expression for pods. Matching pods will be excluded from Sumo. The logs will still be sent to FluentD. 100 | `EXCLUDE_PRIORITY_REGEX`|A regular expression for syslog [priorities](https://en.wikipedia.org/wiki/Syslog#Severity_level). Matching priorities will be excluded from Sumo. The logs will still be sent to FluentD. 101 | `EXCLUDE_UNIT_REGEX` |A regular expression for `systemd` units. Matching units will be excluded from Sumo. The logs will still be sent to FluentD. 102 | `FLUENTD_SOURCE`|Fluentd can use log tail, systemd query or forward as the source, Allowable values: `file`, `systemd`, `forward`.

Default: `file` 103 | `FLUENTD_USER_CONFIG_DIR`|A directory of user-defined fluentd configuration files, which must be in the `*.conf` directory in the container. 104 | `FLUSH_INTERVAL` |How frequently to push logs to Sumo.

Default: `5s` 105 | `KUBERNETES_META`|Include or exclude Kubernetes metadata such as `namespace` and `pod_name` if using JSON log format.

Default: `true` 106 | `KUBERNETES_META_REDUCE`| Reduces redundant Kubernetes metadata, see [_Reducing Kubernetes Metadata_](#reducing-kubernetes-metadata).

Default: `false` 107 | `LOG_FORMAT`|Format in which to post logs to Sumo. Allowable values:

`text`—Logs will appear in SumoLogic in text format.
`json`—Logs will appear in SumoLogic in json format.
`json_merge`—Same as json but if the container logs in json format to stdout it will merge in the container json log at the root level and remove the log field.

Default: `json` 108 | `MULTILINE_START_REGEXP`|The regular expression for the `concat` plugin to use when merging multi-line messages. Defaults to Julian dates, for example, Jul 29, 2017. 109 | `NUM_THREADS`|Set the number of HTTP threads to Sumo. It might be necessary to do so in heavy-logging clusters.

Default: `1` 110 | `READ_FROM_HEAD`|Start to read the logs from the head of file, not bottom. Only applies to containers log files. See in_tail doc for more information.

Default: `true` 111 | `SOURCE_CATEGORY` |Set the `_sourceCategory` metadata field in Sumo.

Default: `"%{namespace}/%{pod_name}"` 112 | `SOURCE_CATEGORY_PREFIX`|Prepends a string that identifies the cluster to the `_sourceCategory` metadata field in Sumo.

Default: `kubernetes/` 113 | `SOURCE_CATEGORY_REPLACE_DASH` |Used to replace a dash (-) character with another character.

Default: `/`

For example, a Pod called `travel-nginx-3629474229-dirmo` within namespace `app` will appear in Sumo with `_sourceCategory=app/travel/nginx`. 114 | `SOURCE_HOST`|Set the `_sourceHost` metadata field in Sumo.

Default: `""` 115 | `SOURCE_NAME`|Set the `_sourceName` metadata field in Sumo.

Default: `"%{namespace}.%{pod}.%{container}"` 116 | `TIME_KEY`|The field name for json formatted sources that should be used as the time. See [time_key](https://docs.fluentd.org/v0.12/articles/formatter_json#time_key-(string,-optional,-defaults-to-%E2%80%9Ctime%E2%80%9D)). Default: `time` 117 | `ADD_TIMESTAMP`|Option to control adding timestamp to logs. Default: `true` 118 | `TIMESTAMP_KEY`|Field name when add_timestamp is on. Default: `timestamp` 119 | `ADD_STREAM`|Option to control adding stream to logs. Default: `true` 120 | `ADD_TIME`|Option to control adding time to logs. Default: `true` 121 | `CONTAINER_LOGS_PATH`|Specify the path in_tail should watch for container logs. Default: `/mnt/log/containers/*.log` 122 | `PROXY_URI`|Add the uri of the proxy environment if present. 123 | `ENABLE_STAT_WATCHER`|Option to control the enabling of [stat_watcher](https://docs.fluentd.org/v1.0/articles/in_tail#enable_stat_watcher). Default: `true` 124 | `K8S_METADATA_FILTER_WATCH`|Option to control the enabling of [metadata filter plugin watch](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#configuration). Default: `true` 125 | `K8S_METADATA_FILTER_CA_FILE`|Option to control the enabling of [metadata filter plugin ca_file](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#configuration). 126 | `K8S_METADATA_FILTER_VERIFY_SSL`|Option to control the enabling of [metadata filter plugin verify_ssl](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#configuration). Default: `true` 127 | `K8S_METADATA_FILTER_CLIENT_CERT`|Option to control the enabling of [metadata filter plugin client_cert](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#configuration). 128 | `K8S_METADATA_FILTER_CLIENT_KEY`|Option to control the enabling of [metadata filter plugin client_key](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#configuration). 129 | `K8S_METADATA_FILTER_BEARER_TOKEN_FILE`|Option to control the enabling of [metadata filter plugin bearer_token_file](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#configuration). 130 | `K8S_METADATA_FILTER_BEARER_CACHE_SIZE`|Option to control the enabling of [metadata filter plugin cache_size](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#configuration). Default: `1000` 131 | `K8S_METADATA_FILTER_BEARER_CACHE_TTL`|Option to control the enabling of [metadata filter plugin cache_ttl](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#configuration). Default: `3600` 132 | `K8S_NODE_NAME`|If set, improves [caching of pod metadata](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#environment-variables-for-kubernetes) and reduces API calls. 133 | `VERIFY_SSL`|Verify ssl certificate of sumologic endpoint. Default: `true` 134 | `FORWARD_INPUT_BIND`|The bind address to listen to if using forward as `FLUENTD_SOURCE`. Default: `0.0.0.0` (all addresses) 135 | `FORWARD_INPUT_PORT`|The port to listen to if using forward as `FLUENTD_SOURCE`. Default: `24224` 136 | 137 | 138 | The following table show which environment variables affect which Fluentd sources. 139 | 140 | | Environment Variable | Containers | Docker | Kubernetes | Systemd | 141 | |----------------------|------------|--------|------------|---------| 142 | | `EXCLUDE_CONTAINER_REGEX` | ✔ | ✘ | ✘ | ✘ | 143 | | `EXCLUDE_FACILITY_REGEX` | ✘ | ✘ | ✘ | ✔ | 144 | | `EXCLUDE_HOST_REGEX `| ✔ | ✘ | ✘ | ✔ | 145 | | `EXCLUDE_NAMESPACE_REGEX` | ✔ | ✘ | ✔ | ✘ | 146 | | `EXCLUDE_PATH` | ✔ | ✔ | ✔ | ✘ | 147 | | `EXCLUDE_PRIORITY_REGEX` | ✘ | ✘ | ✘ | ✔ | 148 | | `EXCLUDE_POD_REGEX` | ✔ | ✘ | ✘ | ✘ | 149 | | `EXCLUDE_UNIT_REGEX` | ✘ | ✘ | ✘ | ✔ | 150 | | `TIME_KEY` | ✔ | ✘ | ✘ | ✘ | 151 | 152 | ### FluentD stops processing logs 153 | When dealing with large volumes of data (TB's from what we have seen), FluentD may stop processing logs, but continue to run. This issue seems to be caused by the [scalability of the inotify process](https://github.com/fluent/fluentd/issues/1630) that is packaged with the FluentD in_tail plugin. If you encounter this situation, setting the `ENABLE_STAT_WATCHER` to `false` should resolve this issue. 154 | 155 | ### Reducing Kubernetes metadata 156 | 157 | You can use the `KUBERNETES_META_REDUCE` environment variable (global) or the `sumologic.com/kubernetes_meta_reduce` annotation (per pod) to reduce the amount of Kubernetes metadata included with each log line under the `kubernetes` field. 158 | 159 | When set, FluentD will remove the following properties: 160 | 161 | * `pod_id` 162 | * `container_id` 163 | * `namespace_id` 164 | * `master_url` 165 | * `labels` 166 | * `annotations` 167 | 168 | Logs will still include: 169 | 170 | * `pod_name` 171 | * `container_name` 172 | * `namespace_name` 173 | * `host` 174 | 175 | These fields still allow you to uniquely identify a pod and look up additional details with the Kubernetes API. 176 | 177 | ```yaml 178 | apiVersion: v1 179 | kind: ReplicationController 180 | metadata: 181 | name: nginx 182 | spec: 183 | replicas: 1 184 | selector: 185 | app: mywebsite 186 | template: 187 | metadata: 188 | name: nginx 189 | labels: 190 | app: mywebsite 191 | annotations: 192 | sumologic.com/kubernetes_meta_reduce: "true" 193 | spec: 194 | containers: 195 | - name: nginx 196 | image: nginx 197 | ports: 198 | - containerPort: 80 199 | ``` 200 | 201 | 202 | ### Override environment variables using annotations 203 | You can override the `LOG_FORMAT`, `KUBERNETES_META_REDUCE`, `SOURCE_CATEGORY` and `SOURCE_NAME` environment variables, per pod, using [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/). For example: 204 | 205 | ``` 206 | apiVersion: v1 207 | kind: ReplicationController 208 | metadata: 209 | name: nginx 210 | spec: 211 | replicas: 1 212 | selector: 213 | app: mywebsite 214 | template: 215 | metadata: 216 | name: nginx 217 | labels: 218 | app: mywebsite 219 | annotations: 220 | sumologic.com/format: "text" 221 | sumologic.com/kubernetes_meta_reduce: "true" 222 | sumologic.com/sourceCategory: "mywebsite/nginx" 223 | sumologic.com/sourceName: "mywebsite_nginx" 224 | spec: 225 | containers: 226 | - name: nginx 227 | image: nginx 228 | ports: 229 | - containerPort: 80 230 | ``` 231 | 232 | ### Exclude data using annotations 233 | 234 | You can also use the `sumologic.com/exclude` annotation to exclude data from Sumo. This data is sent to FluentD, but not to Sumo. 235 | 236 | ``` 237 | apiVersion: v1 238 | kind: ReplicationController 239 | metadata: 240 | name: nginx 241 | spec: 242 | replicas: 1 243 | selector: 244 | app: mywebsite 245 | template: 246 | metadata: 247 | name: nginx 248 | labels: 249 | app: mywebsite 250 | annotations: 251 | sumologic.com/format: "text" 252 | sumologic.com/sourceCategory: "mywebsite/nginx" 253 | sumologic.com/sourceName: "mywebsite_nginx" 254 | sumologic.com/exclude: "true" 255 | spec: 256 | containers: 257 | - name: nginx 258 | image: nginx 259 | ports: 260 | - containerPort: 80 261 | ``` 262 | 263 | ### Include excluded using annotations 264 | 265 | If you excluded a whole namespace, but still need one or few pods to be still included for shipping to Sumologic, you can use the `sumologic.com/include` annotation to include data to Sumo. It takes precedence over the exclusion described above. 266 | 267 | ``` 268 | apiVersion: v1 269 | kind: ReplicationController 270 | metadata: 271 | name: nginx 272 | spec: 273 | replicas: 1 274 | selector: 275 | app: mywebsite 276 | template: 277 | metadata: 278 | name: nginx 279 | labels: 280 | app: mywebsite 281 | annotations: 282 | sumologic.com/format: "text" 283 | sumologic.com/sourceCategory: "mywebsite/nginx" 284 | sumologic.com/sourceName: "mywebsite_nginx" 285 | sumologic.com/include: "true" 286 | spec: 287 | containers: 288 | - name: nginx 289 | image: nginx 290 | ports: 291 | - containerPort: 80 292 | ``` 293 | 294 | # Step 4 Set up Heapster for metric collection 295 | 296 | The recommended way to collect metrics from Kubernetes clusters is to use Heapster and a Sumo collector with a Graphite source. 297 | 298 | Heapster aggregates metrics across a Kubenetes cluster. Heapster runs as a pod in the cluster, and discovers all nodes in the cluster and queries usage information from each node's `kubelet`—the on-machine Kubernetes agent. 299 | 300 | Heapster provides metrics at the cluster, node and pod level. 301 | 302 | 1. Install Heapster in your Kubernetes cluster and configure a Graphite Sink to send the data in Graphite format to Sumo. For instructions, see 303 | https://github.com/kubernetes/heapster/blob/master/docs/sink-configuration.md#graphitecarbon. Assuming you have used the below YAML files to configure your system, then the sink option in graphite would be `--sink=graphite:tcp://sumo-graphite.kube-system.svc:2003`. You may need to change this depending on the namespace you run the deployment in, the name of the service or the port number for your Graphite source. 304 | 305 | 2. Use the Sumo Docker container. For instructions, see https://hub.docker.com/r/sumologic/collector/. 306 | 307 | 3. The following sections contain an example configmap, which contains the `sources.json` configuration, an example service, and an example deployment. Create these manifests in Kubernetes using `kubectl`. 308 | 309 | 310 | ## Kubernetes ConfigMap 311 | ``` 312 | kind: ConfigMap 313 | apiVersion: v1 314 | metadata: 315 | name: "sumo-sources" 316 | data: 317 | sources.json: |- 318 | { 319 | "api.version": "v1", 320 | "sources": [ 321 | { 322 | "name": "SOURCE_NAME", 323 | "category": "SOURCE_CATEGORY", 324 | "automaticDateParsing": true, 325 | "contentType": "Graphite", 326 | "timeZone": "UTC", 327 | "encoding": "UTF-8", 328 | "protocol": "TCP", 329 | "port": 2003, 330 | "sourceType": "Graphite" 331 | } 332 | ] 333 | } 334 | 335 | ``` 336 | ## Kubernetes Service 337 | ``` 338 | apiVersion: v1 339 | kind: Service 340 | metadata: 341 | name: sumo-graphite 342 | spec: 343 | ports: 344 | - port: 2003 345 | selector: 346 | app: sumo-graphite 347 | ``` 348 | ## Kubernetes Deployment 349 | ``` 350 | apiVersion: extensions/v1beta1 351 | kind: Deployment 352 | metadata: 353 | labels: 354 | app: sumo-graphite 355 | name: sumo-graphite 356 | spec: 357 | replicas: 2 358 | template: 359 | metadata: 360 | labels: 361 | app: sumo-graphite 362 | spec: 363 | volumes: 364 | - name: sumo-sources 365 | configMap: 366 | name: sumo-sources 367 | items: 368 | - key: sources.json 369 | path: sources.json 370 | containers: 371 | - name: sumo-graphite 372 | image: sumologic/collector:latest 373 | ports: 374 | - containerPort: 2003 375 | volumeMounts: 376 | - mountPath: /sumo 377 | name: sumo-sources 378 | env: 379 | - name: SUMO_ACCESS_ID 380 | value: 381 | - name: SUMO_ACCESS_KEY 382 | value: 383 | - name: SUMO_SOURCES_JSON 384 | value: /sumo/sources.json 385 | 386 | ``` 387 | 388 | # Templating Kubernetes metadata 389 | The following Kubernetes metadata is available for string templating: 390 | 391 | | String template | Description | 392 | | --------------- | ------------------------------------------------------ | 393 | | `%{namespace}` | Namespace name | 394 | | `%{pod}` | Full pod name (e.g. `travel-products-4136654265-zpovl`) | 395 | | `%{pod_name}` | Friendly pod name (e.g. `travel-products`) | 396 | | `%{pod_id}` | The pod's uid (a UUID) | 397 | | `%{container}` | Container name | 398 | | `%{source_host}` | Host | 399 | | `%{label:foo}` | The value of label `foo` | 400 | 401 | ## Missing labels 402 | Unlike the other templates, labels are not guaranteed to exist, so missing labels interpolate as `"undefined"`. 403 | 404 | For example, if you have only the label `app: travel` but you define `SOURCE_NAME="%{label:app}@%{label:version}"`, the source name will appear as `travel@undefined`. 405 | 406 | # Log data 407 | After performing the configuration described above, your logs should start streaming to SumoLogic in `json` or text format with the appropriate metadata. If you are using `json` format you can auto extract fields, for example `_sourceCategory=some/app | json auto`. 408 | 409 | ## Docker 410 | ![Docker Logs](/screenshots/docker.png) 411 | 412 | ## Kubelet 413 | Note that Kubelet logs are only collected if you are using systemd. Kubernetes no longer outputs the kubelet logs to a file. 414 | ![Docker Logs](/screenshots/kubelet.png) 415 | 416 | ## Containers 417 | ![Docker Logs](/screenshots/container.png) 418 | 419 | # Taints and Tolerations 420 | By default, the fluentd pods will schedule on, and therefore collect logs from, any worker nodes that do not have a taint and any master node that does not have a taint beyond the default master taint. If you would like to schedule pods on all nodes, regardless of taints, uncomment the following line from fluentd.yaml before applying it. 421 | 422 | ``` 423 | tolerations: 424 | #- operator: "Exists" 425 | ``` 426 | 427 | # Running On OpenShift 428 | 429 | This daemonset setting mounts /var/log as service account FluentD so you need to run containers as privileged container. Here is command example: 430 | 431 | ``` 432 | oc adm policy add-scc-to-user privileged system:serviceaccount:logging:fluentd 433 | oc adm policy add-cluster-role-to-user cluster-reader system:serviceaccount:logging:fluentd 434 | oc label node —all logging-sumologic-fluentd=true 435 | oc patch ds fluentd-sumologic -p "spec: 436 | template: 437 | spec: 438 | containers: 439 | - image: sumologic/fluentd-kubernetes-sumologic:latest 440 | name: fluentd 441 | securityContext: 442 | privileged: true" 443 | oc delete pod -l name = fluentd-sumologic 444 | ``` 445 | 446 | ## Running on Kubernetes versions <1.8 447 | 448 | In order to run this plugin on Kubernetes <1.8 you will need to make some changes the yaml file prior to deploying it. 449 | 450 | Replace: 451 | 452 | ``` 453 | - name: pos-files 454 | hostPath: 455 | path: /var/run/fluentd-pos 456 | type: "" 457 | ``` 458 | With: 459 | 460 | ``` 461 | - name: pos-files 462 | emptyDir: {} 463 | ``` 464 | 465 | ## Output to S3 466 | 467 | If you need to also send data to S3 (i.e. as a secondary backup/audit trail) the image includes the `fluent-plugin-s3` plugin. In order to send the logs from FluentD to multiple outputs, you must use the `copy` plugin. This image comes with an [OOB configuration](conf.d/out.sumo.conf) to output the logs to Sumo Logic. In order to output to multiple destinations, you need to modify that existing configuration. 468 | 469 | **Example:** Send all logs to S3 and Sumo: 470 | 471 | ``` 472 | 473 | @type copy 474 | 475 | @type sumologic 476 | log_key log 477 | endpoint "#{ENV['COLLECTOR_URL']}" 478 | verify_ssl "#{ENV['VERIFY_SSL']}" 479 | log_format "#{ENV['LOG_FORMAT']}" 480 | flush_interval "#{ENV['FLUSH_INTERVAL']}" 481 | num_threads "#{ENV['NUM_THREADS']}" 482 | open_timeout 60 483 | add_timestamp "#{ENV['ADD_TIMESTAMP']}" 484 | proxy_uri "#{ENV['PROXY_URI']}" 485 | 486 | 487 | @type s3 488 | 489 | aws_key_id YOUR_AWS_KEY_ID 490 | aws_sec_key YOUR_AWS_SECRET_KEY 491 | s3_bucket YOUR_S3_BUCKET_NAME 492 | s3_region us-west-1 493 | path logs/ 494 | buffer_path /var/log/fluent/s3 495 | 496 | time_slice_format %Y%m%d%H 497 | time_slice_wait 10m 498 | utc 499 | 500 | buffer_chunk_limit 256m 501 | 502 | 503 | ``` 504 | 505 | You can replace the OOB configuration by creating a new Docker image from our image or by using a configmap to inject the new configuration to the pod. 506 | 507 | More details about the S3 plugin can be found [in the docs](https://docs.fluentd.org/v0.12/articles/out_s3). 508 | 509 | ## Upgrading to v2.0.0 510 | 511 | In version 2.0.0, some legacy FluentD configuration has been removed that could lead to [duplicate logs being ingested into Sumo Logic](https://github.com/SumoLogic/fluentd-kubernetes-sumologic/issues/79). These logs were control plane components. This version was done as a major release as it breaks the current version of the [Kubernetes App](https://help.sumologic.com/Send-Data/Applications-and-Other-Data-Sources/Kubernetes/Install_the_Kubernetes_App_and_View_the_Dashboards) you may have installed in Sumo Logic. 512 | 513 | After upgrading to this version, you will need to reinstall the [Kubernetes App](https://help.sumologic.com/Send-Data/Applications-and-Other-Data-Sources/Kubernetes/Install_the_Kubernetes_App_and_View_the_Dashboards) in Sumo Logic. If you do not some of the panels in the dashboards will not render properly. 514 | 515 | If you have other content outside the app (Partitions, Scheduled Views, Field Extraction Rules or Scheduled Searches and Alerts), these may need to be updated after upgrading to v2.0.0. The logs, while the same content, have a different format and the same parsing logic and metadata may not apply. 516 | 517 | The previous log format that is removed in v2.0.0: 518 | ```json 519 | { 520 | "timestamp": 1538776281387, 521 | "severity": "I", 522 | "pid": "1", 523 | "source": "wrap.go:42", 524 | "message": "GET /api/v1/namespaces/kube-system/endpoints/kube-scheduler: (3.514372ms) 200 [[kube-scheduler/v1.10.5 (linux/amd64) kubernetes/32ac1c9/leader-election] 127.0.0.1:46290]" 525 | } 526 | ``` 527 | Is replaced by the following version. It is the same log line in a different format enriched with the same metadata the plugin applies to all pod logs. 528 | ```json 529 | { 530 | "timestamp": 1538776282152, 531 | "log": "I1005 21:51:21.387204 1 wrap.go:42] GET /api/v1/namespaces/kube-system/endpoints/kube-scheduler: (3.514372ms) 200 [[kube-scheduler/v1.10.5 (linux/amd64) kubernetes/32ac1c9/leader-election] 127.0.0.1:46290]", 532 | "stream": "stdout", 533 | "time": "2018-10-05T21:51:21.387477546Z", 534 | "docker": { 535 | "container_id": "a442fd2982dfdc09ab6235941f8d661a0a5c8df5e1d21f23ff48a9923ac14739" 536 | }, 537 | "kubernetes": { 538 | "container_name": "kube-apiserver", 539 | "namespace_name": "kube-system", 540 | "pod_name": "kube-apiserver-ip-172-20-122-71.us-west-2.compute.internal", 541 | "pod_id": "80fa5e13-c8b9-11e8-a456-0a8c1424d0d4", 542 | "labels": { 543 | "k8s-app": "kube-apiserver" 544 | }, 545 | "host": "ip-172-20-122-71.us-west-2.compute.internal", 546 | "master_url": "https://100.64.0.1:443/api", 547 | "namespace_id": "9b9b75b7-aa16-11e8-9d62-06df85b5d3bc" 548 | } 549 | } 550 | ``` 551 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/gem_tasks' 2 | require 'rake/testtask' 3 | 4 | Rake::TestTask.new(:test) do |test| 5 | test.libs << 'test' 6 | test.pattern = 'test/**/test_*.rb' 7 | test.verbose = true 8 | test.warning = false 9 | end 10 | 11 | task :default => :test 12 | -------------------------------------------------------------------------------- /ci/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "Starting build process in: `pwd`" 4 | set -e 5 | 6 | VERSION="${TRAVIS_TAG:-0.0.0}" 7 | VERSION="${VERSION#v}" 8 | : "${DOCKER_TAG:=sumologic/fluentd-kubernetes-sumologic}" 9 | : "${DOCKER_USERNAME:=sumodocker}" 10 | PLUGIN_NAME="fluent-plugin-kubernetes_sumologic" 11 | 12 | echo "Building for tag $VERSION, modify .gemspec file..." 13 | sed -i.bak "s/0.0.0/$VERSION/g" ./$PLUGIN_NAME.gemspec 14 | rm -f ./$PLUGIN_NAME.gemspec.bak 15 | 16 | echo "Install bundler..." 17 | bundle install 18 | 19 | echo "Run unit tests..." 20 | bundle exec rake 21 | 22 | echo "Build gem $PLUGIN_NAME $VERSION..." 23 | gem build $PLUGIN_NAME 24 | 25 | echo "Building docker image with $DOCKER_TAG:$VERSION in `pwd`..." 26 | docker build . -f ./Dockerfile -t $DOCKER_TAG:v$VERSION --no-cache 27 | if [ -z "$DOCKER_PASSWORD" ] || [ -z "$TRAVIS_TAG" ]; then 28 | echo "Skip Docker pushing" 29 | else 30 | echo "Pushing docker image with $DOCKER_TAG:$VERSION..." 31 | echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin 32 | docker push $DOCKER_TAG:v$VERSION 33 | fi 34 | 35 | rm -f ./*.gem 36 | 37 | echo "DONE" 38 | -------------------------------------------------------------------------------- /daemonset/nonrbac/fluentd.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | name: fluentd-sumologic 5 | labels: 6 | app: fluentd-sumologic 7 | version: v1 8 | spec: 9 | template: 10 | metadata: 11 | labels: 12 | name: fluentd-sumologic 13 | spec: 14 | volumes: 15 | - name: pos-files 16 | hostPath: 17 | path: /var/run/fluentd-pos 18 | type: "" 19 | # User data 1 mount point for Kubernetes running in OCI Container Engine for Kubernetes (OKE) 20 | - name: "u01" 21 | hostPath: 22 | path: /u01 23 | - name: host-logs 24 | hostPath: 25 | path: /var/log/ 26 | - name: docker-logs 27 | hostPath: 28 | path: /var/lib/docker 29 | - name: fluentd-sumologic-config 30 | configMap: 31 | name: fluentd-sumologic-config 32 | containers: 33 | - image: sumologic/fluentd-kubernetes-sumologic:v2.4.2 34 | name: fluentd 35 | imagePullPolicy: IfNotPresent 36 | volumeMounts: 37 | - name: fluentd-sumologic-config 38 | mountPath: /fluentd/etc 39 | - name: host-logs 40 | mountPath: /mnt/log/ 41 | readOnly: true 42 | - name: host-logs 43 | mountPath: /var/log/ 44 | readOnly: true 45 | - name: docker-logs 46 | mountPath: /var/lib/docker/ 47 | readOnly: true 48 | - name: "u01" 49 | mountPath: /u01 50 | readOnly: true 51 | - name: pos-files 52 | mountPath: /mnt/pos/ 53 | env: 54 | - name: COLLECTOR_URL 55 | valueFrom: 56 | secretKeyRef: 57 | name: sumologic 58 | key: collector-url 59 | # Improve GC for memory limited envs like docker. 60 | - name: RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR 61 | value: "0.9" 62 | tolerations: 63 | - operator: "Exists" 64 | - effect: "NoSchedule" 65 | key: "node-role.kubernetes.io/master" 66 | 67 | --- 68 | apiVersion: v1 69 | kind: ConfigMap 70 | metadata: 71 | name: fluentd-sumologic-config 72 | labels: 73 | app: fluentd-sumologic 74 | data: 75 | out.sumo.conf: |- 76 | 77 | @type sumologic 78 | log_key log 79 | endpoint "#{ENV['COLLECTOR_URL']}" 80 | verify_ssl "#{ENV['VERIFY_SSL']}" 81 | log_format "#{ENV['LOG_FORMAT']}" 82 | flush_interval "#{ENV['FLUSH_INTERVAL']}" 83 | num_threads "#{ENV['NUM_THREADS']}" 84 | open_timeout 60 85 | add_timestamp "#{ENV['ADD_TIMESTAMP']}" 86 | timestamp_key "#{ENV['TIMESTAMP_KEY']}" 87 | proxy_uri "#{ENV['PROXY_URI']}" 88 | 89 | 90 | fluent.file.conf: |- 91 | 92 | @type null 93 | 94 | 95 | 96 | @type monitor_agent 97 | bind 0.0.0.0 98 | port 24220 99 | 100 | 101 | @include /fluentd/etc/file.source.*.conf 102 | @include /fluentd/etc/user/*.conf 103 | @include /fluentd/etc/out.sumo.conf 104 | 105 | fluent.forward.conf: |- 106 | ## built-in TCP input 107 | ## $ echo | fluent-cat 108 | 109 | @type forward 110 | @id forward_input 111 | port "#{ENV['FORWARD_INPUT_PORT']}" 112 | bind "#{ENV['FORWARD_INPUT_BIND']}" 113 | 114 | 115 | 116 | @type monitor_agent 117 | bind 0.0.0.0 118 | port 24220 119 | 120 | 121 | @include /fluentd/etc/forward.source.*.conf 122 | @include /fluentd/etc/user/*.conf 123 | @include /fluentd/etc/out.sumo.conf 124 | 125 | fluent.systemd.conf: |- 126 | 127 | @type null 128 | 129 | 130 | 131 | @type monitor_agent 132 | bind 0.0.0.0 133 | port 24220 134 | 135 | 136 | @include /fluentd/etc/systemd.source.*.conf 137 | @include /fluentd/etc/user/*.conf 138 | @include /fluentd/etc/out.sumo.conf 139 | 140 | file.source.containers.conf: |- 141 | 142 | @type tail 143 | format json 144 | time_key "#{ENV['TIME_KEY']}" 145 | path "#{ENV['CONTAINER_LOGS_PATH']}" 146 | exclude_path "#{ENV['EXCLUDE_PATH']}" 147 | pos_file /mnt/pos/ggcp-containers.log.pos 148 | time_format %Y-%m-%dT%H:%M:%S.%NZ 149 | tag containers.* 150 | read_from_head "#{ENV['READ_FROM_HEAD']}" 151 | enable_stat_watcher "#{ENV['ENABLE_STAT_WATCHER']}" 152 | 153 | 154 | 155 | @type concat 156 | key log 157 | multiline_start_regexp "#{ENV['MULTILINE_START_REGEXP']}" 158 | separator "#{ENV['CONCAT_SEPARATOR']}" 159 | timeout_label @NORMAL 160 | 161 | 162 | 163 | @type relabel 164 | @label @NORMAL 165 | 166 | 167 | 205 | 206 | file.source.docker.conf: |- 207 | # Examples: 208 | # time="2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json" 209 | # time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404 210 | 211 | @type tail 212 | format /^time="(?