├── Dockerfile ├── Dockerfile.centos ├── LICENSE ├── README.md ├── common-install.sh ├── fluentd-check.sh ├── fluentd-forwarder-build-config-template.yaml ├── fluentd-forwarder-centos-build-config-template.yaml ├── fluentd-forwarder-template.yaml ├── fluentd.conf.template ├── passwd.template └── run.sh /Dockerfile: -------------------------------------------------------------------------------- 1 | # start based on a centos image 2 | FROM rhel7 3 | 4 | ENV HOME=/opt/app-root/src \ 5 | PATH=/opt/rh/rh-ruby22/root/usr/bin:/opt/app-root/src/bin:/opt/app-root/bin${PATH:+:${PATH}} \ 6 | LD_LIBRARY_PATH=/opt/rh/rh-ruby22/root/usr/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} \ 7 | MANPATH=/opt/rh/rh-ruby22/root/usr/share/man:$MANPATH \ 8 | PKG_CONFIG_PATH=/opt/rh/rh-ruby22/root/usr/lib64/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} \ 9 | XDG_DATA_DIRS=/opt/rh/rh-ruby22/root/usr/share${XDG_DATA_DIRS:+:${XDG_DATA_DIRS}} \ 10 | RUBY_VERSION=2.2 \ 11 | FLUENTD_VERSION=0.12.32 \ 12 | GEM_HOME=/opt/app-root/src \ 13 | DATA_VERSION=1.6.0 \ 14 | TARGET_TYPE=remote_syslog \ 15 | TARGET_HOST=localhost \ 16 | TARGET_PORT=24284 \ 17 | IS_SECURE=yes \ 18 | STRICT_VERIFICATION=yes \ 19 | CA_PATH=/etc/pki/CA/certs/ca.crt \ 20 | CERT_PATH=/etc/pki/tls/certs/local.crt \ 21 | KEY_PATH=/etc/pki/tls/private/local.key \ 22 | KEY_PASSPHRASE= \ 23 | SHARED_KEY=ocpaggregatedloggingsharedkey 24 | 25 | LABEL io.k8s.description="Fluentd container for collecting logs from other fluentd instances" \ 26 | io.k8s.display-name="Fluentd Forwarder (${FLUENTD_VERSION})" \ 27 | io.openshift.expose-services="24284:tcp" \ 28 | io.openshift.tags="logging,fluentd,forwarder" \ 29 | name="fluentd-forwarder" \ 30 | architecture=x86_64 31 | 32 | # add files 33 | ADD run.sh fluentd.conf.template passwd.template fluentd-check.sh ${HOME}/ 34 | ADD common-*.sh /tmp/ 35 | 36 | # set permissions on files 37 | RUN chmod g+rx ${HOME}/fluentd-check.sh && \ 38 | chmod +x /tmp/common-*.sh 39 | 40 | # execute files and remove when done 41 | RUN /tmp/common-install.sh && \ 42 | rm -f /tmp/common-*.sh 43 | 44 | # set working dir 45 | WORKDIR ${HOME} 46 | 47 | # external port 48 | EXPOSE 24284 49 | 50 | CMD ["sh", "run.sh"] 51 | -------------------------------------------------------------------------------- /Dockerfile.centos: -------------------------------------------------------------------------------- 1 | # start based on a centos image 2 | FROM centos:7 3 | 4 | ENV HOME=/opt/app-root/src \ 5 | PATH=/opt/rh/rh-ruby22/root/usr/bin:/opt/app-root/src/bin:/opt/app-root/bin${PATH:+:${PATH}} \ 6 | LD_LIBRARY_PATH=/opt/rh/rh-ruby22/root/usr/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} \ 7 | MANPATH=/opt/rh/rh-ruby22/root/usr/share/man:$MANPATH \ 8 | PKG_CONFIG_PATH=/opt/rh/rh-ruby22/root/usr/lib64/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} \ 9 | XDG_DATA_DIRS=/opt/rh/rh-ruby22/root/usr/share${XDG_DATA_DIRS:+:${XDG_DATA_DIRS}} \ 10 | RUBY_VERSION=2.2 \ 11 | FLUENTD_VERSION=0.12.32 \ 12 | GEM_HOME=/opt/app-root/src \ 13 | DATA_VERSION=1.6.0 \ 14 | TARGET_TYPE=remote_syslog \ 15 | TARGET_HOST=localhost \ 16 | TARGET_PORT=24284 \ 17 | IS_SECURE=yes \ 18 | STRICT_VERIFICATION=yes \ 19 | CA_PATH=/etc/pki/CA/certs/ca.crt \ 20 | CERT_PATH=/etc/pki/tls/certs/local.crt \ 21 | KEY_PATH=/etc/pki/tls/private/local.key \ 22 | KEY_PASSPHRASE= \ 23 | SHARED_KEY=ocpaggregatedloggingsharedkey 24 | 25 | LABEL io.k8s.description="Fluentd container for collecting logs from other fluentd instances" \ 26 | io.k8s.display-name="Fluentd Forwarder (${FLUENTD_VERSION})" \ 27 | io.openshift.expose-services="24284:tcp" \ 28 | io.openshift.tags="logging,fluentd,forwarder" \ 29 | name="fluentd-forwarder" \ 30 | architecture=x86_64 31 | 32 | # add files 33 | ADD run.sh fluentd.conf.template passwd.template fluentd-check.sh ${HOME}/ 34 | ADD common-*.sh /tmp/ 35 | 36 | # set permissions on files 37 | RUN chmod g+rx ${HOME}/fluentd-check.sh && \ 38 | chmod +x /tmp/common-*.sh 39 | 40 | # execute files and remove when done 41 | RUN /tmp/common-install.sh && \ 42 | rm -f /tmp/common-*.sh 43 | 44 | # set working dir 45 | WORKDIR ${HOME} 46 | 47 | # external port 48 | EXPOSE 24284 49 | 50 | CMD ["sh", "run.sh"] 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Fluentd Forwarder Container 2 | 3 | ## Table of Contents 4 | 5 | * [Overview](#overview) 6 | * [Public Domain](#public-domain) 7 | * [License](#license) 8 | * [Bill of Materials](#bill-of-materials) 9 | * [Environment Specifications](#environment-specifications) 10 | * [Template Files](#template-files) 11 | * [Config Files](#config-files) 12 | * [Setup Instructions](#setup-instructions) 13 | * [Presenter Notes](#presenter-notes) 14 | * [Environment Setup](#environment-setup) 15 | * [Create Build Configuration](#create-build-configuration) 16 | * [Create Fluentd Forwarder](#create-fluentd-forwarder) 17 | * [RHEL](#rhel) 18 | * [RHEL Rsyslog](#rhel-rsyslog) 19 | * [RHEL splunkex](#rhel-splunkex) 20 | * [RHEL splunkhec](#rhel-splunkhec) 21 | * [CentOS](#centos) 22 | * [CentOS Rsyslog](#centos-rsyslog) 23 | * [CentOS splunkex](#centos-splunkex) 24 | * [CentOS splunkhec](#centos-splunkhec) 25 | * [Configure Fluentd Loggers](#configure-fluentd-loggers) 26 | * [Additional Configuration](#additional-configuration) 27 | * [Filtering](#filtering) 28 | * [Validating the Application](#validating-the-application) 29 | * [Resources](#resources) 30 | * [Privacy](#privacy) 31 | * [Contributing](#contributing) 32 | * [Records](#records) 33 | 34 | ## Overview 35 | OpenShift can be configured to host an EFK stack that stores and indexes log data but at some sites a log aggregation system is already in place. A forwarding fluentd can be configured to forward log data to a remote collection point. Using a containerized version that runs within OCP both simplifies some of the infrastructure and certificate management and allows rapid deployment with resiliancy. 36 | 37 | ## Public Domain 38 | This project constitutes a work of the United States Government and is not 39 | subject to domestic copyright protection under 17 USC § 105. This project is in 40 | the public domain within the United States, and copyright and related rights in 41 | the work worldwide are waived through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/). 42 | All contributions to this project will be released under the CC0 dedication. By 43 | submitting a pull request you are agreeing to comply with this waiver of 44 | copyright interest. 45 | 46 | ## License 47 | The project utilizes code licensed under the terms of the Apache Software 48 | License and therefore is licensed under ASL v2 or later. 49 | 50 | This program is free software: you can redistribute it and/or modify it under 51 | the terms of the Apache Software License version 2, or (at your option) any 52 | later version. 53 | 54 | This program is distributed in the hope that it will be useful, but WITHOUT ANY 55 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A 56 | PARTICULAR PURPOSE. See the Apache Software License for more details. 57 | 58 | You should have received a copy of the Apache Software License along with this 59 | program. If not, see http://www.apache.org/licenses/LICENSE-2.0.html 60 | 61 | ## Privacy 62 | This project contains only non-sensitive, publicly available data and 63 | information. All material and community participation is covered by the 64 | Surveillance Platform [Disclaimer](https://github.com/CDCgov/template/blob/master/DISCLAIMER.md) 65 | and [Code of Conduct](https://github.com/CDCgov/template/blob/master/code-of-conduct.md). 66 | For more information about CDC's privacy policy, please visit [http://www.cdc.gov/privacy.html](http://www.cdc.gov/privacy.html). 67 | 68 | ## Bill of Materials 69 | 70 | ### Environment Specifications 71 | 72 | This quickstart should be run on an installation of OpenShift Enterprise V3 with an existing EFK deployment. 73 | 74 | ### Template Files 75 | 76 | * Build Configurations 77 | * [RHEL](./fluentd-forwarder-build-config-template.yaml) 78 | * [CentOS](./fluentd-forwarder-centos-config-template.yaml) 79 | * [Application Deployment Template](./fluentd-forwarder-template.yaml) 80 | 81 | ### Config Files 82 | 83 | * [Fluentd Forwarding Configuration](./fluentd.conf.template) 84 | * Echoed in [ConfigMap section in Application Template](./fluentd-forwarder-template.yaml) 85 | 86 | ## Setup Instructions 87 | 88 | Have the `[fluentd-forwarder-build-config-template](./fluentd-forwarder-build-config-template.yaml)` and the `[fluentd-forwarder-template](./fluentd-forwarder-template.yaml)` available for adding to the cluster. These templates will be needed for creating builds and deploying the application. 89 | 90 | ### Environment Setup 91 | 92 | The EFK stack should already be configured in the "logging" namespace. 93 | 94 | ### Create Build Configuration 95 | 96 | Choose the RHEL (default) or CentOS (-centos) flavor of build configuration. Add the build configuration template to the logging namespace. 97 | ```bash 98 | oc project logging 99 | oc apply -f fluentd-forwarder-build-config-template.yaml 100 | ``` 101 | 102 | For CentOS use the -centos template. 103 | ```bash 104 | oc project logging 105 | oc apply -f fluentd-forwarder-centos-build-config-template.yaml 106 | ``` 107 | 108 | Process the template to create a build, using any relevant variables. In the general case the defaults are fine. 109 | ```bash 110 | oc project logging 111 | oc process fluentd-forwarder | oc apply -f - 112 | ``` 113 | 114 | For CentOS process the -centos template. 115 | ```bash 116 | oc project logging 117 | oc process fluentd-forwarder-centos | oc apply -f - 118 | ``` 119 | 120 | By default the build will disable all repositories in the base image, enabling only the ones required for installing the required packages. If you want to use the build process to use the existing repository config as is (e.g. if you're using a custom base image) then set the `USE_SYSTEM_REPOS` environment variable to any value in the BuildConfig object. 121 | ``` 122 | oc project logging 123 | oc set env bc/fluentd-forwarder USE_SYSTEM_REPOS=1 124 | ``` 125 | 126 | On CentOS: 127 | ``` 128 | oc project logging 129 | oc set env bc/fluentd-forwarder-centos USE_SYSTEM_REPOS=1 130 | ``` 131 | 132 | Build the fluentd-forwarder 133 | ```bash 134 | oc project logging 135 | oc start-build fluentd-forwarder-build 136 | ``` 137 | 138 | To build with CentOS use the -centos build configuration. 139 | ```bash 140 | oc project logging 141 | oc start-build fluentd-forwarder-centos-build 142 | ``` 143 | 144 | ### Create Fluentd Forwarder 145 | 146 | Add the template to the logging namespace: 147 | ```bash 148 | oc project logging 149 | oc apply -f fluentd-forwarder-template.yaml 150 | ``` 151 | 152 | #### RHEL 153 | 154 | ##### RHEL Rsyslog 155 | 156 | Create the new rsyslog logging forwarder application deployment: 157 | ```bash 158 | oc project logging 159 | oc new-app fluentd-forwarder \ 160 | -p "P_TARGET_TYPE=remote_syslog" \ 161 | -p "P_TARGET_HOST=rsyslog.internal.company.com" \ 162 | -p "P_TARGET_PORT=514" \ 163 | -p "P_SHARED_KEY=changeme" 164 | ``` 165 | 166 | ##### RHEL splunkex 167 | 168 | To create the new splunk-ex logging forwarder application deployment: 169 | ```bash 170 | oc project logging && \ 171 | oc process -f fluentd-forwarder-template.yaml \ 172 | -p "P_TARGET_TYPE=splunk_ex" \ 173 | -p "P_TARGET_HOST=10.10.10.10" \ 174 | -p "P_TARGET_PORT=9997" \ 175 | -p "P_SHARED_KEY=changeme" \ 176 | -p "P_ADDITIONAL_OPTS=output_format json" 177 | ``` 178 | 179 | ##### RHEL splunkhec 180 | 181 | To create the new splunkhec logging forwarder application deployment: 182 | ```bash 183 | oc project logging 184 | oc new-app fluentd-forwarder \ 185 | -p P_TARGET_TYPE="splunkhec" \ 186 | -p P_TARGET_HOST="examplehec.example.com" \ 187 | -p P_TARGET_PORT="8088" \ 188 | -p P_SHARED_KEY="changeme" \ 189 | -p P_ADDITIONAL_OPTS="token " 190 | ``` 191 | 192 | #### CentOS 193 | 194 | ##### CentOS Rsyslog 195 | 196 | To do the same for CentOS you need to reference the ImageStream created by that build. 197 | ```bash 198 | oc project logging 199 | oc new-app fluentd-forwarder \ 200 | -p "P_IMAGE_NAME=fluentd-forwarder-centos" \ 201 | -p "P_TARGET_TYPE=remote_syslog" \ 202 | -p "P_TARGET_HOST=rsyslog.internal.company.com" \ 203 | -p "P_TARGET_PORT=514" \ 204 | -p "P_SHARED_KEY=changeme" 205 | ``` 206 | 207 | ##### CentOS splunkex 208 | 209 | To create the new splunk-ex logging forwarder application deployment: 210 | ```bash 211 | oc project logging && \ 212 | oc process -f fluentd-forwarder-template.yaml \ 213 | -p "P_IMAGE_NAME=fluentd-forwarder-centos" \ 214 | -p "P_TARGET_TYPE=splunk_ex" \ 215 | -p "P_TARGET_HOST=10.10.10.10" \ 216 | -p "P_TARGET_PORT=9997" \ 217 | -p "P_SHARED_KEY=changeme" \ 218 | -p "P_ADDITIONAL_OPTS=output_format json" 219 | ``` 220 | 221 | ##### CentOS splunkhec 222 | 223 | To create the new splunkhec logging forwarder application deployment: 224 | ```bash 225 | oc project logging 226 | oc new-app fluentd-forwarder \ 227 | -p "P_IMAGE_NAME=fluentd-forwarder-centos" \ 228 | -p P_TARGET_TYPE="splunkhec" \ 229 | -p P_TARGET_HOST="examplehec.example.com" \ 230 | -p P_TARGET_PORT="8088" \ 231 | -p P_SHARED_KEY="changeme" \ 232 | -p P_ADDITIONAL_OPTS="token " 233 | ``` 234 | 235 | 236 | A full list of parameters can be found in the [template](./fluentd-forwarder-template.yaml). Additional non-parameterized parameters and environment variables can be found in the [Dockerfile](./Dockerfile). 237 | 238 | ### Configure Fluentd Loggers 239 | 240 | The "logging-fluentd" configmap's "data.secure-forward.conf" key needs to be edited as well. 241 | ```bash 242 | oc edit configmap -n logging logging-fluentd 243 | ``` 244 | 245 | Edit the following YAML: 246 | 247 | ```yaml 248 | data: 249 | secure-forward.conf: | 250 | 251 | @type secure_forward 252 | 253 | self_hostname ${HOSTNAME} 254 | shared_key changeme 255 | 256 | secure yes 257 | enable_strict_verification yes 258 | 259 | ca_cert_path /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt 260 | 261 | 262 | host fluentd-forwarder.logging.svc.cluster.local 263 | port 24284 264 | 265 | 266 | ``` 267 | 268 | This will cause each individual fluentd logger to begin forwarding to the service address `fluentd-forwarder.logging.svc.cluster.local` which was created with the new-app command. That service has it's own cluster-generated certificates and the "ca_cert_path" value here is used to trust the cluster's service signer CA. 269 | 270 | After saving the above changes the logging-fluentd pods need to be restarted. Delete them and they will be recreated. 271 | ```bash 272 | oc delete pod -l component=fluentd 273 | ``` 274 | 275 | ### Additional Configuration 276 | After creating the application you can edit the configuration for the logging forwarder in a more direct manner by manipulating the configuration map. 277 | ```bash 278 | oc edit configmap -n logging fluentd-forwarder 279 | ``` 280 | 281 | This will allow you to edit a copy of the configuration template and override the one provided in the Docker container without performing a rebuild. 282 | 283 | Any environment variables (like `SHARED_KEY` or `TARGET_TYPE`) will be substituted during the Pod startup just as with the built-in template using the `envsubst` command. Additional parameters can be added to the deployment config or directly edited here. 284 | 285 | ```yaml 286 | data: 287 | fluentd.conf: | 288 | 289 | @type secure_forward 290 | self_hostname "#{ENV['HOSTNAME']}" 291 | bind 0.0.0.0 292 | port 24284 293 | 294 | shared_key ${SHARED_KEY} 295 | 296 | secure ${IS_SECURE} 297 | enable_strict_verification ${STRICT_VERIFICATION} 298 | 299 | ca_cert_path ${CA_PATH} 300 | cert_path ${CERT_PATH} 301 | private_key_path ${KEY_PATH} 302 | 303 | private_key_passphrase ${KEY_PASSPHRASE} 304 | 305 | 306 | 307 | @type record_transformer 308 | 309 | forwarded_by "#{ENV['HOSTNAME']}" 310 | source_component "OCP" 311 | 312 | 313 | 314 | 315 | type ${TARGET_TYPE} 316 | host ${TARGET_HOST} 317 | port ${TARGET_PORT} 318 | output_format json 319 | 320 | ``` 321 | 322 | If you save changes to this configuration map you will need to delete the pods for the deployment so they can be recreated. 323 | 324 | ```bash 325 | oc delete pods -l name=fluentd-forwarder 326 | ``` 327 | 328 | #### Filtering 329 | In some use cases it might be necessary to perform filtering at the external fluentd process. This would be done to reduce the number or type of messages that are forwared. 330 | 331 | Using the fluentd.conf file from above a new record will be added to the json message. The record `kubernetes_namespace_name` will be set to the OpenShift namespace from where the messages originated. 332 | 333 | Using the appened records, a filter is applied to all messages. Messages where `kubernetes_namespace_name` match the specified regex pattern `null|devnull|logging|default|kube-public|kube-service-catalog|kube-system|logging|management-infra|openshift|openshift-ansible-service-broker|openshift-infra|openshift-metrics|openshift-node` are dropped and not forwared on. 334 | 335 | ```yaml 336 | data: 337 | fluentd.conf: | 338 | 339 | @type secure_forward 340 | self_hostname "#{ENV['HOSTNAME']}" 341 | bind 0.0.0.0 342 | port 24284 343 | 344 | shared_key ${SHARED_KEY} 345 | 346 | secure ${IS_SECURE} 347 | enable_strict_verification ${STRICT_VERIFICATION} 348 | 349 | ca_cert_path ${CA_PATH} 350 | cert_path ${CERT_PATH} 351 | private_key_path ${KEY_PATH} 352 | 353 | private_key_passphrase ${KEY_PASSPHRASE} 354 | 355 | 356 | 357 | @type record_transformer 358 | enable_ruby yes 359 | auto_typecast yes 360 | 361 | kubernetes_namespace_name ${record["kubernetes"].nil? ? 'devnull' : record["kubernetes"]["namespace_name"].nil? ? 'devnull' : record["kubernetes"]["namespace_name"]} 362 | forwarded_by "#{ENV['HOSTNAME']}" 363 | source_component "OCP" 364 | 365 | 366 | 367 | #Run filter on kube messages 368 | 369 | @type grep 370 | #Always filter out the restricted namespaces 371 | exclude1 kubernetes_namespace_name (null|devnull|logging|default|kube-public|kube-service-catalog|kube-system|logging|management-infra|openshift|openshift-ansible-service-broker|openshift-infra|openshift-metrics|openshift-node) 372 | 373 | 374 | 375 | 376 | @type ${TARGET_TYPE} 377 | #host ${TARGET_HOST} 378 | #port ${TARGET_PORT} 379 | ${ADDITIONAL_OPTS} 380 | 381 | ``` 382 | 383 | All system level messages would be dropped in the example above. To filter system messages filter on the `system.**` tag. 384 | 385 | ```yaml 386 | data: 387 | fluentd.conf: | 388 | 389 | @type secure_forward 390 | self_hostname "#{ENV['HOSTNAME']}" 391 | bind 0.0.0.0 392 | port 24284 393 | 394 | shared_key ${SHARED_KEY} 395 | 396 | secure ${IS_SECURE} 397 | enable_strict_verification ${STRICT_VERIFICATION} 398 | 399 | ca_cert_path ${CA_PATH} 400 | cert_path ${CERT_PATH} 401 | private_key_path ${KEY_PATH} 402 | 403 | private_key_passphrase ${KEY_PASSPHRASE} 404 | 405 | 406 | 407 | #Add system filtering logic here. 408 | 409 | 410 | 411 | @type ${TARGET_TYPE} 412 | #host ${TARGET_HOST} 413 | #port ${TARGET_PORT} 414 | ${ADDITIONAL_OPTS} 415 | 416 | 417 | #Toss the rest of the records. 418 | 419 | @type null 420 | 421 | ``` 422 | 423 | 424 | ### Validating the Application 425 | The best verification is that logs are showing up in the remote location. The application sets two tags "forwarded_by" which is set to the pod's hostname and "source_component" which is always set to "OCP". You can use those tags to search the logging collection facility for the logs being produced. 426 | 427 | If VERBOSE is set as an environment variable in the deployment config (`oc edit dc fluentd-forwarder`) then you can tail the logs of the fluentd-forwarder container and you should see a lot of information about reads. This is not the most reliable test but it will at least point in the right direction. 428 | 429 | ```bash 430 | oc logs fluentd-forwarder-1-a3zdf 431 | 2017-06-19 21:05:20 +0000 [debug]: plugin/input_session.rb:122:on_read: on_read 432 | 2017-06-19 21:05:23 +0000 [debug]: plugin/input_session.rb:122:on_read: on_read 433 | 2017-06-19 21:05:24 +0000 [debug]: plugin/input_session.rb:122:on_read: on_read 434 | 2017-06-19 21:05:25 +0000 [debug]: plugin/input_session.rb:122:on_read: on_read 435 | 2017-06-19 21:05:25 +0000 [debug]: plugin/input_session.rb:122:on_read: on_read 436 | 2017-06-19 21:05:25 +0000 [debug]: plugin/input_session.rb:122:on_read: on_read 437 | 2017-06-19 21:05:25 +0000 [debug]: plugin/input_session.rb:122:on_read: on_read 438 | 2017-06-19 21:05:26 +0000 [debug]: plugin/input_session.rb:122:on_read: on_read 439 | 2017-06-19 21:05:26 +0000 [debug]: plugin/input_session.rb:122:on_read: on_read 440 | ``` 441 | 442 | ## Resources 443 | * [Secure Forwarding with Splunk](http://v1.uncontained.io/playbooks/operationalizing/secure-forward-splunk.html) 444 | * [Origin Fluentd Image Source](https://github.com/openshift/origin-aggregated-logging/blob/master/fluentd/Dockerfile) 445 | * [Fluentd Filter Plugin Overview](http://docs.fluentd.org/v0.12/articles/filter-plugin-overview) 446 | 447 | ## Privacy 448 | This project contains only non-sensitive, publicly available data and 449 | information. All material and community participation is covered by the 450 | Surveillance Platform [Disclaimer](https://github.com/CDCgov/template/blob/master/DISCLAIMER.md) 451 | and [Code of Conduct](https://github.com/CDCgov/template/blob/master/code-of-conduct.md). 452 | For more information about CDC's privacy policy, please visit [http://www.cdc.gov/privacy.html](http://www.cdc.gov/privacy.html). 453 | 454 | ## Contributing 455 | Anyone is encouraged to contribute to the project by [forking](https://help.github.com/articles/fork-a-repo) 456 | and submitting a pull request. (If you are new to GitHub, you might start with a 457 | [basic tutorial](https://help.github.com/articles/set-up-git).) By contributing 458 | to this project, you grant a world-wide, royalty-free, perpetual, irrevocable, 459 | non-exclusive, transferable license to all users under the terms of the 460 | [Apache Software License v2](http://www.apache.org/licenses/LICENSE-2.0.html) or 461 | later. 462 | 463 | All comments, messages, pull requests, and other submissions received through 464 | CDC including this GitHub page are subject to the [Presidential Records Act](http://www.archives.gov/about/laws/presidential-records.html) 465 | and may be archived. Learn more at [http://www.cdc.gov/other/privacy.html](http://www.cdc.gov/other/privacy.html). 466 | 467 | ## Records 468 | This project is not a source of government records, but is a copy to increase 469 | collaboration and collaborative potential. All government records will be 470 | published through the [CDC web site](http://www.cdc.gov). 471 | -------------------------------------------------------------------------------- /common-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # get release version 4 | RELEASE=$(cat /etc/redhat-release) 5 | YUM_ARGS="--setopt=tsflags=nodocs" 6 | 7 | 8 | # shared packages 9 | # - build tools for building gems +# add files 10 | # - iproute needed for ip command to get ip addresses +ADD run.sh fluentd.conf.template passwd.template fluentd-check.sh ${HOME}/ 11 | # - nss_wrapper used to support username identity +ADD common-*.sh /tmp/ 12 | # - bc for calculations in run.conf 13 | PACKAGES="gem gcc-c++ libcurl-devel make bc gettext nss_wrapper hostname iproute" 14 | 15 | # ruby packages 16 | PACKAGES="${PACKAGES} rh-ruby22 rh-ruby22-rubygems rh-ruby22-ruby-devel" 17 | 18 | # if the release is a red hat version then we need to set additional arguments for yum repositories 19 | RED_HAT_MATCH='^Red Hat.*$' 20 | if [[ $RELEASE =~ $RED_HAT_MATCH && -z "$USE_SYSTEM_REPOS" ]]; then 21 | #NOTE: Until the first yum command is run, /etc/yum.repos.d/redhat.repo contains no repositories, so yum-config-manager will not enable/disable anything. 22 | #This command will force the population of said file, see #https://access.redhat.com/solutions/1443553 23 | yum repolist --disablerepo=* && yum-config-manager --disable \* > /dev/null 24 | #Set YUM_ARGS 25 | YUM_ARGS="${YUM_ARGS} --enablerepo=rhel-7-server-rpms --enablerepo=rhel-server-rhscl-7-rpms --enablerepo=rhel-7-server-optional-rpms" 26 | fi 27 | 28 | # enable epel when on CentOS 29 | CENTOS_MATCH='^CentOS.*' 30 | if [[ $RELEASE =~ $CENTOS_MATCH && -z "$USE_SYSTEM_REPOS" ]]; then 31 | rpmkeys --import file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 32 | yum install -y epel-release centos-release-scl-rh 33 | fi 34 | 35 | # ensure latest versions 36 | yum update $YUM_ARGS -y 37 | 38 | # install all required packages 39 | yum install -y $YUM_ARGS $PACKAGES 40 | 41 | # clean up yum to make sure image isn't larger because of installations/updates 42 | yum clean all 43 | rm -rf /var/cache/yum/* 44 | rm -rf /var/lib/yum/* 45 | 46 | # set home directory 47 | mkdir -p ${HOME} && \ 48 | 49 | # install gems for target version of fluentd, eventually 50 | # update to fluentd version that matches version deployed 51 | # into openshift 52 | gem install -N --conservative --minimal-deps --no-document \ 53 | fluentd:${FLUENTD_VERSION} \ 54 | 'activesupport:<5' \ 55 | 'public_suffix:<3.0.0' \ 56 | 'fluent-plugin-record-modifier:<1.0.0' \ 57 | 'fluent-plugin-rewrite-tag-filter:<2.0.0' \ 58 | fluent-plugin-kubernetes_metadata_filter \ 59 | fluent-plugin-rewrite-tag-filter \ 60 | fluent-plugin-secure-forward \ 61 | 'fluent-plugin-remote_syslog:<1.0.0' \ 62 | fluent-plugin-splunk-ex \ 63 | fluent-plugin-splunkhec 64 | 65 | # set up directores so that group 0 can have access like specified in 66 | # https://docs.openshift.com/container-platform/3.7/creating_images/guidelines.html 67 | # https://docs.openshift.com/container-platform/3.7/creating_images/guidelines.html#openshift-specific-guidelines 68 | mkdir -p /etc/fluent 69 | chgrp -R 0 /etc/fluent 70 | chmod -R g+rwX /etc/fluent 71 | chgrp -R 0 ${HOME} 72 | chmod -R g+rwX ${HOME} 73 | chgrp -R 0 /etc/pki 74 | chmod -R g+rwX /etc/pki 75 | mkdir /secrets 76 | chgrp -R 0 /secrets 77 | chmod -R g+rwX /secrets 78 | chgrp -R 0 /var/log 79 | chmod -R g+rwX /var/log 80 | -------------------------------------------------------------------------------- /fluentd-check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # test that something is listening on the fluentd secure_forward port 4 | RESULT=`ss -tnlp 2>/dev/null | grep 24284` 5 | 6 | # if no port is running, fail 7 | if [ "" == "${RESULT}" ]; then 8 | exit 1 9 | fi 10 | 11 | # otherwise explicit clean exit 12 | exit 0 -------------------------------------------------------------------------------- /fluentd-forwarder-build-config-template.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Template 4 | metadata: 5 | name: fluentd-forwarder-build 6 | annotations: 7 | description: Build Config template for collecting and forwarding fluentd logs to a remote log collection point like rsyslog or splunk. 8 | labels: 9 | name: fluentd-forwarder 10 | objects: 11 | - apiVersion: v1 12 | kind: ImageStream 13 | metadata: 14 | name: rhel7 15 | labels: 16 | app: fluentd-forwarder 17 | spec: 18 | tags: 19 | - name: latest 20 | annotations: 21 | openshift.io/imported-from: registry.access.redhat.com/rhel7/rhel 22 | from: 23 | kind: DockerImage 24 | name: registry.access.redhat.com/rhel7/rhel 25 | referencePolicy: 26 | type: Source 27 | - apiVersion: v1 28 | kind: ImageStream 29 | metadata: 30 | name: ${IMAGE_NAME} 31 | labels: 32 | app: fluentd-forwarder 33 | - apiVersion: v1 34 | kind: BuildConfig 35 | metadata: 36 | name: ${IMAGE_NAME} 37 | labels: 38 | app: ${IMAGE_NAME} 39 | spec: 40 | runPolicy: Serial 41 | source: 42 | type: Git 43 | git: 44 | uri: ${GIT_URI} 45 | ref: ${GIT_REF} 46 | strategy: 47 | type: Docker 48 | dockerStrategy: 49 | from: 50 | kind: ImageStreamTag 51 | name: 'rhel7:latest' 52 | output: 53 | to: 54 | kind: ImageStreamTag 55 | name: '${IMAGE_NAME}:latest' 56 | parameters: 57 | - name: GIT_URI 58 | description: The Git URI. 59 | value: "https://github.com/CDCgov/openshift-fluentd-forwarder.git" 60 | required: true 61 | - name: GIT_REF 62 | description: The git reference (tag, branch or other reference) to build from. 63 | value: master 64 | required: true 65 | - name: IMAGE_NAME 66 | description: The name of the image to be used when performing the pull operation. 67 | value: fluentd-forwarder 68 | required: true 69 | -------------------------------------------------------------------------------- /fluentd-forwarder-centos-build-config-template.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Template 4 | metadata: 5 | name: fluentd-forwarder-centos-build 6 | annotations: 7 | description: Build Config template for collecting and forwarding fluentd logs to a remote log collection point like rsyslog or splunk. 8 | labels: 9 | name: fluentd-forwarder 10 | objects: 11 | - apiVersion: v1 12 | kind: ImageStream 13 | metadata: 14 | name: centos 15 | labels: 16 | app: fluentd-forwarder 17 | spec: 18 | tags: 19 | - name: latest 20 | annotations: 21 | openshift.io/imported-from: docker.io/centos/7 22 | from: 23 | kind: DockerImage 24 | name: docker.io/centos/7 25 | referencePolicy: 26 | type: Source 27 | - apiVersion: v1 28 | kind: ImageStream 29 | metadata: 30 | name: ${IMAGE_NAME} 31 | labels: 32 | app: fluentd-forwarder 33 | - apiVersion: v1 34 | kind: BuildConfig 35 | metadata: 36 | name: ${IMAGE_NAME} 37 | labels: 38 | app: ${IMAGE_NAME} 39 | spec: 40 | runPolicy: Serial 41 | source: 42 | type: Git 43 | git: 44 | uri: ${GIT_URI} 45 | ref: ${GIT_REF} 46 | strategy: 47 | type: Docker 48 | dockerStrategy: 49 | from: 50 | kind: ImageStreamTag 51 | name: 'centos:latest' 52 | dockerfilePath: Dockerfile.centos 53 | output: 54 | to: 55 | kind: ImageStreamTag 56 | name: '${IMAGE_NAME}:latest' 57 | parameters: 58 | - name: GIT_URI 59 | description: The Git URI. 60 | value: "https://github.com/CDCgov/openshift-fluentd-forwarder.git" 61 | required: true 62 | - name: GIT_REF 63 | description: The git reference (tag, branch or other reference) to build from. 64 | value: master 65 | required: true 66 | - name: IMAGE_NAME 67 | description: The name of the image to be used when performing the pull operation. 68 | value: fluentd-forwarder-centos 69 | required: true 70 | -------------------------------------------------------------------------------- /fluentd-forwarder-template.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Template 4 | metadata: 5 | name: fluentd-forwarder 6 | annotations: 7 | description: Template for collecting and forwarding fluentd logs to a remote log collection point like rsyslog or splunk. 8 | labels: 9 | name: fluentd-forwarder 10 | objects: 11 | - apiVersion: v1 12 | kind: DeploymentConfig 13 | metadata: 14 | labels: 15 | name: fluentd-forwarder 16 | name: fluentd-forwarder 17 | spec: 18 | replicas: 1 19 | triggers: 20 | - type: ImageChange 21 | imageChangeParams: 22 | containerNames: 23 | - fluentd-forwarder 24 | from: 25 | kind: ImageStreamTag 26 | namespace: ${P_NAMESPACE} 27 | name: ${P_IMAGE_NAME}:${P_IMAGE_VERSION} 28 | - type: ConfigChange 29 | selector: 30 | name: fluentd-forwarder 31 | template: 32 | metadata: 33 | labels: 34 | name: fluentd-forwarder 35 | spec: 36 | containers: 37 | - name: fluentd-forwarder 38 | image: ${P_NAMESPACE}/${P_IMAGE_NAME}:${P_IMAGE_VERSION} 39 | env: 40 | - name: TARGET_TYPE 41 | value: ${P_TARGET_TYPE} 42 | - name: TARGET_HOST 43 | value: ${P_TARGET_HOST} 44 | - name: TARGET_PORT 45 | value: ${P_TARGET_PORT} 46 | - name: ADDITIONAL_OPTS 47 | value: ${P_ADDITIONAL_OPTS} 48 | - name: CA_PATH 49 | value: ${P_CA_PATH} 50 | - name: CERT_PATH 51 | value: ${P_CERT_PATH} 52 | - name: KEY_PATH 53 | value: ${P_KEY_PATH} 54 | - name: SHARED_KEY 55 | value: ${P_SHARED_KEY} 56 | volumeMounts: 57 | - mountPath: /secrets 58 | name: fluentd-forwarder-secret-mount 59 | readOnly: true 60 | - mountPath: /tmp/fluentd-config 61 | name: fluentd-forwarder-config 62 | readOnly: true 63 | name: fluentd-forwarder 64 | ports: 65 | - containerPort: 24284 66 | protocol: TCP 67 | readinessProbe: 68 | exec: 69 | command: 70 | - /opt/app-root/src/fluentd-check.sh 71 | failureThreshold: 3 72 | periodSeconds: 10 73 | successThreshold: 1 74 | timeoutSeconds: 5 75 | livenessProbe: 76 | exec: 77 | command: 78 | - /opt/app-root/src/fluentd-check.sh 79 | failureThreshold: 3 80 | initialDelaySeconds: 15 81 | periodSeconds: 30 82 | successThreshold: 1 83 | timeoutSeconds: 5 84 | volumes: 85 | - name: fluentd-forwarder-secret-mount 86 | secret: 87 | secretName: fluentd-forwarder-certs 88 | defaultMode: 420 89 | - name: fluentd-forwarder-config 90 | configMap: 91 | name: fluentd-forwarder 92 | defaultMode: 420 93 | - apiVersion: v1 94 | kind: Service 95 | metadata: 96 | annotations: 97 | service.alpha.openshift.io/serving-cert-secret-name: fluentd-forwarder-certs 98 | labels: 99 | name: fluentd-forwarder 100 | name: fluentd-forwarder 101 | spec: 102 | ports: 103 | - name: fluentd-forwarder 104 | port: 24284 105 | protocol: TCP 106 | targetPort: 24284 107 | selector: 108 | name: fluentd-forwarder 109 | type: ClusterIP 110 | - apiVersion: v1 111 | kind: ConfigMap 112 | metadata: 113 | labels: 114 | name: fluentd-forwarder 115 | name: fluentd-forwarder 116 | data: 117 | fluentd.conf: | 118 | 119 | @type secure_forward 120 | self_hostname "#{ENV['HOSTNAME']}" 121 | bind 0.0.0.0 122 | port 24284 123 | 124 | shared_key ${SHARED_KEY} 125 | 126 | secure ${IS_SECURE} 127 | enable_strict_verification ${STRICT_VERIFICATION} 128 | 129 | ca_cert_path ${CA_PATH} 130 | cert_path ${CERT_PATH} 131 | private_key_path ${KEY_PATH} 132 | 133 | private_key_passphrase ${KEY_PASSPHRASE} 134 | 135 | 136 | 137 | @type record_transformer 138 | 139 | forwarded_by "#{ENV['HOSTNAME']}" 140 | source_component "OCP" 141 | 142 | 143 | 144 | 145 | @type ${TARGET_TYPE} 146 | host ${TARGET_HOST} 147 | port ${TARGET_PORT} 148 | ${ADDITIONAL_OPTS} 149 | 150 | parameters: 151 | - name: P_NAMESPACE 152 | description: Target namespace for image. Used to reference the ImageStream source for the image. 153 | value: logging 154 | required: true 155 | - name: P_IMAGE_NAME 156 | description: The name of the image to be used when performing the pull operation. 157 | value: fluentd-forwarder 158 | required: true 159 | - name: P_IMAGE_VERSION 160 | description: The version of the fluentd-forwarder image to use. 161 | value: "latest" 162 | - name: P_TARGET_TYPE 163 | description: The output target's type. Can be 'remote_syslog' or 'splunk_ex'. 164 | required: true 165 | value: remote_syslog 166 | - name: P_TARGET_HOST 167 | description: The remote host that is the target for the logging data. 168 | value: remote-syslog-host.lan 169 | required: true 170 | - name: P_TARGET_PORT 171 | description: The remote port on the host that is the target for the logging data. The normal value for syslog is 514 and the normal value for splunk is 9997. 172 | required: true 173 | value: "514" 174 | - name: P_ADDITIONAL_OPTS 175 | description: Additional options passed to the forwarder plugin. The normal value for splunk is output_format json. 176 | value: "" 177 | - name: P_SHARED_KEY 178 | description: "A key shared between the logging providers and this forwarder to ensure secure operation. Default: 'ocpaggregatedloggingsharedkey'." 179 | value: ocpaggregatedloggingsharedkey 180 | required: true 181 | - name: P_CA_PATH 182 | description: Path to the CA certificate required for certificate verification. By default it uses the OCP master's signing certificate. 183 | value: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt 184 | - name: P_CERT_PATH 185 | description: Path to the certificate that should be used to identify the server. Defaults to the path outlined for secret mount. 186 | value: /secrets/tls.crt 187 | - name: P_KEY_PATH 188 | description: Path to the key that should be used for the server PKI. Defaults the the path outlined for secret mount. 189 | value: /secrets/tls.key 190 | -------------------------------------------------------------------------------- /fluentd.conf.template: -------------------------------------------------------------------------------- 1 | 2 | @type secure_forward 3 | self_hostname "#{ENV['HOSTNAME']}" 4 | bind 0.0.0.0 5 | port 24284 6 | 7 | shared_key ${SHARED_KEY} 8 | 9 | secure ${IS_SECURE} 10 | enable_strict_verification ${STRICT_VERIFICATION} 11 | 12 | ca_cert_path ${CA_PATH} 13 | cert_path ${CERT_PATH} 14 | private_key_path ${KEY_PATH} 15 | 16 | private_key_passphrase ${KEY_PASSPHRASE} 17 | 18 | 19 | 20 | @type record_transformer 21 | 22 | forwarded_by "#{ENV['HOSTNAME']}" 23 | source_component "OCP" 24 | 25 | 26 | 27 | 28 | @type ${TARGET_TYPE} 29 | host ${TARGET_HOST} 30 | port ${TARGET_PORT} 31 | ${ADDITIONAL_OPTS} 32 | 33 | -------------------------------------------------------------------------------- /passwd.template: -------------------------------------------------------------------------------- 1 | root:x:0:0:root:/root:/bin/bash 2 | bin:x:1:1:bin:/bin:/sbin/nologin 3 | daemon:x:2:2:daemon:/sbin:/sbin/nologin 4 | adm:x:3:4:adm:/var/adm:/sbin/nologin 5 | lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin 6 | sync:x:5:0:sync:/sbin:/bin/sync 7 | shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown 8 | halt:x:7:0:halt:/sbin:/sbin/halt 9 | mail:x:8:12:mail:/var/spool/mail:/sbin/nologin 10 | operator:x:11:0:operator:/root:/sbin/nologin 11 | games:x:12:100:games:/usr/games:/sbin/nologin 12 | ftp:x:14:50:FTP User:/var/ftp:/sbin/nologin 13 | nobody:x:99:99:Nobody:/:/sbin/nologin 14 | fluentd:x:${USER_ID}:${GROUP_ID}:fluentd daemon runner:${HOME}:/bin/bash 15 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # set up user id into passwd wrapper 4 | export USER_ID=$(id -u) 5 | export GROUP_ID=$(id -g) 6 | cat passwd.template | envsubst > /tmp/passwd 7 | export LD_PRELOAD=/usr/lib64/libnss_wrapper.so 8 | export NSS_WRAPPER_PASSWD=/tmp/passwd 9 | export NSS_WRAPPER_GROUP=/etc/group 10 | USER_NAME=$(id -un) 11 | 12 | # show that alternate user IDs are being honored 13 | echo "Running fluentd as user ${USER_NAME} (${USER_ID})" 14 | 15 | # copy openshift configmap templte if avaiable, otherwise use built-in template 16 | if [ -f /tmp/fluentd-config/fluentd.conf ]; then 17 | echo "Using OpenShift ConfigMap configuration" 18 | cat /tmp/fluentd-config/fluentd.conf | envsubst > /etc/fluent/fluentd.conf 19 | else 20 | echo "Using Docker image configuration" 21 | cat ~/fluentd.conf.template | envsubst > /etc/fluent/fluentd.conf 22 | fi 23 | 24 | ADDITIONAL_OPTS="" 25 | # set additional options if TARGET_TYPE is splunk_ex 26 | if [ "splunk_ex" == "${TARGET_TYPE}" ]; then 27 | ADDITIONAL_OPTS="output_format json" 28 | fi 29 | export ADDITIONAL_OPTS 30 | 31 | # set base args to point to fluentd.conf 32 | fluentdargs="-c /etc/fluent/fluentd.conf" 33 | 34 | # if verbose then set output to be verbose and print configuration before it is loaded 35 | if [[ $VERBOSE ]]; then 36 | echo "Using Raw Configuration: " 37 | cat /etc/fluent/fluentd.conf 38 | 39 | set -ex 40 | fluentdargs="-vv ${fluentdargs}" 41 | else 42 | set -e 43 | fi 44 | 45 | # set up IPADDR for fluentd - from OpenShift/Origin logging-fluentd 46 | IPADDR4=`/usr/sbin/ip -4 addr show dev eth0 | grep inet | sed -e "s/[ \t]*inet \([0-9.]*\).*/\1/"` 47 | IPADDR6=`/usr/sbin/ip -6 addr show dev eth0 | grep inet6 | sed "s/[ \t]*inet6 \([a-f0-9:]*\).*/\1/"` 48 | export IPADDR4 IPADDR6 49 | 50 | # set up resource limits for fluentd - from OpenShift/Origin logging-fluentd 51 | BUFFER_SIZE_LIMIT=${BUFFER_SIZE_LIMIT:-1048576} 52 | FLUENTD_CPU_LIMIT=${FLUENTD_CPU_LIMIT:-100m} 53 | FLUENTD_MEMORY_LIMIT=${FLUENTD_MEMORY_LIMIT:-512Mi} 54 | 55 | MEMORY_LIMIT=`echo $FLUENTD_MEMORY_LIMIT | sed -e "s/[Kk]/*1024/g;s/[Mm]/*1024*1024/g;s/[Gg]/*1024*1024*1024/g;s/i//g" | bc` 56 | BUFFER_SIZE_LIMIT=`echo $BUFFER_SIZE_LIMIT | sed -e "s/[Kk]/*1024/g;s/[Mm]/*1024*1024/g;s/[Gg]/*1024*1024*1024/g;s/i//g" | bc` 57 | if [ $BUFFER_SIZE_LIMIT -eq 0 ]; then 58 | BUFFER_SIZE_LIMIT=1048576 59 | fi 60 | 61 | BUFFER_QUEUE_LIMIT=`expr $MEMORY_LIMIT / $BUFFER_SIZE_LIMIT` 62 | if [ $BUFFER_QUEUE_LIMIT -eq 0 ]; then 63 | BUFFER_QUEUE_LIMIT=1024 64 | fi 65 | export BUFFER_QUEUE_LIMIT BUFFER_SIZE_LIMIT 66 | 67 | # launch fluentd - from OpenShift/Origin logging-fluentd 68 | if [[ $DEBUG ]] ; then 69 | exec fluentd $fluentdargs > /var/log/fluentd.log 2>&1 70 | else 71 | exec fluentd $fluentdargs 72 | fi --------------------------------------------------------------------------------