├── parsers-almost-json.conf ├── parser-pii.conf ├── fluentbit-pii.conf ├── minimal-config.conf ├── fluentbit-almost-json.conf ├── .github └── workflows │ └── repolinter.yml ├── dual_s3_output.conf ├── README.md └── LICENSE.md /parsers-almost-json.conf: -------------------------------------------------------------------------------- 1 | [PARSER] 2 | Name almost.json 3 | Format regex 4 | Regex ^[^{]*(?.*)$ -------------------------------------------------------------------------------- /parser-pii.conf: -------------------------------------------------------------------------------- 1 | [PARSER] 2 | # This example used this sample json log file: 3 | ## {"firstname":"John","email":"pii@sensitivedata.com","city":"Atlanta"} 4 | Name pii 5 | Format json 6 | -------------------------------------------------------------------------------- /fluentbit-pii.conf: -------------------------------------------------------------------------------- 1 | [INPUT] 2 | Name tail 3 | # tag used for fluentbit filtering down below. These tags do not get sent upstream to NR. 4 | Tag leon.email1 5 | Path /etc/newrelic-infra/logging.d/pii1.log 6 | # Path_Key adds the filepath and filename to the log record 7 | Path_Key filename 8 | # Name of the parser used 9 | Parser pii 10 | DB /var/db/newrelic-infra/newrelic-integrations/logging/input1.db 11 | # Read from the beginning of the file 12 | Read_from_Head True 13 | 14 | [INPUT] 15 | Name tail 16 | Tag remove.pii 17 | Path /etc/newrelic-infra/logging.d/pii.log 18 | Path_Key filename 19 | DB /var/db/newrelic-infra/newrelic-integrations/logging/piidb.db 20 | Read_from_Head True 21 | 22 | [FILTER] 23 | Name modify 24 | Match remove.pii 25 | Set email nopii@here.com 26 | -------------------------------------------------------------------------------- /minimal-config.conf: -------------------------------------------------------------------------------- 1 | [SERVICE] 2 | Flush 1 3 | Log_File /var/log/fluentbit.log 4 | Log_Level error 5 | Daemon off 6 | Parsers_File parsers.conf 7 | HTTP_Server On 8 | HTTP_Listen 0.0.0.0 9 | HTTP_Port 2020 10 | 11 | [INPUT] 12 | Name cpu 13 | Tag cpu.test 14 | Interval_Sec 10 15 | 16 | [INPUT] 17 | Name tail 18 | Tag test.file 19 | Path /var/log/apache2/error.log 20 | DB /var/log/apache2_error.db 21 | Path_Key filename 22 | Parser apache2 23 | Mem_Buf_Limit 8MB 24 | Skip_Long_Lines On 25 | Refresh_Interval 30 26 | 27 | [FILTER] 28 | Name record_modifier 29 | Match * 30 | Record logtype nginx 31 | Record hostname ${HOSTNAME} 32 | Record service_name Sample-App-Name 33 | 34 | [OUTPUT] 35 | Name newrelic 36 | Match * 37 | apiKey ${API_KEY} 38 | -------------------------------------------------------------------------------- /fluentbit-almost-json.conf: -------------------------------------------------------------------------------- 1 | [INPUT] 2 | # when tailing a file, the Name should always be 'tail'. It's not really a name, more of a input type. 3 | Name tail 4 | # Path of the file to be tailed. 5 | Path /etc/newrelic-infra/logging.d/json.log 6 | # Path_Key adds the filepath and filename to each log record so you can see the source. 7 | ## This comes in by default when (only) using a .yml file for logging. 8 | Path_Key filename 9 | # just increasing the buffer here, may not be needed for a lower number of files. 10 | Buffer_Max_Size 128k 11 | # The parser we're using is below, named almost.json 12 | Parser almost.json 13 | # The DB is where FB keeps track of what it has processed thus far. 14 | DB /var/db/newrelic-infra/newrelic-integrations/logging/fb.db 15 | 16 | # This [FILTER] clause is so we can add attributes to the log records in a attribute value format 17 | [FILTER] 18 | Name record_modifier 19 | Match *.* 20 | Record attribute value.here -------------------------------------------------------------------------------- /.github/workflows/repolinter.yml: -------------------------------------------------------------------------------- 1 | # NOTE: This file should always be named `repolinter.yml` to allow 2 | # workflow_dispatch to work properly 3 | name: Repolinter Action 4 | 5 | # NOTE: This workflow will ONLY check the default branch! 6 | # Currently there is no elegant way to specify the default 7 | # branch in the event filtering, so branches are instead 8 | # filtered in the "Test Default Branch" step. 9 | on: [push, workflow_dispatch] 10 | 11 | jobs: 12 | repolint: 13 | name: Run Repolinter 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Test Default Branch 17 | id: default-branch 18 | uses: actions/github-script@v2 19 | with: 20 | script: | 21 | const data = await github.repos.get(context.repo) 22 | return data.data && data.data.default_branch === context.ref.split('/').slice(-1)[0] 23 | - name: Checkout Self 24 | if: ${{ steps.default-branch.outputs.result == 'true' }} 25 | uses: actions/checkout@v2 26 | - name: Run Repolinter 27 | if: ${{ steps.default-branch.outputs.result == 'true' }} 28 | uses: newrelic/repolinter-action@v1 29 | with: 30 | config_url: https://raw.githubusercontent.com/newrelic/.github/main/repolinter-rulesets/example-code.yml 31 | output_type: issue 32 | -------------------------------------------------------------------------------- /dual_s3_output.conf: -------------------------------------------------------------------------------- 1 | # Fluent-Bit config to send logs to NR and AWS S3 Bucket 2 | # 3 | # Add the following to your systemctl service config for New Relic Infra Agent: 4 | # 5 | # [Service] 6 | # Environment="AWS_SHARED_CREDENTIALS_FILE=/root/.aws/credentials" 7 | # 8 | # In /etc/newrelic-infra/logging.d/dual-s3-logs.yml, add the following stanza to use 9 | # external Fluent-Bit configuration: 10 | # 11 | # logs: 12 | # - name: fluentbit_dual_s3_logs 13 | # fluentbit: 14 | # config_file: /etc/newrelic-infra/logging.d/dual-s3-output.conf 15 | # 16 | # Configuration below sends sample.log to both NR and AWS S3 Bucket. 17 | # If you want to send all logs, you can leave out the [INPUT] stanza and 18 | # use match * in the [OUTPUT] stanza 19 | # 20 | # Example of file written in bucket: 21 | # s3:///fluent-bit-logs/sample.log//////sample.log..log 22 | # 23 | # 24 | [INPUT] 25 | Name tail 26 | Path /var/log/sample.log 27 | Path_Key filePath 28 | tag sample.log 29 | Mem_Buf_Limit 8MB 30 | Skip_Long_Lines On 31 | Refresh_Interval 30 32 | 33 | [OUTPUT] 34 | name s3 35 | match sample.log 36 | bucket 37 | region 38 | s3_key_format /$TAG/${HOSTNAME}/%Y/%m/%d/%H/$TAG.$UUID.log 39 | s3_key_format_tag_delimiters .- 40 | total_file_size 50M 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Example Code header](https://github.com/newrelic/opensource-website/raw/master/src/images/categories/Example_Code.png)](https://opensource.newrelic.com/oss-category/#example-code) 2 | 3 | # Example Configurations for Fluent Bit 4 | 5 | ## Service 6 | 7 | There are some elements of Fluent Bit that are configured for the entire service; use this to set global configurations like the flush interval or troubleshooting mechanisms like the [HTTP server]((https://docs.fluentbit.io/manual/administration/monitoring#http-server)). You can find an example in our Kubernetes Fluent Bit daemonset configuration [found here](https://github.com/newrelic/kubernetes-logging/blob/master/fluent-conf.yml). 8 | 9 | ``` 10 | [SERVICE] 11 | Flush 1 12 | Log_File /var/log/fluentbit.log 13 | Log_Level error 14 | Daemon off 15 | Parsers_File parsers.conf 16 | HTTP_Server On 17 | HTTP_Listen 0.0.0.0 18 | HTTP_Port 2020 19 | 20 | @INCLUDE input-kubernetes.conf 21 | @INCLUDE output-newrelic.conf 22 | @INCLUDE filter-kubernetes.conf 23 | ``` 24 | 25 | You will notice in the example below that we are making use of the [@INCLUDE](https://fluentbit.io/documentation/0.13/configuration/file.html#config_include_file) configuration command. This allows you to break your configuration up into different modular files and include them as well. The Log_File and Log_Level are used to set how Fluent Bit creates diagnostic logs for itself; this does not have any impact on the logs you monitor. 26 | 27 | ## Routing 28 | 29 | Routing is a core feature that allows to route your data through Filters and finally to one or multiple destinations. Please take the time to read the [official documentation](https://docs.fluentbit.io/manual/concepts/data-pipeline/router) on the subject. You will learn how the Tag value you set on an input relates to what filtering and outputs will match the data. 30 | 31 | ## Input 32 | 33 | Input plugins are how logs are read or accepted into Fluent Bit. Common examples are syslog or tail. Syslog listens on a port for syslog messages, and tail follows a log file and forwards logs as they are added. A list of available input plugins can be [found here](https://docs.fluentbit.io/manual/input). 34 | 35 | ### File Input 36 | 37 | One of the most common types of log input is tailing a file. The in_tail input plugin allows you to read from a text log file as though you were running the _tail -f_ command. Full documentation on this plugin can be found [here](https://fluentbit.io/documentation/0.13/input/tail.html). 38 | 39 | ``` 40 | [INPUT] 41 | Name tail 42 | Tag test.file 43 | Path /var/log/apache2/error.log 44 | DB /var/log/apache2_error.db 45 | Path_Key filename 46 | Parser apache2 47 | Mem_Buf_Limit 8MB 48 | Skip_Long_Lines On 49 | Refresh_Interval 30 50 | ``` 51 | 52 | In this tail example we are tailing an Apache error log and parsing it. This is using the pre-defined parser for apache2; you can also define custom parsers. We recommend using the DB option to keep track of what you have monitored, and to set the Path_Key so that an attribute is populated in the output that will help you differentiate the file source of the logs you aggregate. 53 | 54 | ## Parser 55 | 56 | [Parsers](https://fluentbit.io/documentation/0.12/parser/) are how unstructured logs are organized or how JSON logs can be transformed. There are a number of existing parsers [already published](https://github.com/fluent/fluent-bit/blob/master/conf/parsers.conf) most of which are done using regex. There is also the option to [use Lua](https://fluentbit.io/documentation/0.14/filter/lua.html) for parsing and filtering, which is very flexible. 57 | 58 | The most time will be spent on custom parsing logic written for customer applications. Those are the most likely to have log output in bespoke formats for which no existing parser has been written. 59 | 60 | ## Filter 61 | 62 | Filter plugins transform the data generated by the input plugins. This transformation can be "parsing" of the data, modification of the data or filtering (excluding) data. 63 | 64 | A list of available filter plugins can be [found here](https://docs.fluentbit.io/manual/filter). 65 | 66 | ### Record Modifier 67 | 68 | We can use the [Record Modifier](https://fluentbit.io/documentation/0.12/filter/record_modifier.html) filter to add brand new attributes and values to the log entry. The example below matches to any input; all entries will have logtype, hostname and service_name added to them. Logtype is an important attribute to add for quick filtering, searching and triggering parsing rules. In the example below, adding nginx as the logtype will result in the built-in Nginx Access log parsing being applied. The hostname record is using an environment variable to get the hostname value. Service_name is a standard field in New Relic Logs that can be used to indicate what application is generating the log data. It is always optimal to match this value up with the same application name you are using in your New Relic APM configuration. 69 | 70 | ``` 71 | [FILTER] 72 | Name record_modifier 73 | Match * 74 | Record logtype nginx 75 | Record hostname ${HOSTNAME} 76 | Record service_name Sample-App-Name 77 | ``` 78 | 79 | ## Output 80 | 81 | Once you have input log data and filtered it, you will want to send it someplace. That is what an [output plugin](https://docs.fluentbit.io/manual/output) is for; hopefully you have already installed [New Relic's output plugin for Fluent Bit](https://docs.newrelic.com/docs/logs/new-relic-logs/enable-logs/enable-new-relic-logs-fluent-bit). The example below will match on everything; when testing be careful. Once you match on an entry it will not be in the pipeline anymore; if the newrelic output plugin is after your test output no logs will be sent to New Relic. 82 | 83 | It is recommended to use an API_KEY if rotating or changing the keys will ever be necessary; alternatively a license key can be used. maxBufferSize and maxRecords are optional and defined in the documentation. In the examples below there are references to environment variables; you can simply put the values right into the configuration as well. 84 | 85 | ``` 86 | [OUTPUT] 87 | Name newrelic 88 | Match * 89 | apiKey ${API_KEY} 90 | maxBufferSize ${BUFFER_SIZE} 91 | maxRecords ${MAX_RECORDS} 92 | ``` 93 | 94 | **A note about vulnerabilities** 95 | 96 | As noted in our [security policy](../../security/policy), New Relic is committed to the privacy and security of our customers and their data. We believe that providing coordinated disclosure by security researchers and engaging with the security community are important means to achieve our security goals. 97 | 98 | If you believe you have found a security vulnerability in this project or any of New Relic's products or websites, we welcome and greatly appreciate you reporting it to New Relic through [HackerOne](https://hackerone.com/newrelic). 99 | 100 | If you would like to contribute to this project, review [these guidelines](https://opensource.newrelic.com/code-of-conduct/). 101 | 102 | ## License 103 | fluentbit-examples is licensed under the [Apache 2.0](http://apache.org/licenses/LICENSE-2.0.txt) License. 104 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2020 New Relic 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | --------------------------------------------------------------------------------