├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.yml │ ├── config.yml │ └── feature_request.yml └── workflows │ ├── add-to-project.yml │ ├── ci.yml │ └── pr.yml ├── .gitignore ├── .rubocop.yml ├── .ruby-version ├── .travis.yml ├── CODE_OF_CONDUCT.md ├── Gemfile ├── LICENCE ├── README.md ├── Rakefile ├── docs ├── matching.md └── upgrading.md ├── fluent-plugin-systemd.gemspec ├── lib └── fluent │ └── plugin │ ├── filter_systemd_entry.rb │ ├── in_systemd.rb │ └── systemd │ └── entry_mutator.rb └── test ├── docker ├── Dockerfile.ruby32 ├── Dockerfile.ruby34 ├── Dockerfile.tdagent-almalinux ├── Dockerfile.tdagent-ubuntu └── Dockerfile.ubuntu ├── fixture ├── corrupt │ └── test.badmsg.journal └── test.journal ├── helper.rb └── plugin ├── systemd └── test_entry_mutator.rb ├── test_filter_systemd_entry.rb └── test_in_systemd.rb /.dockerignore: -------------------------------------------------------------------------------- 1 | Gemfile.lock 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.yml: -------------------------------------------------------------------------------- 1 | name: Bug Report 2 | description: Create a report with a procedure for reproducing the bug 3 | labels: "waiting-for-triage" 4 | body: 5 | - type: textarea 6 | id: description 7 | attributes: 8 | label: Describe the bug 9 | description: A clear and concise description of what the bug is 10 | validations: 11 | required: true 12 | - type: textarea 13 | id: reproduce 14 | attributes: 15 | label: To Reproduce 16 | description: Steps to reproduce the behavior 17 | validations: 18 | required: true 19 | - type: textarea 20 | id: expected 21 | attributes: 22 | label: Expected behavior 23 | description: A clear and concise description of what you expected to happen 24 | validations: 25 | required: true 26 | - type: textarea 27 | id: environment 28 | attributes: 29 | label: Your Environment 30 | description: | 31 | - Fluentd or its package version: `fluentd --version` (Fluentd, fluent-package) or `td-agent --version` (td-agent) 32 | - Operating system: `cat /etc/os-release` 33 | - Kernel version: `uname -r` 34 | 35 | Tip: If you hit the problem with older fluent-plugin-systemd version, try latest version first. 36 | value: | 37 | - Fluentd version: 38 | - Package version: 39 | - Operating system: 40 | - Kernel version: 41 | render: markdown 42 | validations: 43 | required: true 44 | - type: textarea 45 | id: configuration 46 | attributes: 47 | label: Your Configuration 48 | description: | 49 | Write your configuration here. Minimum reproducible fluentd.conf is recommended. 50 | render: apache 51 | validations: 52 | required: true 53 | - type: textarea 54 | id: logs 55 | attributes: 56 | label: Your Error Log 57 | description: Write your ALL error log here 58 | render: shell 59 | validations: 60 | required: true 61 | - type: textarea 62 | id: addtional-context 63 | attributes: 64 | label: Additional context 65 | description: Add any other context about the problem here. 66 | validations: 67 | required: false 68 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: Ask a Question 4 | url: https://github.com/fluent/fluentd/discussions 5 | about: I have questions about Fluentd and plugins. Please ask and answer questions at https://github.com/fluent/fluentd/discussions 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.yml: -------------------------------------------------------------------------------- 1 | name: Feature request 2 | description: Suggest an idea for this project 3 | labels: ["waiting-for-triage", "enhancement"] 4 | body: 5 | - type: textarea 6 | id: description 7 | attributes: 8 | label: Is your feature request related to a problem? Please describe. 9 | description: | 10 | A clear and concise description of what the problem is. 11 | Ex. I'm always frustrated when [...] 12 | validations: 13 | required: true 14 | - type: textarea 15 | id: solution 16 | attributes: 17 | label: Describe the solution you'd like 18 | description: A clear and concise description of what you want to happen. 19 | validations: 20 | required: true 21 | - type: textarea 22 | id: alternative 23 | attributes: 24 | label: Describe alternatives you've considered 25 | description: A clear and concise description of any alternative solutions or features you've considered. 26 | validations: 27 | required: true 28 | - type: textarea 29 | id: addtional-context 30 | attributes: 31 | label: Additional context 32 | description: Add any other context or screenshots about the feature request here. 33 | validations: 34 | required: false 35 | 36 | -------------------------------------------------------------------------------- /.github/workflows/add-to-project.yml: -------------------------------------------------------------------------------- 1 | name: Add bugs to fluent project 2 | 3 | on: 4 | issues: 5 | types: 6 | - opened 7 | 8 | jobs: 9 | add-to-project: 10 | name: Add issue to project 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/add-to-project@v1.0.2 14 | with: 15 | project-url: https://github.com/orgs/fluent/projects/4 16 | github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} 17 | labeled: waiting-for-triage 18 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | docker: 10 | name: ${{ matrix.target.env }} 11 | strategy: 12 | matrix: 13 | target: 14 | - env: ubuntu 15 | dockerfile: Dockerfile.ubuntu 16 | - env: td-agent-deb 17 | dockerfile: Dockerfile.tdagent-ubuntu 18 | - env: td-agent-rpm 19 | dockerfile: Dockerfile.tdagent-almalinux 20 | - env: ruby32 21 | dockerfile: Dockerfile.ruby32 22 | - env: ruby34 23 | dockerfile: Dockerfile.ruby34 24 | runs-on: ubuntu-latest 25 | steps: 26 | - uses: actions/checkout@v3 27 | - name: Set up Docker Buildx 28 | uses: docker/setup-buildx-action@v3 29 | - name: Build and push 30 | id: docker_build 31 | uses: docker/build-push-action@v6 32 | with: 33 | file: test/docker/${{ matrix.target.dockerfile }} 34 | rubocop: 35 | runs-on: ubuntu-latest 36 | steps: 37 | - 38 | uses: actions/checkout@v4 39 | - 40 | uses: ruby/setup-ruby@v1 41 | with: 42 | bundler-cache: true 43 | - 44 | run: bundle exec rake rubocop 45 | -------------------------------------------------------------------------------- /.github/workflows/pr.yml: -------------------------------------------------------------------------------- 1 | name: Pull Request Checks 2 | 3 | on: 4 | pull_request: 5 | 6 | jobs: 7 | docker: 8 | name: ${{ matrix.target.env }} 9 | strategy: 10 | matrix: 11 | target: 12 | - env: ubuntu 13 | dockerfile: Dockerfile.ubuntu 14 | - env: td-agent-deb 15 | dockerfile: Dockerfile.tdagent-ubuntu 16 | - env: td-agent-rpm 17 | dockerfile: Dockerfile.tdagent-almalinux 18 | - env: ruby32 19 | dockerfile: Dockerfile.ruby32 20 | - env: ruby34 21 | dockerfile: Dockerfile.ruby34 22 | runs-on: ubuntu-latest 23 | steps: 24 | - 25 | uses: actions/checkout@v4 26 | - 27 | name: Set up Docker Buildx 28 | uses: docker/setup-buildx-action@v3 29 | - 30 | name: Build and push 31 | id: docker_build 32 | uses: docker/build-push-action@v6 33 | with: 34 | file: test/docker/${{ matrix.target.dockerfile }} 35 | rubocop: 36 | runs-on: ubuntu-latest 37 | steps: 38 | - 39 | uses: actions/checkout@v4 40 | - 41 | uses: ruby/setup-ruby@v1 42 | with: 43 | bundler-cache: true 44 | - 45 | run: bundle exec rake rubocop 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | Gemfile.lock 2 | pkg/* 3 | .DS_Store 4 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AllCops: 2 | Exclude: 3 | - '*.gemspec' 4 | - 'vendor/**/*' 5 | - 'vendor/**/.*' 6 | SuggestExtensions: false 7 | NewCops: enable 8 | 9 | Layout/LineLength: 10 | Max: 120 11 | Exclude: 12 | - 'test/**/*' 13 | Metrics/ClassLength: 14 | Exclude: 15 | - 'test/**/*' 16 | Metrics/AbcSize: 17 | Exclude: 18 | - 'test/**/*' 19 | Metrics/MethodLength: 20 | Exclude: 21 | - 'test/**/*' 22 | Naming/VariableNumber: 23 | Exclude: 24 | - 'test/**/*' 25 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 3.0.1 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | sudo: required 3 | services: 4 | - docker 5 | rvm: 6 | - 2.4 7 | 8 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the maintainer at edward-robinson@cookpad.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The maintainer is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [http://contributor-covenant.org/version/1/4][version] 72 | 73 | [homepage]: http://contributor-covenant.org 74 | [version]: http://contributor-covenant.org/version/1/4/ 75 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'https://rubygems.org' 4 | 5 | gemspec 6 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2015-2018 Edward Robinson 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # systemd plugin for [Fluentd](http://github.com/fluent/fluentd) 2 | 3 | [![Build Status](https://github.com/fluent-plugin-systemd/fluent-plugin-systemd/actions/workflows/ci.yml/badge.svg)](https://github.com/fluent-plugin-systemd/fluent-plugin-systemd/actions/workflows/ci.yml) [![Maintainability](https://api.codeclimate.com/v1/badges/5c773a4c1c6a6964fa4b/maintainability)](https://codeclimate.com/github/fluent-plugin-systemd/fluent-plugin-systemd/maintainability) [![Gem Version](https://badge.fury.io/rb/fluent-plugin-systemd.svg)](https://rubygems.org/gems/fluent-plugin-systemd) 4 | 5 | ## Overview 6 | 7 | * **systemd** input plugin to read logs from the systemd journal 8 | * **systemd** filter plugin for basic manipulation of systemd journal entries 9 | 10 | ## Support 11 | 12 | Join the #plugin-systemd channel on the [Fluentd Slack](http://slack.fluentd.org/) 13 | 14 | ## Requirements 15 | 16 | |fluent-plugin-systemd|fluentd|td-agent|ruby| 17 | |----|----|----|----| 18 | | > 0.1.0 | >= 0.14.11, < 2 | 3 | >= 2.1 | 19 | | 0.0.x | ~> 0.12.0 | 2 | >= 1.9 | 20 | 21 | |fluent-plugin-systemd|fluentd|fluent-package|ruby| 22 | |----|----|----|----| 23 | | >= 1.1.0 | >= 0.14.11, < 2 | 5 | >= 3.0 | 24 | 25 | * The 1.x.x series is developed from this branch (master) 26 | * The 0.0.x series (compatible with fluentd v0.12, and td-agent 2) is maintained on the [0.0.x branch](https://github.com/reevoo/fluent-plugin-systemd/tree/0.0.x) 27 | 28 | ## Installation 29 | 30 | Simply use RubyGems: 31 | 32 | gem install fluent-plugin-systemd -v 1.0.3 33 | 34 | or 35 | 36 | td-agent-gem install fluent-plugin-systemd -v 1.0.3 37 | 38 | ## Upgrading 39 | 40 | If you are upgrading to version 1.0 from a previous version of this plugin take a look at the [upgrade documentation](docs/upgrading.md). A number of deprecated config options were removed so you might need to update your configuration. 41 | 42 | ## Input Plugin Configuration 43 | 44 | 45 | @type systemd 46 | tag kubelet 47 | path /var/log/journal 48 | matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] 49 | read_from_head true 50 | 51 | 52 | @type local 53 | path /var/log/fluentd-journald-kubelet-cursor.json 54 | 55 | 56 | 57 | fields_strip_underscores true 58 | fields_lowercase true 59 | 60 | 61 | 62 | 63 | @type stdout 64 | 65 | 66 | 67 | root_dir /var/log/fluentd 68 | 69 | 70 | **`path`** 71 | 72 | Path to the systemd journal, defaults to `/var/log/journal` 73 | 74 | **`filters`** 75 | 76 | _This parameter name is depreciated and should be renamed to `matches`_ 77 | 78 | **`matches`** 79 | 80 | Expects an array of hashes defining desired matches to filter the log 81 | messages with. When this property is not specified, this plugin will default to 82 | reading all logs from the journal. 83 | 84 | See [matching details](docs/matching.md) for a more exhaustive 85 | description of this property and how to use it. 86 | 87 | **`storage`** 88 | 89 | Configuration for a [storage plugin](https://docs.fluentd.org/storage) used to store the journald cursor. 90 | 91 | **`read_from_head`** 92 | 93 | If true reads all available journal from head, otherwise starts reading from tail, 94 | ignored if cursor exists in storage (and is valid). Defaults to false. 95 | 96 | **`entry`** 97 | 98 | Optional configuration for an embedded systemd entry filter. See the [Filter Plugin Configuration](#filter-plugin-configuration) for config reference. 99 | 100 | **`tag`** 101 | 102 | _Required_ 103 | 104 | A tag that will be added to events generated by this input. 105 | 106 | 107 | ## Filter Plugin Configuration 108 | 109 | ``` 110 | 111 | @type systemd_entry 112 | field_map {"MESSAGE": "log", "_PID": ["process", "pid"], "_CMDLINE": "process", "_COMM": "cmd"} 113 | field_map_strict false 114 | fields_lowercase true 115 | fields_strip_underscores true 116 | 117 | ``` 118 | 119 | _Note that the following configurations can be embedded in a systemd source block, within an entry block, you only need to use a filter directly for more complicated workflows._ 120 | 121 | **`field_map`** 122 | 123 | Object / hash defining a mapping of source fields to destination fields. Destination fields may be existing or new user-defined fields. If multiple source fields are mapped to the same destination field, the contents of the fields will be appended to the destination field in the order defined in the mapping. A field map declaration takes the form of: 124 | 125 | { 126 | "": "", 127 | "": ["", ""], 128 | ... 129 | } 130 | Defaults to an empty map. 131 | 132 | **`field_map_strict`** 133 | 134 | If true, only destination fields from `field_map` are included in the result. Defaults to false. 135 | 136 | **`fields_lowercase`** 137 | 138 | If true, lowercase all non-mapped fields. Defaults to false. 139 | 140 | **`fields_strip_underscores`** 141 | 142 | If true, strip leading underscores from all non-mapped fields. Defaults to false. 143 | 144 | ### Filter Example 145 | 146 | Given a systemd journal source entry: 147 | ``` 148 | { 149 | "_MACHINE_ID": "bb9d0a52a41243829ecd729b40ac0bce" 150 | "_HOSTNAME": "arch" 151 | "MESSAGE": "this is a log message", 152 | "_PID": "123" 153 | "_CMDLINE": "login -- root" 154 | "_COMM": "login" 155 | } 156 | ``` 157 | The resulting entry using the above sample configuration: 158 | ``` 159 | { 160 | "machine_id": "bb9d0a52a41243829ecd729b40ac0bce" 161 | "hostname": "arch", 162 | "msg": "this is a log message", 163 | "pid": "123" 164 | "cmd": "login" 165 | "process": "123 login -- root" 166 | } 167 | ``` 168 | 169 | ## Common Issues 170 | 171 | > ### When I look at fluentd logs, everything looks fine but no journal logs are read ? 172 | 173 | This is commonly caused when the user running fluentd does not have the correct permissions 174 | to read the systemd journal. 175 | 176 | According to the [systemd documentation](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html): 177 | > Journal files are, by default, owned and readable by the "systemd-journal" system group but are not writable. Adding a user to this group thus enables her/him to read the journal files. 178 | 179 | > ### How can I deal with multi-line logs ? 180 | 181 | Ideally you want to ensure that your logs are saved to the systemd journal as a single entry regardless of how many lines they span. 182 | 183 | It is possible for applications to naively support this (but only if they have tight integration with systemd it seems) see: https://github.com/systemd/systemd/issues/5188. 184 | 185 | Typically you would not be able to this, so another way is to configure your logger to replace newline characters with something else. See this blog post for an example configuring a Java logging library to do this https://fabianlee.org/2018/03/09/java-collapsing-multiline-stack-traces-into-a-single-log-event-using-spring-backed-by-logback-or-log4j2/ 186 | 187 | Another strategy would be to use a plugin like [fluent-plugin-concat](https://github.com/fluent-plugins-nursery/fluent-plugin-concat) to combine multi line logs into a single event, this is more tricky though because you need to be able to identify the first and last lines of a multi line message with a regex. 188 | 189 | > ### How can I use this plugin inside of a docker container ? 190 | 191 | * Install the [systemd dependencies](#dependencies) if required 192 | * You can use an [offical fluentd docker](https://github.com/fluent/fluentd-docker-image) image as a base, (choose the debian based version, as alpine linux doesn't support systemd). 193 | * Bind mount `/var/log/journal` into your container. 194 | 195 | > ### I am seeing lots of logs being generated very rapidly! 196 | 197 | This commonly occurs when a loop is created when fluentd is logging to STDOUT, and the collected logs are then written to the systemd journal. This could happen if you run fluentd as a systemd serivce, or as a docker container with the systemd log driver. 198 | 199 | Workarounds include: 200 | 201 | * Use another fluentd output 202 | * Don't read every message from the journal, set some `matches` so you only read the messages you are interested in. 203 | * Disable the systemd log driver when you launch your fluentd docker container, e.g. by passing `--log-driver json-file` 204 | 205 | ### Example 206 | 207 | For an example of a full working setup including the plugin, take a look at [the fluentd kubernetes daemonset](https://github.com/fluent/fluentd-kubernetes-daemonset) 208 | 209 | ## Dependencies 210 | 211 | This plugin depends on libsystemd 212 | 213 | On Debian or Ubuntu you might need to install the libsystemd0 package: 214 | 215 | ``` 216 | apt-get install libsystemd0 217 | ``` 218 | 219 | On AlmaLinux or RHEL you might need to install the systemd package: 220 | 221 | ``` 222 | yum install -y systemd 223 | ``` 224 | 225 | If you want to do this in a AlmaLinux docker image you might first need to remove the `fakesystemd` package. 226 | 227 | ``` 228 | yum remove -y fakesystemd 229 | ``` 230 | 231 | ## Running the tests 232 | 233 | To run the tests with docker on several distros simply run `rake` 234 | 235 | For systems with systemd installed you can run the tests against your installed libsystemd with `rake test` 236 | 237 | ## License 238 | 239 | [Apache-2.0](LICENCE) 240 | 241 | ## Contributions 242 | 243 | Issues and pull requests are very welcome. 244 | 245 | If you want to make a contribution but need some help or advice feel free to message me @errm on the [Fluentd Slack](http://slack.fluentd.org/), or send me an email edward-robinson@cookpad.com 246 | 247 | We have adopted the [Contributor Covenant](CODE_OF_CONDUCT.md) and thus expect anyone interacting with contributors, maintainers and users of this project to abide by it. 248 | 249 | ## Maintainer 250 | 251 | * [Ed Robinson](https://github.com/errm) 252 | 253 | ## Contributors 254 | 255 | Many thanks to our fantastic contributors 256 | 257 | * [Erik Maciejewski](https://github.com/emacski) 258 | * [Ernie Hershey](https://github.com/ehershey) 259 | * [Hiroshi Hatake](https://github.com/cosmo0920) 260 | * [Jesus Rafael Carrillo](https://github.com/jescarri) 261 | * [Joel Gerber](https://github.com/Jitsusama) 262 | * [John Thomas Wile II](https://github.com/jtwile2) 263 | * [Kazuhiro Suzuki](https://github.com/ksauzz) 264 | * [Marius Grigoriu](https://github.com/mariusgrigoriu) 265 | * [Masahiro Nakagawa](https://github.com/repeatedly) 266 | * [Mike Kaplinskiy](https://github.com/mikekap) 267 | * [Richard Megginson](https://github.com/richm) 268 | * [Sadayuki Furuhashi](https://github.com/frsyuki) 269 | * [Seiichi Nishikata](https://github.com/neko-neko) 270 | * [Kentaro Hayashi](https://github.com/kenhys) 271 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'bundler/gem_tasks' 4 | require 'rake/testtask' 5 | require 'rubocop/rake_task' 6 | require 'fileutils' 7 | 8 | RuboCop::RakeTask.new(:rubocop) 9 | 10 | Rake::TestTask.new(:test) do |t| 11 | t.test_files = Dir['test/**/test_*.rb'] 12 | end 13 | 14 | task default: 'docker:test' 15 | task build: 'docker:test' 16 | task default: :rubocop 17 | 18 | namespace :docker do 19 | distros = %i[ubuntu tdagent-ubuntu tdagent-almalinux] 20 | task test: distros 21 | 22 | distros.each do |distro| 23 | task distro do 24 | puts "testing on #{distro}" 25 | sh "docker build . -f test/docker/Dockerfile.#{distro}" 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /docs/matching.md: -------------------------------------------------------------------------------- 1 | # Matching Details 2 | 3 | ## Overview 4 | 5 | This application takes an array of hashes passed to the `matches` parameter 6 | within a `systemd` typed source definition in your `fluent.conf` configuration 7 | file and then parses them into a format understood by `libsystemd`'s journal 8 | API. The basis behind what `libsystemd`'s API expects can be found documented in 9 | the `journalctl` [man 10 | page](https://www.freedesktop.org/software/systemd/man/journalctl.html). 11 | 12 | The result of this is that only logs which match the defined set of matching 13 | rules will be further processed. 14 | 15 | ## Usage Information 16 | 17 | In order to utilize this plugin's matching capabilities, you will need to 18 | understand how this plugin transforms the passed array of hashes into a format 19 | that is understood by `libsystemd`. 20 | 21 | The best way to describe this process is probably by example. The following 22 | sub-sections lists out various scenarios that you might wish to perform with 23 | this plugin's matching mechanism and describes both how to configure them, 24 | while also mapping them to examples from the `journalctl` [man 25 | page](https://www.freedesktop.org/software/systemd/man/journalctl.html). 26 | 27 | ### No Filters 28 | 29 | You can leave the `matches` property out altogether, or include a `matches` 30 | property with an empty array (as shown below) to specify that no matching 31 | should occur. 32 | 33 | matches [] 34 | 35 | Which coincides with this part of the `journalctl` man page: 36 | 37 | > Without arguments, all collected logs are shown unfiltered: 38 | > 39 | > `journalctl` 40 | 41 | ### Single Filter 42 | 43 | You can pass a single hash map to the `matches` array with a single key/value 44 | pair specified to only process log entries that match the given field/value 45 | combination. 46 | 47 | For example: 48 | 49 | matches [{"_SYSTEMD_UNIT": "avahi-daemon.service"}] 50 | 51 | Which coincides with this part of the the `journalctl` man page: 52 | 53 | > With one match specified, all entries with a field matching the expression are 54 | > shown: 55 | > 56 | > `journalctl _SYSTEMD_UNIT=avahi-daemon.service` 57 | 58 | ### Multi-Field Filters 59 | 60 | You can pass a single hash map to the `matches` array with multiple key/value 61 | pairs to only process log entries that match the combination of all of the 62 | specified key/value combinations. 63 | 64 | The passed key/value pairs are treated as a logical `AND`, such that all of the 65 | pairs must be true in order to allow further processing of the current log 66 | entry. 67 | 68 | For Example: 69 | 70 | matches [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}] 71 | 72 | Which coincides with this part of the the `journalctl` man page: 73 | 74 | > If two different fields are matched, only entries matching both expressions at 75 | > the same time are shown: 76 | > 77 | > `journalctl _SYSTEMD_UNIT=avahi-daemon.service _PID=28097` 78 | 79 | You can also perform a logical `OR` by splitting key/value pairs across multiple 80 | hashes passed to the `matches` array like so: 81 | 82 | matches [{"_SYSTEMD_UNIT": "avahi-daemon.service"}, {"_PID": 28097}] 83 | 84 | You can combine both `AND` and `OR` combinations together; using a single hash 85 | map to define conditions that `AND` together and using multiple hash maps to 86 | define conditions that `OR` together like so: 87 | 88 | matches [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}, {"_SYSTEMD_UNIT": "dbus.service"}] 89 | 90 | This can be expressed in psuedo-code like so: 91 | 92 | IF (_SYSTEMD_UNIT=avahi-daemon.service AND _PID=28097) OR _SYSTEMD_UNIT=dbus.service 93 | THEN PASS 94 | ELSE DENY 95 | 96 | Which coincides with this part of the `journalctl` man page: 97 | 98 | > If the separator "+" is used, two expressions may be combined in a logical OR. 99 | > The following will show all messages from the Avahi service process with the 100 | > PID 28097 plus all messages from the D-Bus service (from any of its 101 | > processes): 102 | > 103 | > `journalctl _SYSTEMD_UNIT=avahi-daemon.service _PID=28097 + _SYSTEMD_UNIT=dbus.service` 104 | 105 | ### Multi-Value Filters 106 | 107 | Fields with arrays as values are treated as a logical `OR` statement. 108 | 109 | For example: 110 | 111 | matches [{"_SYSTEMD_UNIT": ["avahi-daemon.service", "dbus.service"]}] 112 | 113 | Which coincides with this part of the `journalctl` man page: 114 | 115 | > If two matches refer to the same field, all entries matching either expression 116 | > are shown: 117 | > 118 | > `journalctl _SYSTEMD_UNIT=avahi-daemon.service _SYSTEMD_UNIT=dbus.service` 119 | 120 | The above example can be equivalently broken into 2 separate hashes. This is 121 | particularly helpful when you want to create aggregate logic 122 | 123 | For example: 124 | 125 | matches [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}, {"_SYSTEMD_UNIT": "dbus.service"}] 126 | 127 | This can be expressed in psuedo-code like so: 128 | 129 | IF (_SYSTEMD_UNIT=avahi-daemon.service AND _PID=28097) OR _SYSTEMD_UNIT=dbus.service 130 | THEN PASS 131 | ELSE DENY 132 | 133 | Which coincides with this part of the `journalctl` man page: 134 | 135 | > If the separator "+" is used, two expressions may be combined in a logical OR. 136 | > The following will show all messages from the Avahi service process with the 137 | > PID 28097 plus all messages from the D-Bus service (from any of its 138 | > processes): 139 | > 140 | > `journalctl _SYSTEMD_UNIT=avahi-daemon.service _PID=28097 + _SYSTEMD_UNIT=dbus.service` 141 | 142 | ### Wildcard Filters 143 | 144 | `systemd`/`journald` does not presently support wild-card filtering, so neither 145 | can this plugin. 146 | -------------------------------------------------------------------------------- /docs/upgrading.md: -------------------------------------------------------------------------------- 1 | # Upgrading 2 | 3 | ## To Version 1.0 4 | 5 | Version 1.0 removes a number of configuration options that had been deprecated by previous versions of the plugin. This was done to reduce the size of the code base and make maintenance simpler. 6 | 7 | If you have been paying attention to (and fixing) the deprecation warnings introduced by previous versions of the plugin then there is nothing for you to do. If you have not already done so it is recommended to first upgrade to version `0.3.1` and fix any warnings before trying version `1.0.0` or above. 8 | 9 | Version 1.0 of fluent-plugin-systemd only supports fluentd 0.14.11 and above (including fluentd 1.0+), if you are using tdagent you need to be using version 3 or above. 10 | 11 | ### `pos_file` 12 | 13 | Previous versions of the plugin used the `pos_file` config value to specify a file that the position or cursor from the systemd journal would be written to. This was replaced by a generic fluentd storage block that allows much more flexibility in how the cursor is persisted. Take a look at the [fluentd documentation](https://docs.fluentd.org/v1.0/articles/storage-section) to find out more about this. 14 | 15 | Before you upgrade to 1.0 you should migrate `pos_file` to a storage block. 16 | 17 | ``` 18 | pos_file /var/log/journald.pos 19 | ``` 20 | 21 | could be rewritten as 22 | 23 | ``` 24 | 25 | @type local 26 | persistent true 27 | path /var/log/journald_pos.json 28 | 29 | ``` 30 | 31 | If you want to update this configuration without skipping any entries if you supply the `pos_file` and a storage block at the same time version `0.3.1` will copy the cursor from the path given in `pos_file` to the given storage. 32 | 33 | ### `strip_underscores` 34 | 35 | The legacy `strip_underscores` method is removed in version `1.0.0` and above. The same functionality can be achieved by setting the `fields_strip_underscores` on an entry block. The entry block allows many more options for mutating journal entries. 36 | 37 | ``` 38 | strip_underscores true 39 | ``` 40 | 41 | should be rewritten as 42 | 43 | ``` 44 | 45 | fields_strip_underscores true 46 | 47 | ``` 48 | 49 | ### `filters` 50 | 51 | In version 1.0.0 the `filters` parameter was renamed as `matches` in order to more closely align the plugin with the names used in the systemd documentation. `filters` is deprecated and will be removed in a future version. Other than renaming the parameter no changes have been made to it's structure or operation. 52 | 53 | -------------------------------------------------------------------------------- /fluent-plugin-systemd.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | # frozen_string_literal: true 3 | 4 | lib = File.expand_path('../lib', __FILE__) 5 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 6 | 7 | Gem::Specification.new do |spec| 8 | spec.name = 'fluent-plugin-systemd' 9 | spec.version = '1.1.1' 10 | spec.authors = ['Ed Robinson'] 11 | spec.email = ['edward-robinson@cookpad.com'] 12 | 13 | spec.summary = 'Input plugin to read from systemd journal.' 14 | spec.description = 'This is a fluentd input plugin. It reads logs from the systemd journal.' 15 | spec.homepage = 'https://github.com/fluent-plugins-nursery/fluent-plugin-systemd' 16 | spec.license = 'Apache-2.0' 17 | 18 | spec.files = Dir['lib/**/**.rb', 'README.md', 'LICENCE'] 19 | spec.require_paths = ['lib'] 20 | 21 | spec.metadata['homepage_uri'] = spec.homepage 22 | spec.metadata['source_code_uri'] = 'https://github.com/fluent-plugins-nursery/fluent-plugin-systemd' 23 | spec.metadata['bug_tracker_uri'] = 'https://github.com/fluent-plugins-nursery/fluent-plugin-systemd/issues' 24 | 25 | spec.add_development_dependency 'bundler', '> 1.10' 26 | spec.add_development_dependency 'rake' 27 | spec.add_development_dependency 'test-unit', '~> 3.4' 28 | spec.add_development_dependency 'rubocop', '1.13.0' 29 | 30 | spec.add_runtime_dependency 'fluentd', ['>= 0.14.11', '< 2'] 31 | spec.add_runtime_dependency 'systemd-journal', '~> 2.1.0' 32 | end 33 | -------------------------------------------------------------------------------- /lib/fluent/plugin/filter_systemd_entry.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Copyright 2015-2018 Edward Robinson 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | require 'fluent/plugin/filter' 18 | require 'fluent/plugin/systemd/entry_mutator' 19 | 20 | module Fluent 21 | module Plugin 22 | # Fluentd systemd/journal filter plugin 23 | class SystemdEntryFilter < Filter 24 | Fluent::Plugin.register_filter('systemd_entry', self) 25 | 26 | config_param :field_map, :hash, default: {} 27 | config_param :field_map_strict, :bool, default: false 28 | config_param :fields_strip_underscores, :bool, default: false 29 | config_param :fields_lowercase, :bool, default: false 30 | 31 | def configure(conf) 32 | super 33 | @mutator = SystemdEntryMutator.new(**@config_root_section.to_h) 34 | @mutator.warnings.each { |warning| log.warn(warning) } 35 | end 36 | 37 | def filter(_tag, _time, entry) 38 | @mutator.run(entry) 39 | end 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/fluent/plugin/in_systemd.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Copyright 2015-2018 Edward Robinson 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | require 'systemd/journal' 18 | require 'fluent/plugin/input' 19 | require 'fluent/plugin/systemd/entry_mutator' 20 | 21 | module Fluent 22 | module Plugin 23 | # Fluentd plugin for reading from the systemd journal 24 | class SystemdInput < Input # rubocop:disable Metrics/ClassLength 25 | Fluent::Plugin.register_input('systemd', self) 26 | 27 | helpers :timer, :storage 28 | 29 | DEFAULT_STORAGE_TYPE = 'local' 30 | 31 | config_param :path, :string, default: '/var/log/journal' 32 | config_param :filters, :array, default: [], deprecated: 'filters has been renamed as matches' 33 | config_param :matches, :array, default: nil 34 | config_param :read_from_head, :bool, default: false 35 | config_param :tag, :string 36 | 37 | config_section :storage do 38 | config_set_default :usage, 'positions' 39 | config_set_default :@type, DEFAULT_STORAGE_TYPE 40 | config_set_default :persistent, false 41 | end 42 | 43 | config_section :entry, param_name: 'entry_opts', required: false, multi: false do 44 | config_param :field_map, :hash, default: {} 45 | config_param :field_map_strict, :bool, default: false 46 | config_param :fields_strip_underscores, :bool, default: false 47 | config_param :fields_lowercase, :bool, default: false 48 | end 49 | 50 | def configure(conf) 51 | super 52 | @journal = nil 53 | @pos_storage = storage_create(usage: 'positions') 54 | @mutator = SystemdEntryMutator.new(**@entry_opts.to_h) 55 | @mutator.warnings.each { |warning| log.warn(warning) } 56 | end 57 | 58 | def start 59 | super 60 | @running = true 61 | timer_execute(:in_systemd_emit_worker, 1, &method(:run)) 62 | end 63 | 64 | def shutdown 65 | @running = false 66 | @journal&.close 67 | @journal = nil 68 | @pos_storage = nil 69 | @mutator = nil 70 | super 71 | end 72 | 73 | private 74 | 75 | def init_journal 76 | # TODO: ruby 2.3 77 | @journal.close if @journal # rubocop:disable Style/SafeNavigation 78 | @journal = Systemd::Journal.new(path: @path, auto_reopen: true) 79 | @journal.filter(*(@matches || @filters)) 80 | seek 81 | true 82 | rescue Systemd::JournalError => e 83 | log.warn("#{e.class}: #{e.message} retrying in 1s") 84 | false 85 | end 86 | 87 | def seek 88 | cursor = @pos_storage.get(:journal) 89 | seek_to(cursor || read_from) 90 | rescue Systemd::JournalError 91 | log.warn( 92 | "Could not seek to cursor #{cursor} found in position file: #{@pos_storage.path}, " \ 93 | "falling back to reading from #{read_from}" 94 | ) 95 | seek_to(read_from) 96 | end 97 | 98 | # according to https://github.com/ledbettj/systemd-journal/issues/64#issuecomment-271056644 99 | # and https://bugs.freedesktop.org/show_bug.cgi?id=64614, after doing a seek(:tail), 100 | # you must move back in such a way that the next move_next will return the last 101 | # record 102 | def seek_to(pos) 103 | @journal.seek(pos) 104 | return if pos == :head 105 | 106 | if pos == :tail 107 | @journal.move(-2) 108 | else 109 | @journal.move(1) 110 | end 111 | end 112 | 113 | def read_from 114 | @read_from_head ? :head : :tail 115 | end 116 | 117 | def run 118 | return unless @journal || init_journal 119 | 120 | init_journal if @journal.wait(0) == :invalidate 121 | watch do |entry| 122 | emit(entry) 123 | end 124 | end 125 | 126 | def emit(entry) 127 | router.emit(@tag, Fluent::EventTime.from_time(entry.realtime_timestamp), formatted(entry)) 128 | rescue Fluent::Plugin::Buffer::BufferOverflowError => e 129 | retries ||= 0 130 | raise e if retries > 10 131 | 132 | retries += 1 133 | sleep 1.5**retries + rand(0..3) 134 | retry 135 | rescue => e # rubocop:disable Style/RescueStandardError 136 | log.error("Exception emitting record: #{e}") 137 | end 138 | 139 | def formatted(entry) 140 | @mutator.run(entry) 141 | end 142 | 143 | def watch(&block) 144 | yield_current_entry(&block) while @running && @journal.move_next 145 | rescue Systemd::JournalError => e 146 | log.warn("Error moving to next Journal entry: #{e.class}: #{e.message}") 147 | end 148 | 149 | def yield_current_entry 150 | yield @journal.current_entry 151 | @pos_storage.put(:journal, @journal.cursor) 152 | rescue Systemd::JournalError => e 153 | log.warn("Error reading from Journal: #{e.class}: #{e.message}") 154 | end 155 | end 156 | end 157 | end 158 | -------------------------------------------------------------------------------- /lib/fluent/plugin/systemd/entry_mutator.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Copyright 2015-2018 Edward Robinson 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | require 'fluent/config/error' 18 | 19 | module Fluent 20 | module Plugin 21 | # A simple stand-alone configurable mutator for systemd journal entries. 22 | # 23 | # Note regarding field mapping: 24 | # The input `field_map` option is meant to have a structure that is 25 | # intuative or logical for humans when declaring a field map. 26 | # { 27 | # "" => "", 28 | # "" => ["", ""] 29 | # } 30 | # Internally the inverse of the human-friendly field_map is 31 | # computed (and cached) upon object creation and used as a "mapped model" 32 | # { 33 | # "" => ["", ""], 34 | # "" => [""] 35 | # } 36 | class SystemdEntryMutator 37 | Options = Struct.new( 38 | :field_map, 39 | :field_map_strict, 40 | :fields_lowercase, 41 | :fields_strip_underscores 42 | ) 43 | 44 | def self.default_opts 45 | Options.new({}, false, false, false) 46 | end 47 | 48 | # Constructor keyword options (all other kwargs are ignored): 49 | # field_map - hash describing the desired field mapping in the form: 50 | # {"" => "", ...} 51 | # where `new_field` is a string or array of strings 52 | # field_map_strict - boolean if true will only include new fields 53 | # defined in `field_map` 54 | # fields_strip_underscores - boolean if true will strip all leading 55 | # underscores from non-mapped fields 56 | # fields_lowercase - boolean if true lowercase all non-mapped fields 57 | # 58 | # raises `Fluent::ConfigError` for invalid options 59 | def initialize(**options) 60 | @opts = options_from_hash(options) 61 | validate_options(@opts) 62 | @map = invert_field_map(@opts.field_map) 63 | @map_src_fields = @opts.field_map.keys 64 | @no_transform = @opts == self.class.default_opts 65 | end 66 | 67 | # Expose config state as read-only instance properties of the mutator. 68 | def method_missing(sym, *args) 69 | return @opts[sym] if @opts.members.include?(sym) 70 | 71 | super 72 | end 73 | 74 | def respond_to_missing?(sym, include_private = false) 75 | @opts.members.include?(sym) || super 76 | end 77 | 78 | # The main run method that performs all configured mutations, if any, 79 | # against a single journal entry. Returns the mutated entry hash. 80 | # entry - hash or `Systemd::Journal:Entry` 81 | def run(entry) 82 | return entry.to_h if @no_transform 83 | return map_fields(entry) if @opts.field_map_strict 84 | 85 | format_fields(entry, map_fields(entry)) 86 | end 87 | 88 | # Run field mapping against a single journal entry. Returns the mutated 89 | # entry hash. 90 | # entry - hash or `Systemd::Journal:Entry` 91 | def map_fields(entry) 92 | @map.each_with_object({}) do |(cstm, sysds), mapped| 93 | vals = sysds.collect { |fld| entry[fld] }.compact 94 | next if vals.empty? # systemd field does not exist in source entry 95 | 96 | mapped[cstm] = join_if_needed(vals) 97 | end 98 | end 99 | 100 | # Run field formatting (mutations applied to all non-mapped fields) 101 | # against a single journal entry. Returns the mutated entry hash. 102 | # entry - hash or `Systemd::Journal:Entry` 103 | # mapped - Optional hash that represents a previously mapped entry to 104 | # which the formatted fields will be added 105 | def format_fields(entry, mapped = nil) 106 | entry.each_with_object(mapped || {}) do |(fld, val), formatted_entry| 107 | # don't mess with explicitly mapped fields 108 | next if @map_src_fields.include?(fld) 109 | 110 | fld = format_field_name(fld) 111 | # account for mapping (appending) to an existing systemd field 112 | formatted_entry[fld] = join_if_needed([val, mapped[fld]]) 113 | end 114 | end 115 | 116 | def warnings 117 | return [] unless field_map_strict && field_map.empty? 118 | 119 | '`field_map_strict` set to true with empty `field_map`, expect no fields' 120 | end 121 | 122 | private 123 | 124 | def join_if_needed(values) 125 | values.compact! 126 | return values.first if values.length == 1 127 | 128 | values.join(' ') 129 | end 130 | 131 | def format_field_name(name) 132 | name = name.gsub(/\A_+/, '') if @opts.fields_strip_underscores 133 | name = name.downcase if @opts.fields_lowercase 134 | name 135 | end 136 | 137 | # Returns a `SystemdEntryMutator::Options` struct derived from the 138 | # elements in the supplied hash merged with the option defaults 139 | def options_from_hash(opts) 140 | merged = self.class.default_opts 141 | merged.each_pair do |k, _| 142 | merged[k] = opts[k] if opts.key?(k) 143 | end 144 | merged 145 | end 146 | 147 | def validate_options(opts) 148 | validate_all_strings opts[:field_map].keys, '`field_map` keys must be strings' 149 | validate_all_strings opts[:field_map].values, '`field_map` values must be strings or an array of strings', true 150 | %i[field_map_strict fields_strip_underscores fields_lowercase].each do |opt| 151 | validate_boolean opts[opt], opt 152 | end 153 | end 154 | 155 | def validate_all_strings(arr, message, allow_nesting = false) # rubocop:disable Style/OptionalBooleanParameter 156 | valid = arr.all? do |value| 157 | value.is_a?(String) || allow_nesting && value.is_a?(Array) && value.all? { |key| key.is_a?(String) } 158 | end 159 | raise Fluent::ConfigError, message unless valid 160 | end 161 | 162 | def validate_boolean(value, name) 163 | raise Fluent::ConfigError, "`#{name}` must be boolean" unless [true, false].include?(value) 164 | end 165 | 166 | # Compute the inverse of a human friendly field map `field_map` which is what 167 | # the mutator uses for the actual mapping. The resulting structure for 168 | # the inverse field map hash is: 169 | # {"" => ["", ...], ...} 170 | def invert_field_map(field_map) 171 | invs = {} 172 | field_map.values.flatten.uniq.each do |cstm| 173 | sysds = field_map.select { |_, v| (v == cstm || v.include?(cstm)) } 174 | invs[cstm] = sysds.keys 175 | end 176 | invs 177 | end 178 | end 179 | end 180 | end 181 | -------------------------------------------------------------------------------- /test/docker/Dockerfile.ruby32: -------------------------------------------------------------------------------- 1 | FROM ruby:3.2 2 | 3 | WORKDIR /usr/local/src 4 | 5 | COPY . . 6 | RUN bundle install -j4 -r3 7 | RUN bundle exec rake test TESTOPTS="-v" 8 | -------------------------------------------------------------------------------- /test/docker/Dockerfile.ruby34: -------------------------------------------------------------------------------- 1 | FROM ruby:3.4 2 | 3 | WORKDIR /usr/local/src 4 | 5 | COPY . . 6 | RUN bundle install -j4 -r3 7 | RUN bundle exec rake test TESTOPTS="-v" 8 | -------------------------------------------------------------------------------- /test/docker/Dockerfile.tdagent-almalinux: -------------------------------------------------------------------------------- 1 | FROM almalinux:9 2 | 3 | RUN rpm --import https://packages.treasuredata.com/GPG-KEY-td-agent \ 4 | && printf "[treasuredata]\nname=TreasureData\nbaseurl=http://packages.treasuredata.com/4/redhat/\$releasever/\$basearch\ngpgcheck=1\ngpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent\n" > /etc/yum.repos.d/td.repo \ 5 | && dnf install -y td-agent make gcc-c++ systemd 6 | 7 | ENV PATH /opt/td-agent/bin/:$PATH 8 | RUN td-agent-gem install bundler 9 | WORKDIR /usr/local/src 10 | COPY Gemfile ./ 11 | COPY fluent-plugin-systemd.gemspec ./ 12 | RUN bundle install 13 | COPY . . 14 | RUN bundle exec rake test TESTOPTS="-v" 15 | -------------------------------------------------------------------------------- /test/docker/Dockerfile.tdagent-ubuntu: -------------------------------------------------------------------------------- 1 | FROM ubuntu:jammy 2 | 3 | RUN apt-get update -q \ 4 | && apt-get install -qy --no-install-recommends \ 5 | build-essential \ 6 | curl \ 7 | ca-certificates \ 8 | gnupg \ 9 | libsystemd0 \ 10 | && curl https://packages.treasuredata.com/GPG-KEY-td-agent | apt-key add - \ 11 | && echo "deb http://packages.treasuredata.com/4/ubuntu/jammy/ jammy contrib" > /etc/apt/sources.list.d/treasure-data.list \ 12 | && apt-get update \ 13 | && apt-get install -y td-agent \ 14 | && apt-get clean \ 15 | && rm -rf /var/lib/apt/lists/* \ 16 | && truncate -s 0 /var/log/*log 17 | 18 | ENV PATH /opt/td-agent/bin/:$PATH 19 | 20 | RUN td-agent-gem install bundler 21 | WORKDIR /usr/local/src 22 | COPY Gemfile ./ 23 | COPY fluent-plugin-systemd.gemspec ./ 24 | RUN bundle check || bundle install 25 | COPY . . 26 | RUN bundle exec rake test TESTOPTS="-v" 27 | -------------------------------------------------------------------------------- /test/docker/Dockerfile.ubuntu: -------------------------------------------------------------------------------- 1 | FROM ubuntu:jammy 2 | 3 | RUN apt-get update -q && apt-get install -qy --no-install-recommends \ 4 | build-essential \ 5 | ruby \ 6 | ruby-bundler \ 7 | ruby-dev \ 8 | libsystemd0 \ 9 | && apt-get clean \ 10 | && rm -rf /var/lib/apt/lists/* \ 11 | && truncate -s 0 /var/log/*log 12 | 13 | WORKDIR /usr/local/src 14 | 15 | COPY Gemfile ./ 16 | COPY fluent-plugin-systemd.gemspec ./ 17 | RUN bundle install -j4 -r3 18 | COPY . . 19 | RUN bundle exec rake test TESTOPTS="-v" 20 | -------------------------------------------------------------------------------- /test/fixture/corrupt/test.badmsg.journal: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent-plugins-nursery/fluent-plugin-systemd/7907fdac0f95e8c83dd87d4da9e0398da0eb5299/test/fixture/corrupt/test.badmsg.journal -------------------------------------------------------------------------------- /test/fixture/test.journal: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent-plugins-nursery/fluent-plugin-systemd/7907fdac0f95e8c83dd87d4da9e0398da0eb5299/test/fixture/test.journal -------------------------------------------------------------------------------- /test/helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Copyright 2015-2018 Edward Robinson 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | require 'test/unit' 18 | require 'fluent/test' 19 | require 'fluent/test/helpers' 20 | -------------------------------------------------------------------------------- /test/plugin/systemd/test_entry_mutator.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Copyright 2015-2018 Edward Robinson 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | require_relative '../../helper' 18 | require 'json' 19 | require 'systemd/journal' 20 | require 'fluent/plugin/systemd/entry_mutator' 21 | 22 | class EntryTestData 23 | # `Systemd::Journal::Entry` to test with 24 | ENTRY = lambda { 25 | j = Systemd::Journal.new(path: 'test/fixture') 26 | j.wait(0) 27 | j.seek(:tail) 28 | j.move(-2) 29 | j.move_next 30 | j.current_entry 31 | }[].freeze 32 | 33 | # field map used for tests 34 | FIELD_MAP = { 35 | '_PID' => %w[msg _PID], 36 | 'MESSAGE' => 'msg', 37 | '_COMM' => '_EXE', 38 | '_CMDLINE' => 'command' 39 | }.freeze 40 | 41 | # string json form of `FIELD_MAP` 42 | FIELD_MAP_JSON = JSON.generate(FIELD_MAP).freeze 43 | 44 | # expected entry mutation results 45 | EXPECTED = { 46 | no_transform: { 47 | '_UID' => '0', 48 | '_GID' => '0', 49 | '_BOOT_ID' => '4737ffc504774b3ba67020bc947f1bc0', 50 | '_MACHINE_ID' => 'bb9d0a52a41243829ecd729b40ac0bce', 51 | '_HOSTNAME' => 'arch', 52 | 'PRIORITY' => '5', 53 | '_TRANSPORT' => 'syslog', 54 | 'SYSLOG_FACILITY' => '10', 55 | 'SYSLOG_IDENTIFIER' => 'login', 56 | '_PID' => '141', 57 | '_COMM' => 'login', 58 | '_EXE' => '/bin/login', 59 | '_AUDIT_SESSION' => '1', 60 | '_AUDIT_LOGINUID' => '0', 61 | 'MESSAGE' => 'ROOT LOGIN ON tty1', 62 | '_CMDLINE' => 'login -- root ', 63 | '_SYSTEMD_CGROUP' => '/user/root/1', 64 | '_SYSTEMD_SESSION' => '1', 65 | '_SYSTEMD_OWNER_UID' => '0', 66 | '_SOURCE_REALTIME_TIMESTAMP' => '1364519243563178' 67 | }, 68 | fields_strip_underscores: { 69 | 'UID' => '0', 70 | 'GID' => '0', 71 | 'BOOT_ID' => '4737ffc504774b3ba67020bc947f1bc0', 72 | 'MACHINE_ID' => 'bb9d0a52a41243829ecd729b40ac0bce', 73 | 'HOSTNAME' => 'arch', 74 | 'PRIORITY' => '5', 75 | 'TRANSPORT' => 'syslog', 76 | 'SYSLOG_FACILITY' => '10', 77 | 'SYSLOG_IDENTIFIER' => 'login', 78 | 'PID' => '141', 79 | 'COMM' => 'login', 80 | 'EXE' => '/bin/login', 81 | 'AUDIT_SESSION' => '1', 82 | 'AUDIT_LOGINUID' => '0', 83 | 'MESSAGE' => 'ROOT LOGIN ON tty1', 84 | 'CMDLINE' => 'login -- root ', 85 | 'SYSTEMD_CGROUP' => '/user/root/1', 86 | 'SYSTEMD_SESSION' => '1', 87 | 'SYSTEMD_OWNER_UID' => '0', 88 | 'SOURCE_REALTIME_TIMESTAMP' => '1364519243563178' 89 | }, 90 | fields_lowercase: { 91 | '_uid' => '0', 92 | '_gid' => '0', 93 | '_boot_id' => '4737ffc504774b3ba67020bc947f1bc0', 94 | '_machine_id' => 'bb9d0a52a41243829ecd729b40ac0bce', 95 | '_hostname' => 'arch', 96 | 'priority' => '5', 97 | '_transport' => 'syslog', 98 | 'syslog_facility' => '10', 99 | 'syslog_identifier' => 'login', 100 | '_pid' => '141', 101 | '_comm' => 'login', 102 | '_exe' => '/bin/login', 103 | '_audit_session' => '1', 104 | '_audit_loginuid' => '0', 105 | 'message' => 'ROOT LOGIN ON tty1', 106 | '_cmdline' => 'login -- root ', 107 | '_systemd_cgroup' => '/user/root/1', 108 | '_systemd_session' => '1', 109 | '_systemd_owner_uid' => '0', 110 | '_source_realtime_timestamp' => '1364519243563178' 111 | }, 112 | field_map: { 113 | '_UID' => '0', 114 | '_GID' => '0', 115 | '_BOOT_ID' => '4737ffc504774b3ba67020bc947f1bc0', 116 | '_MACHINE_ID' => 'bb9d0a52a41243829ecd729b40ac0bce', 117 | '_HOSTNAME' => 'arch', 118 | 'PRIORITY' => '5', 119 | '_TRANSPORT' => 'syslog', 120 | 'SYSLOG_FACILITY' => '10', 121 | 'SYSLOG_IDENTIFIER' => 'login', 122 | '_PID' => '141', 123 | '_EXE' => '/bin/login login', 124 | '_AUDIT_SESSION' => '1', 125 | '_AUDIT_LOGINUID' => '0', 126 | 'msg' => '141 ROOT LOGIN ON tty1', 127 | 'command' => 'login -- root ', 128 | '_SYSTEMD_CGROUP' => '/user/root/1', 129 | '_SYSTEMD_SESSION' => '1', 130 | '_SYSTEMD_OWNER_UID' => '0', 131 | '_SOURCE_REALTIME_TIMESTAMP' => '1364519243563178' 132 | }, 133 | field_map_strict: { 134 | '_PID' => '141', 135 | '_EXE' => 'login', 136 | 'msg' => '141 ROOT LOGIN ON tty1', 137 | 'command' => 'login -- root ' 138 | } 139 | }.freeze 140 | end 141 | 142 | class EntryMutatorTest < Test::Unit::TestCase 143 | # option validation test data in the form: 144 | # { test_name: option_hash } 145 | @validation_tests = { 146 | bad_fmap_opt_1: { field_map: { 1 => 'one' } }, 147 | bad_fmap_opt_2: { field_map: { 'one' => 1 } }, 148 | bad_fmap_opt_3: { field_map: { 'One' => ['one', 1] } }, 149 | bad_fmap_strict_opt: { field_map_strict: 1 }, 150 | bad_underscores_opt: { fields_strip_underscores: 1 }, 151 | bad_lowercase_opt: { fields_lowercase: 1 } 152 | } 153 | # mutate test data in the form: 154 | # { test_name: [transform_params, expected_entry], ... } 155 | @mutate_tests = { 156 | empty_options: [ 157 | {}, 158 | EntryTestData::EXPECTED[:no_transform] 159 | ], 160 | fields_strip_underscores: [ 161 | { fields_strip_underscores: true }, 162 | EntryTestData::EXPECTED[:fields_strip_underscores] 163 | ], 164 | fields_lowercase: [ 165 | { fields_lowercase: true }, 166 | EntryTestData::EXPECTED[:fields_lowercase] 167 | ], 168 | field_map: [ 169 | { field_map: EntryTestData::FIELD_MAP }, 170 | EntryTestData::EXPECTED[:field_map] 171 | ], 172 | field_map_strict: [ 173 | { field_map: EntryTestData::FIELD_MAP, field_map_strict: true }, 174 | EntryTestData::EXPECTED[:field_map_strict] 175 | ] 176 | } 177 | 178 | data(@validation_tests) 179 | def test_validation(opt) 180 | assert_raise Fluent::ConfigError do 181 | Fluent::Plugin::SystemdEntryMutator.new(**opt) 182 | end 183 | end 184 | 185 | # tests using Systemd::Journal::Entry 186 | 187 | def test_mutate_default_opts_journal_entry 188 | m = Fluent::Plugin::SystemdEntryMutator.new 189 | mutated = m.run(EntryTestData::ENTRY) 190 | assert_equal(EntryTestData::EXPECTED[:no_transform], mutated) 191 | end 192 | 193 | data(@mutate_tests) 194 | def test_mutate_with_journal_entry(data) 195 | options, expected = data 196 | m = Fluent::Plugin::SystemdEntryMutator.new(**options) 197 | mutated = m.run(EntryTestData::ENTRY) 198 | assert_equal(expected, mutated) 199 | end 200 | 201 | # tests using an entry hash 202 | 203 | def test_mutate_default_opts_hash_entry 204 | m = Fluent::Plugin::SystemdEntryMutator.new 205 | mutated = m.run(EntryTestData::ENTRY.to_h) 206 | assert_equal(EntryTestData::EXPECTED[:no_transform], mutated) 207 | end 208 | 209 | data(@mutate_tests) 210 | def test_mutate_with_hash_entry(data) 211 | options, expected = data 212 | m = Fluent::Plugin::SystemdEntryMutator.new(**options) 213 | mutated = m.run(EntryTestData::ENTRY.to_h) 214 | assert_equal(expected, mutated) 215 | end 216 | end 217 | -------------------------------------------------------------------------------- /test/plugin/test_filter_systemd_entry.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Copyright 2015-2018 Edward Robinson 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | require_relative '../helper' 18 | require_relative './systemd/test_entry_mutator' 19 | require 'fluent/test/driver/filter' 20 | require 'fluent/plugin/filter_systemd_entry' 21 | 22 | class SystemdEntryFilterTest < Test::Unit::TestCase 23 | include Fluent::Test::Helpers 24 | # filter test data in the form: 25 | # { test_name: [filter_config, expected_entry], ... } 26 | @tests = { 27 | no_transform: [ 28 | '', 29 | EntryTestData::EXPECTED[:no_transform] 30 | ], 31 | fields_strip_underscores: [ 32 | %( 33 | fields_strip_underscores true 34 | ), 35 | EntryTestData::EXPECTED[:fields_strip_underscores] 36 | ], 37 | fields_lowercase: [ 38 | %( 39 | fields_lowercase true 40 | ), 41 | EntryTestData::EXPECTED[:fields_lowercase] 42 | ], 43 | field_map: [ 44 | %( 45 | field_map #{EntryTestData::FIELD_MAP_JSON} 46 | ), 47 | EntryTestData::EXPECTED[:field_map] 48 | ], 49 | field_map_strict: [ 50 | %( 51 | field_map #{EntryTestData::FIELD_MAP_JSON} 52 | field_map_strict true 53 | ), 54 | EntryTestData::EXPECTED[:field_map_strict] 55 | ] 56 | } 57 | 58 | def setup 59 | Fluent::Test.setup 60 | end 61 | 62 | def create_driver(config) 63 | Fluent::Test::Driver::Filter.new(Fluent::Plugin::SystemdEntryFilter).configure(config) 64 | end 65 | 66 | data(@tests) 67 | def test_filter(data) 68 | conf, expect = data 69 | d = create_driver(conf) 70 | d.run do 71 | d.feed('test', 1_364_519_243, EntryTestData::ENTRY.to_h) 72 | end 73 | expected = [[ 74 | 1_364_519_243, 75 | expect 76 | ]] 77 | assert_equal(expected, d.filtered) 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /test/plugin/test_in_systemd.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Copyright 2015-2018 Edward Robinson 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | require_relative '../helper' 18 | require_relative './systemd/test_entry_mutator' 19 | require 'tempfile' 20 | require 'fluent/test/driver/input' 21 | require 'fluent/plugin/in_systemd' 22 | 23 | class SystemdInputTest < Test::Unit::TestCase 24 | include Fluent::Test::Helpers 25 | 26 | @base_config = %( 27 | tag test 28 | path test/fixture 29 | ) 30 | # entry test data in the form: 31 | # { test_name: [plugin_config, expected_entry], ... } 32 | @entry_tests = { 33 | fields_strip_underscores: [ 34 | @base_config + %( 35 | 36 | fields_strip_underscores true 37 | 38 | ), 39 | EntryTestData::EXPECTED[:fields_strip_underscores] 40 | ], 41 | fields_lowercase: [ 42 | @base_config + %( 43 | 44 | fields_lowercase true 45 | 46 | ), 47 | EntryTestData::EXPECTED[:fields_lowercase] 48 | ], 49 | field_map: [ 50 | @base_config + %( 51 | 52 | field_map #{EntryTestData::FIELD_MAP_JSON} 53 | 54 | ), 55 | EntryTestData::EXPECTED[:field_map] 56 | ], 57 | field_map_strict: [ 58 | @base_config + %( 59 | 60 | field_map #{EntryTestData::FIELD_MAP_JSON} 61 | field_map_strict true 62 | 63 | ), 64 | EntryTestData::EXPECTED[:field_map_strict] 65 | ] 66 | } 67 | 68 | def setup 69 | Fluent::Test.setup 70 | 71 | @base_config = %( 72 | tag test 73 | path test/fixture 74 | ) 75 | 76 | @storage_path = File.join(Dir.mktmpdir('pos_dir'), 'storage.json') 77 | 78 | @storage_config = @base_config + %( 79 | 80 | @type local 81 | persistent true 82 | path #{storage_path} 83 | 84 | ) 85 | 86 | @head_config = @storage_config + %( 87 | read_from_head true 88 | ) 89 | 90 | @filter_config = @head_config + %( 91 | filters [{ "_SYSTEMD_UNIT": "systemd-journald.service" }] 92 | ) 93 | 94 | @matches_config = @head_config + %( 95 | matches [{ "_SYSTEMD_UNIT": "systemd-journald.service" }] 96 | ) 97 | 98 | @tail_config = @storage_config + %( 99 | read_from_head false 100 | ) 101 | 102 | @not_present_config = %( 103 | tag test 104 | path test/not_a_real_path 105 | ) 106 | 107 | @corrupt_entries_config = %( 108 | tag test 109 | path test/fixture/corrupt 110 | read_from_head true 111 | ) 112 | end 113 | 114 | attr_reader :journal, :base_config, :head_config, 115 | :matches_config, :filter_config, :tail_config, :not_present_config, 116 | :storage_path, :storage_config, :corrupt_entries_config 117 | 118 | def create_driver(config) 119 | Fluent::Test::Driver::Input.new(Fluent::Plugin::SystemdInput).configure(config) 120 | end 121 | 122 | def read_pos 123 | JSON.parse(File.read(storage_path))['journal'] 124 | end 125 | 126 | def write_pos(pos) 127 | File.write(storage_path, JSON.dump(journal: pos)) 128 | end 129 | 130 | def test_configure_requires_tag 131 | assert_raise Fluent::ConfigError do 132 | create_driver('') 133 | end 134 | end 135 | 136 | def test_configuring_tag 137 | d = create_driver(base_config) 138 | assert_equal d.instance.tag, 'test' 139 | end 140 | 141 | def test_reading_from_the_journal_tail 142 | d = create_driver(base_config) 143 | expected = [[ 144 | 'test', 145 | 1_364_519_243, 146 | EntryTestData::EXPECTED[:no_transform] 147 | ]] 148 | d.run(expect_emits: 1) 149 | assert_equal(expected, d.events) 150 | end 151 | 152 | data(@entry_tests) 153 | def test_reading_from_the_journal_tail_mutate_entry(data) 154 | conf, expect = data 155 | d = create_driver(conf) 156 | expected = [[ 157 | 'test', 158 | 1_364_519_243, 159 | expect 160 | ]] 161 | d.run(expect_emits: 1) 162 | assert_equal(expected, d.events) 163 | end 164 | 165 | def test_storage_file_is_written 166 | d = create_driver(storage_config) 167 | d.run(expect_emits: 1) 168 | assert_equal 's=add4782f78ca4b6e84aa88d34e5b4a9d;i=1cd;b=4737ffc504774b3ba67020bc947f1bc0;m=42f2dd;t=4d905e4cd5a92;x=25b3f86ff2774ac4', read_pos 169 | end 170 | 171 | def test_reading_from_head 172 | d = create_driver(head_config) 173 | d.end_if do 174 | d.events.size >= 461 175 | end 176 | d.run(timeout: 5) 177 | assert_equal 461, d.events.size 178 | end 179 | 180 | class BufferErrorDriver < Fluent::Test::Driver::Input 181 | def initialize(klass, opts: {}, &block) 182 | @called = 0 183 | super 184 | end 185 | 186 | def emit_event_stream(tag, event_stream) 187 | unless @called > 1 188 | @called += 1 189 | raise Fluent::Plugin::Buffer::BufferOverflowError, 'buffer space has too many data' 190 | end 191 | 192 | super 193 | end 194 | end 195 | 196 | def test_backoff_on_buffer_error 197 | d = BufferErrorDriver.new(Fluent::Plugin::SystemdInput).configure(base_config) 198 | d.run(expect_emits: 1) 199 | end 200 | 201 | # deprecated and replaced with matches 202 | def test_reading_with_filters 203 | d = create_driver(filter_config) 204 | d.end_if do 205 | d.events.size >= 3 206 | end 207 | d.run(timeout: 5) 208 | assert_equal 3, d.events.size 209 | end 210 | 211 | def test_reading_with_matches 212 | d = create_driver(matches_config) 213 | d.end_if do 214 | d.events.size >= 3 215 | end 216 | d.run(timeout: 5) 217 | assert_equal 3, d.events.size 218 | end 219 | 220 | def test_reading_from_a_pos 221 | write_pos 's=add4782f78ca4b6e84aa88d34e5b4a9d;i=13f;b=4737ffc504774b3ba67020bc947f1bc0;m=ffadd;t=4d905e49a6291;x=9a11dd9ffee96e9f' 222 | d = create_driver(head_config) 223 | d.end_if do 224 | d.events.size >= 142 225 | end 226 | d.run(timeout: 5) 227 | assert_equal 142, d.events.size 228 | end 229 | 230 | def test_reading_from_an_invalid_pos 231 | write_pos 'thisisinvalid' 232 | 233 | # It continues as if the pos file did not exist 234 | d = create_driver(head_config) 235 | d.end_if do 236 | d.events.size >= 461 237 | end 238 | d.run(timeout: 5) 239 | assert_equal 461, d.events.size 240 | assert_match( 241 | "Could not seek to cursor thisisinvalid found in position file: #{storage_path}, falling back to reading from head", 242 | d.logs.last 243 | ) 244 | end 245 | 246 | def test_reading_from_the_journal_tail_explicit_setting 247 | d = create_driver(tail_config) 248 | expected = [[ 249 | 'test', 250 | 1_364_519_243, 251 | EntryTestData::EXPECTED[:no_transform] 252 | ]] 253 | d.run(expect_emits: 1) 254 | assert_equal(expected, d.events) 255 | end 256 | 257 | def test_journal_not_present 258 | d = create_driver(not_present_config) 259 | d.end_if { d.logs.size > 1 } 260 | d.run(timeout: 5) 261 | assert_match 'Systemd::JournalError: No such file or directory retrying in 1s', d.logs.last 262 | end 263 | 264 | def test_reading_from_a_journal_with_corrupted_entries 265 | # One corrupted entry exists in 461 entries. (The 3rd entry is corrupted.) 266 | d = create_driver(corrupt_entries_config) 267 | d.run(expect_emits: 460) 268 | # Since libsystemd v250, it can read this corrupted record. 269 | assert { d.events.size == 460 or d.events.size == 461 } 270 | end 271 | end 272 | --------------------------------------------------------------------------------