├── .ansible-lint ├── .github ├── FUNDING.yml └── workflows │ ├── ci.yml │ ├── release.yml │ └── stale.yml ├── .gitignore ├── .yamllint ├── LICENSE ├── README.md ├── defaults └── main.yml ├── handlers └── main.yml ├── meta └── main.yml ├── molecule └── default │ ├── converge.yml │ └── molecule.yml ├── tasks ├── main.yml ├── setup-Debian.yml └── setup-RedHat.yml └── templates ├── td-agent.conf.j2 └── td.repo.j2 /.ansible-lint: -------------------------------------------------------------------------------- 1 | skip_list: 2 | - 'yaml' 3 | - 'role-name' 4 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | --- 3 | github: geerlingguy 4 | patreon: geerlingguy 5 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: CI 3 | 'on': 4 | pull_request: 5 | push: 6 | branches: 7 | - master 8 | schedule: 9 | - cron: "30 1 * * 1" 10 | 11 | defaults: 12 | run: 13 | working-directory: 'geerlingguy.fluentd' 14 | 15 | jobs: 16 | 17 | lint: 18 | name: Lint 19 | runs-on: ubuntu-latest 20 | steps: 21 | - name: Check out the codebase. 22 | uses: actions/checkout@v2 23 | with: 24 | path: 'geerlingguy.fluentd' 25 | 26 | - name: Set up Python 3. 27 | uses: actions/setup-python@v2 28 | with: 29 | python-version: '3.x' 30 | 31 | - name: Install test dependencies. 32 | run: pip3 install yamllint 33 | 34 | - name: Lint code. 35 | run: | 36 | yamllint . 37 | 38 | molecule: 39 | name: Molecule 40 | runs-on: ubuntu-latest 41 | strategy: 42 | matrix: 43 | distro: 44 | - rockylinux8 45 | - ubuntu2004 46 | - debian10 47 | 48 | steps: 49 | - name: Check out the codebase. 50 | uses: actions/checkout@v2 51 | with: 52 | path: 'geerlingguy.fluentd' 53 | 54 | - name: Set up Python 3. 55 | uses: actions/setup-python@v2 56 | with: 57 | python-version: '3.x' 58 | 59 | - name: Install test dependencies. 60 | run: pip3 install ansible molecule molecule-plugins[docker] docker 61 | 62 | - name: Run Molecule tests. 63 | run: molecule test 64 | env: 65 | PY_COLORS: '1' 66 | ANSIBLE_FORCE_COLOR: '1' 67 | MOLECULE_DISTRO: ${{ matrix.distro }} 68 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This workflow requires a GALAXY_API_KEY secret present in the GitHub 3 | # repository or organization. 4 | # 5 | # See: https://github.com/marketplace/actions/publish-ansible-role-to-galaxy 6 | # See: https://github.com/ansible/galaxy/issues/46 7 | 8 | name: Release 9 | 'on': 10 | push: 11 | tags: 12 | - '*' 13 | 14 | defaults: 15 | run: 16 | working-directory: 'geerlingguy.fluentd' 17 | 18 | jobs: 19 | 20 | release: 21 | name: Release 22 | runs-on: ubuntu-latest 23 | steps: 24 | - name: Check out the codebase. 25 | uses: actions/checkout@v2 26 | with: 27 | path: 'geerlingguy.fluentd' 28 | 29 | - name: Set up Python 3. 30 | uses: actions/setup-python@v2 31 | with: 32 | python-version: '3.x' 33 | 34 | - name: Install Ansible. 35 | run: pip3 install ansible-core 36 | 37 | - name: Trigger a new import on Galaxy. 38 | run: >- 39 | ansible-galaxy role import --api-key ${{ secrets.GALAXY_API_KEY }} 40 | $(echo ${{ github.repository }} | cut -d/ -f1) $(echo ${{ github.repository }} | cut -d/ -f2) 41 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Close inactive issues 3 | 'on': 4 | schedule: 5 | - cron: "55 17 * * 0" # semi-random time 6 | 7 | jobs: 8 | close-issues: 9 | runs-on: ubuntu-latest 10 | permissions: 11 | issues: write 12 | pull-requests: write 13 | steps: 14 | - uses: actions/stale@v8 15 | with: 16 | days-before-stale: 120 17 | days-before-close: 60 18 | exempt-issue-labels: bug,pinned,security,planned 19 | exempt-pr-labels: bug,pinned,security,planned 20 | stale-issue-label: "stale" 21 | stale-pr-label: "stale" 22 | stale-issue-message: | 23 | This issue has been marked 'stale' due to lack of recent activity. If there is no further activity, the issue will be closed in another 30 days. Thank you for your contribution! 24 | 25 | Please read [this blog post](https://www.jeffgeerling.com/blog/2020/enabling-stale-issue-bot-on-my-github-repositories) to see the reasons why I mark issues as stale. 26 | close-issue-message: | 27 | This issue has been closed due to inactivity. If you feel this is in error, please reopen the issue or file a new issue with the relevant details. 28 | stale-pr-message: | 29 | This pr has been marked 'stale' due to lack of recent activity. If there is no further activity, the issue will be closed in another 30 days. Thank you for your contribution! 30 | 31 | Please read [this blog post](https://www.jeffgeerling.com/blog/2020/enabling-stale-issue-bot-on-my-github-repositories) to see the reasons why I mark issues as stale. 32 | close-pr-message: | 33 | This pr has been closed due to inactivity. If you feel this is in error, please reopen the issue or file a new issue with the relevant details. 34 | repo-token: ${{ secrets.GITHUB_TOKEN }} 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.retry 2 | */__pycache__ 3 | *.pyc 4 | .cache 5 | 6 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | 4 | rules: 5 | line-length: 6 | max: 160 7 | level: warning 8 | 9 | ignore: | 10 | .github/workflows/stale.yml 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2020 Jeff Geerling 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ansible Role: Fluentd 2 | 3 | [![CI](https://github.com/geerlingguy/ansible-role-fluentd/workflows/CI/badge.svg?event=push)](https://github.com/geerlingguy/ansible-role-fluentd/actions?query=workflow%3ACI) 4 | 5 | An Ansible Role that installs Fluentd on RedHat/CentOS or Debian/Ubuntu. This role installs `td-agent`, which is a standalone version that doesn't require Ruby to be installed on the system separately. See [differences between td-agent and Fluentd here](https://www.fluentd.org/faqs). 6 | 7 | ## Requirements 8 | 9 | N/A 10 | 11 | ## Role Variables 12 | 13 | Available variables are listed below, along with default values (see `defaults/main.yml`): 14 | 15 | fluentd_version: 3 16 | 17 | The `td-agent` version to install. See more details about the [differences between v2, v3, and v4](https://docs.fluentd.org/quickstart/td-agent-v2-vs-v3-vs-v4). 18 | 19 | fluentd_package_state: present 20 | 21 | The `td-agent` Fluentd package state; set to `latest` to upgrade or change versions. 22 | 23 | fluentd_service_name: td-agent 24 | fluentd_service_state: started 25 | fluentd_service_enabled: true 26 | 27 | Controls the Fluentd service options. 28 | 29 | fluentd_plugins: 30 | - fluent-plugin-elasticsearch 31 | 32 | # Alternative format: 33 | fluentd_plugins: 34 | - name: fluent-plugin-elasticsearch 35 | version: '4.0.6' 36 | state: present 37 | 38 | A list of Fluentd plugins to install. 39 | 40 | fluentd_conf_sources: | 41 | [see defaults/main.yml for default content] 42 | 43 | fluentd_conf_filters: | 44 | [see defaults/main.yml for default content] 45 | 46 | fluentd_conf_matches: | 47 | [see defaults/main.yml for default content] 48 | 49 | The configuration which will be placed into the `td-agent.conf` file which controls how Fluentd listens for, filters, and routes log data. The defaults set up some basic options which can direct data to Treasure Data, but you should override these values with what's appropriate for your logs. 50 | 51 | For example, if you want to monitor an Apache HTTP server's access log, you would add a source: 52 | 53 | fluentd_conf_sources: | 54 | 55 | @type tail 56 | @id input_tail 57 | 58 | @type apache2 59 | 60 | path /var/log/httpd-access.log 61 | tag apache.access 62 | 63 | 64 | And then you could route `apache` log entries to Elasticsearch using: 65 | 66 | fluentd_conf_matches: | 67 | 68 | @type elasticsearch 69 | host log.example.com 70 | port 9200 71 | user myuser 72 | password mypassword 73 | logstash_format true 74 | 75 | 76 | Note that Elasticsearch would require the Fluentd plugin `fluent-plugin-elasticsearch` to be installed. 77 | 78 | ## Dependencies 79 | 80 | N/A 81 | 82 | ## Example Playbook 83 | 84 | ```yaml 85 | - hosts: search 86 | 87 | vars: 88 | fluentd_conf_sources: | 89 | 90 | @type tail 91 | @id input_tail 92 | 93 | @type apache2 94 | 95 | path /var/log/httpd-access.log 96 | tag apache.access 97 | 98 | 99 | roles: 100 | - geerlingguy.fluentd 101 | ``` 102 | 103 | ## License 104 | 105 | MIT / BSD 106 | 107 | ## Author Information 108 | 109 | This role was created in 2020 by [Jeff Geerling](https://www.jeffgeerling.com/), author of [Ansible for DevOps](https://www.ansiblefordevops.com/). 110 | -------------------------------------------------------------------------------- /defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | fluentd_version: 3 3 | fluentd_package_state: present 4 | 5 | fluentd_service_name: td-agent 6 | fluentd_service_state: started 7 | fluentd_service_enabled: true 8 | 9 | fluentd_plugins: 10 | - fluent-plugin-elasticsearch 11 | 12 | fluentd_conf_sources: | 13 | ## built-in TCP input 14 | ## @see http://docs.fluentd.org/articles/in_forward 15 | 16 | @type forward 17 | @id input_forward 18 | 19 | 20 | ## built-in UNIX socket input 21 | # 22 | # type unix 23 | # 24 | 25 | # HTTP input 26 | # POST http://localhost:8888/?json= 27 | # POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"} 28 | # @see http://docs.fluentd.org/articles/in_http 29 | 30 | @type http 31 | @id input_http 32 | port 8888 33 | 34 | 35 | ## live debugging agent 36 | 37 | @type debug_agent 38 | @id input_debug_agent 39 | bind 127.0.0.1 40 | port 24230 41 | 42 | 43 | fluentd_conf_filters: | 44 | ## Filters go here. 45 | 46 | fluentd_conf_matches: | 47 | # This section matches events whose tag is td.DATABASE.TABLE 48 | 49 | @type tdlog 50 | @id output_td 51 | apikey YOUR_API_KEY 52 | 53 | auto_create_table 54 | 55 | @type file 56 | path /var/log/td-agent/buffer/td 57 | 58 | 59 | 60 | @type file 61 | path /var/log/td-agent/failed_records 62 | 63 | 64 | 65 | ## match tag=debug.** and dump to console 66 | 67 | @type stdout 68 | @id output_stdout 69 | 70 | -------------------------------------------------------------------------------- /handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart fluentd 3 | service: name={{ fluentd_service_name }} state=restarted 4 | -------------------------------------------------------------------------------- /meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: [] 3 | 4 | galaxy_info: 5 | role_name: fluentd 6 | author: geerlingguy 7 | description: Fluentd for Linux. 8 | company: "Midwestern Mac, LLC" 9 | license: "license (BSD, MIT)" 10 | min_ansible_version: 2.10 11 | platforms: 12 | - name: Debian 13 | versions: 14 | - all 15 | - name: Ubuntu 16 | versions: 17 | - all 18 | galaxy_tags: 19 | - web 20 | - system 21 | - monitoring 22 | - logging 23 | - fluentd 24 | - td 25 | - agent 26 | - fluent 27 | - elasticsearch 28 | - elk 29 | - efk 30 | - kubernetes 31 | - containers 32 | -------------------------------------------------------------------------------- /molecule/default/converge.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Converge 3 | hosts: all 4 | become: true 5 | 6 | pre_tasks: 7 | - name: Update apt cache. 8 | apt: update_cache=true cache_valid_time=600 9 | when: ansible_os_family == 'Debian' 10 | changed_when: false 11 | 12 | - name: Set fluentd_version for Ubuntu 20.04. 13 | set_fact: 14 | fluentd_version: 4 15 | when: ansible_distribution_version == '20.04' 16 | 17 | roles: 18 | - role: geerlingguy.fluentd 19 | -------------------------------------------------------------------------------- /molecule/default/molecule.yml: -------------------------------------------------------------------------------- 1 | --- 2 | role_name_check: 1 3 | dependency: 4 | name: galaxy 5 | driver: 6 | name: docker 7 | platforms: 8 | - name: instance 9 | image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" 10 | command: ${MOLECULE_DOCKER_COMMAND:-""} 11 | volumes: 12 | - /sys/fs/cgroup:/sys/fs/cgroup:rw 13 | cgroupns_mode: host 14 | privileged: true 15 | pre_build_image: true 16 | provisioner: 17 | name: ansible 18 | playbooks: 19 | converge: ${MOLECULE_PLAYBOOK:-converge.yml} 20 | -------------------------------------------------------------------------------- /tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include_tasks: setup-RedHat.yml 3 | when: ansible_os_family == 'RedHat' 4 | 5 | - include_tasks: setup-Debian.yml 6 | when: ansible_os_family == 'Debian' 7 | 8 | - name: Install td-agent. 9 | package: 10 | name: td-agent 11 | state: "{{ fluentd_package_state }}" 12 | 13 | - name: Configure Fluentd. 14 | template: 15 | src: td-agent.conf.j2 16 | dest: /etc/td-agent/td-agent.conf 17 | owner: root 18 | group: root 19 | mode: 0644 20 | notify: restart fluentd 21 | 22 | - name: Determine fluent-gem executable location for td-agent < v4 23 | set_fact: 24 | fluent_gem_executable: /opt/td-agent/embedded/bin/fluent-gem 25 | when: fluentd_version < 4 26 | 27 | - name: Determine fluent-gem executable location for td-agent v4 28 | set_fact: 29 | fluent_gem_executable: /opt/td-agent/bin/fluent-gem 30 | when: fluentd_version >= 4 31 | 32 | - name: Ensure Fluentd plugins are installed. 33 | gem: 34 | name: "{{ item.name | default(item) }}" 35 | executable: "{{ fluent_gem_executable }}" 36 | state: "{{ item.state | default('present') }}" 37 | version: "{{ item.version | default(omit) }}" 38 | user_install: false 39 | with_items: "{{ fluentd_plugins }}" 40 | 41 | - name: Start Fluentd. 42 | service: 43 | name: "{{ fluentd_service_name }}" 44 | state: "{{ fluentd_service_state }}" 45 | enabled: "{{ fluentd_service_enabled }}" 46 | -------------------------------------------------------------------------------- /tasks/setup-Debian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Add required dependencies. 3 | apt: 4 | name: 5 | - apt-transport-https 6 | - gnupg2 7 | state: present 8 | 9 | - name: Add td-agent apt key. 10 | apt_key: 11 | url: https://packages.treasuredata.com/GPG-KEY-td-agent 12 | state: present 13 | 14 | - name: Add td-agent repository. 15 | apt_repository: 16 | repo: >- 17 | deb 18 | http://packages.treasuredata.com/{{ fluentd_version }}/{{ ansible_distribution | lower }}/{{ ansible_distribution_release | lower }}/ 19 | {{ ansible_distribution_release | lower }} contrib 20 | state: present 21 | update_cache: true 22 | -------------------------------------------------------------------------------- /tasks/setup-RedHat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Add td-agent GPG key. 3 | rpm_key: 4 | key: https://packages.treasuredata.com/GPG-KEY-td-agent 5 | state: present 6 | 7 | - name: Add td-agent repository. 8 | template: 9 | src: td.repo.j2 10 | dest: /etc/yum.repos.d/td.repo 11 | mode: 0644 12 | -------------------------------------------------------------------------------- /templates/td-agent.conf.j2: -------------------------------------------------------------------------------- 1 | #### 2 | ## Source descriptions: 3 | ## 4 | 5 | {{ fluentd_conf_sources }} 6 | 7 | #### 8 | ## Filter descriptions: 9 | ## 10 | 11 | {{ fluentd_conf_filters }} 12 | 13 | #### 14 | ## Match descriptions: 15 | ## 16 | 17 | {{ fluentd_conf_matches }} 18 | -------------------------------------------------------------------------------- /templates/td.repo.j2: -------------------------------------------------------------------------------- 1 | [treasuredata] 2 | name=TreasureData 3 | baseurl=http://packages.treasuredata.com/{{ fluentd_version }}/redhat/$releasever/$basearch 4 | gpgcheck=1 5 | gpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent 6 | --------------------------------------------------------------------------------